Lines Matching +full:adreno +full:- +full:gmu
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
13 #include <linux/nvmem-consumer.h>
14 #include <linux/soc/qcom/llcc-qcom.h>
23 /* Check that the GMU is idle */ in _a6xx_check_idle()
24 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu)) in _a6xx_check_idle()
44 gpu->name, __builtin_return_address(0), in a6xx_idle()
61 if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) { in update_shadow_rptr()
75 spin_lock_irqsave(&ring->preempt_lock, flags); in a6xx_flush()
78 ring->cur = ring->next; in a6xx_flush()
83 spin_unlock_irqrestore(&ring->preempt_lock, flags); in a6xx_flush()
109 if (ctx->seqno == a6xx_gpu->cur_ctx_seqno) in a6xx_set_pagetable()
112 if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid)) in a6xx_set_pagetable()
142 a6xx_gpu->cur_ctx_seqno = ctx->seqno; in a6xx_set_pagetable()
147 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; in a6xx_submit()
148 struct msm_drm_private *priv = gpu->dev->dev_private; in a6xx_submit()
151 struct msm_ringbuffer *ring = submit->ring; in a6xx_submit()
154 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx); in a6xx_submit()
160 * For PM4 the GMU register offsets are calculated from the base of the in a6xx_submit()
175 for (i = 0; i < submit->nr_cmds; i++) { in a6xx_submit()
176 switch (submit->cmd[i].type) { in a6xx_submit()
180 if (priv->lastctx == submit->queue->ctx) in a6xx_submit()
185 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); in a6xx_submit()
186 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); in a6xx_submit()
187 OUT_RING(ring, submit->cmd[i].size); in a6xx_submit()
193 * Periodically update shadow-wptr if needed, so that we in a6xx_submit()
210 OUT_RING(ring, submit->seqno); in a6xx_submit()
221 OUT_RING(ring, submit->seqno); in a6xx_submit()
502 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_set_hwcg() local
507 if (!adreno_gpu->info->hwcg) in a6xx_set_hwcg()
517 /* Don't re-program the registers if they are already correct */ in a6xx_set_hwcg()
522 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); in a6xx_set_hwcg()
524 for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++) in a6xx_set_hwcg()
525 gpu_write(gpu, reg->offset, state ? reg->value : 0); in a6xx_set_hwcg()
528 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); in a6xx_set_hwcg()
683 for (i = 0; i < count - 1; i++) in a6xx_set_cp_protect()
686 gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]); in a6xx_set_cp_protect()
729 struct msm_ringbuffer *ring = gpu->rb[0]; in a6xx_cp_init()
753 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; in a6xx_cp_init()
763 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a6xx_ucode_check_version()
764 struct msm_gpu *gpu = &adreno_gpu->base; in a6xx_ucode_check_version()
765 const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE]; in a6xx_ucode_check_version()
795 a6xx_gpu->has_whereami = true; in a6xx_ucode_check_version()
800 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
809 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
815 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
828 if (!a6xx_gpu->sqe_bo) { in a6xx_ucode_init()
829 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu, in a6xx_ucode_init()
830 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova); in a6xx_ucode_init()
832 if (IS_ERR(a6xx_gpu->sqe_bo)) { in a6xx_ucode_init()
833 int ret = PTR_ERR(a6xx_gpu->sqe_bo); in a6xx_ucode_init()
835 a6xx_gpu->sqe_bo = NULL; in a6xx_ucode_init()
836 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_init()
842 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw"); in a6xx_ucode_init()
843 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) { in a6xx_ucode_init()
844 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); in a6xx_ucode_init()
845 drm_gem_object_put(a6xx_gpu->sqe_bo); in a6xx_ucode_init()
847 a6xx_gpu->sqe_bo = NULL; in a6xx_ucode_init()
848 return -EPERM; in a6xx_ucode_init()
853 REG_A6XX_CP_SQE_INSTR_BASE+1, a6xx_gpu->sqe_iova); in a6xx_ucode_init()
890 /* Make sure the GMU keeps the GPU on while we set it up */ in hw_init()
891 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); in hw_init()
896 * Disable the trusted memory range - we don't actually supported secure in hw_init()
949 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ in hw_init()
955 0x00100000 + adreno_gpu->gmem - 1); in hw_init()
1029 if (gpu->hw_apriv) { in hw_init()
1047 gpu->rb[0]->iova); in hw_init()
1053 if (adreno_gpu->base.hw_apriv) in hw_init()
1064 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) { in hw_init()
1065 if (!a6xx_gpu->shadow_bo) { in hw_init()
1066 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, in hw_init()
1067 sizeof(u32) * gpu->nr_rings, in hw_init()
1069 gpu->aspace, &a6xx_gpu->shadow_bo, in hw_init()
1070 &a6xx_gpu->shadow_iova); in hw_init()
1072 if (IS_ERR(a6xx_gpu->shadow)) in hw_init()
1073 return PTR_ERR(a6xx_gpu->shadow); in hw_init()
1078 shadowptr(a6xx_gpu, gpu->rb[0])); in hw_init()
1082 a6xx_gpu->cur_ring = gpu->rb[0]; in hw_init()
1084 a6xx_gpu->cur_ctx_seqno = 0; in hw_init()
1102 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); in hw_init()
1103 OUT_RING(gpu->rb[0], 0x00000000); in hw_init()
1105 a6xx_flush(gpu, gpu->rb[0]); in hw_init()
1106 if (!a6xx_idle(gpu, gpu->rb[0])) in hw_init()
1107 return -EINVAL; in hw_init()
1108 } else if (ret == -ENODEV) { in hw_init()
1115 dev_warn_once(gpu->dev->dev, in hw_init()
1116 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); in hw_init()
1125 * Tell the GMU that we are done touching the GPU and it can start power in hw_init()
1128 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); in hw_init()
1130 if (a6xx_gpu->gmu.legacy) { in hw_init()
1131 /* Take the GMU out of its special boot mode */ in hw_init()
1132 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); in hw_init()
1144 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_hw_init()
1146 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_hw_init()
1153 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n", in a6xx_dump()
1170 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i, in a6xx_recover()
1180 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); in a6xx_recover()
1182 gpu->funcs->pm_suspend(gpu); in a6xx_recover()
1183 gpu->funcs->pm_resume(gpu); in a6xx_recover()
1238 bool do_devcoredump = info && !READ_ONCE(gpu->crashstate); in a6xx_fault_handler()
1245 gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu); in a6xx_fault_handler()
1250 * adreno-smmu-priv in a6xx_fault_handler()
1263 if (info->fsr & ARM_SMMU_FSR_TF) in a6xx_fault_handler()
1265 else if (info->fsr & ARM_SMMU_FSR_PF) in a6xx_fault_handler()
1267 else if (info->fsr & ARM_SMMU_FSR_EF) in a6xx_fault_handler()
1270 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff); in a6xx_fault_handler()
1273 info->ttbr0, iova, in a6xx_fault_handler()
1283 del_timer(&gpu->hangcheck_timer); in a6xx_fault_handler()
1285 gpu->fault_info.ttbr0 = info->ttbr0; in a6xx_fault_handler()
1286 gpu->fault_info.iova = iova; in a6xx_fault_handler()
1287 gpu->fault_info.flags = flags; in a6xx_fault_handler()
1288 gpu->fault_info.type = type; in a6xx_fault_handler()
1289 gpu->fault_info.block = block; in a6xx_fault_handler()
1291 kthread_queue_work(gpu->worker, &gpu->fault_work); in a6xx_fault_handler()
1306 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1312 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1316 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n", in a6xx_cp_hw_err_irq()
1322 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1329 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n"); in a6xx_cp_hw_err_irq()
1332 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n"); in a6xx_cp_hw_err_irq()
1335 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n"); in a6xx_cp_hw_err_irq()
1343 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); in a6xx_fault_detect_irq()
1358 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); in a6xx_fault_detect_irq()
1360 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_fault_detect_irq()
1362 ring ? ring->id : -1, ring ? ring->seqno : 0, in a6xx_fault_detect_irq()
1372 del_timer(&gpu->hangcheck_timer); in a6xx_fault_detect_irq()
1374 kthread_queue_work(gpu->worker, &gpu->recover_work); in a6xx_fault_detect_irq()
1387 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n"); in a6xx_irq()
1393 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); in a6xx_irq()
1396 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); in a6xx_irq()
1399 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); in a6xx_irq()
1409 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or); in a6xx_llc_rmw()
1414 return msm_writel(value, a6xx_gpu->llc_mmio + (reg << 2)); in a6xx_llc_write()
1419 llcc_slice_deactivate(a6xx_gpu->llc_slice); in a6xx_llc_deactivate()
1420 llcc_slice_deactivate(a6xx_gpu->htw_llc_slice); in a6xx_llc_deactivate()
1425 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a6xx_llc_activate()
1426 struct msm_gpu *gpu = &adreno_gpu->base; in a6xx_llc_activate()
1429 if (IS_ERR(a6xx_gpu->llc_mmio)) in a6xx_llc_activate()
1432 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { in a6xx_llc_activate()
1433 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); in a6xx_llc_activate()
1444 if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) { in a6xx_llc_activate()
1445 if (!a6xx_gpu->have_mmu500) { in a6xx_llc_activate()
1446 u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice); in a6xx_llc_activate()
1460 if (!a6xx_gpu->have_mmu500) { in a6xx_llc_activate()
1485 llcc_slice_putd(a6xx_gpu->llc_slice); in a6xx_llc_slices_destroy()
1486 llcc_slice_putd(a6xx_gpu->htw_llc_slice); in a6xx_llc_slices_destroy()
1498 phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0); in a6xx_llc_slices_init()
1499 a6xx_gpu->have_mmu500 = (phandle && in a6xx_llc_slices_init()
1500 of_device_is_compatible(phandle, "arm,mmu-500")); in a6xx_llc_slices_init()
1503 if (a6xx_gpu->have_mmu500) in a6xx_llc_slices_init()
1504 a6xx_gpu->llc_mmio = NULL; in a6xx_llc_slices_init()
1506 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem", "gpu_cx"); in a6xx_llc_slices_init()
1508 a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU); in a6xx_llc_slices_init()
1509 a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); in a6xx_llc_slices_init()
1511 if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) in a6xx_llc_slices_init()
1512 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL); in a6xx_llc_slices_init()
1521 gpu->needs_hw_init = true; in a6xx_pm_resume()
1525 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_pm_resume()
1527 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_pm_resume()
1550 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_pm_suspend()
1552 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_pm_suspend()
1556 if (a6xx_gpu->shadow_bo) in a6xx_pm_suspend()
1557 for (i = 0; i < gpu->nr_rings; i++) in a6xx_pm_suspend()
1558 a6xx_gpu->shadow[i] = 0; in a6xx_pm_suspend()
1568 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_get_timestamp()
1571 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_get_timestamp()
1576 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_get_timestamp()
1578 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_get_timestamp()
1588 return a6xx_gpu->cur_ring; in a6xx_active_ring()
1596 if (a6xx_gpu->sqe_bo) { in a6xx_destroy()
1597 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); in a6xx_destroy()
1598 drm_gem_object_put(a6xx_gpu->sqe_bo); in a6xx_destroy()
1601 if (a6xx_gpu->shadow_bo) { in a6xx_destroy()
1602 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace); in a6xx_destroy()
1603 drm_gem_object_put(a6xx_gpu->shadow_bo); in a6xx_destroy()
1623 if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0) in a6xx_gpu_busy()
1626 busy_cycles = gmu_read64(&a6xx_gpu->gmu, in a6xx_gpu_busy()
1630 busy_time = (busy_cycles - gpu->devfreq.busy_cycles) * 10; in a6xx_gpu_busy()
1633 gpu->devfreq.busy_cycles = busy_cycles; in a6xx_gpu_busy()
1635 pm_runtime_put(a6xx_gpu->gmu.dev); in a6xx_gpu_busy()
1648 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_gpu_set_freq()
1650 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_gpu_set_freq()
1671 if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) in a6xx_create_address_space()
1674 mmu = msm_iommu_new(&pdev->dev, iommu); in a6xx_create_address_space()
1685 start = max_t(u64, SZ_16M, iommu->geometry.aperture_start); in a6xx_create_address_space()
1686 size = iommu->geometry.aperture_end - start + 1; in a6xx_create_address_space()
1692 mmu->funcs->destroy(mmu); in a6xx_create_address_space()
1702 mmu = msm_iommu_pagetable_create(gpu->aspace->mmu); in a6xx_create_private_address_space()
1716 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) in a6xx_get_rptr()
1717 return a6xx_gpu->shadow[ring->id]; in a6xx_get_rptr()
1719 return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR); in a6xx_get_rptr()
1743 "missing support for speed-bin: %u. Some OPPs may not be supported by hardware", in fuse_to_supp_hw()
1759 * -ENOENT means that the platform doesn't support speedbin which is in a6xx_set_supported_hw()
1762 if (ret == -ENOENT) { in a6xx_set_supported_hw()
1766 "failed to read speed-bin (%d). Some OPPs may not be supported by hardware", in a6xx_set_supported_hw()
1811 struct msm_drm_private *priv = dev->dev_private; in a6xx_gpu_init()
1812 struct platform_device *pdev = priv->gpu_pdev; in a6xx_gpu_init()
1813 struct adreno_platform_config *config = pdev->dev.platform_data; in a6xx_gpu_init()
1823 return ERR_PTR(-ENOMEM); in a6xx_gpu_init()
1825 adreno_gpu = &a6xx_gpu->base; in a6xx_gpu_init()
1826 gpu = &adreno_gpu->base; in a6xx_gpu_init()
1828 adreno_gpu->registers = NULL; in a6xx_gpu_init()
1835 info = adreno_info(config->rev); in a6xx_gpu_init()
1837 if (info && (info->revn == 650 || info->revn == 660 || in a6xx_gpu_init()
1838 adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), info->rev))) in a6xx_gpu_init()
1839 adreno_gpu->base.hw_apriv = true; in a6xx_gpu_init()
1845 if (info && (info->revn == 618)) in a6xx_gpu_init()
1846 gpu->clamp_to_idle = true; in a6xx_gpu_init()
1850 ret = a6xx_set_supported_hw(&pdev->dev, config->rev); in a6xx_gpu_init()
1852 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
1858 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
1862 /* Check if there is a GMU phandle and set it up */ in a6xx_gpu_init()
1863 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0); in a6xx_gpu_init()
1870 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
1874 if (gpu->aspace) in a6xx_gpu_init()
1875 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, in a6xx_gpu_init()