17 static void a5xx_dump(struct msm_gpu *gpu);
21 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in update_shadow_rptr()  argument
23 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in update_shadow_rptr()
33 void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,  in a5xx_flush()  argument
36 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_flush()
46 		update_shadow_rptr(gpu, ring);  in a5xx_flush()
63 		gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);  in a5xx_flush()
66 static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)  in a5xx_submit_in_rb()  argument
78 			if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)  in a5xx_submit_in_rb()
112 	a5xx_flush(gpu, ring, true);  in a5xx_submit_in_rb()
113 	a5xx_preempt_trigger(gpu);  in a5xx_submit_in_rb()
119 	a5xx_idle(gpu, ring);  in a5xx_submit_in_rb()
121 	msm_gpu_retire(gpu);  in a5xx_submit_in_rb()
124 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)  in a5xx_submit()  argument
126 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_submit()
132 		gpu->cur_ctx_seqno = 0;  in a5xx_submit()
133 		a5xx_submit_in_rb(gpu, submit);  in a5xx_submit()
167 			if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)  in a5xx_submit()
187 			update_shadow_rptr(gpu, ring);  in a5xx_submit()
236 	a5xx_flush(gpu, ring, false);  in a5xx_submit()
239 	a5xx_preempt_trigger(gpu);  in a5xx_submit()
436 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)  in a5xx_set_hwcg()  argument
438 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_set_hwcg()
454 		gpu_write(gpu, regs[i].offset,  in a5xx_set_hwcg()
458 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);  in a5xx_set_hwcg()
459 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);  in a5xx_set_hwcg()
462 	gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);  in a5xx_set_hwcg()
463 	gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);  in a5xx_set_hwcg()
466 static int a5xx_me_init(struct msm_gpu *gpu)  in a5xx_me_init()  argument
468 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_me_init()
469 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_me_init()
503 	a5xx_flush(gpu, ring, true);  in a5xx_me_init()
504 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_me_init()
507 static int a5xx_preempt_start(struct msm_gpu *gpu)  in a5xx_preempt_start()  argument
509 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_preempt_start()
511 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_preempt_start()
513 	if (gpu->nr_rings == 1)  in a5xx_preempt_start()
546 	a5xx_flush(gpu, ring, false);  in a5xx_preempt_start()
548 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_preempt_start()
570 static int a5xx_ucode_init(struct msm_gpu *gpu)  in a5xx_ucode_init()  argument
572 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_ucode_init()
577 		a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_init()
584 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",  in a5xx_ucode_init()
593 		a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_init()
599 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",  in a5xx_ucode_init()
608 	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,  in a5xx_ucode_init()
611 	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,  in a5xx_ucode_init()
619 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)  in a5xx_zap_shader_resume()  argument
621 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_zap_shader_resume()
634 			gpu->name, ret);  in a5xx_zap_shader_resume()
639 static int a5xx_zap_shader_init(struct msm_gpu *gpu)  in a5xx_zap_shader_init()  argument
649 		return a5xx_zap_shader_resume(gpu);  in a5xx_zap_shader_init()
651 	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);  in a5xx_zap_shader_init()
670 static int a5xx_hw_init(struct msm_gpu *gpu)  in a5xx_hw_init()  argument
672 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_hw_init()
677 	gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);  in a5xx_hw_init()
681 		gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);  in a5xx_hw_init()
683 	/* Make all blocks contribute to the GPU BUSY perf counter */  in a5xx_hw_init()
684 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);  in a5xx_hw_init()
687 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);  in a5xx_hw_init()
695 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,  in a5xx_hw_init()
697 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,  in a5xx_hw_init()
699 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,  in a5xx_hw_init()
701 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,  in a5xx_hw_init()
703 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,  in a5xx_hw_init()
705 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,  in a5xx_hw_init()
707 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,  in a5xx_hw_init()
709 		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,  in a5xx_hw_init()
714 	gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,  in a5xx_hw_init()
718 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);  in a5xx_hw_init()
721 	gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);  in a5xx_hw_init()
724 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);  in a5xx_hw_init()
727 	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);  in a5xx_hw_init()
730 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);  in a5xx_hw_init()
731 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);  in a5xx_hw_init()
732 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);  in a5xx_hw_init()
733 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);  in a5xx_hw_init()
735 	/* Set the GMEM VA range (0 to gpu->gmem) */  in a5xx_hw_init()
736 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);  in a5xx_hw_init()
737 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);  in a5xx_hw_init()
738 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,  in a5xx_hw_init()
740 	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);  in a5xx_hw_init()
744 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);  in a5xx_hw_init()
746 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);  in a5xx_hw_init()
748 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);  in a5xx_hw_init()
749 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);  in a5xx_hw_init()
750 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);  in a5xx_hw_init()
752 		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);  in a5xx_hw_init()
754 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);  in a5xx_hw_init()
756 			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);  in a5xx_hw_init()
757 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);  in a5xx_hw_init()
758 		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);  in a5xx_hw_init()
762 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,  in a5xx_hw_init()
766 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,  in a5xx_hw_init()
769 		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,  in a5xx_hw_init()
773 		gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));  in a5xx_hw_init()
781 		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9));  in a5xx_hw_init()
784 	gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29));  in a5xx_hw_init()
787 	gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);  in a5xx_hw_init()
790 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);  in a5xx_hw_init()
797 	 *  CCU to be interpreted differently. This can cause gpu fault. This  in a5xx_hw_init()
803 		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);  in a5xx_hw_init()
806 	a5xx_set_hwcg(gpu, true);  in a5xx_hw_init()
808 	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);  in a5xx_hw_init()
816 	gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, regbit << 7);  in a5xx_hw_init()
817 	gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, regbit << 1);  in a5xx_hw_init()
821 		gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, regbit);  in a5xx_hw_init()
824 	gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10));  in a5xx_hw_init()
827 	gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);  in a5xx_hw_init()
830 	gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));  in a5xx_hw_init()
831 	gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));  in a5xx_hw_init()
832 	gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));  in a5xx_hw_init()
833 	gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));  in a5xx_hw_init()
834 	gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));  in a5xx_hw_init()
835 	gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));  in a5xx_hw_init()
838 	gpu_write(gpu, REG_A5XX_CP_PROTECT(6),  in a5xx_hw_init()
841 	gpu_write(gpu, REG_A5XX_CP_PROTECT(7),  in a5xx_hw_init()
845 	gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));  in a5xx_hw_init()
846 	gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));  in a5xx_hw_init()
847 	gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));  in a5xx_hw_init()
848 	gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));  in a5xx_hw_init()
851 	gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));  in a5xx_hw_init()
852 	gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));  in a5xx_hw_init()
855 	gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));  in a5xx_hw_init()
856 	gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16));  in a5xx_hw_init()
859 	gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));  in a5xx_hw_init()
862 	gpu_write(gpu, REG_A5XX_CP_PROTECT(17),  in a5xx_hw_init()
865 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);  in a5xx_hw_init()
871 	gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,  in a5xx_hw_init()
873 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);  in a5xx_hw_init()
875 	/* Put the GPU into 64 bit by default */  in a5xx_hw_init()
876 	gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
877 	gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
878 	gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
879 	gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
880 	gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
881 	gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
882 	gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
883 	gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
884 	gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
885 	gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
886 	gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
887 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);  in a5xx_hw_init()
895 		gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));  in a5xx_hw_init()
896 		gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);  in a5xx_hw_init()
899 	ret = adreno_hw_init(gpu);  in a5xx_hw_init()
904 		a5xx_gpmu_ucode_init(gpu);  in a5xx_hw_init()
906 	ret = a5xx_ucode_init(gpu);  in a5xx_hw_init()
911 	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, REG_A5XX_CP_RB_BASE_HI,  in a5xx_hw_init()
912 		gpu->rb[0]->iova);  in a5xx_hw_init()
920 	gpu_write(gpu, REG_A5XX_CP_RB_CNTL,  in a5xx_hw_init()
926 			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,  in a5xx_hw_init()
927 				sizeof(u32) * gpu->nr_rings,  in a5xx_hw_init()
929 				gpu->aspace, &a5xx_gpu->shadow_bo,  in a5xx_hw_init()
938 		gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,  in a5xx_hw_init()
939 			REG_A5XX_CP_RB_RPTR_ADDR_HI, shadowptr(a5xx_gpu, gpu->rb[0]));  in a5xx_hw_init()
940 	} else if (gpu->nr_rings > 1) {  in a5xx_hw_init()
942 		a5xx_preempt_fini(gpu);  in a5xx_hw_init()
943 		gpu->nr_rings = 1;  in a5xx_hw_init()
946 	a5xx_preempt_hw_init(gpu);  in a5xx_hw_init()
949 	gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);  in a5xx_hw_init()
952 	gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);  in a5xx_hw_init()
953 	ret = a5xx_me_init(gpu);  in a5xx_hw_init()
957 	ret = a5xx_power_init(gpu);  in a5xx_hw_init()
966 		OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);  in a5xx_hw_init()
967 		OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));  in a5xx_hw_init()
969 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
970 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
982 	ret = a5xx_zap_shader_init(gpu);  in a5xx_hw_init()
984 		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);  in a5xx_hw_init()
985 		OUT_RING(gpu->rb[0], 0x00000000);  in a5xx_hw_init()
987 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
988 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
997 		dev_warn_once(gpu->dev->dev,  in a5xx_hw_init()
999 		gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);  in a5xx_hw_init()
1005 	a5xx_preempt_start(gpu);  in a5xx_hw_init()
1010 static void a5xx_recover(struct msm_gpu *gpu)  in a5xx_recover()  argument
1014 	adreno_dump_info(gpu);  in a5xx_recover()
1018 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));  in a5xx_recover()
1022 		a5xx_dump(gpu);  in a5xx_recover()
1024 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);  in a5xx_recover()
1025 	gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);  in a5xx_recover()
1026 	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);  in a5xx_recover()
1027 	adreno_recover(gpu);  in a5xx_recover()
1030 static void a5xx_destroy(struct msm_gpu *gpu)  in a5xx_destroy()  argument
1032 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_destroy()
1035 	DBG("%s", gpu->name);  in a5xx_destroy()
1037 	a5xx_preempt_fini(gpu);  in a5xx_destroy()
1040 		msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);  in a5xx_destroy()
1045 		msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);  in a5xx_destroy()
1050 		msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);  in a5xx_destroy()
1055 		msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);  in a5xx_destroy()
1063 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)  in _a5xx_check_idle()  argument
1065 	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)  in _a5xx_check_idle()
1069 	 * Nearly every abnormality ends up pausing the GPU and triggering a  in _a5xx_check_idle()
1072 	return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &  in _a5xx_check_idle()
1076 bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a5xx_idle()  argument
1078 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_idle()
1087 	if (!adreno_idle(gpu, ring))  in a5xx_idle()
1090 	if (spin_until(_a5xx_check_idle(gpu))) {  in a5xx_idle()
1091 		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",  in a5xx_idle()
1092 			gpu->name, __builtin_return_address(0),  in a5xx_idle()
1093 			gpu_read(gpu, REG_A5XX_RBBM_STATUS),  in a5xx_idle()
1094 			gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),  in a5xx_idle()
1095 			gpu_read(gpu, REG_A5XX_CP_RB_RPTR),  in a5xx_idle()
1096 			gpu_read(gpu, REG_A5XX_CP_RB_WPTR));  in a5xx_idle()
1105 	struct msm_gpu *gpu = arg;  in a5xx_fault_handler()  local
1106 	pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",  in a5xx_fault_handler()
1108 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),  in a5xx_fault_handler()
1109 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),  in a5xx_fault_handler()
1110 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),  in a5xx_fault_handler()
1111 			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));  in a5xx_fault_handler()
1116 static void a5xx_cp_err_irq(struct msm_gpu *gpu)  in a5xx_cp_err_irq()  argument
1118 	u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);  in a5xx_cp_err_irq()
1123 		gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);  in a5xx_cp_err_irq()
1130 		gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);  in a5xx_cp_err_irq()
1131 		val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);  in a5xx_cp_err_irq()
1133 		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",  in a5xx_cp_err_irq()
1138 		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",  in a5xx_cp_err_irq()
1139 			gpu_read(gpu, REG_A5XX_CP_HW_FAULT));  in a5xx_cp_err_irq()
1142 		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");  in a5xx_cp_err_irq()
1145 		u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);  in a5xx_cp_err_irq()
1147 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
1154 		u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);  in a5xx_cp_err_irq()
1160 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
1167 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)  in a5xx_rbbm_err_irq()  argument
1170 		u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);  in a5xx_rbbm_err_irq()
1172 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_rbbm_err_irq()
1179 		gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));  in a5xx_rbbm_err_irq()
1182 		gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,  in a5xx_rbbm_err_irq()
1187 		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");  in a5xx_rbbm_err_irq()
1190 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1191 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1194 		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1195 			gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1198 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1199 			gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));  in a5xx_rbbm_err_irq()
1202 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");  in a5xx_rbbm_err_irq()
1205 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");  in a5xx_rbbm_err_irq()
1208 static void a5xx_uche_err_irq(struct msm_gpu *gpu)  in a5xx_uche_err_irq()  argument
1210 	uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);  in a5xx_uche_err_irq()
1212 	addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);  in a5xx_uche_err_irq()
1214 	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",  in a5xx_uche_err_irq()
1218 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)  in a5xx_gpmu_err_irq()  argument
1220 	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");  in a5xx_gpmu_err_irq()
1223 static void a5xx_fault_detect_irq(struct msm_gpu *gpu)  in a5xx_fault_detect_irq()  argument
1225 	struct drm_device *dev = gpu->dev;  in a5xx_fault_detect_irq()
1226 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in a5xx_fault_detect_irq()
1229 	 * If stalled on SMMU fault, we could trip the GPU's hang detection,  in a5xx_fault_detect_irq()
1234 	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24))  in a5xx_fault_detect_irq()
1237 …DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4…  in a5xx_fault_detect_irq()
1239 		gpu_read(gpu, REG_A5XX_RBBM_STATUS),  in a5xx_fault_detect_irq()
1240 		gpu_read(gpu, REG_A5XX_CP_RB_RPTR),  in a5xx_fault_detect_irq()
1241 		gpu_read(gpu, REG_A5XX_CP_RB_WPTR),  in a5xx_fault_detect_irq()
1242 		gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI),  in a5xx_fault_detect_irq()
1243 		gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),  in a5xx_fault_detect_irq()
1244 		gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI),  in a5xx_fault_detect_irq()
1245 		gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));  in a5xx_fault_detect_irq()
1248 	del_timer(&gpu->hangcheck_timer);  in a5xx_fault_detect_irq()
1250 	kthread_queue_work(gpu->worker, &gpu->recover_work);  in a5xx_fault_detect_irq()
1261 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)  in a5xx_irq()  argument
1263 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a5xx_irq()
1264 	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);  in a5xx_irq()
1270 	gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,  in a5xx_irq()
1280 		a5xx_rbbm_err_irq(gpu, status);  in a5xx_irq()
1283 		a5xx_cp_err_irq(gpu);  in a5xx_irq()
1286 		a5xx_fault_detect_irq(gpu);  in a5xx_irq()
1289 		a5xx_uche_err_irq(gpu);  in a5xx_irq()
1292 		a5xx_gpmu_err_irq(gpu);  in a5xx_irq()
1295 		a5xx_preempt_trigger(gpu);  in a5xx_irq()
1296 		msm_gpu_retire(gpu);  in a5xx_irq()
1300 		a5xx_preempt_irq(gpu);  in a5xx_irq()
1336 static void a5xx_dump(struct msm_gpu *gpu)  in a5xx_dump()  argument
1338 	DRM_DEV_INFO(gpu->dev->dev, "status:   %08x\n",  in a5xx_dump()
1339 		gpu_read(gpu, REG_A5XX_RBBM_STATUS));  in a5xx_dump()
1340 	adreno_dump(gpu);  in a5xx_dump()
1343 static int a5xx_pm_resume(struct msm_gpu *gpu)  in a5xx_pm_resume()  argument
1345 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_pm_resume()
1349 	ret = msm_gpu_pm_resume(gpu);  in a5xx_pm_resume()
1356 		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);  in a5xx_pm_resume()
1357 		a5xx_set_hwcg(gpu, true);  in a5xx_pm_resume()
1359 		gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);  in a5xx_pm_resume()
1364 	gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);  in a5xx_pm_resume()
1369 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,  in a5xx_pm_resume()
1373 			gpu->name,  in a5xx_pm_resume()
1374 			gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));  in a5xx_pm_resume()
1379 	gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);  in a5xx_pm_resume()
1380 	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,  in a5xx_pm_resume()
1384 			gpu->name);  in a5xx_pm_resume()
1389 static int a5xx_pm_suspend(struct msm_gpu *gpu)  in a5xx_pm_suspend()  argument
1391 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_pm_suspend()
1402 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);  in a5xx_pm_suspend()
1403 	spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &  in a5xx_pm_suspend()
1406 	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);  in a5xx_pm_suspend()
1413 		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);  in a5xx_pm_suspend()
1414 		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);  in a5xx_pm_suspend()
1417 	ret = msm_gpu_pm_suspend(gpu);  in a5xx_pm_suspend()
1422 		for (i = 0; i < gpu->nr_rings; i++)  in a5xx_pm_suspend()
1428 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)  in a5xx_get_timestamp()  argument
1430 	*value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO,  in a5xx_get_timestamp()
1447 static int a5xx_crashdumper_init(struct msm_gpu *gpu,  in a5xx_crashdumper_init()  argument
1450 	dumper->ptr = msm_gem_kernel_new(gpu->dev,  in a5xx_crashdumper_init()
1451 		SZ_1M, MSM_BO_WC, gpu->aspace,  in a5xx_crashdumper_init()
1460 static int a5xx_crashdumper_run(struct msm_gpu *gpu,  in a5xx_crashdumper_run()  argument
1468 	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO,  in a5xx_crashdumper_run()
1471 	gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);  in a5xx_crashdumper_run()
1473 	return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,  in a5xx_crashdumper_run()
1504 static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,  in a5xx_gpu_state_get_hlsq_regs()  argument
1512 	if (a5xx_crashdumper_init(gpu, &dumper))  in a5xx_gpu_state_get_hlsq_regs()
1550 	if (a5xx_crashdumper_run(gpu, &dumper)) {  in a5xx_gpu_state_get_hlsq_regs()
1552 		msm_gem_kernel_put(dumper.bo, gpu->aspace);  in a5xx_gpu_state_get_hlsq_regs()
1560 	msm_gem_kernel_put(dumper.bo, gpu->aspace);  in a5xx_gpu_state_get_hlsq_regs()
1563 static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)  in a5xx_gpu_state_get()  argument
1567 	bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24));  in a5xx_gpu_state_get()
1573 	a5xx_set_hwcg(gpu, false);  in a5xx_gpu_state_get()
1576 	adreno_gpu_state_get(gpu, &(a5xx_state->base));  in a5xx_gpu_state_get()
1578 	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);  in a5xx_gpu_state_get()
1586 		a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);  in a5xx_gpu_state_get()
1588 	a5xx_set_hwcg(gpu, true);  in a5xx_gpu_state_get()
1616 static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,  in a5xx_show()  argument
1627 	adreno_show(gpu, state, p);  in a5xx_show()
1657 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)  in a5xx_active_ring()  argument
1659 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_active_ring()
1665 static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)  in a5xx_gpu_busy()  argument
1669 	busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,  in a5xx_gpu_busy()
1671 	*out_sample_rate = clk_get_rate(gpu->core_clk);  in a5xx_gpu_busy()
1676 static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a5xx_get_rptr()  argument
1678 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a5xx_get_rptr()
1684 	return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);  in a5xx_get_rptr()
1752 	struct msm_gpu *gpu;  in a5xx_gpu_init()  local
1765 	gpu = &adreno_gpu->base;  in a5xx_gpu_init()
1779 	if (gpu->aspace)  in a5xx_gpu_init()
1780 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);  in a5xx_gpu_init()
1783 	a5xx_preempt_init(gpu);  in a5xx_gpu_init()
1785 	return gpu;  in a5xx_gpu_init()