15 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)  in _a6xx_check_idle()  argument
17 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in _a6xx_check_idle()
25 	if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &  in _a6xx_check_idle()
29 	return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &  in _a6xx_check_idle()
33 bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a6xx_idle()  argument
36 	if (!adreno_idle(gpu, ring))  in a6xx_idle()
39 	if (spin_until(_a6xx_check_idle(gpu))) {  in a6xx_idle()
41 			gpu->name, __builtin_return_address(0),  in a6xx_idle()
42 			gpu_read(gpu, REG_A6XX_RBBM_STATUS),  in a6xx_idle()
43 			gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),  in a6xx_idle()
44 			gpu_read(gpu, REG_A6XX_CP_RB_RPTR),  in a6xx_idle()
45 			gpu_read(gpu, REG_A6XX_CP_RB_WPTR));  in a6xx_idle()
52 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a6xx_flush()  argument
70 	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);  in a6xx_flush()
82 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,  in a6xx_submit()  argument
86 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a6xx_submit()
87 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_submit()
151 	a6xx_flush(gpu, ring);  in a6xx_submit()
265 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)  in a6xx_set_hwcg()  argument
267 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_set_hwcg()
273 	val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);  in a6xx_set_hwcg()
283 		gpu_write(gpu, a6xx_hwcg[i].offset,  in a6xx_set_hwcg()
289 	gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0);  in a6xx_set_hwcg()
292 static int a6xx_cp_init(struct msm_gpu *gpu)  in a6xx_cp_init()  argument
294 	struct msm_ringbuffer *ring = gpu->rb[0];  in a6xx_cp_init()
317 	a6xx_flush(gpu, ring);  in a6xx_cp_init()
318 	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;  in a6xx_cp_init()
321 static int a6xx_ucode_init(struct msm_gpu *gpu)  in a6xx_ucode_init()  argument
323 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_ucode_init()
327 		a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,  in a6xx_ucode_init()
334 			DRM_DEV_ERROR(&gpu->pdev->dev,  in a6xx_ucode_init()
343 	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO,  in a6xx_ucode_init()
349 static int a6xx_zap_shader_init(struct msm_gpu *gpu)  in a6xx_zap_shader_init()  argument
357 	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);  in a6xx_zap_shader_init()
375 static int a6xx_hw_init(struct msm_gpu *gpu)  in a6xx_hw_init()  argument
377 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_hw_init()
384 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);  in a6xx_hw_init()
391 	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,  in a6xx_hw_init()
393 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);  in a6xx_hw_init()
396 	gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
397 	gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
398 	gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
399 	gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
400 	gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
401 	gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
402 	gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
403 	gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
404 	gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
405 	gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
406 	gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
407 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);  in a6xx_hw_init()
410 	a6xx_set_hwcg(gpu, true);  in a6xx_hw_init()
413 	gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);  in a6xx_hw_init()
414 	gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);  in a6xx_hw_init()
417 	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);  in a6xx_hw_init()
420 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);  in a6xx_hw_init()
421 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);  in a6xx_hw_init()
422 	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);  in a6xx_hw_init()
423 	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);  in a6xx_hw_init()
424 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);  in a6xx_hw_init()
425 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);  in a6xx_hw_init()
428 	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,  in a6xx_hw_init()
431 	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,  in a6xx_hw_init()
435 	gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);  in a6xx_hw_init()
436 	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);  in a6xx_hw_init()
438 	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);  in a6xx_hw_init()
439 	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);  in a6xx_hw_init()
442 	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);  in a6xx_hw_init()
445 	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));  in a6xx_hw_init()
448 	gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);  in a6xx_hw_init()
451 	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);  in a6xx_hw_init()
454 	gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);  in a6xx_hw_init()
456 	gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
457 	gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
458 	gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
459 	gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21);  in a6xx_hw_init()
462 	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,  in a6xx_hw_init()
465 	gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);  in a6xx_hw_init()
468 	gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);  in a6xx_hw_init()
470 	gpu_write(gpu, REG_A6XX_CP_PROTECT(0),  in a6xx_hw_init()
472 	gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2));  in a6xx_hw_init()
473 	gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13));  in a6xx_hw_init()
474 	gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8));  in a6xx_hw_init()
475 	gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1));  in a6xx_hw_init()
476 	gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187));  in a6xx_hw_init()
477 	gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810));  in a6xx_hw_init()
478 	gpu_write(gpu, REG_A6XX_CP_PROTECT(7),  in a6xx_hw_init()
480 	gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0));  in a6xx_hw_init()
481 	gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0));  in a6xx_hw_init()
482 	gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0));  in a6xx_hw_init()
483 	gpu_write(gpu, REG_A6XX_CP_PROTECT(11),  in a6xx_hw_init()
485 	gpu_write(gpu, REG_A6XX_CP_PROTECT(12),  in a6xx_hw_init()
487 	gpu_write(gpu, REG_A6XX_CP_PROTECT(13),  in a6xx_hw_init()
489 	gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe));  in a6xx_hw_init()
490 	gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0));  in a6xx_hw_init()
491 	gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf));  in a6xx_hw_init()
492 	gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0));  in a6xx_hw_init()
493 	gpu_write(gpu, REG_A6XX_CP_PROTECT(18),  in a6xx_hw_init()
495 	gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82));  in a6xx_hw_init()
496 	gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8));  in a6xx_hw_init()
497 	gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19));  in a6xx_hw_init()
498 	gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d));  in a6xx_hw_init()
499 	gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76));  in a6xx_hw_init()
500 	gpu_write(gpu, REG_A6XX_CP_PROTECT(24),  in a6xx_hw_init()
502 	gpu_write(gpu, REG_A6XX_CP_PROTECT(25), A6XX_PROTECT_RW(0xa630, 0x0));  in a6xx_hw_init()
505 	gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);  in a6xx_hw_init()
507 	ret = adreno_hw_init(gpu);  in a6xx_hw_init()
511 	ret = a6xx_ucode_init(gpu);  in a6xx_hw_init()
516 	a6xx_gpu->cur_ring = gpu->rb[0];  in a6xx_hw_init()
519 	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);  in a6xx_hw_init()
521 	ret = a6xx_cp_init(gpu);  in a6xx_hw_init()
532 	ret = a6xx_zap_shader_init(gpu);  in a6xx_hw_init()
534 		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);  in a6xx_hw_init()
535 		OUT_RING(gpu->rb[0], 0x00000000);  in a6xx_hw_init()
537 		a6xx_flush(gpu, gpu->rb[0]);  in a6xx_hw_init()
538 		if (!a6xx_idle(gpu, gpu->rb[0]))  in a6xx_hw_init()
542 		dev_warn_once(gpu->dev->dev,  in a6xx_hw_init()
544 		gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);  in a6xx_hw_init()
561 static void a6xx_dump(struct msm_gpu *gpu)  in a6xx_dump()  argument
563 	DRM_DEV_INFO(&gpu->pdev->dev, "status:   %08x\n",  in a6xx_dump()
564 			gpu_read(gpu, REG_A6XX_RBBM_STATUS));  in a6xx_dump()
565 	adreno_dump(gpu);  in a6xx_dump()
571 static void a6xx_recover(struct msm_gpu *gpu)  in a6xx_recover()  argument
573 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_recover()
577 	adreno_dump_info(gpu);  in a6xx_recover()
580 		DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,  in a6xx_recover()
581 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));  in a6xx_recover()
584 		a6xx_dump(gpu);  in a6xx_recover()
592 	gpu->funcs->pm_suspend(gpu);  in a6xx_recover()
593 	gpu->funcs->pm_resume(gpu);  in a6xx_recover()
595 	msm_gpu_hw_init(gpu);  in a6xx_recover()
600 	struct msm_gpu *gpu = arg;  in a6xx_fault_handler()  local
604 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),  in a6xx_fault_handler()
605 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),  in a6xx_fault_handler()
606 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),  in a6xx_fault_handler()
607 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));  in a6xx_fault_handler()
612 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)  in a6xx_cp_hw_err_irq()  argument
614 	u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);  in a6xx_cp_hw_err_irq()
619 		gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);  in a6xx_cp_hw_err_irq()
620 		val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);  in a6xx_cp_hw_err_irq()
621 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
627 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
631 		dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",  in a6xx_cp_hw_err_irq()
632 			gpu_read(gpu, REG_A6XX_CP_HW_FAULT));  in a6xx_cp_hw_err_irq()
635 		u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);  in a6xx_cp_hw_err_irq()
637 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
644 		dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");  in a6xx_cp_hw_err_irq()
647 		dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");  in a6xx_cp_hw_err_irq()
650 		dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");  in a6xx_cp_hw_err_irq()
654 static void a6xx_fault_detect_irq(struct msm_gpu *gpu)  in a6xx_fault_detect_irq()  argument
656 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_fault_detect_irq()
658 	struct drm_device *dev = gpu->dev;  in a6xx_fault_detect_irq()
660 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in a6xx_fault_detect_irq()
668 	DRM_DEV_ERROR(&gpu->pdev->dev,  in a6xx_fault_detect_irq()
671 		gpu_read(gpu, REG_A6XX_RBBM_STATUS),  in a6xx_fault_detect_irq()
672 		gpu_read(gpu, REG_A6XX_CP_RB_RPTR),  in a6xx_fault_detect_irq()
673 		gpu_read(gpu, REG_A6XX_CP_RB_WPTR),  in a6xx_fault_detect_irq()
674 		gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),  in a6xx_fault_detect_irq()
675 		gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),  in a6xx_fault_detect_irq()
676 		gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),  in a6xx_fault_detect_irq()
677 		gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));  in a6xx_fault_detect_irq()
680 	del_timer(&gpu->hangcheck_timer);  in a6xx_fault_detect_irq()
682 	queue_work(priv->wq, &gpu->recover_work);  in a6xx_fault_detect_irq()
685 static irqreturn_t a6xx_irq(struct msm_gpu *gpu)  in a6xx_irq()  argument
687 	u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);  in a6xx_irq()
689 	gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);  in a6xx_irq()
692 		a6xx_fault_detect_irq(gpu);  in a6xx_irq()
695 		dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");  in a6xx_irq()
698 		a6xx_cp_hw_err_irq(gpu);  in a6xx_irq()
701 		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");  in a6xx_irq()
704 		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");  in a6xx_irq()
707 		dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");  in a6xx_irq()
710 		msm_gpu_retire(gpu);  in a6xx_irq()
727 static int a6xx_pm_resume(struct msm_gpu *gpu)  in a6xx_pm_resume()  argument
729 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_pm_resume()
733 	gpu->needs_hw_init = true;  in a6xx_pm_resume()
739 	msm_gpu_resume_devfreq(gpu);  in a6xx_pm_resume()
744 static int a6xx_pm_suspend(struct msm_gpu *gpu)  in a6xx_pm_suspend()  argument
746 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_pm_suspend()
749 	devfreq_suspend_device(gpu->devfreq.devfreq);  in a6xx_pm_suspend()
754 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)  in a6xx_get_timestamp()  argument
756 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_get_timestamp()
762 	*value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO,  in a6xx_get_timestamp()
769 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)  in a6xx_active_ring()  argument
771 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_active_ring()
777 static void a6xx_destroy(struct msm_gpu *gpu)  in a6xx_destroy()  argument
779 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_destroy()
783 		msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);  in a6xx_destroy()
793 static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)  in a6xx_gpu_busy()  argument
795 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_gpu_busy()
803 	busy_time = (busy_cycles - gpu->devfreq.busy_cycles) * 10;  in a6xx_gpu_busy()
806 	gpu->devfreq.busy_cycles = busy_cycles;  in a6xx_gpu_busy()
847 	struct msm_gpu *gpu;  in a6xx_gpu_init()  local
855 	gpu = &adreno_gpu->base;  in a6xx_gpu_init()
878 	if (gpu->aspace)  in a6xx_gpu_init()
879 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,  in a6xx_gpu_init()
882 	return gpu;  in a6xx_gpu_init()