10 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)  in _a6xx_check_idle()  argument
12 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in _a6xx_check_idle()
20 	if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &  in _a6xx_check_idle()
24 	return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &  in _a6xx_check_idle()
28 bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a6xx_idle()  argument
31 	if (!adreno_idle(gpu, ring))  in a6xx_idle()
34 	if (spin_until(_a6xx_check_idle(gpu))) {  in a6xx_idle()
36 			gpu->name, __builtin_return_address(0),  in a6xx_idle()
37 			gpu_read(gpu, REG_A6XX_RBBM_STATUS),  in a6xx_idle()
38 			gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),  in a6xx_idle()
39 			gpu_read(gpu, REG_A6XX_CP_RB_RPTR),  in a6xx_idle()
40 			gpu_read(gpu, REG_A6XX_CP_RB_WPTR));  in a6xx_idle()
47 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)  in a6xx_flush()  argument
65 	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);  in a6xx_flush()
68 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,  in a6xx_submit()  argument
71 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a6xx_submit()
113 	a6xx_flush(gpu, ring);  in a6xx_submit()
227 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)  in a6xx_set_hwcg()  argument
229 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_set_hwcg()
235 	val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);  in a6xx_set_hwcg()
245 		gpu_write(gpu, a6xx_hwcg[i].offset,  in a6xx_set_hwcg()
251 	gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0);  in a6xx_set_hwcg()
254 static int a6xx_cp_init(struct msm_gpu *gpu)  in a6xx_cp_init()  argument
256 	struct msm_ringbuffer *ring = gpu->rb[0];  in a6xx_cp_init()
279 	a6xx_flush(gpu, ring);  in a6xx_cp_init()
280 	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;  in a6xx_cp_init()
283 static int a6xx_ucode_init(struct msm_gpu *gpu)  in a6xx_ucode_init()  argument
285 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_ucode_init()
289 		a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,  in a6xx_ucode_init()
296 			DRM_DEV_ERROR(&gpu->pdev->dev,  in a6xx_ucode_init()
303 	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO,  in a6xx_ucode_init()
321 static int a6xx_hw_init(struct msm_gpu *gpu)  in a6xx_hw_init()  argument
323 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_hw_init()
330 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);  in a6xx_hw_init()
337 	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,  in a6xx_hw_init()
339 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);  in a6xx_hw_init()
342 	a6xx_set_hwcg(gpu, true);  in a6xx_hw_init()
345 	gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);  in a6xx_hw_init()
346 	gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);  in a6xx_hw_init()
349 	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);  in a6xx_hw_init()
352 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);  in a6xx_hw_init()
353 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);  in a6xx_hw_init()
354 	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);  in a6xx_hw_init()
355 	gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);  in a6xx_hw_init()
356 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);  in a6xx_hw_init()
357 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);  in a6xx_hw_init()
360 	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,  in a6xx_hw_init()
363 	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,  in a6xx_hw_init()
367 	gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);  in a6xx_hw_init()
368 	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);  in a6xx_hw_init()
370 	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);  in a6xx_hw_init()
371 	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);  in a6xx_hw_init()
374 	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);  in a6xx_hw_init()
377 	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));  in a6xx_hw_init()
380 	gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);  in a6xx_hw_init()
383 	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);  in a6xx_hw_init()
386 	gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);  in a6xx_hw_init()
396 	gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
397 	gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
398 	gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1);  in a6xx_hw_init()
399 	gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21);  in a6xx_hw_init()
402 	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,  in a6xx_hw_init()
405 	gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);  in a6xx_hw_init()
408 	gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);  in a6xx_hw_init()
410 	gpu_write(gpu, REG_A6XX_CP_PROTECT(0),  in a6xx_hw_init()
412 	gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2));  in a6xx_hw_init()
413 	gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13));  in a6xx_hw_init()
414 	gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8));  in a6xx_hw_init()
415 	gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1));  in a6xx_hw_init()
416 	gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187));  in a6xx_hw_init()
417 	gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810));  in a6xx_hw_init()
418 	gpu_write(gpu, REG_A6XX_CP_PROTECT(7),  in a6xx_hw_init()
420 	gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0));  in a6xx_hw_init()
421 	gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0));  in a6xx_hw_init()
422 	gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0));  in a6xx_hw_init()
423 	gpu_write(gpu, REG_A6XX_CP_PROTECT(11),  in a6xx_hw_init()
425 	gpu_write(gpu, REG_A6XX_CP_PROTECT(12),  in a6xx_hw_init()
427 	gpu_write(gpu, REG_A6XX_CP_PROTECT(13),  in a6xx_hw_init()
429 	gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe));  in a6xx_hw_init()
430 	gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0));  in a6xx_hw_init()
431 	gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf));  in a6xx_hw_init()
432 	gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0));  in a6xx_hw_init()
433 	gpu_write(gpu, REG_A6XX_CP_PROTECT(18),  in a6xx_hw_init()
435 	gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82));  in a6xx_hw_init()
436 	gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8));  in a6xx_hw_init()
437 	gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19));  in a6xx_hw_init()
438 	gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d));  in a6xx_hw_init()
439 	gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76));  in a6xx_hw_init()
440 	gpu_write(gpu, REG_A6XX_CP_PROTECT(24),  in a6xx_hw_init()
442 	gpu_write(gpu, REG_A6XX_CP_PROTECT(25),  in a6xx_hw_init()
444 	gpu_write(gpu, REG_A6XX_CP_PROTECT(26), A6XX_PROTECT_RW(0xa630, 0x0));  in a6xx_hw_init()
447 	gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);  in a6xx_hw_init()
449 	ret = adreno_hw_init(gpu);  in a6xx_hw_init()
453 	ret = a6xx_ucode_init(gpu);  in a6xx_hw_init()
458 	a6xx_gpu->cur_ring = gpu->rb[0];  in a6xx_hw_init()
461 	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);  in a6xx_hw_init()
463 	ret = a6xx_cp_init(gpu);  in a6xx_hw_init()
467 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);  in a6xx_hw_init()
482 static void a6xx_dump(struct msm_gpu *gpu)  in a6xx_dump()  argument
484 	dev_info(&gpu->pdev->dev, "status:   %08x\n",  in a6xx_dump()
485 			gpu_read(gpu, REG_A6XX_RBBM_STATUS));  in a6xx_dump()
486 	adreno_dump(gpu);  in a6xx_dump()
492 static void a6xx_recover(struct msm_gpu *gpu)  in a6xx_recover()  argument
494 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_recover()
498 	adreno_dump_info(gpu);  in a6xx_recover()
501 		dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,  in a6xx_recover()
502 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));  in a6xx_recover()
505 		a6xx_dump(gpu);  in a6xx_recover()
513 	gpu->funcs->pm_suspend(gpu);  in a6xx_recover()
514 	gpu->funcs->pm_resume(gpu);  in a6xx_recover()
516 	msm_gpu_hw_init(gpu);  in a6xx_recover()
521 	struct msm_gpu *gpu = arg;  in a6xx_fault_handler()  local
525 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),  in a6xx_fault_handler()
526 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),  in a6xx_fault_handler()
527 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),  in a6xx_fault_handler()
528 			gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));  in a6xx_fault_handler()
533 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)  in a6xx_cp_hw_err_irq()  argument
535 	u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);  in a6xx_cp_hw_err_irq()
540 		gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);  in a6xx_cp_hw_err_irq()
541 		val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);  in a6xx_cp_hw_err_irq()
542 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
548 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
552 		dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",  in a6xx_cp_hw_err_irq()
553 			gpu_read(gpu, REG_A6XX_CP_HW_FAULT));  in a6xx_cp_hw_err_irq()
556 		u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);  in a6xx_cp_hw_err_irq()
558 		dev_err_ratelimited(&gpu->pdev->dev,  in a6xx_cp_hw_err_irq()
565 		dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");  in a6xx_cp_hw_err_irq()
568 		dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");  in a6xx_cp_hw_err_irq()
571 		dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");  in a6xx_cp_hw_err_irq()
575 static void a6xx_fault_detect_irq(struct msm_gpu *gpu)  in a6xx_fault_detect_irq()  argument
577 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_fault_detect_irq()
579 	struct drm_device *dev = gpu->dev;  in a6xx_fault_detect_irq()
581 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in a6xx_fault_detect_irq()
589 	DRM_DEV_ERROR(&gpu->pdev->dev,  in a6xx_fault_detect_irq()
592 		gpu_read(gpu, REG_A6XX_RBBM_STATUS),  in a6xx_fault_detect_irq()
593 		gpu_read(gpu, REG_A6XX_CP_RB_RPTR),  in a6xx_fault_detect_irq()
594 		gpu_read(gpu, REG_A6XX_CP_RB_WPTR),  in a6xx_fault_detect_irq()
595 		gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),  in a6xx_fault_detect_irq()
596 		gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),  in a6xx_fault_detect_irq()
597 		gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),  in a6xx_fault_detect_irq()
598 		gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));  in a6xx_fault_detect_irq()
601 	del_timer(&gpu->hangcheck_timer);  in a6xx_fault_detect_irq()
603 	queue_work(priv->wq, &gpu->recover_work);  in a6xx_fault_detect_irq()
606 static irqreturn_t a6xx_irq(struct msm_gpu *gpu)  in a6xx_irq()  argument
608 	u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);  in a6xx_irq()
610 	gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);  in a6xx_irq()
613 		a6xx_fault_detect_irq(gpu);  in a6xx_irq()
616 		dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");  in a6xx_irq()
619 		a6xx_cp_hw_err_irq(gpu);  in a6xx_irq()
622 		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");  in a6xx_irq()
625 		dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");  in a6xx_irq()
628 		dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");  in a6xx_irq()
631 		msm_gpu_retire(gpu);  in a6xx_irq()
675 static int a6xx_pm_resume(struct msm_gpu *gpu)  in a6xx_pm_resume()  argument
677 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_pm_resume()
683 	gpu->needs_hw_init = true;  in a6xx_pm_resume()
688 static int a6xx_pm_suspend(struct msm_gpu *gpu)  in a6xx_pm_suspend()  argument
690 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_pm_suspend()
701 	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);  in a6xx_pm_suspend()
702 	spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf);  in a6xx_pm_suspend()
703 	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);  in a6xx_pm_suspend()
708 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)  in a6xx_get_timestamp()  argument
710 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_get_timestamp()
716 	*value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO,  in a6xx_get_timestamp()
724 static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,  in a6xx_show()  argument
727 	adreno_show(gpu, state, p);  in a6xx_show()
731 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)  in a6xx_active_ring()  argument
733 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_active_ring()
739 static void a6xx_destroy(struct msm_gpu *gpu)  in a6xx_destroy()  argument
741 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);  in a6xx_destroy()
746 			msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace);  in a6xx_destroy()
782 	struct msm_gpu *gpu;  in a6xx_gpu_init()  local
790 	gpu = &adreno_gpu->base;  in a6xx_gpu_init()
813 	if (gpu->aspace)  in a6xx_gpu_init()
814 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,  in a6xx_gpu_init()
817 	return gpu;  in a6xx_gpu_init()