Lines Matching refs:adev
49 static int gmc_v11_0_ecc_interrupt_state(struct amdgpu_device *adev, in gmc_v11_0_ecc_interrupt_state() argument
58 gmc_v11_0_vm_fault_interrupt_state(struct amdgpu_device *adev, in gmc_v11_0_vm_fault_interrupt_state() argument
65 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_0, false); in gmc_v11_0_vm_fault_interrupt_state()
67 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_0, false); in gmc_v11_0_vm_fault_interrupt_state()
71 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_0, true); in gmc_v11_0_vm_fault_interrupt_state()
73 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_0, true); in gmc_v11_0_vm_fault_interrupt_state()
82 static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev, in gmc_v11_0_process_interrupt() argument
86 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; in gmc_v11_0_process_interrupt()
93 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_process_interrupt()
110 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); in gmc_v11_0_process_interrupt()
112 dev_err(adev->dev, in gmc_v11_0_process_interrupt()
119 dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", in gmc_v11_0_process_interrupt()
121 if (!amdgpu_sriov_vf(adev)) in gmc_v11_0_process_interrupt()
122 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); in gmc_v11_0_process_interrupt()
138 static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_irq_funcs() argument
140 adev->gmc.vm_fault.num_types = 1; in gmc_v11_0_set_irq_funcs()
141 adev->gmc.vm_fault.funcs = &gmc_v11_0_irq_funcs; in gmc_v11_0_set_irq_funcs()
143 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_set_irq_funcs()
144 adev->gmc.ecc_irq.num_types = 1; in gmc_v11_0_set_irq_funcs()
145 adev->gmc.ecc_irq.funcs = &gmc_v11_0_ecc_funcs; in gmc_v11_0_set_irq_funcs()
156 static bool gmc_v11_0_use_invalidate_semaphore(struct amdgpu_device *adev, in gmc_v11_0_use_invalidate_semaphore() argument
160 (!amdgpu_sriov_vf(adev))); in gmc_v11_0_use_invalidate_semaphore()
164 struct amdgpu_device *adev, in gmc_v11_0_get_vmid_pasid_mapping_info() argument
179 static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, in gmc_v11_0_flush_vm_hub() argument
182 bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(adev, vmhub); in gmc_v11_0_flush_vm_hub()
183 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; in gmc_v11_0_flush_vm_hub()
194 spin_lock(&adev->gmc.invalidate_lock); in gmc_v11_0_flush_vm_hub()
204 for (i = 0; i < adev->usec_timeout; i++) { in gmc_v11_0_flush_vm_hub()
213 if (i >= adev->usec_timeout) in gmc_v11_0_flush_vm_hub()
220 for (i = 0; i < adev->usec_timeout; i++) { in gmc_v11_0_flush_vm_hub()
242 !amdgpu_sriov_vf(adev)) { in gmc_v11_0_flush_vm_hub()
252 spin_unlock(&adev->gmc.invalidate_lock); in gmc_v11_0_flush_vm_hub()
254 if (i < adev->usec_timeout) in gmc_v11_0_flush_vm_hub()
268 static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, in gmc_v11_0_flush_gpu_tlb() argument
271 if ((vmhub == AMDGPU_GFXHUB_0) && !adev->gfx.is_poweron) in gmc_v11_0_flush_gpu_tlb()
275 adev->hdp.funcs->flush_hdp(adev, NULL); in gmc_v11_0_flush_gpu_tlb()
280 if ((adev->gfx.kiq.ring.sched.ready || adev->mes.ring.sched.ready) && in gmc_v11_0_flush_gpu_tlb()
281 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { in gmc_v11_0_flush_gpu_tlb()
282 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; in gmc_v11_0_flush_gpu_tlb()
288 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, in gmc_v11_0_flush_gpu_tlb()
293 mutex_lock(&adev->mman.gtt_window_lock); in gmc_v11_0_flush_gpu_tlb()
294 gmc_v11_0_flush_vm_hub(adev, vmid, vmhub, 0); in gmc_v11_0_flush_gpu_tlb()
295 mutex_unlock(&adev->mman.gtt_window_lock); in gmc_v11_0_flush_gpu_tlb()
307 static int gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, in gmc_v11_0_flush_gpu_tlb_pasid() argument
316 struct amdgpu_ring *ring = &adev->gfx.kiq.ring; in gmc_v11_0_flush_gpu_tlb_pasid()
317 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gmc_v11_0_flush_gpu_tlb_pasid()
320 spin_lock(&adev->gfx.kiq.ring_lock); in gmc_v11_0_flush_gpu_tlb_pasid()
328 spin_unlock(&adev->gfx.kiq.ring_lock); in gmc_v11_0_flush_gpu_tlb_pasid()
333 spin_unlock(&adev->gfx.kiq.ring_lock); in gmc_v11_0_flush_gpu_tlb_pasid()
334 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); in gmc_v11_0_flush_gpu_tlb_pasid()
336 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); in gmc_v11_0_flush_gpu_tlb_pasid()
345 ret = gmc_v11_0_get_vmid_pasid_mapping_info(adev, vmid, in gmc_v11_0_flush_gpu_tlb_pasid()
349 for (i = 0; i < adev->num_vmhubs; i++) in gmc_v11_0_flush_gpu_tlb_pasid()
350 gmc_v11_0_flush_gpu_tlb(adev, vmid, in gmc_v11_0_flush_gpu_tlb_pasid()
353 gmc_v11_0_flush_gpu_tlb(adev, vmid, in gmc_v11_0_flush_gpu_tlb_pasid()
365 bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); in gmc_v11_0_emit_flush_gpu_tlb()
366 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub]; in gmc_v11_0_emit_flush_gpu_tlb()
413 struct amdgpu_device *adev = ring->adev; in gmc_v11_0_emit_pasid_mapping() local
460 static uint64_t gmc_v11_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) in gmc_v11_0_map_mtype() argument
478 static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level, in gmc_v11_0_get_vm_pde() argument
482 *addr = adev->vm_manager.vram_base_offset + *addr - in gmc_v11_0_get_vm_pde()
483 adev->gmc.vram_start; in gmc_v11_0_get_vm_pde()
486 if (!adev->gmc.translate_further) in gmc_v11_0_get_vm_pde()
502 static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev, in gmc_v11_0_get_vm_pte() argument
524 static unsigned gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev) in gmc_v11_0_get_vbios_fb_size() argument
540 static void gmc_v11_0_set_gmc_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_gmc_funcs() argument
542 adev->gmc.gmc_funcs = &gmc_v11_0_gmc_funcs; in gmc_v11_0_set_gmc_funcs()
545 static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_umc_funcs() argument
547 switch (adev->ip_versions[UMC_HWIP][0]) { in gmc_v11_0_set_umc_funcs()
549 adev->umc.channel_inst_num = UMC_V8_10_CHANNEL_INSTANCE_NUM; in gmc_v11_0_set_umc_funcs()
550 adev->umc.umc_inst_num = UMC_V8_10_UMC_INSTANCE_NUM; in gmc_v11_0_set_umc_funcs()
551 adev->umc.node_inst_num = adev->gmc.num_umc; in gmc_v11_0_set_umc_funcs()
552 adev->umc.max_ras_err_cnt_per_query = UMC_V8_10_TOTAL_CHANNEL_NUM(adev); in gmc_v11_0_set_umc_funcs()
553 adev->umc.channel_offs = UMC_V8_10_PER_CHANNEL_OFFSET; in gmc_v11_0_set_umc_funcs()
554 adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl[0][0][0]; in gmc_v11_0_set_umc_funcs()
555 adev->umc.ras = &umc_v8_10_ras; in gmc_v11_0_set_umc_funcs()
563 if (adev->umc.ras) { in gmc_v11_0_set_umc_funcs()
564 amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block); in gmc_v11_0_set_umc_funcs()
566 strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc"); in gmc_v11_0_set_umc_funcs()
567 adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC; in gmc_v11_0_set_umc_funcs()
568 adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; in gmc_v11_0_set_umc_funcs()
569 adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm; in gmc_v11_0_set_umc_funcs()
572 if (!adev->umc.ras->ras_block.ras_late_init) in gmc_v11_0_set_umc_funcs()
573 adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init; in gmc_v11_0_set_umc_funcs()
576 if (!adev->umc.ras->ras_block.ras_cb) in gmc_v11_0_set_umc_funcs()
577 adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb; in gmc_v11_0_set_umc_funcs()
582 static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_mmhub_funcs() argument
584 switch (adev->ip_versions[MMHUB_HWIP][0]) { in gmc_v11_0_set_mmhub_funcs()
586 adev->mmhub.funcs = &mmhub_v3_0_1_funcs; in gmc_v11_0_set_mmhub_funcs()
589 adev->mmhub.funcs = &mmhub_v3_0_2_funcs; in gmc_v11_0_set_mmhub_funcs()
592 adev->mmhub.funcs = &mmhub_v3_0_funcs; in gmc_v11_0_set_mmhub_funcs()
597 static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_gfxhub_funcs() argument
599 switch (adev->ip_versions[GC_HWIP][0]) { in gmc_v11_0_set_gfxhub_funcs()
601 adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs; in gmc_v11_0_set_gfxhub_funcs()
604 adev->gfxhub.funcs = &gfxhub_v3_0_funcs; in gmc_v11_0_set_gfxhub_funcs()
611 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_early_init() local
613 gmc_v11_0_set_gfxhub_funcs(adev); in gmc_v11_0_early_init()
614 gmc_v11_0_set_mmhub_funcs(adev); in gmc_v11_0_early_init()
615 gmc_v11_0_set_gmc_funcs(adev); in gmc_v11_0_early_init()
616 gmc_v11_0_set_irq_funcs(adev); in gmc_v11_0_early_init()
617 gmc_v11_0_set_umc_funcs(adev); in gmc_v11_0_early_init()
619 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; in gmc_v11_0_early_init()
620 adev->gmc.shared_aperture_end = in gmc_v11_0_early_init()
621 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; in gmc_v11_0_early_init()
622 adev->gmc.private_aperture_start = 0x1000000000000000ULL; in gmc_v11_0_early_init()
623 adev->gmc.private_aperture_end = in gmc_v11_0_early_init()
624 adev->gmc.private_aperture_start + (4ULL << 30) - 1; in gmc_v11_0_early_init()
631 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_late_init() local
634 r = amdgpu_gmc_allocate_vm_inv_eng(adev); in gmc_v11_0_late_init()
638 r = amdgpu_gmc_ras_late_init(adev); in gmc_v11_0_late_init()
642 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); in gmc_v11_0_late_init()
645 static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev, in gmc_v11_0_vram_gtt_location() argument
650 base = adev->mmhub.funcs->get_fb_location(adev); in gmc_v11_0_vram_gtt_location()
652 amdgpu_gmc_vram_location(adev, &adev->gmc, base); in gmc_v11_0_vram_gtt_location()
653 amdgpu_gmc_gart_location(adev, mc); in gmc_v11_0_vram_gtt_location()
656 if (amdgpu_sriov_vf(adev)) in gmc_v11_0_vram_gtt_location()
657 adev->vm_manager.vram_base_offset = 0; in gmc_v11_0_vram_gtt_location()
659 adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev); in gmc_v11_0_vram_gtt_location()
671 static int gmc_v11_0_mc_init(struct amdgpu_device *adev) in gmc_v11_0_mc_init() argument
676 adev->gmc.mc_vram_size = in gmc_v11_0_mc_init()
677 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; in gmc_v11_0_mc_init()
678 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; in gmc_v11_0_mc_init()
680 if (!(adev->flags & AMD_IS_APU)) { in gmc_v11_0_mc_init()
681 r = amdgpu_device_resize_fb_bar(adev); in gmc_v11_0_mc_init()
685 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); in gmc_v11_0_mc_init()
686 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); in gmc_v11_0_mc_init()
689 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) { in gmc_v11_0_mc_init()
690 adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev); in gmc_v11_0_mc_init()
691 adev->gmc.aper_size = adev->gmc.real_vram_size; in gmc_v11_0_mc_init()
695 adev->gmc.visible_vram_size = adev->gmc.aper_size; in gmc_v11_0_mc_init()
696 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) in gmc_v11_0_mc_init()
697 adev->gmc.visible_vram_size = adev->gmc.real_vram_size; in gmc_v11_0_mc_init()
701 adev->gmc.gart_size = 512ULL << 20; in gmc_v11_0_mc_init()
703 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; in gmc_v11_0_mc_init()
705 gmc_v11_0_vram_gtt_location(adev, &adev->gmc); in gmc_v11_0_mc_init()
710 static int gmc_v11_0_gart_init(struct amdgpu_device *adev) in gmc_v11_0_gart_init() argument
714 if (adev->gart.bo) { in gmc_v11_0_gart_init()
720 r = amdgpu_gart_init(adev); in gmc_v11_0_gart_init()
724 adev->gart.table_size = adev->gart.num_gpu_pages * 8; in gmc_v11_0_gart_init()
725 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC) | in gmc_v11_0_gart_init()
728 return amdgpu_gart_table_vram_alloc(adev); in gmc_v11_0_gart_init()
734 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_sw_init() local
736 adev->mmhub.funcs->init(adev); in gmc_v11_0_sw_init()
738 spin_lock_init(&adev->gmc.invalidate_lock); in gmc_v11_0_sw_init()
740 r = amdgpu_atomfirmware_get_vram_info(adev, in gmc_v11_0_sw_init()
742 adev->gmc.vram_width = vram_width; in gmc_v11_0_sw_init()
744 adev->gmc.vram_type = vram_type; in gmc_v11_0_sw_init()
745 adev->gmc.vram_vendor = vram_vendor; in gmc_v11_0_sw_init()
747 switch (adev->ip_versions[GC_HWIP][0]) { in gmc_v11_0_sw_init()
752 adev->num_vmhubs = 2; in gmc_v11_0_sw_init()
758 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); in gmc_v11_0_sw_init()
765 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC, in gmc_v11_0_sw_init()
767 &adev->gmc.vm_fault); in gmc_v11_0_sw_init()
772 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, in gmc_v11_0_sw_init()
774 &adev->gmc.vm_fault); in gmc_v11_0_sw_init()
778 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_sw_init()
780 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0, in gmc_v11_0_sw_init()
781 &adev->gmc.ecc_irq); in gmc_v11_0_sw_init()
790 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ in gmc_v11_0_sw_init()
792 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); in gmc_v11_0_sw_init()
798 adev->need_swiotlb = drm_need_swiotlb(44); in gmc_v11_0_sw_init()
800 r = gmc_v11_0_mc_init(adev); in gmc_v11_0_sw_init()
804 amdgpu_gmc_get_vbios_allocations(adev); in gmc_v11_0_sw_init()
807 r = amdgpu_bo_init(adev); in gmc_v11_0_sw_init()
811 r = gmc_v11_0_gart_init(adev); in gmc_v11_0_sw_init()
821 adev->vm_manager.first_kfd_vmid = 8; in gmc_v11_0_sw_init()
823 amdgpu_vm_manager_init(adev); in gmc_v11_0_sw_init()
835 static void gmc_v11_0_gart_fini(struct amdgpu_device *adev) in gmc_v11_0_gart_fini() argument
837 amdgpu_gart_table_vram_free(adev); in gmc_v11_0_gart_fini()
842 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_sw_fini() local
844 amdgpu_vm_manager_fini(adev); in gmc_v11_0_sw_fini()
845 gmc_v11_0_gart_fini(adev); in gmc_v11_0_sw_fini()
846 amdgpu_gem_force_release(adev); in gmc_v11_0_sw_fini()
847 amdgpu_bo_fini(adev); in gmc_v11_0_sw_fini()
852 static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev) in gmc_v11_0_init_golden_registers() argument
861 static int gmc_v11_0_gart_enable(struct amdgpu_device *adev) in gmc_v11_0_gart_enable() argument
866 if (adev->gart.bo == NULL) { in gmc_v11_0_gart_enable()
867 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); in gmc_v11_0_gart_enable()
871 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr); in gmc_v11_0_gart_enable()
873 r = adev->mmhub.funcs->gart_enable(adev); in gmc_v11_0_gart_enable()
878 adev->hdp.funcs->flush_hdp(adev, NULL); in gmc_v11_0_gart_enable()
883 adev->mmhub.funcs->set_fault_enable_default(adev, value); in gmc_v11_0_gart_enable()
884 gmc_v11_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB_0, 0); in gmc_v11_0_gart_enable()
887 (unsigned)(adev->gmc.gart_size >> 20), in gmc_v11_0_gart_enable()
888 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); in gmc_v11_0_gart_enable()
896 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_hw_init() local
899 gmc_v11_0_init_golden_registers(adev); in gmc_v11_0_hw_init()
901 r = gmc_v11_0_gart_enable(adev); in gmc_v11_0_hw_init()
905 if (adev->umc.funcs && adev->umc.funcs->init_registers) in gmc_v11_0_hw_init()
906 adev->umc.funcs->init_registers(adev); in gmc_v11_0_hw_init()
918 static void gmc_v11_0_gart_disable(struct amdgpu_device *adev) in gmc_v11_0_gart_disable() argument
920 adev->mmhub.funcs->gart_disable(adev); in gmc_v11_0_gart_disable()
925 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_hw_fini() local
927 if (amdgpu_sriov_vf(adev)) { in gmc_v11_0_hw_fini()
933 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); in gmc_v11_0_hw_fini()
934 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); in gmc_v11_0_hw_fini()
935 gmc_v11_0_gart_disable(adev); in gmc_v11_0_hw_fini()
942 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_suspend() local
944 gmc_v11_0_hw_fini(adev); in gmc_v11_0_suspend()
952 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_resume() local
954 r = gmc_v11_0_hw_init(adev); in gmc_v11_0_resume()
958 amdgpu_vmid_reset_all(adev); in gmc_v11_0_resume()
984 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_set_clockgating_state() local
986 r = adev->mmhub.funcs->set_clockgating(adev, state); in gmc_v11_0_set_clockgating_state()
990 return athub_v3_0_set_clockgating(adev, state); in gmc_v11_0_set_clockgating_state()
995 struct amdgpu_device *adev = (struct amdgpu_device *)handle; in gmc_v11_0_get_clockgating_state() local
997 adev->mmhub.funcs->get_clockgating(adev, flags); in gmc_v11_0_get_clockgating_state()
999 athub_v3_0_get_clockgating(adev, flags); in gmc_v11_0_get_clockgating_state()