40 static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev);
41 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
70 static void gmc_v6_0_mc_stop(struct amdgpu_device *adev)  in gmc_v6_0_mc_stop()  argument
74 	gmc_v6_0_wait_for_idle((void *)adev);  in gmc_v6_0_mc_stop()
90 static void gmc_v6_0_mc_resume(struct amdgpu_device *adev)  in gmc_v6_0_mc_resume()  argument
104 static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)  in gmc_v6_0_init_microcode()  argument
113 	switch (adev->asic_type) {  in gmc_v6_0_init_microcode()
140 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);  in gmc_v6_0_init_microcode()
144 	err = amdgpu_ucode_validate(adev->gmc.fw);  in gmc_v6_0_init_microcode()
148 		dev_err(adev->dev,  in gmc_v6_0_init_microcode()
151 		release_firmware(adev->gmc.fw);  in gmc_v6_0_init_microcode()
152 		adev->gmc.fw = NULL;  in gmc_v6_0_init_microcode()
157 static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)  in gmc_v6_0_mc_load_microcode()  argument
165 	if (!adev->gmc.fw)  in gmc_v6_0_mc_load_microcode()
168 	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;  in gmc_v6_0_mc_load_microcode()
172 	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);  in gmc_v6_0_mc_load_microcode()
175 		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in gmc_v6_0_mc_load_microcode()
178 		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in gmc_v6_0_mc_load_microcode()
204 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_mc_load_microcode()
209 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_mc_load_microcode()
220 static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,  in gmc_v6_0_vram_gtt_location()  argument
226 	amdgpu_device_vram_location(adev, &adev->gmc, base);  in gmc_v6_0_vram_gtt_location()
227 	amdgpu_device_gart_location(adev, mc);  in gmc_v6_0_vram_gtt_location()
230 static void gmc_v6_0_mc_program(struct amdgpu_device *adev)  in gmc_v6_0_mc_program()  argument
244 	if (gmc_v6_0_wait_for_idle((void *)adev)) {  in gmc_v6_0_mc_program()
245 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v6_0_mc_program()
248 	if (adev->mode_info.num_crtc) {  in gmc_v6_0_mc_program()
263 	       adev->gmc.vram_start >> 12);  in gmc_v6_0_mc_program()
265 	       adev->gmc.vram_end >> 12);  in gmc_v6_0_mc_program()
267 	       adev->vram_scratch.gpu_addr >> 12);  in gmc_v6_0_mc_program()
272 	if (gmc_v6_0_wait_for_idle((void *)adev)) {  in gmc_v6_0_mc_program()
273 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v6_0_mc_program()
277 static int gmc_v6_0_mc_init(struct amdgpu_device *adev)  in gmc_v6_0_mc_init()  argument
323 	adev->gmc.vram_width = numchan * chansize;  in gmc_v6_0_mc_init()
325 	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v6_0_mc_init()
326 	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v6_0_mc_init()
328 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v6_0_mc_init()
329 		r = amdgpu_device_resize_fb_bar(adev);  in gmc_v6_0_mc_init()
333 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);  in gmc_v6_0_mc_init()
334 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);  in gmc_v6_0_mc_init()
335 	adev->gmc.visible_vram_size = adev->gmc.aper_size;  in gmc_v6_0_mc_init()
339 		switch (adev->asic_type) {  in gmc_v6_0_mc_init()
342 			adev->gmc.gart_size = 256ULL << 20;  in gmc_v6_0_mc_init()
348 			adev->gmc.gart_size = 1024ULL << 20;  in gmc_v6_0_mc_init()
352 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;  in gmc_v6_0_mc_init()
355 	gmc_v6_0_vram_gtt_location(adev, &adev->gmc);  in gmc_v6_0_mc_init()
360 static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)  in gmc_v6_0_flush_gpu_tlb()  argument
383 static int gmc_v6_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,  in gmc_v6_0_set_pte_pde()  argument
397 static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,  in gmc_v6_0_get_vm_pte_flags()  argument
412 static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,  in gmc_v6_0_get_vm_pde()  argument
418 static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,  in gmc_v6_0_set_fault_enable_default()  argument
445 static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)  in gmc_v6_0_set_prt()  argument
449 	if (enable && !adev->gmc.prt_warning) {  in gmc_v6_0_set_prt()
450 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");  in gmc_v6_0_set_prt()
451 		adev->gmc.prt_warning = true;  in gmc_v6_0_set_prt()
471 		uint32_t high = adev->vm_manager.max_pfn -  in gmc_v6_0_set_prt()
494 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)  in gmc_v6_0_gart_enable()  argument
499 	if (adev->gart.robj == NULL) {  in gmc_v6_0_gart_enable()
500 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");  in gmc_v6_0_gart_enable()
503 	r = amdgpu_gart_table_vram_pin(adev);  in gmc_v6_0_gart_enable()
526 	field = adev->vm_manager.fragment_size;  in gmc_v6_0_gart_enable()
532 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);  in gmc_v6_0_gart_enable()
533 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);  in gmc_v6_0_gart_enable()
534 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);  in gmc_v6_0_gart_enable()
536 			(u32)(adev->dummy_page_addr >> 12));  in gmc_v6_0_gart_enable()
550 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);  in gmc_v6_0_gart_enable()
558 			       adev->gart.table_addr >> 12);  in gmc_v6_0_gart_enable()
561 			       adev->gart.table_addr >> 12);  in gmc_v6_0_gart_enable()
566 	       (u32)(adev->dummy_page_addr >> 12));  in gmc_v6_0_gart_enable()
571 	       ((adev->vm_manager.block_size - 9)  in gmc_v6_0_gart_enable()
574 		gmc_v6_0_set_fault_enable_default(adev, false);  in gmc_v6_0_gart_enable()
576 		gmc_v6_0_set_fault_enable_default(adev, true);  in gmc_v6_0_gart_enable()
578 	gmc_v6_0_flush_gpu_tlb(adev, 0);  in gmc_v6_0_gart_enable()
579 	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",  in gmc_v6_0_gart_enable()
580 		 (unsigned)(adev->gmc.gart_size >> 20),  in gmc_v6_0_gart_enable()
581 		 (unsigned long long)adev->gart.table_addr);  in gmc_v6_0_gart_enable()
582 	adev->gart.ready = true;  in gmc_v6_0_gart_enable()
586 static int gmc_v6_0_gart_init(struct amdgpu_device *adev)  in gmc_v6_0_gart_init()  argument
590 	if (adev->gart.robj) {  in gmc_v6_0_gart_init()
591 		dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");  in gmc_v6_0_gart_init()
594 	r = amdgpu_gart_init(adev);  in gmc_v6_0_gart_init()
597 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;  in gmc_v6_0_gart_init()
598 	adev->gart.gart_pte_flags = 0;  in gmc_v6_0_gart_init()
599 	return amdgpu_gart_table_vram_alloc(adev);  in gmc_v6_0_gart_init()
602 static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)  in gmc_v6_0_gart_disable()  argument
632 	amdgpu_gart_table_vram_unpin(adev);  in gmc_v6_0_gart_disable()
635 static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,  in gmc_v6_0_vm_decode_fault()  argument
648 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",  in gmc_v6_0_vm_decode_fault()
804 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_early_init()  local
806 	gmc_v6_0_set_gmc_funcs(adev);  in gmc_v6_0_early_init()
807 	gmc_v6_0_set_irq_funcs(adev);  in gmc_v6_0_early_init()
814 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_late_init()  local
816 	amdgpu_bo_late_init(adev);  in gmc_v6_0_late_init()
819 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);  in gmc_v6_0_late_init()
824 static unsigned gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev)  in gmc_v6_0_get_vbios_fb_size()  argument
838 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in gmc_v6_0_get_vbios_fb_size()
847 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_sw_init()  local
849 	if (adev->flags & AMD_IS_APU) {  in gmc_v6_0_sw_init()
850 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;  in gmc_v6_0_sw_init()
854 		adev->gmc.vram_type = gmc_v6_0_convert_vram_type(tmp);  in gmc_v6_0_sw_init()
857 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);  in gmc_v6_0_sw_init()
861 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);  in gmc_v6_0_sw_init()
865 	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);  in gmc_v6_0_sw_init()
867 	adev->gmc.mc_mask = 0xffffffffffULL;  in gmc_v6_0_sw_init()
869 	adev->need_dma32 = false;  in gmc_v6_0_sw_init()
870 	dma_bits = adev->need_dma32 ? 32 : 40;  in gmc_v6_0_sw_init()
871 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v6_0_sw_init()
873 		adev->need_dma32 = true;  in gmc_v6_0_sw_init()
875 		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");  in gmc_v6_0_sw_init()
877 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v6_0_sw_init()
879 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));  in gmc_v6_0_sw_init()
880 		dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n");  in gmc_v6_0_sw_init()
882 	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);  in gmc_v6_0_sw_init()
884 	r = gmc_v6_0_init_microcode(adev);  in gmc_v6_0_sw_init()
886 		dev_err(adev->dev, "Failed to load mc firmware!\n");  in gmc_v6_0_sw_init()
890 	r = gmc_v6_0_mc_init(adev);  in gmc_v6_0_sw_init()
894 	adev->gmc.stolen_size = gmc_v6_0_get_vbios_fb_size(adev);  in gmc_v6_0_sw_init()
896 	r = amdgpu_bo_init(adev);  in gmc_v6_0_sw_init()
900 	r = gmc_v6_0_gart_init(adev);  in gmc_v6_0_sw_init()
910 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v6_0_sw_init()
911 	amdgpu_vm_manager_init(adev);  in gmc_v6_0_sw_init()
914 	if (adev->flags & AMD_IS_APU) {  in gmc_v6_0_sw_init()
918 		adev->vm_manager.vram_base_offset = tmp;  in gmc_v6_0_sw_init()
920 		adev->vm_manager.vram_base_offset = 0;  in gmc_v6_0_sw_init()
928 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_sw_fini()  local
930 	amdgpu_gem_force_release(adev);  in gmc_v6_0_sw_fini()
931 	amdgpu_vm_manager_fini(adev);  in gmc_v6_0_sw_fini()
932 	amdgpu_gart_table_vram_free(adev);  in gmc_v6_0_sw_fini()
933 	amdgpu_bo_fini(adev);  in gmc_v6_0_sw_fini()
934 	amdgpu_gart_fini(adev);  in gmc_v6_0_sw_fini()
935 	release_firmware(adev->gmc.fw);  in gmc_v6_0_sw_fini()
936 	adev->gmc.fw = NULL;  in gmc_v6_0_sw_fini()
944 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_hw_init()  local
946 	gmc_v6_0_mc_program(adev);  in gmc_v6_0_hw_init()
948 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v6_0_hw_init()
949 		r = gmc_v6_0_mc_load_microcode(adev);  in gmc_v6_0_hw_init()
951 			dev_err(adev->dev, "Failed to load MC firmware!\n");  in gmc_v6_0_hw_init()
956 	r = gmc_v6_0_gart_enable(adev);  in gmc_v6_0_hw_init()
965 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_hw_fini()  local
967 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);  in gmc_v6_0_hw_fini()
968 	gmc_v6_0_gart_disable(adev);  in gmc_v6_0_hw_fini()
975 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_suspend()  local
977 	gmc_v6_0_hw_fini(adev);  in gmc_v6_0_suspend()
985 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_resume()  local
987 	r = gmc_v6_0_hw_init(adev);  in gmc_v6_0_resume()
991 	amdgpu_vmid_reset_all(adev);  in gmc_v6_0_resume()
998 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_is_idle()  local
1011 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_wait_for_idle()  local
1013 	for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_wait_for_idle()
1024 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_soft_reset()  local
1034 		if (!(adev->flags & AMD_IS_APU))  in gmc_v6_0_soft_reset()
1040 		gmc_v6_0_mc_stop(adev);  in gmc_v6_0_soft_reset()
1041 		if (gmc_v6_0_wait_for_idle(adev)) {  in gmc_v6_0_soft_reset()
1042 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");  in gmc_v6_0_soft_reset()
1048 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in gmc_v6_0_soft_reset()
1060 		gmc_v6_0_mc_resume(adev);  in gmc_v6_0_soft_reset()
1067 static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,  in gmc_v6_0_vm_fault_interrupt_state()  argument
1104 static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,  in gmc_v6_0_process_interrupt()  argument
1118 		gmc_v6_0_set_fault_enable_default(adev, false);  in gmc_v6_0_process_interrupt()
1121 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",  in gmc_v6_0_process_interrupt()
1123 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in gmc_v6_0_process_interrupt()
1125 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in gmc_v6_0_process_interrupt()
1127 		gmc_v6_0_vm_decode_fault(adev, status, addr, 0);  in gmc_v6_0_process_interrupt()
1176 static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev)  in gmc_v6_0_set_gmc_funcs()  argument
1178 	if (adev->gmc.gmc_funcs == NULL)  in gmc_v6_0_set_gmc_funcs()
1179 		adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs;  in gmc_v6_0_set_gmc_funcs()
1182 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)  in gmc_v6_0_set_irq_funcs()  argument
1184 	adev->gmc.vm_fault.num_types = 1;  in gmc_v6_0_set_irq_funcs()
1185 	adev->gmc.vm_fault.funcs = &gmc_v6_0_irq_funcs;  in gmc_v6_0_set_irq_funcs()