44 static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev);
45 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
75 static void gmc_v6_0_mc_stop(struct amdgpu_device *adev)  in gmc_v6_0_mc_stop()  argument
79 	gmc_v6_0_wait_for_idle((void *)adev);  in gmc_v6_0_mc_stop()
95 static void gmc_v6_0_mc_resume(struct amdgpu_device *adev)  in gmc_v6_0_mc_resume()  argument
109 static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)  in gmc_v6_0_init_microcode()  argument
118 	switch (adev->asic_type) {  in gmc_v6_0_init_microcode()
145 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);  in gmc_v6_0_init_microcode()
149 	err = amdgpu_ucode_validate(adev->gmc.fw);  in gmc_v6_0_init_microcode()
153 		dev_err(adev->dev,  in gmc_v6_0_init_microcode()
156 		release_firmware(adev->gmc.fw);  in gmc_v6_0_init_microcode()
157 		adev->gmc.fw = NULL;  in gmc_v6_0_init_microcode()
162 static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)  in gmc_v6_0_mc_load_microcode()  argument
170 	if (!adev->gmc.fw)  in gmc_v6_0_mc_load_microcode()
173 	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;  in gmc_v6_0_mc_load_microcode()
177 	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);  in gmc_v6_0_mc_load_microcode()
180 		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in gmc_v6_0_mc_load_microcode()
183 		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in gmc_v6_0_mc_load_microcode()
209 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_mc_load_microcode()
214 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_mc_load_microcode()
225 static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,  in gmc_v6_0_vram_gtt_location()  argument
231 	amdgpu_gmc_vram_location(adev, mc, base);  in gmc_v6_0_vram_gtt_location()
232 	amdgpu_gmc_gart_location(adev, mc);  in gmc_v6_0_vram_gtt_location()
235 static void gmc_v6_0_mc_program(struct amdgpu_device *adev)  in gmc_v6_0_mc_program()  argument
249 	if (gmc_v6_0_wait_for_idle((void *)adev)) {  in gmc_v6_0_mc_program()
250 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v6_0_mc_program()
253 	if (adev->mode_info.num_crtc) {  in gmc_v6_0_mc_program()
268 	       adev->gmc.vram_start >> 12);  in gmc_v6_0_mc_program()
270 	       adev->gmc.vram_end >> 12);  in gmc_v6_0_mc_program()
272 	       adev->vram_scratch.gpu_addr >> 12);  in gmc_v6_0_mc_program()
277 	if (gmc_v6_0_wait_for_idle((void *)adev)) {  in gmc_v6_0_mc_program()
278 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v6_0_mc_program()
282 static int gmc_v6_0_mc_init(struct amdgpu_device *adev)  in gmc_v6_0_mc_init()  argument
328 	adev->gmc.vram_width = numchan * chansize;  in gmc_v6_0_mc_init()
330 	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v6_0_mc_init()
331 	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v6_0_mc_init()
333 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v6_0_mc_init()
334 		r = amdgpu_device_resize_fb_bar(adev);  in gmc_v6_0_mc_init()
338 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);  in gmc_v6_0_mc_init()
339 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);  in gmc_v6_0_mc_init()
340 	adev->gmc.visible_vram_size = adev->gmc.aper_size;  in gmc_v6_0_mc_init()
344 		switch (adev->asic_type) {  in gmc_v6_0_mc_init()
347 			adev->gmc.gart_size = 256ULL << 20;  in gmc_v6_0_mc_init()
353 			adev->gmc.gart_size = 1024ULL << 20;  in gmc_v6_0_mc_init()
357 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;  in gmc_v6_0_mc_init()
360 	gmc_v6_0_vram_gtt_location(adev, &adev->gmc);  in gmc_v6_0_mc_init()
365 static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,  in gmc_v6_0_flush_gpu_tlb()  argument
389 static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,  in gmc_v6_0_get_vm_pte_flags()  argument
404 static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,  in gmc_v6_0_get_vm_pde()  argument
410 static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,  in gmc_v6_0_set_fault_enable_default()  argument
437 static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)  in gmc_v6_0_set_prt()  argument
441 	if (enable && !adev->gmc.prt_warning) {  in gmc_v6_0_set_prt()
442 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");  in gmc_v6_0_set_prt()
443 		adev->gmc.prt_warning = true;  in gmc_v6_0_set_prt()
463 		uint32_t high = adev->vm_manager.max_pfn -  in gmc_v6_0_set_prt()
486 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)  in gmc_v6_0_gart_enable()  argument
492 	if (adev->gart.bo == NULL) {  in gmc_v6_0_gart_enable()
493 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");  in gmc_v6_0_gart_enable()
496 	r = amdgpu_gart_table_vram_pin(adev);  in gmc_v6_0_gart_enable()
500 	table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);  in gmc_v6_0_gart_enable()
522 	field = adev->vm_manager.fragment_size;  in gmc_v6_0_gart_enable()
528 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);  in gmc_v6_0_gart_enable()
529 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);  in gmc_v6_0_gart_enable()
532 			(u32)(adev->dummy_page_addr >> 12));  in gmc_v6_0_gart_enable()
546 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);  in gmc_v6_0_gart_enable()
562 	       (u32)(adev->dummy_page_addr >> 12));  in gmc_v6_0_gart_enable()
567 	       ((adev->vm_manager.block_size - 9)  in gmc_v6_0_gart_enable()
570 		gmc_v6_0_set_fault_enable_default(adev, false);  in gmc_v6_0_gart_enable()
572 		gmc_v6_0_set_fault_enable_default(adev, true);  in gmc_v6_0_gart_enable()
574 	gmc_v6_0_flush_gpu_tlb(adev, 0, 0, 0);  in gmc_v6_0_gart_enable()
575 	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",  in gmc_v6_0_gart_enable()
576 		 (unsigned)(adev->gmc.gart_size >> 20),  in gmc_v6_0_gart_enable()
578 	adev->gart.ready = true;  in gmc_v6_0_gart_enable()
582 static int gmc_v6_0_gart_init(struct amdgpu_device *adev)  in gmc_v6_0_gart_init()  argument
586 	if (adev->gart.bo) {  in gmc_v6_0_gart_init()
587 		dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");  in gmc_v6_0_gart_init()
590 	r = amdgpu_gart_init(adev);  in gmc_v6_0_gart_init()
593 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;  in gmc_v6_0_gart_init()
594 	adev->gart.gart_pte_flags = 0;  in gmc_v6_0_gart_init()
595 	return amdgpu_gart_table_vram_alloc(adev);  in gmc_v6_0_gart_init()
598 static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)  in gmc_v6_0_gart_disable()  argument
628 	amdgpu_gart_table_vram_unpin(adev);  in gmc_v6_0_gart_disable()
631 static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,  in gmc_v6_0_vm_decode_fault()  argument
644 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",  in gmc_v6_0_vm_decode_fault()
800 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_early_init()  local
802 	gmc_v6_0_set_gmc_funcs(adev);  in gmc_v6_0_early_init()
803 	gmc_v6_0_set_irq_funcs(adev);  in gmc_v6_0_early_init()
810 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_late_init()  local
812 	amdgpu_bo_late_init(adev);  in gmc_v6_0_late_init()
815 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);  in gmc_v6_0_late_init()
820 static unsigned gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev)  in gmc_v6_0_get_vbios_fb_size()  argument
834 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in gmc_v6_0_get_vbios_fb_size()
842 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_sw_init()  local
844 	adev->num_vmhubs = 1;  in gmc_v6_0_sw_init()
846 	if (adev->flags & AMD_IS_APU) {  in gmc_v6_0_sw_init()
847 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;  in gmc_v6_0_sw_init()
851 		adev->gmc.vram_type = gmc_v6_0_convert_vram_type(tmp);  in gmc_v6_0_sw_init()
854 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);  in gmc_v6_0_sw_init()
858 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);  in gmc_v6_0_sw_init()
862 	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);  in gmc_v6_0_sw_init()
864 	adev->gmc.mc_mask = 0xffffffffffULL;  in gmc_v6_0_sw_init()
866 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));  in gmc_v6_0_sw_init()
868 		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");  in gmc_v6_0_sw_init()
871 	adev->need_swiotlb = drm_need_swiotlb(44);  in gmc_v6_0_sw_init()
873 	r = gmc_v6_0_init_microcode(adev);  in gmc_v6_0_sw_init()
875 		dev_err(adev->dev, "Failed to load mc firmware!\n");  in gmc_v6_0_sw_init()
879 	r = gmc_v6_0_mc_init(adev);  in gmc_v6_0_sw_init()
883 	adev->gmc.stolen_size = gmc_v6_0_get_vbios_fb_size(adev);  in gmc_v6_0_sw_init()
885 	r = amdgpu_bo_init(adev);  in gmc_v6_0_sw_init()
889 	r = gmc_v6_0_gart_init(adev);  in gmc_v6_0_sw_init()
899 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v6_0_sw_init()
900 	amdgpu_vm_manager_init(adev);  in gmc_v6_0_sw_init()
903 	if (adev->flags & AMD_IS_APU) {  in gmc_v6_0_sw_init()
907 		adev->vm_manager.vram_base_offset = tmp;  in gmc_v6_0_sw_init()
909 		adev->vm_manager.vram_base_offset = 0;  in gmc_v6_0_sw_init()
917 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_sw_fini()  local
919 	amdgpu_gem_force_release(adev);  in gmc_v6_0_sw_fini()
920 	amdgpu_vm_manager_fini(adev);  in gmc_v6_0_sw_fini()
921 	amdgpu_gart_table_vram_free(adev);  in gmc_v6_0_sw_fini()
922 	amdgpu_bo_fini(adev);  in gmc_v6_0_sw_fini()
923 	amdgpu_gart_fini(adev);  in gmc_v6_0_sw_fini()
924 	release_firmware(adev->gmc.fw);  in gmc_v6_0_sw_fini()
925 	adev->gmc.fw = NULL;  in gmc_v6_0_sw_fini()
933 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_hw_init()  local
935 	gmc_v6_0_mc_program(adev);  in gmc_v6_0_hw_init()
937 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v6_0_hw_init()
938 		r = gmc_v6_0_mc_load_microcode(adev);  in gmc_v6_0_hw_init()
940 			dev_err(adev->dev, "Failed to load MC firmware!\n");  in gmc_v6_0_hw_init()
945 	r = gmc_v6_0_gart_enable(adev);  in gmc_v6_0_hw_init()
954 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_hw_fini()  local
956 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);  in gmc_v6_0_hw_fini()
957 	gmc_v6_0_gart_disable(adev);  in gmc_v6_0_hw_fini()
964 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_suspend()  local
966 	gmc_v6_0_hw_fini(adev);  in gmc_v6_0_suspend()
974 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_resume()  local
976 	r = gmc_v6_0_hw_init(adev);  in gmc_v6_0_resume()
980 	amdgpu_vmid_reset_all(adev);  in gmc_v6_0_resume()
987 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_is_idle()  local
1000 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_wait_for_idle()  local
1002 	for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v6_0_wait_for_idle()
1013 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v6_0_soft_reset()  local
1023 		if (!(adev->flags & AMD_IS_APU))  in gmc_v6_0_soft_reset()
1029 		gmc_v6_0_mc_stop(adev);  in gmc_v6_0_soft_reset()
1030 		if (gmc_v6_0_wait_for_idle(adev)) {  in gmc_v6_0_soft_reset()
1031 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");  in gmc_v6_0_soft_reset()
1037 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in gmc_v6_0_soft_reset()
1049 		gmc_v6_0_mc_resume(adev);  in gmc_v6_0_soft_reset()
1056 static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,  in gmc_v6_0_vm_fault_interrupt_state()  argument
1093 static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,  in gmc_v6_0_process_interrupt()  argument
1107 		gmc_v6_0_set_fault_enable_default(adev, false);  in gmc_v6_0_process_interrupt()
1110 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",  in gmc_v6_0_process_interrupt()
1112 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in gmc_v6_0_process_interrupt()
1114 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in gmc_v6_0_process_interrupt()
1116 		gmc_v6_0_vm_decode_fault(adev, status, addr, 0);  in gmc_v6_0_process_interrupt()
1164 static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev)  in gmc_v6_0_set_gmc_funcs()  argument
1166 	adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs;  in gmc_v6_0_set_gmc_funcs()
1169 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)  in gmc_v6_0_set_irq_funcs()  argument
1171 	adev->gmc.vm_fault.num_types = 1;  in gmc_v6_0_set_irq_funcs()
1172 	adev->gmc.vm_fault.funcs = &gmc_v6_0_irq_funcs;  in gmc_v6_0_set_irq_funcs()