49 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev);
50 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
70 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)  in gmc_v7_0_init_golden_registers()  argument
72 	switch (adev->asic_type) {  in gmc_v7_0_init_golden_registers()
74 		amdgpu_device_program_register_sequence(adev,  in gmc_v7_0_init_golden_registers()
77 		amdgpu_device_program_register_sequence(adev,  in gmc_v7_0_init_golden_registers()
86 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)  in gmc_v7_0_mc_stop()  argument
90 	gmc_v7_0_wait_for_idle((void *)adev);  in gmc_v7_0_mc_stop()
105 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)  in gmc_v7_0_mc_resume()  argument
128 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)  in gmc_v7_0_init_microcode()  argument
136 	switch (adev->asic_type) {  in gmc_v7_0_init_microcode()
155 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);  in gmc_v7_0_init_microcode()
158 	err = amdgpu_ucode_validate(adev->gmc.fw);  in gmc_v7_0_init_microcode()
163 		release_firmware(adev->gmc.fw);  in gmc_v7_0_init_microcode()
164 		adev->gmc.fw = NULL;  in gmc_v7_0_init_microcode()
177 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)  in gmc_v7_0_mc_load_microcode()  argument
185 	if (!adev->gmc.fw)  in gmc_v7_0_mc_load_microcode()
188 	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;  in gmc_v7_0_mc_load_microcode()
191 	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);  in gmc_v7_0_mc_load_microcode()
194 		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in gmc_v7_0_mc_load_microcode()
197 		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in gmc_v7_0_mc_load_microcode()
221 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_mc_load_microcode()
227 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_mc_load_microcode()
238 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,  in gmc_v7_0_vram_gtt_location()  argument
244 	amdgpu_device_vram_location(adev, &adev->gmc, base);  in gmc_v7_0_vram_gtt_location()
245 	amdgpu_device_gart_location(adev, mc);  in gmc_v7_0_vram_gtt_location()
256 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)  in gmc_v7_0_mc_program()  argument
271 	if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_mc_program()
272 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v7_0_mc_program()
274 	if (adev->mode_info.num_crtc) {  in gmc_v7_0_mc_program()
287 	       adev->gmc.vram_start >> 12);  in gmc_v7_0_mc_program()
289 	       adev->gmc.vram_end >> 12);  in gmc_v7_0_mc_program()
291 	       adev->vram_scratch.gpu_addr >> 12);  in gmc_v7_0_mc_program()
295 	if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_mc_program()
296 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v7_0_mc_program()
318 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)  in gmc_v7_0_mc_init()  argument
322 	adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev);  in gmc_v7_0_mc_init()
323 	if (!adev->gmc.vram_width) {  in gmc_v7_0_mc_init()
365 		adev->gmc.vram_width = numchan * chansize;  in gmc_v7_0_mc_init()
368 	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v7_0_mc_init()
369 	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v7_0_mc_init()
371 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_mc_init()
372 		r = amdgpu_device_resize_fb_bar(adev);  in gmc_v7_0_mc_init()
376 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);  in gmc_v7_0_mc_init()
377 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);  in gmc_v7_0_mc_init()
380 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_mc_init()
381 		adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;  in gmc_v7_0_mc_init()
382 		adev->gmc.aper_size = adev->gmc.real_vram_size;  in gmc_v7_0_mc_init()
387 	adev->gmc.visible_vram_size = adev->gmc.aper_size;  in gmc_v7_0_mc_init()
388 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)  in gmc_v7_0_mc_init()
389 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;  in gmc_v7_0_mc_init()
393 		switch (adev->asic_type) {  in gmc_v7_0_mc_init()
396 			adev->gmc.gart_size = 256ULL << 20;  in gmc_v7_0_mc_init()
404 			adev->gmc.gart_size = 1024ULL << 20;  in gmc_v7_0_mc_init()
409 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;  in gmc_v7_0_mc_init()
412 	gmc_v7_0_vram_gtt_location(adev, &adev->gmc);  in gmc_v7_0_mc_init()
432 static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)  in gmc_v7_0_flush_gpu_tlb()  argument
472 static int gmc_v7_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,  in gmc_v7_0_set_pte_pde()  argument
486 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,  in gmc_v7_0_get_vm_pte_flags()  argument
501 static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,  in gmc_v7_0_get_vm_pde()  argument
513 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,  in gmc_v7_0_set_fault_enable_default()  argument
540 static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)  in gmc_v7_0_set_prt()  argument
544 	if (enable && !adev->gmc.prt_warning) {  in gmc_v7_0_set_prt()
545 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");  in gmc_v7_0_set_prt()
546 		adev->gmc.prt_warning = true;  in gmc_v7_0_set_prt()
568 		uint32_t high = adev->vm_manager.max_pfn -  in gmc_v7_0_set_prt()
602 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)  in gmc_v7_0_gart_enable()  argument
607 	if (adev->gart.robj == NULL) {  in gmc_v7_0_gart_enable()
608 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");  in gmc_v7_0_gart_enable()
611 	r = amdgpu_gart_table_vram_pin(adev);  in gmc_v7_0_gart_enable()
636 	field = adev->vm_manager.fragment_size;  in gmc_v7_0_gart_enable()
643 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);  in gmc_v7_0_gart_enable()
644 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);  in gmc_v7_0_gart_enable()
645 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);  in gmc_v7_0_gart_enable()
647 			(u32)(adev->dummy_page_addr >> 12));  in gmc_v7_0_gart_enable()
665 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);  in gmc_v7_0_gart_enable()
669 			       adev->gart.table_addr >> 12);  in gmc_v7_0_gart_enable()
672 			       adev->gart.table_addr >> 12);  in gmc_v7_0_gart_enable()
677 	       (u32)(adev->dummy_page_addr >> 12));  in gmc_v7_0_gart_enable()
683 			    adev->vm_manager.block_size - 9);  in gmc_v7_0_gart_enable()
686 		gmc_v7_0_set_fault_enable_default(adev, false);  in gmc_v7_0_gart_enable()
688 		gmc_v7_0_set_fault_enable_default(adev, true);  in gmc_v7_0_gart_enable()
690 	if (adev->asic_type == CHIP_KAVERI) {  in gmc_v7_0_gart_enable()
696 	gmc_v7_0_flush_gpu_tlb(adev, 0);  in gmc_v7_0_gart_enable()
698 		 (unsigned)(adev->gmc.gart_size >> 20),  in gmc_v7_0_gart_enable()
699 		 (unsigned long long)adev->gart.table_addr);  in gmc_v7_0_gart_enable()
700 	adev->gart.ready = true;  in gmc_v7_0_gart_enable()
704 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)  in gmc_v7_0_gart_init()  argument
708 	if (adev->gart.robj) {  in gmc_v7_0_gart_init()
713 	r = amdgpu_gart_init(adev);  in gmc_v7_0_gart_init()
716 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;  in gmc_v7_0_gart_init()
717 	adev->gart.gart_pte_flags = 0;  in gmc_v7_0_gart_init()
718 	return amdgpu_gart_table_vram_alloc(adev);  in gmc_v7_0_gart_init()
728 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)  in gmc_v7_0_gart_disable()  argument
746 	amdgpu_gart_table_vram_unpin(adev);  in gmc_v7_0_gart_disable()
758 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,  in gmc_v7_0_vm_decode_fault()  argument
771 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",  in gmc_v7_0_vm_decode_fault()
815 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,  in gmc_v7_0_enable_mc_ls()  argument
823 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))  in gmc_v7_0_enable_mc_ls()
832 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,  in gmc_v7_0_enable_mc_mgcg()  argument
840 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))  in gmc_v7_0_enable_mc_mgcg()
849 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,  in gmc_v7_0_enable_bif_mgls()  argument
856 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {  in gmc_v7_0_enable_bif_mgls()
872 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,  in gmc_v7_0_enable_hdp_mgcg()  argument
879 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))  in gmc_v7_0_enable_hdp_mgcg()
888 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,  in gmc_v7_0_enable_hdp_ls()  argument
895 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))  in gmc_v7_0_enable_hdp_ls()
928 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_early_init()  local
930 	gmc_v7_0_set_gmc_funcs(adev);  in gmc_v7_0_early_init()
931 	gmc_v7_0_set_irq_funcs(adev);  in gmc_v7_0_early_init()
933 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;  in gmc_v7_0_early_init()
934 	adev->gmc.shared_aperture_end =  in gmc_v7_0_early_init()
935 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;  in gmc_v7_0_early_init()
936 	adev->gmc.private_aperture_start =  in gmc_v7_0_early_init()
937 		adev->gmc.shared_aperture_end + 1;  in gmc_v7_0_early_init()
938 	adev->gmc.private_aperture_end =  in gmc_v7_0_early_init()
939 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;  in gmc_v7_0_early_init()
946 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_late_init()  local
948 	amdgpu_bo_late_init(adev);  in gmc_v7_0_late_init()
951 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);  in gmc_v7_0_late_init()
956 static unsigned gmc_v7_0_get_vbios_fb_size(struct amdgpu_device *adev)  in gmc_v7_0_get_vbios_fb_size()  argument
970 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in gmc_v7_0_get_vbios_fb_size()
979 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_sw_init()  local
981 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_sw_init()
982 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;  in gmc_v7_0_sw_init()
986 		adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp);  in gmc_v7_0_sw_init()
989 …r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &ad…  in gmc_v7_0_sw_init()
993 …r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &ad…  in gmc_v7_0_sw_init()
1001 	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);  in gmc_v7_0_sw_init()
1007 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */  in gmc_v7_0_sw_init()
1014 	adev->need_dma32 = false;  in gmc_v7_0_sw_init()
1015 	dma_bits = adev->need_dma32 ? 32 : 40;  in gmc_v7_0_sw_init()
1016 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v7_0_sw_init()
1018 		adev->need_dma32 = true;  in gmc_v7_0_sw_init()
1022 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v7_0_sw_init()
1024 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));  in gmc_v7_0_sw_init()
1027 	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);  in gmc_v7_0_sw_init()
1029 	r = gmc_v7_0_init_microcode(adev);  in gmc_v7_0_sw_init()
1035 	r = gmc_v7_0_mc_init(adev);  in gmc_v7_0_sw_init()
1039 	adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev);  in gmc_v7_0_sw_init()
1042 	r = amdgpu_bo_init(adev);  in gmc_v7_0_sw_init()
1046 	r = gmc_v7_0_gart_init(adev);  in gmc_v7_0_sw_init()
1056 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v7_0_sw_init()
1057 	amdgpu_vm_manager_init(adev);  in gmc_v7_0_sw_init()
1060 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_sw_init()
1064 		adev->vm_manager.vram_base_offset = tmp;  in gmc_v7_0_sw_init()
1066 		adev->vm_manager.vram_base_offset = 0;  in gmc_v7_0_sw_init()
1069 	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),  in gmc_v7_0_sw_init()
1071 	if (!adev->gmc.vm_fault_info)  in gmc_v7_0_sw_init()
1073 	atomic_set(&adev->gmc.vm_fault_info_updated, 0);  in gmc_v7_0_sw_init()
1080 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_sw_fini()  local
1082 	amdgpu_gem_force_release(adev);  in gmc_v7_0_sw_fini()
1083 	amdgpu_vm_manager_fini(adev);  in gmc_v7_0_sw_fini()
1084 	kfree(adev->gmc.vm_fault_info);  in gmc_v7_0_sw_fini()
1085 	amdgpu_gart_table_vram_free(adev);  in gmc_v7_0_sw_fini()
1086 	amdgpu_bo_fini(adev);  in gmc_v7_0_sw_fini()
1087 	amdgpu_gart_fini(adev);  in gmc_v7_0_sw_fini()
1088 	release_firmware(adev->gmc.fw);  in gmc_v7_0_sw_fini()
1089 	adev->gmc.fw = NULL;  in gmc_v7_0_sw_fini()
1097 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_hw_init()  local
1099 	gmc_v7_0_init_golden_registers(adev);  in gmc_v7_0_hw_init()
1101 	gmc_v7_0_mc_program(adev);  in gmc_v7_0_hw_init()
1103 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_hw_init()
1104 		r = gmc_v7_0_mc_load_microcode(adev);  in gmc_v7_0_hw_init()
1111 	r = gmc_v7_0_gart_enable(adev);  in gmc_v7_0_hw_init()
1120 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_hw_fini()  local
1122 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);  in gmc_v7_0_hw_fini()
1123 	gmc_v7_0_gart_disable(adev);  in gmc_v7_0_hw_fini()
1130 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_suspend()  local
1132 	gmc_v7_0_hw_fini(adev);  in gmc_v7_0_suspend()
1140 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_resume()  local
1142 	r = gmc_v7_0_hw_init(adev);  in gmc_v7_0_resume()
1146 	amdgpu_vmid_reset_all(adev);  in gmc_v7_0_resume()
1153 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_is_idle()  local
1167 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_wait_for_idle()  local
1169 	for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_wait_for_idle()
1186 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_soft_reset()  local
1196 		if (!(adev->flags & AMD_IS_APU))  in gmc_v7_0_soft_reset()
1202 		gmc_v7_0_mc_stop(adev);  in gmc_v7_0_soft_reset()
1203 		if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_soft_reset()
1204 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");  in gmc_v7_0_soft_reset()
1210 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in gmc_v7_0_soft_reset()
1223 		gmc_v7_0_mc_resume(adev);  in gmc_v7_0_soft_reset()
1230 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,  in gmc_v7_0_vm_fault_interrupt_state()  argument
1271 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,  in gmc_v7_0_process_interrupt()  argument
1287 		gmc_v7_0_set_fault_enable_default(adev, false);  in gmc_v7_0_process_interrupt()
1290 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",  in gmc_v7_0_process_interrupt()
1292 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in gmc_v7_0_process_interrupt()
1294 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in gmc_v7_0_process_interrupt()
1296 		gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client,  in gmc_v7_0_process_interrupt()
1302 	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)  in gmc_v7_0_process_interrupt()
1303 		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {  in gmc_v7_0_process_interrupt()
1304 		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;  in gmc_v7_0_process_interrupt()
1320 		atomic_set(&adev->gmc.vm_fault_info_updated, 1);  in gmc_v7_0_process_interrupt()
1330 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_set_clockgating_state()  local
1335 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_set_clockgating_state()
1336 		gmc_v7_0_enable_mc_mgcg(adev, gate);  in gmc_v7_0_set_clockgating_state()
1337 		gmc_v7_0_enable_mc_ls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1339 	gmc_v7_0_enable_bif_mgls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1340 	gmc_v7_0_enable_hdp_mgcg(adev, gate);  in gmc_v7_0_set_clockgating_state()
1341 	gmc_v7_0_enable_hdp_ls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1384 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev)  in gmc_v7_0_set_gmc_funcs()  argument
1386 	if (adev->gmc.gmc_funcs == NULL)  in gmc_v7_0_set_gmc_funcs()
1387 		adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs;  in gmc_v7_0_set_gmc_funcs()
1390 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)  in gmc_v7_0_set_irq_funcs()  argument
1392 	adev->gmc.vm_fault.num_types = 1;  in gmc_v7_0_set_irq_funcs()
1393 	adev->gmc.vm_fault.funcs = &gmc_v7_0_irq_funcs;  in gmc_v7_0_set_irq_funcs()