53 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev);
54 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
74 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)  in gmc_v7_0_init_golden_registers()  argument
76 	switch (adev->asic_type) {  in gmc_v7_0_init_golden_registers()
78 		amdgpu_device_program_register_sequence(adev,  in gmc_v7_0_init_golden_registers()
81 		amdgpu_device_program_register_sequence(adev,  in gmc_v7_0_init_golden_registers()
90 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)  in gmc_v7_0_mc_stop()  argument
94 	gmc_v7_0_wait_for_idle((void *)adev);  in gmc_v7_0_mc_stop()
109 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)  in gmc_v7_0_mc_resume()  argument
132 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)  in gmc_v7_0_init_microcode()  argument
140 	switch (adev->asic_type) {  in gmc_v7_0_init_microcode()
159 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);  in gmc_v7_0_init_microcode()
162 	err = amdgpu_ucode_validate(adev->gmc.fw);  in gmc_v7_0_init_microcode()
167 		release_firmware(adev->gmc.fw);  in gmc_v7_0_init_microcode()
168 		adev->gmc.fw = NULL;  in gmc_v7_0_init_microcode()
181 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)  in gmc_v7_0_mc_load_microcode()  argument
189 	if (!adev->gmc.fw)  in gmc_v7_0_mc_load_microcode()
192 	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;  in gmc_v7_0_mc_load_microcode()
195 	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);  in gmc_v7_0_mc_load_microcode()
198 		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in gmc_v7_0_mc_load_microcode()
201 		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in gmc_v7_0_mc_load_microcode()
225 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_mc_load_microcode()
231 		for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_mc_load_microcode()
242 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,  in gmc_v7_0_vram_gtt_location()  argument
248 	amdgpu_gmc_vram_location(adev, mc, base);  in gmc_v7_0_vram_gtt_location()
249 	amdgpu_gmc_gart_location(adev, mc);  in gmc_v7_0_vram_gtt_location()
260 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)  in gmc_v7_0_mc_program()  argument
275 	if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_mc_program()
276 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v7_0_mc_program()
278 	if (adev->mode_info.num_crtc) {  in gmc_v7_0_mc_program()
291 	       adev->gmc.vram_start >> 12);  in gmc_v7_0_mc_program()
293 	       adev->gmc.vram_end >> 12);  in gmc_v7_0_mc_program()
295 	       adev->vram_scratch.gpu_addr >> 12);  in gmc_v7_0_mc_program()
299 	if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_mc_program()
300 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");  in gmc_v7_0_mc_program()
322 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)  in gmc_v7_0_mc_init()  argument
326 	adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev);  in gmc_v7_0_mc_init()
327 	if (!adev->gmc.vram_width) {  in gmc_v7_0_mc_init()
369 		adev->gmc.vram_width = numchan * chansize;  in gmc_v7_0_mc_init()
372 	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v7_0_mc_init()
373 	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in gmc_v7_0_mc_init()
375 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_mc_init()
376 		r = amdgpu_device_resize_fb_bar(adev);  in gmc_v7_0_mc_init()
380 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);  in gmc_v7_0_mc_init()
381 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);  in gmc_v7_0_mc_init()
384 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_mc_init()
385 		adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;  in gmc_v7_0_mc_init()
386 		adev->gmc.aper_size = adev->gmc.real_vram_size;  in gmc_v7_0_mc_init()
391 	adev->gmc.visible_vram_size = adev->gmc.aper_size;  in gmc_v7_0_mc_init()
392 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)  in gmc_v7_0_mc_init()
393 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;  in gmc_v7_0_mc_init()
397 		switch (adev->asic_type) {  in gmc_v7_0_mc_init()
400 			adev->gmc.gart_size = 256ULL << 20;  in gmc_v7_0_mc_init()
408 			adev->gmc.gart_size = 1024ULL << 20;  in gmc_v7_0_mc_init()
413 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;  in gmc_v7_0_mc_init()
416 	gmc_v7_0_vram_gtt_location(adev, &adev->gmc);  in gmc_v7_0_mc_init()
436 static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,  in gmc_v7_0_flush_gpu_tlb()  argument
466 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,  in gmc_v7_0_get_vm_pte_flags()  argument
481 static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,  in gmc_v7_0_get_vm_pde()  argument
493 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,  in gmc_v7_0_set_fault_enable_default()  argument
520 static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)  in gmc_v7_0_set_prt()  argument
524 	if (enable && !adev->gmc.prt_warning) {  in gmc_v7_0_set_prt()
525 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");  in gmc_v7_0_set_prt()
526 		adev->gmc.prt_warning = true;  in gmc_v7_0_set_prt()
548 		uint32_t high = adev->vm_manager.max_pfn -  in gmc_v7_0_set_prt()
582 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)  in gmc_v7_0_gart_enable()  argument
588 	if (adev->gart.bo == NULL) {  in gmc_v7_0_gart_enable()
589 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");  in gmc_v7_0_gart_enable()
592 	r = amdgpu_gart_table_vram_pin(adev);  in gmc_v7_0_gart_enable()
596 	table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);  in gmc_v7_0_gart_enable()
620 	field = adev->vm_manager.fragment_size;  in gmc_v7_0_gart_enable()
627 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);  in gmc_v7_0_gart_enable()
628 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);  in gmc_v7_0_gart_enable()
631 			(u32)(adev->dummy_page_addr >> 12));  in gmc_v7_0_gart_enable()
649 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);  in gmc_v7_0_gart_enable()
661 	       (u32)(adev->dummy_page_addr >> 12));  in gmc_v7_0_gart_enable()
667 			    adev->vm_manager.block_size - 9);  in gmc_v7_0_gart_enable()
670 		gmc_v7_0_set_fault_enable_default(adev, false);  in gmc_v7_0_gart_enable()
672 		gmc_v7_0_set_fault_enable_default(adev, true);  in gmc_v7_0_gart_enable()
674 	if (adev->asic_type == CHIP_KAVERI) {  in gmc_v7_0_gart_enable()
680 	gmc_v7_0_flush_gpu_tlb(adev, 0, 0, 0);  in gmc_v7_0_gart_enable()
682 		 (unsigned)(adev->gmc.gart_size >> 20),  in gmc_v7_0_gart_enable()
684 	adev->gart.ready = true;  in gmc_v7_0_gart_enable()
688 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)  in gmc_v7_0_gart_init()  argument
692 	if (adev->gart.bo) {  in gmc_v7_0_gart_init()
697 	r = amdgpu_gart_init(adev);  in gmc_v7_0_gart_init()
700 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;  in gmc_v7_0_gart_init()
701 	adev->gart.gart_pte_flags = 0;  in gmc_v7_0_gart_init()
702 	return amdgpu_gart_table_vram_alloc(adev);  in gmc_v7_0_gart_init()
712 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)  in gmc_v7_0_gart_disable()  argument
730 	amdgpu_gart_table_vram_unpin(adev);  in gmc_v7_0_gart_disable()
742 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,  in gmc_v7_0_vm_decode_fault()  argument
755 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",  in gmc_v7_0_vm_decode_fault()
799 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,  in gmc_v7_0_enable_mc_ls()  argument
807 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))  in gmc_v7_0_enable_mc_ls()
816 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,  in gmc_v7_0_enable_mc_mgcg()  argument
824 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))  in gmc_v7_0_enable_mc_mgcg()
833 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,  in gmc_v7_0_enable_bif_mgls()  argument
840 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {  in gmc_v7_0_enable_bif_mgls()
856 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,  in gmc_v7_0_enable_hdp_mgcg()  argument
863 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))  in gmc_v7_0_enable_hdp_mgcg()
872 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,  in gmc_v7_0_enable_hdp_ls()  argument
879 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))  in gmc_v7_0_enable_hdp_ls()
912 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_early_init()  local
914 	gmc_v7_0_set_gmc_funcs(adev);  in gmc_v7_0_early_init()
915 	gmc_v7_0_set_irq_funcs(adev);  in gmc_v7_0_early_init()
917 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;  in gmc_v7_0_early_init()
918 	adev->gmc.shared_aperture_end =  in gmc_v7_0_early_init()
919 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;  in gmc_v7_0_early_init()
920 	adev->gmc.private_aperture_start =  in gmc_v7_0_early_init()
921 		adev->gmc.shared_aperture_end + 1;  in gmc_v7_0_early_init()
922 	adev->gmc.private_aperture_end =  in gmc_v7_0_early_init()
923 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;  in gmc_v7_0_early_init()
930 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_late_init()  local
932 	amdgpu_bo_late_init(adev);  in gmc_v7_0_late_init()
935 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);  in gmc_v7_0_late_init()
940 static unsigned gmc_v7_0_get_vbios_fb_size(struct amdgpu_device *adev)  in gmc_v7_0_get_vbios_fb_size()  argument
954 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in gmc_v7_0_get_vbios_fb_size()
962 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_sw_init()  local
964 	adev->num_vmhubs = 1;  in gmc_v7_0_sw_init()
966 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_sw_init()
967 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;  in gmc_v7_0_sw_init()
971 		adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp);  in gmc_v7_0_sw_init()
974 …r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &a…  in gmc_v7_0_sw_init()
978 …r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &a…  in gmc_v7_0_sw_init()
986 	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);  in gmc_v7_0_sw_init()
992 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */  in gmc_v7_0_sw_init()
994 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));  in gmc_v7_0_sw_init()
999 	adev->need_swiotlb = drm_need_swiotlb(40);  in gmc_v7_0_sw_init()
1001 	r = gmc_v7_0_init_microcode(adev);  in gmc_v7_0_sw_init()
1007 	r = gmc_v7_0_mc_init(adev);  in gmc_v7_0_sw_init()
1011 	adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev);  in gmc_v7_0_sw_init()
1014 	r = amdgpu_bo_init(adev);  in gmc_v7_0_sw_init()
1018 	r = gmc_v7_0_gart_init(adev);  in gmc_v7_0_sw_init()
1028 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v7_0_sw_init()
1029 	amdgpu_vm_manager_init(adev);  in gmc_v7_0_sw_init()
1032 	if (adev->flags & AMD_IS_APU) {  in gmc_v7_0_sw_init()
1036 		adev->vm_manager.vram_base_offset = tmp;  in gmc_v7_0_sw_init()
1038 		adev->vm_manager.vram_base_offset = 0;  in gmc_v7_0_sw_init()
1041 	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),  in gmc_v7_0_sw_init()
1043 	if (!adev->gmc.vm_fault_info)  in gmc_v7_0_sw_init()
1045 	atomic_set(&adev->gmc.vm_fault_info_updated, 0);  in gmc_v7_0_sw_init()
1052 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_sw_fini()  local
1054 	amdgpu_gem_force_release(adev);  in gmc_v7_0_sw_fini()
1055 	amdgpu_vm_manager_fini(adev);  in gmc_v7_0_sw_fini()
1056 	kfree(adev->gmc.vm_fault_info);  in gmc_v7_0_sw_fini()
1057 	amdgpu_gart_table_vram_free(adev);  in gmc_v7_0_sw_fini()
1058 	amdgpu_bo_fini(adev);  in gmc_v7_0_sw_fini()
1059 	amdgpu_gart_fini(adev);  in gmc_v7_0_sw_fini()
1060 	release_firmware(adev->gmc.fw);  in gmc_v7_0_sw_fini()
1061 	adev->gmc.fw = NULL;  in gmc_v7_0_sw_fini()
1069 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_hw_init()  local
1071 	gmc_v7_0_init_golden_registers(adev);  in gmc_v7_0_hw_init()
1073 	gmc_v7_0_mc_program(adev);  in gmc_v7_0_hw_init()
1075 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_hw_init()
1076 		r = gmc_v7_0_mc_load_microcode(adev);  in gmc_v7_0_hw_init()
1083 	r = gmc_v7_0_gart_enable(adev);  in gmc_v7_0_hw_init()
1092 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_hw_fini()  local
1094 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);  in gmc_v7_0_hw_fini()
1095 	gmc_v7_0_gart_disable(adev);  in gmc_v7_0_hw_fini()
1102 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_suspend()  local
1104 	gmc_v7_0_hw_fini(adev);  in gmc_v7_0_suspend()
1112 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_resume()  local
1114 	r = gmc_v7_0_hw_init(adev);  in gmc_v7_0_resume()
1118 	amdgpu_vmid_reset_all(adev);  in gmc_v7_0_resume()
1125 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_is_idle()  local
1139 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_wait_for_idle()  local
1141 	for (i = 0; i < adev->usec_timeout; i++) {  in gmc_v7_0_wait_for_idle()
1158 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_soft_reset()  local
1168 		if (!(adev->flags & AMD_IS_APU))  in gmc_v7_0_soft_reset()
1174 		gmc_v7_0_mc_stop(adev);  in gmc_v7_0_soft_reset()
1175 		if (gmc_v7_0_wait_for_idle((void *)adev)) {  in gmc_v7_0_soft_reset()
1176 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");  in gmc_v7_0_soft_reset()
1182 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in gmc_v7_0_soft_reset()
1195 		gmc_v7_0_mc_resume(adev);  in gmc_v7_0_soft_reset()
1202 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,  in gmc_v7_0_vm_fault_interrupt_state()  argument
1243 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,  in gmc_v7_0_process_interrupt()  argument
1259 		gmc_v7_0_set_fault_enable_default(adev, false);  in gmc_v7_0_process_interrupt()
1262 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",  in gmc_v7_0_process_interrupt()
1264 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in gmc_v7_0_process_interrupt()
1266 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in gmc_v7_0_process_interrupt()
1268 		gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client,  in gmc_v7_0_process_interrupt()
1274 	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)  in gmc_v7_0_process_interrupt()
1275 		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {  in gmc_v7_0_process_interrupt()
1276 		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;  in gmc_v7_0_process_interrupt()
1292 		atomic_set(&adev->gmc.vm_fault_info_updated, 1);  in gmc_v7_0_process_interrupt()
1302 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v7_0_set_clockgating_state()  local
1307 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v7_0_set_clockgating_state()
1308 		gmc_v7_0_enable_mc_mgcg(adev, gate);  in gmc_v7_0_set_clockgating_state()
1309 		gmc_v7_0_enable_mc_ls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1311 	gmc_v7_0_enable_bif_mgls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1312 	gmc_v7_0_enable_hdp_mgcg(adev, gate);  in gmc_v7_0_set_clockgating_state()
1313 	gmc_v7_0_enable_hdp_ls(adev, gate);  in gmc_v7_0_set_clockgating_state()
1355 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev)  in gmc_v7_0_set_gmc_funcs()  argument
1357 	adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs;  in gmc_v7_0_set_gmc_funcs()
1360 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)  in gmc_v7_0_set_irq_funcs()  argument
1362 	adev->gmc.vm_fault.num_types = 1;  in gmc_v7_0_set_irq_funcs()
1363 	adev->gmc.vm_fault.funcs = &gmc_v7_0_irq_funcs;  in gmc_v7_0_set_irq_funcs()