200 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,  in gmc_v9_0_vm_fault_interrupt_state()  argument
219 			hub = &adev->vmhub[j];  in gmc_v9_0_vm_fault_interrupt_state()
230 			hub = &adev->vmhub[j];  in gmc_v9_0_vm_fault_interrupt_state()
245 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,  in gmc_v9_0_process_interrupt()  argument
249 	struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];  in gmc_v9_0_process_interrupt()
256 	if (!amdgpu_sriov_vf(adev)) {  in gmc_v9_0_process_interrupt()
264 		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);  in gmc_v9_0_process_interrupt()
266 		dev_err(adev->dev,  in gmc_v9_0_process_interrupt()
272 		dev_err(adev->dev, "  at address 0x%016llx from %d\n",  in gmc_v9_0_process_interrupt()
274 		if (!amdgpu_sriov_vf(adev))  in gmc_v9_0_process_interrupt()
275 			dev_err(adev->dev,  in gmc_v9_0_process_interrupt()
288 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)  in gmc_v9_0_set_irq_funcs()  argument
290 	adev->gmc.vm_fault.num_types = 1;  in gmc_v9_0_set_irq_funcs()
291 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;  in gmc_v9_0_set_irq_funcs()
328 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,  in gmc_v9_0_flush_gpu_tlb()  argument
335 	spin_lock(&adev->gmc.invalidate_lock);  in gmc_v9_0_flush_gpu_tlb()
338 		struct amdgpu_vmhub *hub = &adev->vmhub[i];  in gmc_v9_0_flush_gpu_tlb()
355 		for (j = 0; j < adev->usec_timeout; j++) {  in gmc_v9_0_flush_gpu_tlb()
362 		if (j < adev->usec_timeout)  in gmc_v9_0_flush_gpu_tlb()
368 	spin_unlock(&adev->gmc.invalidate_lock);  in gmc_v9_0_flush_gpu_tlb()
374 	struct amdgpu_device *adev = ring->adev;  in gmc_v9_0_emit_flush_gpu_tlb()  local
375 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];  in gmc_v9_0_emit_flush_gpu_tlb()
380 	amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags);  in gmc_v9_0_emit_flush_gpu_tlb()
399 	struct amdgpu_device *adev = ring->adev;  in gmc_v9_0_emit_pasid_mapping()  local
421 static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,  in gmc_v9_0_set_pte_pde()  argument
469 static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,  in gmc_v9_0_get_vm_pte_flags()  argument
509 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,  in gmc_v9_0_get_vm_pde()  argument
513 		*addr = adev->vm_manager.vram_base_offset + *addr -  in gmc_v9_0_get_vm_pde()
514 			adev->gmc.vram_start;  in gmc_v9_0_get_vm_pde()
517 	if (!adev->gmc.translate_further)  in gmc_v9_0_get_vm_pde()
542 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)  in gmc_v9_0_set_gmc_funcs()  argument
544 	if (adev->gmc.gmc_funcs == NULL)  in gmc_v9_0_set_gmc_funcs()
545 		adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;  in gmc_v9_0_set_gmc_funcs()
550 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_early_init()  local
552 	gmc_v9_0_set_gmc_funcs(adev);  in gmc_v9_0_early_init()
553 	gmc_v9_0_set_irq_funcs(adev);  in gmc_v9_0_early_init()
555 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;  in gmc_v9_0_early_init()
556 	adev->gmc.shared_aperture_end =  in gmc_v9_0_early_init()
557 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;  in gmc_v9_0_early_init()
558 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;  in gmc_v9_0_early_init()
559 	adev->gmc.private_aperture_end =  in gmc_v9_0_early_init()
560 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;  in gmc_v9_0_early_init()
565 static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)  in gmc_v9_0_ecc_available()  argument
646 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_late_init()  local
665 	for(i = 0; i < adev->num_rings; ++i) {  in gmc_v9_0_late_init()
666 		struct amdgpu_ring *ring = adev->rings[i];  in gmc_v9_0_late_init()
670 		dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",  in gmc_v9_0_late_init()
679 	if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {  in gmc_v9_0_late_init()
680 		r = gmc_v9_0_ecc_available(adev);  in gmc_v9_0_late_init()
685 			adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);  in gmc_v9_0_late_init()
692 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);  in gmc_v9_0_late_init()
695 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,  in gmc_v9_0_vram_gtt_location()  argument
699 	if (!amdgpu_sriov_vf(adev))  in gmc_v9_0_vram_gtt_location()
700 		base = mmhub_v1_0_get_fb_location(adev);  in gmc_v9_0_vram_gtt_location()
701 	amdgpu_device_vram_location(adev, &adev->gmc, base);  in gmc_v9_0_vram_gtt_location()
702 	amdgpu_device_gart_location(adev, mc);  in gmc_v9_0_vram_gtt_location()
704 	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);  in gmc_v9_0_vram_gtt_location()
716 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)  in gmc_v9_0_mc_init()  argument
722 		adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);  in gmc_v9_0_mc_init()
723 	if (!adev->gmc.vram_width) {  in gmc_v9_0_mc_init()
725 		if (adev->flags & AMD_IS_APU)  in gmc_v9_0_mc_init()
730 		numchan = adev->df_funcs->get_hbm_channel_number(adev);  in gmc_v9_0_mc_init()
731 		adev->gmc.vram_width = numchan * chansize;  in gmc_v9_0_mc_init()
735 	adev->gmc.mc_vram_size =  in gmc_v9_0_mc_init()
736 		adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;  in gmc_v9_0_mc_init()
737 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;  in gmc_v9_0_mc_init()
739 	if (!(adev->flags & AMD_IS_APU)) {  in gmc_v9_0_mc_init()
740 		r = amdgpu_device_resize_fb_bar(adev);  in gmc_v9_0_mc_init()
744 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);  in gmc_v9_0_mc_init()
745 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);  in gmc_v9_0_mc_init()
748 	if (adev->flags & AMD_IS_APU) {  in gmc_v9_0_mc_init()
749 		adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);  in gmc_v9_0_mc_init()
750 		adev->gmc.aper_size = adev->gmc.real_vram_size;  in gmc_v9_0_mc_init()
754 	adev->gmc.visible_vram_size = adev->gmc.aper_size;  in gmc_v9_0_mc_init()
755 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)  in gmc_v9_0_mc_init()
756 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;  in gmc_v9_0_mc_init()
760 		switch (adev->asic_type) {  in gmc_v9_0_mc_init()
765 			adev->gmc.gart_size = 512ULL << 20;  in gmc_v9_0_mc_init()
768 			adev->gmc.gart_size = 1024ULL << 20;  in gmc_v9_0_mc_init()
772 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;  in gmc_v9_0_mc_init()
775 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);  in gmc_v9_0_mc_init()
780 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)  in gmc_v9_0_gart_init()  argument
784 	if (adev->gart.robj) {  in gmc_v9_0_gart_init()
789 	r = amdgpu_gart_init(adev);  in gmc_v9_0_gart_init()
792 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;  in gmc_v9_0_gart_init()
793 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) |  in gmc_v9_0_gart_init()
795 	return amdgpu_gart_table_vram_alloc(adev);  in gmc_v9_0_gart_init()
798 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)  in gmc_v9_0_get_vbios_fb_size()  argument
817 		switch (adev->asic_type) {  in gmc_v9_0_get_vbios_fb_size()
837 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in gmc_v9_0_get_vbios_fb_size()
848 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_sw_init()  local
850 	gfxhub_v1_0_init(adev);  in gmc_v9_0_sw_init()
851 	mmhub_v1_0_init(adev);  in gmc_v9_0_sw_init()
853 	spin_lock_init(&adev->gmc.invalidate_lock);  in gmc_v9_0_sw_init()
855 	adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);  in gmc_v9_0_sw_init()
856 	switch (adev->asic_type) {  in gmc_v9_0_sw_init()
858 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {  in gmc_v9_0_sw_init()
859 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);  in gmc_v9_0_sw_init()
862 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);  in gmc_v9_0_sw_init()
863 			adev->gmc.translate_further =  in gmc_v9_0_sw_init()
864 				adev->vm_manager.num_level > 1;  in gmc_v9_0_sw_init()
875 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);  in gmc_v9_0_sw_init()
882 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,  in gmc_v9_0_sw_init()
883 				&adev->gmc.vm_fault);  in gmc_v9_0_sw_init()
884 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,  in gmc_v9_0_sw_init()
885 				&adev->gmc.vm_fault);  in gmc_v9_0_sw_init()
894 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */  in gmc_v9_0_sw_init()
901 	adev->need_dma32 = false;  in gmc_v9_0_sw_init()
902 	dma_bits = adev->need_dma32 ? 32 : 44;  in gmc_v9_0_sw_init()
903 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v9_0_sw_init()
905 		adev->need_dma32 = true;  in gmc_v9_0_sw_init()
909 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));  in gmc_v9_0_sw_init()
911 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));  in gmc_v9_0_sw_init()
914 	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);  in gmc_v9_0_sw_init()
916 	r = gmc_v9_0_mc_init(adev);  in gmc_v9_0_sw_init()
920 	adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev);  in gmc_v9_0_sw_init()
923 	r = amdgpu_bo_init(adev);  in gmc_v9_0_sw_init()
927 	r = gmc_v9_0_gart_init(adev);  in gmc_v9_0_sw_init()
937 	adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v9_0_sw_init()
938 	adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;  in gmc_v9_0_sw_init()
940 	amdgpu_vm_manager_init(adev);  in gmc_v9_0_sw_init()
947 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_sw_fini()  local
949 	amdgpu_gem_force_release(adev);  in gmc_v9_0_sw_fini()
950 	amdgpu_vm_manager_fini(adev);  in gmc_v9_0_sw_fini()
961 	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);  in gmc_v9_0_sw_fini()
963 	amdgpu_gart_table_vram_free(adev);  in gmc_v9_0_sw_fini()
964 	amdgpu_bo_fini(adev);  in gmc_v9_0_sw_fini()
965 	amdgpu_gart_fini(adev);  in gmc_v9_0_sw_fini()
970 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)  in gmc_v9_0_init_golden_registers()  argument
973 	switch (adev->asic_type) {  in gmc_v9_0_init_golden_registers()
976 		soc15_program_register_sequence(adev,  in gmc_v9_0_init_golden_registers()
979 		soc15_program_register_sequence(adev,  in gmc_v9_0_init_golden_registers()
986 		soc15_program_register_sequence(adev,  in gmc_v9_0_init_golden_registers()
1000 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)  in gmc_v9_0_gart_enable()  argument
1006 	amdgpu_device_program_register_sequence(adev,  in gmc_v9_0_gart_enable()
1010 	if (adev->gart.robj == NULL) {  in gmc_v9_0_gart_enable()
1011 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");  in gmc_v9_0_gart_enable()
1014 	r = amdgpu_gart_table_vram_pin(adev);  in gmc_v9_0_gart_enable()
1018 	switch (adev->asic_type) {  in gmc_v9_0_gart_enable()
1020 		mmhub_v1_0_initialize_power_gating(adev);  in gmc_v9_0_gart_enable()
1021 		mmhub_v1_0_update_power_gating(adev, true);  in gmc_v9_0_gart_enable()
1027 	r = gfxhub_v1_0_gart_enable(adev);  in gmc_v9_0_gart_enable()
1031 	r = mmhub_v1_0_gart_enable(adev);  in gmc_v9_0_gart_enable()
1041 	adev->nbio_funcs->hdp_flush(adev, NULL);  in gmc_v9_0_gart_enable()
1048 	gfxhub_v1_0_set_fault_enable_default(adev, value);  in gmc_v9_0_gart_enable()
1049 	mmhub_v1_0_set_fault_enable_default(adev, value);  in gmc_v9_0_gart_enable()
1050 	gmc_v9_0_flush_gpu_tlb(adev, 0);  in gmc_v9_0_gart_enable()
1053 		 (unsigned)(adev->gmc.gart_size >> 20),  in gmc_v9_0_gart_enable()
1054 		 (unsigned long long)adev->gart.table_addr);  in gmc_v9_0_gart_enable()
1055 	adev->gart.ready = true;  in gmc_v9_0_gart_enable()
1062 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_hw_init()  local
1065 	gmc_v9_0_init_golden_registers(adev);  in gmc_v9_0_hw_init()
1067 	if (adev->mode_info.num_crtc) {  in gmc_v9_0_hw_init()
1075 	r = gmc_v9_0_gart_enable(adev);  in gmc_v9_0_hw_init()
1087 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)  in gmc_v9_0_gart_disable()  argument
1089 	gfxhub_v1_0_gart_disable(adev);  in gmc_v9_0_gart_disable()
1090 	mmhub_v1_0_gart_disable(adev);  in gmc_v9_0_gart_disable()
1091 	amdgpu_gart_table_vram_unpin(adev);  in gmc_v9_0_gart_disable()
1096 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_hw_fini()  local
1098 	if (amdgpu_sriov_vf(adev)) {  in gmc_v9_0_hw_fini()
1104 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);  in gmc_v9_0_hw_fini()
1105 	gmc_v9_0_gart_disable(adev);  in gmc_v9_0_hw_fini()
1112 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_suspend()  local
1114 	return gmc_v9_0_hw_fini(adev);  in gmc_v9_0_suspend()
1120 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_resume()  local
1122 	r = gmc_v9_0_hw_init(adev);  in gmc_v9_0_resume()
1126 	amdgpu_vmid_reset_all(adev);  in gmc_v9_0_resume()
1152 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_set_clockgating_state()  local
1154 	return mmhub_v1_0_set_clockgating(adev, state);  in gmc_v9_0_set_clockgating_state()
1159 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in gmc_v9_0_get_clockgating_state()  local
1161 	mmhub_v1_0_get_clockgating(adev, flags);  in gmc_v9_0_get_clockgating_state()