44 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev)  in amdgpu_gmc_pdb0_alloc()  argument
48 	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;  in amdgpu_gmc_pdb0_alloc()
49 	uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21;  in amdgpu_gmc_pdb0_alloc()
62 	r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo);  in amdgpu_gmc_pdb0_alloc()
66 	r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false);  in amdgpu_gmc_pdb0_alloc()
70 	r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM);  in amdgpu_gmc_pdb0_alloc()
73 	r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0);  in amdgpu_gmc_pdb0_alloc()
77 	amdgpu_bo_unreserve(adev->gmc.pdb0_bo);  in amdgpu_gmc_pdb0_alloc()
81 	amdgpu_bo_unpin(adev->gmc.pdb0_bo);  in amdgpu_gmc_pdb0_alloc()
83 	amdgpu_bo_unreserve(adev->gmc.pdb0_bo);  in amdgpu_gmc_pdb0_alloc()
85 	amdgpu_bo_unref(&adev->gmc.pdb0_bo);  in amdgpu_gmc_pdb0_alloc()
102 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_gmc_get_pde_for_bo()  local
116 	amdgpu_gmc_get_vm_pde(adev, level, addr, flags);  in amdgpu_gmc_get_pde_for_bo()
124 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_gmc_pd_addr()  local
128 	if (adev->asic_type >= CHIP_VEGA10) {  in amdgpu_gmc_pd_addr()
150 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,  in amdgpu_gmc_set_pte_pde()  argument
158 	if (!drm_dev_enter(&adev->ddev, &idx))  in amdgpu_gmc_set_pte_pde()
183 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_gmc_agp_addr()  local
188 	if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size)  in amdgpu_gmc_agp_addr()
191 	return adev->gmc.agp_start + bo->ttm->dma_address[0];  in amdgpu_gmc_agp_addr()
204 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,  in amdgpu_gmc_vram_location()  argument
218 	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",  in amdgpu_gmc_vram_location()
239 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)  in amdgpu_gmc_sysvm_location()  argument
249 	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",  in amdgpu_gmc_sysvm_location()
252 	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",  in amdgpu_gmc_sysvm_location()
266 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)  in amdgpu_gmc_gart_location()  argument
271 	u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1);  in amdgpu_gmc_gart_location()
280 		dev_warn(adev->dev, "limiting GART\n");  in amdgpu_gmc_gart_location()
292 	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",  in amdgpu_gmc_gart_location()
307 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)  in amdgpu_gmc_agp_location()  argument
313 	if (amdgpu_sriov_vf(adev)) {  in amdgpu_gmc_agp_location()
340 	dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n",  in amdgpu_gmc_agp_location()
367 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,  in amdgpu_gmc_filter_faults()  argument
370 	struct amdgpu_gmc *gmc = &adev->gmc;  in amdgpu_gmc_filter_faults()
419 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,  in amdgpu_gmc_filter_faults_remove()  argument
422 	struct amdgpu_gmc *gmc = &adev->gmc;  in amdgpu_gmc_filter_faults_remove()
439 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)  in amdgpu_gmc_ras_late_init()  argument
443 	if (adev->umc.ras_funcs &&  in amdgpu_gmc_ras_late_init()
444 	    adev->umc.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
445 		r = adev->umc.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
450 	if (adev->mmhub.ras_funcs &&  in amdgpu_gmc_ras_late_init()
451 	    adev->mmhub.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
452 		r = adev->mmhub.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
457 	if (!adev->gmc.xgmi.connected_to_cpu)  in amdgpu_gmc_ras_late_init()
458 		adev->gmc.xgmi.ras_funcs = &xgmi_ras_funcs;  in amdgpu_gmc_ras_late_init()
460 	if (adev->gmc.xgmi.ras_funcs &&  in amdgpu_gmc_ras_late_init()
461 	    adev->gmc.xgmi.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
462 		r = adev->gmc.xgmi.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
467 	if (adev->hdp.ras_funcs &&  in amdgpu_gmc_ras_late_init()
468 	    adev->hdp.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
469 		r = adev->hdp.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
474 	if (adev->mca.mp0.ras_funcs &&  in amdgpu_gmc_ras_late_init()
475 	    adev->mca.mp0.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
476 		r = adev->mca.mp0.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
481 	if (adev->mca.mp1.ras_funcs &&  in amdgpu_gmc_ras_late_init()
482 	    adev->mca.mp1.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
483 		r = adev->mca.mp1.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
488 	if (adev->mca.mpio.ras_funcs &&  in amdgpu_gmc_ras_late_init()
489 	    adev->mca.mpio.ras_funcs->ras_late_init) {  in amdgpu_gmc_ras_late_init()
490 		r = adev->mca.mpio.ras_funcs->ras_late_init(adev);  in amdgpu_gmc_ras_late_init()
498 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev)  in amdgpu_gmc_ras_fini()  argument
500 	if (adev->umc.ras_funcs &&  in amdgpu_gmc_ras_fini()
501 	    adev->umc.ras_funcs->ras_fini)  in amdgpu_gmc_ras_fini()
502 		adev->umc.ras_funcs->ras_fini(adev);  in amdgpu_gmc_ras_fini()
504 	if (adev->mmhub.ras_funcs &&  in amdgpu_gmc_ras_fini()
505 	    adev->mmhub.ras_funcs->ras_fini)  in amdgpu_gmc_ras_fini()
506 		adev->mmhub.ras_funcs->ras_fini(adev);  in amdgpu_gmc_ras_fini()
508 	if (adev->gmc.xgmi.ras_funcs &&  in amdgpu_gmc_ras_fini()
509 	    adev->gmc.xgmi.ras_funcs->ras_fini)  in amdgpu_gmc_ras_fini()
510 		adev->gmc.xgmi.ras_funcs->ras_fini(adev);  in amdgpu_gmc_ras_fini()
512 	if (adev->hdp.ras_funcs &&  in amdgpu_gmc_ras_fini()
513 	    adev->hdp.ras_funcs->ras_fini)  in amdgpu_gmc_ras_fini()
514 		adev->hdp.ras_funcs->ras_fini(adev);  in amdgpu_gmc_ras_fini()
527 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)  in amdgpu_gmc_allocate_vm_inv_eng()  argument
536 	for (i = 0; i < adev->num_rings; ++i) {  in amdgpu_gmc_allocate_vm_inv_eng()
537 		ring = adev->rings[i];  in amdgpu_gmc_allocate_vm_inv_eng()
540 		if (ring == &adev->mes.ring)  in amdgpu_gmc_allocate_vm_inv_eng()
545 			dev_err(adev->dev, "no VM inv eng for ring %s\n",  in amdgpu_gmc_allocate_vm_inv_eng()
553 		dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n",  in amdgpu_gmc_allocate_vm_inv_eng()
567 void amdgpu_gmc_tmz_set(struct amdgpu_device *adev)  in amdgpu_gmc_tmz_set()  argument
569 	switch (adev->asic_type) {  in amdgpu_gmc_tmz_set()
573 			adev->gmc.tmz_enabled = false;  in amdgpu_gmc_tmz_set()
574 			dev_info(adev->dev,  in amdgpu_gmc_tmz_set()
577 			adev->gmc.tmz_enabled = true;  in amdgpu_gmc_tmz_set()
578 			dev_info(adev->dev,  in amdgpu_gmc_tmz_set()
590 			adev->gmc.tmz_enabled = false;  in amdgpu_gmc_tmz_set()
591 			dev_info(adev->dev,  in amdgpu_gmc_tmz_set()
594 			adev->gmc.tmz_enabled = true;  in amdgpu_gmc_tmz_set()
595 			dev_info(adev->dev,  in amdgpu_gmc_tmz_set()
600 		adev->gmc.tmz_enabled = false;  in amdgpu_gmc_tmz_set()
601 		dev_info(adev->dev,  in amdgpu_gmc_tmz_set()
614 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)  in amdgpu_gmc_noretry_set()  argument
616 	struct amdgpu_gmc *gmc = &adev->gmc;  in amdgpu_gmc_noretry_set()
618 	switch (adev->asic_type) {  in amdgpu_gmc_noretry_set()
653 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,  in amdgpu_gmc_set_vm_fault_masks()  argument
659 	hub = &adev->vmhub[hub_type];  in amdgpu_gmc_set_vm_fault_masks()
678 void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev)  in amdgpu_gmc_get_vbios_allocations()  argument
690 	switch (adev->asic_type) {  in amdgpu_gmc_get_vbios_allocations()
694 		adev->mman.keep_stolen_vga_memory = true;  in amdgpu_gmc_get_vbios_allocations()
697 		adev->mman.keep_stolen_vga_memory = false;  in amdgpu_gmc_get_vbios_allocations()
701 	if (amdgpu_sriov_vf(adev) ||  in amdgpu_gmc_get_vbios_allocations()
702 	    !amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_DCE)) {  in amdgpu_gmc_get_vbios_allocations()
705 		size = amdgpu_gmc_get_vbios_fb_size(adev);  in amdgpu_gmc_get_vbios_allocations()
707 		if (adev->mman.keep_stolen_vga_memory)  in amdgpu_gmc_get_vbios_allocations()
712 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))  in amdgpu_gmc_get_vbios_allocations()
716 		adev->mman.stolen_vga_size = AMDGPU_VBIOS_VGA_ALLOCATION;  in amdgpu_gmc_get_vbios_allocations()
717 		adev->mman.stolen_extended_size = size - adev->mman.stolen_vga_size;  in amdgpu_gmc_get_vbios_allocations()
719 		adev->mman.stolen_vga_size = size;  in amdgpu_gmc_get_vbios_allocations()
720 		adev->mman.stolen_extended_size = 0;  in amdgpu_gmc_get_vbios_allocations()
740 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)  in amdgpu_gmc_init_pdb0()  argument
743 	uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW?  in amdgpu_gmc_init_pdb0()
746 	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;  in amdgpu_gmc_init_pdb0()
747 	u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21;  in amdgpu_gmc_init_pdb0()
748 	u64 vram_addr = adev->vm_manager.vram_base_offset -  in amdgpu_gmc_init_pdb0()
749 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;  in amdgpu_gmc_init_pdb0()
751 	u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo);  in amdgpu_gmc_init_pdb0()
756 	flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1));  in amdgpu_gmc_init_pdb0()
763 		amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags);  in amdgpu_gmc_init_pdb0()
772 	amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags);  in amdgpu_gmc_init_pdb0()
782 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr)  in amdgpu_gmc_vram_mc2pa()  argument
784 	return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset;  in amdgpu_gmc_vram_mc2pa()
794 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo)  in amdgpu_gmc_vram_pa()  argument
796 	return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo));  in amdgpu_gmc_vram_pa()
806 uint64_t amdgpu_gmc_vram_cpu_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo)  in amdgpu_gmc_vram_cpu_pa()  argument
808 	return amdgpu_bo_gpu_offset(bo) - adev->gmc.vram_start + adev->gmc.aper_base;  in amdgpu_gmc_vram_cpu_pa()
811 void amdgpu_gmc_get_reserved_allocation(struct amdgpu_device *adev)  in amdgpu_gmc_get_reserved_allocation()  argument
815 	adev->mman.stolen_reserved_offset = 0;  in amdgpu_gmc_get_reserved_allocation()
816 	adev->mman.stolen_reserved_size = 0;  in amdgpu_gmc_get_reserved_allocation()
818 	switch (adev->asic_type) {  in amdgpu_gmc_get_reserved_allocation()
821 			adev->mman.stolen_reserved_offset = 0x1ffb0000;  in amdgpu_gmc_get_reserved_allocation()
822 			adev->mman.stolen_reserved_size = 64 * PAGE_SIZE;  in amdgpu_gmc_get_reserved_allocation()