Lines Matching refs:adev
95 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
107 struct amdgpu_device *adev = dev->dev_private; in amdgpu_device_is_px() local
109 if (adev->flags & AMD_IS_PX) in amdgpu_device_is_px()
126 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, in amdgpu_mm_rreg() argument
131 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) in amdgpu_mm_rreg()
132 return amdgpu_virt_kiq_rreg(adev, reg); in amdgpu_mm_rreg()
134 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) in amdgpu_mm_rreg()
135 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_rreg()
139 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_mm_rreg()
140 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); in amdgpu_mm_rreg()
141 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); in amdgpu_mm_rreg()
142 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_mm_rreg()
144 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); in amdgpu_mm_rreg()
162 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) { in amdgpu_mm_rreg8() argument
163 if (offset < adev->rmmio_size) in amdgpu_mm_rreg8()
164 return (readb(adev->rmmio + offset)); in amdgpu_mm_rreg8()
183 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) { in amdgpu_mm_wreg8() argument
184 if (offset < adev->rmmio_size) in amdgpu_mm_wreg8()
185 writeb(value, adev->rmmio + offset); in amdgpu_mm_wreg8()
200 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, in amdgpu_mm_wreg() argument
203 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); in amdgpu_mm_wreg()
205 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { in amdgpu_mm_wreg()
206 adev->last_mm_index = v; in amdgpu_mm_wreg()
209 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) in amdgpu_mm_wreg()
210 return amdgpu_virt_kiq_wreg(adev, reg, v); in amdgpu_mm_wreg()
212 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) in amdgpu_mm_wreg()
213 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_wreg()
217 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_mm_wreg()
218 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); in amdgpu_mm_wreg()
219 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); in amdgpu_mm_wreg()
220 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_mm_wreg()
223 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { in amdgpu_mm_wreg()
236 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) in amdgpu_io_rreg() argument
238 if ((reg * 4) < adev->rio_mem_size) in amdgpu_io_rreg()
239 return ioread32(adev->rio_mem + (reg * 4)); in amdgpu_io_rreg()
241 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); in amdgpu_io_rreg()
242 return ioread32(adev->rio_mem + (mmMM_DATA * 4)); in amdgpu_io_rreg()
255 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) in amdgpu_io_wreg() argument
257 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { in amdgpu_io_wreg()
258 adev->last_mm_index = v; in amdgpu_io_wreg()
261 if ((reg * 4) < adev->rio_mem_size) in amdgpu_io_wreg()
262 iowrite32(v, adev->rio_mem + (reg * 4)); in amdgpu_io_wreg()
264 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); in amdgpu_io_wreg()
265 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4)); in amdgpu_io_wreg()
268 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { in amdgpu_io_wreg()
282 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) in amdgpu_mm_rdoorbell() argument
284 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_rdoorbell()
285 return readl(adev->doorbell.ptr + index); in amdgpu_mm_rdoorbell()
302 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) in amdgpu_mm_wdoorbell() argument
304 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_wdoorbell()
305 writel(v, adev->doorbell.ptr + index); in amdgpu_mm_wdoorbell()
320 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) in amdgpu_mm_rdoorbell64() argument
322 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_rdoorbell64()
323 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); in amdgpu_mm_rdoorbell64()
340 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) in amdgpu_mm_wdoorbell64() argument
342 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_wdoorbell64()
343 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); in amdgpu_mm_wdoorbell64()
359 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) in amdgpu_invalid_rreg() argument
376 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) in amdgpu_invalid_wreg() argument
394 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, in amdgpu_block_invalid_rreg() argument
414 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, in amdgpu_block_invalid_wreg() argument
431 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) in amdgpu_device_vram_scratch_init() argument
433 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, in amdgpu_device_vram_scratch_init()
435 &adev->vram_scratch.robj, in amdgpu_device_vram_scratch_init()
436 &adev->vram_scratch.gpu_addr, in amdgpu_device_vram_scratch_init()
437 (void **)&adev->vram_scratch.ptr); in amdgpu_device_vram_scratch_init()
447 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) in amdgpu_device_vram_scratch_fini() argument
449 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); in amdgpu_device_vram_scratch_fini()
462 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, in amdgpu_device_program_register_sequence() argument
496 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) in amdgpu_device_pci_config_reset() argument
498 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); in amdgpu_device_pci_config_reset()
512 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) in amdgpu_device_doorbell_init() argument
515 if (adev->asic_type < CHIP_BONAIRE) { in amdgpu_device_doorbell_init()
516 adev->doorbell.base = 0; in amdgpu_device_doorbell_init()
517 adev->doorbell.size = 0; in amdgpu_device_doorbell_init()
518 adev->doorbell.num_doorbells = 0; in amdgpu_device_doorbell_init()
519 adev->doorbell.ptr = NULL; in amdgpu_device_doorbell_init()
523 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) in amdgpu_device_doorbell_init()
527 adev->doorbell.base = pci_resource_start(adev->pdev, 2); in amdgpu_device_doorbell_init()
528 adev->doorbell.size = pci_resource_len(adev->pdev, 2); in amdgpu_device_doorbell_init()
530 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), in amdgpu_device_doorbell_init()
532 if (adev->doorbell.num_doorbells == 0) in amdgpu_device_doorbell_init()
535 adev->doorbell.ptr = ioremap(adev->doorbell.base, in amdgpu_device_doorbell_init()
536 adev->doorbell.num_doorbells * in amdgpu_device_doorbell_init()
538 if (adev->doorbell.ptr == NULL) in amdgpu_device_doorbell_init()
551 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) in amdgpu_device_doorbell_fini() argument
553 iounmap(adev->doorbell.ptr); in amdgpu_device_doorbell_fini()
554 adev->doorbell.ptr = NULL; in amdgpu_device_doorbell_fini()
573 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) in amdgpu_device_wb_fini() argument
575 if (adev->wb.wb_obj) { in amdgpu_device_wb_fini()
576 amdgpu_bo_free_kernel(&adev->wb.wb_obj, in amdgpu_device_wb_fini()
577 &adev->wb.gpu_addr, in amdgpu_device_wb_fini()
578 (void **)&adev->wb.wb); in amdgpu_device_wb_fini()
579 adev->wb.wb_obj = NULL; in amdgpu_device_wb_fini()
592 static int amdgpu_device_wb_init(struct amdgpu_device *adev) in amdgpu_device_wb_init() argument
596 if (adev->wb.wb_obj == NULL) { in amdgpu_device_wb_init()
598 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, in amdgpu_device_wb_init()
600 &adev->wb.wb_obj, &adev->wb.gpu_addr, in amdgpu_device_wb_init()
601 (void **)&adev->wb.wb); in amdgpu_device_wb_init()
603 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); in amdgpu_device_wb_init()
607 adev->wb.num_wb = AMDGPU_MAX_WB; in amdgpu_device_wb_init()
608 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); in amdgpu_device_wb_init()
611 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); in amdgpu_device_wb_init()
626 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) in amdgpu_device_wb_get() argument
628 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); in amdgpu_device_wb_get()
630 if (offset < adev->wb.num_wb) { in amdgpu_device_wb_get()
631 __set_bit(offset, adev->wb.used); in amdgpu_device_wb_get()
647 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) in amdgpu_device_wb_free() argument
650 if (wb < adev->wb.num_wb) in amdgpu_device_wb_free()
651 __clear_bit(wb, adev->wb.used); in amdgpu_device_wb_free()
664 void amdgpu_device_vram_location(struct amdgpu_device *adev, in amdgpu_device_vram_location() argument
673 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", in amdgpu_device_vram_location()
689 void amdgpu_device_gart_location(struct amdgpu_device *adev, in amdgpu_device_gart_location() argument
694 mc->gart_size += adev->pm.smu_prv_buffer_size; in amdgpu_device_gart_location()
696 size_af = adev->gmc.mc_mask - mc->vram_end; in amdgpu_device_gart_location()
700 dev_warn(adev->dev, "limiting GART\n"); in amdgpu_device_gart_location()
706 dev_warn(adev->dev, "limiting GART\n"); in amdgpu_device_gart_location()
715 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", in amdgpu_device_gart_location()
728 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) in amdgpu_device_resize_fb_bar() argument
730 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); in amdgpu_device_resize_fb_bar()
739 if (amdgpu_sriov_vf(adev)) in amdgpu_device_resize_fb_bar()
743 root = adev->pdev->bus; in amdgpu_device_resize_fb_bar()
758 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); in amdgpu_device_resize_fb_bar()
759 pci_write_config_word(adev->pdev, PCI_COMMAND, in amdgpu_device_resize_fb_bar()
763 amdgpu_device_doorbell_fini(adev); in amdgpu_device_resize_fb_bar()
764 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_resize_fb_bar()
765 pci_release_resource(adev->pdev, 2); in amdgpu_device_resize_fb_bar()
767 pci_release_resource(adev->pdev, 0); in amdgpu_device_resize_fb_bar()
769 r = pci_resize_resource(adev->pdev, 0, rbar_size); in amdgpu_device_resize_fb_bar()
775 pci_assign_unassigned_bus_resources(adev->pdev->bus); in amdgpu_device_resize_fb_bar()
780 r = amdgpu_device_doorbell_init(adev); in amdgpu_device_resize_fb_bar()
781 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) in amdgpu_device_resize_fb_bar()
784 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); in amdgpu_device_resize_fb_bar()
801 bool amdgpu_device_need_post(struct amdgpu_device *adev) in amdgpu_device_need_post() argument
805 if (amdgpu_sriov_vf(adev)) in amdgpu_device_need_post()
808 if (amdgpu_passthrough(adev)) { in amdgpu_device_need_post()
814 if (adev->asic_type == CHIP_FIJI) { in amdgpu_device_need_post()
817 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); in amdgpu_device_need_post()
822 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); in amdgpu_device_need_post()
828 if (adev->has_hw_reset) { in amdgpu_device_need_post()
829 adev->has_hw_reset = false; in amdgpu_device_need_post()
834 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_need_post()
835 return amdgpu_atombios_scratch_need_asic_init(adev); in amdgpu_device_need_post()
838 reg = amdgpu_asic_get_config_memsize(adev); in amdgpu_device_need_post()
858 struct amdgpu_device *adev = cookie; in amdgpu_device_vga_set_decode() local
859 amdgpu_asic_set_vga_state(adev, state); in amdgpu_device_vga_set_decode()
877 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) in amdgpu_device_check_block_size() argument
886 dev_warn(adev->dev, "VM page table size (%d) too small\n", in amdgpu_device_check_block_size()
900 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) in amdgpu_device_check_vm_size() argument
907 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", in amdgpu_device_check_vm_size()
913 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) in amdgpu_device_check_smu_prv_buffer_size() argument
943 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; in amdgpu_device_check_smu_prv_buffer_size()
950 adev->pm.smu_prv_buffer_size = 0; in amdgpu_device_check_smu_prv_buffer_size()
961 static void amdgpu_device_check_arguments(struct amdgpu_device *adev) in amdgpu_device_check_arguments() argument
964 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", in amdgpu_device_check_arguments()
968 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
975 dev_warn(adev->dev, "gart size (%d) too small\n", in amdgpu_device_check_arguments()
982 dev_warn(adev->dev, "gtt size (%d) too small\n", in amdgpu_device_check_arguments()
990 dev_warn(adev->dev, "valid range is between 4 and 9\n"); in amdgpu_device_check_arguments()
994 amdgpu_device_check_smu_prv_buffer_size(adev); in amdgpu_device_check_arguments()
996 amdgpu_device_check_vm_size(adev); in amdgpu_device_check_arguments()
998 amdgpu_device_check_block_size(adev); in amdgpu_device_check_arguments()
1002 dev_warn(adev->dev, "invalid VRAM page split (%d)\n", in amdgpu_device_check_arguments()
1008 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n"); in amdgpu_device_check_arguments()
1012 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); in amdgpu_device_check_arguments()
1091 struct amdgpu_device *adev = dev; in amdgpu_device_ip_set_clockgating_state() local
1094 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_clockgating_state()
1095 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_clockgating_state()
1097 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_clockgating_state()
1099 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) in amdgpu_device_ip_set_clockgating_state()
1101 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( in amdgpu_device_ip_set_clockgating_state()
1102 (void *)adev, state); in amdgpu_device_ip_set_clockgating_state()
1105 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_clockgating_state()
1125 struct amdgpu_device *adev = dev; in amdgpu_device_ip_set_powergating_state() local
1128 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_powergating_state()
1129 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_powergating_state()
1131 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_powergating_state()
1133 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) in amdgpu_device_ip_set_powergating_state()
1135 r = adev->ip_blocks[i].version->funcs->set_powergating_state( in amdgpu_device_ip_set_powergating_state()
1136 (void *)adev, state); in amdgpu_device_ip_set_powergating_state()
1139 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_powergating_state()
1155 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, in amdgpu_device_ip_get_clockgating_state() argument
1160 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_get_clockgating_state()
1161 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_get_clockgating_state()
1163 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) in amdgpu_device_ip_get_clockgating_state()
1164 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); in amdgpu_device_ip_get_clockgating_state()
1177 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, in amdgpu_device_ip_wait_for_idle() argument
1182 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_wait_for_idle()
1183 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_wait_for_idle()
1185 if (adev->ip_blocks[i].version->type == block_type) { in amdgpu_device_ip_wait_for_idle()
1186 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); in amdgpu_device_ip_wait_for_idle()
1205 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, in amdgpu_device_ip_is_idle() argument
1210 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_is_idle()
1211 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_is_idle()
1213 if (adev->ip_blocks[i].version->type == block_type) in amdgpu_device_ip_is_idle()
1214 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); in amdgpu_device_ip_is_idle()
1230 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, in amdgpu_device_ip_get_ip_block() argument
1235 for (i = 0; i < adev->num_ip_blocks; i++) in amdgpu_device_ip_get_ip_block()
1236 if (adev->ip_blocks[i].version->type == type) in amdgpu_device_ip_get_ip_block()
1237 return &adev->ip_blocks[i]; in amdgpu_device_ip_get_ip_block()
1253 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, in amdgpu_device_ip_block_version_cmp() argument
1257 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); in amdgpu_device_ip_block_version_cmp()
1276 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, in amdgpu_device_ip_block_add() argument
1282 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, in amdgpu_device_ip_block_add()
1285 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; in amdgpu_device_ip_block_add()
1302 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) in amdgpu_device_enable_virtual_display() argument
1304 adev->enable_virtual_display = false; in amdgpu_device_enable_virtual_display()
1307 struct drm_device *ddev = adev->ddev; in amdgpu_device_enable_virtual_display()
1320 adev->enable_virtual_display = true; in amdgpu_device_enable_virtual_display()
1331 adev->mode_info.num_crtc = num_crtc; in amdgpu_device_enable_virtual_display()
1333 adev->mode_info.num_crtc = 1; in amdgpu_device_enable_virtual_display()
1341 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_enable_virtual_display()
1357 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) in amdgpu_device_parse_gpu_info_fw() argument
1364 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_parse_gpu_info_fw()
1366 switch (adev->asic_type) { in amdgpu_device_parse_gpu_info_fw()
1405 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); in amdgpu_device_parse_gpu_info_fw()
1407 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1412 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); in amdgpu_device_parse_gpu_info_fw()
1414 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1420 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; in amdgpu_device_parse_gpu_info_fw()
1427 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1430 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); in amdgpu_device_parse_gpu_info_fw()
1431 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); in amdgpu_device_parse_gpu_info_fw()
1432 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); in amdgpu_device_parse_gpu_info_fw()
1433 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); in amdgpu_device_parse_gpu_info_fw()
1434 adev->gfx.config.max_texture_channel_caches = in amdgpu_device_parse_gpu_info_fw()
1436 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); in amdgpu_device_parse_gpu_info_fw()
1437 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); in amdgpu_device_parse_gpu_info_fw()
1438 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); in amdgpu_device_parse_gpu_info_fw()
1439 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); in amdgpu_device_parse_gpu_info_fw()
1440 adev->gfx.config.double_offchip_lds_buf = in amdgpu_device_parse_gpu_info_fw()
1442 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); in amdgpu_device_parse_gpu_info_fw()
1443 adev->gfx.cu_info.max_waves_per_simd = in amdgpu_device_parse_gpu_info_fw()
1445 adev->gfx.cu_info.max_scratch_slots_per_cu = in amdgpu_device_parse_gpu_info_fw()
1447 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); in amdgpu_device_parse_gpu_info_fw()
1451 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1470 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) in amdgpu_device_ip_early_init() argument
1474 amdgpu_device_enable_virtual_display(adev); in amdgpu_device_ip_early_init()
1476 switch (adev->asic_type) { in amdgpu_device_ip_early_init()
1486 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) in amdgpu_device_ip_early_init()
1487 adev->family = AMDGPU_FAMILY_CZ; in amdgpu_device_ip_early_init()
1489 adev->family = AMDGPU_FAMILY_VI; in amdgpu_device_ip_early_init()
1491 r = vi_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1501 adev->family = AMDGPU_FAMILY_SI; in amdgpu_device_ip_early_init()
1502 r = si_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1513 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII)) in amdgpu_device_ip_early_init()
1514 adev->family = AMDGPU_FAMILY_CI; in amdgpu_device_ip_early_init()
1516 adev->family = AMDGPU_FAMILY_KV; in amdgpu_device_ip_early_init()
1518 r = cik_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1527 if (adev->asic_type == CHIP_RAVEN) in amdgpu_device_ip_early_init()
1528 adev->family = AMDGPU_FAMILY_RV; in amdgpu_device_ip_early_init()
1530 adev->family = AMDGPU_FAMILY_AI; in amdgpu_device_ip_early_init()
1532 r = soc15_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1541 r = amdgpu_device_parse_gpu_info_fw(adev); in amdgpu_device_ip_early_init()
1545 amdgpu_amdkfd_device_probe(adev); in amdgpu_device_ip_early_init()
1547 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_ip_early_init()
1548 r = amdgpu_virt_request_full_gpu(adev, true); in amdgpu_device_ip_early_init()
1553 adev->powerplay.pp_feature = amdgpu_pp_feature_mask; in amdgpu_device_ip_early_init()
1555 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_early_init()
1558 i, adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_early_init()
1559 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
1561 if (adev->ip_blocks[i].version->funcs->early_init) { in amdgpu_device_ip_early_init()
1562 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); in amdgpu_device_ip_early_init()
1564 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
1567 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_early_init()
1570 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
1573 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
1578 adev->cg_flags &= amdgpu_cg_mask; in amdgpu_device_ip_early_init()
1579 adev->pg_flags &= amdgpu_pg_mask; in amdgpu_device_ip_early_init()
1595 static int amdgpu_device_ip_init(struct amdgpu_device *adev) in amdgpu_device_ip_init() argument
1599 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
1600 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_init()
1602 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); in amdgpu_device_ip_init()
1605 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_init()
1608 adev->ip_blocks[i].status.sw = true; in amdgpu_device_ip_init()
1611 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_init()
1612 r = amdgpu_device_vram_scratch_init(adev); in amdgpu_device_ip_init()
1617 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
1622 r = amdgpu_device_wb_init(adev); in amdgpu_device_ip_init()
1627 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
1630 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_ip_init()
1631 r = amdgpu_allocate_static_csa(adev); in amdgpu_device_ip_init()
1640 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
1641 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_init()
1643 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_init()
1645 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
1648 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_init()
1651 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
1654 amdgpu_amdkfd_device_init(adev); in amdgpu_device_ip_init()
1656 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_init()
1657 amdgpu_virt_release_full_gpu(adev, true); in amdgpu_device_ip_init()
1671 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) in amdgpu_device_fill_reset_magic() argument
1673 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); in amdgpu_device_fill_reset_magic()
1686 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) in amdgpu_device_check_vram_lost() argument
1688 return !!memcmp(adev->gart.ptr, adev->reset_magic, in amdgpu_device_check_vram_lost()
1703 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev) in amdgpu_device_ip_late_set_cg_state() argument
1710 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_set_cg_state()
1711 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_late_set_cg_state()
1714 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_ip_late_set_cg_state()
1715 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_ip_late_set_cg_state()
1716 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_ip_late_set_cg_state()
1717 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_ip_late_set_cg_state()
1719 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_ip_late_set_cg_state()
1723 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_set_cg_state()
1732 static int amdgpu_device_ip_late_set_pg_state(struct amdgpu_device *adev) in amdgpu_device_ip_late_set_pg_state() argument
1739 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_set_pg_state()
1740 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_late_set_pg_state()
1743 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_ip_late_set_pg_state()
1744 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_ip_late_set_pg_state()
1745 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_ip_late_set_pg_state()
1746 adev->ip_blocks[i].version->funcs->set_powergating_state) { in amdgpu_device_ip_late_set_pg_state()
1748 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, in amdgpu_device_ip_late_set_pg_state()
1752 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_set_pg_state()
1772 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) in amdgpu_device_ip_late_init() argument
1776 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_init()
1777 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_late_init()
1779 if (adev->ip_blocks[i].version->funcs->late_init) { in amdgpu_device_ip_late_init()
1780 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); in amdgpu_device_ip_late_init()
1783 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_init()
1786 adev->ip_blocks[i].status.late_initialized = true; in amdgpu_device_ip_late_init()
1790 amdgpu_device_ip_late_set_cg_state(adev); in amdgpu_device_ip_late_init()
1791 amdgpu_device_ip_late_set_pg_state(adev); in amdgpu_device_ip_late_init()
1793 queue_delayed_work(system_wq, &adev->late_init_work, in amdgpu_device_ip_late_init()
1796 amdgpu_device_fill_reset_magic(adev); in amdgpu_device_ip_late_init()
1812 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) in amdgpu_device_ip_fini() argument
1816 amdgpu_amdkfd_device_fini(adev); in amdgpu_device_ip_fini()
1818 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_fini()
1819 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini()
1821 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC && in amdgpu_device_ip_fini()
1822 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_ip_fini()
1824 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_ip_fini()
1828 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
1831 if (adev->powerplay.pp_funcs->set_powergating_by_smu) in amdgpu_device_ip_fini()
1832 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); in amdgpu_device_ip_fini()
1833 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini()
1837 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
1839 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini()
1844 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
1845 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini()
1848 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_ip_fini()
1849 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_ip_fini()
1850 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_ip_fini()
1851 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_ip_fini()
1853 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_ip_fini()
1857 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
1862 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini()
1866 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
1869 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini()
1873 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
1874 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_fini()
1877 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_fini()
1878 amdgpu_free_static_csa(adev); in amdgpu_device_ip_fini()
1879 amdgpu_device_wb_fini(adev); in amdgpu_device_ip_fini()
1880 amdgpu_device_vram_scratch_fini(adev); in amdgpu_device_ip_fini()
1883 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); in amdgpu_device_ip_fini()
1887 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
1889 adev->ip_blocks[i].status.sw = false; in amdgpu_device_ip_fini()
1890 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_fini()
1893 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
1894 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_ip_fini()
1896 if (adev->ip_blocks[i].version->funcs->late_fini) in amdgpu_device_ip_fini()
1897 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); in amdgpu_device_ip_fini()
1898 adev->ip_blocks[i].status.late_initialized = false; in amdgpu_device_ip_fini()
1901 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_fini()
1902 if (amdgpu_virt_release_full_gpu(adev, false)) in amdgpu_device_ip_fini()
1919 struct amdgpu_device *adev = in amdgpu_device_ip_late_init_func_handler() local
1923 r = amdgpu_ib_ring_tests(adev); in amdgpu_device_ip_late_init_func_handler()
1939 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) in amdgpu_device_ip_suspend_phase1() argument
1943 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend_phase1()
1944 amdgpu_virt_request_full_gpu(adev, false); in amdgpu_device_ip_suspend_phase1()
1946 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase1()
1947 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase1()
1950 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { in amdgpu_device_ip_suspend_phase1()
1952 if (adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_ip_suspend_phase1()
1953 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_ip_suspend_phase1()
1957 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase1()
1961 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase1()
1965 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase1()
1970 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend_phase1()
1971 amdgpu_virt_release_full_gpu(adev, false); in amdgpu_device_ip_suspend_phase1()
1987 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) in amdgpu_device_ip_suspend_phase2() argument
1991 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend_phase2()
1992 amdgpu_virt_request_full_gpu(adev, false); in amdgpu_device_ip_suspend_phase2()
1995 r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC, in amdgpu_device_ip_suspend_phase2()
2002 if (adev->powerplay.pp_funcs->set_powergating_by_smu) in amdgpu_device_ip_suspend_phase2()
2003 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); in amdgpu_device_ip_suspend_phase2()
2005 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase2()
2006 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase2()
2009 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase2()
2012 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC && in amdgpu_device_ip_suspend_phase2()
2013 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_ip_suspend_phase2()
2014 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_ip_suspend_phase2()
2018 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase2()
2022 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase2()
2026 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase2()
2030 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend_phase2()
2031 amdgpu_virt_release_full_gpu(adev, false); in amdgpu_device_ip_suspend_phase2()
2047 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) in amdgpu_device_ip_suspend() argument
2051 r = amdgpu_device_ip_suspend_phase1(adev); in amdgpu_device_ip_suspend()
2054 r = amdgpu_device_ip_suspend_phase2(adev); in amdgpu_device_ip_suspend()
2059 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) in amdgpu_device_ip_reinit_early_sriov() argument
2074 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_early_sriov()
2075 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_early_sriov()
2081 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_early_sriov()
2091 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) in amdgpu_device_ip_reinit_late_sriov() argument
2108 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_late_sriov()
2109 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_late_sriov()
2115 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_late_sriov()
2137 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) in amdgpu_device_ip_resume_phase1() argument
2141 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase1()
2142 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_resume_phase1()
2144 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase1()
2145 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase1()
2146 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_resume_phase1()
2147 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase1()
2150 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase1()
2172 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) in amdgpu_device_ip_resume_phase2() argument
2176 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase2()
2177 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_resume_phase2()
2179 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase2()
2180 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase2()
2181 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) in amdgpu_device_ip_resume_phase2()
2183 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase2()
2186 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase2()
2206 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) in amdgpu_device_ip_resume() argument
2210 r = amdgpu_device_ip_resume_phase1(adev); in amdgpu_device_ip_resume()
2213 r = amdgpu_device_ip_resume_phase2(adev); in amdgpu_device_ip_resume()
2225 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) in amdgpu_device_detect_sriov_bios() argument
2227 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_detect_sriov_bios()
2228 if (adev->is_atom_fw) { in amdgpu_device_detect_sriov_bios()
2229 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev)) in amdgpu_device_detect_sriov_bios()
2230 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
2232 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) in amdgpu_device_detect_sriov_bios()
2233 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
2236 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) in amdgpu_device_detect_sriov_bios()
2237 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); in amdgpu_device_detect_sriov_bios()
2294 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) in amdgpu_device_has_dc_support() argument
2296 if (amdgpu_sriov_vf(adev)) in amdgpu_device_has_dc_support()
2299 return amdgpu_device_asic_has_dc_support(adev->asic_type); in amdgpu_device_has_dc_support()
2314 int amdgpu_device_init(struct amdgpu_device *adev, in amdgpu_device_init() argument
2323 adev->shutdown = false; in amdgpu_device_init()
2324 adev->dev = &pdev->dev; in amdgpu_device_init()
2325 adev->ddev = ddev; in amdgpu_device_init()
2326 adev->pdev = pdev; in amdgpu_device_init()
2327 adev->flags = flags; in amdgpu_device_init()
2328 adev->asic_type = flags & AMD_ASIC_MASK; in amdgpu_device_init()
2329 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; in amdgpu_device_init()
2331 adev->usec_timeout *= 2; in amdgpu_device_init()
2332 adev->gmc.gart_size = 512 * 1024 * 1024; in amdgpu_device_init()
2333 adev->accel_working = false; in amdgpu_device_init()
2334 adev->num_rings = 0; in amdgpu_device_init()
2335 adev->mman.buffer_funcs = NULL; in amdgpu_device_init()
2336 adev->mman.buffer_funcs_ring = NULL; in amdgpu_device_init()
2337 adev->vm_manager.vm_pte_funcs = NULL; in amdgpu_device_init()
2338 adev->vm_manager.vm_pte_num_rings = 0; in amdgpu_device_init()
2339 adev->gmc.gmc_funcs = NULL; in amdgpu_device_init()
2340 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); in amdgpu_device_init()
2341 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in amdgpu_device_init()
2343 adev->smc_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2344 adev->smc_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2345 adev->pcie_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2346 adev->pcie_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2347 adev->pciep_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2348 adev->pciep_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2349 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2350 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2351 adev->didt_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2352 adev->didt_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2353 adev->gc_cac_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2354 adev->gc_cac_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2355 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; in amdgpu_device_init()
2356 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; in amdgpu_device_init()
2359 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, in amdgpu_device_init()
2364 atomic_set(&adev->irq.ih.lock, 0); in amdgpu_device_init()
2365 mutex_init(&adev->firmware.mutex); in amdgpu_device_init()
2366 mutex_init(&adev->pm.mutex); in amdgpu_device_init()
2367 mutex_init(&adev->gfx.gpu_clock_mutex); in amdgpu_device_init()
2368 mutex_init(&adev->srbm_mutex); in amdgpu_device_init()
2369 mutex_init(&adev->gfx.pipe_reserve_mutex); in amdgpu_device_init()
2370 mutex_init(&adev->grbm_idx_mutex); in amdgpu_device_init()
2371 mutex_init(&adev->mn_lock); in amdgpu_device_init()
2372 mutex_init(&adev->virt.vf_errors.lock); in amdgpu_device_init()
2373 hash_init(adev->mn_hash); in amdgpu_device_init()
2374 mutex_init(&adev->lock_reset); in amdgpu_device_init()
2376 amdgpu_device_check_arguments(adev); in amdgpu_device_init()
2378 spin_lock_init(&adev->mmio_idx_lock); in amdgpu_device_init()
2379 spin_lock_init(&adev->smc_idx_lock); in amdgpu_device_init()
2380 spin_lock_init(&adev->pcie_idx_lock); in amdgpu_device_init()
2381 spin_lock_init(&adev->uvd_ctx_idx_lock); in amdgpu_device_init()
2382 spin_lock_init(&adev->didt_idx_lock); in amdgpu_device_init()
2383 spin_lock_init(&adev->gc_cac_idx_lock); in amdgpu_device_init()
2384 spin_lock_init(&adev->se_cac_idx_lock); in amdgpu_device_init()
2385 spin_lock_init(&adev->audio_endpt_idx_lock); in amdgpu_device_init()
2386 spin_lock_init(&adev->mm_stats.lock); in amdgpu_device_init()
2388 INIT_LIST_HEAD(&adev->shadow_list); in amdgpu_device_init()
2389 mutex_init(&adev->shadow_list_lock); in amdgpu_device_init()
2391 INIT_LIST_HEAD(&adev->ring_lru_list); in amdgpu_device_init()
2392 spin_lock_init(&adev->ring_lru_list_lock); in amdgpu_device_init()
2394 INIT_DELAYED_WORK(&adev->late_init_work, in amdgpu_device_init()
2397 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false; in amdgpu_device_init()
2401 if (adev->asic_type >= CHIP_BONAIRE) { in amdgpu_device_init()
2402 adev->rmmio_base = pci_resource_start(adev->pdev, 5); in amdgpu_device_init()
2403 adev->rmmio_size = pci_resource_len(adev->pdev, 5); in amdgpu_device_init()
2405 adev->rmmio_base = pci_resource_start(adev->pdev, 2); in amdgpu_device_init()
2406 adev->rmmio_size = pci_resource_len(adev->pdev, 2); in amdgpu_device_init()
2409 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); in amdgpu_device_init()
2410 if (adev->rmmio == NULL) { in amdgpu_device_init()
2413 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); in amdgpu_device_init()
2414 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); in amdgpu_device_init()
2417 amdgpu_device_doorbell_init(adev); in amdgpu_device_init()
2421 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) { in amdgpu_device_init()
2422 adev->rio_mem_size = pci_resource_len(adev->pdev, i); in amdgpu_device_init()
2423 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size); in amdgpu_device_init()
2427 if (adev->rio_mem == NULL) in amdgpu_device_init()
2430 amdgpu_device_get_pcie_info(adev); in amdgpu_device_init()
2433 r = amdgpu_device_ip_early_init(adev); in amdgpu_device_init()
2440 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode); in amdgpu_device_init()
2444 if (!pci_is_thunderbolt_attached(adev->pdev)) in amdgpu_device_init()
2445 vga_switcheroo_register_client(adev->pdev, in amdgpu_device_init()
2448 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); in amdgpu_device_init()
2452 emu_soc_asic_init(adev); in amdgpu_device_init()
2457 if (!amdgpu_get_bios(adev)) { in amdgpu_device_init()
2462 r = amdgpu_atombios_init(adev); in amdgpu_device_init()
2464 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); in amdgpu_device_init()
2465 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); in amdgpu_device_init()
2470 amdgpu_device_detect_sriov_bios(adev); in amdgpu_device_init()
2473 if (amdgpu_device_need_post(adev)) { in amdgpu_device_init()
2474 if (!adev->bios) { in amdgpu_device_init()
2475 dev_err(adev->dev, "no vBIOS found\n"); in amdgpu_device_init()
2480 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_init()
2482 dev_err(adev->dev, "gpu post error!\n"); in amdgpu_device_init()
2487 if (adev->is_atom_fw) { in amdgpu_device_init()
2489 r = amdgpu_atomfirmware_get_clock_info(adev); in amdgpu_device_init()
2491 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); in amdgpu_device_init()
2492 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); in amdgpu_device_init()
2497 r = amdgpu_atombios_get_clock_info(adev); in amdgpu_device_init()
2499 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); in amdgpu_device_init()
2500 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); in amdgpu_device_init()
2504 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_init()
2505 amdgpu_atombios_i2c_init(adev); in amdgpu_device_init()
2510 r = amdgpu_fence_driver_init(adev); in amdgpu_device_init()
2512 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); in amdgpu_device_init()
2513 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); in amdgpu_device_init()
2518 drm_mode_config_init(adev->ddev); in amdgpu_device_init()
2520 r = amdgpu_device_ip_init(adev); in amdgpu_device_init()
2523 if (amdgpu_sriov_vf(adev) && in amdgpu_device_init()
2524 !amdgpu_sriov_runtime(adev) && in amdgpu_device_init()
2525 amdgpu_virt_mmio_blocked(adev) && in amdgpu_device_init()
2526 !amdgpu_virt_wait_reset(adev)) { in amdgpu_device_init()
2527 dev_err(adev->dev, "VF exclusive mode timeout\n"); in amdgpu_device_init()
2529 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; in amdgpu_device_init()
2530 adev->virt.ops = NULL; in amdgpu_device_init()
2534 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); in amdgpu_device_init()
2535 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); in amdgpu_device_init()
2539 adev->accel_working = true; in amdgpu_device_init()
2541 amdgpu_vm_check_compute_bug(adev); in amdgpu_device_init()
2549 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); in amdgpu_device_init()
2551 r = amdgpu_ib_pool_init(adev); in amdgpu_device_init()
2553 dev_err(adev->dev, "IB initialization failed (%d).\n", r); in amdgpu_device_init()
2554 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); in amdgpu_device_init()
2558 if (amdgpu_sriov_vf(adev)) in amdgpu_device_init()
2559 amdgpu_virt_init_data_exchange(adev); in amdgpu_device_init()
2561 amdgpu_fbdev_init(adev); in amdgpu_device_init()
2563 r = amdgpu_pm_sysfs_init(adev); in amdgpu_device_init()
2567 r = amdgpu_debugfs_gem_init(adev); in amdgpu_device_init()
2571 r = amdgpu_debugfs_regs_init(adev); in amdgpu_device_init()
2575 r = amdgpu_debugfs_firmware_init(adev); in amdgpu_device_init()
2579 r = amdgpu_debugfs_init(adev); in amdgpu_device_init()
2584 if (adev->accel_working) in amdgpu_device_init()
2585 amdgpu_test_moves(adev); in amdgpu_device_init()
2590 if (adev->accel_working) in amdgpu_device_init()
2591 amdgpu_benchmark(adev, amdgpu_benchmarking); in amdgpu_device_init()
2599 r = amdgpu_device_ip_late_init(adev); in amdgpu_device_init()
2601 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); in amdgpu_device_init()
2602 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); in amdgpu_device_init()
2609 amdgpu_vf_error_trans_all(adev); in amdgpu_device_init()
2611 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_init()
2624 void amdgpu_device_fini(struct amdgpu_device *adev) in amdgpu_device_fini() argument
2629 adev->shutdown = true; in amdgpu_device_fini()
2631 amdgpu_irq_disable_all(adev); in amdgpu_device_fini()
2632 if (adev->mode_info.mode_config_initialized){ in amdgpu_device_fini()
2633 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_fini()
2634 drm_crtc_force_disable_all(adev->ddev); in amdgpu_device_fini()
2636 drm_atomic_helper_shutdown(adev->ddev); in amdgpu_device_fini()
2638 amdgpu_ib_pool_fini(adev); in amdgpu_device_fini()
2639 amdgpu_fence_driver_fini(adev); in amdgpu_device_fini()
2640 amdgpu_pm_sysfs_fini(adev); in amdgpu_device_fini()
2641 amdgpu_fbdev_fini(adev); in amdgpu_device_fini()
2642 r = amdgpu_device_ip_fini(adev); in amdgpu_device_fini()
2643 if (adev->firmware.gpu_info_fw) { in amdgpu_device_fini()
2644 release_firmware(adev->firmware.gpu_info_fw); in amdgpu_device_fini()
2645 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_fini()
2647 adev->accel_working = false; in amdgpu_device_fini()
2648 cancel_delayed_work_sync(&adev->late_init_work); in amdgpu_device_fini()
2650 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_fini()
2651 amdgpu_i2c_fini(adev); in amdgpu_device_fini()
2654 amdgpu_atombios_fini(adev); in amdgpu_device_fini()
2656 kfree(adev->bios); in amdgpu_device_fini()
2657 adev->bios = NULL; in amdgpu_device_fini()
2658 if (!pci_is_thunderbolt_attached(adev->pdev)) in amdgpu_device_fini()
2659 vga_switcheroo_unregister_client(adev->pdev); in amdgpu_device_fini()
2660 if (adev->flags & AMD_IS_PX) in amdgpu_device_fini()
2661 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_fini()
2662 vga_client_register(adev->pdev, NULL, NULL, NULL); in amdgpu_device_fini()
2663 if (adev->rio_mem) in amdgpu_device_fini()
2664 pci_iounmap(adev->pdev, adev->rio_mem); in amdgpu_device_fini()
2665 adev->rio_mem = NULL; in amdgpu_device_fini()
2666 iounmap(adev->rmmio); in amdgpu_device_fini()
2667 adev->rmmio = NULL; in amdgpu_device_fini()
2668 amdgpu_device_doorbell_fini(adev); in amdgpu_device_fini()
2669 amdgpu_debugfs_regs_cleanup(adev); in amdgpu_device_fini()
2689 struct amdgpu_device *adev; in amdgpu_device_suspend() local
2698 adev = dev->dev_private; in amdgpu_device_suspend()
2706 amdgpu_fbdev_set_suspend(adev, 1); in amdgpu_device_suspend()
2708 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_suspend()
2735 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { in amdgpu_device_suspend()
2745 amdgpu_amdkfd_suspend(adev); in amdgpu_device_suspend()
2747 r = amdgpu_device_ip_suspend_phase1(adev); in amdgpu_device_suspend()
2750 amdgpu_bo_evict_vram(adev); in amdgpu_device_suspend()
2752 amdgpu_fence_driver_suspend(adev); in amdgpu_device_suspend()
2754 r = amdgpu_device_ip_suspend_phase2(adev); in amdgpu_device_suspend()
2760 amdgpu_bo_evict_vram(adev); in amdgpu_device_suspend()
2768 r = amdgpu_asic_reset(adev); in amdgpu_device_suspend()
2790 struct amdgpu_device *adev = dev->dev_private; in amdgpu_device_resume() local
2806 if (amdgpu_device_need_post(adev)) { in amdgpu_device_resume()
2807 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_resume()
2812 r = amdgpu_device_ip_resume(adev); in amdgpu_device_resume()
2817 amdgpu_fence_driver_resume(adev); in amdgpu_device_resume()
2820 r = amdgpu_device_ip_late_init(adev); in amdgpu_device_resume()
2824 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_resume()
2842 r = amdgpu_amdkfd_resume(adev); in amdgpu_device_resume()
2847 flush_delayed_work(&adev->late_init_work); in amdgpu_device_resume()
2851 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_resume()
2862 amdgpu_fbdev_set_suspend(adev, 0); in amdgpu_device_resume()
2879 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_resume()
2899 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_check_soft_reset() argument
2904 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_check_soft_reset()
2907 if (amdgpu_asic_need_full_reset(adev)) in amdgpu_device_ip_check_soft_reset()
2910 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_check_soft_reset()
2911 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_check_soft_reset()
2913 if (adev->ip_blocks[i].version->funcs->check_soft_reset) in amdgpu_device_ip_check_soft_reset()
2914 adev->ip_blocks[i].status.hang = in amdgpu_device_ip_check_soft_reset()
2915 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); in amdgpu_device_ip_check_soft_reset()
2916 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_check_soft_reset()
2917 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_check_soft_reset()
2935 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_pre_soft_reset() argument
2939 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_pre_soft_reset()
2940 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_pre_soft_reset()
2942 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_pre_soft_reset()
2943 adev->ip_blocks[i].version->funcs->pre_soft_reset) { in amdgpu_device_ip_pre_soft_reset()
2944 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); in amdgpu_device_ip_pre_soft_reset()
2962 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) in amdgpu_device_ip_need_full_reset() argument
2966 if (amdgpu_asic_need_full_reset(adev)) in amdgpu_device_ip_need_full_reset()
2969 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_need_full_reset()
2970 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_need_full_reset()
2972 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || in amdgpu_device_ip_need_full_reset()
2973 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || in amdgpu_device_ip_need_full_reset()
2974 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || in amdgpu_device_ip_need_full_reset()
2975 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || in amdgpu_device_ip_need_full_reset()
2976 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_need_full_reset()
2977 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_need_full_reset()
2997 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_soft_reset() argument
3001 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_soft_reset()
3002 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_soft_reset()
3004 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_soft_reset()
3005 adev->ip_blocks[i].version->funcs->soft_reset) { in amdgpu_device_ip_soft_reset()
3006 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); in amdgpu_device_ip_soft_reset()
3026 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_post_soft_reset() argument
3030 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_post_soft_reset()
3031 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_post_soft_reset()
3033 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_post_soft_reset()
3034 adev->ip_blocks[i].version->funcs->post_soft_reset) in amdgpu_device_ip_post_soft_reset()
3035 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); in amdgpu_device_ip_post_soft_reset()
3056 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev, in amdgpu_device_recover_vram_from_shadow() argument
3079 r = amdgpu_bo_restore_from_shadow(adev, ring, bo, in amdgpu_device_recover_vram_from_shadow()
3101 static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev) in amdgpu_device_handle_vram_lost() argument
3103 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_device_handle_vram_lost()
3110 if (amdgpu_sriov_runtime(adev)) in amdgpu_device_handle_vram_lost()
3116 mutex_lock(&adev->shadow_list_lock); in amdgpu_device_handle_vram_lost()
3117 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { in amdgpu_device_handle_vram_lost()
3119 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next); in amdgpu_device_handle_vram_lost()
3137 mutex_unlock(&adev->shadow_list_lock); in amdgpu_device_handle_vram_lost()
3165 static int amdgpu_device_reset(struct amdgpu_device *adev) in amdgpu_device_reset() argument
3170 need_full_reset = amdgpu_device_ip_need_full_reset(adev); in amdgpu_device_reset()
3173 amdgpu_device_ip_pre_soft_reset(adev); in amdgpu_device_reset()
3174 r = amdgpu_device_ip_soft_reset(adev); in amdgpu_device_reset()
3175 amdgpu_device_ip_post_soft_reset(adev); in amdgpu_device_reset()
3176 if (r || amdgpu_device_ip_check_soft_reset(adev)) { in amdgpu_device_reset()
3183 r = amdgpu_device_ip_suspend(adev); in amdgpu_device_reset()
3186 r = amdgpu_asic_reset(adev); in amdgpu_device_reset()
3188 amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_reset()
3191 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n"); in amdgpu_device_reset()
3192 r = amdgpu_device_ip_resume_phase1(adev); in amdgpu_device_reset()
3196 vram_lost = amdgpu_device_check_vram_lost(adev); in amdgpu_device_reset()
3199 atomic_inc(&adev->vram_lost_counter); in amdgpu_device_reset()
3203 &adev->mman.bdev.man[TTM_PL_TT]); in amdgpu_device_reset()
3207 r = amdgpu_device_ip_resume_phase2(adev); in amdgpu_device_reset()
3212 amdgpu_device_fill_reset_magic(adev); in amdgpu_device_reset()
3218 amdgpu_irq_gpu_reset_resume_helper(adev); in amdgpu_device_reset()
3219 r = amdgpu_ib_ring_tests(adev); in amdgpu_device_reset()
3221 dev_err(adev->dev, "ib ring test failed (%d).\n", r); in amdgpu_device_reset()
3222 r = amdgpu_device_ip_suspend(adev); in amdgpu_device_reset()
3228 if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost)) in amdgpu_device_reset()
3229 r = amdgpu_device_handle_vram_lost(adev); in amdgpu_device_reset()
3243 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, in amdgpu_device_reset_sriov() argument
3249 r = amdgpu_virt_request_full_gpu(adev, true); in amdgpu_device_reset_sriov()
3251 r = amdgpu_virt_reset_gpu(adev); in amdgpu_device_reset_sriov()
3256 r = amdgpu_device_ip_reinit_early_sriov(adev); in amdgpu_device_reset_sriov()
3261 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]); in amdgpu_device_reset_sriov()
3264 r = amdgpu_device_ip_reinit_late_sriov(adev); in amdgpu_device_reset_sriov()
3268 amdgpu_irq_gpu_reset_resume_helper(adev); in amdgpu_device_reset_sriov()
3269 r = amdgpu_ib_ring_tests(adev); in amdgpu_device_reset_sriov()
3272 amdgpu_virt_release_full_gpu(adev, true); in amdgpu_device_reset_sriov()
3273 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { in amdgpu_device_reset_sriov()
3274 atomic_inc(&adev->vram_lost_counter); in amdgpu_device_reset_sriov()
3275 r = amdgpu_device_handle_vram_lost(adev); in amdgpu_device_reset_sriov()
3291 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, in amdgpu_device_gpu_recover() argument
3296 if (!force && !amdgpu_device_ip_check_soft_reset(adev)) { in amdgpu_device_gpu_recover()
3302 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))) { in amdgpu_device_gpu_recover()
3307 dev_info(adev->dev, "GPU reset begin!\n"); in amdgpu_device_gpu_recover()
3309 mutex_lock(&adev->lock_reset); in amdgpu_device_gpu_recover()
3310 atomic_inc(&adev->gpu_reset_counter); in amdgpu_device_gpu_recover()
3311 adev->in_gpu_reset = 1; in amdgpu_device_gpu_recover()
3314 amdgpu_amdkfd_pre_reset(adev); in amdgpu_device_gpu_recover()
3317 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); in amdgpu_device_gpu_recover()
3321 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_gpu_recover()
3337 if (amdgpu_sriov_vf(adev)) in amdgpu_device_gpu_recover()
3338 r = amdgpu_device_reset_sriov(adev, job ? false : true); in amdgpu_device_gpu_recover()
3340 r = amdgpu_device_reset(adev); in amdgpu_device_gpu_recover()
3343 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_gpu_recover()
3358 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_gpu_recover()
3359 drm_helper_resume_force_mode(adev->ddev); in amdgpu_device_gpu_recover()
3362 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched); in amdgpu_device_gpu_recover()
3366 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
3367 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); in amdgpu_device_gpu_recover()
3369 dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
3373 amdgpu_amdkfd_post_reset(adev); in amdgpu_device_gpu_recover()
3374 amdgpu_vf_error_trans_all(adev); in amdgpu_device_gpu_recover()
3375 adev->in_gpu_reset = 0; in amdgpu_device_gpu_recover()
3376 mutex_unlock(&adev->lock_reset); in amdgpu_device_gpu_recover()
3389 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) in amdgpu_device_get_pcie_info() argument
3396 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; in amdgpu_device_get_pcie_info()
3399 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; in amdgpu_device_get_pcie_info()
3402 if (pci_is_root_bus(adev->pdev->bus)) { in amdgpu_device_get_pcie_info()
3403 if (adev->pm.pcie_gen_mask == 0) in amdgpu_device_get_pcie_info()
3404 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; in amdgpu_device_get_pcie_info()
3405 if (adev->pm.pcie_mlw_mask == 0) in amdgpu_device_get_pcie_info()
3406 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
3410 if (adev->pm.pcie_gen_mask == 0) { in amdgpu_device_get_pcie_info()
3412 pdev = adev->pdev; in amdgpu_device_get_pcie_info()
3415 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3420 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3425 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3429 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3432 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
3435 pdev = adev->ddev->pdev->bus->self; in amdgpu_device_get_pcie_info()
3438 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3442 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3447 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3451 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
3454 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
3458 if (adev->pm.pcie_mlw_mask == 0) { in amdgpu_device_get_pcie_info()
3459 pdev = adev->ddev->pdev->bus->self; in amdgpu_device_get_pcie_info()
3462 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
3466 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
3475 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
3483 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
3490 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
3496 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
3501 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
3505 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()