Lines Matching refs:adev
126 struct amdgpu_device *adev = ddev->dev_private; in amdgpu_device_get_pcie_replay_count() local
127 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); in amdgpu_device_get_pcie_replay_count()
135 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
147 struct amdgpu_device *adev = dev->dev_private; in amdgpu_device_is_px() local
149 if (adev->flags & AMD_IS_PX) in amdgpu_device_is_px()
166 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, in amdgpu_mm_rreg() argument
171 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) in amdgpu_mm_rreg()
172 return amdgpu_virt_kiq_rreg(adev, reg); in amdgpu_mm_rreg()
174 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) in amdgpu_mm_rreg()
175 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_rreg()
179 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_mm_rreg()
180 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); in amdgpu_mm_rreg()
181 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); in amdgpu_mm_rreg()
182 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_mm_rreg()
184 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); in amdgpu_mm_rreg()
202 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) { in amdgpu_mm_rreg8() argument
203 if (offset < adev->rmmio_size) in amdgpu_mm_rreg8()
204 return (readb(adev->rmmio + offset)); in amdgpu_mm_rreg8()
223 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) { in amdgpu_mm_wreg8() argument
224 if (offset < adev->rmmio_size) in amdgpu_mm_wreg8()
225 writeb(value, adev->rmmio + offset); in amdgpu_mm_wreg8()
240 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, in amdgpu_mm_wreg() argument
243 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); in amdgpu_mm_wreg()
245 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { in amdgpu_mm_wreg()
246 adev->last_mm_index = v; in amdgpu_mm_wreg()
249 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) in amdgpu_mm_wreg()
250 return amdgpu_virt_kiq_wreg(adev, reg, v); in amdgpu_mm_wreg()
252 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) in amdgpu_mm_wreg()
253 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_wreg()
257 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_mm_wreg()
258 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); in amdgpu_mm_wreg()
259 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); in amdgpu_mm_wreg()
260 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_mm_wreg()
263 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { in amdgpu_mm_wreg()
276 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) in amdgpu_io_rreg() argument
278 if ((reg * 4) < adev->rio_mem_size) in amdgpu_io_rreg()
279 return ioread32(adev->rio_mem + (reg * 4)); in amdgpu_io_rreg()
281 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); in amdgpu_io_rreg()
282 return ioread32(adev->rio_mem + (mmMM_DATA * 4)); in amdgpu_io_rreg()
295 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) in amdgpu_io_wreg() argument
297 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { in amdgpu_io_wreg()
298 adev->last_mm_index = v; in amdgpu_io_wreg()
301 if ((reg * 4) < adev->rio_mem_size) in amdgpu_io_wreg()
302 iowrite32(v, adev->rio_mem + (reg * 4)); in amdgpu_io_wreg()
304 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); in amdgpu_io_wreg()
305 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4)); in amdgpu_io_wreg()
308 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { in amdgpu_io_wreg()
322 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) in amdgpu_mm_rdoorbell() argument
324 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_rdoorbell()
325 return readl(adev->doorbell.ptr + index); in amdgpu_mm_rdoorbell()
342 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) in amdgpu_mm_wdoorbell() argument
344 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_wdoorbell()
345 writel(v, adev->doorbell.ptr + index); in amdgpu_mm_wdoorbell()
360 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) in amdgpu_mm_rdoorbell64() argument
362 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_rdoorbell64()
363 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); in amdgpu_mm_rdoorbell64()
380 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) in amdgpu_mm_wdoorbell64() argument
382 if (index < adev->doorbell.num_doorbells) { in amdgpu_mm_wdoorbell64()
383 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); in amdgpu_mm_wdoorbell64()
399 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) in amdgpu_invalid_rreg() argument
416 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) in amdgpu_invalid_wreg() argument
433 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) in amdgpu_invalid_rreg64() argument
450 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) in amdgpu_invalid_wreg64() argument
468 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, in amdgpu_block_invalid_rreg() argument
488 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, in amdgpu_block_invalid_wreg() argument
505 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) in amdgpu_device_vram_scratch_init() argument
507 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, in amdgpu_device_vram_scratch_init()
509 &adev->vram_scratch.robj, in amdgpu_device_vram_scratch_init()
510 &adev->vram_scratch.gpu_addr, in amdgpu_device_vram_scratch_init()
511 (void **)&adev->vram_scratch.ptr); in amdgpu_device_vram_scratch_init()
521 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) in amdgpu_device_vram_scratch_fini() argument
523 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); in amdgpu_device_vram_scratch_fini()
536 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, in amdgpu_device_program_register_sequence() argument
556 if (adev->family >= AMDGPU_FAMILY_AI) in amdgpu_device_program_register_sequence()
573 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) in amdgpu_device_pci_config_reset() argument
575 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); in amdgpu_device_pci_config_reset()
589 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) in amdgpu_device_doorbell_init() argument
593 if (adev->asic_type < CHIP_BONAIRE) { in amdgpu_device_doorbell_init()
594 adev->doorbell.base = 0; in amdgpu_device_doorbell_init()
595 adev->doorbell.size = 0; in amdgpu_device_doorbell_init()
596 adev->doorbell.num_doorbells = 0; in amdgpu_device_doorbell_init()
597 adev->doorbell.ptr = NULL; in amdgpu_device_doorbell_init()
601 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) in amdgpu_device_doorbell_init()
604 amdgpu_asic_init_doorbell_index(adev); in amdgpu_device_doorbell_init()
607 adev->doorbell.base = pci_resource_start(adev->pdev, 2); in amdgpu_device_doorbell_init()
608 adev->doorbell.size = pci_resource_len(adev->pdev, 2); in amdgpu_device_doorbell_init()
610 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), in amdgpu_device_doorbell_init()
611 adev->doorbell_index.max_assignment+1); in amdgpu_device_doorbell_init()
612 if (adev->doorbell.num_doorbells == 0) in amdgpu_device_doorbell_init()
621 if (adev->asic_type >= CHIP_VEGA10) in amdgpu_device_doorbell_init()
622 adev->doorbell.num_doorbells += 0x400; in amdgpu_device_doorbell_init()
624 adev->doorbell.ptr = ioremap(adev->doorbell.base, in amdgpu_device_doorbell_init()
625 adev->doorbell.num_doorbells * in amdgpu_device_doorbell_init()
627 if (adev->doorbell.ptr == NULL) in amdgpu_device_doorbell_init()
640 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) in amdgpu_device_doorbell_fini() argument
642 iounmap(adev->doorbell.ptr); in amdgpu_device_doorbell_fini()
643 adev->doorbell.ptr = NULL; in amdgpu_device_doorbell_fini()
662 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) in amdgpu_device_wb_fini() argument
664 if (adev->wb.wb_obj) { in amdgpu_device_wb_fini()
665 amdgpu_bo_free_kernel(&adev->wb.wb_obj, in amdgpu_device_wb_fini()
666 &adev->wb.gpu_addr, in amdgpu_device_wb_fini()
667 (void **)&adev->wb.wb); in amdgpu_device_wb_fini()
668 adev->wb.wb_obj = NULL; in amdgpu_device_wb_fini()
681 static int amdgpu_device_wb_init(struct amdgpu_device *adev) in amdgpu_device_wb_init() argument
685 if (adev->wb.wb_obj == NULL) { in amdgpu_device_wb_init()
687 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, in amdgpu_device_wb_init()
689 &adev->wb.wb_obj, &adev->wb.gpu_addr, in amdgpu_device_wb_init()
690 (void **)&adev->wb.wb); in amdgpu_device_wb_init()
692 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); in amdgpu_device_wb_init()
696 adev->wb.num_wb = AMDGPU_MAX_WB; in amdgpu_device_wb_init()
697 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); in amdgpu_device_wb_init()
700 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); in amdgpu_device_wb_init()
715 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) in amdgpu_device_wb_get() argument
717 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); in amdgpu_device_wb_get()
719 if (offset < adev->wb.num_wb) { in amdgpu_device_wb_get()
720 __set_bit(offset, adev->wb.used); in amdgpu_device_wb_get()
736 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) in amdgpu_device_wb_free() argument
739 if (wb < adev->wb.num_wb) in amdgpu_device_wb_free()
740 __clear_bit(wb, adev->wb.used); in amdgpu_device_wb_free()
752 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) in amdgpu_device_resize_fb_bar() argument
754 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); in amdgpu_device_resize_fb_bar()
763 if (amdgpu_sriov_vf(adev)) in amdgpu_device_resize_fb_bar()
767 root = adev->pdev->bus; in amdgpu_device_resize_fb_bar()
782 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); in amdgpu_device_resize_fb_bar()
783 pci_write_config_word(adev->pdev, PCI_COMMAND, in amdgpu_device_resize_fb_bar()
787 amdgpu_device_doorbell_fini(adev); in amdgpu_device_resize_fb_bar()
788 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_resize_fb_bar()
789 pci_release_resource(adev->pdev, 2); in amdgpu_device_resize_fb_bar()
791 pci_release_resource(adev->pdev, 0); in amdgpu_device_resize_fb_bar()
793 r = pci_resize_resource(adev->pdev, 0, rbar_size); in amdgpu_device_resize_fb_bar()
799 pci_assign_unassigned_bus_resources(adev->pdev->bus); in amdgpu_device_resize_fb_bar()
804 r = amdgpu_device_doorbell_init(adev); in amdgpu_device_resize_fb_bar()
805 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) in amdgpu_device_resize_fb_bar()
808 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); in amdgpu_device_resize_fb_bar()
825 bool amdgpu_device_need_post(struct amdgpu_device *adev) in amdgpu_device_need_post() argument
829 if (amdgpu_sriov_vf(adev)) in amdgpu_device_need_post()
832 if (amdgpu_passthrough(adev)) { in amdgpu_device_need_post()
838 if (adev->asic_type == CHIP_FIJI) { in amdgpu_device_need_post()
841 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); in amdgpu_device_need_post()
846 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); in amdgpu_device_need_post()
852 if (adev->has_hw_reset) { in amdgpu_device_need_post()
853 adev->has_hw_reset = false; in amdgpu_device_need_post()
858 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_need_post()
859 return amdgpu_atombios_scratch_need_asic_init(adev); in amdgpu_device_need_post()
862 reg = amdgpu_asic_get_config_memsize(adev); in amdgpu_device_need_post()
882 struct amdgpu_device *adev = cookie; in amdgpu_device_vga_set_decode() local
883 amdgpu_asic_set_vga_state(adev, state); in amdgpu_device_vga_set_decode()
901 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) in amdgpu_device_check_block_size() argument
910 dev_warn(adev->dev, "VM page table size (%d) too small\n", in amdgpu_device_check_block_size()
924 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) in amdgpu_device_check_vm_size() argument
931 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", in amdgpu_device_check_vm_size()
937 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) in amdgpu_device_check_smu_prv_buffer_size() argument
967 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; in amdgpu_device_check_smu_prv_buffer_size()
974 adev->pm.smu_prv_buffer_size = 0; in amdgpu_device_check_smu_prv_buffer_size()
985 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) in amdgpu_device_check_arguments() argument
990 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", in amdgpu_device_check_arguments()
994 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1001 dev_warn(adev->dev, "gart size (%d) too small\n", in amdgpu_device_check_arguments()
1008 dev_warn(adev->dev, "gtt size (%d) too small\n", in amdgpu_device_check_arguments()
1016 dev_warn(adev->dev, "valid range is between 4 and 9\n"); in amdgpu_device_check_arguments()
1020 amdgpu_device_check_smu_prv_buffer_size(adev); in amdgpu_device_check_arguments()
1022 amdgpu_device_check_vm_size(adev); in amdgpu_device_check_arguments()
1024 amdgpu_device_check_block_size(adev); in amdgpu_device_check_arguments()
1026 ret = amdgpu_device_get_job_timeout_settings(adev); in amdgpu_device_check_arguments()
1028 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); in amdgpu_device_check_arguments()
1032 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); in amdgpu_device_check_arguments()
1113 struct amdgpu_device *adev = dev; in amdgpu_device_ip_set_clockgating_state() local
1116 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_clockgating_state()
1117 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_clockgating_state()
1119 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_clockgating_state()
1121 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) in amdgpu_device_ip_set_clockgating_state()
1123 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( in amdgpu_device_ip_set_clockgating_state()
1124 (void *)adev, state); in amdgpu_device_ip_set_clockgating_state()
1127 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_clockgating_state()
1147 struct amdgpu_device *adev = dev; in amdgpu_device_ip_set_powergating_state() local
1150 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_powergating_state()
1151 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_powergating_state()
1153 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_powergating_state()
1155 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) in amdgpu_device_ip_set_powergating_state()
1157 r = adev->ip_blocks[i].version->funcs->set_powergating_state( in amdgpu_device_ip_set_powergating_state()
1158 (void *)adev, state); in amdgpu_device_ip_set_powergating_state()
1161 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_powergating_state()
1177 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, in amdgpu_device_ip_get_clockgating_state() argument
1182 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_get_clockgating_state()
1183 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_get_clockgating_state()
1185 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) in amdgpu_device_ip_get_clockgating_state()
1186 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); in amdgpu_device_ip_get_clockgating_state()
1199 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, in amdgpu_device_ip_wait_for_idle() argument
1204 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_wait_for_idle()
1205 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_wait_for_idle()
1207 if (adev->ip_blocks[i].version->type == block_type) { in amdgpu_device_ip_wait_for_idle()
1208 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); in amdgpu_device_ip_wait_for_idle()
1227 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, in amdgpu_device_ip_is_idle() argument
1232 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_is_idle()
1233 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_is_idle()
1235 if (adev->ip_blocks[i].version->type == block_type) in amdgpu_device_ip_is_idle()
1236 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); in amdgpu_device_ip_is_idle()
1252 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, in amdgpu_device_ip_get_ip_block() argument
1257 for (i = 0; i < adev->num_ip_blocks; i++) in amdgpu_device_ip_get_ip_block()
1258 if (adev->ip_blocks[i].version->type == type) in amdgpu_device_ip_get_ip_block()
1259 return &adev->ip_blocks[i]; in amdgpu_device_ip_get_ip_block()
1275 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, in amdgpu_device_ip_block_version_cmp() argument
1279 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); in amdgpu_device_ip_block_version_cmp()
1298 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, in amdgpu_device_ip_block_add() argument
1304 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, in amdgpu_device_ip_block_add()
1307 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; in amdgpu_device_ip_block_add()
1324 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) in amdgpu_device_enable_virtual_display() argument
1326 adev->enable_virtual_display = false; in amdgpu_device_enable_virtual_display()
1329 struct drm_device *ddev = adev->ddev; in amdgpu_device_enable_virtual_display()
1342 adev->enable_virtual_display = true; in amdgpu_device_enable_virtual_display()
1353 adev->mode_info.num_crtc = num_crtc; in amdgpu_device_enable_virtual_display()
1355 adev->mode_info.num_crtc = 1; in amdgpu_device_enable_virtual_display()
1363 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_enable_virtual_display()
1379 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) in amdgpu_device_parse_gpu_info_fw() argument
1386 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_parse_gpu_info_fw()
1388 switch (adev->asic_type) { in amdgpu_device_parse_gpu_info_fw()
1422 if (adev->rev_id >= 8) in amdgpu_device_parse_gpu_info_fw()
1424 else if (adev->pdev->device == 0x15d8) in amdgpu_device_parse_gpu_info_fw()
1447 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); in amdgpu_device_parse_gpu_info_fw()
1449 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1454 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); in amdgpu_device_parse_gpu_info_fw()
1456 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1462 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; in amdgpu_device_parse_gpu_info_fw()
1469 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1472 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); in amdgpu_device_parse_gpu_info_fw()
1473 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); in amdgpu_device_parse_gpu_info_fw()
1474 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); in amdgpu_device_parse_gpu_info_fw()
1475 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); in amdgpu_device_parse_gpu_info_fw()
1476 adev->gfx.config.max_texture_channel_caches = in amdgpu_device_parse_gpu_info_fw()
1478 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); in amdgpu_device_parse_gpu_info_fw()
1479 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); in amdgpu_device_parse_gpu_info_fw()
1480 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); in amdgpu_device_parse_gpu_info_fw()
1481 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); in amdgpu_device_parse_gpu_info_fw()
1482 adev->gfx.config.double_offchip_lds_buf = in amdgpu_device_parse_gpu_info_fw()
1484 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); in amdgpu_device_parse_gpu_info_fw()
1485 adev->gfx.cu_info.max_waves_per_simd = in amdgpu_device_parse_gpu_info_fw()
1487 adev->gfx.cu_info.max_scratch_slots_per_cu = in amdgpu_device_parse_gpu_info_fw()
1489 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); in amdgpu_device_parse_gpu_info_fw()
1492 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1494 adev->gfx.config.num_sc_per_sh = in amdgpu_device_parse_gpu_info_fw()
1496 adev->gfx.config.num_packer_per_sc = in amdgpu_device_parse_gpu_info_fw()
1502 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1504 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; in amdgpu_device_parse_gpu_info_fw()
1510 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1529 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) in amdgpu_device_ip_early_init() argument
1533 amdgpu_device_enable_virtual_display(adev); in amdgpu_device_ip_early_init()
1535 switch (adev->asic_type) { in amdgpu_device_ip_early_init()
1545 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) in amdgpu_device_ip_early_init()
1546 adev->family = AMDGPU_FAMILY_CZ; in amdgpu_device_ip_early_init()
1548 adev->family = AMDGPU_FAMILY_VI; in amdgpu_device_ip_early_init()
1550 r = vi_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1560 adev->family = AMDGPU_FAMILY_SI; in amdgpu_device_ip_early_init()
1561 r = si_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1572 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII)) in amdgpu_device_ip_early_init()
1573 adev->family = AMDGPU_FAMILY_CI; in amdgpu_device_ip_early_init()
1575 adev->family = AMDGPU_FAMILY_KV; in amdgpu_device_ip_early_init()
1577 r = cik_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1588 if (adev->asic_type == CHIP_RAVEN || in amdgpu_device_ip_early_init()
1589 adev->asic_type == CHIP_RENOIR) in amdgpu_device_ip_early_init()
1590 adev->family = AMDGPU_FAMILY_RV; in amdgpu_device_ip_early_init()
1592 adev->family = AMDGPU_FAMILY_AI; in amdgpu_device_ip_early_init()
1594 r = soc15_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1601 adev->family = AMDGPU_FAMILY_NV; in amdgpu_device_ip_early_init()
1603 r = nv_set_ip_blocks(adev); in amdgpu_device_ip_early_init()
1612 r = amdgpu_device_parse_gpu_info_fw(adev); in amdgpu_device_ip_early_init()
1616 amdgpu_amdkfd_device_probe(adev); in amdgpu_device_ip_early_init()
1618 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_ip_early_init()
1619 r = amdgpu_virt_request_full_gpu(adev, true); in amdgpu_device_ip_early_init()
1624 adev->pm.pp_feature = amdgpu_pp_feature_mask; in amdgpu_device_ip_early_init()
1625 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_early_init()
1626 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in amdgpu_device_ip_early_init()
1628 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_early_init()
1631 i, adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_early_init()
1632 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
1634 if (adev->ip_blocks[i].version->funcs->early_init) { in amdgpu_device_ip_early_init()
1635 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); in amdgpu_device_ip_early_init()
1637 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
1640 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_early_init()
1643 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
1646 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
1650 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_early_init()
1652 if (!amdgpu_get_bios(adev)) in amdgpu_device_ip_early_init()
1655 r = amdgpu_atombios_init(adev); in amdgpu_device_ip_early_init()
1657 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); in amdgpu_device_ip_early_init()
1658 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); in amdgpu_device_ip_early_init()
1664 adev->cg_flags &= amdgpu_cg_mask; in amdgpu_device_ip_early_init()
1665 adev->pg_flags &= amdgpu_pg_mask; in amdgpu_device_ip_early_init()
1670 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) in amdgpu_device_ip_hw_init_phase1() argument
1674 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase1()
1675 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase1()
1677 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase1()
1679 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_hw_init_phase1()
1680 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || in amdgpu_device_ip_hw_init_phase1()
1681 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_hw_init_phase1()
1682 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase1()
1685 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase1()
1688 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase1()
1695 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) in amdgpu_device_ip_hw_init_phase2() argument
1699 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase2()
1700 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase2()
1702 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase2()
1704 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase2()
1707 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase2()
1710 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase2()
1716 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) in amdgpu_device_fw_loading() argument
1722 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_device_fw_loading()
1723 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_fw_loading()
1724 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_fw_loading()
1728 if (adev->ip_blocks[i].status.hw == true) in amdgpu_device_fw_loading()
1731 if (adev->in_gpu_reset || adev->in_suspend) { in amdgpu_device_fw_loading()
1732 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_fw_loading()
1735 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
1739 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_fw_loading()
1742 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
1747 adev->ip_blocks[i].status.hw = true; in amdgpu_device_fw_loading()
1752 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); in amdgpu_device_fw_loading()
1768 static int amdgpu_device_ip_init(struct amdgpu_device *adev) in amdgpu_device_ip_init() argument
1772 r = amdgpu_ras_init(adev); in amdgpu_device_ip_init()
1776 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
1777 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_init()
1779 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); in amdgpu_device_ip_init()
1782 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_init()
1785 adev->ip_blocks[i].status.sw = true; in amdgpu_device_ip_init()
1788 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_init()
1789 r = amdgpu_device_vram_scratch_init(adev); in amdgpu_device_ip_init()
1794 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
1799 r = amdgpu_device_wb_init(adev); in amdgpu_device_ip_init()
1804 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
1807 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { in amdgpu_device_ip_init()
1808 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, in amdgpu_device_ip_init()
1819 r = amdgpu_ib_pool_init(adev); in amdgpu_device_ip_init()
1821 dev_err(adev->dev, "IB initialization failed (%d).\n", r); in amdgpu_device_ip_init()
1822 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); in amdgpu_device_ip_init()
1826 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ in amdgpu_device_ip_init()
1830 r = amdgpu_device_ip_hw_init_phase1(adev); in amdgpu_device_ip_init()
1834 r = amdgpu_device_fw_loading(adev); in amdgpu_device_ip_init()
1838 r = amdgpu_device_ip_hw_init_phase2(adev); in amdgpu_device_ip_init()
1842 if (adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_ip_init()
1843 amdgpu_xgmi_add_device(adev); in amdgpu_device_ip_init()
1844 amdgpu_amdkfd_device_init(adev); in amdgpu_device_ip_init()
1847 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_ip_init()
1849 amdgpu_virt_init_data_exchange(adev); in amdgpu_device_ip_init()
1850 amdgpu_virt_release_full_gpu(adev, true); in amdgpu_device_ip_init()
1865 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) in amdgpu_device_fill_reset_magic() argument
1867 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); in amdgpu_device_fill_reset_magic()
1880 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) in amdgpu_device_check_vram_lost() argument
1882 return !!memcmp(adev->gart.ptr, adev->reset_magic, in amdgpu_device_check_vram_lost()
1898 static int amdgpu_device_set_cg_state(struct amdgpu_device *adev, in amdgpu_device_set_cg_state() argument
1906 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_cg_state()
1907 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_cg_state()
1908 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_cg_state()
1911 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_cg_state()
1912 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_cg_state()
1913 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_cg_state()
1914 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_set_cg_state()
1916 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_set_cg_state()
1920 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_cg_state()
1929 static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state) in amdgpu_device_set_pg_state() argument
1936 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_pg_state()
1937 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_pg_state()
1938 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_pg_state()
1941 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_pg_state()
1942 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_pg_state()
1943 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_pg_state()
1944 adev->ip_blocks[i].version->funcs->set_powergating_state) { in amdgpu_device_set_pg_state()
1946 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, in amdgpu_device_set_pg_state()
1950 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_pg_state()
1961 struct amdgpu_device *adev; in amdgpu_device_enable_mgpu_fan_boost() local
1976 adev = gpu_ins->adev; in amdgpu_device_enable_mgpu_fan_boost()
1977 if (!(adev->flags & AMD_IS_APU) && in amdgpu_device_enable_mgpu_fan_boost()
1979 adev->powerplay.pp_funcs && in amdgpu_device_enable_mgpu_fan_boost()
1980 adev->powerplay.pp_funcs->enable_mgpu_fan_boost) { in amdgpu_device_enable_mgpu_fan_boost()
1981 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); in amdgpu_device_enable_mgpu_fan_boost()
2007 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) in amdgpu_device_ip_late_init() argument
2011 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_init()
2012 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_late_init()
2014 if (adev->ip_blocks[i].version->funcs->late_init) { in amdgpu_device_ip_late_init()
2015 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); in amdgpu_device_ip_late_init()
2018 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_init()
2022 adev->ip_blocks[i].status.late_initialized = true; in amdgpu_device_ip_late_init()
2025 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); in amdgpu_device_ip_late_init()
2026 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); in amdgpu_device_ip_late_init()
2028 amdgpu_device_fill_reset_magic(adev); in amdgpu_device_ip_late_init()
2035 amdgpu_xgmi_set_pstate(adev, 0); in amdgpu_device_ip_late_init()
2051 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) in amdgpu_device_ip_fini() argument
2055 amdgpu_ras_pre_fini(adev); in amdgpu_device_ip_fini()
2057 if (adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_ip_fini()
2058 amdgpu_xgmi_remove_device(adev); in amdgpu_device_ip_fini()
2060 amdgpu_amdkfd_device_fini(adev); in amdgpu_device_ip_fini()
2062 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); in amdgpu_device_ip_fini()
2063 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); in amdgpu_device_ip_fini()
2066 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_fini()
2067 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini()
2069 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_fini()
2070 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini()
2074 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
2076 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini()
2081 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
2082 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini()
2085 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini()
2089 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
2092 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini()
2096 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
2097 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_fini()
2100 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_fini()
2101 amdgpu_ucode_free_bo(adev); in amdgpu_device_ip_fini()
2102 amdgpu_free_static_csa(&adev->virt.csa_obj); in amdgpu_device_ip_fini()
2103 amdgpu_device_wb_fini(adev); in amdgpu_device_ip_fini()
2104 amdgpu_device_vram_scratch_fini(adev); in amdgpu_device_ip_fini()
2105 amdgpu_ib_pool_fini(adev); in amdgpu_device_ip_fini()
2108 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); in amdgpu_device_ip_fini()
2112 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
2114 adev->ip_blocks[i].status.sw = false; in amdgpu_device_ip_fini()
2115 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_fini()
2118 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
2119 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_ip_fini()
2121 if (adev->ip_blocks[i].version->funcs->late_fini) in amdgpu_device_ip_fini()
2122 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); in amdgpu_device_ip_fini()
2123 adev->ip_blocks[i].status.late_initialized = false; in amdgpu_device_ip_fini()
2126 amdgpu_ras_fini(adev); in amdgpu_device_ip_fini()
2128 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_fini()
2129 if (amdgpu_virt_release_full_gpu(adev, false)) in amdgpu_device_ip_fini()
2142 struct amdgpu_device *adev = in amdgpu_device_delayed_init_work_handler() local
2146 r = amdgpu_ib_ring_tests(adev); in amdgpu_device_delayed_init_work_handler()
2153 struct amdgpu_device *adev = in amdgpu_device_delay_enable_gfx_off() local
2156 mutex_lock(&adev->gfx.gfx_off_mutex); in amdgpu_device_delay_enable_gfx_off()
2157 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { in amdgpu_device_delay_enable_gfx_off()
2158 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) in amdgpu_device_delay_enable_gfx_off()
2159 adev->gfx.gfx_off_state = true; in amdgpu_device_delay_enable_gfx_off()
2161 mutex_unlock(&adev->gfx.gfx_off_mutex); in amdgpu_device_delay_enable_gfx_off()
2175 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) in amdgpu_device_ip_suspend_phase1() argument
2179 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); in amdgpu_device_ip_suspend_phase1()
2180 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); in amdgpu_device_ip_suspend_phase1()
2182 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase1()
2183 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase1()
2186 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { in amdgpu_device_ip_suspend_phase1()
2188 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase1()
2192 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase1()
2195 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase1()
2213 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) in amdgpu_device_ip_suspend_phase2() argument
2217 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase2()
2218 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase2()
2221 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase2()
2224 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase2()
2228 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase2()
2230 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
2232 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_suspend_phase2()
2233 if (is_support_sw_smu(adev)) { in amdgpu_device_ip_suspend_phase2()
2235 } else if (adev->powerplay.pp_funcs && in amdgpu_device_ip_suspend_phase2()
2236 adev->powerplay.pp_funcs->set_mp1_state) { in amdgpu_device_ip_suspend_phase2()
2237 r = adev->powerplay.pp_funcs->set_mp1_state( in amdgpu_device_ip_suspend_phase2()
2238 adev->powerplay.pp_handle, in amdgpu_device_ip_suspend_phase2()
2239 adev->mp1_state); in amdgpu_device_ip_suspend_phase2()
2242 adev->mp1_state, r); in amdgpu_device_ip_suspend_phase2()
2248 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
2265 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) in amdgpu_device_ip_suspend() argument
2269 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend()
2270 amdgpu_virt_request_full_gpu(adev, false); in amdgpu_device_ip_suspend()
2272 r = amdgpu_device_ip_suspend_phase1(adev); in amdgpu_device_ip_suspend()
2275 r = amdgpu_device_ip_suspend_phase2(adev); in amdgpu_device_ip_suspend()
2277 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_suspend()
2278 amdgpu_virt_release_full_gpu(adev, false); in amdgpu_device_ip_suspend()
2283 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) in amdgpu_device_ip_reinit_early_sriov() argument
2298 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_early_sriov()
2299 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_early_sriov()
2306 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_early_sriov()
2317 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) in amdgpu_device_ip_reinit_late_sriov() argument
2334 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_late_sriov()
2335 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_late_sriov()
2342 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_late_sriov()
2365 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) in amdgpu_device_ip_resume_phase1() argument
2369 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase1()
2370 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase1()
2372 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase1()
2373 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase1()
2374 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_resume_phase1()
2376 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase1()
2379 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase1()
2382 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase1()
2402 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) in amdgpu_device_ip_resume_phase2() argument
2406 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase2()
2407 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase2()
2409 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase2()
2410 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase2()
2411 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase2()
2412 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_resume_phase2()
2414 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase2()
2417 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase2()
2420 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase2()
2438 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) in amdgpu_device_ip_resume() argument
2442 r = amdgpu_device_ip_resume_phase1(adev); in amdgpu_device_ip_resume()
2446 r = amdgpu_device_fw_loading(adev); in amdgpu_device_ip_resume()
2450 r = amdgpu_device_ip_resume_phase2(adev); in amdgpu_device_ip_resume()
2462 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) in amdgpu_device_detect_sriov_bios() argument
2464 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_detect_sriov_bios()
2465 if (adev->is_atom_fw) { in amdgpu_device_detect_sriov_bios()
2466 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev)) in amdgpu_device_detect_sriov_bios()
2467 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
2469 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) in amdgpu_device_detect_sriov_bios()
2470 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
2473 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) in amdgpu_device_detect_sriov_bios()
2474 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); in amdgpu_device_detect_sriov_bios()
2539 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) in amdgpu_device_has_dc_support() argument
2541 if (amdgpu_sriov_vf(adev)) in amdgpu_device_has_dc_support()
2544 return amdgpu_device_asic_has_dc_support(adev->asic_type); in amdgpu_device_has_dc_support()
2550 struct amdgpu_device *adev = in amdgpu_device_xgmi_reset_func() local
2553 adev->asic_reset_res = amdgpu_asic_reset(adev); in amdgpu_device_xgmi_reset_func()
2554 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
2556 adev->asic_reset_res, adev->ddev->unique); in amdgpu_device_xgmi_reset_func()
2572 int amdgpu_device_init(struct amdgpu_device *adev, in amdgpu_device_init() argument
2581 adev->shutdown = false; in amdgpu_device_init()
2582 adev->dev = &pdev->dev; in amdgpu_device_init()
2583 adev->ddev = ddev; in amdgpu_device_init()
2584 adev->pdev = pdev; in amdgpu_device_init()
2585 adev->flags = flags; in amdgpu_device_init()
2586 adev->asic_type = flags & AMD_ASIC_MASK; in amdgpu_device_init()
2587 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; in amdgpu_device_init()
2589 adev->usec_timeout *= 2; in amdgpu_device_init()
2590 adev->gmc.gart_size = 512 * 1024 * 1024; in amdgpu_device_init()
2591 adev->accel_working = false; in amdgpu_device_init()
2592 adev->num_rings = 0; in amdgpu_device_init()
2593 adev->mman.buffer_funcs = NULL; in amdgpu_device_init()
2594 adev->mman.buffer_funcs_ring = NULL; in amdgpu_device_init()
2595 adev->vm_manager.vm_pte_funcs = NULL; in amdgpu_device_init()
2596 adev->vm_manager.vm_pte_num_rqs = 0; in amdgpu_device_init()
2597 adev->gmc.gmc_funcs = NULL; in amdgpu_device_init()
2598 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); in amdgpu_device_init()
2599 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in amdgpu_device_init()
2601 adev->smc_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2602 adev->smc_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2603 adev->pcie_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2604 adev->pcie_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2605 adev->pciep_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2606 adev->pciep_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2607 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; in amdgpu_device_init()
2608 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; in amdgpu_device_init()
2609 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2610 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2611 adev->didt_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2612 adev->didt_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2613 adev->gc_cac_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
2614 adev->gc_cac_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
2615 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; in amdgpu_device_init()
2616 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; in amdgpu_device_init()
2619 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, in amdgpu_device_init()
2624 atomic_set(&adev->irq.ih.lock, 0); in amdgpu_device_init()
2625 mutex_init(&adev->firmware.mutex); in amdgpu_device_init()
2626 mutex_init(&adev->pm.mutex); in amdgpu_device_init()
2627 mutex_init(&adev->gfx.gpu_clock_mutex); in amdgpu_device_init()
2628 mutex_init(&adev->srbm_mutex); in amdgpu_device_init()
2629 mutex_init(&adev->gfx.pipe_reserve_mutex); in amdgpu_device_init()
2630 mutex_init(&adev->gfx.gfx_off_mutex); in amdgpu_device_init()
2631 mutex_init(&adev->grbm_idx_mutex); in amdgpu_device_init()
2632 mutex_init(&adev->mn_lock); in amdgpu_device_init()
2633 mutex_init(&adev->virt.vf_errors.lock); in amdgpu_device_init()
2634 hash_init(adev->mn_hash); in amdgpu_device_init()
2635 mutex_init(&adev->lock_reset); in amdgpu_device_init()
2636 mutex_init(&adev->virt.dpm_mutex); in amdgpu_device_init()
2637 mutex_init(&adev->psp.mutex); in amdgpu_device_init()
2639 r = amdgpu_device_check_arguments(adev); in amdgpu_device_init()
2643 spin_lock_init(&adev->mmio_idx_lock); in amdgpu_device_init()
2644 spin_lock_init(&adev->smc_idx_lock); in amdgpu_device_init()
2645 spin_lock_init(&adev->pcie_idx_lock); in amdgpu_device_init()
2646 spin_lock_init(&adev->uvd_ctx_idx_lock); in amdgpu_device_init()
2647 spin_lock_init(&adev->didt_idx_lock); in amdgpu_device_init()
2648 spin_lock_init(&adev->gc_cac_idx_lock); in amdgpu_device_init()
2649 spin_lock_init(&adev->se_cac_idx_lock); in amdgpu_device_init()
2650 spin_lock_init(&adev->audio_endpt_idx_lock); in amdgpu_device_init()
2651 spin_lock_init(&adev->mm_stats.lock); in amdgpu_device_init()
2653 INIT_LIST_HEAD(&adev->shadow_list); in amdgpu_device_init()
2654 mutex_init(&adev->shadow_list_lock); in amdgpu_device_init()
2656 INIT_LIST_HEAD(&adev->ring_lru_list); in amdgpu_device_init()
2657 spin_lock_init(&adev->ring_lru_list_lock); in amdgpu_device_init()
2659 INIT_DELAYED_WORK(&adev->delayed_init_work, in amdgpu_device_init()
2661 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, in amdgpu_device_init()
2664 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); in amdgpu_device_init()
2666 adev->gfx.gfx_off_req_count = 1; in amdgpu_device_init()
2667 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false; in amdgpu_device_init()
2671 if (adev->asic_type >= CHIP_BONAIRE) { in amdgpu_device_init()
2672 adev->rmmio_base = pci_resource_start(adev->pdev, 5); in amdgpu_device_init()
2673 adev->rmmio_size = pci_resource_len(adev->pdev, 5); in amdgpu_device_init()
2675 adev->rmmio_base = pci_resource_start(adev->pdev, 2); in amdgpu_device_init()
2676 adev->rmmio_size = pci_resource_len(adev->pdev, 2); in amdgpu_device_init()
2679 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); in amdgpu_device_init()
2680 if (adev->rmmio == NULL) { in amdgpu_device_init()
2683 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); in amdgpu_device_init()
2684 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); in amdgpu_device_init()
2688 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) { in amdgpu_device_init()
2689 adev->rio_mem_size = pci_resource_len(adev->pdev, i); in amdgpu_device_init()
2690 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size); in amdgpu_device_init()
2694 if (adev->rio_mem == NULL) in amdgpu_device_init()
2698 r = pci_enable_atomic_ops_to_root(adev->pdev, in amdgpu_device_init()
2702 adev->have_atomics_support = false; in amdgpu_device_init()
2705 adev->have_atomics_support = true; in amdgpu_device_init()
2708 amdgpu_device_get_pcie_info(adev); in amdgpu_device_init()
2713 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10) in amdgpu_device_init()
2714 adev->enable_mes = true; in amdgpu_device_init()
2716 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) { in amdgpu_device_init()
2717 r = amdgpu_discovery_init(adev); in amdgpu_device_init()
2719 dev_err(adev->dev, "amdgpu_discovery_init failed\n"); in amdgpu_device_init()
2725 r = amdgpu_device_ip_early_init(adev); in amdgpu_device_init()
2730 amdgpu_device_doorbell_init(adev); in amdgpu_device_init()
2735 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode); in amdgpu_device_init()
2739 if (!pci_is_thunderbolt_attached(adev->pdev)) in amdgpu_device_init()
2740 vga_switcheroo_register_client(adev->pdev, in amdgpu_device_init()
2743 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); in amdgpu_device_init()
2747 emu_soc_asic_init(adev); in amdgpu_device_init()
2752 amdgpu_device_detect_sriov_bios(adev); in amdgpu_device_init()
2757 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { in amdgpu_device_init()
2758 r = amdgpu_asic_reset(adev); in amdgpu_device_init()
2760 dev_err(adev->dev, "asic reset on init failed\n"); in amdgpu_device_init()
2766 if (amdgpu_device_need_post(adev)) { in amdgpu_device_init()
2767 if (!adev->bios) { in amdgpu_device_init()
2768 dev_err(adev->dev, "no vBIOS found\n"); in amdgpu_device_init()
2773 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_init()
2775 dev_err(adev->dev, "gpu post error!\n"); in amdgpu_device_init()
2780 if (adev->is_atom_fw) { in amdgpu_device_init()
2782 r = amdgpu_atomfirmware_get_clock_info(adev); in amdgpu_device_init()
2784 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); in amdgpu_device_init()
2785 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); in amdgpu_device_init()
2790 r = amdgpu_atombios_get_clock_info(adev); in amdgpu_device_init()
2792 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); in amdgpu_device_init()
2793 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); in amdgpu_device_init()
2797 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_init()
2798 amdgpu_atombios_i2c_init(adev); in amdgpu_device_init()
2803 r = amdgpu_fence_driver_init(adev); in amdgpu_device_init()
2805 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); in amdgpu_device_init()
2806 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); in amdgpu_device_init()
2811 drm_mode_config_init(adev->ddev); in amdgpu_device_init()
2813 r = amdgpu_device_ip_init(adev); in amdgpu_device_init()
2816 if (amdgpu_sriov_vf(adev) && in amdgpu_device_init()
2817 !amdgpu_sriov_runtime(adev) && in amdgpu_device_init()
2818 amdgpu_virt_mmio_blocked(adev) && in amdgpu_device_init()
2819 !amdgpu_virt_wait_reset(adev)) { in amdgpu_device_init()
2820 dev_err(adev->dev, "VF exclusive mode timeout\n"); in amdgpu_device_init()
2822 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; in amdgpu_device_init()
2823 adev->virt.ops = NULL; in amdgpu_device_init()
2827 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); in amdgpu_device_init()
2828 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); in amdgpu_device_init()
2829 if (amdgpu_virt_request_full_gpu(adev, false)) in amdgpu_device_init()
2830 amdgpu_virt_release_full_gpu(adev, false); in amdgpu_device_init()
2834 adev->accel_working = true; in amdgpu_device_init()
2836 amdgpu_vm_check_compute_bug(adev); in amdgpu_device_init()
2844 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); in amdgpu_device_init()
2846 amdgpu_fbdev_init(adev); in amdgpu_device_init()
2848 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)) in amdgpu_device_init()
2849 amdgpu_pm_virt_sysfs_init(adev); in amdgpu_device_init()
2851 r = amdgpu_pm_sysfs_init(adev); in amdgpu_device_init()
2855 r = amdgpu_ucode_sysfs_init(adev); in amdgpu_device_init()
2859 r = amdgpu_debugfs_gem_init(adev); in amdgpu_device_init()
2863 r = amdgpu_debugfs_regs_init(adev); in amdgpu_device_init()
2867 r = amdgpu_debugfs_firmware_init(adev); in amdgpu_device_init()
2871 r = amdgpu_debugfs_init(adev); in amdgpu_device_init()
2876 if (adev->accel_working) in amdgpu_device_init()
2877 amdgpu_test_moves(adev); in amdgpu_device_init()
2882 if (adev->accel_working) in amdgpu_device_init()
2883 amdgpu_benchmark(adev, amdgpu_benchmarking); in amdgpu_device_init()
2893 amdgpu_register_gpu_instance(adev); in amdgpu_device_init()
2898 r = amdgpu_device_ip_late_init(adev); in amdgpu_device_init()
2900 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); in amdgpu_device_init()
2901 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); in amdgpu_device_init()
2906 amdgpu_ras_resume(adev); in amdgpu_device_init()
2908 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_init()
2911 r = device_create_file(adev->dev, &dev_attr_pcie_replay_count); in amdgpu_device_init()
2913 dev_err(adev->dev, "Could not create pcie_replay_count"); in amdgpu_device_init()
2918 r = amdgpu_pmu_init(adev); in amdgpu_device_init()
2920 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); in amdgpu_device_init()
2925 amdgpu_vf_error_trans_all(adev); in amdgpu_device_init()
2927 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_init()
2940 void amdgpu_device_fini(struct amdgpu_device *adev) in amdgpu_device_fini() argument
2945 adev->shutdown = true; in amdgpu_device_fini()
2947 amdgpu_irq_disable_all(adev); in amdgpu_device_fini()
2948 if (adev->mode_info.mode_config_initialized){ in amdgpu_device_fini()
2949 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_fini()
2950 drm_helper_force_disable_all(adev->ddev); in amdgpu_device_fini()
2952 drm_atomic_helper_shutdown(adev->ddev); in amdgpu_device_fini()
2954 amdgpu_fence_driver_fini(adev); in amdgpu_device_fini()
2955 amdgpu_pm_sysfs_fini(adev); in amdgpu_device_fini()
2956 amdgpu_fbdev_fini(adev); in amdgpu_device_fini()
2957 r = amdgpu_device_ip_fini(adev); in amdgpu_device_fini()
2958 if (adev->firmware.gpu_info_fw) { in amdgpu_device_fini()
2959 release_firmware(adev->firmware.gpu_info_fw); in amdgpu_device_fini()
2960 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_fini()
2962 adev->accel_working = false; in amdgpu_device_fini()
2963 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_fini()
2965 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_fini()
2966 amdgpu_i2c_fini(adev); in amdgpu_device_fini()
2969 amdgpu_atombios_fini(adev); in amdgpu_device_fini()
2971 kfree(adev->bios); in amdgpu_device_fini()
2972 adev->bios = NULL; in amdgpu_device_fini()
2973 if (!pci_is_thunderbolt_attached(adev->pdev)) in amdgpu_device_fini()
2974 vga_switcheroo_unregister_client(adev->pdev); in amdgpu_device_fini()
2975 if (adev->flags & AMD_IS_PX) in amdgpu_device_fini()
2976 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_fini()
2977 vga_client_register(adev->pdev, NULL, NULL, NULL); in amdgpu_device_fini()
2978 if (adev->rio_mem) in amdgpu_device_fini()
2979 pci_iounmap(adev->pdev, adev->rio_mem); in amdgpu_device_fini()
2980 adev->rio_mem = NULL; in amdgpu_device_fini()
2981 iounmap(adev->rmmio); in amdgpu_device_fini()
2982 adev->rmmio = NULL; in amdgpu_device_fini()
2983 amdgpu_device_doorbell_fini(adev); in amdgpu_device_fini()
2984 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)) in amdgpu_device_fini()
2985 amdgpu_pm_virt_sysfs_fini(adev); in amdgpu_device_fini()
2987 amdgpu_debugfs_regs_cleanup(adev); in amdgpu_device_fini()
2988 device_remove_file(adev->dev, &dev_attr_pcie_replay_count); in amdgpu_device_fini()
2989 amdgpu_ucode_sysfs_fini(adev); in amdgpu_device_fini()
2991 amdgpu_pmu_fini(adev); in amdgpu_device_fini()
2992 amdgpu_debugfs_preempt_cleanup(adev); in amdgpu_device_fini()
2993 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) in amdgpu_device_fini()
2994 amdgpu_discovery_fini(adev); in amdgpu_device_fini()
3014 struct amdgpu_device *adev; in amdgpu_device_suspend() local
3023 adev = dev->dev_private; in amdgpu_device_suspend()
3028 adev->in_suspend = true; in amdgpu_device_suspend()
3032 amdgpu_fbdev_set_suspend(adev, 1); in amdgpu_device_suspend()
3034 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_suspend()
3036 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_suspend()
3049 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { in amdgpu_device_suspend()
3063 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { in amdgpu_device_suspend()
3073 amdgpu_amdkfd_suspend(adev); in amdgpu_device_suspend()
3075 amdgpu_ras_suspend(adev); in amdgpu_device_suspend()
3077 r = amdgpu_device_ip_suspend_phase1(adev); in amdgpu_device_suspend()
3080 amdgpu_bo_evict_vram(adev); in amdgpu_device_suspend()
3082 amdgpu_fence_driver_suspend(adev); in amdgpu_device_suspend()
3084 r = amdgpu_device_ip_suspend_phase2(adev); in amdgpu_device_suspend()
3090 amdgpu_bo_evict_vram(adev); in amdgpu_device_suspend()
3098 r = amdgpu_asic_reset(adev); in amdgpu_device_suspend()
3120 struct amdgpu_device *adev = dev->dev_private; in amdgpu_device_resume() local
3136 if (amdgpu_device_need_post(adev)) { in amdgpu_device_resume()
3137 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_resume()
3142 r = amdgpu_device_ip_resume(adev); in amdgpu_device_resume()
3147 amdgpu_fence_driver_resume(adev); in amdgpu_device_resume()
3150 r = amdgpu_device_ip_late_init(adev); in amdgpu_device_resume()
3154 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_resume()
3157 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_resume()
3162 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { in amdgpu_device_resume()
3175 r = amdgpu_amdkfd_resume(adev); in amdgpu_device_resume()
3180 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_resume()
3184 if (!amdgpu_device_has_dc_support(adev)) { in amdgpu_device_resume()
3195 amdgpu_fbdev_set_suspend(adev, 0); in amdgpu_device_resume()
3200 amdgpu_ras_resume(adev); in amdgpu_device_resume()
3214 if (!amdgpu_device_has_dc_support(adev)) in amdgpu_device_resume()
3221 adev->in_suspend = false; in amdgpu_device_resume()
3236 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_check_soft_reset() argument
3241 if (amdgpu_sriov_vf(adev)) in amdgpu_device_ip_check_soft_reset()
3244 if (amdgpu_asic_need_full_reset(adev)) in amdgpu_device_ip_check_soft_reset()
3247 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_check_soft_reset()
3248 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_check_soft_reset()
3250 if (adev->ip_blocks[i].version->funcs->check_soft_reset) in amdgpu_device_ip_check_soft_reset()
3251 adev->ip_blocks[i].status.hang = in amdgpu_device_ip_check_soft_reset()
3252 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); in amdgpu_device_ip_check_soft_reset()
3253 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_check_soft_reset()
3254 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_check_soft_reset()
3272 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_pre_soft_reset() argument
3276 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_pre_soft_reset()
3277 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_pre_soft_reset()
3279 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_pre_soft_reset()
3280 adev->ip_blocks[i].version->funcs->pre_soft_reset) { in amdgpu_device_ip_pre_soft_reset()
3281 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); in amdgpu_device_ip_pre_soft_reset()
3299 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) in amdgpu_device_ip_need_full_reset() argument
3303 if (amdgpu_asic_need_full_reset(adev)) in amdgpu_device_ip_need_full_reset()
3306 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_need_full_reset()
3307 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_need_full_reset()
3309 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || in amdgpu_device_ip_need_full_reset()
3310 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || in amdgpu_device_ip_need_full_reset()
3311 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || in amdgpu_device_ip_need_full_reset()
3312 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || in amdgpu_device_ip_need_full_reset()
3313 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_need_full_reset()
3314 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_need_full_reset()
3334 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_soft_reset() argument
3338 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_soft_reset()
3339 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_soft_reset()
3341 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_soft_reset()
3342 adev->ip_blocks[i].version->funcs->soft_reset) { in amdgpu_device_ip_soft_reset()
3343 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); in amdgpu_device_ip_soft_reset()
3363 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) in amdgpu_device_ip_post_soft_reset() argument
3367 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_post_soft_reset()
3368 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_post_soft_reset()
3370 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_post_soft_reset()
3371 adev->ip_blocks[i].version->funcs->post_soft_reset) in amdgpu_device_ip_post_soft_reset()
3372 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); in amdgpu_device_ip_post_soft_reset()
3392 static int amdgpu_device_recover_vram(struct amdgpu_device *adev) in amdgpu_device_recover_vram() argument
3398 if (amdgpu_sriov_runtime(adev)) in amdgpu_device_recover_vram()
3404 mutex_lock(&adev->shadow_list_lock); in amdgpu_device_recover_vram()
3405 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) { in amdgpu_device_recover_vram()
3432 mutex_unlock(&adev->shadow_list_lock); in amdgpu_device_recover_vram()
3457 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, in amdgpu_device_reset_sriov() argument
3463 r = amdgpu_virt_request_full_gpu(adev, true); in amdgpu_device_reset_sriov()
3465 r = amdgpu_virt_reset_gpu(adev); in amdgpu_device_reset_sriov()
3469 amdgpu_amdkfd_pre_reset(adev); in amdgpu_device_reset_sriov()
3472 r = amdgpu_device_ip_reinit_early_sriov(adev); in amdgpu_device_reset_sriov()
3477 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]); in amdgpu_device_reset_sriov()
3479 r = amdgpu_device_fw_loading(adev); in amdgpu_device_reset_sriov()
3484 r = amdgpu_device_ip_reinit_late_sriov(adev); in amdgpu_device_reset_sriov()
3488 amdgpu_irq_gpu_reset_resume_helper(adev); in amdgpu_device_reset_sriov()
3489 r = amdgpu_ib_ring_tests(adev); in amdgpu_device_reset_sriov()
3490 amdgpu_amdkfd_post_reset(adev); in amdgpu_device_reset_sriov()
3493 amdgpu_virt_init_data_exchange(adev); in amdgpu_device_reset_sriov()
3494 amdgpu_virt_release_full_gpu(adev, true); in amdgpu_device_reset_sriov()
3495 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { in amdgpu_device_reset_sriov()
3496 amdgpu_inc_vram_lost(adev); in amdgpu_device_reset_sriov()
3497 r = amdgpu_device_recover_vram(adev); in amdgpu_device_reset_sriov()
3511 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) in amdgpu_device_should_recover_gpu() argument
3513 if (!amdgpu_device_ip_check_soft_reset(adev)) { in amdgpu_device_should_recover_gpu()
3521 if (amdgpu_sriov_vf(adev)) in amdgpu_device_should_recover_gpu()
3525 switch (adev->asic_type) { in amdgpu_device_should_recover_gpu()
3553 static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, in amdgpu_device_pre_asic_reset() argument
3562 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_pre_asic_reset()
3575 if (!amdgpu_sriov_vf(adev)) { in amdgpu_device_pre_asic_reset()
3578 need_full_reset = amdgpu_device_ip_need_full_reset(adev); in amdgpu_device_pre_asic_reset()
3581 amdgpu_device_ip_pre_soft_reset(adev); in amdgpu_device_pre_asic_reset()
3582 r = amdgpu_device_ip_soft_reset(adev); in amdgpu_device_pre_asic_reset()
3583 amdgpu_device_ip_post_soft_reset(adev); in amdgpu_device_pre_asic_reset()
3584 if (r || amdgpu_device_ip_check_soft_reset(adev)) { in amdgpu_device_pre_asic_reset()
3591 r = amdgpu_device_ip_suspend(adev); in amdgpu_device_pre_asic_reset()
3725 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock) in amdgpu_device_lock_adev() argument
3728 if (!mutex_trylock(&adev->lock_reset)) in amdgpu_device_lock_adev()
3731 mutex_lock(&adev->lock_reset); in amdgpu_device_lock_adev()
3733 atomic_inc(&adev->gpu_reset_counter); in amdgpu_device_lock_adev()
3734 adev->in_gpu_reset = 1; in amdgpu_device_lock_adev()
3735 switch (amdgpu_asic_reset_method(adev)) { in amdgpu_device_lock_adev()
3737 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; in amdgpu_device_lock_adev()
3740 adev->mp1_state = PP_MP1_STATE_RESET; in amdgpu_device_lock_adev()
3743 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_lock_adev()
3747 if (!amdgpu_sriov_vf(adev)) in amdgpu_device_lock_adev()
3748 amdgpu_amdkfd_pre_reset(adev); in amdgpu_device_lock_adev()
3753 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) in amdgpu_device_unlock_adev() argument
3756 if (!amdgpu_sriov_vf(adev)) in amdgpu_device_unlock_adev()
3757 amdgpu_amdkfd_post_reset(adev); in amdgpu_device_unlock_adev()
3758 amdgpu_vf_error_trans_all(adev); in amdgpu_device_unlock_adev()
3759 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_unlock_adev()
3760 adev->in_gpu_reset = 0; in amdgpu_device_unlock_adev()
3761 mutex_unlock(&adev->lock_reset); in amdgpu_device_unlock_adev()
3776 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, in amdgpu_device_gpu_recover() argument
3788 dev_info(adev->dev, "GPU reset begin!\n"); in amdgpu_device_gpu_recover()
3790 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_gpu_recover()
3792 hive = amdgpu_get_xgmi_hive(adev, false); in amdgpu_device_gpu_recover()
3809 if (!amdgpu_device_lock_adev(adev, !hive)) { in amdgpu_device_gpu_recover()
3816 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_gpu_recover()
3818 amdgpu_device_unlock_adev(adev); in amdgpu_device_gpu_recover()
3829 list_add_tail(&adev->gmc.xgmi.head, &device_list); in amdgpu_device_gpu_recover()
3867 if (!amdgpu_device_ip_need_full_reset(adev)) in amdgpu_device_gpu_recover()
3871 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); in amdgpu_device_gpu_recover()
3877 r = amdgpu_device_pre_asic_reset(adev, job, &need_full_reset); in amdgpu_device_gpu_recover()
3881 r, adev->ddev->unique); in amdgpu_device_gpu_recover()
3882 adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
3888 if (tmp_adev == adev) in amdgpu_device_gpu_recover()
3905 if (amdgpu_sriov_vf(adev)) { in amdgpu_device_gpu_recover()
3906 r = amdgpu_device_reset_sriov(adev, job ? false : true); in amdgpu_device_gpu_recover()
3908 adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
3940 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
3943 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
3953 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); in amdgpu_device_gpu_recover()
3966 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) in amdgpu_device_get_pcie_info() argument
3973 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; in amdgpu_device_get_pcie_info()
3976 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; in amdgpu_device_get_pcie_info()
3979 if (pci_is_root_bus(adev->pdev->bus)) { in amdgpu_device_get_pcie_info()
3980 if (adev->pm.pcie_gen_mask == 0) in amdgpu_device_get_pcie_info()
3981 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; in amdgpu_device_get_pcie_info()
3982 if (adev->pm.pcie_mlw_mask == 0) in amdgpu_device_get_pcie_info()
3983 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
3987 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) in amdgpu_device_get_pcie_info()
3990 pcie_bandwidth_available(adev->pdev, NULL, in amdgpu_device_get_pcie_info()
3993 if (adev->pm.pcie_gen_mask == 0) { in amdgpu_device_get_pcie_info()
3995 pdev = adev->pdev; in amdgpu_device_get_pcie_info()
3998 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4003 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4008 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4012 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4015 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
4019 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4023 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4028 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4032 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
4035 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
4039 if (adev->pm.pcie_mlw_mask == 0) { in amdgpu_device_get_pcie_info()
4041 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
4045 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
4054 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
4062 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
4069 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
4075 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
4080 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
4084 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()