121 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_hotplug_work_func()  local
123 	struct drm_device *dev = adev_to_drm(adev);  in amdgpu_hotplug_work_func()
145 void amdgpu_irq_disable_all(struct amdgpu_device *adev)  in amdgpu_irq_disable_all()  argument
151 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
153 		if (!adev->irq.client[i].sources)  in amdgpu_irq_disable_all()
157 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_disable_all()
164 				r = src->funcs->set(adev, src, k,  in amdgpu_irq_disable_all()
172 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
189 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_irq_handler()  local
192 	ret = amdgpu_ih_process(adev, &adev->irq.ih);  in amdgpu_irq_handler()
201 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF)) {  in amdgpu_irq_handler()
202 		if (adev->nbio.ras_funcs &&  in amdgpu_irq_handler()
203 		    adev->nbio.ras_funcs->handle_ras_controller_intr_no_bifring)  in amdgpu_irq_handler()
204 			adev->nbio.ras_funcs->handle_ras_controller_intr_no_bifring(adev);  in amdgpu_irq_handler()
206 		if (adev->nbio.ras_funcs &&  in amdgpu_irq_handler()
207 		    adev->nbio.ras_funcs->handle_ras_err_event_athub_intr_no_bifring)  in amdgpu_irq_handler()
208 			adev->nbio.ras_funcs->handle_ras_err_event_athub_intr_no_bifring(adev);  in amdgpu_irq_handler()
223 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih1()  local
226 	amdgpu_ih_process(adev, &adev->irq.ih1);  in amdgpu_irq_handle_ih1()
238 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih2()  local
241 	amdgpu_ih_process(adev, &adev->irq.ih2);  in amdgpu_irq_handle_ih2()
253 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih_soft()  local
256 	amdgpu_ih_process(adev, &adev->irq.ih_soft);  in amdgpu_irq_handle_ih_soft()
270 static bool amdgpu_msi_ok(struct amdgpu_device *adev)  in amdgpu_msi_ok()  argument
280 static void amdgpu_restore_msix(struct amdgpu_device *adev)  in amdgpu_restore_msix()  argument
284 	pci_read_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);  in amdgpu_restore_msix()
290 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
292 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
306 int amdgpu_irq_init(struct amdgpu_device *adev)  in amdgpu_irq_init()  argument
311 	spin_lock_init(&adev->irq.lock);  in amdgpu_irq_init()
314 	adev->irq.msi_enabled = false;  in amdgpu_irq_init()
316 	if (amdgpu_msi_ok(adev)) {  in amdgpu_irq_init()
317 		int nvec = pci_msix_vec_count(adev->pdev);  in amdgpu_irq_init()
326 		nvec = pci_alloc_irq_vectors(adev->pdev, 1, 1, flags);  in amdgpu_irq_init()
328 			adev->irq.msi_enabled = true;  in amdgpu_irq_init()
329 			dev_dbg(adev->dev, "using MSI/MSI-X.\n");  in amdgpu_irq_init()
333 	if (!amdgpu_device_has_dc_support(adev)) {  in amdgpu_irq_init()
334 		if (!adev->enable_virtual_display)  in amdgpu_irq_init()
337 			adev_to_drm(adev)->vblank_disable_immediate = true;  in amdgpu_irq_init()
339 		r = drm_vblank_init(adev_to_drm(adev), adev->mode_info.num_crtc);  in amdgpu_irq_init()
344 		INIT_WORK(&adev->hotplug_work,  in amdgpu_irq_init()
348 	INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1);  in amdgpu_irq_init()
349 	INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2);  in amdgpu_irq_init()
350 	INIT_WORK(&adev->irq.ih_soft_work, amdgpu_irq_handle_ih_soft);  in amdgpu_irq_init()
353 	r = pci_irq_vector(adev->pdev, 0);  in amdgpu_irq_init()
359 	r = request_irq(irq, amdgpu_irq_handler, IRQF_SHARED, adev_to_drm(adev)->driver->name,  in amdgpu_irq_init()
360 			adev_to_drm(adev));  in amdgpu_irq_init()
362 		if (!amdgpu_device_has_dc_support(adev))  in amdgpu_irq_init()
363 			flush_work(&adev->hotplug_work);  in amdgpu_irq_init()
366 	adev->irq.installed = true;  in amdgpu_irq_init()
367 	adev->irq.irq = irq;  in amdgpu_irq_init()
368 	adev_to_drm(adev)->max_vblank_count = 0x00ffffff;  in amdgpu_irq_init()
375 void amdgpu_irq_fini_hw(struct amdgpu_device *adev)  in amdgpu_irq_fini_hw()  argument
377 	if (adev->irq.installed) {  in amdgpu_irq_fini_hw()
378 		free_irq(adev->irq.irq, adev_to_drm(adev));  in amdgpu_irq_fini_hw()
379 		adev->irq.installed = false;  in amdgpu_irq_fini_hw()
380 		if (adev->irq.msi_enabled)  in amdgpu_irq_fini_hw()
381 			pci_free_irq_vectors(adev->pdev);  in amdgpu_irq_fini_hw()
383 		if (!amdgpu_device_has_dc_support(adev))  in amdgpu_irq_fini_hw()
384 			flush_work(&adev->hotplug_work);  in amdgpu_irq_fini_hw()
387 	amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft);  in amdgpu_irq_fini_hw()
388 	amdgpu_ih_ring_fini(adev, &adev->irq.ih);  in amdgpu_irq_fini_hw()
389 	amdgpu_ih_ring_fini(adev, &adev->irq.ih1);  in amdgpu_irq_fini_hw()
390 	amdgpu_ih_ring_fini(adev, &adev->irq.ih2);  in amdgpu_irq_fini_hw()
402 void amdgpu_irq_fini_sw(struct amdgpu_device *adev)  in amdgpu_irq_fini_sw()  argument
407 		if (!adev->irq.client[i].sources)  in amdgpu_irq_fini_sw()
411 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_fini_sw()
419 		kfree(adev->irq.client[i].sources);  in amdgpu_irq_fini_sw()
420 		adev->irq.client[i].sources = NULL;  in amdgpu_irq_fini_sw()
437 int amdgpu_irq_add_id(struct amdgpu_device *adev,  in amdgpu_irq_add_id()  argument
450 	if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_add_id()
451 		adev->irq.client[client_id].sources =  in amdgpu_irq_add_id()
455 		if (!adev->irq.client[client_id].sources)  in amdgpu_irq_add_id()
459 	if (adev->irq.client[client_id].sources[src_id] != NULL)  in amdgpu_irq_add_id()
473 	adev->irq.client[client_id].sources[src_id] = source;  in amdgpu_irq_add_id()
485 void amdgpu_irq_dispatch(struct amdgpu_device *adev,  in amdgpu_irq_dispatch()  argument
497 	amdgpu_ih_decode_iv(adev, &entry);  in amdgpu_irq_dispatch()
499 	trace_amdgpu_iv(ih - &adev->irq.ih, &entry);  in amdgpu_irq_dispatch()
511 		   adev->irq.virq[src_id]) {  in amdgpu_irq_dispatch()
512 		generic_handle_domain_irq(adev->irq.domain, src_id);  in amdgpu_irq_dispatch()
514 	} else if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_dispatch()
518 	} else if ((src = adev->irq.client[client_id].sources[src_id])) {  in amdgpu_irq_dispatch()
519 		r = src->funcs->process(adev, src, &entry);  in amdgpu_irq_dispatch()
531 		amdgpu_amdkfd_interrupt(adev, entry.iv_entry);  in amdgpu_irq_dispatch()
544 void amdgpu_irq_delegate(struct amdgpu_device *adev,  in amdgpu_irq_delegate()  argument
548 	amdgpu_ih_ring_write(&adev->irq.ih_soft, entry->iv_entry, num_dw);  in amdgpu_irq_delegate()
549 	schedule_work(&adev->irq.ih_soft_work);  in amdgpu_irq_delegate()
561 int amdgpu_irq_update(struct amdgpu_device *adev,  in amdgpu_irq_update()  argument
568 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
572 	if (amdgpu_irq_enabled(adev, src, type))  in amdgpu_irq_update()
577 	r = src->funcs->set(adev, src, type, state);  in amdgpu_irq_update()
578 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
590 void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)  in amdgpu_irq_gpu_reset_resume_helper()  argument
594 	if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))  in amdgpu_irq_gpu_reset_resume_helper()
595 		amdgpu_restore_msix(adev);  in amdgpu_irq_gpu_reset_resume_helper()
598 		if (!adev->irq.client[i].sources)  in amdgpu_irq_gpu_reset_resume_helper()
602 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_gpu_reset_resume_helper()
607 				amdgpu_irq_update(adev, src, k);  in amdgpu_irq_gpu_reset_resume_helper()
624 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_get()  argument
627 	if (!adev->irq.installed)  in amdgpu_irq_get()
637 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_get()
654 int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_put()  argument
657 	if (!adev->irq.installed)  in amdgpu_irq_put()
667 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_put()
685 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_enabled()  argument
688 	if (!adev->irq.installed)  in amdgpu_irq_enabled()
758 int amdgpu_irq_add_domain(struct amdgpu_device *adev)  in amdgpu_irq_add_domain()  argument
760 	adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,  in amdgpu_irq_add_domain()
761 						 &amdgpu_hw_irqdomain_ops, adev);  in amdgpu_irq_add_domain()
762 	if (!adev->irq.domain) {  in amdgpu_irq_add_domain()
778 void amdgpu_irq_remove_domain(struct amdgpu_device *adev)  in amdgpu_irq_remove_domain()  argument
780 	if (adev->irq.domain) {  in amdgpu_irq_remove_domain()
781 		irq_domain_remove(adev->irq.domain);  in amdgpu_irq_remove_domain()
782 		adev->irq.domain = NULL;  in amdgpu_irq_remove_domain()
799 unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)  in amdgpu_irq_create_mapping()  argument
801 	adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);  in amdgpu_irq_create_mapping()
803 	return adev->irq.virq[src_id];  in amdgpu_irq_create_mapping()