121 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_hotplug_work_func()  local
123 	struct drm_device *dev = adev_to_drm(adev);  in amdgpu_hotplug_work_func()
145 void amdgpu_irq_disable_all(struct amdgpu_device *adev)  in amdgpu_irq_disable_all()  argument
151 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
153 		if (!adev->irq.client[i].sources)  in amdgpu_irq_disable_all()
157 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_disable_all()
164 				r = src->funcs->set(adev, src, k,  in amdgpu_irq_disable_all()
172 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
189 	struct amdgpu_device *adev = drm_to_adev(dev);  in amdgpu_irq_handler()  local
192 	ret = amdgpu_ih_process(adev, &adev->irq.ih);  in amdgpu_irq_handler()
196 	amdgpu_ras_interrupt_fatal_error_handler(adev);  in amdgpu_irq_handler()
210 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih1()  local
213 	amdgpu_ih_process(adev, &adev->irq.ih1);  in amdgpu_irq_handle_ih1()
225 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih2()  local
228 	amdgpu_ih_process(adev, &adev->irq.ih2);  in amdgpu_irq_handle_ih2()
240 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,  in amdgpu_irq_handle_ih_soft()  local
243 	amdgpu_ih_process(adev, &adev->irq.ih_soft);  in amdgpu_irq_handle_ih_soft()
257 static bool amdgpu_msi_ok(struct amdgpu_device *adev)  in amdgpu_msi_ok()  argument
267 static void amdgpu_restore_msix(struct amdgpu_device *adev)  in amdgpu_restore_msix()  argument
271 	pci_read_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);  in amdgpu_restore_msix()
277 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
279 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
293 int amdgpu_irq_init(struct amdgpu_device *adev)  in amdgpu_irq_init()  argument
298 	spin_lock_init(&adev->irq.lock);  in amdgpu_irq_init()
301 	adev->irq.msi_enabled = false;  in amdgpu_irq_init()
303 	if (amdgpu_msi_ok(adev)) {  in amdgpu_irq_init()
304 		int nvec = pci_msix_vec_count(adev->pdev);  in amdgpu_irq_init()
313 		nvec = pci_alloc_irq_vectors(adev->pdev, 1, 1, flags);  in amdgpu_irq_init()
315 			adev->irq.msi_enabled = true;  in amdgpu_irq_init()
316 			dev_dbg(adev->dev, "using MSI/MSI-X.\n");  in amdgpu_irq_init()
320 	if (!amdgpu_device_has_dc_support(adev)) {  in amdgpu_irq_init()
321 		if (!adev->enable_virtual_display)  in amdgpu_irq_init()
324 			adev_to_drm(adev)->vblank_disable_immediate = true;  in amdgpu_irq_init()
326 		r = drm_vblank_init(adev_to_drm(adev), adev->mode_info.num_crtc);  in amdgpu_irq_init()
331 		INIT_WORK(&adev->hotplug_work,  in amdgpu_irq_init()
335 	INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1);  in amdgpu_irq_init()
336 	INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2);  in amdgpu_irq_init()
337 	INIT_WORK(&adev->irq.ih_soft_work, amdgpu_irq_handle_ih_soft);  in amdgpu_irq_init()
340 	r = pci_irq_vector(adev->pdev, 0);  in amdgpu_irq_init()
346 	r = request_irq(irq, amdgpu_irq_handler, IRQF_SHARED, adev_to_drm(adev)->driver->name,  in amdgpu_irq_init()
347 			adev_to_drm(adev));  in amdgpu_irq_init()
349 		if (!amdgpu_device_has_dc_support(adev))  in amdgpu_irq_init()
350 			flush_work(&adev->hotplug_work);  in amdgpu_irq_init()
353 	adev->irq.installed = true;  in amdgpu_irq_init()
354 	adev->irq.irq = irq;  in amdgpu_irq_init()
355 	adev_to_drm(adev)->max_vblank_count = 0x00ffffff;  in amdgpu_irq_init()
362 void amdgpu_irq_fini_hw(struct amdgpu_device *adev)  in amdgpu_irq_fini_hw()  argument
364 	if (adev->irq.installed) {  in amdgpu_irq_fini_hw()
365 		free_irq(adev->irq.irq, adev_to_drm(adev));  in amdgpu_irq_fini_hw()
366 		adev->irq.installed = false;  in amdgpu_irq_fini_hw()
367 		if (adev->irq.msi_enabled)  in amdgpu_irq_fini_hw()
368 			pci_free_irq_vectors(adev->pdev);  in amdgpu_irq_fini_hw()
370 		if (!amdgpu_device_has_dc_support(adev))  in amdgpu_irq_fini_hw()
371 			flush_work(&adev->hotplug_work);  in amdgpu_irq_fini_hw()
374 	amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft);  in amdgpu_irq_fini_hw()
375 	amdgpu_ih_ring_fini(adev, &adev->irq.ih);  in amdgpu_irq_fini_hw()
376 	amdgpu_ih_ring_fini(adev, &adev->irq.ih1);  in amdgpu_irq_fini_hw()
377 	amdgpu_ih_ring_fini(adev, &adev->irq.ih2);  in amdgpu_irq_fini_hw()
389 void amdgpu_irq_fini_sw(struct amdgpu_device *adev)  in amdgpu_irq_fini_sw()  argument
394 		if (!adev->irq.client[i].sources)  in amdgpu_irq_fini_sw()
398 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_fini_sw()
406 		kfree(adev->irq.client[i].sources);  in amdgpu_irq_fini_sw()
407 		adev->irq.client[i].sources = NULL;  in amdgpu_irq_fini_sw()
424 int amdgpu_irq_add_id(struct amdgpu_device *adev,  in amdgpu_irq_add_id()  argument
437 	if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_add_id()
438 		adev->irq.client[client_id].sources =  in amdgpu_irq_add_id()
442 		if (!adev->irq.client[client_id].sources)  in amdgpu_irq_add_id()
446 	if (adev->irq.client[client_id].sources[src_id] != NULL)  in amdgpu_irq_add_id()
460 	adev->irq.client[client_id].sources[src_id] = source;  in amdgpu_irq_add_id()
472 void amdgpu_irq_dispatch(struct amdgpu_device *adev,  in amdgpu_irq_dispatch()  argument
484 	amdgpu_ih_decode_iv(adev, &entry);  in amdgpu_irq_dispatch()
486 	trace_amdgpu_iv(ih - &adev->irq.ih, &entry);  in amdgpu_irq_dispatch()
498 		   adev->irq.virq[src_id]) {  in amdgpu_irq_dispatch()
499 		generic_handle_domain_irq(adev->irq.domain, src_id);  in amdgpu_irq_dispatch()
501 	} else if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_dispatch()
505 	} else if ((src = adev->irq.client[client_id].sources[src_id])) {  in amdgpu_irq_dispatch()
506 		r = src->funcs->process(adev, src, &entry);  in amdgpu_irq_dispatch()
518 		amdgpu_amdkfd_interrupt(adev, entry.iv_entry);  in amdgpu_irq_dispatch()
534 void amdgpu_irq_delegate(struct amdgpu_device *adev,  in amdgpu_irq_delegate()  argument
538 	amdgpu_ih_ring_write(&adev->irq.ih_soft, entry->iv_entry, num_dw);  in amdgpu_irq_delegate()
539 	schedule_work(&adev->irq.ih_soft_work);  in amdgpu_irq_delegate()
551 int amdgpu_irq_update(struct amdgpu_device *adev,  in amdgpu_irq_update()  argument
558 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
562 	if (amdgpu_irq_enabled(adev, src, type))  in amdgpu_irq_update()
567 	r = src->funcs->set(adev, src, type, state);  in amdgpu_irq_update()
568 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
580 void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)  in amdgpu_irq_gpu_reset_resume_helper()  argument
584 	if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))  in amdgpu_irq_gpu_reset_resume_helper()
585 		amdgpu_restore_msix(adev);  in amdgpu_irq_gpu_reset_resume_helper()
588 		if (!adev->irq.client[i].sources)  in amdgpu_irq_gpu_reset_resume_helper()
592 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_gpu_reset_resume_helper()
597 				amdgpu_irq_update(adev, src, k);  in amdgpu_irq_gpu_reset_resume_helper()
614 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_get()  argument
617 	if (!adev->irq.installed)  in amdgpu_irq_get()
627 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_get()
644 int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_put()  argument
647 	if (!adev->irq.installed)  in amdgpu_irq_put()
657 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_put()
675 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_enabled()  argument
678 	if (!adev->irq.installed)  in amdgpu_irq_enabled()
748 int amdgpu_irq_add_domain(struct amdgpu_device *adev)  in amdgpu_irq_add_domain()  argument
750 	adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,  in amdgpu_irq_add_domain()
751 						 &amdgpu_hw_irqdomain_ops, adev);  in amdgpu_irq_add_domain()
752 	if (!adev->irq.domain) {  in amdgpu_irq_add_domain()
768 void amdgpu_irq_remove_domain(struct amdgpu_device *adev)  in amdgpu_irq_remove_domain()  argument
770 	if (adev->irq.domain) {  in amdgpu_irq_remove_domain()
771 		irq_domain_remove(adev->irq.domain);  in amdgpu_irq_remove_domain()
772 		adev->irq.domain = NULL;  in amdgpu_irq_remove_domain()
789 unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)  in amdgpu_irq_create_mapping()  argument
791 	adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);  in amdgpu_irq_create_mapping()
793 	return adev->irq.virq[src_id];  in amdgpu_irq_create_mapping()