50 	struct kvm_kernel_irqfd *irqfd =  in irqfd_inject()  local
52 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
54 	if (!irqfd->resampler) {  in irqfd_inject()
55 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
57 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
61 			    irqfd->gsi, 1, false);  in irqfd_inject()
74 	struct kvm_kernel_irqfd *irqfd;  in irqfd_resampler_ack()  local
86 	list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)  in irqfd_resampler_ack()
87 		eventfd_signal(irqfd->resamplefd, 1);  in irqfd_resampler_ack()
93 irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)  in irqfd_resampler_shutdown()  argument
95 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
100 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
120 	struct kvm_kernel_irqfd *irqfd =  in irqfd_shutdown()  local
122 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()
132 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
138 	flush_work(&irqfd->inject);  in irqfd_shutdown()
140 	if (irqfd->resampler) {  in irqfd_shutdown()
141 		irqfd_resampler_shutdown(irqfd);  in irqfd_shutdown()
142 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
149 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
151 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
152 	kfree(irqfd);  in irqfd_shutdown()
158 irqfd_is_active(struct kvm_kernel_irqfd *irqfd)  in irqfd_is_active()  argument
160 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
169 irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)  in irqfd_deactivate()  argument
171 	BUG_ON(!irqfd_is_active(irqfd));  in irqfd_deactivate()
173 	list_del_init(&irqfd->list);  in irqfd_deactivate()
175 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
193 	struct kvm_kernel_irqfd *irqfd =  in irqfd_wakeup()  local
197 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
204 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
205 			irq = irqfd->irq_entry;  in irqfd_wakeup()
206 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
211 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
230 		if (irqfd_is_active(irqfd))  in irqfd_wakeup()
231 			irqfd_deactivate(irqfd);  in irqfd_wakeup()
243 	struct kvm_kernel_irqfd *irqfd =  in irqfd_ptable_queue_proc()  local
245 	add_wait_queue(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
249 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)  in irqfd_update()  argument
255 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
257 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
261 		irqfd->irq_entry = *e;  in irqfd_update()
263 		irqfd->irq_entry.type = 0;  in irqfd_update()
265 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
290 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_assign()  local
300 	irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL);  in kvm_irqfd_assign()
301 	if (!irqfd)  in kvm_irqfd_assign()
304 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
305 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
306 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
307 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
308 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
309 	seqcount_init(&irqfd->irq_entry_sc);  in kvm_irqfd_assign()
323 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
334 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
335 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
341 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
342 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
347 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
357 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
364 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
367 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
377 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
378 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
384 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
393 	irqfd_update(kvm, irqfd);  in kvm_irqfd_assign()
395 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
403 	events = vfs_poll(f.file, &irqfd->pt);  in kvm_irqfd_assign()
406 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
410 		irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
411 		irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
412 		irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
413 		irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
414 		irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
415 		ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
418 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
432 	if (irqfd->resampler)  in kvm_irqfd_assign()
433 		irqfd_resampler_shutdown(irqfd);  in kvm_irqfd_assign()
444 	kfree(irqfd);  in kvm_irqfd_assign()
531 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_deassign()  local
540 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
541 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
548 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
549 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
550 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
551 			irqfd_deactivate(irqfd);  in kvm_irqfd_deassign()
587 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_release()  local
591 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
592 		irqfd_deactivate(irqfd);  in kvm_irqfd_release()
610 	struct kvm_kernel_irqfd *irqfd;  in kvm_irq_routing_update()  local
614 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
615 		irqfd_update(kvm, irqfd);  in kvm_irq_routing_update()
618 		if (irqfd->producer) {  in kvm_irq_routing_update()
620 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
621 					irqfd->gsi, 1);  in kvm_irq_routing_update()