44 	struct kvm_kernel_irqfd *irqfd =  in irqfd_inject()  local
46 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
48 	if (!irqfd->resampler) {  in irqfd_inject()
49 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
51 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
55 			    irqfd->gsi, 1, false);  in irqfd_inject()
68 	struct kvm_kernel_irqfd *irqfd;  in irqfd_resampler_ack()  local
80 	list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)  in irqfd_resampler_ack()
81 		eventfd_signal(irqfd->resamplefd, 1);  in irqfd_resampler_ack()
87 irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)  in irqfd_resampler_shutdown()  argument
89 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
94 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
114 	struct kvm_kernel_irqfd *irqfd =  in irqfd_shutdown()  local
116 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()
126 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
132 	flush_work(&irqfd->inject);  in irqfd_shutdown()
134 	if (irqfd->resampler) {  in irqfd_shutdown()
135 		irqfd_resampler_shutdown(irqfd);  in irqfd_shutdown()
136 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
143 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
145 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
146 	kfree(irqfd);  in irqfd_shutdown()
152 irqfd_is_active(struct kvm_kernel_irqfd *irqfd)  in irqfd_is_active()  argument
154 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
163 irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)  in irqfd_deactivate()  argument
165 	BUG_ON(!irqfd_is_active(irqfd));  in irqfd_deactivate()
167 	list_del_init(&irqfd->list);  in irqfd_deactivate()
169 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
187 	struct kvm_kernel_irqfd *irqfd =  in irqfd_wakeup()  local
191 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
198 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
199 			irq = irqfd->irq_entry;  in irqfd_wakeup()
200 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
205 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
224 		if (irqfd_is_active(irqfd))  in irqfd_wakeup()
225 			irqfd_deactivate(irqfd);  in irqfd_wakeup()
237 	struct kvm_kernel_irqfd *irqfd =  in irqfd_ptable_queue_proc()  local
239 	add_wait_queue(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
243 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)  in irqfd_update()  argument
249 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
251 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
255 		irqfd->irq_entry = *e;  in irqfd_update()
257 		irqfd->irq_entry.type = 0;  in irqfd_update()
259 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
284 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_assign()  local
297 	irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL_ACCOUNT);  in kvm_irqfd_assign()
298 	if (!irqfd)  in kvm_irqfd_assign()
301 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
302 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
303 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
304 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
305 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
306 	seqcount_init(&irqfd->irq_entry_sc);  in kvm_irqfd_assign()
320 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
331 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
332 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
338 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
339 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
344 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
355 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
362 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
365 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
375 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
376 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
382 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
391 	irqfd_update(kvm, irqfd);  in kvm_irqfd_assign()
393 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
401 	events = vfs_poll(f.file, &irqfd->pt);  in kvm_irqfd_assign()
404 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
408 		irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
409 		irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
410 		irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
411 		irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
412 		irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
413 		ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
416 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
430 	if (irqfd->resampler)  in kvm_irqfd_assign()
431 		irqfd_resampler_shutdown(irqfd);  in kvm_irqfd_assign()
442 	kfree(irqfd);  in kvm_irqfd_assign()
529 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_deassign()  local
538 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
539 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
546 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
547 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
548 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
549 			irqfd_deactivate(irqfd);  in kvm_irqfd_deassign()
585 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_release()  local
589 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
590 		irqfd_deactivate(irqfd);  in kvm_irqfd_release()
608 	struct kvm_kernel_irqfd *irqfd;  in kvm_irq_routing_update()  local
612 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
613 		irqfd_update(kvm, irqfd);  in kvm_irq_routing_update()
616 		if (irqfd->producer) {  in kvm_irq_routing_update()
618 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
619 					irqfd->gsi, 1);  in kvm_irq_routing_update()