236 static void vhost_worker_queue(struct vhost_worker *worker,  in vhost_worker_queue()  argument
244 		llist_add(&work->node, &worker->work_list);  in vhost_worker_queue()
245 		vhost_task_wake(worker->vtsk);  in vhost_worker_queue()
251 	struct vhost_worker *worker;  in vhost_vq_work_queue()  local
255 	worker = rcu_dereference(vq->worker);  in vhost_vq_work_queue()
256 	if (worker) {  in vhost_vq_work_queue()
258 		vhost_worker_queue(worker, work);  in vhost_vq_work_queue()
285 static void vhost_worker_flush(struct vhost_worker *worker)  in vhost_worker_flush()  argument
292 	vhost_worker_queue(worker, &flush.work);  in vhost_worker_flush()
298 	struct vhost_worker *worker;  in vhost_dev_flush()  local
301 	xa_for_each(&dev->worker_xa, i, worker) {  in vhost_dev_flush()
302 		mutex_lock(&worker->mutex);  in vhost_dev_flush()
303 		if (!worker->attachment_cnt) {  in vhost_dev_flush()
304 			mutex_unlock(&worker->mutex);  in vhost_dev_flush()
307 		vhost_worker_flush(worker);  in vhost_dev_flush()
308 		mutex_unlock(&worker->mutex);  in vhost_dev_flush()
316 	struct vhost_worker *worker;  in vhost_vq_has_work()  local
320 	worker = rcu_dereference(vq->worker);  in vhost_vq_has_work()
321 	if (worker && !llist_empty(&worker->work_list))  in vhost_vq_has_work()
390 	rcu_assign_pointer(vq->worker, NULL);  in vhost_vq_reset()
397 	struct vhost_worker *worker = data;  in vhost_worker()  local
401 	node = llist_del_all(&worker->work_list);  in vhost_worker()
410 			kcov_remote_start_common(worker->kcov_handle);  in vhost_worker()
592 				 struct vhost_worker *worker)  in vhost_worker_destroy()  argument
594 	if (!worker)  in vhost_worker_destroy()
597 	WARN_ON(!llist_empty(&worker->work_list));  in vhost_worker_destroy()
598 	xa_erase(&dev->worker_xa, worker->id);  in vhost_worker_destroy()
599 	vhost_task_stop(worker->vtsk);  in vhost_worker_destroy()
600 	kfree(worker);  in vhost_worker_destroy()
605 	struct vhost_worker *worker;  in vhost_workers_free()  local
612 		rcu_assign_pointer(dev->vqs[i]->worker, NULL);  in vhost_workers_free()
617 	xa_for_each(&dev->worker_xa, i, worker)  in vhost_workers_free()
618 		vhost_worker_destroy(dev, worker);  in vhost_workers_free()
624 	struct vhost_worker *worker;  in vhost_worker_create()  local
630 	worker = kzalloc(sizeof(*worker), GFP_KERNEL_ACCOUNT);  in vhost_worker_create()
631 	if (!worker)  in vhost_worker_create()
636 	vtsk = vhost_task_create(vhost_worker, worker, name);  in vhost_worker_create()
640 	mutex_init(&worker->mutex);  in vhost_worker_create()
641 	init_llist_head(&worker->work_list);  in vhost_worker_create()
642 	worker->kcov_handle = kcov_common_handle();  in vhost_worker_create()
643 	worker->vtsk = vtsk;  in vhost_worker_create()
647 	ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL);  in vhost_worker_create()
650 	worker->id = id;  in vhost_worker_create()
652 	return worker;  in vhost_worker_create()
657 	kfree(worker);  in vhost_worker_create()
663 				     struct vhost_worker *worker)  in __vhost_vq_attach_worker()  argument
667 	old_worker = rcu_dereference_check(vq->worker,  in __vhost_vq_attach_worker()
670 	mutex_lock(&worker->mutex);  in __vhost_vq_attach_worker()
671 	worker->attachment_cnt++;  in __vhost_vq_attach_worker()
672 	mutex_unlock(&worker->mutex);  in __vhost_vq_attach_worker()
673 	rcu_assign_pointer(vq->worker, worker);  in __vhost_vq_attach_worker()
718 	struct vhost_worker *worker;  in vhost_vq_attach_worker()  local
723 	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);  in vhost_vq_attach_worker()
724 	if (!worker || worker->id != info->worker_id)  in vhost_vq_attach_worker()
727 	__vhost_vq_attach_worker(vq, worker);  in vhost_vq_attach_worker()
735 	struct vhost_worker *worker;  in vhost_new_worker()  local
737 	worker = vhost_worker_create(dev);  in vhost_new_worker()
738 	if (!worker)  in vhost_new_worker()
741 	info->worker_id = worker->id;  in vhost_new_worker()
750 	struct vhost_worker *worker;  in vhost_free_worker()  local
752 	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);  in vhost_free_worker()
753 	if (!worker || worker->id != info->worker_id)  in vhost_free_worker()
756 	mutex_lock(&worker->mutex);  in vhost_free_worker()
757 	if (worker->attachment_cnt) {  in vhost_free_worker()
758 		mutex_unlock(&worker->mutex);  in vhost_free_worker()
761 	mutex_unlock(&worker->mutex);  in vhost_free_worker()
763 	vhost_worker_destroy(dev, worker);  in vhost_free_worker()
794 	struct vhost_worker *worker;  in vhost_worker_ioctl()  local
842 		worker = rcu_dereference_check(vq->worker,  in vhost_worker_ioctl()
844 		if (!worker) {  in vhost_worker_ioctl()
850 		ring_worker.worker_id = worker->id;  in vhost_worker_ioctl()
867 	struct vhost_worker *worker;  in vhost_dev_set_owner()  local
889 		worker = vhost_worker_create(dev);  in vhost_dev_set_owner()
890 		if (!worker) {  in vhost_dev_set_owner()
896 			__vhost_vq_attach_worker(dev->vqs[i], worker);  in vhost_dev_set_owner()