643 void __kthread_init_worker(struct kthread_worker *worker,  in __kthread_init_worker()  argument
647 	memset(worker, 0, sizeof(struct kthread_worker));  in __kthread_init_worker()
648 	raw_spin_lock_init(&worker->lock);  in __kthread_init_worker()
649 	lockdep_set_class_and_name(&worker->lock, key, name);  in __kthread_init_worker()
650 	INIT_LIST_HEAD(&worker->work_list);  in __kthread_init_worker()
651 	INIT_LIST_HEAD(&worker->delayed_work_list);  in __kthread_init_worker()
672 	struct kthread_worker *worker = worker_ptr;  in kthread_worker_fn()  local
679 	WARN_ON(worker->task && worker->task != current);  in kthread_worker_fn()
680 	worker->task = current;  in kthread_worker_fn()
682 	if (worker->flags & KTW_FREEZABLE)  in kthread_worker_fn()
690 		raw_spin_lock_irq(&worker->lock);  in kthread_worker_fn()
691 		worker->task = NULL;  in kthread_worker_fn()
692 		raw_spin_unlock_irq(&worker->lock);  in kthread_worker_fn()
697 	raw_spin_lock_irq(&worker->lock);  in kthread_worker_fn()
698 	if (!list_empty(&worker->work_list)) {  in kthread_worker_fn()
699 		work = list_first_entry(&worker->work_list,  in kthread_worker_fn()
703 	worker->current_work = work;  in kthread_worker_fn()
704 	raw_spin_unlock_irq(&worker->lock);  in kthread_worker_fn()
722 	struct kthread_worker *worker;  in __kthread_create_worker()  local
726 	worker = kzalloc(sizeof(*worker), GFP_KERNEL);  in __kthread_create_worker()
727 	if (!worker)  in __kthread_create_worker()
730 	kthread_init_worker(worker);  in __kthread_create_worker()
735 	task = __kthread_create_on_node(kthread_worker_fn, worker,  in __kthread_create_worker()
743 	worker->flags = flags;  in __kthread_create_worker()
744 	worker->task = task;  in __kthread_create_worker()
746 	return worker;  in __kthread_create_worker()
749 	kfree(worker);  in __kthread_create_worker()
765 	struct kthread_worker *worker;  in kthread_create_worker()  local
769 	worker = __kthread_create_worker(-1, flags, namefmt, args);  in kthread_create_worker()
772 	return worker;  in kthread_create_worker()
797 	struct kthread_worker *worker;  in kthread_create_worker_on_cpu()  local
801 	worker = __kthread_create_worker(cpu, flags, namefmt, args);  in kthread_create_worker_on_cpu()
804 	return worker;  in kthread_create_worker_on_cpu()
813 static inline bool queuing_blocked(struct kthread_worker *worker,  in queuing_blocked()  argument
816 	lockdep_assert_held(&worker->lock);  in queuing_blocked()
821 static void kthread_insert_work_sanity_check(struct kthread_worker *worker,  in kthread_insert_work_sanity_check()  argument
824 	lockdep_assert_held(&worker->lock);  in kthread_insert_work_sanity_check()
827 	WARN_ON_ONCE(work->worker && work->worker != worker);  in kthread_insert_work_sanity_check()
831 static void kthread_insert_work(struct kthread_worker *worker,  in kthread_insert_work()  argument
835 	kthread_insert_work_sanity_check(worker, work);  in kthread_insert_work()
838 	work->worker = worker;  in kthread_insert_work()
839 	if (!worker->current_work && likely(worker->task))  in kthread_insert_work()
840 		wake_up_process(worker->task);  in kthread_insert_work()
855 bool kthread_queue_work(struct kthread_worker *worker,  in kthread_queue_work()  argument
861 	raw_spin_lock_irqsave(&worker->lock, flags);  in kthread_queue_work()
862 	if (!queuing_blocked(worker, work)) {  in kthread_queue_work()
863 		kthread_insert_work(worker, work, &worker->work_list);  in kthread_queue_work()
866 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in kthread_queue_work()
883 	struct kthread_worker *worker = work->worker;  in kthread_delayed_work_timer_fn()  local
890 	if (WARN_ON_ONCE(!worker))  in kthread_delayed_work_timer_fn()
893 	raw_spin_lock_irqsave(&worker->lock, flags);  in kthread_delayed_work_timer_fn()
895 	WARN_ON_ONCE(work->worker != worker);  in kthread_delayed_work_timer_fn()
901 		kthread_insert_work(worker, work, &worker->work_list);  in kthread_delayed_work_timer_fn()
903 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in kthread_delayed_work_timer_fn()
907 static void __kthread_queue_delayed_work(struct kthread_worker *worker,  in __kthread_queue_delayed_work()  argument
923 		kthread_insert_work(worker, work, &worker->work_list);  in __kthread_queue_delayed_work()
928 	kthread_insert_work_sanity_check(worker, work);  in __kthread_queue_delayed_work()
930 	list_add(&work->node, &worker->delayed_work_list);  in __kthread_queue_delayed_work()
931 	work->worker = worker;  in __kthread_queue_delayed_work()
951 bool kthread_queue_delayed_work(struct kthread_worker *worker,  in kthread_queue_delayed_work()  argument
959 	raw_spin_lock_irqsave(&worker->lock, flags);  in kthread_queue_delayed_work()
961 	if (!queuing_blocked(worker, work)) {  in kthread_queue_delayed_work()
962 		__kthread_queue_delayed_work(worker, dwork, delay);  in kthread_queue_delayed_work()
966 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in kthread_queue_delayed_work()
995 	struct kthread_worker *worker;  in kthread_flush_work()  local
998 	worker = work->worker;  in kthread_flush_work()
999 	if (!worker)  in kthread_flush_work()
1002 	raw_spin_lock_irq(&worker->lock);  in kthread_flush_work()
1004 	WARN_ON_ONCE(work->worker != worker);  in kthread_flush_work()
1007 		kthread_insert_work(worker, &fwork.work, work->node.next);  in kthread_flush_work()
1008 	else if (worker->current_work == work)  in kthread_flush_work()
1009 		kthread_insert_work(worker, &fwork.work,  in kthread_flush_work()
1010 				    worker->work_list.next);  in kthread_flush_work()
1014 	raw_spin_unlock_irq(&worker->lock);  in kthread_flush_work()
1038 		struct kthread_worker *worker = work->worker;  in __kthread_cancel_work()  local
1047 		raw_spin_unlock_irqrestore(&worker->lock, *flags);  in __kthread_cancel_work()
1049 		raw_spin_lock_irqsave(&worker->lock, *flags);  in __kthread_cancel_work()
1088 bool kthread_mod_delayed_work(struct kthread_worker *worker,  in kthread_mod_delayed_work()  argument
1096 	raw_spin_lock_irqsave(&worker->lock, flags);  in kthread_mod_delayed_work()
1099 	if (!work->worker)  in kthread_mod_delayed_work()
1103 	WARN_ON_ONCE(work->worker != worker);  in kthread_mod_delayed_work()
1111 	__kthread_queue_delayed_work(worker, dwork, delay);  in kthread_mod_delayed_work()
1113 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in kthread_mod_delayed_work()
1120 	struct kthread_worker *worker = work->worker;  in __kthread_cancel_work_sync()  local
1124 	if (!worker)  in __kthread_cancel_work_sync()
1127 	raw_spin_lock_irqsave(&worker->lock, flags);  in __kthread_cancel_work_sync()
1129 	WARN_ON_ONCE(work->worker != worker);  in __kthread_cancel_work_sync()
1133 	if (worker->current_work != work)  in __kthread_cancel_work_sync()
1141 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in __kthread_cancel_work_sync()
1143 	raw_spin_lock_irqsave(&worker->lock, flags);  in __kthread_cancel_work_sync()
1147 	raw_spin_unlock_irqrestore(&worker->lock, flags);  in __kthread_cancel_work_sync()
1196 void kthread_flush_worker(struct kthread_worker *worker)  in kthread_flush_worker()  argument
1203 	kthread_queue_work(worker, &fwork.work);  in kthread_flush_worker()
1216 void kthread_destroy_worker(struct kthread_worker *worker)  in kthread_destroy_worker()  argument
1220 	task = worker->task;  in kthread_destroy_worker()
1224 	kthread_flush_worker(worker);  in kthread_destroy_worker()
1226 	WARN_ON(!list_empty(&worker->work_list));  in kthread_destroy_worker()
1227 	kfree(worker);  in kthread_destroy_worker()