597 void __kthread_init_worker(struct kthread_worker *worker,  in __kthread_init_worker()  argument
601 	memset(worker, 0, sizeof(struct kthread_worker));  in __kthread_init_worker()
602 	spin_lock_init(&worker->lock);  in __kthread_init_worker()
603 	lockdep_set_class_and_name(&worker->lock, key, name);  in __kthread_init_worker()
604 	INIT_LIST_HEAD(&worker->work_list);  in __kthread_init_worker()
605 	INIT_LIST_HEAD(&worker->delayed_work_list);  in __kthread_init_worker()
626 	struct kthread_worker *worker = worker_ptr;  in kthread_worker_fn()  local
633 	WARN_ON(worker->task && worker->task != current);  in kthread_worker_fn()
634 	worker->task = current;  in kthread_worker_fn()
636 	if (worker->flags & KTW_FREEZABLE)  in kthread_worker_fn()
644 		spin_lock_irq(&worker->lock);  in kthread_worker_fn()
645 		worker->task = NULL;  in kthread_worker_fn()
646 		spin_unlock_irq(&worker->lock);  in kthread_worker_fn()
651 	spin_lock_irq(&worker->lock);  in kthread_worker_fn()
652 	if (!list_empty(&worker->work_list)) {  in kthread_worker_fn()
653 		work = list_first_entry(&worker->work_list,  in kthread_worker_fn()
657 	worker->current_work = work;  in kthread_worker_fn()
658 	spin_unlock_irq(&worker->lock);  in kthread_worker_fn()
676 	struct kthread_worker *worker;  in __kthread_create_worker()  local
680 	worker = kzalloc(sizeof(*worker), GFP_KERNEL);  in __kthread_create_worker()
681 	if (!worker)  in __kthread_create_worker()
684 	kthread_init_worker(worker);  in __kthread_create_worker()
689 	task = __kthread_create_on_node(kthread_worker_fn, worker,  in __kthread_create_worker()
697 	worker->flags = flags;  in __kthread_create_worker()
698 	worker->task = task;  in __kthread_create_worker()
700 	return worker;  in __kthread_create_worker()
703 	kfree(worker);  in __kthread_create_worker()
719 	struct kthread_worker *worker;  in kthread_create_worker()  local
723 	worker = __kthread_create_worker(-1, flags, namefmt, args);  in kthread_create_worker()
726 	return worker;  in kthread_create_worker()
751 	struct kthread_worker *worker;  in kthread_create_worker_on_cpu()  local
755 	worker = __kthread_create_worker(cpu, flags, namefmt, args);  in kthread_create_worker_on_cpu()
758 	return worker;  in kthread_create_worker_on_cpu()
767 static inline bool queuing_blocked(struct kthread_worker *worker,  in queuing_blocked()  argument
770 	lockdep_assert_held(&worker->lock);  in queuing_blocked()
775 static void kthread_insert_work_sanity_check(struct kthread_worker *worker,  in kthread_insert_work_sanity_check()  argument
778 	lockdep_assert_held(&worker->lock);  in kthread_insert_work_sanity_check()
781 	WARN_ON_ONCE(work->worker && work->worker != worker);  in kthread_insert_work_sanity_check()
785 static void kthread_insert_work(struct kthread_worker *worker,  in kthread_insert_work()  argument
789 	kthread_insert_work_sanity_check(worker, work);  in kthread_insert_work()
792 	work->worker = worker;  in kthread_insert_work()
793 	if (!worker->current_work && likely(worker->task))  in kthread_insert_work()
794 		wake_up_process(worker->task);  in kthread_insert_work()
809 bool kthread_queue_work(struct kthread_worker *worker,  in kthread_queue_work()  argument
815 	spin_lock_irqsave(&worker->lock, flags);  in kthread_queue_work()
816 	if (!queuing_blocked(worker, work)) {  in kthread_queue_work()
817 		kthread_insert_work(worker, work, &worker->work_list);  in kthread_queue_work()
820 	spin_unlock_irqrestore(&worker->lock, flags);  in kthread_queue_work()
837 	struct kthread_worker *worker = work->worker;  in kthread_delayed_work_timer_fn()  local
843 	if (WARN_ON_ONCE(!worker))  in kthread_delayed_work_timer_fn()
846 	spin_lock(&worker->lock);  in kthread_delayed_work_timer_fn()
848 	WARN_ON_ONCE(work->worker != worker);  in kthread_delayed_work_timer_fn()
853 	kthread_insert_work(worker, work, &worker->work_list);  in kthread_delayed_work_timer_fn()
855 	spin_unlock(&worker->lock);  in kthread_delayed_work_timer_fn()
859 void __kthread_queue_delayed_work(struct kthread_worker *worker,  in __kthread_queue_delayed_work()  argument
875 		kthread_insert_work(worker, work, &worker->work_list);  in __kthread_queue_delayed_work()
880 	kthread_insert_work_sanity_check(worker, work);  in __kthread_queue_delayed_work()
882 	list_add(&work->node, &worker->delayed_work_list);  in __kthread_queue_delayed_work()
883 	work->worker = worker;  in __kthread_queue_delayed_work()
903 bool kthread_queue_delayed_work(struct kthread_worker *worker,  in kthread_queue_delayed_work()  argument
911 	spin_lock_irqsave(&worker->lock, flags);  in kthread_queue_delayed_work()
913 	if (!queuing_blocked(worker, work)) {  in kthread_queue_delayed_work()
914 		__kthread_queue_delayed_work(worker, dwork, delay);  in kthread_queue_delayed_work()
918 	spin_unlock_irqrestore(&worker->lock, flags);  in kthread_queue_delayed_work()
947 	struct kthread_worker *worker;  in kthread_flush_work()  local
950 	worker = work->worker;  in kthread_flush_work()
951 	if (!worker)  in kthread_flush_work()
954 	spin_lock_irq(&worker->lock);  in kthread_flush_work()
956 	WARN_ON_ONCE(work->worker != worker);  in kthread_flush_work()
959 		kthread_insert_work(worker, &fwork.work, work->node.next);  in kthread_flush_work()
960 	else if (worker->current_work == work)  in kthread_flush_work()
961 		kthread_insert_work(worker, &fwork.work,  in kthread_flush_work()
962 				    worker->work_list.next);  in kthread_flush_work()
966 	spin_unlock_irq(&worker->lock);  in kthread_flush_work()
990 		struct kthread_worker *worker = work->worker;  in __kthread_cancel_work()  local
999 		spin_unlock_irqrestore(&worker->lock, *flags);  in __kthread_cancel_work()
1001 		spin_lock_irqsave(&worker->lock, *flags);  in __kthread_cancel_work()
1040 bool kthread_mod_delayed_work(struct kthread_worker *worker,  in kthread_mod_delayed_work()  argument
1048 	spin_lock_irqsave(&worker->lock, flags);  in kthread_mod_delayed_work()
1051 	if (!work->worker)  in kthread_mod_delayed_work()
1055 	WARN_ON_ONCE(work->worker != worker);  in kthread_mod_delayed_work()
1063 	__kthread_queue_delayed_work(worker, dwork, delay);  in kthread_mod_delayed_work()
1065 	spin_unlock_irqrestore(&worker->lock, flags);  in kthread_mod_delayed_work()
1072 	struct kthread_worker *worker = work->worker;  in __kthread_cancel_work_sync()  local
1076 	if (!worker)  in __kthread_cancel_work_sync()
1079 	spin_lock_irqsave(&worker->lock, flags);  in __kthread_cancel_work_sync()
1081 	WARN_ON_ONCE(work->worker != worker);  in __kthread_cancel_work_sync()
1085 	if (worker->current_work != work)  in __kthread_cancel_work_sync()
1093 	spin_unlock_irqrestore(&worker->lock, flags);  in __kthread_cancel_work_sync()
1095 	spin_lock_irqsave(&worker->lock, flags);  in __kthread_cancel_work_sync()
1099 	spin_unlock_irqrestore(&worker->lock, flags);  in __kthread_cancel_work_sync()
1148 void kthread_flush_worker(struct kthread_worker *worker)  in kthread_flush_worker()  argument
1155 	kthread_queue_work(worker, &fwork.work);  in kthread_flush_worker()
1168 void kthread_destroy_worker(struct kthread_worker *worker)  in kthread_destroy_worker()  argument
1172 	task = worker->task;  in kthread_destroy_worker()
1176 	kthread_flush_worker(worker);  in kthread_destroy_worker()
1178 	WARN_ON(!list_empty(&worker->work_list));  in kthread_destroy_worker()
1179 	kfree(worker);  in kthread_destroy_worker()