440 	struct work_struct *work = addr;  in work_is_static_object()  local
442 	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));  in work_is_static_object()
451 	struct work_struct *work = addr;  in work_fixup_init()  local
455 		cancel_work_sync(work);  in work_fixup_init()
456 		debug_object_init(work, &work_debug_descr);  in work_fixup_init()
469 	struct work_struct *work = addr;  in work_fixup_free()  local
473 		cancel_work_sync(work);  in work_fixup_free()
474 		debug_object_free(work, &work_debug_descr);  in work_fixup_free()
489 static inline void debug_work_activate(struct work_struct *work)  in debug_work_activate()  argument
491 	debug_object_activate(work, &work_debug_descr);  in debug_work_activate()
494 static inline void debug_work_deactivate(struct work_struct *work)  in debug_work_deactivate()  argument
496 	debug_object_deactivate(work, &work_debug_descr);  in debug_work_deactivate()
499 void __init_work(struct work_struct *work, int onstack)  in __init_work()  argument
502 		debug_object_init_on_stack(work, &work_debug_descr);  in __init_work()
504 		debug_object_init(work, &work_debug_descr);  in __init_work()
508 void destroy_work_on_stack(struct work_struct *work)  in destroy_work_on_stack()  argument
510 	debug_object_free(work, &work_debug_descr);  in destroy_work_on_stack()
514 void destroy_delayed_work_on_stack(struct delayed_work *work)  in destroy_delayed_work_on_stack()  argument
516 	destroy_timer_on_stack(&work->timer);  in destroy_delayed_work_on_stack()
517 	debug_object_free(&work->work, &work_debug_descr);  in destroy_delayed_work_on_stack()
522 static inline void debug_work_activate(struct work_struct *work) { }  in debug_work_activate()  argument
523 static inline void debug_work_deactivate(struct work_struct *work) { }  in debug_work_deactivate()  argument
582 static int get_work_color(struct work_struct *work)  in get_work_color()  argument
584 	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &  in get_work_color()
613 static inline void set_work_data(struct work_struct *work, unsigned long data,  in set_work_data()  argument
616 	WARN_ON_ONCE(!work_pending(work));  in set_work_data()
617 	atomic_long_set(&work->data, data | flags | work_static(work));  in set_work_data()
620 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,  in set_work_pwq()  argument
623 	set_work_data(work, (unsigned long)pwq,  in set_work_pwq()
627 static void set_work_pool_and_keep_pending(struct work_struct *work,  in set_work_pool_and_keep_pending()  argument
630 	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,  in set_work_pool_and_keep_pending()
634 static void set_work_pool_and_clear_pending(struct work_struct *work,  in set_work_pool_and_clear_pending()  argument
644 	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);  in set_work_pool_and_clear_pending()
676 static void clear_work_data(struct work_struct *work)  in clear_work_data()  argument
679 	set_work_data(work, WORK_STRUCT_NO_POOL, 0);  in clear_work_data()
682 static struct pool_workqueue *get_work_pwq(struct work_struct *work)  in get_work_pwq()  argument
684 	unsigned long data = atomic_long_read(&work->data);  in get_work_pwq()
707 static struct worker_pool *get_work_pool(struct work_struct *work)  in get_work_pool()  argument
709 	unsigned long data = atomic_long_read(&work->data);  in get_work_pool()
732 static int get_work_pool_id(struct work_struct *work)  in get_work_pool_id()  argument
734 	unsigned long data = atomic_long_read(&work->data);  in get_work_pool_id()
743 static void mark_work_canceling(struct work_struct *work)  in mark_work_canceling()  argument
745 	unsigned long pool_id = get_work_pool_id(work);  in mark_work_canceling()
748 	set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);  in mark_work_canceling()
751 static bool work_is_canceling(struct work_struct *work)  in work_is_canceling()  argument
753 	unsigned long data = atomic_long_read(&work->data);  in work_is_canceling()
1000 						 struct work_struct *work)  in find_worker_executing_work()  argument
1005 			       (unsigned long)work)  in find_worker_executing_work()
1006 		if (worker->current_work == work &&  in find_worker_executing_work()
1007 		    worker->current_func == work->func)  in find_worker_executing_work()
1030 static void move_linked_works(struct work_struct *work, struct list_head *head,  in move_linked_works()  argument
1039 	list_for_each_entry_safe_from(work, n, NULL, entry) {  in move_linked_works()
1040 		list_move_tail(&work->entry, head);  in move_linked_works()
1041 		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))  in move_linked_works()
1112 static void pwq_activate_delayed_work(struct work_struct *work)  in pwq_activate_delayed_work()  argument
1114 	struct pool_workqueue *pwq = get_work_pwq(work);  in pwq_activate_delayed_work()
1116 	trace_workqueue_activate_work(work);  in pwq_activate_delayed_work()
1119 	move_linked_works(work, &pwq->pool->worklist, NULL);  in pwq_activate_delayed_work()
1120 	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));  in pwq_activate_delayed_work()
1126 	struct work_struct *work = list_first_entry(&pwq->delayed_works,  in pwq_activate_first_delayed()  local
1129 	pwq_activate_delayed_work(work);  in pwq_activate_first_delayed()
1206 static int try_to_grab_pending(struct work_struct *work, bool is_dwork,  in try_to_grab_pending()  argument
1216 		struct delayed_work *dwork = to_delayed_work(work);  in try_to_grab_pending()
1228 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))  in try_to_grab_pending()
1235 	pool = get_work_pool(work);  in try_to_grab_pending()
1248 	pwq = get_work_pwq(work);  in try_to_grab_pending()
1250 		debug_work_deactivate(work);  in try_to_grab_pending()
1259 		if (*work_data_bits(work) & WORK_STRUCT_DELAYED)  in try_to_grab_pending()
1260 			pwq_activate_delayed_work(work);  in try_to_grab_pending()
1262 		list_del_init(&work->entry);  in try_to_grab_pending()
1263 		pwq_dec_nr_in_flight(pwq, get_work_color(work));  in try_to_grab_pending()
1266 		set_work_pool_and_keep_pending(work, pool->id);  in try_to_grab_pending()
1274 	if (work_is_canceling(work))  in try_to_grab_pending()
1293 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,  in insert_work()  argument
1299 	set_work_pwq(work, pwq, extra_flags);  in insert_work()
1300 	list_add_tail(&work->entry, head);  in insert_work()
1364 			 struct work_struct *work)  in __queue_work()  argument
1380 	debug_work_activate(work);  in __queue_work()
1401 	last_pool = get_work_pool(work);  in __queue_work()
1407 		worker = find_worker_executing_work(last_pool, work);  in __queue_work()
1440 	trace_workqueue_queue_work(req_cpu, pwq, work);  in __queue_work()
1442 	if (WARN_ON(!list_empty(&work->entry))) {  in __queue_work()
1451 		trace_workqueue_activate_work(work);  in __queue_work()
1461 	insert_work(pwq, work, worklist, work_flags);  in __queue_work()
1478 		   struct work_struct *work)  in queue_work_on()  argument
1485 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {  in queue_work_on()
1486 		__queue_work(cpu, wq, work);  in queue_work_on()
1500 	__queue_work(dwork->cpu, dwork->wq, &dwork->work);  in delayed_work_timer_fn()
1508 	struct work_struct *work = &dwork->work;  in __queue_delayed_work()  local
1513 	WARN_ON_ONCE(!list_empty(&work->entry));  in __queue_delayed_work()
1522 		__queue_work(cpu, wq, &dwork->work);  in __queue_delayed_work()
1550 	struct work_struct *work = &dwork->work;  in queue_delayed_work_on()  local
1557 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {  in queue_delayed_work_on()
1592 		ret = try_to_grab_pending(&dwork->work, true, &flags);  in mod_delayed_work_on()
1611 	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);  in rcu_work_rcufn()
1627 	struct work_struct *work = &rwork->work;  in queue_rcu_work()  local
1629 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {  in queue_rcu_work()
1895 static void send_mayday(struct work_struct *work)  in send_mayday()  argument
1897 	struct pool_workqueue *pwq = get_work_pwq(work);  in send_mayday()
1921 	struct work_struct *work;  in pool_mayday_timeout()  local
1933 		list_for_each_entry(work, &pool->worklist, entry)  in pool_mayday_timeout()
1934 			send_mayday(work);  in pool_mayday_timeout()
2046 static void process_one_work(struct worker *worker, struct work_struct *work)  in process_one_work()  argument
2050 	struct pool_workqueue *pwq = get_work_pwq(work);  in process_one_work()
2065 	lockdep_copy_map(&lockdep_map, &work->lockdep_map);  in process_one_work()
2077 	collision = find_worker_executing_work(pool, work);  in process_one_work()
2079 		move_linked_works(work, &collision->scheduled, NULL);  in process_one_work()
2084 	debug_work_deactivate(work);  in process_one_work()
2085 	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);  in process_one_work()
2086 	worker->current_work = work;  in process_one_work()
2087 	worker->current_func = work->func;  in process_one_work()
2089 	work_color = get_work_color(work);  in process_one_work()
2097 	list_del_init(&work->entry);  in process_one_work()
2124 	set_work_pool_and_clear_pending(work, pool->id);  in process_one_work()
2152 	trace_workqueue_execute_start(work);  in process_one_work()
2153 	worker->current_func(work);  in process_one_work()
2158 	trace_workqueue_execute_end(work);  in process_one_work()
2210 		struct work_struct *work = list_first_entry(&worker->scheduled,  in process_scheduled_works()  local
2212 		process_one_work(worker, work);  in process_scheduled_works()
2288 		struct work_struct *work =  in worker_thread()  local
2294 		if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {  in worker_thread()
2296 			process_one_work(worker, work);  in worker_thread()
2300 			move_linked_works(work, &worker->scheduled, NULL);  in worker_thread()
2376 		struct work_struct *work, *n;  in rescuer_thread()  local
2393 		list_for_each_entry_safe(work, n, &pool->worklist, entry) {  in rescuer_thread()
2394 			if (get_work_pwq(work) == pwq) {  in rescuer_thread()
2397 				move_linked_works(work, scheduled, &n);  in rescuer_thread()
2490 	struct work_struct	work;  member
2495 static void wq_barrier_func(struct work_struct *work)  in wq_barrier_func()  argument
2497 	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);  in wq_barrier_func()
2538 	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);  in insert_wq_barrier()
2539 	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));  in insert_wq_barrier()
2560 	debug_work_activate(&barr->work);  in insert_wq_barrier()
2561 	insert_work(pwq, &barr->work, head,  in insert_wq_barrier()
2849 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,  in start_flush_work()  argument
2859 	pool = get_work_pool(work);  in start_flush_work()
2867 	pwq = get_work_pwq(work);  in start_flush_work()
2872 		worker = find_worker_executing_work(pool, work);  in start_flush_work()
2878 	check_flush_dependency(pwq->wq, work);  in start_flush_work()
2880 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
2904 static bool __flush_work(struct work_struct *work, bool from_cancel)  in __flush_work()  argument
2912 		lock_map_acquire(&work->lockdep_map);  in __flush_work()
2913 		lock_map_release(&work->lockdep_map);  in __flush_work()
2916 	if (start_flush_work(work, &barr, from_cancel)) {  in __flush_work()
2918 		destroy_work_on_stack(&barr.work);  in __flush_work()
2936 bool flush_work(struct work_struct *work)  in flush_work()  argument
2938 	return __flush_work(work, false);  in flush_work()
2944 	struct work_struct	*work;  member
2951 	if (cwait->work != key)  in cwt_wakefn()
2956 static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)  in __cancel_work_timer()  argument
2963 		ret = try_to_grab_pending(work, is_dwork, &flags);  in __cancel_work_timer()
2985 			cwait.work = work;  in __cancel_work_timer()
2989 			if (work_is_canceling(work))  in __cancel_work_timer()
2996 	mark_work_canceling(work);  in __cancel_work_timer()
3004 		__flush_work(work, true);  in __cancel_work_timer()
3006 	clear_work_data(work);  in __cancel_work_timer()
3015 		__wake_up(&cancel_waitq, TASK_NORMAL, 1, work);  in __cancel_work_timer()
3038 bool cancel_work_sync(struct work_struct *work)  in cancel_work_sync()  argument
3040 	return __cancel_work_timer(work, false);  in cancel_work_sync()
3060 		__queue_work(dwork->cpu, dwork->wq, &dwork->work);  in flush_delayed_work()
3062 	return flush_work(&dwork->work);  in flush_delayed_work()
3076 	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {  in flush_rcu_work()
3078 		flush_work(&rwork->work);  in flush_rcu_work()
3081 		return flush_work(&rwork->work);  in flush_rcu_work()
3086 static bool __cancel_work(struct work_struct *work, bool is_dwork)  in __cancel_work()  argument
3092 		ret = try_to_grab_pending(work, is_dwork, &flags);  in __cancel_work()
3098 	set_work_pool_and_clear_pending(work, get_work_pool_id(work));  in __cancel_work()
3121 	return __cancel_work(&dwork->work, true);  in cancel_delayed_work()
3136 	return __cancel_work_timer(&dwork->work, true);  in cancel_delayed_work_sync()
3163 		struct work_struct *work = per_cpu_ptr(works, cpu);  in schedule_on_each_cpu()  local
3165 		INIT_WORK(work, func);  in schedule_on_each_cpu()
3166 		schedule_work_on(cpu, work);  in schedule_on_each_cpu()
3192 		fn(&ew->work);  in execute_in_process_context()
3196 	INIT_WORK(&ew->work, fn);  in execute_in_process_context()
3197 	schedule_work(&ew->work);  in execute_in_process_context()
3486 static void pwq_unbound_release_workfn(struct work_struct *work)  in pwq_unbound_release_workfn()  argument
3488 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,  in pwq_unbound_release_workfn()
4336 unsigned int work_busy(struct work_struct *work)  in work_busy()  argument
4342 	if (work_pending(work))  in work_busy()
4346 	pool = get_work_pool(work);  in work_busy()
4349 		if (find_worker_executing_work(pool, work))  in work_busy()
4439 static void pr_cont_work(bool comma, struct work_struct *work)  in pr_cont_work()  argument
4441 	if (work->func == wq_barrier_func) {  in pr_cont_work()
4444 		barr = container_of(work, struct wq_barrier, work);  in pr_cont_work()
4449 		pr_cont("%s %pf", comma ? "," : "", work->func);  in pr_cont_work()
4456 	struct work_struct *work;  in show_pwq()  local
4485 			list_for_each_entry(work, &worker->scheduled, entry)  in show_pwq()
4486 				pr_cont_work(false, work);  in show_pwq()
4492 	list_for_each_entry(work, &pool->worklist, entry) {  in show_pwq()
4493 		if (get_work_pwq(work) == pwq) {  in show_pwq()
4502 		list_for_each_entry(work, &pool->worklist, entry) {  in show_pwq()
4503 			if (get_work_pwq(work) != pwq)  in show_pwq()
4506 			pr_cont_work(comma, work);  in show_pwq()
4507 			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);  in show_pwq()
4516 		list_for_each_entry(work, &pwq->delayed_works, entry) {  in show_pwq()
4517 			pr_cont_work(comma, work);  in show_pwq()
4518 			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);  in show_pwq()
4867 	struct work_struct work;  member
4873 static void work_for_cpu_fn(struct work_struct *work)  in work_for_cpu_fn()  argument
4875 	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);  in work_for_cpu_fn()
4895 	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);  in work_on_cpu()
4896 	schedule_work_on(cpu, &wfc.work);  in work_on_cpu()
4897 	flush_work(&wfc.work);  in work_on_cpu()
4898 	destroy_work_on_stack(&wfc.work);  in work_on_cpu()