440 static void show_pwq(struct pool_workqueue *pwq);
506 #define for_each_pwq(pwq, wq)						\  argument
507 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
672 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,  in set_work_pwq()  argument
675 	set_work_data(work, (unsigned long)pwq,  in set_work_pwq()
1327 	struct pool_workqueue *pwq = worker->current_pwq;  in wq_worker_tick()  local
1330 	if (!pwq)  in wq_worker_tick()
1333 	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;  in wq_worker_tick()
1359 	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;  in wq_worker_tick()
1362 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;  in wq_worker_tick()
1405 static void get_pwq(struct pool_workqueue *pwq)  in get_pwq()  argument
1407 	lockdep_assert_held(&pwq->pool->lock);  in get_pwq()
1408 	WARN_ON_ONCE(pwq->refcnt <= 0);  in get_pwq()
1409 	pwq->refcnt++;  in get_pwq()
1419 static void put_pwq(struct pool_workqueue *pwq)  in put_pwq()  argument
1421 	lockdep_assert_held(&pwq->pool->lock);  in put_pwq()
1422 	if (likely(--pwq->refcnt))  in put_pwq()
1428 	kthread_queue_work(pwq_release_worker, &pwq->release_work);  in put_pwq()
1437 static void put_pwq_unlocked(struct pool_workqueue *pwq)  in put_pwq_unlocked()  argument
1439 	if (pwq) {  in put_pwq_unlocked()
1444 		raw_spin_lock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1445 		put_pwq(pwq);  in put_pwq_unlocked()
1446 		raw_spin_unlock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1452 	struct pool_workqueue *pwq = get_work_pwq(work);  in pwq_activate_inactive_work()  local
1455 	if (list_empty(&pwq->pool->worklist))  in pwq_activate_inactive_work()
1456 		pwq->pool->watchdog_ts = jiffies;  in pwq_activate_inactive_work()
1457 	move_linked_works(work, &pwq->pool->worklist, NULL);  in pwq_activate_inactive_work()
1459 	pwq->nr_active++;  in pwq_activate_inactive_work()
1462 static void pwq_activate_first_inactive(struct pool_workqueue *pwq)  in pwq_activate_first_inactive()  argument
1464 	struct work_struct *work = list_first_entry(&pwq->inactive_works,  in pwq_activate_first_inactive()
1481 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)  in pwq_dec_nr_in_flight()  argument
1486 		pwq->nr_active--;  in pwq_dec_nr_in_flight()
1487 		if (!list_empty(&pwq->inactive_works)) {  in pwq_dec_nr_in_flight()
1489 			if (pwq->nr_active < pwq->max_active)  in pwq_dec_nr_in_flight()
1490 				pwq_activate_first_inactive(pwq);  in pwq_dec_nr_in_flight()
1494 	pwq->nr_in_flight[color]--;  in pwq_dec_nr_in_flight()
1497 	if (likely(pwq->flush_color != color))  in pwq_dec_nr_in_flight()
1501 	if (pwq->nr_in_flight[color])  in pwq_dec_nr_in_flight()
1505 	pwq->flush_color = -1;  in pwq_dec_nr_in_flight()
1511 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))  in pwq_dec_nr_in_flight()
1512 		complete(&pwq->wq->first_flusher->done);  in pwq_dec_nr_in_flight()
1514 	put_pwq(pwq);  in pwq_dec_nr_in_flight()
1551 	struct pool_workqueue *pwq;  in try_to_grab_pending()  local
1590 	pwq = get_work_pwq(work);  in try_to_grab_pending()
1591 	if (pwq && pwq->pool == pool) {  in try_to_grab_pending()
1609 		pwq_dec_nr_in_flight(pwq, *work_data_bits(work));  in try_to_grab_pending()
1641 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,  in insert_work()  argument
1650 	set_work_pwq(work, pwq, extra_flags);  in insert_work()
1652 	get_pwq(pwq);  in insert_work()
1705 	struct pool_workqueue *pwq;  in __queue_work()  local
1737 	pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));  in __queue_work()
1738 	pool = pwq->pool;  in __queue_work()
1754 			pwq = worker->current_pwq;  in __queue_work()
1755 			pool = pwq->pool;  in __queue_work()
1773 	if (unlikely(!pwq->refcnt)) {  in __queue_work()
1785 	trace_workqueue_queue_work(req_cpu, pwq, work);  in __queue_work()
1790 	pwq->nr_in_flight[pwq->work_color]++;  in __queue_work()
1791 	work_flags = work_color_to_flags(pwq->work_color);  in __queue_work()
1793 	if (likely(pwq->nr_active < pwq->max_active)) {  in __queue_work()
1798 		pwq->nr_active++;  in __queue_work()
1799 		insert_work(pwq, work, &pool->worklist, work_flags);  in __queue_work()
1803 		insert_work(pwq, work, &pwq->inactive_works, work_flags);  in __queue_work()
2386 	struct pool_workqueue *pwq = get_work_pwq(work);  in send_mayday()  local
2387 	struct workqueue_struct *wq = pwq->wq;  in send_mayday()
2395 	if (list_empty(&pwq->mayday_node)) {  in send_mayday()
2401 		get_pwq(pwq);  in send_mayday()
2402 		list_add_tail(&pwq->mayday_node, &wq->maydays);  in send_mayday()
2404 		pwq->stats[PWQ_STAT_MAYDAY]++;  in send_mayday()
2540 	struct pool_workqueue *pwq = get_work_pwq(work);  in process_one_work()  local
2564 	worker->current_pwq = pwq;  in process_one_work()
2573 	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);  in process_one_work()
2583 	if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))  in process_one_work()
2602 	pwq->stats[PWQ_STAT_STARTED]++;  in process_one_work()
2605 	lock_map_acquire(&pwq->wq->lockdep_map);  in process_one_work()
2636 	pwq->stats[PWQ_STAT_COMPLETED]++;  in process_one_work()
2638 	lock_map_release(&pwq->wq->lockdep_map);  in process_one_work()
2677 	pwq_dec_nr_in_flight(pwq, work_data);  in process_one_work()
2854 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,  in rescuer_thread()  local
2856 		struct worker_pool *pool = pwq->pool;  in rescuer_thread()
2860 		list_del_init(&pwq->mayday_node);  in rescuer_thread()
2874 			if (get_work_pwq(work) == pwq &&  in rescuer_thread()
2876 				pwq->stats[PWQ_STAT_RESCUED]++;  in rescuer_thread()
2891 			if (pwq->nr_active && need_to_create_worker(pool)) {  in rescuer_thread()
2897 				if (wq->rescuer && list_empty(&pwq->mayday_node)) {  in rescuer_thread()
2898 					get_pwq(pwq);  in rescuer_thread()
2899 					list_add_tail(&pwq->mayday_node, &wq->maydays);  in rescuer_thread()
2909 		put_pwq(pwq);  in rescuer_thread()
3006 static void insert_wq_barrier(struct pool_workqueue *pwq,  in insert_wq_barrier()  argument
3047 	pwq->nr_in_flight[work_color]++;  in insert_wq_barrier()
3050 	insert_work(pwq, &barr->work, head, work_flags);  in insert_wq_barrier()
3088 	struct pool_workqueue *pwq;  in flush_workqueue_prep_pwqs()  local
3095 	for_each_pwq(pwq, wq) {  in flush_workqueue_prep_pwqs()
3096 		struct worker_pool *pool = pwq->pool;  in flush_workqueue_prep_pwqs()
3101 			WARN_ON_ONCE(pwq->flush_color != -1);  in flush_workqueue_prep_pwqs()
3103 			if (pwq->nr_in_flight[flush_color]) {  in flush_workqueue_prep_pwqs()
3104 				pwq->flush_color = flush_color;  in flush_workqueue_prep_pwqs()
3111 			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));  in flush_workqueue_prep_pwqs()
3112 			pwq->work_color = work_color;  in flush_workqueue_prep_pwqs()
3296 	struct pool_workqueue *pwq;  in drain_workqueue()  local
3312 	for_each_pwq(pwq, wq) {  in drain_workqueue()
3315 		raw_spin_lock_irq(&pwq->pool->lock);  in drain_workqueue()
3316 		drained = !pwq->nr_active && list_empty(&pwq->inactive_works);  in drain_workqueue()
3317 		raw_spin_unlock_irq(&pwq->pool->lock);  in drain_workqueue()
3342 	struct pool_workqueue *pwq;  in start_flush_work()  local
3355 	pwq = get_work_pwq(work);  in start_flush_work()
3356 	if (pwq) {  in start_flush_work()
3357 		if (unlikely(pwq->pool != pool))  in start_flush_work()
3363 		pwq = worker->current_pwq;  in start_flush_work()
3366 	check_flush_dependency(pwq->wq, work);  in start_flush_work()
3368 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
3381 	    (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {  in start_flush_work()
3382 		lock_map_acquire(&pwq->wq->lockdep_map);  in start_flush_work()
3383 		lock_map_release(&pwq->wq->lockdep_map);  in start_flush_work()
4107 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,  in pwq_release_workfn()  local
4109 	struct workqueue_struct *wq = pwq->wq;  in pwq_release_workfn()
4110 	struct worker_pool *pool = pwq->pool;  in pwq_release_workfn()
4117 	if (!list_empty(&pwq->pwqs_node)) {  in pwq_release_workfn()
4119 		list_del_rcu(&pwq->pwqs_node);  in pwq_release_workfn()
4130 	call_rcu(&pwq->rcu, rcu_free_pwq);  in pwq_release_workfn()
4150 static void pwq_adjust_max_active(struct pool_workqueue *pwq)  in pwq_adjust_max_active()  argument
4152 	struct workqueue_struct *wq = pwq->wq;  in pwq_adjust_max_active()
4160 	if (!freezable && pwq->max_active == wq->saved_max_active)  in pwq_adjust_max_active()
4164 	raw_spin_lock_irqsave(&pwq->pool->lock, flags);  in pwq_adjust_max_active()
4172 		pwq->max_active = wq->saved_max_active;  in pwq_adjust_max_active()
4174 		while (!list_empty(&pwq->inactive_works) &&  in pwq_adjust_max_active()
4175 		       pwq->nr_active < pwq->max_active)  in pwq_adjust_max_active()
4176 			pwq_activate_first_inactive(pwq);  in pwq_adjust_max_active()
4178 		kick_pool(pwq->pool);  in pwq_adjust_max_active()
4180 		pwq->max_active = 0;  in pwq_adjust_max_active()
4183 	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);  in pwq_adjust_max_active()
4187 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,  in init_pwq()  argument
4190 	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);  in init_pwq()
4192 	memset(pwq, 0, sizeof(*pwq));  in init_pwq()
4194 	pwq->pool = pool;  in init_pwq()
4195 	pwq->wq = wq;  in init_pwq()
4196 	pwq->flush_color = -1;  in init_pwq()
4197 	pwq->refcnt = 1;  in init_pwq()
4198 	INIT_LIST_HEAD(&pwq->inactive_works);  in init_pwq()
4199 	INIT_LIST_HEAD(&pwq->pwqs_node);  in init_pwq()
4200 	INIT_LIST_HEAD(&pwq->mayday_node);  in init_pwq()
4201 	kthread_init_work(&pwq->release_work, pwq_release_workfn);  in init_pwq()
4205 static void link_pwq(struct pool_workqueue *pwq)  in link_pwq()  argument
4207 	struct workqueue_struct *wq = pwq->wq;  in link_pwq()
4212 	if (!list_empty(&pwq->pwqs_node))  in link_pwq()
4216 	pwq->work_color = wq->work_color;  in link_pwq()
4219 	pwq_adjust_max_active(pwq);  in link_pwq()
4222 	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);  in link_pwq()
4230 	struct pool_workqueue *pwq;  in alloc_unbound_pwq()  local
4238 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);  in alloc_unbound_pwq()
4239 	if (!pwq) {  in alloc_unbound_pwq()
4244 	init_pwq(pwq, wq, pool);  in alloc_unbound_pwq()
4245 	return pwq;  in alloc_unbound_pwq()
4291 					int cpu, struct pool_workqueue *pwq)  in install_unbound_pwq()  argument
4299 	link_pwq(pwq);  in install_unbound_pwq()
4302 	rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);  in install_unbound_pwq()
4512 	struct pool_workqueue *old_pwq = NULL, *pwq;  in wq_update_pod()  local
4532 	pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),  in wq_update_pod()
4534 	if (wqattrs_equal(target_attrs, pwq->pool->attrs))  in wq_update_pod()
4538 	pwq = alloc_unbound_pwq(wq, target_attrs);  in wq_update_pod()
4539 	if (!pwq) {  in wq_update_pod()
4547 	old_pwq = install_unbound_pwq(wq, cpu, pwq);  in wq_update_pod()
4614 			struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);  in alloc_and_link_pwqs()  local
4616 			if (pwq)  in alloc_and_link_pwqs()
4617 				kmem_cache_free(pwq_cache, pwq);  in alloc_and_link_pwqs()
4678 	struct pool_workqueue *pwq;  in alloc_workqueue()  local
4741 	for_each_pwq(pwq, wq)  in alloc_workqueue()
4742 		pwq_adjust_max_active(pwq);  in alloc_workqueue()
4764 static bool pwq_busy(struct pool_workqueue *pwq)  in pwq_busy()  argument
4769 		if (pwq->nr_in_flight[i])  in pwq_busy()
4772 	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))  in pwq_busy()
4774 	if (pwq->nr_active || !list_empty(&pwq->inactive_works))  in pwq_busy()
4788 	struct pool_workqueue *pwq;  in destroy_workqueue()  local
4825 	for_each_pwq(pwq, wq) {  in destroy_workqueue()
4826 		raw_spin_lock_irq(&pwq->pool->lock);  in destroy_workqueue()
4827 		if (WARN_ON(pwq_busy(pwq))) {  in destroy_workqueue()
4830 			show_pwq(pwq);  in destroy_workqueue()
4831 			raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
4837 		raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
4856 		pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));  in destroy_workqueue()
4858 		put_pwq_unlocked(pwq);  in destroy_workqueue()
4880 	struct pool_workqueue *pwq;  in workqueue_set_max_active()  local
4893 	for_each_pwq(pwq, wq)  in workqueue_set_max_active()
4894 		pwq_adjust_max_active(pwq);  in workqueue_set_max_active()
4952 	struct pool_workqueue *pwq;  in workqueue_congested()  local
4961 	pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);  in workqueue_congested()
4962 	ret = !list_empty(&pwq->inactive_works);  in workqueue_congested()
5046 	struct pool_workqueue *pwq = NULL;  in print_worker_info()  local
5064 	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));  in print_worker_info()
5065 	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));  in print_worker_info()
5129 static void show_pwq(struct pool_workqueue *pwq)  in show_pwq()  argument
5132 	struct worker_pool *pool = pwq->pool;  in show_pwq()
5142 		pwq->nr_active, pwq->max_active, pwq->refcnt,  in show_pwq()
5143 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");  in show_pwq()
5146 		if (worker->current_pwq == pwq) {  in show_pwq()
5156 			if (worker->current_pwq != pwq)  in show_pwq()
5172 		if (get_work_pwq(work) == pwq) {  in show_pwq()
5182 			if (get_work_pwq(work) != pwq)  in show_pwq()
5192 	if (!list_empty(&pwq->inactive_works)) {  in show_pwq()
5196 		list_for_each_entry(work, &pwq->inactive_works, entry) {  in show_pwq()
5211 	struct pool_workqueue *pwq;  in show_one_workqueue()  local
5215 	for_each_pwq(pwq, wq) {  in show_one_workqueue()
5216 		if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {  in show_one_workqueue()
5226 	for_each_pwq(pwq, wq) {  in show_one_workqueue()
5227 		raw_spin_lock_irqsave(&pwq->pool->lock, flags);  in show_one_workqueue()
5228 		if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {  in show_one_workqueue()
5235 			show_pwq(pwq);  in show_one_workqueue()
5238 		raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);  in show_one_workqueue()
5686 	struct pool_workqueue *pwq;  in freeze_workqueues_begin()  local
5695 		for_each_pwq(pwq, wq)  in freeze_workqueues_begin()
5696 			pwq_adjust_max_active(pwq);  in freeze_workqueues_begin()
5720 	struct pool_workqueue *pwq;  in freeze_workqueues_busy()  local
5734 		for_each_pwq(pwq, wq) {  in freeze_workqueues_busy()
5735 			WARN_ON_ONCE(pwq->nr_active < 0);  in freeze_workqueues_busy()
5736 			if (pwq->nr_active) {  in freeze_workqueues_busy()
5761 	struct pool_workqueue *pwq;  in thaw_workqueues()  local
5773 		for_each_pwq(pwq, wq)  in thaw_workqueues()
5774 			pwq_adjust_max_active(pwq);  in thaw_workqueues()