377 static void show_pwq(struct pool_workqueue *pwq);
442 #define for_each_pwq(pwq, wq)						\  argument
443 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
637 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,  in set_work_pwq()  argument
640 	set_work_data(work, (unsigned long)pwq,  in set_work_pwq()
1105 static void get_pwq(struct pool_workqueue *pwq)  in get_pwq()  argument
1107 	lockdep_assert_held(&pwq->pool->lock);  in get_pwq()
1108 	WARN_ON_ONCE(pwq->refcnt <= 0);  in get_pwq()
1109 	pwq->refcnt++;  in get_pwq()
1119 static void put_pwq(struct pool_workqueue *pwq)  in put_pwq()  argument
1121 	lockdep_assert_held(&pwq->pool->lock);  in put_pwq()
1122 	if (likely(--pwq->refcnt))  in put_pwq()
1124 	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))  in put_pwq()
1134 	schedule_work(&pwq->unbound_release_work);  in put_pwq()
1143 static void put_pwq_unlocked(struct pool_workqueue *pwq)  in put_pwq_unlocked()  argument
1145 	if (pwq) {  in put_pwq_unlocked()
1150 		raw_spin_lock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1151 		put_pwq(pwq);  in put_pwq_unlocked()
1152 		raw_spin_unlock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1158 	struct pool_workqueue *pwq = get_work_pwq(work);  in pwq_activate_inactive_work()  local
1161 	if (list_empty(&pwq->pool->worklist))  in pwq_activate_inactive_work()
1162 		pwq->pool->watchdog_ts = jiffies;  in pwq_activate_inactive_work()
1163 	move_linked_works(work, &pwq->pool->worklist, NULL);  in pwq_activate_inactive_work()
1165 	pwq->nr_active++;  in pwq_activate_inactive_work()
1168 static void pwq_activate_first_inactive(struct pool_workqueue *pwq)  in pwq_activate_first_inactive()  argument
1170 	struct work_struct *work = list_first_entry(&pwq->inactive_works,  in pwq_activate_first_inactive()
1187 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)  in pwq_dec_nr_in_flight()  argument
1192 		pwq->nr_active--;  in pwq_dec_nr_in_flight()
1193 		if (!list_empty(&pwq->inactive_works)) {  in pwq_dec_nr_in_flight()
1195 			if (pwq->nr_active < pwq->max_active)  in pwq_dec_nr_in_flight()
1196 				pwq_activate_first_inactive(pwq);  in pwq_dec_nr_in_flight()
1200 	pwq->nr_in_flight[color]--;  in pwq_dec_nr_in_flight()
1203 	if (likely(pwq->flush_color != color))  in pwq_dec_nr_in_flight()
1207 	if (pwq->nr_in_flight[color])  in pwq_dec_nr_in_flight()
1211 	pwq->flush_color = -1;  in pwq_dec_nr_in_flight()
1217 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))  in pwq_dec_nr_in_flight()
1218 		complete(&pwq->wq->first_flusher->done);  in pwq_dec_nr_in_flight()
1220 	put_pwq(pwq);  in pwq_dec_nr_in_flight()
1257 	struct pool_workqueue *pwq;  in try_to_grab_pending()  local
1296 	pwq = get_work_pwq(work);  in try_to_grab_pending()
1297 	if (pwq && pwq->pool == pool) {  in try_to_grab_pending()
1315 		pwq_dec_nr_in_flight(pwq, *work_data_bits(work));  in try_to_grab_pending()
1347 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,  in insert_work()  argument
1350 	struct worker_pool *pool = pwq->pool;  in insert_work()
1356 	set_work_pwq(work, pwq, extra_flags);  in insert_work()
1358 	get_pwq(pwq);  in insert_work()
1423 	struct pool_workqueue *pwq;  in __queue_work()  local
1448 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in __queue_work()
1452 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in __queue_work()
1461 	if (last_pool && last_pool != pwq->pool) {  in __queue_work()
1469 			pwq = worker->current_pwq;  in __queue_work()
1473 			raw_spin_lock(&pwq->pool->lock);  in __queue_work()
1476 		raw_spin_lock(&pwq->pool->lock);  in __queue_work()
1487 	if (unlikely(!pwq->refcnt)) {  in __queue_work()
1489 			raw_spin_unlock(&pwq->pool->lock);  in __queue_work()
1499 	trace_workqueue_queue_work(req_cpu, pwq, work);  in __queue_work()
1504 	pwq->nr_in_flight[pwq->work_color]++;  in __queue_work()
1505 	work_flags = work_color_to_flags(pwq->work_color);  in __queue_work()
1507 	if (likely(pwq->nr_active < pwq->max_active)) {  in __queue_work()
1509 		pwq->nr_active++;  in __queue_work()
1510 		worklist = &pwq->pool->worklist;  in __queue_work()
1512 			pwq->pool->watchdog_ts = jiffies;  in __queue_work()
1515 		worklist = &pwq->inactive_works;  in __queue_work()
1519 	insert_work(pwq, work, worklist, work_flags);  in __queue_work()
1522 	raw_spin_unlock(&pwq->pool->lock);  in __queue_work()
2040 	struct pool_workqueue *pwq = get_work_pwq(work);  in send_mayday()  local
2041 	struct workqueue_struct *wq = pwq->wq;  in send_mayday()
2049 	if (list_empty(&pwq->mayday_node)) {  in send_mayday()
2055 		get_pwq(pwq);  in send_mayday()
2056 		list_add_tail(&pwq->mayday_node, &wq->maydays);  in send_mayday()
2193 	struct pool_workqueue *pwq = get_work_pwq(work);  in process_one_work()  local
2195 	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;  in process_one_work()
2231 	worker->current_pwq = pwq;  in process_one_work()
2239 	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);  in process_one_work()
2272 	lock_map_acquire(&pwq->wq->lockdep_map);  in process_one_work()
2304 	lock_map_release(&pwq->wq->lockdep_map);  in process_one_work()
2340 	pwq_dec_nr_in_flight(pwq, work_data);  in process_one_work()
2521 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,  in rescuer_thread()  local
2523 		struct worker_pool *pool = pwq->pool;  in rescuer_thread()
2528 		list_del_init(&pwq->mayday_node);  in rescuer_thread()
2542 			if (get_work_pwq(work) == pwq) {  in rescuer_thread()
2562 			if (pwq->nr_active && need_to_create_worker(pool)) {  in rescuer_thread()
2568 				if (wq->rescuer && list_empty(&pwq->mayday_node)) {  in rescuer_thread()
2569 					get_pwq(pwq);  in rescuer_thread()
2570 					list_add_tail(&pwq->mayday_node, &wq->maydays);  in rescuer_thread()
2580 		put_pwq(pwq);  in rescuer_thread()
2679 static void insert_wq_barrier(struct pool_workqueue *pwq,  in insert_wq_barrier()  argument
2720 	pwq->nr_in_flight[work_color]++;  in insert_wq_barrier()
2724 	insert_work(pwq, &barr->work, head, work_flags);  in insert_wq_barrier()
2762 	struct pool_workqueue *pwq;  in flush_workqueue_prep_pwqs()  local
2769 	for_each_pwq(pwq, wq) {  in flush_workqueue_prep_pwqs()
2770 		struct worker_pool *pool = pwq->pool;  in flush_workqueue_prep_pwqs()
2775 			WARN_ON_ONCE(pwq->flush_color != -1);  in flush_workqueue_prep_pwqs()
2777 			if (pwq->nr_in_flight[flush_color]) {  in flush_workqueue_prep_pwqs()
2778 				pwq->flush_color = flush_color;  in flush_workqueue_prep_pwqs()
2785 			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));  in flush_workqueue_prep_pwqs()
2786 			pwq->work_color = work_color;  in flush_workqueue_prep_pwqs()
2970 	struct pool_workqueue *pwq;  in drain_workqueue()  local
2986 	for_each_pwq(pwq, wq) {  in drain_workqueue()
2989 		raw_spin_lock_irq(&pwq->pool->lock);  in drain_workqueue()
2990 		drained = !pwq->nr_active && list_empty(&pwq->inactive_works);  in drain_workqueue()
2991 		raw_spin_unlock_irq(&pwq->pool->lock);  in drain_workqueue()
3016 	struct pool_workqueue *pwq;  in start_flush_work()  local
3029 	pwq = get_work_pwq(work);  in start_flush_work()
3030 	if (pwq) {  in start_flush_work()
3031 		if (unlikely(pwq->pool != pool))  in start_flush_work()
3037 		pwq = worker->current_pwq;  in start_flush_work()
3040 	check_flush_dependency(pwq->wq, work);  in start_flush_work()
3042 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
3055 	    (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {  in start_flush_work()
3056 		lock_map_acquire(&pwq->wq->lockdep_map);  in start_flush_work()
3057 		lock_map_release(&pwq->wq->lockdep_map);  in start_flush_work()
3705 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,  in pwq_unbound_release_workfn()  local
3707 	struct workqueue_struct *wq = pwq->wq;  in pwq_unbound_release_workfn()
3708 	struct worker_pool *pool = pwq->pool;  in pwq_unbound_release_workfn()
3715 	if (!list_empty(&pwq->pwqs_node)) {  in pwq_unbound_release_workfn()
3720 		list_del_rcu(&pwq->pwqs_node);  in pwq_unbound_release_workfn()
3729 	call_rcu(&pwq->rcu, rcu_free_pwq);  in pwq_unbound_release_workfn()
3749 static void pwq_adjust_max_active(struct pool_workqueue *pwq)  in pwq_adjust_max_active()  argument
3751 	struct workqueue_struct *wq = pwq->wq;  in pwq_adjust_max_active()
3759 	if (!freezable && pwq->max_active == wq->saved_max_active)  in pwq_adjust_max_active()
3763 	raw_spin_lock_irqsave(&pwq->pool->lock, flags);  in pwq_adjust_max_active()
3773 		pwq->max_active = wq->saved_max_active;  in pwq_adjust_max_active()
3775 		while (!list_empty(&pwq->inactive_works) &&  in pwq_adjust_max_active()
3776 		       pwq->nr_active < pwq->max_active) {  in pwq_adjust_max_active()
3777 			pwq_activate_first_inactive(pwq);  in pwq_adjust_max_active()
3788 			wake_up_worker(pwq->pool);  in pwq_adjust_max_active()
3790 		pwq->max_active = 0;  in pwq_adjust_max_active()
3793 	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);  in pwq_adjust_max_active()
3797 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,  in init_pwq()  argument
3800 	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);  in init_pwq()
3802 	memset(pwq, 0, sizeof(*pwq));  in init_pwq()
3804 	pwq->pool = pool;  in init_pwq()
3805 	pwq->wq = wq;  in init_pwq()
3806 	pwq->flush_color = -1;  in init_pwq()
3807 	pwq->refcnt = 1;  in init_pwq()
3808 	INIT_LIST_HEAD(&pwq->inactive_works);  in init_pwq()
3809 	INIT_LIST_HEAD(&pwq->pwqs_node);  in init_pwq()
3810 	INIT_LIST_HEAD(&pwq->mayday_node);  in init_pwq()
3811 	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);  in init_pwq()
3815 static void link_pwq(struct pool_workqueue *pwq)  in link_pwq()  argument
3817 	struct workqueue_struct *wq = pwq->wq;  in link_pwq()
3822 	if (!list_empty(&pwq->pwqs_node))  in link_pwq()
3826 	pwq->work_color = wq->work_color;  in link_pwq()
3829 	pwq_adjust_max_active(pwq);  in link_pwq()
3832 	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);  in link_pwq()
3840 	struct pool_workqueue *pwq;  in alloc_unbound_pwq()  local
3848 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);  in alloc_unbound_pwq()
3849 	if (!pwq) {  in alloc_unbound_pwq()
3854 	init_pwq(pwq, wq, pool);  in alloc_unbound_pwq()
3855 	return pwq;  in alloc_unbound_pwq()
3913 						   struct pool_workqueue *pwq)  in numa_pwq_tbl_install()  argument
3921 	link_pwq(pwq);  in numa_pwq_tbl_install()
3924 	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);  in numa_pwq_tbl_install()
4146 	struct pool_workqueue *old_pwq = NULL, *pwq;  in wq_update_unbound_numa()  local
4165 	pwq = unbound_pwq_by_node(wq, node);  in wq_update_unbound_numa()
4174 		if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))  in wq_update_unbound_numa()
4181 	pwq = alloc_unbound_pwq(wq, target_attrs);  in wq_update_unbound_numa()
4182 	if (!pwq) {  in wq_update_unbound_numa()
4190 	old_pwq = numa_pwq_tbl_install(wq, node, pwq);  in wq_update_unbound_numa()
4215 			struct pool_workqueue *pwq =  in alloc_and_link_pwqs()  local
4220 			init_pwq(pwq, wq, &cpu_pools[highpri]);  in alloc_and_link_pwqs()
4223 			link_pwq(pwq);  in alloc_and_link_pwqs()
4295 	struct pool_workqueue *pwq;  in alloc_workqueue()  local
4362 	for_each_pwq(pwq, wq)  in alloc_workqueue()
4363 		pwq_adjust_max_active(pwq);  in alloc_workqueue()
4385 static bool pwq_busy(struct pool_workqueue *pwq)  in pwq_busy()  argument
4390 		if (pwq->nr_in_flight[i])  in pwq_busy()
4393 	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))  in pwq_busy()
4395 	if (pwq->nr_active || !list_empty(&pwq->inactive_works))  in pwq_busy()
4409 	struct pool_workqueue *pwq;  in destroy_workqueue()  local
4441 	for_each_pwq(pwq, wq) {  in destroy_workqueue()
4442 		raw_spin_lock_irq(&pwq->pool->lock);  in destroy_workqueue()
4443 		if (WARN_ON(pwq_busy(pwq))) {  in destroy_workqueue()
4446 			show_pwq(pwq);  in destroy_workqueue()
4447 			raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
4453 		raw_spin_unlock_irq(&pwq->pool->lock);  in destroy_workqueue()
4478 			pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);  in destroy_workqueue()
4480 			put_pwq_unlocked(pwq);  in destroy_workqueue()
4487 		pwq = wq->dfl_pwq;  in destroy_workqueue()
4489 		put_pwq_unlocked(pwq);  in destroy_workqueue()
4506 	struct pool_workqueue *pwq;  in workqueue_set_max_active()  local
4519 	for_each_pwq(pwq, wq)  in workqueue_set_max_active()
4520 		pwq_adjust_max_active(pwq);  in workqueue_set_max_active()
4577 	struct pool_workqueue *pwq;  in workqueue_congested()  local
4587 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in workqueue_congested()
4589 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in workqueue_congested()
4591 	ret = !list_empty(&pwq->inactive_works);  in workqueue_congested()
4674 	struct pool_workqueue *pwq = NULL;  in print_worker_info()  local
4692 	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));  in print_worker_info()
4693 	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));  in print_worker_info()
4727 static void show_pwq(struct pool_workqueue *pwq)  in show_pwq()  argument
4729 	struct worker_pool *pool = pwq->pool;  in show_pwq()
4739 		pwq->nr_active, pwq->max_active, pwq->refcnt,  in show_pwq()
4740 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");  in show_pwq()
4743 		if (worker->current_pwq == pwq) {  in show_pwq()
4753 			if (worker->current_pwq != pwq)  in show_pwq()
4768 		if (get_work_pwq(work) == pwq) {  in show_pwq()
4778 			if (get_work_pwq(work) != pwq)  in show_pwq()
4787 	if (!list_empty(&pwq->inactive_works)) {  in show_pwq()
4791 		list_for_each_entry(work, &pwq->inactive_works, entry) {  in show_pwq()
4817 		struct pool_workqueue *pwq;  in show_workqueue_state()  local
4820 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4821 			if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {  in show_workqueue_state()
4831 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4832 			raw_spin_lock_irqsave(&pwq->pool->lock, flags);  in show_workqueue_state()
4833 			if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {  in show_workqueue_state()
4840 				show_pwq(pwq);  in show_workqueue_state()
4843 			raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);  in show_workqueue_state()
5239 	struct pool_workqueue *pwq;  in freeze_workqueues_begin()  local
5248 		for_each_pwq(pwq, wq)  in freeze_workqueues_begin()
5249 			pwq_adjust_max_active(pwq);  in freeze_workqueues_begin()
5273 	struct pool_workqueue *pwq;  in freeze_workqueues_busy()  local
5287 		for_each_pwq(pwq, wq) {  in freeze_workqueues_busy()
5288 			WARN_ON_ONCE(pwq->nr_active < 0);  in freeze_workqueues_busy()
5289 			if (pwq->nr_active) {  in freeze_workqueues_busy()
5314 	struct pool_workqueue *pwq;  in thaw_workqueues()  local
5326 		for_each_pwq(pwq, wq)  in thaw_workqueues()
5327 			pwq_adjust_max_active(pwq);  in thaw_workqueues()