Lines Matching refs:pwq
424 #define for_each_pwq(pwq, wq) \ argument
425 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
620 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq, in set_work_pwq() argument
623 set_work_data(work, (unsigned long)pwq, in set_work_pwq()
1061 static void get_pwq(struct pool_workqueue *pwq) in get_pwq() argument
1063 lockdep_assert_held(&pwq->pool->lock); in get_pwq()
1064 WARN_ON_ONCE(pwq->refcnt <= 0); in get_pwq()
1065 pwq->refcnt++; in get_pwq()
1075 static void put_pwq(struct pool_workqueue *pwq) in put_pwq() argument
1077 lockdep_assert_held(&pwq->pool->lock); in put_pwq()
1078 if (likely(--pwq->refcnt)) in put_pwq()
1080 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND))) in put_pwq()
1090 schedule_work(&pwq->unbound_release_work); in put_pwq()
1099 static void put_pwq_unlocked(struct pool_workqueue *pwq) in put_pwq_unlocked() argument
1101 if (pwq) { in put_pwq_unlocked()
1106 spin_lock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1107 put_pwq(pwq); in put_pwq_unlocked()
1108 spin_unlock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1114 struct pool_workqueue *pwq = get_work_pwq(work); in pwq_activate_delayed_work() local
1117 if (list_empty(&pwq->pool->worklist)) in pwq_activate_delayed_work()
1118 pwq->pool->watchdog_ts = jiffies; in pwq_activate_delayed_work()
1119 move_linked_works(work, &pwq->pool->worklist, NULL); in pwq_activate_delayed_work()
1121 pwq->nr_active++; in pwq_activate_delayed_work()
1124 static void pwq_activate_first_delayed(struct pool_workqueue *pwq) in pwq_activate_first_delayed() argument
1126 struct work_struct *work = list_first_entry(&pwq->delayed_works, in pwq_activate_first_delayed()
1143 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color) in pwq_dec_nr_in_flight() argument
1149 pwq->nr_in_flight[color]--; in pwq_dec_nr_in_flight()
1151 pwq->nr_active--; in pwq_dec_nr_in_flight()
1152 if (!list_empty(&pwq->delayed_works)) { in pwq_dec_nr_in_flight()
1154 if (pwq->nr_active < pwq->max_active) in pwq_dec_nr_in_flight()
1155 pwq_activate_first_delayed(pwq); in pwq_dec_nr_in_flight()
1159 if (likely(pwq->flush_color != color)) in pwq_dec_nr_in_flight()
1163 if (pwq->nr_in_flight[color]) in pwq_dec_nr_in_flight()
1167 pwq->flush_color = -1; in pwq_dec_nr_in_flight()
1173 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) in pwq_dec_nr_in_flight()
1174 complete(&pwq->wq->first_flusher->done); in pwq_dec_nr_in_flight()
1176 put_pwq(pwq); in pwq_dec_nr_in_flight()
1210 struct pool_workqueue *pwq; in try_to_grab_pending() local
1248 pwq = get_work_pwq(work); in try_to_grab_pending()
1249 if (pwq && pwq->pool == pool) { in try_to_grab_pending()
1263 pwq_dec_nr_in_flight(pwq, get_work_color(work)); in try_to_grab_pending()
1293 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work, in insert_work() argument
1296 struct worker_pool *pool = pwq->pool; in insert_work()
1299 set_work_pwq(work, pwq, extra_flags); in insert_work()
1301 get_pwq(pwq); in insert_work()
1366 struct pool_workqueue *pwq; in __queue_work() local
1392 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in __queue_work()
1394 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); in __queue_work()
1402 if (last_pool && last_pool != pwq->pool) { in __queue_work()
1410 pwq = worker->current_pwq; in __queue_work()
1414 spin_lock(&pwq->pool->lock); in __queue_work()
1417 spin_lock(&pwq->pool->lock); in __queue_work()
1428 if (unlikely(!pwq->refcnt)) { in __queue_work()
1430 spin_unlock(&pwq->pool->lock); in __queue_work()
1440 trace_workqueue_queue_work(req_cpu, pwq, work); in __queue_work()
1443 spin_unlock(&pwq->pool->lock); in __queue_work()
1447 pwq->nr_in_flight[pwq->work_color]++; in __queue_work()
1448 work_flags = work_color_to_flags(pwq->work_color); in __queue_work()
1450 if (likely(pwq->nr_active < pwq->max_active)) { in __queue_work()
1452 pwq->nr_active++; in __queue_work()
1453 worklist = &pwq->pool->worklist; in __queue_work()
1455 pwq->pool->watchdog_ts = jiffies; in __queue_work()
1458 worklist = &pwq->delayed_works; in __queue_work()
1461 insert_work(pwq, work, worklist, work_flags); in __queue_work()
1463 spin_unlock(&pwq->pool->lock); in __queue_work()
1897 struct pool_workqueue *pwq = get_work_pwq(work); in send_mayday() local
1898 struct workqueue_struct *wq = pwq->wq; in send_mayday()
1906 if (list_empty(&pwq->mayday_node)) { in send_mayday()
1912 get_pwq(pwq); in send_mayday()
1913 list_add_tail(&pwq->mayday_node, &wq->maydays); in send_mayday()
2050 struct pool_workqueue *pwq = get_work_pwq(work); in process_one_work() local
2052 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; in process_one_work()
2088 worker->current_pwq = pwq; in process_one_work()
2095 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
2128 lock_map_acquire(&pwq->wq->lockdep_map); in process_one_work()
2160 lock_map_release(&pwq->wq->lockdep_map); in process_one_work()
2192 pwq_dec_nr_in_flight(pwq, work_color); in process_one_work()
2373 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, in rescuer_thread() local
2375 struct worker_pool *pool = pwq->pool; in rescuer_thread()
2380 list_del_init(&pwq->mayday_node); in rescuer_thread()
2394 if (get_work_pwq(work) == pwq) { in rescuer_thread()
2416 get_pwq(pwq); in rescuer_thread()
2417 list_move_tail(&pwq->mayday_node, &wq->maydays); in rescuer_thread()
2426 put_pwq(pwq); in rescuer_thread()
2525 static void insert_wq_barrier(struct pool_workqueue *pwq, in insert_wq_barrier() argument
2561 insert_work(pwq, &barr->work, head, in insert_wq_barrier()
2600 struct pool_workqueue *pwq; in flush_workqueue_prep_pwqs() local
2607 for_each_pwq(pwq, wq) { in flush_workqueue_prep_pwqs()
2608 struct worker_pool *pool = pwq->pool; in flush_workqueue_prep_pwqs()
2613 WARN_ON_ONCE(pwq->flush_color != -1); in flush_workqueue_prep_pwqs()
2615 if (pwq->nr_in_flight[flush_color]) { in flush_workqueue_prep_pwqs()
2616 pwq->flush_color = flush_color; in flush_workqueue_prep_pwqs()
2623 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); in flush_workqueue_prep_pwqs()
2624 pwq->work_color = work_color; in flush_workqueue_prep_pwqs()
2808 struct pool_workqueue *pwq; in drain_workqueue() local
2824 for_each_pwq(pwq, wq) { in drain_workqueue()
2827 spin_lock_irq(&pwq->pool->lock); in drain_workqueue()
2828 drained = !pwq->nr_active && list_empty(&pwq->delayed_works); in drain_workqueue()
2829 spin_unlock_irq(&pwq->pool->lock); in drain_workqueue()
2854 struct pool_workqueue *pwq; in start_flush_work() local
2867 pwq = get_work_pwq(work); in start_flush_work()
2868 if (pwq) { in start_flush_work()
2869 if (unlikely(pwq->pool != pool)) in start_flush_work()
2875 pwq = worker->current_pwq; in start_flush_work()
2878 check_flush_dependency(pwq->wq, work); in start_flush_work()
2880 insert_wq_barrier(pwq, barr, work, worker); in start_flush_work()
2893 (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) { in start_flush_work()
2894 lock_map_acquire(&pwq->wq->lockdep_map); in start_flush_work()
2895 lock_map_release(&pwq->wq->lockdep_map); in start_flush_work()
3488 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue, in pwq_unbound_release_workfn() local
3490 struct workqueue_struct *wq = pwq->wq; in pwq_unbound_release_workfn()
3491 struct worker_pool *pool = pwq->pool; in pwq_unbound_release_workfn()
3498 list_del_rcu(&pwq->pwqs_node); in pwq_unbound_release_workfn()
3506 call_rcu_sched(&pwq->rcu, rcu_free_pwq); in pwq_unbound_release_workfn()
3524 static void pwq_adjust_max_active(struct pool_workqueue *pwq) in pwq_adjust_max_active() argument
3526 struct workqueue_struct *wq = pwq->wq; in pwq_adjust_max_active()
3534 if (!freezable && pwq->max_active == wq->saved_max_active) in pwq_adjust_max_active()
3538 spin_lock_irqsave(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3546 pwq->max_active = wq->saved_max_active; in pwq_adjust_max_active()
3548 while (!list_empty(&pwq->delayed_works) && in pwq_adjust_max_active()
3549 pwq->nr_active < pwq->max_active) in pwq_adjust_max_active()
3550 pwq_activate_first_delayed(pwq); in pwq_adjust_max_active()
3556 wake_up_worker(pwq->pool); in pwq_adjust_max_active()
3558 pwq->max_active = 0; in pwq_adjust_max_active()
3561 spin_unlock_irqrestore(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3565 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, in init_pwq() argument
3568 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK); in init_pwq()
3570 memset(pwq, 0, sizeof(*pwq)); in init_pwq()
3572 pwq->pool = pool; in init_pwq()
3573 pwq->wq = wq; in init_pwq()
3574 pwq->flush_color = -1; in init_pwq()
3575 pwq->refcnt = 1; in init_pwq()
3576 INIT_LIST_HEAD(&pwq->delayed_works); in init_pwq()
3577 INIT_LIST_HEAD(&pwq->pwqs_node); in init_pwq()
3578 INIT_LIST_HEAD(&pwq->mayday_node); in init_pwq()
3579 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); in init_pwq()
3583 static void link_pwq(struct pool_workqueue *pwq) in link_pwq() argument
3585 struct workqueue_struct *wq = pwq->wq; in link_pwq()
3590 if (!list_empty(&pwq->pwqs_node)) in link_pwq()
3594 pwq->work_color = wq->work_color; in link_pwq()
3597 pwq_adjust_max_active(pwq); in link_pwq()
3600 list_add_rcu(&pwq->pwqs_node, &wq->pwqs); in link_pwq()
3608 struct pool_workqueue *pwq; in alloc_unbound_pwq() local
3616 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); in alloc_unbound_pwq()
3617 if (!pwq) { in alloc_unbound_pwq()
3622 init_pwq(pwq, wq, pool); in alloc_unbound_pwq()
3623 return pwq; in alloc_unbound_pwq()
3681 struct pool_workqueue *pwq) in numa_pwq_tbl_install() argument
3689 link_pwq(pwq); in numa_pwq_tbl_install()
3692 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq); in numa_pwq_tbl_install()
3911 struct pool_workqueue *old_pwq = NULL, *pwq; in wq_update_unbound_numa() local
3930 pwq = unbound_pwq_by_node(wq, node); in wq_update_unbound_numa()
3939 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) in wq_update_unbound_numa()
3946 pwq = alloc_unbound_pwq(wq, target_attrs); in wq_update_unbound_numa()
3947 if (!pwq) { in wq_update_unbound_numa()
3955 old_pwq = numa_pwq_tbl_install(wq, node, pwq); in wq_update_unbound_numa()
3980 struct pool_workqueue *pwq = in alloc_and_link_pwqs() local
3985 init_pwq(pwq, wq, &cpu_pools[highpri]); in alloc_and_link_pwqs()
3988 link_pwq(pwq); in alloc_and_link_pwqs()
4056 struct pool_workqueue *pwq; in __alloc_workqueue_key() local
4123 for_each_pwq(pwq, wq) in __alloc_workqueue_key()
4124 pwq_adjust_max_active(pwq); in __alloc_workqueue_key()
4151 struct pool_workqueue *pwq; in destroy_workqueue() local
4159 for_each_pwq(pwq, wq) { in destroy_workqueue()
4163 if (WARN_ON(pwq->nr_in_flight[i])) { in destroy_workqueue()
4170 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) || in destroy_workqueue()
4171 WARN_ON(pwq->nr_active) || in destroy_workqueue()
4172 WARN_ON(!list_empty(&pwq->delayed_works))) { in destroy_workqueue()
4206 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in destroy_workqueue()
4208 put_pwq_unlocked(pwq); in destroy_workqueue()
4215 pwq = wq->dfl_pwq; in destroy_workqueue()
4217 put_pwq_unlocked(pwq); in destroy_workqueue()
4234 struct pool_workqueue *pwq; in workqueue_set_max_active() local
4247 for_each_pwq(pwq, wq) in workqueue_set_max_active()
4248 pwq_adjust_max_active(pwq); in workqueue_set_max_active()
4305 struct pool_workqueue *pwq; in workqueue_congested() local
4314 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in workqueue_congested()
4316 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); in workqueue_congested()
4318 ret = !list_empty(&pwq->delayed_works); in workqueue_congested()
4400 struct pool_workqueue *pwq = NULL; in print_worker_info() local
4418 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq)); in print_worker_info()
4419 probe_kernel_read(&wq, &pwq->wq, sizeof(wq)); in print_worker_info()
4453 static void show_pwq(struct pool_workqueue *pwq) in show_pwq() argument
4455 struct worker_pool *pool = pwq->pool; in show_pwq()
4464 pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active, in show_pwq()
4465 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); in show_pwq()
4468 if (worker->current_pwq == pwq) { in show_pwq()
4478 if (worker->current_pwq != pwq) in show_pwq()
4483 worker == pwq->wq->rescuer ? "(RESCUER)" : "", in show_pwq()
4493 if (get_work_pwq(work) == pwq) { in show_pwq()
4503 if (get_work_pwq(work) != pwq) in show_pwq()
4512 if (!list_empty(&pwq->delayed_works)) { in show_pwq()
4516 list_for_each_entry(work, &pwq->delayed_works, entry) { in show_pwq()
4542 struct pool_workqueue *pwq; in show_workqueue_state() local
4545 for_each_pwq(pwq, wq) { in show_workqueue_state()
4546 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) { in show_workqueue_state()
4556 for_each_pwq(pwq, wq) { in show_workqueue_state()
4557 spin_lock_irqsave(&pwq->pool->lock, flags); in show_workqueue_state()
4558 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) in show_workqueue_state()
4559 show_pwq(pwq); in show_workqueue_state()
4560 spin_unlock_irqrestore(&pwq->pool->lock, flags); in show_workqueue_state()
4942 struct pool_workqueue *pwq; in freeze_workqueues_begin() local
4951 for_each_pwq(pwq, wq) in freeze_workqueues_begin()
4952 pwq_adjust_max_active(pwq); in freeze_workqueues_begin()
4976 struct pool_workqueue *pwq; in freeze_workqueues_busy() local
4990 for_each_pwq(pwq, wq) { in freeze_workqueues_busy()
4991 WARN_ON_ONCE(pwq->nr_active < 0); in freeze_workqueues_busy()
4992 if (pwq->nr_active) { in freeze_workqueues_busy()
5017 struct pool_workqueue *pwq; in thaw_workqueues() local
5029 for_each_pwq(pwq, wq) in thaw_workqueues()
5030 pwq_adjust_max_active(pwq); in thaw_workqueues()