Lines Matching +full:full +full:- +full:pwr +full:- +full:cycle
1 // SPDX-License-Identifier: GPL-2.0-only
3 * kernel/workqueue.c - generic async execution with shared worker pool
10 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
22 * pools for workqueues which are not bound to any specific CPU - the
25 * Please read Documentation/core-api/workqueue.rst for details.
90 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
115 * I: Modifiable by initialization/destruction paths and read-only for
121 * L: pool->lock protected. Access with pool->lock held.
123 * X: During normal operation, modification requires pool->lock and should
125 * cpu or grabbing pool->lock is enough for read access. If
134 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
136 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
139 * WQ: wq->mutex protected.
141 * WR: wq->mutex protected for writes. RCU protected for reads.
195 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
196 * of work_struct->data are used for flags and the remaining high bits
212 * When pwq->nr_active >= max_active, new work item is queued to
213 * pwq->inactive_works instead of pool->worklist and marked with
217 * in pwq->nr_active and all work items in pwq->inactive_works are
219 * work items are in pwq->inactive_works. Some of them are ready to
220 * run in pool->worklist or worker->scheduled. Those work itmes are
222 * not participate in pwq->nr_active. For non-barrier work item, it
223 * is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
228 struct list_head pwqs_node; /* WR: node on wq->pwqs */
229 struct list_head mayday_node; /* MD: node on wq->maydays */
235 * determined without grabbing wq->mutex.
296 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
297 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
321 static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
336 * following always forces round-robin CPU selection on unbound work items
346 /* the per-cpu worker pools */
351 /* PL: hash of all unbound pools keyed by pool->attrs */
389 !lockdep_is_held(&wq->mutex) && \
391 "RCU, wq->mutex or wq_pool_mutex should be held")
399 * for_each_pool - iterate through all worker_pools in the system
416 * for_each_pool_worker - iterate through all workers of a worker_pool
426 list_for_each_entry((worker), &(pool)->workers, node) \
431 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
435 * This must be called either with wq->mutex held or RCU read locked.
443 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
444 lockdep_is_held(&(wq->mutex)))
452 return ((struct work_struct *) addr)->func; in work_debug_hint()
464 * - an active object is initialized
482 * - an active object is freed
533 destroy_timer_on_stack(&work->timer); in destroy_delayed_work_on_stack()
534 debug_object_free(&work->work, &work_debug_descr); in destroy_delayed_work_on_stack()
544 * worker_pool_assign_id - allocate ID and assign it to @pool
548 * successfully, -errno on failure.
559 pool->id = ret; in worker_pool_assign_id()
566 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
570 * This must be called with any of wq_pool_mutex, wq->mutex or RCU
584 * delayed item is pending. The plan is to keep CPU -> NODE in unbound_pwq_by_node()
589 return wq->dfl_pwq; in unbound_pwq_by_node()
591 return rcu_dereference_raw(wq->numa_pwq_tbl[node]); in unbound_pwq_by_node()
602 ((1 << WORK_STRUCT_COLOR_BITS) - 1); in get_work_color()
617 * work->data. These functions should only be called while the work is
618 * owned - ie. while the PENDING bit is set.
634 atomic_long_set(&work->data, data | flags | work_static(work)); in set_work_data()
665 * work->current_func, which is executed afterwards. This possible in set_work_pool_and_clear_pending()
670 * ---------------------------- -------------------------------- in set_work_pool_and_clear_pending()
678 * 7 work->current_func() { in set_work_pool_and_clear_pending()
682 * Without an explicit full barrier speculative LOAD on line 8 can in set_work_pool_and_clear_pending()
701 unsigned long data = atomic_long_read(&work->data); in get_work_pwq()
710 * get_work_pool - return the worker_pool a given work was associated with
726 unsigned long data = atomic_long_read(&work->data); in get_work_pool()
733 (data & WORK_STRUCT_WQ_DATA_MASK))->pool; in get_work_pool()
743 * get_work_pool_id - return the worker pool ID a given work is associated with
751 unsigned long data = atomic_long_read(&work->data); in get_work_pool_id()
755 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; in get_work_pool_id()
770 unsigned long data = atomic_long_read(&work->data); in work_is_canceling()
778 * they're being called with pool->lock held.
783 return !atomic_read(&pool->nr_running); in __need_more_worker()
796 return !list_empty(&pool->worklist) && __need_more_worker(pool); in need_more_worker()
802 return pool->nr_idle; in may_start_working()
808 return !list_empty(&pool->worklist) && in keep_working()
809 atomic_read(&pool->nr_running) <= 1; in keep_working()
821 bool managing = pool->flags & POOL_MANAGER_ACTIVE; in too_many_workers()
822 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ in too_many_workers()
823 int nr_busy = pool->nr_workers - nr_idle; in too_many_workers()
825 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; in too_many_workers()
835 if (unlikely(list_empty(&pool->idle_list))) in first_idle_worker()
838 return list_first_entry(&pool->idle_list, struct worker, entry); in first_idle_worker()
842 * wake_up_worker - wake up an idle worker
848 * raw_spin_lock_irq(pool->lock).
855 wake_up_process(worker->task); in wake_up_worker()
859 * wq_worker_running - a worker is running again
868 if (!worker->sleeping) in wq_worker_running()
870 if (!(worker->flags & WORKER_NOT_RUNNING)) in wq_worker_running()
871 atomic_inc(&worker->pool->nr_running); in wq_worker_running()
872 worker->sleeping = 0; in wq_worker_running()
876 * wq_worker_sleeping - a worker is going to sleep
880 * going to sleep. Preemption needs to be disabled to protect ->sleeping
893 if (worker->flags & WORKER_NOT_RUNNING) in wq_worker_sleeping()
896 pool = worker->pool; in wq_worker_sleeping()
899 if (worker->sleeping) in wq_worker_sleeping()
902 worker->sleeping = 1; in wq_worker_sleeping()
903 raw_spin_lock_irq(&pool->lock); in wq_worker_sleeping()
916 if (atomic_dec_and_test(&pool->nr_running) && in wq_worker_sleeping()
917 !list_empty(&pool->worklist)) { in wq_worker_sleeping()
920 wake_up_process(next->task); in wq_worker_sleeping()
922 raw_spin_unlock_irq(&pool->lock); in wq_worker_sleeping()
926 * wq_worker_last_func - retrieve worker's last work function
933 * raw_spin_lock_irq(rq->lock)
937 * dequeuing, to allow periodic aggregation to shut-off when that
940 * As this function doesn't involve any workqueue-related locking, it
953 return worker->last_func; in wq_worker_last_func()
957 * worker_set_flags - set worker flags and adjust nr_running accordingly
961 * Set @flags in @worker->flags and adjust nr_running accordingly.
964 * raw_spin_lock_irq(pool->lock)
968 struct worker_pool *pool = worker->pool; in worker_set_flags()
970 WARN_ON_ONCE(worker->task != current); in worker_set_flags()
974 !(worker->flags & WORKER_NOT_RUNNING)) { in worker_set_flags()
975 atomic_dec(&pool->nr_running); in worker_set_flags()
978 worker->flags |= flags; in worker_set_flags()
982 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
986 * Clear @flags in @worker->flags and adjust nr_running accordingly.
989 * raw_spin_lock_irq(pool->lock)
993 struct worker_pool *pool = worker->pool; in worker_clr_flags()
994 unsigned int oflags = worker->flags; in worker_clr_flags()
996 WARN_ON_ONCE(worker->task != current); in worker_clr_flags()
998 worker->flags &= ~flags; in worker_clr_flags()
1006 if (!(worker->flags & WORKER_NOT_RUNNING)) in worker_clr_flags()
1007 atomic_inc(&pool->nr_running); in worker_clr_flags()
1011 * find_worker_executing_work - find worker which is executing a work
1016 * @pool->busy_hash which is keyed by the address of @work. For a worker
1037 * raw_spin_lock_irq(pool->lock).
1048 hash_for_each_possible(pool->busy_hash, worker, hentry, in find_worker_executing_work()
1050 if (worker->current_work == work && in find_worker_executing_work()
1051 worker->current_func == work->func) in find_worker_executing_work()
1058 * move_linked_works - move linked works to a list
1072 * raw_spin_lock_irq(pool->lock).
1084 list_move_tail(&work->entry, head); in move_linked_works()
1099 * get_pwq - get an extra reference on the specified pool_workqueue
1103 * @pwq has positive refcnt and be holding the matching pool->lock.
1107 lockdep_assert_held(&pwq->pool->lock); in get_pwq()
1108 WARN_ON_ONCE(pwq->refcnt <= 0); in get_pwq()
1109 pwq->refcnt++; in get_pwq()
1113 * put_pwq - put a pool_workqueue reference
1117 * destruction. The caller should be holding the matching pool->lock.
1121 lockdep_assert_held(&pwq->pool->lock); in put_pwq()
1122 if (likely(--pwq->refcnt)) in put_pwq()
1124 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND))) in put_pwq()
1127 * @pwq can't be released under pool->lock, bounce to in put_pwq()
1129 * pool->lock as this path is taken only for unbound workqueues and in put_pwq()
1130 * the release work item is scheduled on a per-cpu workqueue. To in put_pwq()
1131 * avoid lockdep warning, unbound pool->locks are given lockdep in put_pwq()
1134 schedule_work(&pwq->unbound_release_work); in put_pwq()
1138 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1150 raw_spin_lock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1152 raw_spin_unlock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1161 if (list_empty(&pwq->pool->worklist)) in pwq_activate_inactive_work()
1162 pwq->pool->watchdog_ts = jiffies; in pwq_activate_inactive_work()
1163 move_linked_works(work, &pwq->pool->worklist, NULL); in pwq_activate_inactive_work()
1165 pwq->nr_active++; in pwq_activate_inactive_work()
1170 struct work_struct *work = list_first_entry(&pwq->inactive_works, in pwq_activate_first_inactive()
1177 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1185 * raw_spin_lock_irq(pool->lock).
1192 pwq->nr_active--; in pwq_dec_nr_in_flight()
1193 if (!list_empty(&pwq->inactive_works)) { in pwq_dec_nr_in_flight()
1195 if (pwq->nr_active < pwq->max_active) in pwq_dec_nr_in_flight()
1200 pwq->nr_in_flight[color]--; in pwq_dec_nr_in_flight()
1203 if (likely(pwq->flush_color != color)) in pwq_dec_nr_in_flight()
1206 /* are there still in-flight works? */ in pwq_dec_nr_in_flight()
1207 if (pwq->nr_in_flight[color]) in pwq_dec_nr_in_flight()
1211 pwq->flush_color = -1; in pwq_dec_nr_in_flight()
1217 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) in pwq_dec_nr_in_flight()
1218 complete(&pwq->wq->first_flusher->done); in pwq_dec_nr_in_flight()
1224 * try_to_grab_pending - steal work item from worklist and disable irq
1230 * stable state - idle, on timer or on worklist.
1237 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1238 * -ENOENT if someone else is canceling @work, this state may persist
1245 * disabled on entry. This, combined with delayed_work->timer being
1246 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1266 * dwork->timer is irqsafe. If del_timer() fails, it's in try_to_grab_pending()
1270 if (likely(del_timer(&dwork->timer))) in try_to_grab_pending()
1281 * steal it from ->worklist without clearing WORK_STRUCT_PENDING. in try_to_grab_pending()
1287 raw_spin_lock(&pool->lock); in try_to_grab_pending()
1289 * work->data is guaranteed to point to pwq only while the work in try_to_grab_pending()
1290 * item is queued on pwq->wq, and both updating work->data to point in try_to_grab_pending()
1292 * pwq->pool->lock. This in turn guarantees that, if work->data in try_to_grab_pending()
1297 if (pwq && pwq->pool == pool) { in try_to_grab_pending()
1302 * pwq->inactive_works since a queued barrier can't be in try_to_grab_pending()
1307 * on the inactive_works list, will confuse pwq->nr_active in try_to_grab_pending()
1314 list_del_init(&work->entry); in try_to_grab_pending()
1317 /* work->data points to pwq iff queued, point to pool */ in try_to_grab_pending()
1318 set_work_pool_and_keep_pending(work, pool->id); in try_to_grab_pending()
1320 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1324 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1329 return -ENOENT; in try_to_grab_pending()
1331 return -EAGAIN; in try_to_grab_pending()
1335 * insert_work - insert a work into a pool
1345 * raw_spin_lock_irq(pool->lock).
1350 struct worker_pool *pool = pwq->pool; in insert_work()
1357 list_add_tail(&work->entry, head); in insert_work()
1384 return worker && worker->current_pwq->wq == wq; in is_chained_work()
1401 pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n"); in wq_select_unbound_cpu()
1431 * steal the PENDING will busy-loop waiting for it to either get in __queue_work()
1439 if (unlikely(wq->flags & __WQ_DRAINING) && in __queue_work()
1445 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1452 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in __queue_work()
1458 * pool to guarantee non-reentrancy. in __queue_work()
1461 if (last_pool && last_pool != pwq->pool) { in __queue_work()
1464 raw_spin_lock(&last_pool->lock); in __queue_work()
1468 if (worker && worker->current_pwq->wq == wq) { in __queue_work()
1469 pwq = worker->current_pwq; in __queue_work()
1472 raw_spin_unlock(&last_pool->lock); in __queue_work()
1473 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1476 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1485 * make forward-progress. in __queue_work()
1487 if (unlikely(!pwq->refcnt)) { in __queue_work()
1488 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1489 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1494 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", in __queue_work()
1495 wq->name, cpu); in __queue_work()
1501 if (WARN_ON(!list_empty(&work->entry))) in __queue_work()
1504 pwq->nr_in_flight[pwq->work_color]++; in __queue_work()
1505 work_flags = work_color_to_flags(pwq->work_color); in __queue_work()
1507 if (likely(pwq->nr_active < pwq->max_active)) { in __queue_work()
1509 pwq->nr_active++; in __queue_work()
1510 worklist = &pwq->pool->worklist; in __queue_work()
1512 pwq->pool->watchdog_ts = jiffies; in __queue_work()
1515 worklist = &pwq->inactive_works; in __queue_work()
1522 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1527 * queue_work_on - queue work on specific cpu
1556 * workqueue_select_cpu_near - Select a CPU based on NUMA node
1589 * queue_work_node - queue work on a "random" cpu for a given NUMA node
1619 * If this is used with a per-cpu workqueue then the logic in in queue_work_node()
1623 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); in queue_work_node()
1644 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in delayed_work_timer_fn()
1651 struct timer_list *timer = &dwork->timer; in __queue_delayed_work()
1652 struct work_struct *work = &dwork->work; in __queue_delayed_work()
1655 WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn); in __queue_delayed_work()
1657 WARN_ON_ONCE(!list_empty(&work->entry)); in __queue_delayed_work()
1660 * If @delay is 0, queue @dwork->work immediately. This is for in __queue_delayed_work()
1666 __queue_work(cpu, wq, &dwork->work); in __queue_delayed_work()
1670 dwork->wq = wq; in __queue_delayed_work()
1671 dwork->cpu = cpu; in __queue_delayed_work()
1672 timer->expires = jiffies + delay; in __queue_delayed_work()
1681 * queue_delayed_work_on - queue work on specific CPU after delay
1694 struct work_struct *work = &dwork->work; in queue_delayed_work_on()
1712 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1736 ret = try_to_grab_pending(&dwork->work, true, &flags); in mod_delayed_work_on()
1737 } while (unlikely(ret == -EAGAIN)); in mod_delayed_work_on()
1744 /* -ENOENT from try_to_grab_pending() becomes %true */ in mod_delayed_work_on()
1755 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); in rcu_work_rcufn()
1760 * queue_rcu_work - queue work after a RCU grace period
1765 * that a full RCU grace period is guaranteed only after a %true return.
1767 * execution may happen before a full RCU grace period has passed.
1771 struct work_struct *work = &rwork->work; in queue_rcu_work()
1774 rwork->wq = wq; in queue_rcu_work()
1775 call_rcu(&rwork->rcu, rcu_work_rcufn); in queue_rcu_work()
1784 * worker_enter_idle - enter idle state
1791 * raw_spin_lock_irq(pool->lock).
1795 struct worker_pool *pool = worker->pool; in worker_enter_idle()
1797 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || in worker_enter_idle()
1798 WARN_ON_ONCE(!list_empty(&worker->entry) && in worker_enter_idle()
1799 (worker->hentry.next || worker->hentry.pprev))) in worker_enter_idle()
1803 worker->flags |= WORKER_IDLE; in worker_enter_idle()
1804 pool->nr_idle++; in worker_enter_idle()
1805 worker->last_active = jiffies; in worker_enter_idle()
1808 list_add(&worker->entry, &pool->idle_list); in worker_enter_idle()
1810 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) in worker_enter_idle()
1811 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); in worker_enter_idle()
1815 * pool->lock between setting %WORKER_UNBOUND and zapping in worker_enter_idle()
1819 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in worker_enter_idle()
1820 pool->nr_workers == pool->nr_idle && in worker_enter_idle()
1821 atomic_read(&pool->nr_running)); in worker_enter_idle()
1825 * worker_leave_idle - leave idle state
1831 * raw_spin_lock_irq(pool->lock).
1835 struct worker_pool *pool = worker->pool; in worker_leave_idle()
1837 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) in worker_leave_idle()
1840 pool->nr_idle--; in worker_leave_idle()
1841 list_del_init(&worker->entry); in worker_leave_idle()
1850 INIT_LIST_HEAD(&worker->entry); in alloc_worker()
1851 INIT_LIST_HEAD(&worker->scheduled); in alloc_worker()
1852 INIT_LIST_HEAD(&worker->node); in alloc_worker()
1854 worker->flags = WORKER_PREP; in alloc_worker()
1860 * worker_attach_to_pool() - attach a worker to a pool
1865 * cpu-binding of @worker are kept coordinated with the pool across
1866 * cpu-[un]hotplugs.
1878 if (pool->flags & POOL_DISASSOCIATED) in worker_attach_to_pool()
1879 worker->flags |= WORKER_UNBOUND; in worker_attach_to_pool()
1881 kthread_set_per_cpu(worker->task, pool->cpu); in worker_attach_to_pool()
1883 if (worker->rescue_wq) in worker_attach_to_pool()
1884 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); in worker_attach_to_pool()
1886 list_add_tail(&worker->node, &pool->workers); in worker_attach_to_pool()
1887 worker->pool = pool; in worker_attach_to_pool()
1893 * worker_detach_from_pool() - detach a worker from its pool
1902 struct worker_pool *pool = worker->pool; in worker_detach_from_pool()
1907 kthread_set_per_cpu(worker->task, -1); in worker_detach_from_pool()
1908 list_del(&worker->node); in worker_detach_from_pool()
1909 worker->pool = NULL; in worker_detach_from_pool()
1911 if (list_empty(&pool->workers)) in worker_detach_from_pool()
1912 detach_completion = pool->detach_completion; in worker_detach_from_pool()
1915 /* clear leftover flags without pool->lock after it is detached */ in worker_detach_from_pool()
1916 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); in worker_detach_from_pool()
1923 * create_worker - create a new workqueue worker
1941 id = ida_alloc(&pool->worker_ida, GFP_KERNEL); in create_worker()
1945 worker = alloc_worker(pool->node); in create_worker()
1949 worker->id = id; in create_worker()
1951 if (pool->cpu >= 0) in create_worker()
1952 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id, in create_worker()
1953 pool->attrs->nice < 0 ? "H" : ""); in create_worker()
1955 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id); in create_worker()
1957 worker->task = kthread_create_on_node(worker_thread, worker, pool->node, in create_worker()
1959 if (IS_ERR(worker->task)) in create_worker()
1962 set_user_nice(worker->task, pool->attrs->nice); in create_worker()
1963 kthread_bind_mask(worker->task, pool->attrs->cpumask); in create_worker()
1969 raw_spin_lock_irq(&pool->lock); in create_worker()
1970 worker->pool->nr_workers++; in create_worker()
1972 wake_up_process(worker->task); in create_worker()
1973 raw_spin_unlock_irq(&pool->lock); in create_worker()
1978 ida_free(&pool->worker_ida, id); in create_worker()
1984 * destroy_worker - destroy a workqueue worker
1991 * raw_spin_lock_irq(pool->lock).
1995 struct worker_pool *pool = worker->pool; in destroy_worker()
1997 lockdep_assert_held(&pool->lock); in destroy_worker()
2000 if (WARN_ON(worker->current_work) || in destroy_worker()
2001 WARN_ON(!list_empty(&worker->scheduled)) || in destroy_worker()
2002 WARN_ON(!(worker->flags & WORKER_IDLE))) in destroy_worker()
2005 pool->nr_workers--; in destroy_worker()
2006 pool->nr_idle--; in destroy_worker()
2008 list_del_init(&worker->entry); in destroy_worker()
2009 worker->flags |= WORKER_DIE; in destroy_worker()
2010 wake_up_process(worker->task); in destroy_worker()
2017 raw_spin_lock_irq(&pool->lock); in idle_worker_timeout()
2024 worker = list_entry(pool->idle_list.prev, struct worker, entry); in idle_worker_timeout()
2025 expires = worker->last_active + IDLE_WORKER_TIMEOUT; in idle_worker_timeout()
2028 mod_timer(&pool->idle_timer, expires); in idle_worker_timeout()
2035 raw_spin_unlock_irq(&pool->lock); in idle_worker_timeout()
2041 struct workqueue_struct *wq = pwq->wq; in send_mayday()
2045 if (!wq->rescuer) in send_mayday()
2049 if (list_empty(&pwq->mayday_node)) { in send_mayday()
2056 list_add_tail(&pwq->mayday_node, &wq->maydays); in send_mayday()
2057 wake_up_process(wq->rescuer->task); in send_mayday()
2066 raw_spin_lock_irq(&pool->lock); in pool_mayday_timeout()
2067 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */ in pool_mayday_timeout()
2076 list_for_each_entry(work, &pool->worklist, entry) in pool_mayday_timeout()
2081 raw_spin_unlock_irq(&pool->lock); in pool_mayday_timeout()
2083 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); in pool_mayday_timeout()
2087 * maybe_create_worker - create a new worker if necessary
2100 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2105 __releases(&pool->lock) in maybe_create_worker()
2106 __acquires(&pool->lock) in maybe_create_worker()
2109 raw_spin_unlock_irq(&pool->lock); in maybe_create_worker()
2112 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); in maybe_create_worker()
2124 del_timer_sync(&pool->mayday_timer); in maybe_create_worker()
2125 raw_spin_lock_irq(&pool->lock); in maybe_create_worker()
2128 * created as @pool->lock was dropped and the new worker might have in maybe_create_worker()
2136 * manage_workers - manage worker pool
2148 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2159 struct worker_pool *pool = worker->pool; in manage_workers()
2161 if (pool->flags & POOL_MANAGER_ACTIVE) in manage_workers()
2164 pool->flags |= POOL_MANAGER_ACTIVE; in manage_workers()
2165 pool->manager = worker; in manage_workers()
2169 pool->manager = NULL; in manage_workers()
2170 pool->flags &= ~POOL_MANAGER_ACTIVE; in manage_workers()
2176 * process_one_work - process single work
2187 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2190 __releases(&pool->lock) in process_one_work()
2191 __acquires(&pool->lock) in process_one_work()
2194 struct worker_pool *pool = worker->pool; in process_one_work()
2195 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; in process_one_work()
2204 * work->lockdep_map, make a copy and use that here. in process_one_work()
2208 lockdep_copy_map(&lockdep_map, &work->lockdep_map); in process_one_work()
2211 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in process_one_work()
2212 raw_smp_processor_id() != pool->cpu); in process_one_work()
2222 move_linked_works(work, &collision->scheduled, NULL); in process_one_work()
2228 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); in process_one_work()
2229 worker->current_work = work; in process_one_work()
2230 worker->current_func = work->func; in process_one_work()
2231 worker->current_pwq = pwq; in process_one_work()
2233 worker->current_color = get_work_color(work_data); in process_one_work()
2239 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
2241 list_del_init(&work->entry); in process_one_work()
2254 * false for normal per-cpu workers since nr_running would always in process_one_work()
2264 * update to @work. Also, do this inside @pool->lock so that in process_one_work()
2268 set_work_pool_and_clear_pending(work, pool->id); in process_one_work()
2270 raw_spin_unlock_irq(&pool->lock); in process_one_work()
2272 lock_map_acquire(&pwq->wq->lockdep_map); in process_one_work()
2285 * Which would create W1->C->W1 dependencies, even though there is no in process_one_work()
2287 * read-recursive acquire on the work(queue) 'locks', but this will then in process_one_work()
2292 * flush_work() and complete() primitives (except for single-threaded in process_one_work()
2297 worker->current_func(work); in process_one_work()
2302 trace_workqueue_execute_end(work, worker->current_func); in process_one_work()
2304 lock_map_release(&pwq->wq->lockdep_map); in process_one_work()
2309 current->comm, preempt_count(), task_pid_nr(current), in process_one_work()
2310 worker->current_func); in process_one_work()
2325 raw_spin_lock_irq(&pool->lock); in process_one_work()
2332 worker->last_func = worker->current_func; in process_one_work()
2335 hash_del(&worker->hentry); in process_one_work()
2336 worker->current_work = NULL; in process_one_work()
2337 worker->current_func = NULL; in process_one_work()
2338 worker->current_pwq = NULL; in process_one_work()
2339 worker->current_color = INT_MAX; in process_one_work()
2344 * process_scheduled_works - process scheduled works
2352 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2357 while (!list_empty(&worker->scheduled)) { in process_scheduled_works()
2358 struct work_struct *work = list_first_entry(&worker->scheduled, in process_scheduled_works()
2368 current->flags |= PF_WQ_WORKER; in set_pf_worker()
2370 current->flags &= ~PF_WQ_WORKER; in set_pf_worker()
2375 * worker_thread - the worker thread function
2378 * The worker thread function. All workers belong to a worker_pool -
2379 * either a per-cpu one or dynamic unbound one. These workers process all
2389 struct worker_pool *pool = worker->pool; in worker_thread()
2394 raw_spin_lock_irq(&pool->lock); in worker_thread()
2397 if (unlikely(worker->flags & WORKER_DIE)) { in worker_thread()
2398 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2399 WARN_ON_ONCE(!list_empty(&worker->entry)); in worker_thread()
2402 set_task_comm(worker->task, "kworker/dying"); in worker_thread()
2403 ida_free(&pool->worker_ida, worker->id); in worker_thread()
2420 * ->scheduled list can only be filled while a worker is in worker_thread()
2424 WARN_ON_ONCE(!list_empty(&worker->scheduled)); in worker_thread()
2437 list_first_entry(&pool->worklist, in worker_thread()
2440 pool->watchdog_ts = jiffies; in worker_thread()
2445 if (unlikely(!list_empty(&worker->scheduled))) in worker_thread()
2448 move_linked_works(work, &worker->scheduled, NULL); in worker_thread()
2456 * pool->lock is held and there's no work to process and no need to in worker_thread()
2458 * pool->lock or from local cpu, so setting the current state in worker_thread()
2459 * before releasing pool->lock is enough to prevent losing any in worker_thread()
2464 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2470 * rescuer_thread - the rescuer thread function
2493 struct workqueue_struct *wq = rescuer->rescue_wq; in rescuer_thread()
2494 struct list_head *scheduled = &rescuer->scheduled; in rescuer_thread()
2509 * shouldn't have any work pending, but @wq->maydays may still have in rescuer_thread()
2510 * pwq(s) queued. This can happen by non-rescuer workers consuming in rescuer_thread()
2512 * @wq->maydays processing before acting on should_stop so that the in rescuer_thread()
2520 while (!list_empty(&wq->maydays)) { in rescuer_thread()
2521 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, in rescuer_thread()
2523 struct worker_pool *pool = pwq->pool; in rescuer_thread()
2528 list_del_init(&pwq->mayday_node); in rescuer_thread()
2534 raw_spin_lock_irq(&pool->lock); in rescuer_thread()
2541 list_for_each_entry_safe(work, n, &pool->worklist, entry) { in rescuer_thread()
2544 pool->watchdog_ts = jiffies; in rescuer_thread()
2558 * that such back-to-back work items, which may be in rescuer_thread()
2562 if (pwq->nr_active && need_to_create_worker(pool)) { in rescuer_thread()
2568 if (wq->rescuer && list_empty(&pwq->mayday_node)) { in rescuer_thread()
2570 list_add_tail(&pwq->mayday_node, &wq->maydays); in rescuer_thread()
2590 raw_spin_unlock_irq(&pool->lock); in rescuer_thread()
2606 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING)); in rescuer_thread()
2612 * check_flush_dependency - check for flush dependency sanity
2619 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2625 work_func_t target_func = target_work ? target_work->func : NULL; in check_flush_dependency()
2628 if (target_wq->flags & WQ_MEM_RECLAIM) in check_flush_dependency()
2633 WARN_ONCE(current->flags & PF_MEMALLOC, in check_flush_dependency()
2635 current->pid, current->comm, target_wq->name, target_func); in check_flush_dependency()
2636 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & in check_flush_dependency()
2639 worker->current_pwq->wq->name, worker->current_func, in check_flush_dependency()
2640 target_wq->name, target_func); in check_flush_dependency()
2652 complete(&barr->done); in wq_barrier_func()
2656 * insert_wq_barrier - insert a barrier work
2673 * Note that when @worker is non-NULL, @target may be modified
2677 * raw_spin_lock_irq(pool->lock).
2688 * debugobject calls are safe here even with pool->lock locked in insert_wq_barrier()
2693 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); in insert_wq_barrier()
2694 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); in insert_wq_barrier()
2696 init_completion_map(&barr->done, &target->lockdep_map); in insert_wq_barrier()
2698 barr->task = current; in insert_wq_barrier()
2700 /* The barrier work item does not participate in pwq->nr_active. */ in insert_wq_barrier()
2708 head = worker->scheduled.next; in insert_wq_barrier()
2709 work_color = worker->current_color; in insert_wq_barrier()
2713 head = target->entry.next; in insert_wq_barrier()
2720 pwq->nr_in_flight[work_color]++; in insert_wq_barrier()
2723 debug_work_activate(&barr->work); in insert_wq_barrier()
2724 insert_work(pwq, &barr->work, head, work_flags); in insert_wq_barrier()
2728 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2730 * @flush_color: new flush color, < 0 for no-op
2731 * @work_color: new work color, < 0 for no-op
2735 * If @flush_color is non-negative, flush_color on all pwqs should be
2736 * -1. If no pwq has in-flight commands at the specified color, all
2737 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2738 * has in flight commands, its pwq->flush_color is set to
2739 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2742 * The caller should have initialized @wq->first_flusher prior to
2743 * calling this function with non-negative @flush_color. If
2747 * If @work_color is non-negative, all pwqs should have the same
2752 * mutex_lock(wq->mutex).
2765 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); in flush_workqueue_prep_pwqs()
2766 atomic_set(&wq->nr_pwqs_to_flush, 1); in flush_workqueue_prep_pwqs()
2770 struct worker_pool *pool = pwq->pool; in flush_workqueue_prep_pwqs()
2772 raw_spin_lock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2775 WARN_ON_ONCE(pwq->flush_color != -1); in flush_workqueue_prep_pwqs()
2777 if (pwq->nr_in_flight[flush_color]) { in flush_workqueue_prep_pwqs()
2778 pwq->flush_color = flush_color; in flush_workqueue_prep_pwqs()
2779 atomic_inc(&wq->nr_pwqs_to_flush); in flush_workqueue_prep_pwqs()
2785 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); in flush_workqueue_prep_pwqs()
2786 pwq->work_color = work_color; in flush_workqueue_prep_pwqs()
2789 raw_spin_unlock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2792 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) in flush_workqueue_prep_pwqs()
2793 complete(&wq->first_flusher->done); in flush_workqueue_prep_pwqs()
2799 * flush_workqueue - ensure that any scheduled work has run to completion.
2809 .flush_color = -1, in flush_workqueue()
2810 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map), in flush_workqueue()
2817 lock_map_acquire(&wq->lockdep_map); in flush_workqueue()
2818 lock_map_release(&wq->lockdep_map); in flush_workqueue()
2820 mutex_lock(&wq->mutex); in flush_workqueue()
2823 * Start-to-wait phase in flush_workqueue()
2825 next_color = work_next_color(wq->work_color); in flush_workqueue()
2827 if (next_color != wq->flush_color) { in flush_workqueue()
2829 * Color space is not full. The current work_color in flush_workqueue()
2833 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); in flush_workqueue()
2834 this_flusher.flush_color = wq->work_color; in flush_workqueue()
2835 wq->work_color = next_color; in flush_workqueue()
2837 if (!wq->first_flusher) { in flush_workqueue()
2839 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2841 wq->first_flusher = &this_flusher; in flush_workqueue()
2843 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, in flush_workqueue()
2844 wq->work_color)) { in flush_workqueue()
2846 wq->flush_color = next_color; in flush_workqueue()
2847 wq->first_flusher = NULL; in flush_workqueue()
2852 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); in flush_workqueue()
2853 list_add_tail(&this_flusher.list, &wq->flusher_queue); in flush_workqueue()
2854 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2858 * Oops, color space is full, wait on overflow queue. in flush_workqueue()
2862 list_add_tail(&this_flusher.list, &wq->flusher_overflow); in flush_workqueue()
2867 mutex_unlock(&wq->mutex); in flush_workqueue()
2872 * Wake-up-and-cascade phase in flush_workqueue()
2875 * handling overflow. Non-first flushers can simply return. in flush_workqueue()
2877 if (READ_ONCE(wq->first_flusher) != &this_flusher) in flush_workqueue()
2880 mutex_lock(&wq->mutex); in flush_workqueue()
2883 if (wq->first_flusher != &this_flusher) in flush_workqueue()
2886 WRITE_ONCE(wq->first_flusher, NULL); in flush_workqueue()
2889 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2895 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { in flush_workqueue()
2896 if (next->flush_color != wq->flush_color) in flush_workqueue()
2898 list_del_init(&next->list); in flush_workqueue()
2899 complete(&next->done); in flush_workqueue()
2902 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && in flush_workqueue()
2903 wq->flush_color != work_next_color(wq->work_color)); in flush_workqueue()
2906 wq->flush_color = work_next_color(wq->flush_color); in flush_workqueue()
2909 if (!list_empty(&wq->flusher_overflow)) { in flush_workqueue()
2913 * flusher_queue. This is the start-to-wait in flush_workqueue()
2916 list_for_each_entry(tmp, &wq->flusher_overflow, list) in flush_workqueue()
2917 tmp->flush_color = wq->work_color; in flush_workqueue()
2919 wq->work_color = work_next_color(wq->work_color); in flush_workqueue()
2921 list_splice_tail_init(&wq->flusher_overflow, in flush_workqueue()
2922 &wq->flusher_queue); in flush_workqueue()
2923 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2926 if (list_empty(&wq->flusher_queue)) { in flush_workqueue()
2927 WARN_ON_ONCE(wq->flush_color != wq->work_color); in flush_workqueue()
2935 WARN_ON_ONCE(wq->flush_color == wq->work_color); in flush_workqueue()
2936 WARN_ON_ONCE(wq->flush_color != next->flush_color); in flush_workqueue()
2938 list_del_init(&next->list); in flush_workqueue()
2939 wq->first_flusher = next; in flush_workqueue()
2941 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) in flush_workqueue()
2948 wq->first_flusher = NULL; in flush_workqueue()
2952 mutex_unlock(&wq->mutex); in flush_workqueue()
2957 * drain_workqueue - drain a workqueue
2974 * hotter than drain_workqueue() and already looks at @wq->flags. in drain_workqueue()
2977 mutex_lock(&wq->mutex); in drain_workqueue()
2978 if (!wq->nr_drainers++) in drain_workqueue()
2979 wq->flags |= __WQ_DRAINING; in drain_workqueue()
2980 mutex_unlock(&wq->mutex); in drain_workqueue()
2984 mutex_lock(&wq->mutex); in drain_workqueue()
2989 raw_spin_lock_irq(&pwq->pool->lock); in drain_workqueue()
2990 drained = !pwq->nr_active && list_empty(&pwq->inactive_works); in drain_workqueue()
2991 raw_spin_unlock_irq(&pwq->pool->lock); in drain_workqueue()
2999 wq->name, __func__, flush_cnt); in drain_workqueue()
3001 mutex_unlock(&wq->mutex); in drain_workqueue()
3005 if (!--wq->nr_drainers) in drain_workqueue()
3006 wq->flags &= ~__WQ_DRAINING; in drain_workqueue()
3007 mutex_unlock(&wq->mutex); in drain_workqueue()
3027 raw_spin_lock_irq(&pool->lock); in start_flush_work()
3031 if (unlikely(pwq->pool != pool)) in start_flush_work()
3037 pwq = worker->current_pwq; in start_flush_work()
3040 check_flush_dependency(pwq->wq, work); in start_flush_work()
3043 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3047 * single-threaded or rescuer equipped workqueue. in start_flush_work()
3055 (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) { in start_flush_work()
3056 lock_map_acquire(&pwq->wq->lockdep_map); in start_flush_work()
3057 lock_map_release(&pwq->wq->lockdep_map); in start_flush_work()
3062 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3074 if (WARN_ON(!work->func)) in __flush_work()
3078 lock_map_acquire(&work->lockdep_map); in __flush_work()
3079 lock_map_release(&work->lockdep_map); in __flush_work()
3092 * flush_work - wait for a work to finish executing the last queueing instance
3117 if (cwait->work != key) in cwt_wakefn()
3135 * CANCELING - flush_work() will return false immediately in __cancel_work_timer()
3137 * return -ENOENT as @work is still being canceled and the in __cancel_work_timer()
3146 if (unlikely(ret == -ENOENT)) { in __cancel_work_timer()
3187 * cancel_work_sync - cancel a work and wait for it to finish
3191 * can be used even if the work re-queues itself or migrates to
3195 * cancel_work_sync(&delayed_work->work) must not be used for
3211 * flush_delayed_work - wait for a dwork to finish executing the last queueing
3225 if (del_timer_sync(&dwork->timer)) in flush_delayed_work()
3226 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in flush_delayed_work()
3228 return flush_work(&dwork->work); in flush_delayed_work()
3233 * flush_rcu_work - wait for a rwork to finish executing the last queueing
3242 if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { in flush_rcu_work()
3244 flush_work(&rwork->work); in flush_rcu_work()
3247 return flush_work(&rwork->work); in flush_rcu_work()
3259 } while (unlikely(ret == -EAGAIN)); in __cancel_work()
3270 * cancel_delayed_work - cancel a delayed work
3280 * it returns %true and the work doesn't re-arm itself. Explicitly flush or
3287 return __cancel_work(&dwork->work, true); in cancel_delayed_work()
3292 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3302 return __cancel_work_timer(&dwork->work, true); in cancel_delayed_work_sync()
3307 * schedule_on_each_cpu - execute a function synchronously on each online CPU
3315 * 0 on success, -errno on failure.
3324 return -ENOMEM; in schedule_on_each_cpu()
3344 * execute_in_process_context - reliably execute the routine with user context
3352 * Return: 0 - function was executed
3353 * 1 - function was scheduled for execution
3358 fn(&ew->work); in execute_in_process_context()
3362 INIT_WORK(&ew->work, fn); in execute_in_process_context()
3363 schedule_work(&ew->work); in execute_in_process_context()
3370 * free_workqueue_attrs - free a workqueue_attrs
3378 free_cpumask_var(attrs->cpumask); in free_workqueue_attrs()
3384 * alloc_workqueue_attrs - allocate a workqueue_attrs
3398 if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL)) in alloc_workqueue_attrs()
3401 cpumask_copy(attrs->cpumask, cpu_possible_mask); in alloc_workqueue_attrs()
3411 to->nice = from->nice; in copy_workqueue_attrs()
3412 cpumask_copy(to->cpumask, from->cpumask); in copy_workqueue_attrs()
3415 * ->no_numa as it is used for both pool and wq attrs. Instead, in copy_workqueue_attrs()
3416 * get_unbound_pool() explicitly clears ->no_numa after copying. in copy_workqueue_attrs()
3418 to->no_numa = from->no_numa; in copy_workqueue_attrs()
3426 hash = jhash_1word(attrs->nice, hash); in wqattrs_hash()
3427 hash = jhash(cpumask_bits(attrs->cpumask), in wqattrs_hash()
3436 if (a->nice != b->nice) in wqattrs_equal()
3438 if (!cpumask_equal(a->cpumask, b->cpumask)) in wqattrs_equal()
3444 * init_worker_pool - initialize a newly zalloc'd worker_pool
3447 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3449 * Return: 0 on success, -errno on failure. Even on failure, all fields
3455 raw_spin_lock_init(&pool->lock); in init_worker_pool()
3456 pool->id = -1; in init_worker_pool()
3457 pool->cpu = -1; in init_worker_pool()
3458 pool->node = NUMA_NO_NODE; in init_worker_pool()
3459 pool->flags |= POOL_DISASSOCIATED; in init_worker_pool()
3460 pool->watchdog_ts = jiffies; in init_worker_pool()
3461 INIT_LIST_HEAD(&pool->worklist); in init_worker_pool()
3462 INIT_LIST_HEAD(&pool->idle_list); in init_worker_pool()
3463 hash_init(pool->busy_hash); in init_worker_pool()
3465 timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); in init_worker_pool()
3467 timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); in init_worker_pool()
3469 INIT_LIST_HEAD(&pool->workers); in init_worker_pool()
3471 ida_init(&pool->worker_ida); in init_worker_pool()
3472 INIT_HLIST_NODE(&pool->hash_node); in init_worker_pool()
3473 pool->refcnt = 1; in init_worker_pool()
3476 pool->attrs = alloc_workqueue_attrs(); in init_worker_pool()
3477 if (!pool->attrs) in init_worker_pool()
3478 return -ENOMEM; in init_worker_pool()
3487 lockdep_register_key(&wq->key); in wq_init_lockdep()
3488 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name); in wq_init_lockdep()
3490 lock_name = wq->name; in wq_init_lockdep()
3492 wq->lock_name = lock_name; in wq_init_lockdep()
3493 lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0); in wq_init_lockdep()
3498 lockdep_unregister_key(&wq->key); in wq_unregister_lockdep()
3503 if (wq->lock_name != wq->name) in wq_free_lockdep()
3504 kfree(wq->lock_name); in wq_free_lockdep()
3527 if (!(wq->flags & WQ_UNBOUND)) in rcu_free_wq()
3528 free_percpu(wq->cpu_pwqs); in rcu_free_wq()
3530 free_workqueue_attrs(wq->unbound_attrs); in rcu_free_wq()
3539 ida_destroy(&pool->worker_ida); in rcu_free_pool()
3540 free_workqueue_attrs(pool->attrs); in rcu_free_pool()
3547 raw_spin_lock_irq(&pool->lock); in wq_manager_inactive()
3549 if (pool->flags & POOL_MANAGER_ACTIVE) { in wq_manager_inactive()
3550 raw_spin_unlock_irq(&pool->lock); in wq_manager_inactive()
3557 * put_unbound_pool - put a worker_pool
3574 if (--pool->refcnt) in put_unbound_pool()
3578 if (WARN_ON(!(pool->cpu < 0)) || in put_unbound_pool()
3579 WARN_ON(!list_empty(&pool->worklist))) in put_unbound_pool()
3583 if (pool->id >= 0) in put_unbound_pool()
3584 idr_remove(&worker_pool_idr, pool->id); in put_unbound_pool()
3585 hash_del(&pool->hash_node); in put_unbound_pool()
3596 pool->flags |= POOL_MANAGER_ACTIVE; in put_unbound_pool()
3600 WARN_ON(pool->nr_workers || pool->nr_idle); in put_unbound_pool()
3601 raw_spin_unlock_irq(&pool->lock); in put_unbound_pool()
3604 if (!list_empty(&pool->workers)) in put_unbound_pool()
3605 pool->detach_completion = &detach_completion; in put_unbound_pool()
3608 if (pool->detach_completion) in put_unbound_pool()
3609 wait_for_completion(pool->detach_completion); in put_unbound_pool()
3612 del_timer_sync(&pool->idle_timer); in put_unbound_pool()
3613 del_timer_sync(&pool->mayday_timer); in put_unbound_pool()
3616 call_rcu(&pool->rcu, rcu_free_pool); in put_unbound_pool()
3620 * get_unbound_pool - get a worker_pool with the specified attributes
3644 if (wqattrs_equal(pool->attrs, attrs)) { in get_unbound_pool()
3645 pool->refcnt++; in get_unbound_pool()
3653 if (cpumask_subset(attrs->cpumask, in get_unbound_pool()
3666 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */ in get_unbound_pool()
3667 copy_workqueue_attrs(pool->attrs, attrs); in get_unbound_pool()
3668 pool->node = target_node; in get_unbound_pool()
3674 pool->attrs->no_numa = false; in get_unbound_pool()
3684 hash_add(unbound_pool_hash, &pool->hash_node, hash); in get_unbound_pool()
3707 struct workqueue_struct *wq = pwq->wq; in pwq_unbound_release_workfn()
3708 struct worker_pool *pool = pwq->pool; in pwq_unbound_release_workfn()
3715 if (!list_empty(&pwq->pwqs_node)) { in pwq_unbound_release_workfn()
3716 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) in pwq_unbound_release_workfn()
3719 mutex_lock(&wq->mutex); in pwq_unbound_release_workfn()
3720 list_del_rcu(&pwq->pwqs_node); in pwq_unbound_release_workfn()
3721 is_last = list_empty(&wq->pwqs); in pwq_unbound_release_workfn()
3722 mutex_unlock(&wq->mutex); in pwq_unbound_release_workfn()
3729 call_rcu(&pwq->rcu, rcu_free_pwq); in pwq_unbound_release_workfn()
3737 call_rcu(&wq->rcu, rcu_free_wq); in pwq_unbound_release_workfn()
3742 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3745 * If @pwq isn't freezing, set @pwq->max_active to the associated
3747 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3751 struct workqueue_struct *wq = pwq->wq; in pwq_adjust_max_active()
3752 bool freezable = wq->flags & WQ_FREEZABLE; in pwq_adjust_max_active()
3755 /* for @wq->saved_max_active */ in pwq_adjust_max_active()
3756 lockdep_assert_held(&wq->mutex); in pwq_adjust_max_active()
3758 /* fast exit for non-freezable wqs */ in pwq_adjust_max_active()
3759 if (!freezable && pwq->max_active == wq->saved_max_active) in pwq_adjust_max_active()
3763 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3773 pwq->max_active = wq->saved_max_active; in pwq_adjust_max_active()
3775 while (!list_empty(&pwq->inactive_works) && in pwq_adjust_max_active()
3776 pwq->nr_active < pwq->max_active) { in pwq_adjust_max_active()
3788 wake_up_worker(pwq->pool); in pwq_adjust_max_active()
3790 pwq->max_active = 0; in pwq_adjust_max_active()
3793 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3804 pwq->pool = pool; in init_pwq()
3805 pwq->wq = wq; in init_pwq()
3806 pwq->flush_color = -1; in init_pwq()
3807 pwq->refcnt = 1; in init_pwq()
3808 INIT_LIST_HEAD(&pwq->inactive_works); in init_pwq()
3809 INIT_LIST_HEAD(&pwq->pwqs_node); in init_pwq()
3810 INIT_LIST_HEAD(&pwq->mayday_node); in init_pwq()
3811 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); in init_pwq()
3817 struct workqueue_struct *wq = pwq->wq; in link_pwq()
3819 lockdep_assert_held(&wq->mutex); in link_pwq()
3822 if (!list_empty(&pwq->pwqs_node)) in link_pwq()
3826 pwq->work_color = wq->work_color; in link_pwq()
3832 list_add_rcu(&pwq->pwqs_node, &wq->pwqs); in link_pwq()
3848 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); in alloc_unbound_pwq()
3859 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3869 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3872 * @attrs->cpumask.
3877 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3883 if (!wq_numa_enabled || attrs->no_numa) in wq_calc_node_cpumask()
3887 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask); in wq_calc_node_cpumask()
3895 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]); in wq_calc_node_cpumask()
3903 return !cpumask_equal(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3906 cpumask_copy(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3918 lockdep_assert_held(&wq->mutex); in numa_pwq_tbl_install()
3923 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in numa_pwq_tbl_install()
3924 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq); in numa_pwq_tbl_install()
3944 put_pwq_unlocked(ctx->pwq_tbl[node]); in apply_wqattrs_cleanup()
3945 put_pwq_unlocked(ctx->dfl_pwq); in apply_wqattrs_cleanup()
3947 free_workqueue_attrs(ctx->attrs); in apply_wqattrs_cleanup()
3977 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3978 if (unlikely(cpumask_empty(new_attrs->cpumask))) in apply_wqattrs_prepare()
3979 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3990 * the default pwq covering whole @attrs->cpumask. Always create in apply_wqattrs_prepare()
3993 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); in apply_wqattrs_prepare()
3994 if (!ctx->dfl_pwq) in apply_wqattrs_prepare()
3998 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) { in apply_wqattrs_prepare()
3999 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs); in apply_wqattrs_prepare()
4000 if (!ctx->pwq_tbl[node]) in apply_wqattrs_prepare()
4003 ctx->dfl_pwq->refcnt++; in apply_wqattrs_prepare()
4004 ctx->pwq_tbl[node] = ctx->dfl_pwq; in apply_wqattrs_prepare()
4010 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask); in apply_wqattrs_prepare()
4011 ctx->attrs = new_attrs; in apply_wqattrs_prepare()
4013 ctx->wq = wq; in apply_wqattrs_prepare()
4030 mutex_lock(&ctx->wq->mutex); in apply_wqattrs_commit()
4032 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); in apply_wqattrs_commit()
4036 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node, in apply_wqattrs_commit()
4037 ctx->pwq_tbl[node]); in apply_wqattrs_commit()
4040 link_pwq(ctx->dfl_pwq); in apply_wqattrs_commit()
4041 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq); in apply_wqattrs_commit()
4043 mutex_unlock(&ctx->wq->mutex); in apply_wqattrs_commit()
4065 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) in apply_workqueue_attrs_locked()
4066 return -EINVAL; in apply_workqueue_attrs_locked()
4069 if (!list_empty(&wq->pwqs)) { in apply_workqueue_attrs_locked()
4070 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in apply_workqueue_attrs_locked()
4071 return -EINVAL; in apply_workqueue_attrs_locked()
4073 wq->flags &= ~__WQ_ORDERED; in apply_workqueue_attrs_locked()
4078 return -ENOMEM; in apply_workqueue_attrs_locked()
4088 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4094 * possibles CPUs in @attrs->cpumask so that work items are affine to the
4095 * NUMA node it was issued on. Older pwqs are released as in-flight work
4097 * back-to-back will stay on its current pwq.
4103 * Return: 0 on success and -errno on failure.
4120 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4130 * falls back to @wq->dfl_pwq which may not be optimal but is always
4136 * affinity and may execute on any CPU. This is similar to how per-cpu
4145 int cpu_off = online ? -1 : cpu; in wq_update_unbound_numa()
4152 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) || in wq_update_unbound_numa()
4153 wq->unbound_attrs->no_numa) in wq_update_unbound_numa()
4162 cpumask = target_attrs->cpumask; in wq_update_unbound_numa()
4164 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); in wq_update_unbound_numa()
4173 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) { in wq_update_unbound_numa()
4174 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) in wq_update_unbound_numa()
4184 wq->name); in wq_update_unbound_numa()
4189 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4194 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4195 raw_spin_lock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4196 get_pwq(wq->dfl_pwq); in wq_update_unbound_numa()
4197 raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4198 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq); in wq_update_unbound_numa()
4200 mutex_unlock(&wq->mutex); in wq_update_unbound_numa()
4206 bool highpri = wq->flags & WQ_HIGHPRI; in alloc_and_link_pwqs()
4209 if (!(wq->flags & WQ_UNBOUND)) { in alloc_and_link_pwqs()
4210 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue); in alloc_and_link_pwqs()
4211 if (!wq->cpu_pwqs) in alloc_and_link_pwqs()
4212 return -ENOMEM; in alloc_and_link_pwqs()
4216 per_cpu_ptr(wq->cpu_pwqs, cpu); in alloc_and_link_pwqs()
4222 mutex_lock(&wq->mutex); in alloc_and_link_pwqs()
4224 mutex_unlock(&wq->mutex); in alloc_and_link_pwqs()
4230 if (wq->flags & __WQ_ORDERED) { in alloc_and_link_pwqs()
4233 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node || in alloc_and_link_pwqs()
4234 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node), in alloc_and_link_pwqs()
4235 "ordering guarantee broken for workqueue %s\n", wq->name); in alloc_and_link_pwqs()
4265 if (!(wq->flags & WQ_MEM_RECLAIM)) in init_rescuer()
4270 return -ENOMEM; in init_rescuer()
4272 rescuer->rescue_wq = wq; in init_rescuer()
4273 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name); in init_rescuer()
4274 if (IS_ERR(rescuer->task)) { in init_rescuer()
4275 ret = PTR_ERR(rescuer->task); in init_rescuer()
4280 wq->rescuer = rescuer; in init_rescuer()
4281 kthread_bind_mask(rescuer->task, cpu_possible_mask); in init_rescuer()
4282 wake_up_process(rescuer->task); in init_rescuer()
4299 * longer the case on NUMA machines due to per-node pools. While in alloc_workqueue()
4313 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]); in alloc_workqueue()
4320 wq->unbound_attrs = alloc_workqueue_attrs(); in alloc_workqueue()
4321 if (!wq->unbound_attrs) in alloc_workqueue()
4326 vsnprintf(wq->name, sizeof(wq->name), fmt, args); in alloc_workqueue()
4330 max_active = wq_clamp_max_active(max_active, flags, wq->name); in alloc_workqueue()
4333 wq->flags = flags; in alloc_workqueue()
4334 wq->saved_max_active = max_active; in alloc_workqueue()
4335 mutex_init(&wq->mutex); in alloc_workqueue()
4336 atomic_set(&wq->nr_pwqs_to_flush, 0); in alloc_workqueue()
4337 INIT_LIST_HEAD(&wq->pwqs); in alloc_workqueue()
4338 INIT_LIST_HEAD(&wq->flusher_queue); in alloc_workqueue()
4339 INIT_LIST_HEAD(&wq->flusher_overflow); in alloc_workqueue()
4340 INIT_LIST_HEAD(&wq->maydays); in alloc_workqueue()
4343 INIT_LIST_HEAD(&wq->list); in alloc_workqueue()
4351 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) in alloc_workqueue()
4361 mutex_lock(&wq->mutex); in alloc_workqueue()
4364 mutex_unlock(&wq->mutex); in alloc_workqueue()
4366 list_add_tail_rcu(&wq->list, &workqueues); in alloc_workqueue()
4376 free_workqueue_attrs(wq->unbound_attrs); in alloc_workqueue()
4390 if (pwq->nr_in_flight[i]) in pwq_busy()
4393 if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1)) in pwq_busy()
4395 if (pwq->nr_active || !list_empty(&pwq->inactive_works)) in pwq_busy()
4402 * destroy_workqueue - safely terminate a workqueue
4422 if (wq->rescuer) { in destroy_workqueue()
4423 struct worker *rescuer = wq->rescuer; in destroy_workqueue()
4427 wq->rescuer = NULL; in destroy_workqueue()
4431 kthread_stop(rescuer->task); in destroy_workqueue()
4436 * Sanity checks - grab all the locks so that we wait for all in destroy_workqueue()
4437 * in-flight operations which may do put_pwq(). in destroy_workqueue()
4440 mutex_lock(&wq->mutex); in destroy_workqueue()
4442 raw_spin_lock_irq(&pwq->pool->lock); in destroy_workqueue()
4445 __func__, wq->name); in destroy_workqueue()
4447 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4448 mutex_unlock(&wq->mutex); in destroy_workqueue()
4453 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4455 mutex_unlock(&wq->mutex); in destroy_workqueue()
4461 list_del_rcu(&wq->list); in destroy_workqueue()
4464 if (!(wq->flags & WQ_UNBOUND)) { in destroy_workqueue()
4467 * The base ref is never dropped on per-cpu pwqs. Directly in destroy_workqueue()
4470 call_rcu(&wq->rcu, rcu_free_wq); in destroy_workqueue()
4478 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in destroy_workqueue()
4479 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL); in destroy_workqueue()
4487 pwq = wq->dfl_pwq; in destroy_workqueue()
4488 wq->dfl_pwq = NULL; in destroy_workqueue()
4495 * workqueue_set_max_active - adjust max_active of a workqueue
4509 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_set_max_active()
4512 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); in workqueue_set_max_active()
4514 mutex_lock(&wq->mutex); in workqueue_set_max_active()
4516 wq->flags &= ~__WQ_ORDERED; in workqueue_set_max_active()
4517 wq->saved_max_active = max_active; in workqueue_set_max_active()
4522 mutex_unlock(&wq->mutex); in workqueue_set_max_active()
4527 * current_work - retrieve %current task's work struct
4538 return worker ? worker->current_work : NULL; in current_work()
4543 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4554 return worker && worker->rescue_wq; in current_is_workqueue_rescuer()
4558 * workqueue_congested - test whether a workqueue is congested
4567 * Note that both per-cpu and unbound workqueues may be associated with
4586 if (!(wq->flags & WQ_UNBOUND)) in workqueue_congested()
4587 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in workqueue_congested()
4591 ret = !list_empty(&pwq->inactive_works); in workqueue_congested()
4600 * work_busy - test whether a work is currently pending or running
4622 raw_spin_lock_irqsave(&pool->lock, flags); in work_busy()
4625 raw_spin_unlock_irqrestore(&pool->lock, flags); in work_busy()
4634 * set_worker_desc - set description for the current work item
4635 * @fmt: printf-style format string
4650 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); in set_worker_desc()
4657 * print_worker_info - print out worker information and description
4678 if (!(task->flags & PF_WQ_WORKER)) in print_worker_info()
4691 copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn)); in print_worker_info()
4692 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq)); in print_worker_info()
4693 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq)); in print_worker_info()
4694 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1); in print_worker_info()
4695 copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1); in print_worker_info()
4707 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask); in pr_cont_pool_info()
4708 if (pool->node != NUMA_NO_NODE) in pr_cont_pool_info()
4709 pr_cont(" node=%d", pool->node); in pr_cont_pool_info()
4710 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice); in pr_cont_pool_info()
4715 if (work->func == wq_barrier_func) { in pr_cont_work()
4721 task_pid_nr(barr->task)); in pr_cont_work()
4723 pr_cont("%s %ps", comma ? "," : "", work->func); in pr_cont_work()
4729 struct worker_pool *pool = pwq->pool; in show_pwq()
4735 pr_info(" pwq %d:", pool->id); in show_pwq()
4739 pwq->nr_active, pwq->max_active, pwq->refcnt, in show_pwq()
4740 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); in show_pwq()
4742 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4743 if (worker->current_pwq == pwq) { in show_pwq()
4751 pr_info(" in-flight:"); in show_pwq()
4752 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4753 if (worker->current_pwq != pwq) in show_pwq()
4757 task_pid_nr(worker->task), in show_pwq()
4758 worker->rescue_wq ? "(RESCUER)" : "", in show_pwq()
4759 worker->current_func); in show_pwq()
4760 list_for_each_entry(work, &worker->scheduled, entry) in show_pwq()
4767 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4777 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4787 if (!list_empty(&pwq->inactive_works)) { in show_pwq()
4791 list_for_each_entry(work, &pwq->inactive_works, entry) { in show_pwq()
4800 * show_workqueue_state - dump workqueue state
4821 if (pwq->nr_active || !list_empty(&pwq->inactive_works)) { in show_workqueue_state()
4829 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); in show_workqueue_state()
4832 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in show_workqueue_state()
4833 if (pwq->nr_active || !list_empty(&pwq->inactive_works)) { in show_workqueue_state()
4843 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in show_workqueue_state()
4846 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4857 raw_spin_lock_irqsave(&pool->lock, flags); in show_workqueue_state()
4858 if (pool->nr_workers == pool->nr_idle) in show_workqueue_state()
4866 pr_info("pool %d:", pool->id); in show_workqueue_state()
4869 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000, in show_workqueue_state()
4870 pool->nr_workers); in show_workqueue_state()
4871 if (pool->manager) in show_workqueue_state()
4873 task_pid_nr(pool->manager->task)); in show_workqueue_state()
4874 list_for_each_entry(worker, &pool->idle_list, entry) { in show_workqueue_state()
4876 task_pid_nr(worker->task)); in show_workqueue_state()
4882 raw_spin_unlock_irqrestore(&pool->lock, flags); in show_workqueue_state()
4885 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4900 off = strscpy(buf, task->comm, size); in wq_worker_comm()
4907 if (task->flags & PF_WQ_WORKER) { in wq_worker_comm()
4909 struct worker_pool *pool = worker->pool; in wq_worker_comm()
4912 raw_spin_lock_irq(&pool->lock); in wq_worker_comm()
4914 * ->desc tracks information (wq name or in wq_worker_comm()
4916 * current, prepend '+', otherwise '-'. in wq_worker_comm()
4918 if (worker->desc[0] != '\0') { in wq_worker_comm()
4919 if (worker->current_work) in wq_worker_comm()
4920 scnprintf(buf + off, size - off, "+%s", in wq_worker_comm()
4921 worker->desc); in wq_worker_comm()
4923 scnprintf(buf + off, size - off, "-%s", in wq_worker_comm()
4924 worker->desc); in wq_worker_comm()
4926 raw_spin_unlock_irq(&pool->lock); in wq_worker_comm()
4957 raw_spin_lock_irq(&pool->lock); in unbind_workers()
4967 worker->flags |= WORKER_UNBOUND; in unbind_workers()
4969 pool->flags |= POOL_DISASSOCIATED; in unbind_workers()
4971 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
4974 kthread_set_per_cpu(worker->task, -1); in unbind_workers()
4975 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); in unbind_workers()
4981 * Call schedule() so that we cross rq->lock and thus can in unbind_workers()
4996 atomic_set(&pool->nr_running, 0); in unbind_workers()
5003 raw_spin_lock_irq(&pool->lock); in unbind_workers()
5005 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
5010 * rebind_workers - rebind all workers of a pool to the associated CPU
5013 * @pool->cpu is coming online. Rebind all workers to the CPU.
5023 * be on the run-queue of the associated CPU before any local in rebind_workers()
5024 * wake-ups for concurrency management happen, restore CPU affinity in rebind_workers()
5029 kthread_set_per_cpu(worker->task, pool->cpu); in rebind_workers()
5030 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, in rebind_workers()
5031 pool->attrs->cpumask) < 0); in rebind_workers()
5034 raw_spin_lock_irq(&pool->lock); in rebind_workers()
5036 pool->flags &= ~POOL_DISASSOCIATED; in rebind_workers()
5039 unsigned int worker_flags = worker->flags; in rebind_workers()
5043 * of the associated CPU for local wake-ups targeting it to in rebind_workers()
5047 * be bound before @pool->lock is released. in rebind_workers()
5050 wake_up_process(worker->task); in rebind_workers()
5057 * it initiates the next execution cycle thus restoring in rebind_workers()
5061 * WRITE_ONCE() is necessary because @worker->flags may be in rebind_workers()
5070 WRITE_ONCE(worker->flags, worker_flags); in rebind_workers()
5073 raw_spin_unlock_irq(&pool->lock); in rebind_workers()
5077 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
5094 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask)) in restore_unbound_workers_cpumask()
5097 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); in restore_unbound_workers_cpumask()
5101 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); in restore_unbound_workers_cpumask()
5109 if (pool->nr_workers) in workqueue_prepare_cpu()
5112 return -ENOMEM; in workqueue_prepare_cpu()
5128 if (pool->cpu == cpu) in workqueue_online_cpu()
5130 else if (pool->cpu < 0) in workqueue_online_cpu()
5148 /* unbinding per-cpu workers should happen on the local CPU */ in workqueue_offline_cpu()
5150 return -1; in workqueue_offline_cpu()
5174 wfc->ret = wfc->fn(wfc->arg); in work_for_cpu_fn()
5178 * work_on_cpu - run a function in thread context on a particular cpu
5201 * work_on_cpu_safe - run a function in thread context on a particular cpu
5213 long ret = -ENODEV; in work_on_cpu_safe()
5227 * freeze_workqueues_begin - begin freezing workqueues
5231 * pool->worklist.
5234 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5247 mutex_lock(&wq->mutex); in freeze_workqueues_begin()
5250 mutex_unlock(&wq->mutex); in freeze_workqueues_begin()
5257 * freeze_workqueues_busy - are freezable workqueues still busy?
5280 if (!(wq->flags & WQ_FREEZABLE)) in freeze_workqueues_busy()
5288 WARN_ON_ONCE(pwq->nr_active < 0); in freeze_workqueues_busy()
5289 if (pwq->nr_active) { in freeze_workqueues_busy()
5303 * thaw_workqueues - thaw workqueues
5309 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5325 mutex_lock(&wq->mutex); in thaw_workqueues()
5328 mutex_unlock(&wq->mutex); in thaw_workqueues()
5346 if (!(wq->flags & WQ_UNBOUND)) in workqueue_apply_unbound_cpumask()
5349 if (wq->flags & __WQ_ORDERED) in workqueue_apply_unbound_cpumask()
5352 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs); in workqueue_apply_unbound_cpumask()
5354 ret = -ENOMEM; in workqueue_apply_unbound_cpumask()
5358 list_add_tail(&ctx->list, &ctxs); in workqueue_apply_unbound_cpumask()
5371 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5374 * The low-level workqueues cpumask is a global cpumask that limits
5378 * Return: 0 - Success
5379 * -EINVAL - Invalid @cpumask
5380 * -ENOMEM - Failed to allocate memory for attrs or pwqs.
5384 int ret = -EINVAL; in workqueue_set_unbound_cpumask()
5388 return -ENOMEM; in workqueue_set_unbound_cpumask()
5422 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
5423 * max_active RW int : maximum number of in-flight work items
5441 return wq_dev->wq; in dev_to_wq()
5449 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); in per_cpu_show()
5458 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); in max_active_show()
5469 return -EINVAL; in max_active_store()
5493 written += scnprintf(buf + written, PAGE_SIZE - written, in wq_pool_ids_show()
5495 unbound_pwq_by_node(wq, node)->pool->id); in wq_pool_ids_show()
5498 written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); in wq_pool_ids_show()
5511 mutex_lock(&wq->mutex); in wq_nice_show()
5512 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); in wq_nice_show()
5513 mutex_unlock(&wq->mutex); in wq_nice_show()
5529 copy_workqueue_attrs(attrs, wq->unbound_attrs); in wq_sysfs_prep_attrs()
5538 int ret = -ENOMEM; in wq_nice_store()
5546 if (sscanf(buf, "%d", &attrs->nice) == 1 && in wq_nice_store()
5547 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE) in wq_nice_store()
5550 ret = -EINVAL; in wq_nice_store()
5564 mutex_lock(&wq->mutex); in wq_cpumask_show()
5566 cpumask_pr_args(wq->unbound_attrs->cpumask)); in wq_cpumask_show()
5567 mutex_unlock(&wq->mutex); in wq_cpumask_show()
5577 int ret = -ENOMEM; in wq_cpumask_store()
5585 ret = cpumask_parse(buf, attrs->cpumask); in wq_cpumask_store()
5601 mutex_lock(&wq->mutex); in wq_numa_show()
5603 !wq->unbound_attrs->no_numa); in wq_numa_show()
5604 mutex_unlock(&wq->mutex); in wq_numa_show()
5614 int v, ret = -ENOMEM; in wq_numa_store()
5622 ret = -EINVAL; in wq_numa_store()
5624 attrs->no_numa = !v; in wq_numa_store()
5667 return -ENOMEM; in wq_unbound_cpumask_store()
5701 * workqueue_sysfs_register - make a workqueue visible in sysfs
5713 * Return: 0 on success, -errno on failure.
5725 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_sysfs_register()
5726 return -EINVAL; in workqueue_sysfs_register()
5728 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL); in workqueue_sysfs_register()
5730 return -ENOMEM; in workqueue_sysfs_register()
5732 wq_dev->wq = wq; in workqueue_sysfs_register()
5733 wq_dev->dev.bus = &wq_subsys; in workqueue_sysfs_register()
5734 wq_dev->dev.release = wq_device_release; in workqueue_sysfs_register()
5735 dev_set_name(&wq_dev->dev, "%s", wq->name); in workqueue_sysfs_register()
5741 dev_set_uevent_suppress(&wq_dev->dev, true); in workqueue_sysfs_register()
5743 ret = device_register(&wq_dev->dev); in workqueue_sysfs_register()
5745 put_device(&wq_dev->dev); in workqueue_sysfs_register()
5746 wq->wq_dev = NULL; in workqueue_sysfs_register()
5750 if (wq->flags & WQ_UNBOUND) { in workqueue_sysfs_register()
5753 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) { in workqueue_sysfs_register()
5754 ret = device_create_file(&wq_dev->dev, attr); in workqueue_sysfs_register()
5756 device_unregister(&wq_dev->dev); in workqueue_sysfs_register()
5757 wq->wq_dev = NULL; in workqueue_sysfs_register()
5763 dev_set_uevent_suppress(&wq_dev->dev, false); in workqueue_sysfs_register()
5764 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD); in workqueue_sysfs_register()
5769 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5776 struct wq_device *wq_dev = wq->wq_dev; in workqueue_sysfs_unregister()
5778 if (!wq->wq_dev) in workqueue_sysfs_unregister()
5781 wq->wq_dev = NULL; in workqueue_sysfs_unregister()
5782 device_unregister(&wq_dev->dev); in workqueue_sysfs_unregister()
5791 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5799 * forward progress is defined as the first item on ->worklist changing.
5838 if (list_empty(&pool->worklist)) in wq_watchdog_timer_fn()
5848 if (pool->cpu >= 0) in wq_watchdog_timer_fn()
5849 touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu)); in wq_watchdog_timer_fn()
5852 pool_ts = READ_ONCE(pool->watchdog_ts); in wq_watchdog_timer_fn()
5862 pr_emerg("BUG: workqueue lockup - pool"); in wq_watchdog_timer_fn()
5865 jiffies_to_msecs(now - pool_ts) / 1000); in wq_watchdog_timer_fn()
5981 * workqueue_init_early - early init for workqueue subsystem
5983 * This is the first half of two-staged workqueue subsystem initialization
5984 * and invoked as soon as the bare basics - memory allocation, cpumasks and
6010 pool->cpu = cpu; in workqueue_init_early()
6011 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu)); in workqueue_init_early()
6012 pool->attrs->nice = std_nice[i++]; in workqueue_init_early()
6013 pool->node = cpu_to_node(cpu); in workqueue_init_early()
6027 attrs->nice = std_nice[i]; in workqueue_init_early()
6036 attrs->nice = std_nice[i]; in workqueue_init_early()
6037 attrs->no_numa = true; in workqueue_init_early()
6060 * workqueue_init - bring workqueue subsystem fully online
6062 * This is the latter half of two-staged workqueue subsystem initialization
6077 * archs such as power and arm64. As per-cpu pools created in workqueue_init()
6089 pool->node = cpu_to_node(cpu); in workqueue_init()
6097 wq->name); in workqueue_init()
6105 pool->flags &= ~POOL_DISASSOCIATED; in workqueue_init()