111 	struct io_wq_acct acct[IO_WQ_ACCT_NR];  member
139 					struct io_wq_acct *acct,
157 	return &wq->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];  in io_get_acct()
189 	struct io_wq_acct *acct = io_wq_get_acct(worker);  in io_worker_cancel_cb()  local
192 	atomic_dec(&acct->nr_running);  in io_worker_cancel_cb()
194 	acct->nr_workers--;  in io_worker_cancel_cb()
245 static inline bool __io_acct_run_queue(struct io_wq_acct *acct)  in __io_acct_run_queue()  argument
247 	return !test_bit(IO_ACCT_STALLED_BIT, &acct->flags) &&  in __io_acct_run_queue()
248 		!wq_list_empty(&acct->work_list);  in __io_acct_run_queue()
255 static inline bool io_acct_run_queue(struct io_wq_acct *acct)  in io_acct_run_queue()  argument
256 	__acquires(&acct->lock)  in io_acct_run_queue()
258 	raw_spin_lock(&acct->lock);  in io_acct_run_queue()
259 	if (__io_acct_run_queue(acct))  in io_acct_run_queue()
262 	raw_spin_unlock(&acct->lock);  in io_acct_run_queue()
271 					struct io_wq_acct *acct)  in io_wq_activate_free_worker()  argument
285 		if (io_wq_get_acct(worker) != acct) {  in io_wq_activate_free_worker()
306 static bool io_wq_create_worker(struct io_wq *wq, struct io_wq_acct *acct)  in io_wq_create_worker()  argument
312 	if (unlikely(!acct->max_workers))  in io_wq_create_worker()
316 	if (acct->nr_workers >= acct->max_workers) {  in io_wq_create_worker()
320 	acct->nr_workers++;  in io_wq_create_worker()
322 	atomic_inc(&acct->nr_running);  in io_wq_create_worker()
324 	return create_io_worker(wq, acct->index);  in io_wq_create_worker()
329 	struct io_wq_acct *acct = io_wq_get_acct(worker);  in io_wq_inc_running()  local
331 	atomic_inc(&acct->nr_running);  in io_wq_inc_running()
339 	struct io_wq_acct *acct;  in create_worker_cb()  local
344 	acct = &wq->acct[worker->create_index];  in create_worker_cb()
347 	if (acct->nr_workers < acct->max_workers) {  in create_worker_cb()
348 		acct->nr_workers++;  in create_worker_cb()
355 		atomic_dec(&acct->nr_running);  in create_worker_cb()
363 				   struct io_wq_acct *acct,  in io_queue_worker_create()  argument
385 	worker->create_index = acct->index;  in io_queue_worker_create()
403 	atomic_dec(&acct->nr_running);  in io_queue_worker_create()
410 	struct io_wq_acct *acct = io_wq_get_acct(worker);  in io_wq_dec_running()  local
416 	if (!atomic_dec_and_test(&acct->nr_running))  in io_wq_dec_running()
418 	if (!io_acct_run_queue(acct))  in io_wq_dec_running()
421 	raw_spin_unlock(&acct->lock);  in io_wq_dec_running()
422 	atomic_inc(&acct->nr_running);  in io_wq_dec_running()
424 	io_queue_worker_create(worker, acct, create_worker_cb);  in io_wq_dec_running()
475 static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,  in io_get_next_work()  argument
477 	__must_hold(acct->lock)  in io_get_next_work()
484 	wq_list_for_each(node, prev, &acct->work_list) {  in io_get_next_work()
491 			wq_list_del(&acct->work_list, node, prev);  in io_get_next_work()
502 			wq_list_cut(&acct->work_list, &tail->list, prev);  in io_get_next_work()
518 		set_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_get_next_work()
519 		raw_spin_unlock(&acct->lock);  in io_get_next_work()
521 		raw_spin_lock(&acct->lock);  in io_get_next_work()
523 			clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_get_next_work()
549 static void io_worker_handle_work(struct io_wq_acct *acct,  in io_worker_handle_work()  argument
551 	__releases(&acct->lock)  in io_worker_handle_work()
566 		work = io_get_next_work(acct, worker);  in io_worker_handle_work()
567 		raw_spin_unlock(&acct->lock);  in io_worker_handle_work()
613 				clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_worker_handle_work()
620 		if (!__io_acct_run_queue(acct))  in io_worker_handle_work()
622 		raw_spin_lock(&acct->lock);  in io_worker_handle_work()
629 	struct io_wq_acct *acct = io_wq_get_acct(worker);  in io_wq_worker()  local
648 		while (io_acct_run_queue(acct))  in io_wq_worker()
649 			io_worker_handle_work(acct, worker);  in io_wq_worker()
656 		if (last_timeout && (exit_mask || acct->nr_workers > 1)) {  in io_wq_worker()
657 			acct->nr_workers--;  in io_wq_worker()
682 	if (test_bit(IO_WQ_BIT_EXIT, &wq->state) && io_acct_run_queue(acct))  in io_wq_worker()
683 		io_worker_handle_work(acct, worker);  in io_wq_worker()
780 		struct io_wq_acct *acct = io_wq_get_acct(worker);  in create_worker_cont()  local
782 		atomic_dec(&acct->nr_running);  in create_worker_cont()
784 		acct->nr_workers--;  in create_worker_cont()
785 		if (!acct->nr_workers) {  in create_worker_cont()
792 			while (io_acct_cancel_pending_work(wq, acct, &match))  in create_worker_cont()
810 	struct io_wq_acct *acct = io_wq_get_acct(worker);  in io_workqueue_create()  local
812 	if (!io_queue_worker_create(worker, acct, create_worker_cont))  in io_workqueue_create()
818 	struct io_wq_acct *acct = &wq->acct[index];  in create_io_worker()  local
827 		atomic_dec(&acct->nr_running);  in create_io_worker()
829 		acct->nr_workers--;  in create_io_worker()
900 	struct io_wq_acct *acct = io_work_get_acct(wq, work);  in io_wq_insert_work()  local
906 		wq_list_add_tail(&work->list, &acct->work_list);  in io_wq_insert_work()
916 	wq_list_add_after(&work->list, &tail->list, &acct->work_list);  in io_wq_insert_work()
926 	struct io_wq_acct *acct = io_work_get_acct(wq, work);  in io_wq_enqueue()  local
941 	raw_spin_lock(&acct->lock);  in io_wq_enqueue()
943 	clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_wq_enqueue()
944 	raw_spin_unlock(&acct->lock);  in io_wq_enqueue()
947 	do_create = !io_wq_activate_free_worker(wq, acct);  in io_wq_enqueue()
951 	    !atomic_read(&acct->nr_running))) {  in io_wq_enqueue()
954 		did_create = io_wq_create_worker(wq, acct);  in io_wq_enqueue()
959 		if (acct->nr_workers) {  in io_wq_enqueue()
970 		io_acct_cancel_pending_work(wq, acct, &match);  in io_wq_enqueue()
1020 	struct io_wq_acct *acct = io_work_get_acct(wq, work);  in io_wq_remove_pending()  local
1032 	wq_list_del(&acct->work_list, &work->list, prev);  in io_wq_remove_pending()
1036 					struct io_wq_acct *acct,  in io_acct_cancel_pending_work()  argument
1042 	raw_spin_lock(&acct->lock);  in io_acct_cancel_pending_work()
1043 	wq_list_for_each(node, prev, &acct->work_list) {  in io_acct_cancel_pending_work()
1048 		raw_spin_unlock(&acct->lock);  in io_acct_cancel_pending_work()
1054 	raw_spin_unlock(&acct->lock);  in io_acct_cancel_pending_work()
1065 		struct io_wq_acct *acct = io_get_acct(wq, i == 0);  in io_wq_cancel_pending_work()  local
1067 		if (io_acct_cancel_pending_work(wq, acct, match)) {  in io_wq_cancel_pending_work()
1132 		struct io_wq_acct *acct = &wq->acct[i];  in io_wq_hash_wake()  local
1134 		if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))  in io_wq_hash_wake()
1135 			io_wq_activate_free_worker(wq, acct);  in io_wq_hash_wake()
1165 	wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;  in io_wq_create()
1166 	wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =  in io_wq_create()
1171 		struct io_wq_acct *acct = &wq->acct[i];  in io_wq_create()  local
1173 		acct->index = i;  in io_wq_create()
1174 		atomic_set(&acct->nr_running, 0);  in io_wq_create()
1175 		INIT_WQ_LIST(&acct->work_list);  in io_wq_create()
1176 		raw_spin_lock_init(&acct->lock);  in io_wq_create()
1338 	struct io_wq_acct *acct;  in io_wq_max_workers()  local
1358 		acct = &wq->acct[i];  in io_wq_max_workers()
1359 		prev[i] = max_t(int, acct->max_workers, prev[i]);  in io_wq_max_workers()
1361 			acct->max_workers = new_count[i];  in io_wq_max_workers()