97 	struct io_wqe_acct acct[IO_WQ_ACCT_NR];  member
146 					struct io_wqe_acct *acct,
164 	return &wqe->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];  in io_get_acct()
186 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_worker_cancel_cb()  local
190 	atomic_dec(&acct->nr_running);  in io_worker_cancel_cb()
192 	acct->nr_workers--;  in io_worker_cancel_cb()
242 static inline bool io_acct_run_queue(struct io_wqe_acct *acct)  in io_acct_run_queue()  argument
246 	raw_spin_lock(&acct->lock);  in io_acct_run_queue()
247 	if (!wq_list_empty(&acct->work_list) &&  in io_acct_run_queue()
248 	    !test_bit(IO_ACCT_STALLED_BIT, &acct->flags))  in io_acct_run_queue()
250 	raw_spin_unlock(&acct->lock);  in io_acct_run_queue()
260 					struct io_wqe_acct *acct)  in io_wqe_activate_free_worker()  argument
274 		if (io_wqe_get_acct(worker) != acct) {  in io_wqe_activate_free_worker()
292 static bool io_wqe_create_worker(struct io_wqe *wqe, struct io_wqe_acct *acct)  in io_wqe_create_worker()  argument
298 	if (unlikely(!acct->max_workers))  in io_wqe_create_worker()
302 	if (acct->nr_workers >= acct->max_workers) {  in io_wqe_create_worker()
306 	acct->nr_workers++;  in io_wqe_create_worker()
308 	atomic_inc(&acct->nr_running);  in io_wqe_create_worker()
310 	return create_io_worker(wqe->wq, wqe, acct->index);  in io_wqe_create_worker()
315 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_wqe_inc_running()  local
317 	atomic_inc(&acct->nr_running);  in io_wqe_inc_running()
325 	struct io_wqe_acct *acct;  in create_worker_cb()  local
331 	acct = &wqe->acct[worker->create_index];  in create_worker_cb()
333 	if (acct->nr_workers < acct->max_workers) {  in create_worker_cb()
334 		acct->nr_workers++;  in create_worker_cb()
341 		atomic_dec(&acct->nr_running);  in create_worker_cb()
349 				   struct io_wqe_acct *acct,  in io_queue_worker_create()  argument
372 	worker->create_index = acct->index;  in io_queue_worker_create()
390 	atomic_dec(&acct->nr_running);  in io_queue_worker_create()
397 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_wqe_dec_running()  local
403 	if (!atomic_dec_and_test(&acct->nr_running))  in io_wqe_dec_running()
405 	if (!io_acct_run_queue(acct))  in io_wqe_dec_running()
408 	atomic_inc(&acct->nr_running);  in io_wqe_dec_running()
410 	io_queue_worker_create(worker, acct, create_worker_cb);  in io_wqe_dec_running()
466 static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,  in io_get_next_work()  argument
468 	__must_hold(acct->lock)  in io_get_next_work()
475 	wq_list_for_each(node, prev, &acct->work_list) {  in io_get_next_work()
482 			wq_list_del(&acct->work_list, node, prev);  in io_get_next_work()
493 			wq_list_cut(&acct->work_list, &tail->list, prev);  in io_get_next_work()
509 		set_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_get_next_work()
510 		raw_spin_unlock(&acct->lock);  in io_get_next_work()
512 		raw_spin_lock(&acct->lock);  in io_get_next_work()
514 			clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_get_next_work()
541 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_worker_handle_work()  local
556 		raw_spin_lock(&acct->lock);  in io_worker_handle_work()
557 		work = io_get_next_work(acct, worker);  in io_worker_handle_work()
558 		raw_spin_unlock(&acct->lock);  in io_worker_handle_work()
604 				clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_worker_handle_work()
616 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_wqe_worker()  local
631 		while (io_acct_run_queue(acct))  in io_wqe_worker()
636 		if (last_timeout && acct->nr_workers > 1) {  in io_wqe_worker()
637 			acct->nr_workers--;  in io_wqe_worker()
757 		struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in create_worker_cont()  local
759 		atomic_dec(&acct->nr_running);  in create_worker_cont()
761 		acct->nr_workers--;  in create_worker_cont()
762 		if (!acct->nr_workers) {  in create_worker_cont()
769 			while (io_acct_cancel_pending_work(wqe, acct, &match))  in create_worker_cont()
787 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);  in io_workqueue_create()  local
789 	if (!io_queue_worker_create(worker, acct, create_worker_cont))  in io_workqueue_create()
795 	struct io_wqe_acct *acct = &wqe->acct[index];  in create_io_worker()  local
804 		atomic_dec(&acct->nr_running);  in create_io_worker()
806 		acct->nr_workers--;  in create_io_worker()
879 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_insert_work()  local
885 		wq_list_add_tail(&work->list, &acct->work_list);  in io_wqe_insert_work()
895 	wq_list_add_after(&work->list, &tail->list, &acct->work_list);  in io_wqe_insert_work()
905 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_enqueue()  local
920 	raw_spin_lock(&acct->lock);  in io_wqe_enqueue()
922 	clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);  in io_wqe_enqueue()
923 	raw_spin_unlock(&acct->lock);  in io_wqe_enqueue()
927 	do_create = !io_wqe_activate_free_worker(wqe, acct);  in io_wqe_enqueue()
933 	    !atomic_read(&acct->nr_running))) {  in io_wqe_enqueue()
936 		did_create = io_wqe_create_worker(wqe, acct);  in io_wqe_enqueue()
941 		if (acct->nr_workers) {  in io_wqe_enqueue()
952 		io_acct_cancel_pending_work(wqe, acct, &match);  in io_wqe_enqueue()
1009 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_remove_pending()  local
1021 	wq_list_del(&acct->work_list, &work->list, prev);  in io_wqe_remove_pending()
1025 					struct io_wqe_acct *acct,  in io_acct_cancel_pending_work()  argument
1031 	raw_spin_lock(&acct->lock);  in io_acct_cancel_pending_work()
1032 	wq_list_for_each(node, prev, &acct->work_list) {  in io_acct_cancel_pending_work()
1037 		raw_spin_unlock(&acct->lock);  in io_acct_cancel_pending_work()
1043 	raw_spin_unlock(&acct->lock);  in io_acct_cancel_pending_work()
1054 		struct io_wqe_acct *acct = io_get_acct(wqe, i == 0);  in io_wqe_cancel_pending_work()  local
1056 		if (io_acct_cancel_pending_work(wqe, acct, match)) {  in io_wqe_cancel_pending_work()
1126 		struct io_wqe_acct *acct = &wqe->acct[i];  in io_wqe_hash_wake()  local
1128 		if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))  in io_wqe_hash_wake()
1129 			io_wqe_activate_free_worker(wqe, acct);  in io_wqe_hash_wake()
1172 		wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;  in io_wq_create()
1173 		wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers =  in io_wq_create()
1178 			struct io_wqe_acct *acct = &wqe->acct[i];  in io_wq_create()  local
1180 			acct->index = i;  in io_wq_create()
1181 			atomic_set(&acct->nr_running, 0);  in io_wq_create()
1182 			INIT_WQ_LIST(&acct->work_list);  in io_wq_create()
1183 			raw_spin_lock_init(&acct->lock);  in io_wq_create()
1379 		struct io_wqe_acct *acct;  in io_wq_max_workers()  local
1383 			acct = &wqe->acct[i];  in io_wq_max_workers()
1385 				prev[i] = max_t(int, acct->max_workers, prev[i]);  in io_wq_max_workers()
1387 				acct->max_workers = new_count[i];  in io_wq_max_workers()