30 	IO_WORKER_F_BOUND	= 8,	/* is doing bounded work */
64 		struct work_struct work;  member
168 						   struct io_wq_work *work)  in io_work_get_acct()  argument
170 	return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND));  in io_work_get_acct()
295 	 * Most likely an attempt to queue unbounded work on an io_wq that  in io_wqe_create_worker()
378 		 * work item after we canceled in io_wq_exit_workers().  in io_queue_worker_create()
414  * Worker will start processing some work. Move it to the busy list, if
428  * No work, worker going to sleep. Move to freelist, and unuse mm if we
443 static inline unsigned int io_get_work_hash(struct io_wq_work *work)  in io_get_work_hash()  argument
445 	return work->flags >> IO_WQ_HASH_SHIFT;  in io_get_work_hash()
471 	struct io_wq_work *work, *tail;  in io_get_next_work()  local
478 		work = container_of(node, struct io_wq_work, list);  in io_get_next_work()
481 		if (!io_wq_is_hashed(work)) {  in io_get_next_work()
483 			return work;  in io_get_next_work()
486 		hash = io_get_work_hash(work);  in io_get_next_work()
487 		/* all items with this hash lie in [work, tail] */  in io_get_next_work()
494 			return work;  in io_get_next_work()
507 		 * work being added and clearing the stalled bit.  in io_get_next_work()
524 				   struct io_wq_work *work)  in io_assign_current_work()  argument
526 	if (work) {  in io_assign_current_work()
532 	worker->cur_work = work;  in io_assign_current_work()
537 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
547 		struct io_wq_work *work;  in io_worker_handle_work()  local
550 		 * If we got some work, mark us as busy. If we didn't, but  in io_worker_handle_work()
551 		 * the list isn't empty, it means we stalled on hashed work.  in io_worker_handle_work()
552 		 * Mark us stalled so we don't keep looking for work when we  in io_worker_handle_work()
553 		 * can't make progress, any work completion or insertion will  in io_worker_handle_work()
557 		work = io_get_next_work(acct, worker);  in io_worker_handle_work()
559 		if (work) {  in io_worker_handle_work()
564 			 * it becomes the active work. That avoids a window  in io_worker_handle_work()
565 			 * where the work has been removed from our general  in io_worker_handle_work()
566 			 * work list, but isn't yet discoverable as the  in io_worker_handle_work()
567 			 * current work item for this worker.  in io_worker_handle_work()
570 			worker->next_work = work;  in io_worker_handle_work()
575 		io_assign_current_work(worker, work);  in io_worker_handle_work()
581 			unsigned int hash = io_get_work_hash(work);  in io_worker_handle_work()
583 			next_hashed = wq_next_work(work);  in io_worker_handle_work()
585 			if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND))  in io_worker_handle_work()
586 				work->flags |= IO_WQ_WORK_CANCEL;  in io_worker_handle_work()
587 			wq->do_work(work);  in io_worker_handle_work()
590 			linked = wq->free_work(work);  in io_worker_handle_work()
591 			work = next_hashed;  in io_worker_handle_work()
592 			if (!work && linked && !io_wq_is_hashed(linked)) {  in io_worker_handle_work()
593 				work = linked;  in io_worker_handle_work()
596 			io_assign_current_work(worker, work);  in io_worker_handle_work()
609 		} while (work);  in io_worker_handle_work()
684  * running and we have work pending, wake up a free one or create a new one.
717 static bool io_wq_work_match_all(struct io_wq_work *work, void *data)  in io_wq_work_match_all()  argument
781 	schedule_work(&worker->work);  in create_worker_cont()
784 static void io_workqueue_create(struct work_struct *work)  in io_workqueue_create()  argument
786 	struct io_worker *worker = container_of(work, struct io_worker, work);  in io_workqueue_create()
827 		INIT_WORK(&worker->work, io_workqueue_create);  in create_io_worker()
828 		schedule_work(&worker->work);  in create_io_worker()
866 static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)  in io_run_cancel()  argument
871 		work->flags |= IO_WQ_WORK_CANCEL;  in io_run_cancel()
872 		wq->do_work(work);  in io_run_cancel()
873 		work = wq->free_work(work);  in io_run_cancel()
874 	} while (work);  in io_run_cancel()
877 static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work)  in io_wqe_insert_work()  argument
879 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_insert_work()
883 	if (!io_wq_is_hashed(work)) {  in io_wqe_insert_work()
885 		wq_list_add_tail(&work->list, &acct->work_list);  in io_wqe_insert_work()
889 	hash = io_get_work_hash(work);  in io_wqe_insert_work()
891 	wqe->hash_tail[hash] = work;  in io_wqe_insert_work()
895 	wq_list_add_after(&work->list, &tail->list, &acct->work_list);  in io_wqe_insert_work()
898 static bool io_wq_work_match_item(struct io_wq_work *work, void *data)  in io_wq_work_match_item()  argument
900 	return work == data;  in io_wq_work_match_item()
903 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)  in io_wqe_enqueue()  argument
905 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_enqueue()
907 	unsigned work_flags = work->flags;  in io_wqe_enqueue()
915 	    (work->flags & IO_WQ_WORK_CANCEL)) {  in io_wqe_enqueue()
916 		io_run_cancel(work, wqe);  in io_wqe_enqueue()
921 	io_wqe_insert_work(wqe, work);  in io_wqe_enqueue()
949 		match.data		= work,  in io_wqe_enqueue()
956 void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)  in io_wq_enqueue()  argument
960 	io_wqe_enqueue(wqe, work);  in io_wq_enqueue()
964  * Work items that hash to the same value will not be done in parallel.
967 void io_wq_hash_work(struct io_wq_work *work, void *val)  in io_wq_hash_work()  argument
972 	work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));  in io_wq_hash_work()
977 				  struct io_wq_work *work)  in __io_wq_worker_cancel()  argument
979 	if (work && match->fn(work, match->data)) {  in __io_wq_worker_cancel()
980 		work->flags |= IO_WQ_WORK_CANCEL;  in __io_wq_worker_cancel()
994 	 * may dereference the passed in work.  in io_wq_worker_cancel()
1006 					 struct io_wq_work *work,  in io_wqe_remove_pending()  argument
1009 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_remove_pending()
1010 	unsigned int hash = io_get_work_hash(work);  in io_wqe_remove_pending()
1013 	if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) {  in io_wqe_remove_pending()
1021 	wq_list_del(&acct->work_list, &work->list, prev);  in io_wqe_remove_pending()
1029 	struct io_wq_work *work;  in io_acct_cancel_pending_work()  local
1033 		work = container_of(node, struct io_wq_work, list);  in io_acct_cancel_pending_work()
1034 		if (!match->fn(work, match->data))  in io_acct_cancel_pending_work()
1036 		io_wqe_remove_pending(wqe, work, prev);  in io_acct_cancel_pending_work()
1038 		io_run_cancel(work, wqe);  in io_acct_cancel_pending_work()
1084 	 * from there. CANCEL_OK means that the work is returned as-new,  in io_wq_cancel_cb()
1087 	 * Then check if a free (going busy) or busy worker has the work  in io_wq_cancel_cb()
1093 	 * we'll find a work item regardless of state.  in io_wq_cancel_cb()