23 typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t);
26  * request flags */
31 /* request for flush sequence */
41 /* use an I/O scheduler for this request */
49 /* runtime pm request */
58 /* The per-zone write lock is held for this request */
80 struct request {  struct
102 		struct request *rq_next;  argument
107 	/* Time that the first bio started allocating this request. */  argument
110 	/* Time that this request was allocated for this IO. */  argument
149 	 * request reaches the dispatch list. The ipi_list is only used  argument
150 	 * to queue the request for softirq completion, which is long
151 	 * after the request has been unhashed (and even removed from
193 static inline enum req_op req_op(const struct request *req)  in req_op()  argument
198 static inline bool blk_rq_is_passthrough(struct request *rq)  in blk_rq_is_passthrough()
203 static inline unsigned short req_get_ioprio(struct request *req)  in req_get_ioprio()
226 	struct request *__req = NULL;			\
236 	struct request *__req = NULL;			\
250 #define rq_list_empty(list)	((list) == (struct request *) NULL)
253  * rq_list_move() - move a struct request from one list to another
256  * @rq: The request to move
257  * @prev: The request preceding @rq in @src (NULL if @rq is the head)
259 static inline void rq_list_move(struct request **src, struct request **dst,  in rq_list_move()
260 				struct request *rq, struct request *prev)  in rq_list_move()
273  * @BLK_EH_RESET_TIMER: Reset the request timer and continue waiting for the
274  *	request to complete.
328 	 * @sched_data: Pointer owned by the IO scheduler attached to a request
333 	 * @queue: Pointer to the request queue that owns this hardware context.
347 	 * pending request in that software queue.
386 	 * assigned when a request is dispatched from a hardware queue.
391 	 * scheduler associated with a request queue, a tag is assigned when
392 	 * that request is allocated. Else, this member is not used.
406 	 * shared across request queues.
410 	/** @cpuhp_online: List to store request if CPU is going to die */
412 	/** @cpuhp_dead: List to store request if some CPU die. */
466  * struct blk_mq_tag_set - tag set that can be shared between request queues
480  * @cmd_size:	   Number of additional bytes to allocate per request. The block
483  * @timeout:	   Request processing timeout in jiffies.
493  * @tag_list:	   List of the request queues that use this tag set. See also
495  * @srcu:	   Use as lock when type of the request queue is blocking
521  * struct blk_mq_queue_data - Data about a request inserted in a queue
523  * @rq:   Request pointer.
524  * @last: If it is the last request in the queue.
527 	struct request *rq;
531 typedef bool (busy_tag_iter_fn)(struct request *, void *);
539 	 * @queue_rq: Queue a new request from block IO.
548 	 * purpose of kicking the hardware (which the last request otherwise
555 	 * that each request belongs to the same queue. If the driver doesn't
559 	void (*queue_rqs)(struct request **rqlist);
562 	 * @get_budget: Reserve budget before queue request, once .queue_rq is
577 	void (*set_rq_budget_token)(struct request *, int);
581 	int (*get_rq_budget_token)(struct request *);
584 	 * @timeout: Called on request timeout.
586 	enum blk_eh_timer_return (*timeout)(struct request *);
594 	 * @complete: Mark the request as complete.
596 	void (*complete)(struct request *);
614 	 * flush request.
616 	int (*init_request)(struct blk_mq_tag_set *set, struct request *,
621 	void (*exit_request)(struct blk_mq_tag_set *set, struct request *,
625 	 * @cleanup_rq: Called before freeing one request which isn't completed
628 	void (*cleanup_rq)(struct request *);
644 	 * information about a request.
646 	void (*show_rq)(struct seq_file *m, struct request *rq);
711 void blk_mq_free_request(struct request *rq);
712 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
726 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
728 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
743 	struct request **rqs;
744 	struct request **static_rqs;
748 	 * used to clear request reference in rqs[] before freeing one
749 	 * request pool
754 static inline struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags,  in blk_mq_tag_to_rq()
770 u32 blk_mq_unique_tag(struct request *rq);
783  * blk_mq_rq_state() - read the current MQ_RQ_* state of a request
784  * @rq: target request.
786 static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)  in blk_mq_rq_state()
791 static inline int blk_mq_request_started(struct request *rq)  in blk_mq_request_started()
796 static inline int blk_mq_request_completed(struct request *rq)  in blk_mq_request_completed()
803  * Set the state to complete when completing a request from inside ->queue_rq.
805  * need access to the request are called on failure, e.g. by nvme for
808 static inline void blk_mq_set_request_complete(struct request *rq)  in blk_mq_set_request_complete()
814  * Complete the request directly instead of deferring it to softirq or
817 static inline void blk_mq_complete_request_direct(struct request *rq,  in blk_mq_complete_request_direct()
818 		   void (*complete)(struct request *rq))  in blk_mq_complete_request_direct()
824 void blk_mq_start_request(struct request *rq);
825 void blk_mq_end_request(struct request *rq, blk_status_t error);
826 void __blk_mq_end_request(struct request *rq, blk_status_t error);
833 static inline bool blk_mq_need_time_stamp(struct request *rq)  in blk_mq_need_time_stamp()
838 static inline bool blk_mq_is_reserved_rq(struct request *rq)  in blk_mq_is_reserved_rq()
847 static inline bool blk_mq_add_to_batch(struct request *req,  in blk_mq_add_to_batch()
852 	 * blk_mq_end_request_batch() can't end request allocated from  in blk_mq_add_to_batch()
868 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
871 void blk_mq_complete_request(struct request *rq);
872 bool blk_mq_complete_request_remote(struct request *rq);
903 unsigned int blk_mq_rq_cpu(struct request *rq);
915  * blk_mq_rq_from_pdu - cast a PDU to a request
918  * Return: request
920  * Driver command data is immediately after the request. So subtract request
921  * size to get back to the original request.
923 static inline struct request *blk_mq_rq_from_pdu(void *pdu)  in blk_mq_rq_from_pdu()
925 	return pdu - sizeof(struct request);  in blk_mq_rq_from_pdu()
929  * blk_mq_rq_to_pdu - cast a request to a PDU
930  * @rq: the request to be casted
934  * Driver command data is immediately after the request. So add request to get
937 static inline void *blk_mq_rq_to_pdu(struct request *rq)  in blk_mq_rq_to_pdu()
949 static inline void blk_mq_cleanup_rq(struct request *rq)  in blk_mq_cleanup_rq()
955 static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,  in blk_rq_bio_prep()
967 static inline bool rq_is_sync(struct request *rq)  in rq_is_sync()
972 void blk_rq_init(struct request_queue *q, struct request *rq);
973 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
976 void blk_rq_unprep_clone(struct request *rq);
977 blk_status_t blk_insert_cloned_request(struct request *rq);
988 int blk_rq_map_user(struct request_queue *, struct request *,
990 int blk_rq_map_user_io(struct request *, struct rq_map_data *,
992 int blk_rq_map_user_iov(struct request_queue *, struct request *,
995 int blk_rq_map_kern(struct request_queue *, struct request *, void *,
997 int blk_rq_append_bio(struct request *rq, struct bio *bio);
998 void blk_execute_rq_nowait(struct request *rq, bool at_head);
999 blk_status_t blk_execute_rq(struct request *rq, bool at_head);
1000 bool blk_rq_is_poll(struct request *rq);
1025  * blk_rq_bytes()		: bytes left in the entire request
1027  * blk_rq_sectors()		: sectors left in the entire request
1029  * blk_rq_stats_sectors()	: sectors of the entire request used for stats
1031 static inline sector_t blk_rq_pos(const struct request *rq)  in blk_rq_pos()
1036 static inline unsigned int blk_rq_bytes(const struct request *rq)  in blk_rq_bytes()
1041 static inline int blk_rq_cur_bytes(const struct request *rq)  in blk_rq_cur_bytes()
1050 static inline unsigned int blk_rq_sectors(const struct request *rq)  in blk_rq_sectors()
1055 static inline unsigned int blk_rq_cur_sectors(const struct request *rq)  in blk_rq_cur_sectors()
1060 static inline unsigned int blk_rq_stats_sectors(const struct request *rq)  in blk_rq_stats_sectors()
1067  * is different from the size of the request.  Any driver that supports such
1071 static inline unsigned int blk_rq_payload_bytes(struct request *rq)  in blk_rq_payload_bytes()
1079  * Return the first full biovec in the request.  The caller needs to check that
1082 static inline struct bio_vec req_bvec(struct request *rq)  in req_bvec()
1089 static inline unsigned int blk_rq_count_bios(struct request *rq)  in blk_rq_count_bios()
1100 void blk_steal_bios(struct bio_list *list, struct request *rq);
1103  * Request completion related functions.
1106  * the request without completing it.
1108 bool blk_update_request(struct request *rq, blk_status_t error,
1110 void blk_abort_request(struct request *);
1121 static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)  in blk_rq_nr_phys_segments()
1130  * Each discard bio merged into a request is counted as one segment.
1132 static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)  in blk_rq_nr_discard_segments()
1137 int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
1139 static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,  in blk_rq_map_sg()
1146 void blk_dump_rq_flags(struct request *, char *);
1149 static inline unsigned int blk_rq_zone_no(struct request *rq)  in blk_rq_zone_no()
1154 static inline unsigned int blk_rq_zone_is_seq(struct request *rq)  in blk_rq_zone_is_seq()
1161  * @rq: Request to examine.
1165 static inline bool blk_rq_is_seq_zoned_write(struct request *rq)  in blk_rq_is_seq_zoned_write()
1171 bool blk_req_needs_zone_write_lock(struct request *rq);
1172 bool blk_req_zone_write_trylock(struct request *rq);
1173 void __blk_req_zone_write_lock(struct request *rq);
1174 void __blk_req_zone_write_unlock(struct request *rq);
1176 static inline void blk_req_zone_write_lock(struct request *rq)  in blk_req_zone_write_lock()
1182 static inline void blk_req_zone_write_unlock(struct request *rq)  in blk_req_zone_write_unlock()
1188 static inline bool blk_req_zone_is_write_locked(struct request *rq)  in blk_req_zone_is_write_locked()
1194 static inline bool blk_req_can_dispatch_to_zone(struct request *rq)  in blk_req_can_dispatch_to_zone()
1201 static inline bool blk_rq_is_seq_zoned_write(struct request *rq)  in blk_rq_is_seq_zoned_write()
1206 static inline bool blk_req_needs_zone_write_lock(struct request *rq)  in blk_req_needs_zone_write_lock()
1211 static inline void blk_req_zone_write_lock(struct request *rq)  in blk_req_zone_write_lock()
1215 static inline void blk_req_zone_write_unlock(struct request *rq)  in blk_req_zone_write_unlock()
1218 static inline bool blk_req_zone_is_write_locked(struct request *rq)  in blk_req_zone_is_write_locked()
1223 static inline bool blk_req_can_dispatch_to_zone(struct request *rq)  in blk_req_can_dispatch_to_zone()