211 static inline bool blk_rq_is_passthrough(struct request *rq)  in blk_rq_is_passthrough()  argument
213 	return blk_op_is_passthrough(req_op(rq));  in blk_rq_is_passthrough()
221 #define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)  argument
223 #define rq_dma_dir(rq) \  argument
224 	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
226 #define rq_list_add(listptr, rq)	do {		\  argument
227 	(rq)->rq_next = *(listptr);			\
228 	*(listptr) = rq;				\
256 #define rq_list_next(rq)	(rq)->rq_next  argument
267 				struct request *rq, struct request *prev)  in rq_list_move()  argument
270 		prev->rq_next = rq->rq_next;  in rq_list_move()
272 		*src = rq->rq_next;  in rq_list_move()
273 	rq_list_add(dst, rq);  in rq_list_move()
533 	struct request *rq;  member
652 	void (*show_rq)(struct seq_file *m, struct request *rq);
717 void blk_mq_free_request(struct request *rq);
775 u32 blk_mq_unique_tag(struct request *rq);
791 static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)  in blk_mq_rq_state()  argument
793 	return READ_ONCE(rq->state);  in blk_mq_rq_state()
796 static inline int blk_mq_request_started(struct request *rq)  in blk_mq_request_started()  argument
798 	return blk_mq_rq_state(rq) != MQ_RQ_IDLE;  in blk_mq_request_started()
801 static inline int blk_mq_request_completed(struct request *rq)  in blk_mq_request_completed()  argument
803 	return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;  in blk_mq_request_completed()
813 static inline void blk_mq_set_request_complete(struct request *rq)  in blk_mq_set_request_complete()  argument
815 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);  in blk_mq_set_request_complete()
822 static inline void blk_mq_complete_request_direct(struct request *rq,  in blk_mq_complete_request_direct()  argument
823 		   void (*complete)(struct request *rq))  in blk_mq_complete_request_direct()  argument
825 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);  in blk_mq_complete_request_direct()
826 	complete(rq);  in blk_mq_complete_request_direct()
829 void blk_mq_start_request(struct request *rq);
830 void blk_mq_end_request(struct request *rq, blk_status_t error);
831 void __blk_mq_end_request(struct request *rq, blk_status_t error);
838 static inline bool blk_mq_need_time_stamp(struct request *rq)  in blk_mq_need_time_stamp()  argument
840 	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));  in blk_mq_need_time_stamp()
843 static inline bool blk_mq_is_reserved_rq(struct request *rq)  in blk_mq_is_reserved_rq()  argument
845 	return rq->rq_flags & RQF_RESV;  in blk_mq_is_reserved_rq()
869 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
872 void blk_mq_complete_request(struct request *rq);
873 bool blk_mq_complete_request_remote(struct request *rq);
902 unsigned int blk_mq_rq_cpu(struct request *rq);
936 static inline void *blk_mq_rq_to_pdu(struct request *rq)  in blk_mq_rq_to_pdu()  argument
938 	return rq + 1;  in blk_mq_rq_to_pdu()
948 static inline void blk_mq_cleanup_rq(struct request *rq)  in blk_mq_cleanup_rq()  argument
950 	if (rq->q->mq_ops->cleanup_rq)  in blk_mq_cleanup_rq()
951 		rq->q->mq_ops->cleanup_rq(rq);  in blk_mq_cleanup_rq()
954 static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,  in blk_rq_bio_prep()  argument
957 	rq->nr_phys_segments = nr_segs;  in blk_rq_bio_prep()
958 	rq->__data_len = bio->bi_iter.bi_size;  in blk_rq_bio_prep()
959 	rq->bio = rq->biotail = bio;  in blk_rq_bio_prep()
960 	rq->ioprio = bio_prio(bio);  in blk_rq_bio_prep()
966 static inline bool rq_is_sync(struct request *rq)  in rq_is_sync()  argument
968 	return op_is_sync(rq->cmd_flags);  in rq_is_sync()
971 void blk_rq_init(struct request_queue *q, struct request *rq);
972 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
975 void blk_rq_unprep_clone(struct request *rq);
976 blk_status_t blk_insert_cloned_request(struct request *rq);
996 int blk_rq_append_bio(struct request *rq, struct bio *bio);
997 void blk_execute_rq_nowait(struct request *rq, bool at_head);
998 blk_status_t blk_execute_rq(struct request *rq, bool at_head);
999 bool blk_rq_is_poll(struct request *rq);
1006 #define __rq_for_each_bio(_bio, rq)	\  argument
1007 	if ((rq->bio))			\
1008 		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
1030 static inline sector_t blk_rq_pos(const struct request *rq)  in blk_rq_pos()  argument
1032 	return rq->__sector;  in blk_rq_pos()
1035 static inline unsigned int blk_rq_bytes(const struct request *rq)  in blk_rq_bytes()  argument
1037 	return rq->__data_len;  in blk_rq_bytes()
1040 static inline int blk_rq_cur_bytes(const struct request *rq)  in blk_rq_cur_bytes()  argument
1042 	if (!rq->bio)  in blk_rq_cur_bytes()
1044 	if (!bio_has_data(rq->bio))	/* dataless requests such as discard */  in blk_rq_cur_bytes()
1045 		return rq->bio->bi_iter.bi_size;  in blk_rq_cur_bytes()
1046 	return bio_iovec(rq->bio).bv_len;  in blk_rq_cur_bytes()
1049 static inline unsigned int blk_rq_sectors(const struct request *rq)  in blk_rq_sectors()  argument
1051 	return blk_rq_bytes(rq) >> SECTOR_SHIFT;  in blk_rq_sectors()
1054 static inline unsigned int blk_rq_cur_sectors(const struct request *rq)  in blk_rq_cur_sectors()  argument
1056 	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;  in blk_rq_cur_sectors()
1059 static inline unsigned int blk_rq_stats_sectors(const struct request *rq)  in blk_rq_stats_sectors()  argument
1061 	return rq->stats_sectors;  in blk_rq_stats_sectors()
1070 static inline unsigned int blk_rq_payload_bytes(struct request *rq)  in blk_rq_payload_bytes()  argument
1072 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in blk_rq_payload_bytes()
1073 		return rq->special_vec.bv_len;  in blk_rq_payload_bytes()
1074 	return blk_rq_bytes(rq);  in blk_rq_payload_bytes()
1081 static inline struct bio_vec req_bvec(struct request *rq)  in req_bvec()  argument
1083 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in req_bvec()
1084 		return rq->special_vec;  in req_bvec()
1085 	return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);  in req_bvec()
1088 static inline unsigned int blk_rq_count_bios(struct request *rq)  in blk_rq_count_bios()  argument
1093 	__rq_for_each_bio(bio, rq)  in blk_rq_count_bios()
1099 void blk_steal_bios(struct bio_list *list, struct request *rq);
1107 bool blk_update_request(struct request *rq, blk_status_t error,
1120 static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)  in blk_rq_nr_phys_segments()  argument
1122 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)  in blk_rq_nr_phys_segments()
1124 	return rq->nr_phys_segments;  in blk_rq_nr_phys_segments()
1131 static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)  in blk_rq_nr_discard_segments()  argument
1133 	return max_t(unsigned short, rq->nr_phys_segments, 1);  in blk_rq_nr_discard_segments()
1136 int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
1138 static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,  in blk_rq_map_sg()  argument
1143 	return __blk_rq_map_sg(q, rq, sglist, &last_sg);  in blk_rq_map_sg()
1148 static inline unsigned int blk_rq_zone_no(struct request *rq)  in blk_rq_zone_no()  argument
1150 	return disk_zone_no(rq->q->disk, blk_rq_pos(rq));  in blk_rq_zone_no()
1153 static inline unsigned int blk_rq_zone_is_seq(struct request *rq)  in blk_rq_zone_is_seq()  argument
1155 	return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq));  in blk_rq_zone_is_seq()
1158 bool blk_req_needs_zone_write_lock(struct request *rq);
1159 bool blk_req_zone_write_trylock(struct request *rq);
1160 void __blk_req_zone_write_lock(struct request *rq);
1161 void __blk_req_zone_write_unlock(struct request *rq);
1163 static inline void blk_req_zone_write_lock(struct request *rq)  in blk_req_zone_write_lock()  argument
1165 	if (blk_req_needs_zone_write_lock(rq))  in blk_req_zone_write_lock()
1166 		__blk_req_zone_write_lock(rq);  in blk_req_zone_write_lock()
1169 static inline void blk_req_zone_write_unlock(struct request *rq)  in blk_req_zone_write_unlock()  argument
1171 	if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)  in blk_req_zone_write_unlock()
1172 		__blk_req_zone_write_unlock(rq);  in blk_req_zone_write_unlock()
1175 static inline bool blk_req_zone_is_write_locked(struct request *rq)  in blk_req_zone_is_write_locked()  argument
1177 	return rq->q->disk->seq_zones_wlock &&  in blk_req_zone_is_write_locked()
1178 		test_bit(blk_rq_zone_no(rq), rq->q->disk->seq_zones_wlock);  in blk_req_zone_is_write_locked()
1181 static inline bool blk_req_can_dispatch_to_zone(struct request *rq)  in blk_req_can_dispatch_to_zone()  argument
1183 	if (!blk_req_needs_zone_write_lock(rq))  in blk_req_can_dispatch_to_zone()
1185 	return !blk_req_zone_is_write_locked(rq);  in blk_req_can_dispatch_to_zone()
1188 static inline bool blk_req_needs_zone_write_lock(struct request *rq)  in blk_req_needs_zone_write_lock()  argument
1193 static inline void blk_req_zone_write_lock(struct request *rq)  in blk_req_zone_write_lock()  argument
1197 static inline void blk_req_zone_write_unlock(struct request *rq)  in blk_req_zone_write_unlock()  argument
1200 static inline bool blk_req_zone_is_write_locked(struct request *rq)  in blk_req_zone_is_write_locked()  argument
1205 static inline bool blk_req_can_dispatch_to_zone(struct request *rq)  in blk_req_can_dispatch_to_zone()  argument