24 #define bio_prio(bio)			(bio)->bi_ioprio  argument
25 #define bio_set_prio(bio, prio)		((bio)->bi_ioprio = prio)  argument
27 #define bio_iter_iovec(bio, iter)				\  argument
28 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
30 #define bio_iter_page(bio, iter)				\  argument
31 	bvec_iter_page((bio)->bi_io_vec, (iter))
32 #define bio_iter_len(bio, iter)					\  argument
33 	bvec_iter_len((bio)->bi_io_vec, (iter))
34 #define bio_iter_offset(bio, iter)				\  argument
35 	bvec_iter_offset((bio)->bi_io_vec, (iter))
37 #define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)  argument
38 #define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)  argument
39 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)  argument
41 #define bio_multiple_segments(bio)				\  argument
42 	((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
47 #define bio_sectors(bio)	bvec_iter_sectors((bio)->bi_iter)  argument
48 #define bio_end_sector(bio)	bvec_iter_end_sector((bio)->bi_iter)  argument
53 #define bio_data_dir(bio) \  argument
54 	(op_is_write(bio_op(bio)) ? WRITE : READ)
59 static inline bool bio_has_data(struct bio *bio)  in bio_has_data()  argument
61 	if (bio &&  in bio_has_data()
62 	    bio->bi_iter.bi_size &&  in bio_has_data()
63 	    bio_op(bio) != REQ_OP_DISCARD &&  in bio_has_data()
64 	    bio_op(bio) != REQ_OP_SECURE_ERASE &&  in bio_has_data()
65 	    bio_op(bio) != REQ_OP_WRITE_ZEROES)  in bio_has_data()
71 static inline bool bio_no_advance_iter(const struct bio *bio)  in bio_no_advance_iter()  argument
73 	return bio_op(bio) == REQ_OP_DISCARD ||  in bio_no_advance_iter()
74 	       bio_op(bio) == REQ_OP_SECURE_ERASE ||  in bio_no_advance_iter()
75 	       bio_op(bio) == REQ_OP_WRITE_SAME ||  in bio_no_advance_iter()
76 	       bio_op(bio) == REQ_OP_WRITE_ZEROES;  in bio_no_advance_iter()
79 static inline bool bio_mergeable(struct bio *bio)  in bio_mergeable()  argument
81 	if (bio->bi_opf & REQ_NOMERGE_FLAGS)  in bio_mergeable()
87 static inline unsigned int bio_cur_bytes(struct bio *bio)  in bio_cur_bytes()  argument
89 	if (bio_has_data(bio))  in bio_cur_bytes()
90 		return bio_iovec(bio).bv_len;  in bio_cur_bytes()
92 		return bio->bi_iter.bi_size;  in bio_cur_bytes()
95 static inline void *bio_data(struct bio *bio)  in bio_data()  argument
97 	if (bio_has_data(bio))  in bio_data()
98 		return page_address(bio_page(bio)) + bio_offset(bio);  in bio_data()
111 static inline bool bio_full(struct bio *bio, unsigned len)  in bio_full()  argument
113 	if (bio->bi_vcnt >= bio->bi_max_vecs)  in bio_full()
116 	if (bio->bi_iter.bi_size > UINT_MAX - len)  in bio_full()
122 static inline bool bio_next_segment(const struct bio *bio,  in bio_next_segment()  argument
125 	if (iter->idx >= bio->bi_vcnt)  in bio_next_segment()
128 	bvec_advance(&bio->bi_io_vec[iter->idx], iter);  in bio_next_segment()
136 #define bio_for_each_segment_all(bvl, bio, iter) \  argument
137 	for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
139 static inline void bio_advance_iter(const struct bio *bio,  in bio_advance_iter()  argument
144 	if (bio_no_advance_iter(bio))  in bio_advance_iter()
147 		bvec_iter_advance(bio->bi_io_vec, iter, bytes);  in bio_advance_iter()
151 #define __bio_for_each_segment(bvl, bio, iter, start)			\  argument
154 		((bvl = bio_iter_iovec((bio), (iter))), 1);		\
155 	     bio_advance_iter((bio), &(iter), (bvl).bv_len))
157 #define bio_for_each_segment(bvl, bio, iter)				\  argument
158 	__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
160 #define __bio_for_each_bvec(bvl, bio, iter, start)		\  argument
163 		((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
164 	     bio_advance_iter((bio), &(iter), (bvl).bv_len))
167 #define bio_for_each_bvec(bvl, bio, iter)			\  argument
168 	__bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
174 #define bio_for_each_bvec_all(bvl, bio, i)		\  argument
175 	for (i = 0, bvl = bio_first_bvec_all(bio);	\
176 	     i < (bio)->bi_vcnt; i++, bvl++)		\
180 static inline unsigned bio_segments(struct bio *bio)  in bio_segments()  argument
191 	switch (bio_op(bio)) {  in bio_segments()
202 	bio_for_each_segment(bv, bio, iter)  in bio_segments()
222 static inline void bio_get(struct bio *bio)  in bio_get()  argument
224 	bio->bi_flags |= (1 << BIO_REFFED);  in bio_get()
226 	atomic_inc(&bio->__bi_cnt);  in bio_get()
229 static inline void bio_cnt_set(struct bio *bio, unsigned int count)  in bio_cnt_set()  argument
232 		bio->bi_flags |= (1 << BIO_REFFED);  in bio_cnt_set()
235 	atomic_set(&bio->__bi_cnt, count);  in bio_cnt_set()
238 static inline bool bio_flagged(struct bio *bio, unsigned int bit)  in bio_flagged()  argument
240 	return (bio->bi_flags & (1U << bit)) != 0;  in bio_flagged()
243 static inline void bio_set_flag(struct bio *bio, unsigned int bit)  in bio_set_flag()  argument
245 	bio->bi_flags |= (1U << bit);  in bio_set_flag()
248 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)  in bio_clear_flag()  argument
250 	bio->bi_flags &= ~(1U << bit);  in bio_clear_flag()
253 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)  in bio_get_first_bvec()  argument
255 	*bv = bio_iovec(bio);  in bio_get_first_bvec()
258 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)  in bio_get_last_bvec()  argument
260 	struct bvec_iter iter = bio->bi_iter;  in bio_get_last_bvec()
263 	if (unlikely(!bio_multiple_segments(bio))) {  in bio_get_last_bvec()
264 		*bv = bio_iovec(bio);  in bio_get_last_bvec()
268 	bio_advance_iter(bio, &iter, iter.bi_size);  in bio_get_last_bvec()
275 	*bv = bio->bi_io_vec[idx];  in bio_get_last_bvec()
285 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)  in bio_first_bvec_all()  argument
287 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));  in bio_first_bvec_all()
288 	return bio->bi_io_vec;  in bio_first_bvec_all()
291 static inline struct page *bio_first_page_all(struct bio *bio)  in bio_first_page_all()  argument
293 	return bio_first_bvec_all(bio)->bv_page;  in bio_first_page_all()
296 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)  in bio_last_bvec_all()  argument
298 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));  in bio_last_bvec_all()
299 	return &bio->bi_io_vec[bio->bi_vcnt - 1];  in bio_last_bvec_all()
314 	struct bio		*bip_bio;	/* parent bio */
333 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)  in bio_integrity()  argument
335 	if (bio->bi_opf & REQ_INTEGRITY)  in bio_integrity()
336 		return bio->bi_integrity;  in bio_integrity()
341 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)  in bio_integrity_flagged()  argument
343 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_flagged()
364 extern void bio_trim(struct bio *bio, int offset, int size);
365 extern struct bio *bio_split(struct bio *bio, int sectors,
378 static inline struct bio *bio_next_split(struct bio *bio, int sectors,  in bio_next_split()  argument
381 	if (sectors >= bio_sectors(bio))  in bio_next_split()
382 		return bio;  in bio_next_split()
384 	return bio_split(bio, sectors, gfp, bs);  in bio_next_split()
396 extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
397 extern void bio_put(struct bio *);
399 extern void __bio_clone_fast(struct bio *, struct bio *);
400 extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
404 static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)  in bio_alloc()
409 static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)  in bio_kmalloc()
414 extern blk_qc_t submit_bio(struct bio *);
416 extern void bio_endio(struct bio *);
418 static inline void bio_io_error(struct bio *bio)  in bio_io_error()  argument
420 	bio->bi_status = BLK_STS_IOERR;  in bio_io_error()
421 	bio_endio(bio);  in bio_io_error()
424 static inline void bio_wouldblock_error(struct bio *bio)  in bio_wouldblock_error()  argument
426 	bio_set_flag(bio, BIO_QUIET);  in bio_wouldblock_error()
427 	bio->bi_status = BLK_STS_AGAIN;  in bio_wouldblock_error()
428 	bio_endio(bio);  in bio_wouldblock_error()
433 extern int submit_bio_wait(struct bio *bio);
434 extern void bio_advance(struct bio *, unsigned);
436 extern void bio_init(struct bio *bio, struct bio_vec *table,
438 extern void bio_uninit(struct bio *);
439 extern void bio_reset(struct bio *);
440 void bio_chain(struct bio *, struct bio *);
442 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
443 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
445 bool __bio_try_merge_page(struct bio *bio, struct page *page,
447 void __bio_add_page(struct bio *bio, struct page *page,
449 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
450 void bio_release_pages(struct bio *bio, bool mark_dirty);
451 extern void bio_set_pages_dirty(struct bio *bio);
452 extern void bio_check_pages_dirty(struct bio *bio);
454 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
455 			       struct bio *src, struct bvec_iter *src_iter);
456 extern void bio_copy_data(struct bio *dst, struct bio *src);
457 extern void bio_list_copy_data(struct bio *dst, struct bio *src);
458 extern void bio_free_pages(struct bio *bio);
459 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
460 void bio_truncate(struct bio *bio, unsigned new_size);
461 void guard_bio_eod(struct bio *bio);
463 static inline void zero_fill_bio(struct bio *bio)  in zero_fill_bio()  argument
465 	zero_fill_bio_iter(bio, bio->bi_iter);  in zero_fill_bio()
471 extern const char *bio_devname(struct bio *bio, char *buffer);
473 #define bio_set_dev(bio, bdev) 			\  argument
475 	if ((bio)->bi_disk != (bdev)->bd_disk)	\
476 		bio_clear_flag(bio, BIO_THROTTLED);\
477 	(bio)->bi_disk = (bdev)->bd_disk;	\
478 	(bio)->bi_partno = (bdev)->bd_partno;	\
479 	bio_associate_blkg(bio);		\
489 #define bio_dev(bio) \  argument
490 	disk_devt((bio)->bi_disk)
493 void bio_associate_blkg(struct bio *bio);
494 void bio_associate_blkg_from_css(struct bio *bio,
496 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
498 static inline void bio_associate_blkg(struct bio *bio) { }  in bio_associate_blkg()  argument
499 static inline void bio_associate_blkg_from_css(struct bio *bio,  in bio_associate_blkg_from_css()  argument
502 static inline void bio_clone_blkg_association(struct bio *dst,  in bio_clone_blkg_association()
503 					      struct bio *src) { }  in bio_clone_blkg_association()
555 	struct bio *head;
556 	struct bio *tail;
571 #define bio_list_for_each(bio, bl) \  argument
572 	for (bio = (bl)->head; bio; bio = bio->bi_next)
577 	struct bio *bio;  in bio_list_size()  local
579 	bio_list_for_each(bio, bl)  in bio_list_size()
585 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)  in bio_list_add()  argument
587 	bio->bi_next = NULL;  in bio_list_add()
590 		bl->tail->bi_next = bio;  in bio_list_add()
592 		bl->head = bio;  in bio_list_add()
594 	bl->tail = bio;  in bio_list_add()
597 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)  in bio_list_add_head()  argument
599 	bio->bi_next = bl->head;  in bio_list_add_head()
601 	bl->head = bio;  in bio_list_add_head()
604 		bl->tail = bio;  in bio_list_add_head()
634 static inline struct bio *bio_list_peek(struct bio_list *bl)  in bio_list_peek()
639 static inline struct bio *bio_list_pop(struct bio_list *bl)  in bio_list_pop()
641 	struct bio *bio = bl->head;  in bio_list_pop()  local
643 	if (bio) {  in bio_list_pop()
648 		bio->bi_next = NULL;  in bio_list_pop()
651 	return bio;  in bio_list_pop()
654 static inline struct bio *bio_list_get(struct bio_list *bl)  in bio_list_get()
656 	struct bio *bio = bl->head;  in bio_list_get()  local
660 	return bio;  in bio_list_get()
667 static inline void bio_inc_remaining(struct bio *bio)  in bio_inc_remaining()  argument
669 	bio_set_flag(bio, BIO_CHAIN);  in bio_inc_remaining()
671 	atomic_inc(&bio->__bi_remaining);  in bio_inc_remaining()
729 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
730 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
731 extern bool bio_integrity_prep(struct bio *);
732 extern void bio_integrity_advance(struct bio *, unsigned int);
733 extern void bio_integrity_trim(struct bio *);
734 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
741 static inline void *bio_integrity(struct bio *bio)  in bio_integrity()  argument
756 static inline bool bio_integrity_prep(struct bio *bio)  in bio_integrity_prep()  argument
761 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,  in bio_integrity_clone()  argument
767 static inline void bio_integrity_advance(struct bio *bio,  in bio_integrity_advance()  argument
773 static inline void bio_integrity_trim(struct bio *bio)  in bio_integrity_trim()  argument
783 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)  in bio_integrity_flagged()  argument
788 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,  in bio_integrity_alloc()  argument
794 static inline int bio_integrity_add_page(struct bio *bio, struct page *page,  in bio_integrity_add_page()  argument
809 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)  in bio_set_polled()  argument
811 	bio->bi_opf |= REQ_HIPRI;  in bio_set_polled()
813 		bio->bi_opf |= REQ_NOWAIT;  in bio_set_polled()