17 		struct request *prev_rq, struct bio *prev, struct bio *next)  in bio_will_gap()
30 		bio_get_first_bvec(prev_rq->bio, &pb);  in bio_will_gap()
52 static inline bool req_gap_back_merge(struct request *req, struct bio *bio)  in req_gap_back_merge()  argument
54 	return bio_will_gap(req->q, req, req->biotail, bio);  in req_gap_back_merge()
57 static inline bool req_gap_front_merge(struct request *req, struct bio *bio)  in req_gap_front_merge()  argument
59 	return bio_will_gap(req->q, NULL, bio, req->bio);  in req_gap_front_merge()
62 static struct bio *blk_bio_discard_split(struct request_queue *q,  in blk_bio_discard_split()
63 					 struct bio *bio,  in blk_bio_discard_split()  argument
86 	if (bio_sectors(bio) <= max_discard_sectors)  in blk_bio_discard_split()
97 	tmp = bio->bi_iter.bi_sector + split_sectors - alignment;  in blk_bio_discard_split()
103 	return bio_split(bio, split_sectors, GFP_NOIO, bs);  in blk_bio_discard_split()
106 static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,  in blk_bio_write_zeroes_split()
107 		struct bio *bio, struct bio_set *bs, unsigned *nsegs)  in blk_bio_write_zeroes_split()  argument
114 	if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)  in blk_bio_write_zeroes_split()
117 	return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);  in blk_bio_write_zeroes_split()
120 static struct bio *blk_bio_write_same_split(struct request_queue *q,  in blk_bio_write_same_split()
121 					    struct bio *bio,  in blk_bio_write_same_split()  argument
130 	if (bio_sectors(bio) <= q->limits.max_write_same_sectors)  in blk_bio_write_same_split()
133 	return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);  in blk_bio_write_same_split()
145 				       struct bio *bio)  in get_max_io_size()  argument
147 	unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector, 0);  in get_max_io_size()
151 	unsigned start_offset = bio->bi_iter.bi_sector & (pbs - 1);  in get_max_io_size()
245 static struct bio *blk_bio_segment_split(struct request_queue *q,  in blk_bio_segment_split()
246 					 struct bio *bio,  in blk_bio_segment_split()  argument
253 	const unsigned max_sectors = get_max_io_size(q, bio);  in blk_bio_segment_split()
256 	bio_for_each_bvec(bv, bio, iter) {  in blk_bio_segment_split()
282 	return bio_split(bio, sectors, GFP_NOIO, bs);  in blk_bio_segment_split()
298 void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)  in __blk_queue_split()  argument
300 	struct request_queue *q = (*bio)->bi_disk->queue;  in __blk_queue_split()
301 	struct bio *split = NULL;  in __blk_queue_split()
303 	switch (bio_op(*bio)) {  in __blk_queue_split()
306 		split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs);  in __blk_queue_split()
309 		split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split,  in __blk_queue_split()
313 		split = blk_bio_write_same_split(q, *bio, &q->bio_split,  in __blk_queue_split()
326 		    (*bio)->bi_vcnt == 1 &&  in __blk_queue_split()
327 		    ((*bio)->bi_io_vec[0].bv_len +  in __blk_queue_split()
328 		     (*bio)->bi_io_vec[0].bv_offset) <= PAGE_SIZE) {  in __blk_queue_split()
332 		split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);  in __blk_queue_split()
340 		bio_chain(split, *bio);  in __blk_queue_split()
341 		trace_block_split(q, split, (*bio)->bi_iter.bi_sector);  in __blk_queue_split()
342 		submit_bio_noacct(*bio);  in __blk_queue_split()
343 		*bio = split;  in __blk_queue_split()
357 void blk_queue_split(struct bio **bio)  in blk_queue_split()  argument
361 	__blk_queue_split(bio, &nr_segs);  in blk_queue_split()
372 	if (!rq->bio)  in blk_recalc_rq_segments()
375 	switch (bio_op(rq->bio)) {  in blk_recalc_rq_segments()
472 static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,  in __blk_bios_map_sg()  argument
481 	for_each_bio(bio) {  in __blk_bios_map_sg()
482 		bio_for_each_bvec(bvec, bio, iter) {  in __blk_bios_map_sg()
499 		if (likely(bio->bi_iter.bi_size)) {  in __blk_bios_map_sg()
519 	else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME)  in __blk_rq_map_sg()
520 		nsegs = __blk_bvec_map_sg(bio_iovec(rq->bio), sglist, last_sg);  in __blk_rq_map_sg()
521 	else if (rq->bio)  in __blk_rq_map_sg()
522 		nsegs = __blk_bios_map_sg(q, rq->bio, sglist, last_sg);  in __blk_rq_map_sg()
544 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,  in ll_new_hw_segment()  argument
550 	if (blk_integrity_merge_bio(req->q, req, bio) == false)  in ll_new_hw_segment()
565 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)  in ll_back_merge_fn()  argument
567 	if (req_gap_back_merge(req, bio))  in ll_back_merge_fn()
570 	    integrity_req_gap_back_merge(req, bio))  in ll_back_merge_fn()
572 	if (!bio_crypt_ctx_back_mergeable(req, bio))  in ll_back_merge_fn()
574 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_back_merge_fn()
580 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_back_merge_fn()
583 static int ll_front_merge_fn(struct request *req, struct bio *bio,  in ll_front_merge_fn()  argument
586 	if (req_gap_front_merge(req, bio))  in ll_front_merge_fn()
589 	    integrity_req_gap_front_merge(req, bio))  in ll_front_merge_fn()
591 	if (!bio_crypt_ctx_front_mergeable(req, bio))  in ll_front_merge_fn()
593 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_front_merge_fn()
594 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {  in ll_front_merge_fn()
599 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_front_merge_fn()
609 	if (blk_rq_sectors(req) + bio_sectors(next->bio) >  in req_attempt_discard_merge()
625 	if (req_gap_back_merge(req, next->bio))  in ll_merge_requests_fn()
662 	struct bio *bio;  in blk_rq_set_mixed_merge()  local
672 	for (bio = rq->bio; bio; bio = bio->bi_next) {  in blk_rq_set_mixed_merge()
673 		WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&  in blk_rq_set_mixed_merge()
674 			     (bio->bi_opf & REQ_FAILFAST_MASK) != ff);  in blk_rq_set_mixed_merge()
675 		bio->bi_opf |= ff;  in blk_rq_set_mixed_merge()
736 	    !blk_write_same_mergeable(req->bio, next->bio))  in attempt_merge()
791 	req->biotail->bi_next = next->bio;  in attempt_merge()
810 	next->bio = NULL;  in attempt_merge()
850 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)  in blk_rq_merge_ok()  argument
852 	if (!rq_mergeable(rq) || !bio_mergeable(bio))  in blk_rq_merge_ok()
855 	if (req_op(rq) != bio_op(bio))  in blk_rq_merge_ok()
859 	if (bio_data_dir(bio) != rq_data_dir(rq))  in blk_rq_merge_ok()
863 	if (rq->rq_disk != bio->bi_disk)  in blk_rq_merge_ok()
867 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)  in blk_rq_merge_ok()
871 	if (!bio_crypt_rq_ctx_compatible(rq, bio))  in blk_rq_merge_ok()
876 	    !blk_write_same_mergeable(rq->bio, bio))  in blk_rq_merge_ok()
883 	if (rq->write_hint != bio->bi_write_hint)  in blk_rq_merge_ok()
886 	if (rq->ioprio != bio_prio(bio))  in blk_rq_merge_ok()
892 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)  in blk_try_merge()  argument
896 	else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)  in blk_try_merge()
898 	else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)  in blk_try_merge()
920 		struct bio *bio, unsigned int nr_segs)  in bio_attempt_back_merge()  argument
922 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;  in bio_attempt_back_merge()
924 	if (!ll_back_merge_fn(req, bio, nr_segs))  in bio_attempt_back_merge()
927 	trace_block_bio_backmerge(req->q, req, bio);  in bio_attempt_back_merge()
928 	rq_qos_merge(req->q, req, bio);  in bio_attempt_back_merge()
933 	req->biotail->bi_next = bio;  in bio_attempt_back_merge()
934 	req->biotail = bio;  in bio_attempt_back_merge()
935 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_back_merge()
937 	bio_crypt_free_ctx(bio);  in bio_attempt_back_merge()
944 		struct bio *bio, unsigned int nr_segs)  in bio_attempt_front_merge()  argument
946 	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;  in bio_attempt_front_merge()
948 	if (!ll_front_merge_fn(req, bio, nr_segs))  in bio_attempt_front_merge()
951 	trace_block_bio_frontmerge(req->q, req, bio);  in bio_attempt_front_merge()
952 	rq_qos_merge(req->q, req, bio);  in bio_attempt_front_merge()
957 	bio->bi_next = req->bio;  in bio_attempt_front_merge()
958 	req->bio = bio;  in bio_attempt_front_merge()
960 	req->__sector = bio->bi_iter.bi_sector;  in bio_attempt_front_merge()
961 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_front_merge()
963 	bio_crypt_do_front_merge(req, bio);  in bio_attempt_front_merge()
970 		struct request *req, struct bio *bio)  in bio_attempt_discard_merge()  argument
976 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in bio_attempt_discard_merge()
980 	rq_qos_merge(q, req, bio);  in bio_attempt_discard_merge()
982 	req->biotail->bi_next = bio;  in bio_attempt_discard_merge()
983 	req->biotail = bio;  in bio_attempt_discard_merge()
984 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_discard_merge()
996 						   struct bio *bio,  in blk_attempt_bio_merge()  argument
1000 	if (!blk_rq_merge_ok(rq, bio))  in blk_attempt_bio_merge()
1003 	switch (blk_try_merge(rq, bio)) {  in blk_attempt_bio_merge()
1005 		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))  in blk_attempt_bio_merge()
1006 			return bio_attempt_back_merge(rq, bio, nr_segs);  in blk_attempt_bio_merge()
1009 		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))  in blk_attempt_bio_merge()
1010 			return bio_attempt_front_merge(rq, bio, nr_segs);  in blk_attempt_bio_merge()
1013 		return bio_attempt_discard_merge(q, rq, bio);  in blk_attempt_bio_merge()
1043 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,  in blk_attempt_plug_merge()  argument
1050 	plug = blk_mq_plug(q, bio);  in blk_attempt_plug_merge()
1069 		if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==  in blk_attempt_plug_merge()
1082 			struct bio *bio, unsigned int nr_segs)  in blk_bio_list_merge()  argument
1091 		switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {  in blk_bio_list_merge()
1106 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,  in blk_mq_sched_try_merge()  argument
1111 	switch (elv_merge(q, &rq, bio)) {  in blk_mq_sched_try_merge()
1113 		if (!blk_mq_sched_allow_merge(q, rq, bio))  in blk_mq_sched_try_merge()
1115 		if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)  in blk_mq_sched_try_merge()
1122 		if (!blk_mq_sched_allow_merge(q, rq, bio))  in blk_mq_sched_try_merge()
1124 		if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)  in blk_mq_sched_try_merge()
1131 		return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;  in blk_mq_sched_try_merge()