52 static inline bool req_gap_back_merge(struct request *req, struct bio *bio)  in req_gap_back_merge()  argument
54 	return bio_will_gap(req->q, req, req->biotail, bio);  in req_gap_back_merge()
57 static inline bool req_gap_front_merge(struct request *req, struct bio *bio)  in req_gap_front_merge()  argument
59 	return bio_will_gap(req->q, NULL, bio, req->bio);  in req_gap_front_merge()
544 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,  in ll_new_hw_segment()  argument
547 	if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))  in ll_new_hw_segment()
550 	if (blk_integrity_merge_bio(req->q, req, bio) == false)  in ll_new_hw_segment()
557 	req->nr_phys_segments += nr_phys_segs;  in ll_new_hw_segment()
561 	req_set_nomerge(req->q, req);  in ll_new_hw_segment()
565 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)  in ll_back_merge_fn()  argument
567 	if (req_gap_back_merge(req, bio))  in ll_back_merge_fn()
569 	if (blk_integrity_rq(req) &&  in ll_back_merge_fn()
570 	    integrity_req_gap_back_merge(req, bio))  in ll_back_merge_fn()
572 	if (!bio_crypt_ctx_back_mergeable(req, bio))  in ll_back_merge_fn()
574 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_back_merge_fn()
575 	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {  in ll_back_merge_fn()
576 		req_set_nomerge(req->q, req);  in ll_back_merge_fn()
580 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_back_merge_fn()
583 static int ll_front_merge_fn(struct request *req, struct bio *bio,  in ll_front_merge_fn()  argument
586 	if (req_gap_front_merge(req, bio))  in ll_front_merge_fn()
588 	if (blk_integrity_rq(req) &&  in ll_front_merge_fn()
589 	    integrity_req_gap_front_merge(req, bio))  in ll_front_merge_fn()
591 	if (!bio_crypt_ctx_front_mergeable(req, bio))  in ll_front_merge_fn()
593 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in ll_front_merge_fn()
594 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {  in ll_front_merge_fn()
595 		req_set_nomerge(req->q, req);  in ll_front_merge_fn()
599 	return ll_new_hw_segment(req, bio, nr_segs);  in ll_front_merge_fn()
602 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,  in req_attempt_discard_merge()  argument
605 	unsigned short segments = blk_rq_nr_discard_segments(req);  in req_attempt_discard_merge()
609 	if (blk_rq_sectors(req) + bio_sectors(next->bio) >  in req_attempt_discard_merge()
610 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in req_attempt_discard_merge()
613 	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);  in req_attempt_discard_merge()
616 	req_set_nomerge(q, req);  in req_attempt_discard_merge()
620 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,  in ll_merge_requests_fn()  argument
625 	if (req_gap_back_merge(req, next->bio))  in ll_merge_requests_fn()
631 	if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >  in ll_merge_requests_fn()
632 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in ll_merge_requests_fn()
635 	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;  in ll_merge_requests_fn()
636 	if (total_phys_segments > blk_rq_get_max_segments(req))  in ll_merge_requests_fn()
639 	if (blk_integrity_merge_rq(q, req, next) == false)  in ll_merge_requests_fn()
642 	if (!bio_crypt_ctx_merge_rq(req, next))  in ll_merge_requests_fn()
646 	req->nr_phys_segments = total_phys_segments;  in ll_merge_requests_fn()
680 static void blk_account_io_merge_request(struct request *req)  in blk_account_io_merge_request()  argument
682 	if (blk_do_io_stat(req)) {  in blk_account_io_merge_request()
684 		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);  in blk_account_io_merge_request()
687 		hd_struct_put(req->part);  in blk_account_io_merge_request()
699 static inline bool blk_discard_mergable(struct request *req)  in blk_discard_mergable()  argument
701 	if (req_op(req) == REQ_OP_DISCARD &&  in blk_discard_mergable()
702 	    queue_max_discard_segments(req->q) > 1)  in blk_discard_mergable()
707 static enum elv_merge blk_try_req_merge(struct request *req,  in blk_try_req_merge()  argument
710 	if (blk_discard_mergable(req))  in blk_try_req_merge()
712 	else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))  in blk_try_req_merge()
723 				     struct request *req, struct request *next)  in attempt_merge()  argument
725 	if (!rq_mergeable(req) || !rq_mergeable(next))  in attempt_merge()
728 	if (req_op(req) != req_op(next))  in attempt_merge()
731 	if (rq_data_dir(req) != rq_data_dir(next)  in attempt_merge()
732 	    || req->rq_disk != next->rq_disk)  in attempt_merge()
735 	if (req_op(req) == REQ_OP_WRITE_SAME &&  in attempt_merge()
736 	    !blk_write_same_mergeable(req->bio, next->bio))  in attempt_merge()
743 	if (req->write_hint != next->write_hint)  in attempt_merge()
746 	if (req->ioprio != next->ioprio)  in attempt_merge()
757 	switch (blk_try_req_merge(req, next)) {  in attempt_merge()
759 		if (!req_attempt_discard_merge(q, req, next))  in attempt_merge()
763 		if (!ll_merge_requests_fn(q, req, next))  in attempt_merge()
776 	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||  in attempt_merge()
777 	    (req->cmd_flags & REQ_FAILFAST_MASK) !=  in attempt_merge()
779 		blk_rq_set_mixed_merge(req);  in attempt_merge()
788 	if (next->start_time_ns < req->start_time_ns)  in attempt_merge()
789 		req->start_time_ns = next->start_time_ns;  in attempt_merge()
791 	req->biotail->bi_next = next->bio;  in attempt_merge()
792 	req->biotail = next->biotail;  in attempt_merge()
794 	req->__data_len += blk_rq_bytes(next);  in attempt_merge()
796 	if (!blk_discard_mergable(req))  in attempt_merge()
797 		elv_merge_requests(q, req, next);  in attempt_merge()
903 static void blk_account_io_merge_bio(struct request *req)  in blk_account_io_merge_bio()  argument
905 	if (!blk_do_io_stat(req))  in blk_account_io_merge_bio()
909 	part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);  in blk_account_io_merge_bio()
919 static enum bio_merge_status bio_attempt_back_merge(struct request *req,  in bio_attempt_back_merge()  argument
924 	if (!ll_back_merge_fn(req, bio, nr_segs))  in bio_attempt_back_merge()
927 	trace_block_bio_backmerge(req->q, req, bio);  in bio_attempt_back_merge()
928 	rq_qos_merge(req->q, req, bio);  in bio_attempt_back_merge()
930 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_back_merge()
931 		blk_rq_set_mixed_merge(req);  in bio_attempt_back_merge()
933 	req->biotail->bi_next = bio;  in bio_attempt_back_merge()
934 	req->biotail = bio;  in bio_attempt_back_merge()
935 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_back_merge()
939 	blk_account_io_merge_bio(req);  in bio_attempt_back_merge()
943 static enum bio_merge_status bio_attempt_front_merge(struct request *req,  in bio_attempt_front_merge()  argument
948 	if (!ll_front_merge_fn(req, bio, nr_segs))  in bio_attempt_front_merge()
951 	trace_block_bio_frontmerge(req->q, req, bio);  in bio_attempt_front_merge()
952 	rq_qos_merge(req->q, req, bio);  in bio_attempt_front_merge()
954 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)  in bio_attempt_front_merge()
955 		blk_rq_set_mixed_merge(req);  in bio_attempt_front_merge()
957 	bio->bi_next = req->bio;  in bio_attempt_front_merge()
958 	req->bio = bio;  in bio_attempt_front_merge()
960 	req->__sector = bio->bi_iter.bi_sector;  in bio_attempt_front_merge()
961 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_front_merge()
963 	bio_crypt_do_front_merge(req, bio);  in bio_attempt_front_merge()
965 	blk_account_io_merge_bio(req);  in bio_attempt_front_merge()
970 		struct request *req, struct bio *bio)  in bio_attempt_discard_merge()  argument
972 	unsigned short segments = blk_rq_nr_discard_segments(req);  in bio_attempt_discard_merge()
976 	if (blk_rq_sectors(req) + bio_sectors(bio) >  in bio_attempt_discard_merge()
977 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))  in bio_attempt_discard_merge()
980 	rq_qos_merge(q, req, bio);  in bio_attempt_discard_merge()
982 	req->biotail->bi_next = bio;  in bio_attempt_discard_merge()
983 	req->biotail = bio;  in bio_attempt_discard_merge()
984 	req->__data_len += bio->bi_iter.bi_size;  in bio_attempt_discard_merge()
985 	req->nr_phys_segments = segments + 1;  in bio_attempt_discard_merge()
987 	blk_account_io_merge_bio(req);  in bio_attempt_discard_merge()
990 	req_set_nomerge(q, req);  in bio_attempt_discard_merge()