Lines Matching refs:bio
229 static void req_bio_endio(struct request *rq, struct bio *bio, in req_bio_endio() argument
233 bio->bi_status = error; in req_bio_endio()
236 bio_set_flag(bio, BIO_QUIET); in req_bio_endio()
238 bio_advance(bio, nbytes); in req_bio_endio()
241 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) in req_bio_endio()
242 bio_endio(bio); in req_bio_endio()
255 rq->bio, rq->biotail, blk_rq_bytes(rq)); in blk_dump_rq_flags()
599 bool bio_attempt_back_merge(struct request *req, struct bio *bio, in bio_attempt_back_merge() argument
602 const int ff = bio->bi_opf & REQ_FAILFAST_MASK; in bio_attempt_back_merge()
604 if (!ll_back_merge_fn(req, bio, nr_segs)) in bio_attempt_back_merge()
607 trace_block_bio_backmerge(req->q, req, bio); in bio_attempt_back_merge()
608 rq_qos_merge(req->q, req, bio); in bio_attempt_back_merge()
613 req->biotail->bi_next = bio; in bio_attempt_back_merge()
614 req->biotail = bio; in bio_attempt_back_merge()
615 req->__data_len += bio->bi_iter.bi_size; in bio_attempt_back_merge()
621 bool bio_attempt_front_merge(struct request *req, struct bio *bio, in bio_attempt_front_merge() argument
624 const int ff = bio->bi_opf & REQ_FAILFAST_MASK; in bio_attempt_front_merge()
626 if (!ll_front_merge_fn(req, bio, nr_segs)) in bio_attempt_front_merge()
629 trace_block_bio_frontmerge(req->q, req, bio); in bio_attempt_front_merge()
630 rq_qos_merge(req->q, req, bio); in bio_attempt_front_merge()
635 bio->bi_next = req->bio; in bio_attempt_front_merge()
636 req->bio = bio; in bio_attempt_front_merge()
638 req->__sector = bio->bi_iter.bi_sector; in bio_attempt_front_merge()
639 req->__data_len += bio->bi_iter.bi_size; in bio_attempt_front_merge()
646 struct bio *bio) in bio_attempt_discard_merge() argument
652 if (blk_rq_sectors(req) + bio_sectors(bio) > in bio_attempt_discard_merge()
656 rq_qos_merge(q, req, bio); in bio_attempt_discard_merge()
658 req->biotail->bi_next = bio; in bio_attempt_discard_merge()
659 req->biotail = bio; in bio_attempt_discard_merge()
660 req->__data_len += bio->bi_iter.bi_size; in bio_attempt_discard_merge()
692 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, in blk_attempt_plug_merge() argument
699 plug = blk_mq_plug(q, bio); in blk_attempt_plug_merge()
717 if (rq->q != q || !blk_rq_merge_ok(rq, bio)) in blk_attempt_plug_merge()
720 switch (blk_try_merge(rq, bio)) { in blk_attempt_plug_merge()
722 merged = bio_attempt_back_merge(rq, bio, nr_segs); in blk_attempt_plug_merge()
725 merged = bio_attempt_front_merge(rq, bio, nr_segs); in blk_attempt_plug_merge()
728 merged = bio_attempt_discard_merge(q, rq, bio); in blk_attempt_plug_merge()
741 static void handle_bad_sector(struct bio *bio, sector_t maxsector) in handle_bad_sector() argument
747 bio_devname(bio, b), bio->bi_opf, in handle_bad_sector()
748 (unsigned long long)bio_end_sector(bio), in handle_bad_sector()
787 static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part) in bio_check_ro() argument
789 const int op = bio_op(bio); in bio_check_ro()
794 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) in bio_check_ro()
800 bio_devname(bio, b), part->partno); in bio_check_ro()
808 static noinline int should_fail_bio(struct bio *bio) in should_fail_bio() argument
810 if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size)) in should_fail_bio()
821 static inline int bio_check_eod(struct bio *bio, sector_t maxsector) in bio_check_eod() argument
823 unsigned int nr_sectors = bio_sectors(bio); in bio_check_eod()
827 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { in bio_check_eod()
828 handle_bad_sector(bio, maxsector); in bio_check_eod()
837 static inline int blk_partition_remap(struct bio *bio) in blk_partition_remap() argument
843 p = __disk_get_part(bio->bi_disk, bio->bi_partno); in blk_partition_remap()
846 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) in blk_partition_remap()
848 if (unlikely(bio_check_ro(bio, p))) in blk_partition_remap()
855 if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) { in blk_partition_remap()
856 if (bio_check_eod(bio, part_nr_sects_read(p))) in blk_partition_remap()
858 bio->bi_iter.bi_sector += p->start_sect; in blk_partition_remap()
859 trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p), in blk_partition_remap()
860 bio->bi_iter.bi_sector - p->start_sect); in blk_partition_remap()
862 bio->bi_partno = 0; in blk_partition_remap()
870 generic_make_request_checks(struct bio *bio) in generic_make_request_checks() argument
873 int nr_sectors = bio_sectors(bio); in generic_make_request_checks()
879 q = bio->bi_disk->queue; in generic_make_request_checks()
884 bio_devname(bio, b), (long long)bio->bi_iter.bi_sector); in generic_make_request_checks()
892 if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q)) in generic_make_request_checks()
895 if (should_fail_bio(bio)) in generic_make_request_checks()
898 if (bio->bi_partno) { in generic_make_request_checks()
899 if (unlikely(blk_partition_remap(bio))) in generic_make_request_checks()
902 if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0))) in generic_make_request_checks()
904 if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) in generic_make_request_checks()
913 if (op_is_flush(bio->bi_opf) && in generic_make_request_checks()
915 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); in generic_make_request_checks()
923 bio->bi_opf &= ~REQ_HIPRI; in generic_make_request_checks()
925 switch (bio_op(bio)) { in generic_make_request_checks()
962 if (!blkcg_bio_issue_check(q, bio)) in generic_make_request_checks()
965 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { in generic_make_request_checks()
966 trace_block_bio_queue(q, bio); in generic_make_request_checks()
970 bio_set_flag(bio, BIO_TRACE_COMPLETION); in generic_make_request_checks()
977 bio->bi_status = status; in generic_make_request_checks()
978 bio_endio(bio); in generic_make_request_checks()
1006 blk_qc_t generic_make_request(struct bio *bio) in generic_make_request() argument
1018 if (!generic_make_request_checks(bio)) in generic_make_request()
1032 bio_list_add(¤t->bio_list[0], bio); in generic_make_request()
1050 BUG_ON(bio->bi_next); in generic_make_request()
1054 struct request_queue *q = bio->bi_disk->queue; in generic_make_request()
1055 blk_mq_req_flags_t flags = bio->bi_opf & REQ_NOWAIT ? in generic_make_request()
1064 ret = q->make_request_fn(q, bio); in generic_make_request()
1073 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) in generic_make_request()
1074 if (q == bio->bi_disk->queue) in generic_make_request()
1075 bio_list_add(&same, bio); in generic_make_request()
1077 bio_list_add(&lower, bio); in generic_make_request()
1084 (bio->bi_opf & REQ_NOWAIT))) in generic_make_request()
1085 bio_wouldblock_error(bio); in generic_make_request()
1087 bio_io_error(bio); in generic_make_request()
1089 bio = bio_list_pop(&bio_list_on_stack[0]); in generic_make_request()
1090 } while (bio); in generic_make_request()
1108 blk_qc_t direct_make_request(struct bio *bio) in direct_make_request() argument
1110 struct request_queue *q = bio->bi_disk->queue; in direct_make_request()
1111 bool nowait = bio->bi_opf & REQ_NOWAIT; in direct_make_request()
1114 if (!generic_make_request_checks(bio)) in direct_make_request()
1119 bio->bi_status = BLK_STS_AGAIN; in direct_make_request()
1121 bio->bi_status = BLK_STS_IOERR; in direct_make_request()
1122 bio_endio(bio); in direct_make_request()
1126 ret = q->make_request_fn(q, bio); in direct_make_request()
1141 blk_qc_t submit_bio(struct bio *bio) in submit_bio() argument
1147 if (blkcg_punt_bio_submit(bio)) in submit_bio()
1154 if (bio_has_data(bio)) { in submit_bio()
1157 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) in submit_bio()
1158 count = queue_logical_block_size(bio->bi_disk->queue) >> 9; in submit_bio()
1160 count = bio_sectors(bio); in submit_bio()
1162 if (op_is_write(bio_op(bio))) { in submit_bio()
1165 if (bio_flagged(bio, BIO_WORKINGSET)) in submit_bio()
1167 task_io_account_read(bio->bi_iter.bi_size); in submit_bio()
1175 op_is_write(bio_op(bio)) ? "WRITE" : "READ", in submit_bio()
1176 (unsigned long long)bio->bi_iter.bi_sector, in submit_bio()
1177 bio_devname(bio, b), count); in submit_bio()
1190 ret = generic_make_request(bio); in submit_bio()
1285 struct bio *bio; in blk_rq_err_bytes() local
1297 for (bio = rq->bio; bio; bio = bio->bi_next) { in blk_rq_err_bytes()
1298 if ((bio->bi_opf & ff) != ff) in blk_rq_err_bytes()
1300 bytes += bio->bi_iter.bi_size; in blk_rq_err_bytes()
1389 if (rq->bio) { in blk_steal_bios()
1391 list->tail->bi_next = rq->bio; in blk_steal_bios()
1393 list->head = rq->bio; in blk_steal_bios()
1396 rq->bio = NULL; in blk_steal_bios()
1437 if (!req->bio) in blk_update_request()
1453 while (req->bio) { in blk_update_request()
1454 struct bio *bio = req->bio; in blk_update_request() local
1455 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); in blk_update_request()
1457 if (bio_bytes == bio->bi_iter.bi_size) in blk_update_request()
1458 req->bio = bio->bi_next; in blk_update_request()
1461 bio_clear_flag(bio, BIO_TRACE_COMPLETION); in blk_update_request()
1462 req_bio_endio(req, bio, bio_bytes, error); in blk_update_request()
1474 if (!req->bio) { in blk_update_request()
1493 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; in blk_update_request()
1570 struct bio *bio; in blk_rq_unprep_clone() local
1572 while ((bio = rq->bio) != NULL) { in blk_rq_unprep_clone()
1573 rq->bio = bio->bi_next; in blk_rq_unprep_clone()
1575 bio_put(bio); in blk_rq_unprep_clone()
1618 int (*bio_ctr)(struct bio *, struct bio *, void *), in blk_rq_prep_clone() argument
1621 struct bio *bio, *bio_src; in blk_rq_prep_clone() local
1627 bio = bio_clone_fast(bio_src, gfp_mask, bs); in blk_rq_prep_clone()
1628 if (!bio) in blk_rq_prep_clone()
1631 if (bio_ctr && bio_ctr(bio, bio_src, data)) in blk_rq_prep_clone()
1634 if (rq->bio) { in blk_rq_prep_clone()
1635 rq->biotail->bi_next = bio; in blk_rq_prep_clone()
1636 rq->biotail = bio; in blk_rq_prep_clone()
1638 rq->bio = rq->biotail = bio; in blk_rq_prep_clone()
1646 if (bio) in blk_rq_prep_clone()
1647 bio_put(bio); in blk_rq_prep_clone()
1795 FIELD_SIZEOF(struct bio, bi_opf)); in blk_dev_init()