Lines Matching refs:bio
239 static void req_bio_endio(struct request *rq, struct bio *bio, in req_bio_endio() argument
243 bio->bi_status = error; in req_bio_endio()
246 bio_set_flag(bio, BIO_QUIET); in req_bio_endio()
248 bio_advance(bio, nbytes); in req_bio_endio()
255 if (bio->bi_iter.bi_size) in req_bio_endio()
256 bio->bi_status = BLK_STS_IOERR; in req_bio_endio()
258 bio->bi_iter.bi_sector = rq->__sector; in req_bio_endio()
262 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) in req_bio_endio()
263 bio_endio(bio); in req_bio_endio()
276 rq->bio, rq->biotail, blk_rq_bytes(rq)); in blk_dump_rq_flags()
476 static inline int bio_queue_enter(struct bio *bio) in bio_queue_enter() argument
478 struct request_queue *q = bio->bi_disk->queue; in bio_queue_enter()
479 bool nowait = bio->bi_opf & REQ_NOWAIT; in bio_queue_enter()
485 bio_wouldblock_error(bio); in bio_queue_enter()
487 bio_io_error(bio); in bio_queue_enter()
649 static void handle_bad_sector(struct bio *bio, sector_t maxsector) in handle_bad_sector() argument
655 bio_devname(bio, b), bio->bi_opf, in handle_bad_sector()
656 bio_end_sector(bio), maxsector); in handle_bad_sector()
694 static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part) in bio_check_ro() argument
696 const int op = bio_op(bio); in bio_check_ro()
701 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) in bio_check_ro()
706 bio_devname(bio, b), part->partno); in bio_check_ro()
714 static noinline int should_fail_bio(struct bio *bio) in should_fail_bio() argument
716 if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size)) in should_fail_bio()
727 static inline int bio_check_eod(struct bio *bio, sector_t maxsector) in bio_check_eod() argument
729 unsigned int nr_sectors = bio_sectors(bio); in bio_check_eod()
733 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { in bio_check_eod()
734 handle_bad_sector(bio, maxsector); in bio_check_eod()
743 static inline int blk_partition_remap(struct bio *bio) in blk_partition_remap() argument
749 p = __disk_get_part(bio->bi_disk, bio->bi_partno); in blk_partition_remap()
752 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) in blk_partition_remap()
754 if (unlikely(bio_check_ro(bio, p))) in blk_partition_remap()
757 if (bio_sectors(bio)) { in blk_partition_remap()
758 if (bio_check_eod(bio, part_nr_sects_read(p))) in blk_partition_remap()
760 bio->bi_iter.bi_sector += p->start_sect; in blk_partition_remap()
761 trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p), in blk_partition_remap()
762 bio->bi_iter.bi_sector - p->start_sect); in blk_partition_remap()
764 bio->bi_partno = 0; in blk_partition_remap()
775 struct bio *bio) in blk_check_zone_append() argument
777 sector_t pos = bio->bi_iter.bi_sector; in blk_check_zone_append()
778 int nr_sectors = bio_sectors(bio); in blk_check_zone_append()
801 bio->bi_opf |= REQ_NOMERGE; in blk_check_zone_append()
806 static noinline_for_stack bool submit_bio_checks(struct bio *bio) in submit_bio_checks() argument
808 struct request_queue *q = bio->bi_disk->queue; in submit_bio_checks()
814 plug = blk_mq_plug(q, bio); in submit_bio_checks()
816 bio->bi_opf |= REQ_NOWAIT; in submit_bio_checks()
822 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q)) in submit_bio_checks()
825 if (should_fail_bio(bio)) in submit_bio_checks()
828 if (bio->bi_partno) { in submit_bio_checks()
829 if (unlikely(blk_partition_remap(bio))) in submit_bio_checks()
832 if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0))) in submit_bio_checks()
834 if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) in submit_bio_checks()
842 if (op_is_flush(bio->bi_opf) && in submit_bio_checks()
844 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); in submit_bio_checks()
845 if (!bio_sectors(bio)) { in submit_bio_checks()
852 bio->bi_opf &= ~REQ_HIPRI; in submit_bio_checks()
854 switch (bio_op(bio)) { in submit_bio_checks()
868 status = blk_check_zone_append(q, bio); in submit_bio_checks()
900 if (blk_throtl_bio(bio)) { in submit_bio_checks()
901 blkcg_bio_issue_init(bio); in submit_bio_checks()
905 blk_cgroup_bio_start(bio); in submit_bio_checks()
906 blkcg_bio_issue_init(bio); in submit_bio_checks()
908 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { in submit_bio_checks()
909 trace_block_bio_queue(q, bio); in submit_bio_checks()
913 bio_set_flag(bio, BIO_TRACE_COMPLETION); in submit_bio_checks()
920 bio->bi_status = status; in submit_bio_checks()
921 bio_endio(bio); in submit_bio_checks()
925 static blk_qc_t __submit_bio(struct bio *bio) in __submit_bio() argument
927 struct gendisk *disk = bio->bi_disk; in __submit_bio()
930 if (blk_crypto_bio_prep(&bio)) { in __submit_bio()
932 return blk_mq_submit_bio(bio); in __submit_bio()
933 ret = disk->fops->submit_bio(bio); in __submit_bio()
958 static blk_qc_t __submit_bio_noacct(struct bio *bio) in __submit_bio_noacct() argument
963 BUG_ON(bio->bi_next); in __submit_bio_noacct()
969 struct request_queue *q = bio->bi_disk->queue; in __submit_bio_noacct()
972 if (unlikely(bio_queue_enter(bio) != 0)) in __submit_bio_noacct()
981 ret = __submit_bio(bio); in __submit_bio_noacct()
989 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) in __submit_bio_noacct()
990 if (q == bio->bi_disk->queue) in __submit_bio_noacct()
991 bio_list_add(&same, bio); in __submit_bio_noacct()
993 bio_list_add(&lower, bio); in __submit_bio_noacct()
1001 } while ((bio = bio_list_pop(&bio_list_on_stack[0]))); in __submit_bio_noacct()
1007 static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) in __submit_bio_noacct_mq() argument
1015 struct gendisk *disk = bio->bi_disk; in __submit_bio_noacct_mq()
1017 if (unlikely(bio_queue_enter(bio) != 0)) in __submit_bio_noacct_mq()
1020 if (!blk_crypto_bio_prep(&bio)) { in __submit_bio_noacct_mq()
1026 ret = blk_mq_submit_bio(bio); in __submit_bio_noacct_mq()
1027 } while ((bio = bio_list_pop(&bio_list[0]))); in __submit_bio_noacct_mq()
1042 blk_qc_t submit_bio_noacct(struct bio *bio) in submit_bio_noacct() argument
1044 if (!submit_bio_checks(bio)) in submit_bio_noacct()
1054 bio_list_add(¤t->bio_list[0], bio); in submit_bio_noacct()
1058 if (!bio->bi_disk->fops->submit_bio) in submit_bio_noacct()
1059 return __submit_bio_noacct_mq(bio); in submit_bio_noacct()
1060 return __submit_bio_noacct(bio); in submit_bio_noacct()
1077 blk_qc_t submit_bio(struct bio *bio) in submit_bio() argument
1079 if (blkcg_punt_bio_submit(bio)) in submit_bio()
1086 if (bio_has_data(bio)) { in submit_bio()
1089 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) in submit_bio()
1090 count = queue_logical_block_size(bio->bi_disk->queue) >> 9; in submit_bio()
1092 count = bio_sectors(bio); in submit_bio()
1094 if (op_is_write(bio_op(bio))) { in submit_bio()
1097 task_io_account_read(bio->bi_iter.bi_size); in submit_bio()
1105 op_is_write(bio_op(bio)) ? "WRITE" : "READ", in submit_bio()
1106 (unsigned long long)bio->bi_iter.bi_sector, in submit_bio()
1107 bio_devname(bio, b), count); in submit_bio()
1117 if (unlikely(bio_op(bio) == REQ_OP_READ && in submit_bio()
1118 bio_flagged(bio, BIO_WORKINGSET))) { in submit_bio()
1123 ret = submit_bio_noacct(bio); in submit_bio()
1129 return submit_bio_noacct(bio); in submit_bio()
1239 struct bio *bio; in blk_rq_err_bytes() local
1251 for (bio = rq->bio; bio; bio = bio->bi_next) { in blk_rq_err_bytes()
1252 if ((bio->bi_opf & ff) != ff) in blk_rq_err_bytes()
1254 bytes += bio->bi_iter.bi_size; in blk_rq_err_bytes()
1344 struct bio *bio) in part_start_io_acct() argument
1346 *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector); in part_start_io_acct()
1348 return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio)); in part_start_io_acct()
1373 void part_end_io_acct(struct hd_struct *part, struct bio *bio, in part_end_io_acct() argument
1376 __part_end_io_acct(part, bio_op(bio), start_time); in part_end_io_acct()
1394 if (rq->bio) { in blk_steal_bios()
1396 list->tail->bi_next = rq->bio; in blk_steal_bios()
1398 list->head = rq->bio; in blk_steal_bios()
1401 rq->bio = NULL; in blk_steal_bios()
1442 if (!req->bio) in blk_update_request()
1458 while (req->bio) { in blk_update_request()
1459 struct bio *bio = req->bio; in blk_update_request() local
1460 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); in blk_update_request()
1462 if (bio_bytes == bio->bi_iter.bi_size) in blk_update_request()
1463 req->bio = bio->bi_next; in blk_update_request()
1466 bio_clear_flag(bio, BIO_TRACE_COMPLETION); in blk_update_request()
1467 req_bio_endio(req, bio, bio_bytes, error); in blk_update_request()
1479 if (!req->bio) { in blk_update_request()
1498 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; in blk_update_request()
1575 struct bio *bio; in blk_rq_unprep_clone() local
1577 while ((bio = rq->bio) != NULL) { in blk_rq_unprep_clone()
1578 rq->bio = bio->bi_next; in blk_rq_unprep_clone()
1580 bio_put(bio); in blk_rq_unprep_clone()
1604 int (*bio_ctr)(struct bio *, struct bio *, void *), in blk_rq_prep_clone() argument
1607 struct bio *bio, *bio_src; in blk_rq_prep_clone() local
1613 bio = bio_clone_fast(bio_src, gfp_mask, bs); in blk_rq_prep_clone()
1614 if (!bio) in blk_rq_prep_clone()
1617 if (bio_ctr && bio_ctr(bio, bio_src, data)) in blk_rq_prep_clone()
1620 if (rq->bio) { in blk_rq_prep_clone()
1621 rq->biotail->bi_next = bio; in blk_rq_prep_clone()
1622 rq->biotail = bio; in blk_rq_prep_clone()
1624 rq->bio = rq->biotail = bio; in blk_rq_prep_clone()
1626 bio = NULL; in blk_rq_prep_clone()
1639 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) in blk_rq_prep_clone()
1645 if (bio) in blk_rq_prep_clone()
1646 bio_put(bio); in blk_rq_prep_clone()
1801 sizeof_field(struct bio, bi_opf)); in blk_dev_init()