40 static void bio_csum(struct bio *bio, struct bkey *k)  in bio_csum()  argument
46 	bio_for_each_segment(bv, bio, iter) {  in bio_csum()
123 	struct bio *bio = op->bio;  in bch_data_invalidate()  local
126 		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);  in bch_data_invalidate()
128 	while (bio_sectors(bio)) {  in bch_data_invalidate()
129 		unsigned int sectors = min(bio_sectors(bio),  in bch_data_invalidate()
135 		bio->bi_iter.bi_sector	+= sectors;  in bch_data_invalidate()
136 		bio->bi_iter.bi_size	-= sectors << 9;  in bch_data_invalidate()
140 				     bio->bi_iter.bi_sector,  in bch_data_invalidate()
146 	bio_put(bio);  in bch_data_invalidate()
181 static void bch_data_insert_endio(struct bio *bio)  in bch_data_insert_endio()  argument
183 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()
186 	if (bio->bi_status) {  in bch_data_insert_endio()
189 			op->status = bio->bi_status;  in bch_data_insert_endio()
196 	bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");  in bch_data_insert_endio()
202 	struct bio *bio = op->bio, *n;  in bch_data_insert_start()  local
207 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)  in bch_data_insert_start()
214 	bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);  in bch_data_insert_start()
232 		SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);  in bch_data_insert_start()
234 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),  in bch_data_insert_start()
239 		n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);  in bch_data_insert_start()
261 	} while (n != bio);  in bch_data_insert_start()
291 		bio_put(bio);  in bch_data_insert_start()
324 	trace_bcache_write(op->c, op->inode, op->bio,  in bch_data_insert()
328 	bio_get(op->bio);  in bch_data_insert()
375 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)  in check_should_bypass()  argument
385 	    (bio_op(bio) == REQ_OP_DISCARD))  in check_should_bypass()
390 	     op_is_write(bio_op(bio))))  in check_should_bypass()
398 	if (bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND) &&  in check_should_bypass()
399 	    !(bio->bi_opf & (REQ_META|REQ_PRIO)))  in check_should_bypass()
402 	if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||  in check_should_bypass()
403 	    bio_sectors(bio) & (c->sb.block_size - 1)) {  in check_should_bypass()
421 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)  in check_should_bypass()
422 		if (i->last == bio->bi_iter.bi_sector &&  in check_should_bypass()
431 	if (i->sequential + bio->bi_iter.bi_size > i->sequential)  in check_should_bypass()
432 		i->sequential	+= bio->bi_iter.bi_size;  in check_should_bypass()
434 	i->last			 = bio_end_sector(bio);  in check_should_bypass()
449 		trace_bcache_bypass_sequential(bio);  in check_should_bypass()
454 		trace_bcache_bypass_congested(bio);  in check_should_bypass()
459 	bch_rescale_priorities(c, bio_sectors(bio));  in check_should_bypass()
462 	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));  in check_should_bypass()
472 	struct bbio		bio;  member
473 	struct bio		*orig_bio;
474 	struct bio		*cache_miss;
489 static void bch_cache_read_endio(struct bio *bio)  in bch_cache_read_endio()  argument
491 	struct bbio *b = container_of(bio, struct bbio, bio);  in bch_cache_read_endio()
492 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()
502 	if (bio->bi_status)  in bch_cache_read_endio()
503 		s->iop.status = bio->bi_status;  in bch_cache_read_endio()
510 	bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache");  in bch_cache_read_endio()
520 	struct bio *n, *bio = &s->bio.bio;  in cache_lookup_fn()  local
524 	if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)  in cache_lookup_fn()
528 	    KEY_START(k) > bio->bi_iter.bi_sector) {  in cache_lookup_fn()
529 		unsigned int bio_sectors = bio_sectors(bio);  in cache_lookup_fn()
532 				KEY_START(k) - bio->bi_iter.bi_sector)  in cache_lookup_fn()
534 		int ret = s->d->cache_miss(b, s, bio, sectors);  in cache_lookup_fn()
554 	n = bio_next_split(bio, min_t(uint64_t, INT_MAX,  in cache_lookup_fn()
555 				      KEY_OFFSET(k) - bio->bi_iter.bi_sector),  in cache_lookup_fn()
558 	bio_key = &container_of(n, struct bbio, bio)->key;  in cache_lookup_fn()
579 	return n == bio ? MAP_DONE : MAP_CONTINUE;  in cache_lookup_fn()
585 	struct bio *bio = &s->bio.bio;  in cache_lookup()  local
592 				 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),  in cache_lookup()
625 static void request_endio(struct bio *bio)  in request_endio()  argument
627 	struct closure *cl = bio->bi_private;  in request_endio()
629 	if (bio->bi_status) {  in request_endio()
632 		s->iop.status = bio->bi_status;  in request_endio()
637 	bio_put(bio);  in request_endio()
641 static void backing_request_endio(struct bio *bio)  in backing_request_endio()  argument
643 	struct closure *cl = bio->bi_private;  in backing_request_endio()
645 	if (bio->bi_status) {  in backing_request_endio()
657 			     bio->bi_opf & REQ_PREFLUSH)) {  in backing_request_endio()
659 				dc->backing_dev_name, bio->bi_status);  in backing_request_endio()
662 			s->iop.status = bio->bi_status;  in backing_request_endio()
666 		bch_count_backing_io_errors(dc, bio);  in backing_request_endio()
669 	bio_put(bio);  in backing_request_endio()
687 			struct bio *orig_bio,  in do_bio_hook()
690 	struct bio *bio = &s->bio.bio;  in do_bio_hook()  local
692 	bio_init(bio, NULL, 0);  in do_bio_hook()
693 	__bio_clone_fast(bio, orig_bio);  in do_bio_hook()
700 	bio->bi_end_io		= end_io_fn;  in do_bio_hook()
701 	bio->bi_private		= &s->cl;  in do_bio_hook()
703 	bio_cnt_set(bio, 3);  in do_bio_hook()
712 	if (s->iop.bio)  in search_free()
713 		bio_put(s->iop.bio);  in search_free()
720 static inline struct search *search_alloc(struct bio *bio,  in search_alloc()  argument
728 	do_bio_hook(s, bio, request_endio);  in search_alloc()
731 	s->orig_bio		= bio;  in search_alloc()
736 	s->write		= op_is_write(bio_op(bio));  in search_alloc()
741 	s->iop.bio		= NULL;  in search_alloc()
747 	s->iop.flush_journal	= op_is_flush(bio->bi_opf);  in search_alloc()
773 	if (s->iop.bio)  in cached_dev_read_error_done()
774 		bio_free_pages(s->iop.bio);  in cached_dev_read_error_done()
782 	struct bio *bio = &s->bio.bio;  in cached_dev_read_error()  local
801 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_read_error()
815 	if (s->iop.bio)  in cached_dev_cache_miss_done()
816 		bio_free_pages(s->iop.bio);  in cached_dev_cache_miss_done()
835 	if (s->iop.bio) {  in cached_dev_read_done()
836 		bio_reset(s->iop.bio);  in cached_dev_read_done()
837 		s->iop.bio->bi_iter.bi_sector =  in cached_dev_read_done()
839 		bio_copy_dev(s->iop.bio, s->cache_miss);  in cached_dev_read_done()
840 		s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;  in cached_dev_read_done()
841 		bch_bio_map(s->iop.bio, NULL);  in cached_dev_read_done()
843 		bio_copy_data(s->cache_miss, s->iop.bio);  in cached_dev_read_done()
855 	if (s->iop.bio &&  in cached_dev_read_done()
875 	else if (s->iop.bio || verify(dc))  in cached_dev_read_done_bh()
882 				 struct bio *bio, unsigned int sectors)  in cached_dev_cache_miss()  argument
887 	struct bio *miss, *cache_bio;  in cached_dev_cache_miss()
892 		miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);  in cached_dev_cache_miss()
893 		ret = miss == bio ? MAP_DONE : MAP_CONTINUE;  in cached_dev_cache_miss()
897 	if (!(bio->bi_opf & REQ_RAHEAD) &&  in cached_dev_cache_miss()
898 	    !(bio->bi_opf & (REQ_META|REQ_PRIO)) &&  in cached_dev_cache_miss()
901 			      get_capacity(bio->bi_disk) - bio_end_sector(bio));  in cached_dev_cache_miss()
903 	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);  in cached_dev_cache_miss()
906 				 bio->bi_iter.bi_sector + s->insert_bio_sectors,  in cached_dev_cache_miss()
915 	miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);  in cached_dev_cache_miss()
918 	ret = miss == bio ? MAP_DONE : -EINTR;  in cached_dev_cache_miss()
941 	s->iop.bio	= cache_bio;  in cached_dev_cache_miss()
979 	struct bio *bio = &s->bio.bio;  in cached_dev_write()  local
980 	struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);  in cached_dev_write()
981 	struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);  in cached_dev_write()
1002 	if (bio_op(bio) == REQ_OP_DISCARD)  in cached_dev_write()
1013 		s->iop.bio = s->orig_bio;  in cached_dev_write()
1014 		bio_get(s->iop.bio);  in cached_dev_write()
1016 		if (bio_op(bio) == REQ_OP_DISCARD &&  in cached_dev_write()
1021 		bio->bi_end_io = backing_request_endio;  in cached_dev_write()
1022 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1026 		s->iop.bio = bio;  in cached_dev_write()
1028 		if (bio->bi_opf & REQ_PREFLUSH) {  in cached_dev_write()
1033 			struct bio *flush;  in cached_dev_write()
1041 			bio_copy_dev(flush, bio);  in cached_dev_write()
1049 		s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);  in cached_dev_write()
1051 		bio->bi_end_io = backing_request_endio;  in cached_dev_write()
1052 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1063 	struct bio *bio = &s->bio.bio;  in cached_dev_nodata()  local
1069 	bio->bi_end_io = backing_request_endio;  in cached_dev_nodata()
1070 	closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_nodata()
1082 static void detached_dev_end_io(struct bio *bio)  in detached_dev_end_io()  argument
1086 	ddip = bio->bi_private;  in detached_dev_end_io()
1087 	bio->bi_end_io = ddip->bi_end_io;  in detached_dev_end_io()
1088 	bio->bi_private = ddip->bi_private;  in detached_dev_end_io()
1090 	generic_end_io_acct(ddip->d->disk->queue, bio_op(bio),  in detached_dev_end_io()
1093 	if (bio->bi_status) {  in detached_dev_end_io()
1097 		bch_count_backing_io_errors(dc, bio);  in detached_dev_end_io()
1101 	bio->bi_end_io(bio);  in detached_dev_end_io()
1104 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio)  in detached_dev_do_request()  argument
1117 	ddip->bi_end_io = bio->bi_end_io;  in detached_dev_do_request()
1118 	ddip->bi_private = bio->bi_private;  in detached_dev_do_request()
1119 	bio->bi_end_io = detached_dev_end_io;  in detached_dev_do_request()
1120 	bio->bi_private = ddip;  in detached_dev_do_request()
1122 	if ((bio_op(bio) == REQ_OP_DISCARD) &&  in detached_dev_do_request()
1124 		bio->bi_end_io(bio);  in detached_dev_do_request()
1126 		generic_make_request(bio);  in detached_dev_do_request()
1170 					struct bio *bio)  in cached_dev_make_request()  argument
1173 	struct bcache_device *d = bio->bi_disk->private_data;  in cached_dev_make_request()
1175 	int rw = bio_data_dir(bio);  in cached_dev_make_request()
1179 		bio->bi_status = BLK_STS_IOERR;  in cached_dev_make_request()
1180 		bio_endio(bio);  in cached_dev_make_request()
1200 			      bio_op(bio),  in cached_dev_make_request()
1201 			      bio_sectors(bio),  in cached_dev_make_request()
1204 	bio_set_dev(bio, dc->bdev);  in cached_dev_make_request()
1205 	bio->bi_iter.bi_sector += dc->sb.data_offset;  in cached_dev_make_request()
1208 		s = search_alloc(bio, d);  in cached_dev_make_request()
1209 		trace_bcache_request_start(s->d, bio);  in cached_dev_make_request()
1211 		if (!bio->bi_iter.bi_size) {  in cached_dev_make_request()
1220 			s->iop.bypass = check_should_bypass(dc, bio);  in cached_dev_make_request()
1229 		detached_dev_do_request(d, bio);  in cached_dev_make_request()
1283 				struct bio *bio, unsigned int sectors)  in flash_dev_cache_miss()  argument
1285 	unsigned int bytes = min(sectors, bio_sectors(bio)) << 9;  in flash_dev_cache_miss()
1287 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1288 	zero_fill_bio(bio);  in flash_dev_cache_miss()
1289 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1291 	bio_advance(bio, bytes);  in flash_dev_cache_miss()
1293 	if (!bio->bi_iter.bi_size)  in flash_dev_cache_miss()
1310 					     struct bio *bio)  in flash_dev_make_request()  argument
1314 	struct bcache_device *d = bio->bi_disk->private_data;  in flash_dev_make_request()
1317 		bio->bi_status = BLK_STS_IOERR;  in flash_dev_make_request()
1318 		bio_endio(bio);  in flash_dev_make_request()
1322 	generic_start_io_acct(q, bio_op(bio), bio_sectors(bio), &d->disk->part0);  in flash_dev_make_request()
1324 	s = search_alloc(bio, d);  in flash_dev_make_request()
1326 	bio = &s->bio.bio;  in flash_dev_make_request()
1328 	trace_bcache_request_start(s->d, bio);  in flash_dev_make_request()
1330 	if (!bio->bi_iter.bi_size) {  in flash_dev_make_request()
1339 	} else if (bio_data_dir(bio)) {  in flash_dev_make_request()
1341 					&KEY(d->id, bio->bi_iter.bi_sector, 0),  in flash_dev_make_request()
1342 					&KEY(d->id, bio_end_sector(bio), 0));  in flash_dev_make_request()
1344 		s->iop.bypass		= (bio_op(bio) == REQ_OP_DISCARD) != 0;  in flash_dev_make_request()
1346 		s->iop.bio		= bio;  in flash_dev_make_request()