Lines Matching refs:iop
486 struct data_insert_op iop; member
503 s->iop.status = bio->bi_status; in bch_cache_read_endio()
505 ptr_stale(s->iop.c, &b->key, 0)) { in bch_cache_read_endio()
506 atomic_long_inc(&s->iop.c->cache_read_races); in bch_cache_read_endio()
507 s->iop.status = BLK_STS_IOERR; in bch_cache_read_endio()
510 bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache"); in bch_cache_read_endio()
524 if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0) in cache_lookup_fn()
527 if (KEY_INODE(k) != s->iop.inode || in cache_lookup_fn()
530 unsigned int sectors = KEY_INODE(k) == s->iop.inode in cache_lookup_fn()
561 bch_cut_front(&KEY(s->iop.inode, n->bi_iter.bi_sector, 0), bio_key); in cache_lookup_fn()
562 bch_cut_back(&KEY(s->iop.inode, bio_end_sector(n), 0), bio_key); in cache_lookup_fn()
584 struct search *s = container_of(cl, struct search, iop.cl); in cache_lookup()
591 ret = bch_btree_map_keys(&s->op, s->iop.c, in cache_lookup()
592 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0), in cache_lookup()
616 if (!s->iop.status) in cache_lookup()
617 s->iop.status = BLK_STS_IOERR; in cache_lookup()
632 s->iop.status = bio->bi_status; in request_endio()
656 if (unlikely(s->iop.writeback && in backing_request_endio()
662 s->iop.status = bio->bi_status; in backing_request_endio()
680 s->orig_bio->bi_status = s->iop.status; in bio_complete()
710 atomic_dec(&s->iop.c->search_inflight); in search_free()
712 if (s->iop.bio) in search_free()
713 bio_put(s->iop.bio); in search_free()
717 mempool_free(s, &s->iop.c->search); in search_free()
740 s->iop.c = d->c; in search_alloc()
741 s->iop.bio = NULL; in search_alloc()
742 s->iop.inode = d->id; in search_alloc()
743 s->iop.write_point = hash_long((unsigned long) current, 16); in search_alloc()
744 s->iop.write_prio = 0; in search_alloc()
745 s->iop.status = 0; in search_alloc()
746 s->iop.flags = 0; in search_alloc()
747 s->iop.flush_journal = op_is_flush(bio->bi_opf); in search_alloc()
748 s->iop.wq = bcache_wq; in search_alloc()
770 if (s->iop.replace_collision) in cached_dev_read_error_done()
771 bch_mark_cache_miss_collision(s->iop.c, s->d); in cached_dev_read_error_done()
773 if (s->iop.bio) in cached_dev_read_error_done()
774 bio_free_pages(s->iop.bio); in cached_dev_read_error_done()
795 s->iop.status = 0; in cached_dev_read_error()
801 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_read_error()
812 if (s->iop.replace_collision) in cached_dev_cache_miss_done()
813 bch_mark_cache_miss_collision(s->iop.c, s->d); in cached_dev_cache_miss_done()
815 if (s->iop.bio) in cached_dev_cache_miss_done()
816 bio_free_pages(s->iop.bio); in cached_dev_cache_miss_done()
835 if (s->iop.bio) { in cached_dev_read_done()
836 bio_reset(s->iop.bio); in cached_dev_read_done()
837 s->iop.bio->bi_iter.bi_sector = in cached_dev_read_done()
839 bio_copy_dev(s->iop.bio, s->cache_miss); in cached_dev_read_done()
840 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9; in cached_dev_read_done()
841 bch_bio_map(s->iop.bio, NULL); in cached_dev_read_done()
843 bio_copy_data(s->cache_miss, s->iop.bio); in cached_dev_read_done()
855 if (s->iop.bio && in cached_dev_read_done()
856 !test_bit(CACHE_SET_STOPPING, &s->iop.c->flags)) { in cached_dev_read_done()
857 BUG_ON(!s->iop.replace); in cached_dev_read_done()
858 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in cached_dev_read_done()
869 bch_mark_cache_accounting(s->iop.c, s->d, in cached_dev_read_done_bh()
870 !s->cache_missed, s->iop.bypass); in cached_dev_read_done_bh()
871 trace_bcache_read(s->orig_bio, !s->cache_missed, s->iop.bypass); in cached_dev_read_done_bh()
873 if (s->iop.status) in cached_dev_read_done_bh()
875 else if (s->iop.bio || verify(dc)) in cached_dev_read_done_bh()
891 if (s->cache_miss || s->iop.bypass) { in cached_dev_cache_miss()
899 s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA) in cached_dev_cache_miss()
905 s->iop.replace_key = KEY(s->iop.inode, in cached_dev_cache_miss()
909 ret = bch_btree_insert_check_key(b, &s->op, &s->iop.replace_key); in cached_dev_cache_miss()
913 s->iop.replace = true; in cached_dev_cache_miss()
938 bch_mark_cache_readahead(s->iop.c, s->d); in cached_dev_cache_miss()
941 s->iop.bio = cache_bio; in cached_dev_cache_miss()
944 closure_bio_submit(s->iop.c, cache_bio, &s->cl); in cached_dev_cache_miss()
953 closure_bio_submit(s->iop.c, miss, &s->cl); in cached_dev_cache_miss()
961 closure_call(&s->iop.cl, cache_lookup, NULL, cl); in cached_dev_read()
983 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, &start, &end); in cached_dev_write()
991 s->iop.bypass = false; in cached_dev_write()
992 s->iop.writeback = true; in cached_dev_write()
1003 s->iop.bypass = true; in cached_dev_write()
1007 s->iop.bypass)) { in cached_dev_write()
1008 s->iop.bypass = false; in cached_dev_write()
1009 s->iop.writeback = true; in cached_dev_write()
1012 if (s->iop.bypass) { in cached_dev_write()
1013 s->iop.bio = s->orig_bio; in cached_dev_write()
1014 bio_get(s->iop.bio); in cached_dev_write()
1022 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1024 } else if (s->iop.writeback) { in cached_dev_write()
1026 s->iop.bio = bio; in cached_dev_write()
1038 s->iop.status = BLK_STS_RESOURCE; in cached_dev_write()
1046 closure_bio_submit(s->iop.c, flush, cl); in cached_dev_write()
1049 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split); in cached_dev_write()
1052 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1056 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in cached_dev_write()
1065 if (s->iop.flush_journal) in cached_dev_nodata()
1066 bch_journal_meta(s->iop.c, cl); in cached_dev_nodata()
1070 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_nodata()
1220 s->iop.bypass = check_should_bypass(dc, bio); in cached_dev_make_request()
1303 if (s->iop.flush_journal) in flash_dev_nodata()
1304 bch_journal_meta(s->iop.c, cl); in flash_dev_nodata()
1340 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, in flash_dev_make_request()
1344 s->iop.bypass = (bio_op(bio) == REQ_OP_DISCARD) != 0; in flash_dev_make_request()
1345 s->iop.writeback = true; in flash_dev_make_request()
1346 s->iop.bio = bio; in flash_dev_make_request()
1348 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in flash_dev_make_request()
1350 closure_call(&s->iop.cl, cache_lookup, NULL, cl); in flash_dev_make_request()