28 static void bch_data_insert_start(struct closure *cl);
58 static void bch_data_insert_keys(struct closure *cl)  in bch_data_insert_keys()  argument
60 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_keys()
73 	while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)  in bch_data_insert_keys()
74 		closure_sync(&s->cl);  in bch_data_insert_keys()
79 					  op->flush_journal ? cl : NULL);  in bch_data_insert_keys()
94 		continue_at(cl, bch_data_insert_start, op->wq);  in bch_data_insert_keys()
99 	closure_return(cl);  in bch_data_insert_keys()
120 static void bch_data_invalidate(struct closure *cl)  in bch_data_invalidate()  argument
122 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_invalidate()
148 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_invalidate()
151 static void bch_data_insert_error(struct closure *cl)  in bch_data_insert_error()  argument
153 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_error()
178 	bch_data_insert_keys(cl);  in bch_data_insert_error()
183 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()  local
184 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_endio()
191 			set_closure_fn(cl, bch_data_insert_error, op->wq);  in bch_data_insert_endio()
193 			set_closure_fn(cl, NULL, NULL);  in bch_data_insert_endio()
199 static void bch_data_insert_start(struct closure *cl)  in bch_data_insert_start()  argument
201 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_start()
205 		return bch_data_invalidate(cl);  in bch_data_insert_start()
225 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
242 		n->bi_private	= cl;  in bch_data_insert_start()
264 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
284 		return bch_data_invalidate(cl);  in bch_data_insert_start()
294 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
296 			closure_return(cl);  in bch_data_insert_start()
320 void bch_data_insert(struct closure *cl)  in bch_data_insert()  argument
322 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert()
329 	bch_data_insert_start(cl);  in bch_data_insert()
470 	struct closure		cl;  member
492 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()  local
493 	struct search *s = container_of(cl, struct search, cl);  in bch_cache_read_endio()
565 	n->bi_private	= &s->cl;  in cache_lookup_fn()
582 static void cache_lookup(struct closure *cl)  in cache_lookup()  argument
584 	struct search *s = container_of(cl, struct search, iop.cl);  in cache_lookup()
595 		continue_at(cl, cache_lookup, bcache_wq);  in cache_lookup()
620 	closure_return(cl);  in cache_lookup()
627 	struct closure *cl = bio->bi_private;  in request_endio()  local
630 		struct search *s = container_of(cl, struct search, cl);  in request_endio()
638 	closure_put(cl);  in request_endio()
643 	struct closure *cl = bio->bi_private;  in backing_request_endio()  local
646 		struct search *s = container_of(cl, struct search, cl);  in backing_request_endio()
670 	closure_put(cl);  in backing_request_endio()
701 	bio->bi_private		= &s->cl;  in do_bio_hook()
706 static void search_free(struct closure *cl)  in search_free()  argument
708 	struct search *s = container_of(cl, struct search, cl);  in search_free()
716 	closure_debug_destroy(cl);  in search_free()
727 	closure_init(&s->cl, NULL);  in search_alloc()
755 static void cached_dev_bio_complete(struct closure *cl)  in cached_dev_bio_complete()  argument
757 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_bio_complete()
761 	search_free(cl);  in cached_dev_bio_complete()
766 static void cached_dev_read_error_done(struct closure *cl)  in cached_dev_read_error_done()  argument
768 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error_done()
776 	cached_dev_bio_complete(cl);  in cached_dev_read_error_done()
779 static void cached_dev_read_error(struct closure *cl)  in cached_dev_read_error()  argument
781 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error()
801 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_read_error()
804 	continue_at(cl, cached_dev_read_error_done, NULL);  in cached_dev_read_error()
807 static void cached_dev_cache_miss_done(struct closure *cl)  in cached_dev_cache_miss_done()  argument
809 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_cache_miss_done()
818 	cached_dev_bio_complete(cl);  in cached_dev_cache_miss_done()
819 	closure_put(&d->cl);  in cached_dev_cache_miss_done()
822 static void cached_dev_read_done(struct closure *cl)  in cached_dev_read_done()  argument
824 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done()
852 	closure_get(&dc->disk.cl);  in cached_dev_read_done()
858 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_read_done()
861 	continue_at(cl, cached_dev_cache_miss_done, NULL);  in cached_dev_read_done()
864 static void cached_dev_read_done_bh(struct closure *cl)  in cached_dev_read_done_bh()  argument
866 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done_bh()
874 		continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);  in cached_dev_read_done_bh()
876 		continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);  in cached_dev_read_done_bh()
878 		continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);  in cached_dev_read_done_bh()
931 	cache_bio->bi_private	= &s->cl;  in cached_dev_cache_miss()
944 	closure_bio_submit(s->iop.c, cache_bio, &s->cl);  in cached_dev_cache_miss()
951 	miss->bi_private	= &s->cl;  in cached_dev_cache_miss()
953 	closure_bio_submit(s->iop.c, miss, &s->cl);  in cached_dev_cache_miss()
959 	struct closure *cl = &s->cl;  in cached_dev_read()  local
961 	closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in cached_dev_read()
962 	continue_at(cl, cached_dev_read_done_bh, NULL);  in cached_dev_read()
967 static void cached_dev_write_complete(struct closure *cl)  in cached_dev_write_complete()  argument
969 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_write_complete()
973 	cached_dev_bio_complete(cl);  in cached_dev_write_complete()
978 	struct closure *cl = &s->cl;  in cached_dev_write()  local
1022 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1043 			flush->bi_private = cl;  in cached_dev_write()
1046 			closure_bio_submit(s->iop.c, flush, cl);  in cached_dev_write()
1052 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1056 	closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_write()
1057 	continue_at(cl, cached_dev_write_complete, NULL);  in cached_dev_write()
1060 static void cached_dev_nodata(struct closure *cl)  in cached_dev_nodata()  argument
1062 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_nodata()
1066 		bch_journal_meta(s->iop.c, cl);  in cached_dev_nodata()
1070 	closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_nodata()
1072 	continue_at(cl, cached_dev_bio_complete, NULL);  in cached_dev_nodata()
1216 			continue_at_nobarrier(&s->cl,  in cached_dev_make_request()
1299 static void flash_dev_nodata(struct closure *cl)  in flash_dev_nodata()  argument
1301 	struct search *s = container_of(cl, struct search, cl);  in flash_dev_nodata()
1304 		bch_journal_meta(s->iop.c, cl);  in flash_dev_nodata()
1306 	continue_at(cl, search_free, NULL);  in flash_dev_nodata()
1313 	struct closure *cl;  in flash_dev_make_request()  local
1325 	cl = &s->cl;  in flash_dev_make_request()
1335 		continue_at_nobarrier(&s->cl,  in flash_dev_make_request()
1348 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in flash_dev_make_request()
1350 		closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in flash_dev_make_request()
1353 	continue_at(cl, search_free, NULL);  in flash_dev_make_request()