328 	struct cache *ca = bio->bi_private;  in write_super_endio()  local
331 	bch_count_io_errors(ca, bio->bi_status, 0,  in write_super_endio()
333 	closure_put(&ca->set->sb_write);  in write_super_endio()
346 	struct cache *ca = c->cache;  in bcache_write_super()  local
347 	struct bio *bio = &ca->sb_bio;  in bcache_write_super()
353 	ca->sb.seq++;  in bcache_write_super()
355 	if (ca->sb.version < version)  in bcache_write_super()
356 		ca->sb.version = version;  in bcache_write_super()
358 	bio_init(bio, ca->sb_bv, 1);  in bcache_write_super()
359 	bio_set_dev(bio, ca->bdev);  in bcache_write_super()
361 	bio->bi_private = ca;  in bcache_write_super()
364 	__write_super(&ca->sb, ca->sb_disk, bio);  in bcache_write_super()
474 	struct cache *ca = c->cache;  in __uuid_write()  local
483 	size =  meta_bucket_pages(&ca->sb) * PAGE_SECTORS;  in __uuid_write()
489 	atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written);  in __uuid_write()
554 	struct cache *ca = bio->bi_private;  in prio_endio()  local
556 	cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");  in prio_endio()
557 	bch_bbio_free(bio, ca->set);  in prio_endio()
558 	closure_put(&ca->prio);  in prio_endio()
561 static void prio_io(struct cache *ca, uint64_t bucket, int op,  in prio_io()  argument
564 	struct closure *cl = &ca->prio;  in prio_io()
565 	struct bio *bio = bch_bbio_alloc(ca->set);  in prio_io()
569 	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;  in prio_io()
570 	bio_set_dev(bio, ca->bdev);  in prio_io()
571 	bio->bi_iter.bi_size	= meta_bucket_bytes(&ca->sb);  in prio_io()
574 	bio->bi_private = ca;  in prio_io()
576 	bch_bio_map(bio, ca->disk_buckets);  in prio_io()
578 	closure_bio_submit(ca->set, bio, &ca->prio);  in prio_io()
582 int bch_prio_write(struct cache *ca, bool wait)  in bch_prio_write()  argument
589 		 fifo_used(&ca->free[RESERVE_PRIO]),  in bch_prio_write()
590 		 fifo_used(&ca->free[RESERVE_NONE]),  in bch_prio_write()
591 		 fifo_used(&ca->free_inc));  in bch_prio_write()
599 		size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) +  in bch_prio_write()
600 			       fifo_used(&ca->free[RESERVE_NONE]);  in bch_prio_write()
601 		if (prio_buckets(ca) > avail)  in bch_prio_write()
607 	lockdep_assert_held(&ca->set->bucket_lock);  in bch_prio_write()
609 	ca->disk_buckets->seq++;  in bch_prio_write()
611 	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),  in bch_prio_write()
612 			&ca->meta_sectors_written);  in bch_prio_write()
614 	for (i = prio_buckets(ca) - 1; i >= 0; --i) {  in bch_prio_write()
616 		struct prio_set *p = ca->disk_buckets;  in bch_prio_write()
618 		struct bucket_disk *end = d + prios_per_bucket(ca);  in bch_prio_write()
620 		for (b = ca->buckets + i * prios_per_bucket(ca);  in bch_prio_write()
621 		     b < ca->buckets + ca->sb.nbuckets && d < end;  in bch_prio_write()
627 		p->next_bucket	= ca->prio_buckets[i + 1];  in bch_prio_write()
628 		p->magic	= pset_magic(&ca->sb);  in bch_prio_write()
629 		p->csum		= bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8);  in bch_prio_write()
631 		bucket = bch_bucket_alloc(ca, RESERVE_PRIO, wait);  in bch_prio_write()
634 		mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
635 		prio_io(ca, bucket, REQ_OP_WRITE, 0);  in bch_prio_write()
636 		mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
638 		ca->prio_buckets[i] = bucket;  in bch_prio_write()
639 		atomic_dec_bug(&ca->buckets[bucket].pin);  in bch_prio_write()
642 	mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
644 	bch_journal_meta(ca->set, &cl);  in bch_prio_write()
647 	mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
653 	for (i = 0; i < prio_buckets(ca); i++) {  in bch_prio_write()
654 		if (ca->prio_last_buckets[i])  in bch_prio_write()
655 			__bch_bucket_free(ca,  in bch_prio_write()
656 				&ca->buckets[ca->prio_last_buckets[i]]);  in bch_prio_write()
658 		ca->prio_last_buckets[i] = ca->prio_buckets[i];  in bch_prio_write()
663 static int prio_read(struct cache *ca, uint64_t bucket)  in prio_read()  argument
665 	struct prio_set *p = ca->disk_buckets;  in prio_read()
666 	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;  in prio_read()
671 	for (b = ca->buckets;  in prio_read()
672 	     b < ca->buckets + ca->sb.nbuckets;  in prio_read()
675 			ca->prio_buckets[bucket_nr] = bucket;  in prio_read()
676 			ca->prio_last_buckets[bucket_nr] = bucket;  in prio_read()
679 			prio_io(ca, bucket, REQ_OP_READ, 0);  in prio_read()
682 			    bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) {  in prio_read()
687 			if (p->magic != pset_magic(&ca->sb)) {  in prio_read()
765 		struct cache *ca = d->c->cache;  in bcache_device_unlink()  local
770 		bd_unlink_disk_holder(ca->bdev, d->disk);  in bcache_device_unlink()
777 	struct cache *ca = c->cache;  in bcache_device_link()  local
780 	bd_link_disk_holder(ca->bdev, d->disk);  in bcache_device_link()
1650 	struct cache *ca;  in cache_set_free()  local
1662 	ca = c->cache;  in cache_set_free()
1663 	if (ca) {  in cache_set_free()
1664 		ca->set = NULL;  in cache_set_free()
1666 		kobject_put(&ca->kobj);  in cache_set_free()
1691 	struct cache *ca = c->cache;  in cache_set_flush()  local
1717 	if (ca->alloc_thread)  in cache_set_flush()
1718 		kthread_stop(ca->alloc_thread);  in cache_set_flush()
1834 	struct cache *ca = container_of(sb, struct cache, sb);  in bch_cache_set_alloc()  local
1858 	c->cache		= ca;  in bch_cache_set_alloc()
1949 	struct cache *ca = c->cache;  in run_cache_set()  local
1956 	c->nbuckets = ca->sb.nbuckets;  in run_cache_set()
1976 		if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]))  in run_cache_set()
2021 		if (bch_cache_allocator_start(ca))  in run_cache_set()
2044 		ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,  in run_cache_set()
2047 		for (j = 0; j < ca->sb.keys; j++)  in run_cache_set()
2048 			ca->sb.d[j] = ca->sb.first_bucket + j;  in run_cache_set()
2053 		if (bch_cache_allocator_start(ca))  in run_cache_set()
2057 		bch_prio_write(ca, true);  in run_cache_set()
2117 static const char *register_cache_set(struct cache *ca)  in register_cache_set()  argument
2124 		if (!memcmp(c->set_uuid, ca->sb.set_uuid, 16)) {  in register_cache_set()
2131 	c = bch_cache_set_alloc(&ca->sb);  in register_cache_set()
2147 	sprintf(buf, "cache%i", ca->sb.nr_this_dev);  in register_cache_set()
2148 	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||  in register_cache_set()
2149 	    sysfs_create_link(&c->kobj, &ca->kobj, buf))  in register_cache_set()
2152 	kobject_get(&ca->kobj);  in register_cache_set()
2153 	ca->set = c;  in register_cache_set()
2154 	ca->set->cache = ca;  in register_cache_set()
2171 	struct cache *ca = container_of(kobj, struct cache, kobj);  in bch_cache_release()  local
2174 	if (ca->set) {  in bch_cache_release()
2175 		BUG_ON(ca->set->cache != ca);  in bch_cache_release()
2176 		ca->set->cache = NULL;  in bch_cache_release()
2179 	free_pages((unsigned long) ca->disk_buckets, ilog2(meta_bucket_pages(&ca->sb)));  in bch_cache_release()
2180 	kfree(ca->prio_buckets);  in bch_cache_release()
2181 	vfree(ca->buckets);  in bch_cache_release()
2183 	free_heap(&ca->heap);  in bch_cache_release()
2184 	free_fifo(&ca->free_inc);  in bch_cache_release()
2187 		free_fifo(&ca->free[i]);  in bch_cache_release()
2189 	if (ca->sb_disk)  in bch_cache_release()
2190 		put_page(virt_to_page(ca->sb_disk));  in bch_cache_release()
2192 	if (!IS_ERR_OR_NULL(ca->bdev))  in bch_cache_release()
2193 		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);  in bch_cache_release()
2195 	kfree(ca);  in bch_cache_release()
2199 static int cache_alloc(struct cache *ca)  in cache_alloc()  argument
2208 	kobject_init(&ca->kobj, &bch_cache_ktype);  in cache_alloc()
2210 	bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8);  in cache_alloc()
2221 	btree_buckets = ca->sb.njournal_buckets ?: 8;  in cache_alloc()
2222 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;  in cache_alloc()
2229 	if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets,  in cache_alloc()
2235 	if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca),  in cache_alloc()
2241 	if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) {  in cache_alloc()
2246 	if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) {  in cache_alloc()
2251 	if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) {  in cache_alloc()
2256 	if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) {  in cache_alloc()
2261 	ca->buckets = vzalloc(array_size(sizeof(struct bucket),  in cache_alloc()
2262 			      ca->sb.nbuckets));  in cache_alloc()
2263 	if (!ca->buckets) {  in cache_alloc()
2268 	ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t),  in cache_alloc()
2269 				   prio_buckets(ca), 2),  in cache_alloc()
2271 	if (!ca->prio_buckets) {  in cache_alloc()
2276 	ca->disk_buckets = alloc_meta_bucket_pages(GFP_KERNEL, &ca->sb);  in cache_alloc()
2277 	if (!ca->disk_buckets) {  in cache_alloc()
2282 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);  in cache_alloc()
2284 	for_each_bucket(b, ca)  in cache_alloc()
2289 	kfree(ca->prio_buckets);  in cache_alloc()
2291 	vfree(ca->buckets);  in cache_alloc()
2293 	free_heap(&ca->heap);  in cache_alloc()
2295 	free_fifo(&ca->free_inc);  in cache_alloc()
2297 	free_fifo(&ca->free[RESERVE_NONE]);  in cache_alloc()
2299 	free_fifo(&ca->free[RESERVE_MOVINGGC]);  in cache_alloc()
2301 	free_fifo(&ca->free[RESERVE_PRIO]);  in cache_alloc()
2303 	free_fifo(&ca->free[RESERVE_BTREE]);  in cache_alloc()
2308 		pr_notice("error %s: %s\n", ca->cache_dev_name, err);  in cache_alloc()
2313 				struct block_device *bdev, struct cache *ca)  in register_cache()  argument
2318 	bdevname(bdev, ca->cache_dev_name);  in register_cache()
2319 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));  in register_cache()
2320 	ca->bdev = bdev;  in register_cache()
2321 	ca->bdev->bd_holder = ca;  in register_cache()
2322 	ca->sb_disk = sb_disk;  in register_cache()
2325 		ca->discard = CACHE_DISCARD(&ca->sb);  in register_cache()
2327 	ret = cache_alloc(ca);  in register_cache()
2345 	if (kobject_add(&ca->kobj,  in register_cache()
2354 	err = register_cache_set(ca);  in register_cache()
2362 	pr_info("registered cache device %s\n", ca->cache_dev_name);  in register_cache()
2365 	kobject_put(&ca->kobj);  in register_cache()
2369 		pr_notice("error %s: %s\n", ca->cache_dev_name, err);  in register_cache()
2406 		struct cache *ca = c->cache;  in bch_is_open_cache()  local
2408 		if (ca->bdev == bdev)  in bch_is_open_cache()
2463 	struct cache *ca;  in register_cache_worker()  local
2465 	ca = kzalloc(sizeof(*ca), GFP_KERNEL);  in register_cache_worker()
2466 	if (!ca) {  in register_cache_worker()
2474 	if (register_cache(args->sb, args->sb_disk, args->bdev, ca) != 0)  in register_cache_worker()
2599 		struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);  in register_bcache()  local
2601 		if (!ca)  in register_bcache()
2605 		if (register_cache(sb, sb_disk, bdev, ca) != 0)  in register_bcache()