273 	struct cache *ca = bio->bi_private;  in write_super_endio()  local
276 	bch_count_io_errors(ca, bio->bi_status, 0,  in write_super_endio()
278 	closure_put(&ca->set->sb_write);  in write_super_endio()
291 	struct cache *ca;  in bcache_write_super()  local
299 	for_each_cache(ca, c, i) {  in bcache_write_super()
300 		struct bio *bio = &ca->sb_bio;  in bcache_write_super()
302 		ca->sb.version		= BCACHE_SB_VERSION_CDEV_WITH_UUID;  in bcache_write_super()
303 		ca->sb.seq		= c->sb.seq;  in bcache_write_super()
304 		ca->sb.last_mount	= c->sb.last_mount;  in bcache_write_super()
306 		SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));  in bcache_write_super()
309 		bio_set_dev(bio, ca->bdev);  in bcache_write_super()
311 		bio->bi_private = ca;  in bcache_write_super()
314 		__write_super(&ca->sb, bio);  in bcache_write_super()
425 	struct cache *ca;  in __uuid_write()  local
438 	ca = PTR_CACHE(c, &k.key, 0);  in __uuid_write()
439 	atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written);  in __uuid_write()
504 	struct cache *ca = bio->bi_private;  in prio_endio()  local
506 	cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");  in prio_endio()
507 	bch_bbio_free(bio, ca->set);  in prio_endio()
508 	closure_put(&ca->prio);  in prio_endio()
511 static void prio_io(struct cache *ca, uint64_t bucket, int op,  in prio_io()  argument
514 	struct closure *cl = &ca->prio;  in prio_io()
515 	struct bio *bio = bch_bbio_alloc(ca->set);  in prio_io()
519 	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;  in prio_io()
520 	bio_set_dev(bio, ca->bdev);  in prio_io()
521 	bio->bi_iter.bi_size	= bucket_bytes(ca);  in prio_io()
524 	bio->bi_private = ca;  in prio_io()
526 	bch_bio_map(bio, ca->disk_buckets);  in prio_io()
528 	closure_bio_submit(ca->set, bio, &ca->prio);  in prio_io()
532 void bch_prio_write(struct cache *ca)  in bch_prio_write()  argument
540 	lockdep_assert_held(&ca->set->bucket_lock);  in bch_prio_write()
542 	ca->disk_buckets->seq++;  in bch_prio_write()
544 	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),  in bch_prio_write()
545 			&ca->meta_sectors_written);  in bch_prio_write()
550 	for (i = prio_buckets(ca) - 1; i >= 0; --i) {  in bch_prio_write()
552 		struct prio_set *p = ca->disk_buckets;  in bch_prio_write()
554 		struct bucket_disk *end = d + prios_per_bucket(ca);  in bch_prio_write()
556 		for (b = ca->buckets + i * prios_per_bucket(ca);  in bch_prio_write()
557 		     b < ca->buckets + ca->sb.nbuckets && d < end;  in bch_prio_write()
563 		p->next_bucket	= ca->prio_buckets[i + 1];  in bch_prio_write()
564 		p->magic	= pset_magic(&ca->sb);  in bch_prio_write()
565 		p->csum		= bch_crc64(&p->magic, bucket_bytes(ca) - 8);  in bch_prio_write()
567 		bucket = bch_bucket_alloc(ca, RESERVE_PRIO, true);  in bch_prio_write()
570 		mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
571 		prio_io(ca, bucket, REQ_OP_WRITE, 0);  in bch_prio_write()
572 		mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
574 		ca->prio_buckets[i] = bucket;  in bch_prio_write()
575 		atomic_dec_bug(&ca->buckets[bucket].pin);  in bch_prio_write()
578 	mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
580 	bch_journal_meta(ca->set, &cl);  in bch_prio_write()
583 	mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
589 	for (i = 0; i < prio_buckets(ca); i++) {  in bch_prio_write()
590 		if (ca->prio_last_buckets[i])  in bch_prio_write()
591 			__bch_bucket_free(ca,  in bch_prio_write()
592 				&ca->buckets[ca->prio_last_buckets[i]]);  in bch_prio_write()
594 		ca->prio_last_buckets[i] = ca->prio_buckets[i];  in bch_prio_write()
598 static void prio_read(struct cache *ca, uint64_t bucket)  in prio_read()  argument
600 	struct prio_set *p = ca->disk_buckets;  in prio_read()
601 	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;  in prio_read()
605 	for (b = ca->buckets;  in prio_read()
606 	     b < ca->buckets + ca->sb.nbuckets;  in prio_read()
609 			ca->prio_buckets[bucket_nr] = bucket;  in prio_read()
610 			ca->prio_last_buckets[bucket_nr] = bucket;  in prio_read()
613 			prio_io(ca, bucket, REQ_OP_READ, 0);  in prio_read()
616 			    bch_crc64(&p->magic, bucket_bytes(ca) - 8))  in prio_read()
619 			if (p->magic != pset_magic(&ca->sb))  in prio_read()
683 		struct cache *ca;  in bcache_device_unlink()  local
688 		for_each_cache(ca, d->c, i)  in bcache_device_unlink()
689 			bd_unlink_disk_holder(ca->bdev, d->disk);  in bcache_device_unlink()
697 	struct cache *ca;  in bcache_device_link()  local
700 	for_each_cache(ca, d->c, i)  in bcache_device_link()
701 		bd_link_disk_holder(ca->bdev, d->disk);  in bcache_device_link()
1547 	struct cache *ca;  in cache_set_free()  local
1557 	for_each_cache(ca, c, i)  in cache_set_free()
1558 		if (ca) {  in cache_set_free()
1559 			ca->set = NULL;  in cache_set_free()
1560 			c->cache[ca->sb.nr_this_dev] = NULL;  in cache_set_free()
1561 			kobject_put(&ca->kobj);  in cache_set_free()
1588 	struct cache *ca;  in cache_set_flush()  local
1615 	for_each_cache(ca, c, i)  in cache_set_flush()
1616 		if (ca->alloc_thread)  in cache_set_flush()
1617 			kthread_stop(ca->alloc_thread);  in cache_set_flush()
1824 	struct cache *ca;  in run_cache_set()  local
1832 	for_each_cache(ca, c, i)  in run_cache_set()
1833 		c->nbuckets += ca->sb.nbuckets;  in run_cache_set()
1853 		for_each_cache(ca, c, i)  in run_cache_set()
1854 			prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]);  in run_cache_set()
1915 		for_each_cache(ca, c, i)  in run_cache_set()
1916 			if (bch_cache_allocator_start(ca))  in run_cache_set()
1938 		for_each_cache(ca, c, i) {  in run_cache_set()
1941 			ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,  in run_cache_set()
1944 			for (j = 0; j < ca->sb.keys; j++)  in run_cache_set()
1945 				ca->sb.d[j] = ca->sb.first_bucket + j;  in run_cache_set()
1951 		for_each_cache(ca, c, i)  in run_cache_set()
1952 			if (bch_cache_allocator_start(ca))  in run_cache_set()
1956 		for_each_cache(ca, c, i)  in run_cache_set()
1957 			bch_prio_write(ca);  in run_cache_set()
2017 static bool can_attach_cache(struct cache *ca, struct cache_set *c)  in can_attach_cache()  argument
2019 	return ca->sb.block_size	== c->sb.block_size &&  in can_attach_cache()
2020 		ca->sb.bucket_size	== c->sb.bucket_size &&  in can_attach_cache()
2021 		ca->sb.nr_in_set	== c->sb.nr_in_set;  in can_attach_cache()
2024 static const char *register_cache_set(struct cache *ca)  in register_cache_set()  argument
2031 		if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) {  in register_cache_set()
2032 			if (c->cache[ca->sb.nr_this_dev])  in register_cache_set()
2035 			if (!can_attach_cache(ca, c))  in register_cache_set()
2038 			if (!CACHE_SYNC(&ca->sb))  in register_cache_set()
2044 	c = bch_cache_set_alloc(&ca->sb);  in register_cache_set()
2060 	sprintf(buf, "cache%i", ca->sb.nr_this_dev);  in register_cache_set()
2061 	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||  in register_cache_set()
2062 	    sysfs_create_link(&c->kobj, &ca->kobj, buf))  in register_cache_set()
2065 	if (ca->sb.seq > c->sb.seq) {  in register_cache_set()
2066 		c->sb.version		= ca->sb.version;  in register_cache_set()
2067 		memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16);  in register_cache_set()
2068 		c->sb.flags             = ca->sb.flags;  in register_cache_set()
2069 		c->sb.seq		= ca->sb.seq;  in register_cache_set()
2073 	kobject_get(&ca->kobj);  in register_cache_set()
2074 	ca->set = c;  in register_cache_set()
2075 	ca->set->cache[ca->sb.nr_this_dev] = ca;  in register_cache_set()
2076 	c->cache_by_alloc[c->caches_loaded++] = ca;  in register_cache_set()
2095 	struct cache *ca = container_of(kobj, struct cache, kobj);  in bch_cache_release()  local
2098 	if (ca->set) {  in bch_cache_release()
2099 		BUG_ON(ca->set->cache[ca->sb.nr_this_dev] != ca);  in bch_cache_release()
2100 		ca->set->cache[ca->sb.nr_this_dev] = NULL;  in bch_cache_release()
2103 	free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));  in bch_cache_release()
2104 	kfree(ca->prio_buckets);  in bch_cache_release()
2105 	vfree(ca->buckets);  in bch_cache_release()
2107 	free_heap(&ca->heap);  in bch_cache_release()
2108 	free_fifo(&ca->free_inc);  in bch_cache_release()
2111 		free_fifo(&ca->free[i]);  in bch_cache_release()
2113 	if (ca->sb_bio.bi_inline_vecs[0].bv_page)  in bch_cache_release()
2114 		put_page(bio_first_page_all(&ca->sb_bio));  in bch_cache_release()
2116 	if (!IS_ERR_OR_NULL(ca->bdev))  in bch_cache_release()
2117 		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);  in bch_cache_release()
2119 	kfree(ca);  in bch_cache_release()
2123 static int cache_alloc(struct cache *ca)  in cache_alloc()  argument
2132 	kobject_init(&ca->kobj, &bch_cache_ktype);  in cache_alloc()
2134 	bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8);  in cache_alloc()
2145 	btree_buckets = ca->sb.njournal_buckets ?: 8;  in cache_alloc()
2146 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;  in cache_alloc()
2153 	if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets,  in cache_alloc()
2159 	if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca),  in cache_alloc()
2165 	if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) {  in cache_alloc()
2170 	if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) {  in cache_alloc()
2175 	if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) {  in cache_alloc()
2180 	if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) {  in cache_alloc()
2185 	ca->buckets = vzalloc(array_size(sizeof(struct bucket),  in cache_alloc()
2186 			      ca->sb.nbuckets));  in cache_alloc()
2187 	if (!ca->buckets) {  in cache_alloc()
2192 	ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t),  in cache_alloc()
2193 				   prio_buckets(ca), 2),  in cache_alloc()
2195 	if (!ca->prio_buckets) {  in cache_alloc()
2200 	ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca);  in cache_alloc()
2201 	if (!ca->disk_buckets) {  in cache_alloc()
2206 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);  in cache_alloc()
2208 	for_each_bucket(b, ca)  in cache_alloc()
2213 	kfree(ca->prio_buckets);  in cache_alloc()
2215 	vfree(ca->buckets);  in cache_alloc()
2217 	free_heap(&ca->heap);  in cache_alloc()
2219 	free_fifo(&ca->free_inc);  in cache_alloc()
2221 	free_fifo(&ca->free[RESERVE_NONE]);  in cache_alloc()
2223 	free_fifo(&ca->free[RESERVE_MOVINGGC]);  in cache_alloc()
2225 	free_fifo(&ca->free[RESERVE_PRIO]);  in cache_alloc()
2227 	free_fifo(&ca->free[RESERVE_BTREE]);  in cache_alloc()
2232 		pr_notice("error %s: %s", ca->cache_dev_name, err);  in cache_alloc()
2237 				struct block_device *bdev, struct cache *ca)  in register_cache()  argument
2242 	bdevname(bdev, ca->cache_dev_name);  in register_cache()
2243 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));  in register_cache()
2244 	ca->bdev = bdev;  in register_cache()
2245 	ca->bdev->bd_holder = ca;  in register_cache()
2247 	bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);  in register_cache()
2248 	bio_first_bvec_all(&ca->sb_bio)->bv_page = sb_page;  in register_cache()
2252 		ca->discard = CACHE_DISCARD(&ca->sb);  in register_cache()
2254 	ret = cache_alloc(ca);  in register_cache()
2272 	if (kobject_add(&ca->kobj,  in register_cache()
2281 	err = register_cache_set(ca);  in register_cache()
2289 	pr_info("registered cache device %s", ca->cache_dev_name);  in register_cache()
2292 	kobject_put(&ca->kobj);  in register_cache()
2296 		pr_notice("error %s: %s", ca->cache_dev_name, err);  in register_cache()
2331 	struct cache *ca;  in bch_is_open_cache()  local
2335 		for_each_cache(ca, c, i)  in bch_is_open_cache()
2336 			if (ca->bdev == bdev)  in bch_is_open_cache()
2415 		struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);  in register_bcache()  local
2417 		if (!ca)  in register_bcache()
2421 		if (register_cache(sb, sb_page, bdev, ca) != 0)  in register_bcache()