269 	struct cache *ca = bio->bi_private;  in write_super_endio()  local
272 	bch_count_io_errors(ca, bio->bi_status, 0,  in write_super_endio()
274 	closure_put(&ca->set->sb_write);  in write_super_endio()
287 	struct cache *ca;  in bcache_write_super()  local
295 	for_each_cache(ca, c, i) {  in bcache_write_super()
296 		struct bio *bio = &ca->sb_bio;  in bcache_write_super()
298 		ca->sb.version		= BCACHE_SB_VERSION_CDEV_WITH_UUID;  in bcache_write_super()
299 		ca->sb.seq		= c->sb.seq;  in bcache_write_super()
300 		ca->sb.last_mount	= c->sb.last_mount;  in bcache_write_super()
302 		SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));  in bcache_write_super()
305 		bio_set_dev(bio, ca->bdev);  in bcache_write_super()
307 		bio->bi_private = ca;  in bcache_write_super()
310 		__write_super(&ca->sb, bio);  in bcache_write_super()
495 	struct cache *ca = bio->bi_private;  in prio_endio()  local
497 	cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");  in prio_endio()
498 	bch_bbio_free(bio, ca->set);  in prio_endio()
499 	closure_put(&ca->prio);  in prio_endio()
502 static void prio_io(struct cache *ca, uint64_t bucket, int op,  in prio_io()  argument
505 	struct closure *cl = &ca->prio;  in prio_io()
506 	struct bio *bio = bch_bbio_alloc(ca->set);  in prio_io()
510 	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;  in prio_io()
511 	bio_set_dev(bio, ca->bdev);  in prio_io()
512 	bio->bi_iter.bi_size	= bucket_bytes(ca);  in prio_io()
515 	bio->bi_private = ca;  in prio_io()
517 	bch_bio_map(bio, ca->disk_buckets);  in prio_io()
519 	closure_bio_submit(ca->set, bio, &ca->prio);  in prio_io()
523 void bch_prio_write(struct cache *ca)  in bch_prio_write()  argument
531 	lockdep_assert_held(&ca->set->bucket_lock);  in bch_prio_write()
533 	ca->disk_buckets->seq++;  in bch_prio_write()
535 	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),  in bch_prio_write()
536 			&ca->meta_sectors_written);  in bch_prio_write()
541 	for (i = prio_buckets(ca) - 1; i >= 0; --i) {  in bch_prio_write()
543 		struct prio_set *p = ca->disk_buckets;  in bch_prio_write()
545 		struct bucket_disk *end = d + prios_per_bucket(ca);  in bch_prio_write()
547 		for (b = ca->buckets + i * prios_per_bucket(ca);  in bch_prio_write()
548 		     b < ca->buckets + ca->sb.nbuckets && d < end;  in bch_prio_write()
554 		p->next_bucket	= ca->prio_buckets[i + 1];  in bch_prio_write()
555 		p->magic	= pset_magic(&ca->sb);  in bch_prio_write()
556 		p->csum		= bch_crc64(&p->magic, bucket_bytes(ca) - 8);  in bch_prio_write()
558 		bucket = bch_bucket_alloc(ca, RESERVE_PRIO, true);  in bch_prio_write()
561 		mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
562 		prio_io(ca, bucket, REQ_OP_WRITE, 0);  in bch_prio_write()
563 		mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
565 		ca->prio_buckets[i] = bucket;  in bch_prio_write()
566 		atomic_dec_bug(&ca->buckets[bucket].pin);  in bch_prio_write()
569 	mutex_unlock(&ca->set->bucket_lock);  in bch_prio_write()
571 	bch_journal_meta(ca->set, &cl);  in bch_prio_write()
574 	mutex_lock(&ca->set->bucket_lock);  in bch_prio_write()
580 	for (i = 0; i < prio_buckets(ca); i++) {  in bch_prio_write()
581 		if (ca->prio_last_buckets[i])  in bch_prio_write()
582 			__bch_bucket_free(ca,  in bch_prio_write()
583 				&ca->buckets[ca->prio_last_buckets[i]]);  in bch_prio_write()
585 		ca->prio_last_buckets[i] = ca->prio_buckets[i];  in bch_prio_write()
589 static void prio_read(struct cache *ca, uint64_t bucket)  in prio_read()  argument
591 	struct prio_set *p = ca->disk_buckets;  in prio_read()
592 	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;  in prio_read()
596 	for (b = ca->buckets;  in prio_read()
597 	     b < ca->buckets + ca->sb.nbuckets;  in prio_read()
600 			ca->prio_buckets[bucket_nr] = bucket;  in prio_read()
601 			ca->prio_last_buckets[bucket_nr] = bucket;  in prio_read()
604 			prio_io(ca, bucket, REQ_OP_READ, 0);  in prio_read()
607 			    bch_crc64(&p->magic, bucket_bytes(ca) - 8))  in prio_read()
610 			if (p->magic != pset_magic(&ca->sb))  in prio_read()
673 		struct cache *ca;  in bcache_device_unlink()  local
678 		for_each_cache(ca, d->c, i)  in bcache_device_unlink()
679 			bd_unlink_disk_holder(ca->bdev, d->disk);  in bcache_device_unlink()
687 	struct cache *ca;  in bcache_device_link()  local
689 	for_each_cache(ca, d->c, i)  in bcache_device_link()
690 		bd_link_disk_holder(ca->bdev, d->disk);  in bcache_device_link()
1507 	struct cache *ca;  in cache_set_free()  local
1517 	for_each_cache(ca, c, i)  in cache_set_free()
1518 		if (ca) {  in cache_set_free()
1519 			ca->set = NULL;  in cache_set_free()
1520 			c->cache[ca->sb.nr_this_dev] = NULL;  in cache_set_free()
1521 			kobject_put(&ca->kobj);  in cache_set_free()
1549 	struct cache *ca;  in cache_set_flush()  local
1572 	for_each_cache(ca, c, i)  in cache_set_flush()
1573 		if (ca->alloc_thread)  in cache_set_flush()
1574 			kthread_stop(ca->alloc_thread);  in cache_set_flush()
1780 	struct cache *ca;  in run_cache_set()  local
1786 	for_each_cache(ca, c, i)  in run_cache_set()
1787 		c->nbuckets += ca->sb.nbuckets;  in run_cache_set()
1808 		for_each_cache(ca, c, i)  in run_cache_set()
1809 			prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]);  in run_cache_set()
1853 		for_each_cache(ca, c, i)  in run_cache_set()
1854 			if (bch_cache_allocator_start(ca))  in run_cache_set()
1874 		for_each_cache(ca, c, i) {  in run_cache_set()
1877 			ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,  in run_cache_set()
1880 			for (j = 0; j < ca->sb.keys; j++)  in run_cache_set()
1881 				ca->sb.d[j] = ca->sb.first_bucket + j;  in run_cache_set()
1887 		for_each_cache(ca, c, i)  in run_cache_set()
1888 			if (bch_cache_allocator_start(ca))  in run_cache_set()
1892 		for_each_cache(ca, c, i)  in run_cache_set()
1893 			bch_prio_write(ca);  in run_cache_set()
1945 static bool can_attach_cache(struct cache *ca, struct cache_set *c)  in can_attach_cache()  argument
1947 	return ca->sb.block_size	== c->sb.block_size &&  in can_attach_cache()
1948 		ca->sb.bucket_size	== c->sb.bucket_size &&  in can_attach_cache()
1949 		ca->sb.nr_in_set	== c->sb.nr_in_set;  in can_attach_cache()
1952 static const char *register_cache_set(struct cache *ca)  in register_cache_set()  argument
1959 		if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) {  in register_cache_set()
1960 			if (c->cache[ca->sb.nr_this_dev])  in register_cache_set()
1963 			if (!can_attach_cache(ca, c))  in register_cache_set()
1966 			if (!CACHE_SYNC(&ca->sb))  in register_cache_set()
1972 	c = bch_cache_set_alloc(&ca->sb);  in register_cache_set()
1988 	sprintf(buf, "cache%i", ca->sb.nr_this_dev);  in register_cache_set()
1989 	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||  in register_cache_set()
1990 	    sysfs_create_link(&c->kobj, &ca->kobj, buf))  in register_cache_set()
1993 	if (ca->sb.seq > c->sb.seq) {  in register_cache_set()
1994 		c->sb.version		= ca->sb.version;  in register_cache_set()
1995 		memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16);  in register_cache_set()
1996 		c->sb.flags             = ca->sb.flags;  in register_cache_set()
1997 		c->sb.seq		= ca->sb.seq;  in register_cache_set()
2001 	kobject_get(&ca->kobj);  in register_cache_set()
2002 	ca->set = c;  in register_cache_set()
2003 	ca->set->cache[ca->sb.nr_this_dev] = ca;  in register_cache_set()
2004 	c->cache_by_alloc[c->caches_loaded++] = ca;  in register_cache_set()
2019 	struct cache *ca = container_of(kobj, struct cache, kobj);  in bch_cache_release()  local
2022 	if (ca->set) {  in bch_cache_release()
2023 		BUG_ON(ca->set->cache[ca->sb.nr_this_dev] != ca);  in bch_cache_release()
2024 		ca->set->cache[ca->sb.nr_this_dev] = NULL;  in bch_cache_release()
2027 	free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));  in bch_cache_release()
2028 	kfree(ca->prio_buckets);  in bch_cache_release()
2029 	vfree(ca->buckets);  in bch_cache_release()
2031 	free_heap(&ca->heap);  in bch_cache_release()
2032 	free_fifo(&ca->free_inc);  in bch_cache_release()
2035 		free_fifo(&ca->free[i]);  in bch_cache_release()
2037 	if (ca->sb_bio.bi_inline_vecs[0].bv_page)  in bch_cache_release()
2038 		put_page(bio_first_page_all(&ca->sb_bio));  in bch_cache_release()
2040 	if (!IS_ERR_OR_NULL(ca->bdev))  in bch_cache_release()
2041 		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);  in bch_cache_release()
2043 	kfree(ca);  in bch_cache_release()
2047 static int cache_alloc(struct cache *ca)  in cache_alloc()  argument
2054 	kobject_init(&ca->kobj, &bch_cache_ktype);  in cache_alloc()
2056 	bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8);  in cache_alloc()
2067 	btree_buckets = ca->sb.njournal_buckets ?: 8;  in cache_alloc()
2068 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;  in cache_alloc()
2070 	if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, GFP_KERNEL) ||  in cache_alloc()
2071 	    !init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||  in cache_alloc()
2072 	    !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) ||  in cache_alloc()
2073 	    !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) ||  in cache_alloc()
2074 	    !init_fifo(&ca->free_inc,	free << 2, GFP_KERNEL) ||  in cache_alloc()
2075 	    !init_heap(&ca->heap,	free << 3, GFP_KERNEL) ||  in cache_alloc()
2076 	    !(ca->buckets	= vzalloc(array_size(sizeof(struct bucket),  in cache_alloc()
2077 						     ca->sb.nbuckets))) ||  in cache_alloc()
2078 	    !(ca->prio_buckets	= kzalloc(array3_size(sizeof(uint64_t),  in cache_alloc()
2079 						      prio_buckets(ca), 2),  in cache_alloc()
2081 	    !(ca->disk_buckets	= alloc_bucket_pages(GFP_KERNEL, ca)))  in cache_alloc()
2084 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);  in cache_alloc()
2086 	for_each_bucket(b, ca)  in cache_alloc()
2093 				struct block_device *bdev, struct cache *ca)  in register_cache()  argument
2098 	bdevname(bdev, ca->cache_dev_name);  in register_cache()
2099 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));  in register_cache()
2100 	ca->bdev = bdev;  in register_cache()
2101 	ca->bdev->bd_holder = ca;  in register_cache()
2103 	bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);  in register_cache()
2104 	bio_first_bvec_all(&ca->sb_bio)->bv_page = sb_page;  in register_cache()
2108 		ca->discard = CACHE_DISCARD(&ca->sb);  in register_cache()
2110 	ret = cache_alloc(ca);  in register_cache()
2120 	if (kobject_add(&ca->kobj,  in register_cache()
2129 	err = register_cache_set(ca);  in register_cache()
2137 	pr_info("registered cache device %s", ca->cache_dev_name);  in register_cache()
2140 	kobject_put(&ca->kobj);  in register_cache()
2144 		pr_notice("error %s: %s", ca->cache_dev_name, err);  in register_cache()
2175 	struct cache *ca;  in bch_is_open_cache()  local
2179 		for_each_cache(ca, c, i)  in bch_is_open_cache()
2180 			if (ca->bdev == bdev)  in bch_is_open_cache()
2251 		struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);  in register_bcache()  local
2253 		if (!ca)  in register_bcache()
2256 		if (register_cache(sb, sb_page, bdev, ca) != 0)  in register_bcache()