Lines Matching +full:dc +full:- +full:dc
1 // SPDX-License-Identifier: GPL-2.0
3 * bcache setup/teardown code, and some metadata io - read a superblock and
66 unsigned int bucket_size = le16_to_cpu(s->bucket_size); in get_bucket_size()
68 if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) { in get_bucket_size()
72 max = sizeof(unsigned int) * BITS_PER_BYTE - 1; in get_bucket_size()
73 order = le16_to_cpu(s->bucket_size); in get_bucket_size()
84 le16_to_cpu(s->obso_bucket_size_hi) << 16; in get_bucket_size()
97 sb->first_bucket= le16_to_cpu(s->first_bucket); in read_super_common()
98 sb->nbuckets = le64_to_cpu(s->nbuckets); in read_super_common()
99 sb->bucket_size = get_bucket_size(sb, s); in read_super_common()
101 sb->nr_in_set = le16_to_cpu(s->nr_in_set); in read_super_common()
102 sb->nr_this_dev = le16_to_cpu(s->nr_this_dev); in read_super_common()
105 if (sb->keys > SB_JOURNAL_BUCKETS) in read_super_common()
109 if (sb->nbuckets > LONG_MAX) in read_super_common()
113 if (sb->nbuckets < 1 << 7) in read_super_common()
117 if (!is_power_of_2(sb->block_size)) in read_super_common()
121 if (sb->block_size > PAGE_SECTORS) in read_super_common()
125 if (!is_power_of_2(sb->bucket_size)) in read_super_common()
129 if (sb->bucket_size < PAGE_SECTORS) in read_super_common()
133 if (get_capacity(bdev->bd_disk) < in read_super_common()
134 sb->bucket_size * sb->nbuckets) in read_super_common()
138 if (bch_is_zero(sb->set_uuid, 16)) in read_super_common()
142 if (!sb->nr_in_set || in read_super_common()
143 sb->nr_in_set <= sb->nr_this_dev || in read_super_common()
144 sb->nr_in_set > MAX_CACHES_PER_SET) in read_super_common()
148 for (i = 0; i < sb->keys; i++) in read_super_common()
149 if (sb->d[i] != sb->first_bucket + i) in read_super_common()
153 if (sb->first_bucket + sb->keys > sb->nbuckets) in read_super_common()
157 if (sb->first_bucket * sb->bucket_size < 16) in read_super_common()
174 page = read_cache_page_gfp(bdev->bd_inode->i_mapping, in read_super()
180 sb->offset = le64_to_cpu(s->offset); in read_super()
181 sb->version = le64_to_cpu(s->version); in read_super()
183 memcpy(sb->magic, s->magic, 16); in read_super()
184 memcpy(sb->uuid, s->uuid, 16); in read_super()
185 memcpy(sb->set_uuid, s->set_uuid, 16); in read_super()
186 memcpy(sb->label, s->label, SB_LABEL_SIZE); in read_super()
188 sb->flags = le64_to_cpu(s->flags); in read_super()
189 sb->seq = le64_to_cpu(s->seq); in read_super()
190 sb->last_mount = le32_to_cpu(s->last_mount); in read_super()
191 sb->keys = le16_to_cpu(s->keys); in read_super()
194 sb->d[i] = le64_to_cpu(s->d[i]); in read_super()
197 sb->version, sb->flags, sb->seq, sb->keys); in read_super()
200 if (sb->offset != SB_SECTOR) in read_super()
204 if (memcmp(sb->magic, bcache_magic, 16)) in read_super()
208 if (s->csum != csum_set(s)) in read_super()
212 if (bch_is_zero(sb->uuid, 16)) in read_super()
215 sb->block_size = le16_to_cpu(s->block_size); in read_super()
218 if (sb->block_size << 9 < bdev_logical_block_size(bdev)) in read_super()
221 switch (sb->version) { in read_super()
223 sb->data_offset = BDEV_DATA_START_DEFAULT; in read_super()
227 sb->data_offset = le64_to_cpu(s->data_offset); in read_super()
230 if (sb->data_offset < BDEV_DATA_START_DEFAULT) in read_super()
245 sb->feature_compat = le64_to_cpu(s->feature_compat); in read_super()
246 sb->feature_incompat = le64_to_cpu(s->feature_incompat); in read_super()
247 sb->feature_ro_compat = le64_to_cpu(s->feature_ro_compat); in read_super()
254 err = "Unsupported read-only compatible feature found"; in read_super()
271 sb->last_mount = (u32)ktime_get_real_seconds(); in read_super()
281 struct cached_dev *dc = bio->bi_private; in write_bdev_super_endio() local
283 if (bio->bi_status) in write_bdev_super_endio()
284 bch_count_backing_io_errors(dc, bio); in write_bdev_super_endio()
286 closure_put(&dc->sb_write); in write_bdev_super_endio()
294 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META; in __write_super()
295 bio->bi_iter.bi_sector = SB_SECTOR; in __write_super()
299 out->offset = cpu_to_le64(sb->offset); in __write_super()
301 memcpy(out->uuid, sb->uuid, 16); in __write_super()
302 memcpy(out->set_uuid, sb->set_uuid, 16); in __write_super()
303 memcpy(out->label, sb->label, SB_LABEL_SIZE); in __write_super()
305 out->flags = cpu_to_le64(sb->flags); in __write_super()
306 out->seq = cpu_to_le64(sb->seq); in __write_super()
308 out->last_mount = cpu_to_le32(sb->last_mount); in __write_super()
309 out->first_bucket = cpu_to_le16(sb->first_bucket); in __write_super()
310 out->keys = cpu_to_le16(sb->keys); in __write_super()
312 for (i = 0; i < sb->keys; i++) in __write_super()
313 out->d[i] = cpu_to_le64(sb->d[i]); in __write_super()
315 if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) { in __write_super()
316 out->feature_compat = cpu_to_le64(sb->feature_compat); in __write_super()
317 out->feature_incompat = cpu_to_le64(sb->feature_incompat); in __write_super()
318 out->feature_ro_compat = cpu_to_le64(sb->feature_ro_compat); in __write_super()
321 out->version = cpu_to_le64(sb->version); in __write_super()
322 out->csum = csum_set(out); in __write_super()
325 sb->version, sb->flags, sb->seq); in __write_super()
332 struct cached_dev *dc = container_of(cl, struct cached_dev, sb_write); in bch_write_bdev_super_unlock() local
334 up(&dc->sb_write_mutex); in bch_write_bdev_super_unlock()
337 void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent) in bch_write_bdev_super() argument
339 struct closure *cl = &dc->sb_write; in bch_write_bdev_super()
340 struct bio *bio = &dc->sb_bio; in bch_write_bdev_super()
342 down(&dc->sb_write_mutex); in bch_write_bdev_super()
345 bio_init(bio, dc->bdev, dc->sb_bv, 1, 0); in bch_write_bdev_super()
346 bio->bi_end_io = write_bdev_super_endio; in bch_write_bdev_super()
347 bio->bi_private = dc; in bch_write_bdev_super()
351 __write_super(&dc->sb, dc->sb_disk, bio); in bch_write_bdev_super()
358 struct cache *ca = bio->bi_private; in write_super_endio()
361 bch_count_io_errors(ca, bio->bi_status, 0, in write_super_endio()
363 closure_put(&ca->set->sb_write); in write_super_endio()
370 up(&c->sb_write_mutex); in bcache_write_super_unlock()
375 struct closure *cl = &c->sb_write; in bcache_write_super()
376 struct cache *ca = c->cache; in bcache_write_super()
377 struct bio *bio = &ca->sb_bio; in bcache_write_super()
380 down(&c->sb_write_mutex); in bcache_write_super()
381 closure_init(cl, &c->cl); in bcache_write_super()
383 ca->sb.seq++; in bcache_write_super()
385 if (ca->sb.version < version) in bcache_write_super()
386 ca->sb.version = version; in bcache_write_super()
388 bio_init(bio, ca->bdev, ca->sb_bv, 1, 0); in bcache_write_super()
389 bio->bi_end_io = write_super_endio; in bcache_write_super()
390 bio->bi_private = ca; in bcache_write_super()
393 __write_super(&ca->sb, ca->sb_disk, bio); in bcache_write_super()
402 struct closure *cl = bio->bi_private; in uuid_endio()
405 cache_set_err_on(bio->bi_status, c, "accessing uuids"); in uuid_endio()
414 up(&c->uuid_write_mutex); in uuid_io_unlock()
420 struct closure *cl = &c->uuid_write; in uuid_io()
426 down(&c->uuid_write_mutex); in uuid_io()
432 bio->bi_opf = opf | REQ_SYNC | REQ_META; in uuid_io()
433 bio->bi_iter.bi_size = KEY_SIZE(k) << 9; in uuid_io()
435 bio->bi_end_io = uuid_endio; in uuid_io()
436 bio->bi_private = cl; in uuid_io()
437 bch_bio_map(bio, c->uuids); in uuid_io()
449 for (u = c->uuids; u < c->uuids + c->nr_uuids; u++) in uuid_io()
450 if (!bch_is_zero(u->uuid, 16)) in uuid_io()
452 u - c->uuids, u->uuid, u->label, in uuid_io()
453 u->first_reg, u->last_reg, u->invalidated); in uuid_io()
460 struct bkey *k = &j->uuid_bucket; in uuid_read()
465 bkey_copy(&c->uuid_bucket, k); in uuid_read()
468 if (j->version < BCACHE_JSET_VERSION_UUIDv1) { in uuid_read()
469 struct uuid_entry_v0 *u0 = (void *) c->uuids; in uuid_read()
470 struct uuid_entry *u1 = (void *) c->uuids; in uuid_read()
481 for (i = c->nr_uuids - 1; in uuid_read()
483 --i) { in uuid_read()
503 struct cache *ca = c->cache; in __uuid_write()
512 size = meta_bucket_pages(&ca->sb) * PAGE_SECTORS; in __uuid_write()
518 atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written); in __uuid_write()
520 bkey_copy(&c->uuid_bucket, &k.key); in __uuid_write()
539 for (u = c->uuids; in uuid_find()
540 u < c->uuids + c->nr_uuids; u++) in uuid_find()
541 if (!memcmp(u->uuid, uuid, 16)) in uuid_find()
566 * it's actually the gens that must be written out at specific times - it's no
583 struct cache *ca = bio->bi_private; in prio_endio()
585 cache_set_err_on(bio->bi_status, ca->set, "accessing priorities"); in prio_endio()
586 bch_bbio_free(bio, ca->set); in prio_endio()
587 closure_put(&ca->prio); in prio_endio()
592 struct closure *cl = &ca->prio; in prio_io()
593 struct bio *bio = bch_bbio_alloc(ca->set); in prio_io()
597 bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size; in prio_io()
598 bio_set_dev(bio, ca->bdev); in prio_io()
599 bio->bi_iter.bi_size = meta_bucket_bytes(&ca->sb); in prio_io()
601 bio->bi_end_io = prio_endio; in prio_io()
602 bio->bi_private = ca; in prio_io()
603 bio->bi_opf = opf | REQ_SYNC | REQ_META; in prio_io()
604 bch_bio_map(bio, ca->disk_buckets); in prio_io()
606 closure_bio_submit(ca->set, bio, &ca->prio); in prio_io()
617 fifo_used(&ca->free[RESERVE_PRIO]), in bch_prio_write()
618 fifo_used(&ca->free[RESERVE_NONE]), in bch_prio_write()
619 fifo_used(&ca->free_inc)); in bch_prio_write()
622 * Pre-check if there are enough free buckets. In the non-blocking in bch_prio_write()
627 size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) + in bch_prio_write()
628 fifo_used(&ca->free[RESERVE_NONE]); in bch_prio_write()
630 return -ENOMEM; in bch_prio_write()
635 lockdep_assert_held(&ca->set->bucket_lock); in bch_prio_write()
637 ca->disk_buckets->seq++; in bch_prio_write()
639 atomic_long_add(ca->sb.bucket_size * prio_buckets(ca), in bch_prio_write()
640 &ca->meta_sectors_written); in bch_prio_write()
642 for (i = prio_buckets(ca) - 1; i >= 0; --i) { in bch_prio_write()
644 struct prio_set *p = ca->disk_buckets; in bch_prio_write()
645 struct bucket_disk *d = p->data; in bch_prio_write()
648 for (b = ca->buckets + i * prios_per_bucket(ca); in bch_prio_write()
649 b < ca->buckets + ca->sb.nbuckets && d < end; in bch_prio_write()
651 d->prio = cpu_to_le16(b->prio); in bch_prio_write()
652 d->gen = b->gen; in bch_prio_write()
655 p->next_bucket = ca->prio_buckets[i + 1]; in bch_prio_write()
656 p->magic = pset_magic(&ca->sb); in bch_prio_write()
657 p->csum = bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8); in bch_prio_write()
660 BUG_ON(bucket == -1); in bch_prio_write()
662 mutex_unlock(&ca->set->bucket_lock); in bch_prio_write()
664 mutex_lock(&ca->set->bucket_lock); in bch_prio_write()
666 ca->prio_buckets[i] = bucket; in bch_prio_write()
667 atomic_dec_bug(&ca->buckets[bucket].pin); in bch_prio_write()
670 mutex_unlock(&ca->set->bucket_lock); in bch_prio_write()
672 bch_journal_meta(ca->set, &cl); in bch_prio_write()
675 mutex_lock(&ca->set->bucket_lock); in bch_prio_write()
682 if (ca->prio_last_buckets[i]) in bch_prio_write()
684 &ca->buckets[ca->prio_last_buckets[i]]); in bch_prio_write()
686 ca->prio_last_buckets[i] = ca->prio_buckets[i]; in bch_prio_write()
693 struct prio_set *p = ca->disk_buckets; in prio_read()
694 struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d; in prio_read()
697 int ret = -EIO; in prio_read()
699 for (b = ca->buckets; in prio_read()
700 b < ca->buckets + ca->sb.nbuckets; in prio_read()
703 ca->prio_buckets[bucket_nr] = bucket; in prio_read()
704 ca->prio_last_buckets[bucket_nr] = bucket; in prio_read()
709 if (p->csum != in prio_read()
710 bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) { in prio_read()
715 if (p->magic != pset_magic(&ca->sb)) { in prio_read()
720 bucket = p->next_bucket; in prio_read()
721 d = p->data; in prio_read()
724 b->prio = le16_to_cpu(d->prio); in prio_read()
725 b->gen = b->last_gc = d->gen; in prio_read()
737 struct bcache_device *d = b->bd_disk->private_data; in open_dev()
739 if (test_bit(BCACHE_DEV_CLOSING, &d->flags)) in open_dev()
740 return -ENXIO; in open_dev()
742 closure_get(&d->cl); in open_dev()
748 struct bcache_device *d = b->private_data; in release_dev()
750 closure_put(&d->cl); in release_dev()
756 struct bcache_device *d = b->bd_disk->private_data; in ioctl_dev()
758 return d->ioctl(d, mode, cmd, arg); in ioctl_dev()
779 if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags)) in bcache_device_stop()
782 * - cached device: cached_dev_flush() in bcache_device_stop()
783 * - flash dev: flash_dev_flush() in bcache_device_stop()
785 closure_queue(&d->cl); in bcache_device_stop()
792 if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) { in bcache_device_unlink()
793 struct cache *ca = d->c->cache; in bcache_device_unlink()
795 sysfs_remove_link(&d->c->kobj, d->name); in bcache_device_unlink()
796 sysfs_remove_link(&d->kobj, "cache"); in bcache_device_unlink()
798 bd_unlink_disk_holder(ca->bdev, d->disk); in bcache_device_unlink()
805 struct cache *ca = c->cache; in bcache_device_link()
808 bd_link_disk_holder(ca->bdev, d->disk); in bcache_device_link()
810 snprintf(d->name, BCACHEDEVNAME_SIZE, in bcache_device_link()
811 "%s%u", name, d->id); in bcache_device_link()
813 ret = sysfs_create_link(&d->kobj, &c->kobj, "cache"); in bcache_device_link()
815 pr_err("Couldn't create device -> cache set symlink\n"); in bcache_device_link()
817 ret = sysfs_create_link(&c->kobj, &d->kobj, d->name); in bcache_device_link()
819 pr_err("Couldn't create cache set -> device symlink\n"); in bcache_device_link()
821 clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags); in bcache_device_link()
828 atomic_dec(&d->c->attached_dev_nr); in bcache_device_detach()
830 if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) { in bcache_device_detach()
831 struct uuid_entry *u = d->c->uuids + d->id; in bcache_device_detach()
834 memcpy(u->uuid, invalid_uuid, 16); in bcache_device_detach()
835 u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds()); in bcache_device_detach()
836 bch_uuid_write(d->c); in bcache_device_detach()
841 d->c->devices[d->id] = NULL; in bcache_device_detach()
842 closure_put(&d->c->caching); in bcache_device_detach()
843 d->c = NULL; in bcache_device_detach()
849 d->id = id; in bcache_device_attach()
850 d->c = c; in bcache_device_attach()
851 c->devices[id] = d; in bcache_device_attach()
853 if (id >= c->devices_max_used) in bcache_device_attach()
854 c->devices_max_used = id + 1; in bcache_device_attach()
856 closure_get(&c->caching); in bcache_device_attach()
871 struct gendisk *disk = d->disk; in bcache_device_free()
876 pr_info("%s stopped\n", disk->disk_name); in bcache_device_free()
880 if (d->c) in bcache_device_free()
885 first_minor_to_idx(disk->first_minor)); in bcache_device_free()
889 bioset_exit(&d->bio_split); in bcache_device_free()
890 kvfree(d->full_dirty_stripes); in bcache_device_free()
891 kvfree(d->stripe_sectors_dirty); in bcache_device_free()
893 closure_debug_destroy(&d->cl); in bcache_device_free()
906 if (!d->stripe_size) in bcache_device_init()
907 d->stripe_size = 1 << 31; in bcache_device_init()
909 n = DIV_ROUND_UP_ULL(sectors, d->stripe_size); in bcache_device_init()
913 return -ENOMEM; in bcache_device_init()
915 d->nr_stripes = n; in bcache_device_init()
917 n = d->nr_stripes * sizeof(atomic_t); in bcache_device_init()
918 d->stripe_sectors_dirty = kvzalloc(n, GFP_KERNEL); in bcache_device_init()
919 if (!d->stripe_sectors_dirty) in bcache_device_init()
920 return -ENOMEM; in bcache_device_init()
922 n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long); in bcache_device_init()
923 d->full_dirty_stripes = kvzalloc(n, GFP_KERNEL); in bcache_device_init()
924 if (!d->full_dirty_stripes) in bcache_device_init()
932 if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio), in bcache_device_init()
936 d->disk = blk_alloc_disk(NUMA_NO_NODE); in bcache_device_init()
937 if (!d->disk) in bcache_device_init()
940 set_capacity(d->disk, sectors); in bcache_device_init()
941 snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx); in bcache_device_init()
943 d->disk->major = bcache_major; in bcache_device_init()
944 d->disk->first_minor = idx_to_first_minor(idx); in bcache_device_init()
945 d->disk->minors = BCACHE_MINORS; in bcache_device_init()
946 d->disk->fops = ops; in bcache_device_init()
947 d->disk->private_data = d; in bcache_device_init()
949 q = d->disk->queue; in bcache_device_init()
950 q->limits.max_hw_sectors = UINT_MAX; in bcache_device_init()
951 q->limits.max_sectors = UINT_MAX; in bcache_device_init()
952 q->limits.max_segment_size = UINT_MAX; in bcache_device_init()
953 q->limits.max_segments = BIO_MAX_VECS; in bcache_device_init()
955 q->limits.discard_granularity = 512; in bcache_device_init()
956 q->limits.io_min = block_size; in bcache_device_init()
957 q->limits.logical_block_size = block_size; in bcache_device_init()
958 q->limits.physical_block_size = block_size; in bcache_device_init()
960 if (q->limits.logical_block_size > PAGE_SIZE && cached_bdev) { in bcache_device_init()
966 d->disk->disk_name, q->limits.logical_block_size, in bcache_device_init()
973 blk_queue_flag_set(QUEUE_FLAG_NONROT, d->disk->queue); in bcache_device_init()
974 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, d->disk->queue); in bcache_device_init()
981 bioset_exit(&d->bio_split); in bcache_device_init()
985 kvfree(d->full_dirty_stripes); in bcache_device_init()
987 kvfree(d->stripe_sectors_dirty); in bcache_device_init()
988 return -ENOMEM; in bcache_device_init()
997 struct cached_dev *dc; in calc_cached_dev_sectors() local
999 list_for_each_entry(dc, &c->cached_devs, list) in calc_cached_dev_sectors()
1000 sectors += bdev_nr_sectors(dc->bdev); in calc_cached_dev_sectors()
1002 c->cached_dev_sectors = sectors; in calc_cached_dev_sectors()
1008 struct cached_dev *dc = arg; in cached_dev_status_update() local
1013 * dc->io_disable might be set via sysfs interface, so check it in cached_dev_status_update()
1016 while (!kthread_should_stop() && !dc->io_disable) { in cached_dev_status_update()
1017 q = bdev_get_queue(dc->bdev); in cached_dev_status_update()
1019 dc->offline_seconds++; in cached_dev_status_update()
1021 dc->offline_seconds = 0; in cached_dev_status_update()
1023 if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) { in cached_dev_status_update()
1025 dc->bdev, in cached_dev_status_update()
1028 dc->disk.name); in cached_dev_status_update()
1029 dc->io_disable = true; in cached_dev_status_update()
1032 bcache_device_stop(&dc->disk); in cached_dev_status_update()
1043 int bch_cached_dev_run(struct cached_dev *dc) in bch_cached_dev_run() argument
1046 struct bcache_device *d = &dc->disk; in bch_cached_dev_run()
1047 char *buf = kmemdup_nul(dc->sb.label, SB_LABEL_SIZE, GFP_KERNEL); in bch_cached_dev_run()
1050 kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid), in bch_cached_dev_run()
1055 if (dc->io_disable) { in bch_cached_dev_run()
1056 pr_err("I/O disabled on cached dev %pg\n", dc->bdev); in bch_cached_dev_run()
1057 ret = -EIO; in bch_cached_dev_run()
1061 if (atomic_xchg(&dc->running, 1)) { in bch_cached_dev_run()
1062 pr_info("cached dev %pg is running already\n", dc->bdev); in bch_cached_dev_run()
1063 ret = -EBUSY; in bch_cached_dev_run()
1067 if (!d->c && in bch_cached_dev_run()
1068 BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) { in bch_cached_dev_run()
1073 SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE); in bch_cached_dev_run()
1074 bch_write_bdev_super(dc, &cl); in bch_cached_dev_run()
1078 ret = add_disk(d->disk); in bch_cached_dev_run()
1081 bd_link_disk_holder(dc->bdev, dc->disk.disk); in bch_cached_dev_run()
1083 * won't show up in the uevent file, use udevadm monitor -e instead in bch_cached_dev_run()
1086 kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env); in bch_cached_dev_run()
1088 if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") || in bch_cached_dev_run()
1089 sysfs_create_link(&disk_to_dev(d->disk)->kobj, in bch_cached_dev_run()
1090 &d->kobj, "bcache")) { in bch_cached_dev_run()
1091 pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n"); in bch_cached_dev_run()
1092 ret = -ENOMEM; in bch_cached_dev_run()
1096 dc->status_update_thread = kthread_run(cached_dev_status_update, in bch_cached_dev_run()
1097 dc, "bcache_status_update"); in bch_cached_dev_run()
1098 if (IS_ERR(dc->status_update_thread)) { in bch_cached_dev_run()
1111 * work dc->writeback_rate_update is running. Wait until the routine
1116 static void cancel_writeback_rate_update_dwork(struct cached_dev *dc) in cancel_writeback_rate_update_dwork() argument
1122 &dc->disk.flags)) in cancel_writeback_rate_update_dwork()
1124 time_out--; in cancel_writeback_rate_update_dwork()
1129 pr_warn("give up waiting for dc->writeback_write_update to quit\n"); in cancel_writeback_rate_update_dwork()
1131 cancel_delayed_work_sync(&dc->writeback_rate_update); in cancel_writeback_rate_update_dwork()
1136 struct cached_dev *dc = container_of(w, struct cached_dev, detach); in cached_dev_detach_finish() local
1137 struct cache_set *c = dc->disk.c; in cached_dev_detach_finish()
1139 BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)); in cached_dev_detach_finish()
1140 BUG_ON(refcount_read(&dc->count)); in cached_dev_detach_finish()
1143 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags)) in cached_dev_detach_finish()
1144 cancel_writeback_rate_update_dwork(dc); in cached_dev_detach_finish()
1146 if (!IS_ERR_OR_NULL(dc->writeback_thread)) { in cached_dev_detach_finish()
1147 kthread_stop(dc->writeback_thread); in cached_dev_detach_finish()
1148 dc->writeback_thread = NULL; in cached_dev_detach_finish()
1153 bcache_device_detach(&dc->disk); in cached_dev_detach_finish()
1154 list_move(&dc->list, &uncached_devices); in cached_dev_detach_finish()
1157 clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags); in cached_dev_detach_finish()
1158 clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags); in cached_dev_detach_finish()
1162 pr_info("Caching disabled for %pg\n", dc->bdev); in cached_dev_detach_finish()
1165 closure_put(&dc->disk.cl); in cached_dev_detach_finish()
1168 void bch_cached_dev_detach(struct cached_dev *dc) in bch_cached_dev_detach() argument
1172 if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags)) in bch_cached_dev_detach()
1175 if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) in bch_cached_dev_detach()
1182 closure_get(&dc->disk.cl); in bch_cached_dev_detach()
1184 bch_writeback_queue(dc); in bch_cached_dev_detach()
1186 cached_dev_put(dc); in bch_cached_dev_detach()
1189 int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, in bch_cached_dev_attach() argument
1197 if ((set_uuid && memcmp(set_uuid, c->set_uuid, 16)) || in bch_cached_dev_attach()
1198 (!set_uuid && memcmp(dc->sb.set_uuid, c->set_uuid, 16))) in bch_cached_dev_attach()
1199 return -ENOENT; in bch_cached_dev_attach()
1201 if (dc->disk.c) { in bch_cached_dev_attach()
1202 pr_err("Can't attach %pg: already attached\n", dc->bdev); in bch_cached_dev_attach()
1203 return -EINVAL; in bch_cached_dev_attach()
1206 if (test_bit(CACHE_SET_STOPPING, &c->flags)) { in bch_cached_dev_attach()
1207 pr_err("Can't attach %pg: shutting down\n", dc->bdev); in bch_cached_dev_attach()
1208 return -EINVAL; in bch_cached_dev_attach()
1211 if (dc->sb.block_size < c->cache->sb.block_size) { in bch_cached_dev_attach()
1214 dc->bdev); in bch_cached_dev_attach()
1215 return -EINVAL; in bch_cached_dev_attach()
1219 list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) { in bch_cached_dev_attach()
1220 if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) { in bch_cached_dev_attach()
1222 dc->bdev); in bch_cached_dev_attach()
1224 return -EINVAL; in bch_cached_dev_attach()
1228 u = uuid_find(c, dc->sb.uuid); in bch_cached_dev_attach()
1231 (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE || in bch_cached_dev_attach()
1232 BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) { in bch_cached_dev_attach()
1233 memcpy(u->uuid, invalid_uuid, 16); in bch_cached_dev_attach()
1234 u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds()); in bch_cached_dev_attach()
1239 if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { in bch_cached_dev_attach()
1240 pr_err("Couldn't find uuid for %pg in set\n", dc->bdev); in bch_cached_dev_attach()
1241 return -ENOENT; in bch_cached_dev_attach()
1246 pr_err("Not caching %pg, no room for UUID\n", dc->bdev); in bch_cached_dev_attach()
1247 return -EINVAL; in bch_cached_dev_attach()
1253 * sysfs_remove_file(&dc->kobj, &sysfs_attach); in bch_cached_dev_attach()
1256 if (bch_is_zero(u->uuid, 16)) { in bch_cached_dev_attach()
1261 memcpy(u->uuid, dc->sb.uuid, 16); in bch_cached_dev_attach()
1262 memcpy(u->label, dc->sb.label, SB_LABEL_SIZE); in bch_cached_dev_attach()
1263 u->first_reg = u->last_reg = rtime; in bch_cached_dev_attach()
1266 memcpy(dc->sb.set_uuid, c->set_uuid, 16); in bch_cached_dev_attach()
1267 SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN); in bch_cached_dev_attach()
1269 bch_write_bdev_super(dc, &cl); in bch_cached_dev_attach()
1272 u->last_reg = rtime; in bch_cached_dev_attach()
1276 bcache_device_attach(&dc->disk, c, u - c->uuids); in bch_cached_dev_attach()
1277 list_move(&dc->list, &c->cached_devs); in bch_cached_dev_attach()
1281 * dc->c must be set before dc->count != 0 - paired with the mb in in bch_cached_dev_attach()
1285 refcount_set(&dc->count, 1); in bch_cached_dev_attach()
1288 down_write(&dc->writeback_lock); in bch_cached_dev_attach()
1289 if (bch_cached_dev_writeback_start(dc)) { in bch_cached_dev_attach()
1290 up_write(&dc->writeback_lock); in bch_cached_dev_attach()
1292 dc->disk.disk->disk_name); in bch_cached_dev_attach()
1293 return -ENOMEM; in bch_cached_dev_attach()
1296 if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { in bch_cached_dev_attach()
1297 atomic_set(&dc->has_dirty, 1); in bch_cached_dev_attach()
1298 bch_writeback_queue(dc); in bch_cached_dev_attach()
1301 bch_sectors_dirty_init(&dc->disk); in bch_cached_dev_attach()
1303 ret = bch_cached_dev_run(dc); in bch_cached_dev_attach()
1304 if (ret && (ret != -EBUSY)) { in bch_cached_dev_attach()
1305 up_write(&dc->writeback_lock); in bch_cached_dev_attach()
1312 kthread_stop(dc->writeback_thread); in bch_cached_dev_attach()
1313 cancel_writeback_rate_update_dwork(dc); in bch_cached_dev_attach()
1314 pr_err("Couldn't run cached device %pg\n", dc->bdev); in bch_cached_dev_attach()
1318 bcache_device_link(&dc->disk, c, "bdev"); in bch_cached_dev_attach()
1319 atomic_inc(&c->attached_dev_nr); in bch_cached_dev_attach()
1321 if (bch_has_feature_obso_large_bucket(&(c->cache->sb))) { in bch_cached_dev_attach()
1322 …pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n"); in bch_cached_dev_attach()
1323 pr_err("Please update to the latest bcache-tools to create the cache device\n"); in bch_cached_dev_attach()
1324 set_disk_ro(dc->disk.disk, 1); in bch_cached_dev_attach()
1328 up_write(&dc->writeback_lock); in bch_cached_dev_attach()
1331 dc->bdev, in bch_cached_dev_attach()
1332 dc->disk.disk->disk_name, in bch_cached_dev_attach()
1333 dc->disk.c->set_uuid); in bch_cached_dev_attach()
1337 /* when dc->disk.kobj released */
1340 struct cached_dev *dc = container_of(kobj, struct cached_dev, in bch_cached_dev_release() local
1342 kfree(dc); in bch_cached_dev_release()
1348 struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl); in cached_dev_free() local
1350 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags)) in cached_dev_free()
1351 cancel_writeback_rate_update_dwork(dc); in cached_dev_free()
1353 if (!IS_ERR_OR_NULL(dc->writeback_thread)) in cached_dev_free()
1354 kthread_stop(dc->writeback_thread); in cached_dev_free()
1355 if (!IS_ERR_OR_NULL(dc->status_update_thread)) in cached_dev_free()
1356 kthread_stop(dc->status_update_thread); in cached_dev_free()
1360 if (atomic_read(&dc->running)) { in cached_dev_free()
1361 bd_unlink_disk_holder(dc->bdev, dc->disk.disk); in cached_dev_free()
1362 del_gendisk(dc->disk.disk); in cached_dev_free()
1364 bcache_device_free(&dc->disk); in cached_dev_free()
1365 list_del(&dc->list); in cached_dev_free()
1369 if (dc->sb_disk) in cached_dev_free()
1370 put_page(virt_to_page(dc->sb_disk)); in cached_dev_free()
1372 if (!IS_ERR_OR_NULL(dc->bdev)) in cached_dev_free()
1373 blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); in cached_dev_free()
1377 kobject_put(&dc->disk.kobj); in cached_dev_free()
1382 struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl); in cached_dev_flush() local
1383 struct bcache_device *d = &dc->disk; in cached_dev_flush()
1389 bch_cache_accounting_destroy(&dc->accounting); in cached_dev_flush()
1390 kobject_del(&d->kobj); in cached_dev_flush()
1395 static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) in cached_dev_init() argument
1399 struct request_queue *q = bdev_get_queue(dc->bdev); in cached_dev_init()
1402 INIT_LIST_HEAD(&dc->list); in cached_dev_init()
1403 closure_init(&dc->disk.cl, NULL); in cached_dev_init()
1404 set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq); in cached_dev_init()
1405 kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype); in cached_dev_init()
1406 INIT_WORK(&dc->detach, cached_dev_detach_finish); in cached_dev_init()
1407 sema_init(&dc->sb_write_mutex, 1); in cached_dev_init()
1408 INIT_LIST_HEAD(&dc->io_lru); in cached_dev_init()
1409 spin_lock_init(&dc->io_lock); in cached_dev_init()
1410 bch_cache_accounting_init(&dc->accounting, &dc->disk.cl); in cached_dev_init()
1412 dc->sequential_cutoff = 4 << 20; in cached_dev_init()
1414 for (io = dc->io; io < dc->io + RECENT_IO; io++) { in cached_dev_init()
1415 list_add(&io->lru, &dc->io_lru); in cached_dev_init()
1416 hlist_add_head(&io->hash, dc->io_hash + RECENT_IO); in cached_dev_init()
1419 dc->disk.stripe_size = q->limits.io_opt >> 9; in cached_dev_init()
1421 if (dc->disk.stripe_size) in cached_dev_init()
1422 dc->partial_stripes_expensive = in cached_dev_init()
1423 q->limits.raid_partial_stripes_expensive; in cached_dev_init()
1425 ret = bcache_device_init(&dc->disk, block_size, in cached_dev_init()
1426 bdev_nr_sectors(dc->bdev) - dc->sb.data_offset, in cached_dev_init()
1427 dc->bdev, &bcache_cached_ops); in cached_dev_init()
1431 blk_queue_io_opt(dc->disk.disk->queue, in cached_dev_init()
1432 max(queue_io_opt(dc->disk.disk->queue), queue_io_opt(q))); in cached_dev_init()
1434 atomic_set(&dc->io_errors, 0); in cached_dev_init()
1435 dc->io_disable = false; in cached_dev_init()
1436 dc->error_limit = DEFAULT_CACHED_DEV_ERROR_LIMIT; in cached_dev_init()
1438 dc->stop_when_cache_set_failed = BCH_CACHED_DEV_STOP_AUTO; in cached_dev_init()
1440 bch_cached_dev_request_init(dc); in cached_dev_init()
1441 bch_cached_dev_writeback_init(dc); in cached_dev_init()
1445 /* Cached device - bcache superblock */
1449 struct cached_dev *dc) in register_bdev() argument
1453 int ret = -ENOMEM; in register_bdev()
1455 memcpy(&dc->sb, sb, sizeof(struct cache_sb)); in register_bdev()
1456 dc->bdev = bdev; in register_bdev()
1457 dc->bdev->bd_holder = dc; in register_bdev()
1458 dc->sb_disk = sb_disk; in register_bdev()
1460 if (cached_dev_init(dc, sb->block_size << 9)) in register_bdev()
1464 if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache")) in register_bdev()
1466 if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj)) in register_bdev()
1469 pr_info("registered backing device %pg\n", dc->bdev); in register_bdev()
1471 list_add(&dc->list, &uncached_devices); in register_bdev()
1474 bch_cached_dev_attach(dc, c, NULL); in register_bdev()
1476 if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE || in register_bdev()
1477 BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) { in register_bdev()
1479 ret = bch_cached_dev_run(dc); in register_bdev()
1486 pr_notice("error %pg: %s\n", dc->bdev, err); in register_bdev()
1487 bcache_device_stop(&dc->disk); in register_bdev()
1493 /* When d->kobj released */
1507 &d->c->flash_dev_dirty_sectors); in flash_dev_free()
1508 del_gendisk(d->disk); in flash_dev_free()
1511 kobject_put(&d->kobj); in flash_dev_free()
1521 kobject_del(&d->kobj); in flash_dev_flush()
1527 int err = -ENOMEM; in flash_dev_run()
1533 closure_init(&d->cl, NULL); in flash_dev_run()
1534 set_closure_fn(&d->cl, flash_dev_flush, system_wq); in flash_dev_run()
1536 kobject_init(&d->kobj, &bch_flash_dev_ktype); in flash_dev_run()
1538 if (bcache_device_init(d, block_bytes(c->cache), u->sectors, in flash_dev_run()
1542 bcache_device_attach(d, c, u - c->uuids); in flash_dev_run()
1545 err = add_disk(d->disk); in flash_dev_run()
1549 err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache"); in flash_dev_run()
1555 if (bch_has_feature_obso_large_bucket(&c->cache->sb)) { in flash_dev_run()
1556 …pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n"); in flash_dev_run()
1557 pr_err("Please update to the latest bcache-tools to create the cache device\n"); in flash_dev_run()
1558 set_disk_ro(d->disk, 1); in flash_dev_run()
1563 kobject_put(&d->kobj); in flash_dev_run()
1573 for (u = c->uuids; in flash_devs_run()
1574 u < c->uuids + c->nr_uuids && !ret; in flash_devs_run()
1586 if (test_bit(CACHE_SET_STOPPING, &c->flags)) in bch_flash_dev_create()
1587 return -EINTR; in bch_flash_dev_create()
1589 if (!test_bit(CACHE_SET_RUNNING, &c->flags)) in bch_flash_dev_create()
1590 return -EPERM; in bch_flash_dev_create()
1595 return -EINVAL; in bch_flash_dev_create()
1598 get_random_bytes(u->uuid, 16); in bch_flash_dev_create()
1599 memset(u->label, 0, 32); in bch_flash_dev_create()
1600 u->first_reg = u->last_reg = cpu_to_le32((u32)ktime_get_real_seconds()); in bch_flash_dev_create()
1603 u->sectors = size >> 9; in bch_flash_dev_create()
1610 bool bch_cached_dev_error(struct cached_dev *dc) in bch_cached_dev_error() argument
1612 if (!dc || test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags)) in bch_cached_dev_error()
1615 dc->io_disable = true; in bch_cached_dev_error()
1620 dc->disk.disk->disk_name, dc->bdev); in bch_cached_dev_error()
1622 bcache_device_stop(&dc->disk); in bch_cached_dev_error()
1634 if (c->on_error != ON_ERROR_PANIC && in bch_cache_set_error()
1635 test_bit(CACHE_SET_STOPPING, &c->flags)) in bch_cache_set_error()
1638 if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags)) in bch_cache_set_error()
1652 c->set_uuid, &vaf); in bch_cache_set_error()
1656 if (c->on_error == ON_ERROR_PANIC) in bch_cache_set_error()
1663 /* When c->kobj released */
1677 debugfs_remove(c->debug); in cache_set_free()
1684 bch_bset_sort_state_free(&c->sort); in cache_set_free()
1685 free_pages((unsigned long) c->uuids, ilog2(meta_bucket_pages(&c->cache->sb))); in cache_set_free()
1687 ca = c->cache; in cache_set_free()
1689 ca->set = NULL; in cache_set_free()
1690 c->cache = NULL; in cache_set_free()
1691 kobject_put(&ca->kobj); in cache_set_free()
1695 if (c->moving_gc_wq) in cache_set_free()
1696 destroy_workqueue(c->moving_gc_wq); in cache_set_free()
1697 bioset_exit(&c->bio_split); in cache_set_free()
1698 mempool_exit(&c->fill_iter); in cache_set_free()
1699 mempool_exit(&c->bio_meta); in cache_set_free()
1700 mempool_exit(&c->search); in cache_set_free()
1701 kfree(c->devices); in cache_set_free()
1703 list_del(&c->list); in cache_set_free()
1706 pr_info("Cache set %pU unregistered\n", c->set_uuid); in cache_set_free()
1709 closure_debug_destroy(&c->cl); in cache_set_free()
1710 kobject_put(&c->kobj); in cache_set_free()
1716 struct cache *ca = c->cache; in cache_set_flush()
1719 bch_cache_accounting_destroy(&c->accounting); in cache_set_flush()
1721 kobject_put(&c->internal); in cache_set_flush()
1722 kobject_del(&c->kobj); in cache_set_flush()
1724 if (!IS_ERR_OR_NULL(c->gc_thread)) in cache_set_flush()
1725 kthread_stop(c->gc_thread); in cache_set_flush()
1727 if (!IS_ERR_OR_NULL(c->root)) in cache_set_flush()
1728 list_add(&c->root->list, &c->btree_cache); in cache_set_flush()
1734 if (!test_bit(CACHE_SET_IO_DISABLE, &c->flags)) in cache_set_flush()
1735 list_for_each_entry(b, &c->btree_cache, list) { in cache_set_flush()
1736 mutex_lock(&b->write_lock); in cache_set_flush()
1739 mutex_unlock(&b->write_lock); in cache_set_flush()
1742 if (ca->alloc_thread) in cache_set_flush()
1743 kthread_stop(ca->alloc_thread); in cache_set_flush()
1745 if (c->journal.cur) { in cache_set_flush()
1746 cancel_delayed_work_sync(&c->journal.work); in cache_set_flush()
1748 c->journal.work.work.func(&c->journal.work.work); in cache_set_flush()
1760 * dc->stop_when_cache_set_failed dc->has_dirty stop bcache device
1772 struct cached_dev *dc) in conditional_stop_bcache_device() argument
1774 if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) { in conditional_stop_bcache_device()
1776 d->disk->disk_name, c->set_uuid); in conditional_stop_bcache_device()
1778 } else if (atomic_read(&dc->has_dirty)) { in conditional_stop_bcache_device()
1780 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO in conditional_stop_bcache_device()
1781 * and dc->has_dirty == 1 in conditional_stop_bcache_device()
1784 d->disk->disk_name); in conditional_stop_bcache_device()
1793 * to a broken cache device, dc->io_disable should be in conditional_stop_bcache_device()
1796 dc->io_disable = true; in conditional_stop_bcache_device()
1802 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO in conditional_stop_bcache_device()
1803 * and dc->has_dirty == 0 in conditional_stop_bcache_device()
1806 d->disk->disk_name); in conditional_stop_bcache_device()
1813 struct cached_dev *dc; in __cache_set_unregister() local
1819 for (i = 0; i < c->devices_max_used; i++) { in __cache_set_unregister()
1820 d = c->devices[i]; in __cache_set_unregister()
1824 if (!UUID_FLASH_ONLY(&c->uuids[i]) && in __cache_set_unregister()
1825 test_bit(CACHE_SET_UNREGISTERING, &c->flags)) { in __cache_set_unregister()
1826 dc = container_of(d, struct cached_dev, disk); in __cache_set_unregister()
1827 bch_cached_dev_detach(dc); in __cache_set_unregister()
1828 if (test_bit(CACHE_SET_IO_DISABLE, &c->flags)) in __cache_set_unregister()
1829 conditional_stop_bcache_device(c, d, dc); in __cache_set_unregister()
1842 if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags)) in bch_cache_set_stop()
1844 closure_queue(&c->caching); in bch_cache_set_stop()
1849 set_bit(CACHE_SET_UNREGISTERING, &c->flags); in bch_cache_set_unregister()
1866 closure_init(&c->cl, NULL); in bch_cache_set_alloc()
1867 set_closure_fn(&c->cl, cache_set_free, system_wq); in bch_cache_set_alloc()
1869 closure_init(&c->caching, &c->cl); in bch_cache_set_alloc()
1870 set_closure_fn(&c->caching, __cache_set_unregister, system_wq); in bch_cache_set_alloc()
1873 closure_set_stopped(&c->cl); in bch_cache_set_alloc()
1874 closure_put(&c->cl); in bch_cache_set_alloc()
1876 kobject_init(&c->kobj, &bch_cache_set_ktype); in bch_cache_set_alloc()
1877 kobject_init(&c->internal, &bch_cache_set_internal_ktype); in bch_cache_set_alloc()
1879 bch_cache_accounting_init(&c->accounting, &c->cl); in bch_cache_set_alloc()
1881 memcpy(c->set_uuid, sb->set_uuid, 16); in bch_cache_set_alloc()
1883 c->cache = ca; in bch_cache_set_alloc()
1884 c->cache->set = c; in bch_cache_set_alloc()
1885 c->bucket_bits = ilog2(sb->bucket_size); in bch_cache_set_alloc()
1886 c->block_bits = ilog2(sb->block_size); in bch_cache_set_alloc()
1887 c->nr_uuids = meta_bucket_bytes(sb) / sizeof(struct uuid_entry); in bch_cache_set_alloc()
1888 c->devices_max_used = 0; in bch_cache_set_alloc()
1889 atomic_set(&c->attached_dev_nr, 0); in bch_cache_set_alloc()
1890 c->btree_pages = meta_bucket_pages(sb); in bch_cache_set_alloc()
1891 if (c->btree_pages > BTREE_MAX_PAGES) in bch_cache_set_alloc()
1892 c->btree_pages = max_t(int, c->btree_pages / 4, in bch_cache_set_alloc()
1895 sema_init(&c->sb_write_mutex, 1); in bch_cache_set_alloc()
1896 mutex_init(&c->bucket_lock); in bch_cache_set_alloc()
1897 init_waitqueue_head(&c->btree_cache_wait); in bch_cache_set_alloc()
1898 spin_lock_init(&c->btree_cannibalize_lock); in bch_cache_set_alloc()
1899 init_waitqueue_head(&c->bucket_wait); in bch_cache_set_alloc()
1900 init_waitqueue_head(&c->gc_wait); in bch_cache_set_alloc()
1901 sema_init(&c->uuid_write_mutex, 1); in bch_cache_set_alloc()
1903 spin_lock_init(&c->btree_gc_time.lock); in bch_cache_set_alloc()
1904 spin_lock_init(&c->btree_split_time.lock); in bch_cache_set_alloc()
1905 spin_lock_init(&c->btree_read_time.lock); in bch_cache_set_alloc()
1909 INIT_LIST_HEAD(&c->list); in bch_cache_set_alloc()
1910 INIT_LIST_HEAD(&c->cached_devs); in bch_cache_set_alloc()
1911 INIT_LIST_HEAD(&c->btree_cache); in bch_cache_set_alloc()
1912 INIT_LIST_HEAD(&c->btree_cache_freeable); in bch_cache_set_alloc()
1913 INIT_LIST_HEAD(&c->btree_cache_freed); in bch_cache_set_alloc()
1914 INIT_LIST_HEAD(&c->data_buckets); in bch_cache_set_alloc()
1916 iter_size = ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size + 1) * in bch_cache_set_alloc()
1919 c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL); in bch_cache_set_alloc()
1920 if (!c->devices) in bch_cache_set_alloc()
1923 if (mempool_init_slab_pool(&c->search, 32, bch_search_cache)) in bch_cache_set_alloc()
1926 if (mempool_init_kmalloc_pool(&c->bio_meta, 2, in bch_cache_set_alloc()
1931 if (mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size)) in bch_cache_set_alloc()
1934 if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio), in bch_cache_set_alloc()
1938 c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb); in bch_cache_set_alloc()
1939 if (!c->uuids) in bch_cache_set_alloc()
1942 c->moving_gc_wq = alloc_workqueue("bcache_gc", WQ_MEM_RECLAIM, 0); in bch_cache_set_alloc()
1943 if (!c->moving_gc_wq) in bch_cache_set_alloc()
1955 if (bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages))) in bch_cache_set_alloc()
1958 c->congested_read_threshold_us = 2000; in bch_cache_set_alloc()
1959 c->congested_write_threshold_us = 20000; in bch_cache_set_alloc()
1960 c->error_limit = DEFAULT_IO_ERROR_LIMIT; in bch_cache_set_alloc()
1961 c->idle_max_writeback_rate_enabled = 1; in bch_cache_set_alloc()
1962 WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags)); in bch_cache_set_alloc()
1973 struct cached_dev *dc, *t; in run_cache_set() local
1974 struct cache *ca = c->cache; in run_cache_set()
1981 c->nbuckets = ca->sb.nbuckets; in run_cache_set()
1984 if (CACHE_SYNC(&c->cache->sb)) { in run_cache_set()
1998 j = &list_entry(journal.prev, struct journal_replay, list)->j; in run_cache_set()
2001 if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev])) in run_cache_set()
2010 k = &j->btree_root; in run_cache_set()
2017 c->root = bch_btree_node_get(c, NULL, k, in run_cache_set()
2018 j->btree_level, in run_cache_set()
2020 if (IS_ERR_OR_NULL(c->root)) in run_cache_set()
2023 list_del_init(&c->root->list); in run_cache_set()
2024 rw_unlock(true, c->root); in run_cache_set()
2041 * gc_gen - this is a hack but oh well. in run_cache_set()
2043 bch_journal_next(&c->journal); in run_cache_set()
2059 if (j->version < BCACHE_JSET_VERSION_UUID) in run_cache_set()
2069 ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7, in run_cache_set()
2072 for (j = 0; j < ca->sb.keys; j++) in run_cache_set()
2073 ca->sb.d[j] = ca->sb.first_bucket + j; in run_cache_set()
2081 mutex_lock(&c->bucket_lock); in run_cache_set()
2083 mutex_unlock(&c->bucket_lock); in run_cache_set()
2090 c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL); in run_cache_set()
2091 if (IS_ERR_OR_NULL(c->root)) in run_cache_set()
2094 mutex_lock(&c->root->write_lock); in run_cache_set()
2095 bkey_copy_key(&c->root->key, &MAX_KEY); in run_cache_set()
2096 bch_btree_node_write(c->root, &cl); in run_cache_set()
2097 mutex_unlock(&c->root->write_lock); in run_cache_set()
2099 bch_btree_set_root(c->root); in run_cache_set()
2100 rw_unlock(true, c->root); in run_cache_set()
2104 * everything is set up - fortunately journal entries won't be in run_cache_set()
2107 SET_CACHE_SYNC(&c->cache->sb, true); in run_cache_set()
2109 bch_journal_next(&c->journal); in run_cache_set()
2118 c->cache->sb.last_mount = (u32)ktime_get_real_seconds(); in run_cache_set()
2121 if (bch_has_feature_obso_large_bucket(&c->cache->sb)) in run_cache_set()
2122 pr_err("Detect obsoleted large bucket layout, all attached bcache device will be read-only\n"); in run_cache_set()
2124 list_for_each_entry_safe(dc, t, &uncached_devices, list) in run_cache_set()
2125 bch_cached_dev_attach(dc, c, NULL); in run_cache_set()
2129 bch_journal_space_reserve(&c->journal); in run_cache_set()
2130 set_bit(CACHE_SET_RUNNING, &c->flags); in run_cache_set()
2135 list_del(&l->list); in run_cache_set()
2143 return -EIO; in run_cache_set()
2153 if (!memcmp(c->set_uuid, ca->sb.set_uuid, 16)) { in register_cache_set()
2154 if (c->cache) in register_cache_set()
2160 c = bch_cache_set_alloc(&ca->sb); in register_cache_set()
2165 if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->set_uuid) || in register_cache_set()
2166 kobject_add(&c->internal, &c->kobj, "internal")) in register_cache_set()
2169 if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj)) in register_cache_set()
2174 list_add(&c->list, &bch_cache_sets); in register_cache_set()
2176 sprintf(buf, "cache%i", ca->sb.nr_this_dev); in register_cache_set()
2177 if (sysfs_create_link(&ca->kobj, &c->kobj, "set") || in register_cache_set()
2178 sysfs_create_link(&c->kobj, &ca->kobj, buf)) in register_cache_set()
2181 kobject_get(&ca->kobj); in register_cache_set()
2182 ca->set = c; in register_cache_set()
2183 ca->set->cache = ca; in register_cache_set()
2197 /* When ca->kobj released */
2203 if (ca->set) { in bch_cache_release()
2204 BUG_ON(ca->set->cache != ca); in bch_cache_release()
2205 ca->set->cache = NULL; in bch_cache_release()
2208 free_pages((unsigned long) ca->disk_buckets, ilog2(meta_bucket_pages(&ca->sb))); in bch_cache_release()
2209 kfree(ca->prio_buckets); in bch_cache_release()
2210 vfree(ca->buckets); in bch_cache_release()
2212 free_heap(&ca->heap); in bch_cache_release()
2213 free_fifo(&ca->free_inc); in bch_cache_release()
2216 free_fifo(&ca->free[i]); in bch_cache_release()
2218 if (ca->sb_disk) in bch_cache_release()
2219 put_page(virt_to_page(ca->sb_disk)); in bch_cache_release()
2221 if (!IS_ERR_OR_NULL(ca->bdev)) in bch_cache_release()
2222 blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); in bch_cache_release()
2233 int ret = -ENOMEM; in cache_alloc()
2237 kobject_init(&ca->kobj, &bch_cache_ktype); in cache_alloc()
2239 bio_init(&ca->journal.bio, NULL, ca->journal.bio.bi_inline_vecs, 8, 0); in cache_alloc()
2242 * when ca->sb.njournal_buckets is not zero, journal exists, in cache_alloc()
2250 btree_buckets = ca->sb.njournal_buckets ?: 8; in cache_alloc()
2251 free = roundup_pow_of_two(ca->sb.nbuckets) >> 10; in cache_alloc()
2253 ret = -EPERM; in cache_alloc()
2254 err = "ca->sb.nbuckets is too small"; in cache_alloc()
2258 if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, in cache_alloc()
2260 err = "ca->free[RESERVE_BTREE] alloc failed"; in cache_alloc()
2264 if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), in cache_alloc()
2266 err = "ca->free[RESERVE_PRIO] alloc failed"; in cache_alloc()
2270 if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) { in cache_alloc()
2271 err = "ca->free[RESERVE_MOVINGGC] alloc failed"; in cache_alloc()
2275 if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) { in cache_alloc()
2276 err = "ca->free[RESERVE_NONE] alloc failed"; in cache_alloc()
2280 if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) { in cache_alloc()
2281 err = "ca->free_inc alloc failed"; in cache_alloc()
2285 if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) { in cache_alloc()
2286 err = "ca->heap alloc failed"; in cache_alloc()
2290 ca->buckets = vzalloc(array_size(sizeof(struct bucket), in cache_alloc()
2291 ca->sb.nbuckets)); in cache_alloc()
2292 if (!ca->buckets) { in cache_alloc()
2293 err = "ca->buckets alloc failed"; in cache_alloc()
2297 ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t), in cache_alloc()
2300 if (!ca->prio_buckets) { in cache_alloc()
2301 err = "ca->prio_buckets alloc failed"; in cache_alloc()
2305 ca->disk_buckets = alloc_meta_bucket_pages(GFP_KERNEL, &ca->sb); in cache_alloc()
2306 if (!ca->disk_buckets) { in cache_alloc()
2307 err = "ca->disk_buckets alloc failed"; in cache_alloc()
2311 ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); in cache_alloc()
2314 atomic_set(&b->pin, 0); in cache_alloc()
2318 kfree(ca->prio_buckets); in cache_alloc()
2320 vfree(ca->buckets); in cache_alloc()
2322 free_heap(&ca->heap); in cache_alloc()
2324 free_fifo(&ca->free_inc); in cache_alloc()
2326 free_fifo(&ca->free[RESERVE_NONE]); in cache_alloc()
2328 free_fifo(&ca->free[RESERVE_MOVINGGC]); in cache_alloc()
2330 free_fifo(&ca->free[RESERVE_PRIO]); in cache_alloc()
2332 free_fifo(&ca->free[RESERVE_BTREE]); in cache_alloc()
2337 pr_notice("error %pg: %s\n", ca->bdev, err); in cache_alloc()
2347 memcpy(&ca->sb, sb, sizeof(struct cache_sb)); in register_cache()
2348 ca->bdev = bdev; in register_cache()
2349 ca->bdev->bd_holder = ca; in register_cache()
2350 ca->sb_disk = sb_disk; in register_cache()
2353 ca->discard = CACHE_DISCARD(&ca->sb); in register_cache()
2358 * If we failed here, it means ca->kobj is not initialized yet, in register_cache()
2364 if (ret == -ENOMEM) in register_cache()
2365 err = "cache_alloc(): -ENOMEM"; in register_cache()
2366 else if (ret == -EPERM) in register_cache()
2373 if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache")) { in register_cache()
2375 ret = -ENOMEM; in register_cache()
2384 ret = -ENODEV; in register_cache()
2388 pr_info("registered cache device %pg\n", ca->bdev); in register_cache()
2391 kobject_put(&ca->kobj); in register_cache()
2395 pr_notice("error %pg: %s\n", ca->bdev, err); in register_cache()
2415 struct cached_dev *dc, *t; in bch_is_open_backing() local
2418 list_for_each_entry_safe(dc, t, &c->cached_devs, list) in bch_is_open_backing()
2419 if (dc->bdev->bd_dev == dev) in bch_is_open_backing()
2421 list_for_each_entry_safe(dc, t, &uncached_devices, list) in bch_is_open_backing()
2422 if (dc->bdev->bd_dev == dev) in bch_is_open_backing()
2432 struct cache *ca = c->cache; in bch_is_open_cache()
2434 if (ca->bdev->bd_dev == dev) in bch_is_open_cache()
2459 struct cached_dev *dc; in register_bdev_worker() local
2461 dc = kzalloc(sizeof(*dc), GFP_KERNEL); in register_bdev_worker()
2462 if (!dc) { in register_bdev_worker()
2464 put_page(virt_to_page(args->sb_disk)); in register_bdev_worker()
2465 blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); in register_bdev_worker()
2470 if (register_bdev(args->sb, args->sb_disk, args->bdev, dc) < 0) in register_bdev_worker()
2477 args->path); in register_bdev_worker()
2478 kfree(args->sb); in register_bdev_worker()
2479 kfree(args->path); in register_bdev_worker()
2494 put_page(virt_to_page(args->sb_disk)); in register_cache_worker()
2495 blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); in register_cache_worker()
2500 if (register_cache(args->sb, args->sb_disk, args->bdev, ca) != 0) in register_cache_worker()
2506 args->path); in register_cache_worker()
2507 kfree(args->sb); in register_cache_worker()
2508 kfree(args->path); in register_cache_worker()
2515 if (SB_IS_BDEV(args->sb)) in register_device_async()
2516 INIT_DELAYED_WORK(&args->reg_work, register_bdev_worker); in register_device_async()
2518 INIT_DELAYED_WORK(&args->reg_work, register_cache_worker); in register_device_async()
2521 queue_delayed_work(system_wq, &args->reg_work, 10); in register_device_async()
2539 ret = -EBUSY; in register_bcache()
2550 ret = -ENOMEM; in register_bcache()
2560 ret = -EINVAL; in register_bcache()
2566 if (bdev == ERR_PTR(-EBUSY)) { in register_bcache()
2598 ret = -ENOMEM; in register_bcache()
2603 args->path = path; in register_bcache()
2604 args->sb = sb; in register_bcache()
2605 args->sb_disk = sb_disk; in register_bcache()
2606 args->bdev = bdev; in register_bcache()
2613 struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL); in register_bcache() local
2615 if (!dc) { in register_bcache()
2616 ret = -ENOMEM; in register_bcache()
2622 ret = register_bdev(sb, sb_disk, bdev, dc); in register_bcache()
2631 ret = -ENOMEM; in register_bcache()
2668 struct cached_dev *dc; member
2678 struct cached_dev *dc, *tdc; in bch_pending_bdevs_cleanup() local
2683 list_for_each_entry_safe(dc, tdc, &uncached_devices, list) { in bch_pending_bdevs_cleanup()
2687 pdev->dc = dc; in bch_pending_bdevs_cleanup()
2688 list_add(&pdev->list, &pending_devs); in bch_pending_bdevs_cleanup()
2692 char *pdev_set_uuid = pdev->dc->sb.set_uuid; in bch_pending_bdevs_cleanup()
2694 char *set_uuid = c->set_uuid; in bch_pending_bdevs_cleanup()
2697 list_del(&pdev->list); in bch_pending_bdevs_cleanup()
2707 list_del(&pdev->list); in bch_pending_bdevs_cleanup()
2708 bcache_device_stop(&pdev->dc->disk); in bch_pending_bdevs_cleanup()
2728 struct cached_dev *dc, *tdc; in bcache_reboot() local
2761 * bcache_reboot() won't be re-entered on by other thread, in bcache_reboot()
2768 list_for_each_entry_safe(dc, tdc, &uncached_devices, list) in bcache_reboot()
2769 bcache_device_stop(&dc->disk); in bcache_reboot()
2780 long timeout = start + 10 * HZ - jiffies; in bcache_reboot()
2922 return -ENOMEM; in bcache_init()