34 	struct writeset_metadata md;  member
94 	ws->md.nr_bits = nr_blocks;  in writeset_init()
95 	r = setup_on_disk_bitset(info, ws->md.nr_bits, &ws->md.root);  in writeset_init()
139 		r = dm_bitset_set_bit(info, ws->md.root, block, &ws->md.root);  in writeset_test_and_set()
301 static int superblock_read_lock(struct era_metadata *md,  in superblock_read_lock()  argument
304 	return dm_bm_read_lock(md->bm, SUPERBLOCK_LOCATION,  in superblock_read_lock()
308 static int superblock_lock_zero(struct era_metadata *md,  in superblock_lock_zero()  argument
311 	return dm_bm_write_lock_zero(md->bm, SUPERBLOCK_LOCATION,  in superblock_lock_zero()
315 static int superblock_lock(struct era_metadata *md,  in superblock_lock()  argument
318 	return dm_bm_write_lock(md->bm, SUPERBLOCK_LOCATION,  in superblock_lock()
368 	struct era_metadata *md = context;  in ws_inc()  local
376 		dm_tm_inc(md->tm, b);  in ws_inc()
382 	struct era_metadata *md = context;  in ws_dec()  local
390 		dm_bitset_del(&md->bitset_info, b);  in ws_dec()
401 static void setup_writeset_tree_info(struct era_metadata *md)  in setup_writeset_tree_info()  argument
403 	struct dm_btree_value_type *vt = &md->writeset_tree_info.value_type;  in setup_writeset_tree_info()
404 	md->writeset_tree_info.tm = md->tm;  in setup_writeset_tree_info()
405 	md->writeset_tree_info.levels = 1;  in setup_writeset_tree_info()
406 	vt->context = md;  in setup_writeset_tree_info()
413 static void setup_era_array_info(struct era_metadata *md)  in setup_era_array_info()  argument
423 	dm_array_info_init(&md->era_array_info, md->tm, &vt);  in setup_era_array_info()
426 static void setup_infos(struct era_metadata *md)  in setup_infos()  argument
428 	dm_disk_bitset_init(md->tm, &md->bitset_info);  in setup_infos()
429 	setup_writeset_tree_info(md);  in setup_infos()
430 	setup_era_array_info(md);  in setup_infos()
435 static int create_fresh_metadata(struct era_metadata *md)  in create_fresh_metadata()  argument
439 	r = dm_tm_create_with_sm(md->bm, SUPERBLOCK_LOCATION,  in create_fresh_metadata()
440 				 &md->tm, &md->sm);  in create_fresh_metadata()
446 	setup_infos(md);  in create_fresh_metadata()
448 	r = dm_btree_empty(&md->writeset_tree_info, &md->writeset_tree_root);  in create_fresh_metadata()
454 	r = dm_array_empty(&md->era_array_info, &md->era_array_root);  in create_fresh_metadata()
463 	dm_sm_destroy(md->sm);  in create_fresh_metadata()
464 	dm_tm_destroy(md->tm);  in create_fresh_metadata()
469 static int save_sm_root(struct era_metadata *md)  in save_sm_root()  argument
474 	r = dm_sm_root_size(md->sm, &metadata_len);  in save_sm_root()
478 	return dm_sm_copy_root(md->sm, &md->metadata_space_map_root,  in save_sm_root()
482 static void copy_sm_root(struct era_metadata *md, struct superblock_disk *disk)  in copy_sm_root()  argument
485 	       &md->metadata_space_map_root,  in copy_sm_root()
486 	       sizeof(md->metadata_space_map_root));  in copy_sm_root()
491  * with every commit (possible optimisation here).  'md' should be fully
494 static void prepare_superblock(struct era_metadata *md, struct superblock_disk *disk)  in prepare_superblock()  argument
503 	copy_sm_root(md, disk);  in prepare_superblock()
505 	disk->data_block_size = cpu_to_le32(md->block_size);  in prepare_superblock()
507 	disk->nr_blocks = cpu_to_le32(md->nr_blocks);  in prepare_superblock()
508 	disk->current_era = cpu_to_le32(md->current_era);  in prepare_superblock()
510 	ws_pack(&md->current_writeset->md, &disk->current_writeset);  in prepare_superblock()
511 	disk->writeset_tree_root = cpu_to_le64(md->writeset_tree_root);  in prepare_superblock()
512 	disk->era_array_root = cpu_to_le64(md->era_array_root);  in prepare_superblock()
513 	disk->metadata_snap = cpu_to_le64(md->metadata_snap);  in prepare_superblock()
516 static int write_superblock(struct era_metadata *md)  in write_superblock()  argument
522 	r = save_sm_root(md);  in write_superblock()
528 	r = superblock_lock_zero(md, &sblock);  in write_superblock()
533 	prepare_superblock(md, disk);  in write_superblock()
535 	return dm_tm_commit(md->tm, sblock);  in write_superblock()
541 static int format_metadata(struct era_metadata *md)  in format_metadata()  argument
545 	r = create_fresh_metadata(md);  in format_metadata()
549 	r = write_superblock(md);  in format_metadata()
551 		dm_sm_destroy(md->sm);  in format_metadata()
552 		dm_tm_destroy(md->tm);  in format_metadata()
559 static int open_metadata(struct era_metadata *md)  in open_metadata()  argument
565 	r = superblock_read_lock(md, &sblock);  in open_metadata()
574 	if (le32_to_cpu(disk->data_block_size) != md->block_size) {  in open_metadata()
576 		      le32_to_cpu(disk->data_block_size), md->block_size);  in open_metadata()
581 	r = dm_tm_open_with_sm(md->bm, SUPERBLOCK_LOCATION,  in open_metadata()
584 			       &md->tm, &md->sm);  in open_metadata()
590 	setup_infos(md);  in open_metadata()
592 	md->nr_blocks = le32_to_cpu(disk->nr_blocks);  in open_metadata()
593 	md->current_era = le32_to_cpu(disk->current_era);  in open_metadata()
595 	ws_unpack(&disk->current_writeset, &md->current_writeset->md);  in open_metadata()
596 	md->writeset_tree_root = le64_to_cpu(disk->writeset_tree_root);  in open_metadata()
597 	md->era_array_root = le64_to_cpu(disk->era_array_root);  in open_metadata()
598 	md->metadata_snap = le64_to_cpu(disk->metadata_snap);  in open_metadata()
599 	md->archived_writesets = true;  in open_metadata()
610 static int open_or_format_metadata(struct era_metadata *md,  in open_or_format_metadata()  argument
616 	r = superblock_all_zeroes(md->bm, &unformatted);  in open_or_format_metadata()
621 		return may_format ? format_metadata(md) : -EPERM;  in open_or_format_metadata()
623 	return open_metadata(md);  in open_or_format_metadata()
626 static int create_persistent_data_objects(struct era_metadata *md,  in create_persistent_data_objects()  argument
631 	md->bm = dm_block_manager_create(md->bdev, DM_ERA_METADATA_BLOCK_SIZE,  in create_persistent_data_objects()
633 	if (IS_ERR(md->bm)) {  in create_persistent_data_objects()
635 		return PTR_ERR(md->bm);  in create_persistent_data_objects()
638 	r = open_or_format_metadata(md, may_format);  in create_persistent_data_objects()
640 		dm_block_manager_destroy(md->bm);  in create_persistent_data_objects()
645 static void destroy_persistent_data_objects(struct era_metadata *md)  in destroy_persistent_data_objects()  argument
647 	dm_sm_destroy(md->sm);  in destroy_persistent_data_objects()
648 	dm_tm_destroy(md->tm);  in destroy_persistent_data_objects()
649 	dm_block_manager_destroy(md->bm);  in destroy_persistent_data_objects()
655 static void swap_writeset(struct era_metadata *md, struct writeset *new_writeset)  in swap_writeset()  argument
657 	rcu_assign_pointer(md->current_writeset, new_writeset);  in swap_writeset()
678 static int metadata_digest_lookup_writeset(struct era_metadata *md,
681 static int metadata_digest_remove_writeset(struct era_metadata *md,  in metadata_digest_remove_writeset()  argument
687 	r = dm_btree_remove(&md->writeset_tree_info, md->writeset_tree_root,  in metadata_digest_remove_writeset()
688 			    &key, &md->writeset_tree_root);  in metadata_digest_remove_writeset()
700 static int metadata_digest_transcribe_writeset(struct era_metadata *md,  in metadata_digest_transcribe_writeset()  argument
718 		r = dm_array_set_value(&md->era_array_info, md->era_array_root,  in metadata_digest_transcribe_writeset()
719 				       b, &d->value, &md->era_array_root);  in metadata_digest_transcribe_writeset()
734 static int metadata_digest_lookup_writeset(struct era_metadata *md,  in metadata_digest_lookup_writeset()  argument
741 	r = dm_btree_find_lowest_key(&md->writeset_tree_info,  in metadata_digest_lookup_writeset()
742 				     md->writeset_tree_root, &key);  in metadata_digest_lookup_writeset()
748 	r = dm_btree_lookup(&md->writeset_tree_info,  in metadata_digest_lookup_writeset()
749 			    md->writeset_tree_root, &key, &disk);  in metadata_digest_lookup_writeset()
767 	dm_disk_bitset_init(md->tm, &d->info);  in metadata_digest_lookup_writeset()
769 	d->nr_bits = min(d->writeset.nr_bits, md->nr_blocks);  in metadata_digest_lookup_writeset()
776 static int metadata_digest_start(struct era_metadata *md, struct digest *d)  in metadata_digest_start()  argument
796 	struct era_metadata *md = kzalloc(sizeof(*md), GFP_KERNEL);  in metadata_open()  local
798 	if (!md)  in metadata_open()
801 	md->bdev = bdev;  in metadata_open()
802 	md->block_size = block_size;  in metadata_open()
804 	md->writesets[0].md.root = INVALID_WRITESET_ROOT;  in metadata_open()
805 	md->writesets[1].md.root = INVALID_WRITESET_ROOT;  in metadata_open()
806 	md->current_writeset = &md->writesets[0];  in metadata_open()
808 	r = create_persistent_data_objects(md, may_format);  in metadata_open()
810 		kfree(md);  in metadata_open()
814 	return md;  in metadata_open()
817 static void metadata_close(struct era_metadata *md)  in metadata_close()  argument
819 	writeset_free(&md->writesets[0]);  in metadata_close()
820 	writeset_free(&md->writesets[1]);  in metadata_close()
821 	destroy_persistent_data_objects(md);  in metadata_close()
822 	kfree(md);  in metadata_close()
834 static int metadata_resize(struct era_metadata *md, void *arg)  in metadata_resize()  argument
846 	writeset_free(&md->writesets[0]);  in metadata_resize()
847 	writeset_free(&md->writesets[1]);  in metadata_resize()
849 	r = writeset_alloc(&md->writesets[0], *new_size);  in metadata_resize()
855 	r = writeset_alloc(&md->writesets[1], *new_size);  in metadata_resize()
858 		writeset_free(&md->writesets[0]);  in metadata_resize()
864 	r = dm_array_resize(&md->era_array_info, md->era_array_root,  in metadata_resize()
865 			    md->nr_blocks, *new_size,  in metadata_resize()
866 			    &value, &md->era_array_root);  in metadata_resize()
869 		writeset_free(&md->writesets[0]);  in metadata_resize()
870 		writeset_free(&md->writesets[1]);  in metadata_resize()
874 	md->nr_blocks = *new_size;  in metadata_resize()
878 static int metadata_era_archive(struct era_metadata *md)  in metadata_era_archive()  argument
884 	r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,  in metadata_era_archive()
885 			    &md->current_writeset->md.root);  in metadata_era_archive()
891 	ws_pack(&md->current_writeset->md, &value);  in metadata_era_archive()
893 	keys[0] = md->current_era;  in metadata_era_archive()
895 	r = dm_btree_insert(&md->writeset_tree_info, md->writeset_tree_root,  in metadata_era_archive()
896 			    keys, &value, &md->writeset_tree_root);  in metadata_era_archive()
903 	md->current_writeset->md.root = INVALID_WRITESET_ROOT;  in metadata_era_archive()
904 	md->archived_writesets = true;  in metadata_era_archive()
909 static struct writeset *next_writeset(struct era_metadata *md)  in next_writeset()  argument
911 	return (md->current_writeset == &md->writesets[0]) ?  in next_writeset()
912 		&md->writesets[1] : &md->writesets[0];  in next_writeset()
915 static int metadata_new_era(struct era_metadata *md)  in metadata_new_era()  argument
918 	struct writeset *new_writeset = next_writeset(md);  in metadata_new_era()
920 	r = writeset_init(&md->bitset_info, new_writeset, md->nr_blocks);  in metadata_new_era()
926 	swap_writeset(md, new_writeset);  in metadata_new_era()
927 	md->current_era++;  in metadata_new_era()
932 static int metadata_era_rollover(struct era_metadata *md)  in metadata_era_rollover()  argument
936 	if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {  in metadata_era_rollover()
937 		r = metadata_era_archive(md);  in metadata_era_rollover()
945 	r = metadata_new_era(md);  in metadata_era_rollover()
955 static bool metadata_current_marked(struct era_metadata *md, dm_block_t block)  in metadata_current_marked()  argument
961 	ws = rcu_dereference(md->current_writeset);  in metadata_current_marked()
968 static int metadata_commit(struct era_metadata *md)  in metadata_commit()  argument
973 	if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {  in metadata_commit()
974 		r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,  in metadata_commit()
975 				    &md->current_writeset->md.root);  in metadata_commit()
982 	r = dm_tm_pre_commit(md->tm);  in metadata_commit()
988 	r = save_sm_root(md);  in metadata_commit()
994 	r = superblock_lock(md, &sblock);  in metadata_commit()
1000 	prepare_superblock(md, dm_block_data(sblock));  in metadata_commit()
1002 	return dm_tm_commit(md->tm, sblock);  in metadata_commit()
1005 static int metadata_checkpoint(struct era_metadata *md)  in metadata_checkpoint()  argument
1011 	return metadata_era_rollover(md);  in metadata_checkpoint()
1017 static int metadata_take_snap(struct era_metadata *md)  in metadata_take_snap()  argument
1022 	if (md->metadata_snap != SUPERBLOCK_LOCATION) {  in metadata_take_snap()
1027 	r = metadata_era_rollover(md);  in metadata_take_snap()
1033 	r = metadata_commit(md);  in metadata_take_snap()
1039 	r = dm_sm_inc_block(md->sm, SUPERBLOCK_LOCATION);  in metadata_take_snap()
1045 	r = dm_tm_shadow_block(md->tm, SUPERBLOCK_LOCATION,  in metadata_take_snap()
1049 		dm_sm_dec_block(md->sm, SUPERBLOCK_LOCATION);  in metadata_take_snap()
1054 	r = dm_sm_inc_block(md->sm, md->writeset_tree_root);  in metadata_take_snap()
1057 		dm_tm_unlock(md->tm, clone);  in metadata_take_snap()
1061 	r = dm_sm_inc_block(md->sm, md->era_array_root);  in metadata_take_snap()
1064 		dm_sm_dec_block(md->sm, md->writeset_tree_root);  in metadata_take_snap()
1065 		dm_tm_unlock(md->tm, clone);  in metadata_take_snap()
1069 	md->metadata_snap = dm_block_location(clone);  in metadata_take_snap()
1071 	dm_tm_unlock(md->tm, clone);  in metadata_take_snap()
1076 static int metadata_drop_snap(struct era_metadata *md)  in metadata_drop_snap()  argument
1083 	if (md->metadata_snap == SUPERBLOCK_LOCATION) {  in metadata_drop_snap()
1088 	r = dm_tm_read_lock(md->tm, md->metadata_snap, &sb_validator, &clone);  in metadata_drop_snap()
1098 	md->metadata_snap = SUPERBLOCK_LOCATION;  in metadata_drop_snap()
1101 	r = dm_btree_del(&md->writeset_tree_info,  in metadata_drop_snap()
1105 		dm_tm_unlock(md->tm, clone);  in metadata_drop_snap()
1109 	r = dm_array_del(&md->era_array_info, le64_to_cpu(disk->era_array_root));  in metadata_drop_snap()
1112 		dm_tm_unlock(md->tm, clone);  in metadata_drop_snap()
1117 	dm_tm_unlock(md->tm, clone);  in metadata_drop_snap()
1119 	return dm_sm_dec_block(md->sm, location);  in metadata_drop_snap()
1129 static int metadata_get_stats(struct era_metadata *md, void *ptr)  in metadata_get_stats()  argument
1135 	r = dm_sm_get_nr_free(md->sm, &nr_free);  in metadata_get_stats()
1141 	r = dm_sm_get_nr_blocks(md->sm, &nr_total);  in metadata_get_stats()
1149 	s->snap = md->metadata_snap;  in metadata_get_stats()
1150 	s->era = md->current_era;  in metadata_get_stats()
1166 	struct era_metadata *md;  member
1233 	r = era->digest.step(era->md, &era->digest);  in process_old_eras()
1250 	struct writeset *ws = era->md->current_writeset;  in process_deferred_bios()
1264 		r = writeset_test_and_set(&era->md->bitset_info, ws,  in process_deferred_bios()
1279 		r = metadata_commit(era->md);  in process_deferred_bios()
1315 		rpc->result = rpc->fn0 ? rpc->fn0(era->md) : rpc->fn1(era->md, rpc->arg);  in process_rpc_calls()
1320 		r = metadata_commit(era->md);  in process_rpc_calls()
1332 	if (era->md->archived_writesets) {  in kick_off_digest()
1333 		era->md->archived_writesets = false;  in kick_off_digest()
1334 		metadata_digest_start(era->md, &era->digest);  in kick_off_digest()
1411 	if (era->md)  in era_destroy()
1412 		metadata_close(era->md);  in era_destroy()
1447 	struct era_metadata *md;  in era_ctr()  local
1500 	md = metadata_open(era->metadata_dev->bdev, era->sectors_per_block, true);  in era_ctr()
1501 	if (IS_ERR(md)) {  in era_ctr()
1504 		return PTR_ERR(md);  in era_ctr()
1506 	era->md = md;  in era_ctr()
1553 	    !metadata_current_marked(era->md, block)) {  in era_map()
1574 	r = metadata_commit(era->md);  in era_postsuspend()
1588 		r = metadata_resize(era->md, &new_size);  in era_preresume()
1594 		r = metadata_commit(era->md);  in era_preresume()