207 #define dmz_zmd_info(zmd, format, args...)	\  argument
208 	DMINFO("(%s): " format, (zmd)->label, ## args)
210 #define dmz_zmd_err(zmd, format, args...)	\  argument
211 	DMERR("(%s): " format, (zmd)->label, ## args)
213 #define dmz_zmd_warn(zmd, format, args...)	\  argument
214 	DMWARN("(%s): " format, (zmd)->label, ## args)
216 #define dmz_zmd_debug(zmd, format, args...)	\  argument
217 	DMDEBUG("(%s): " format, (zmd)->label, ## args)
221 static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_dev_zone_id()  argument
229 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_sect()  argument
231 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_sect()
233 	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;  in dmz_start_sect()
236 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_block()  argument
238 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_block()
240 	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;  in dmz_start_block()
243 unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks()  argument
245 	return zmd->zone_nr_blocks;  in dmz_zone_nr_blocks()
248 unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks_shift()  argument
250 	return zmd->zone_nr_blocks_shift;  in dmz_zone_nr_blocks_shift()
253 unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors()  argument
255 	return zmd->zone_nr_sectors;  in dmz_zone_nr_sectors()
258 unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors_shift()  argument
260 	return zmd->zone_nr_sectors_shift;  in dmz_zone_nr_sectors_shift()
263 unsigned int dmz_nr_zones(struct dmz_metadata *zmd)  in dmz_nr_zones()  argument
265 	return zmd->nr_zones;  in dmz_nr_zones()
268 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)  in dmz_nr_chunks()  argument
270 	return zmd->nr_chunks;  in dmz_nr_chunks()
273 unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_rnd_zones()  argument
275 	return zmd->dev[idx].nr_rnd;  in dmz_nr_rnd_zones()
278 unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_rnd_zones()  argument
280 	return atomic_read(&zmd->dev[idx].unmap_nr_rnd);  in dmz_nr_unmap_rnd_zones()
283 unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_cache_zones()  argument
285 	return zmd->nr_cache;  in dmz_nr_cache_zones()
288 unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_unmap_cache_zones()  argument
290 	return atomic_read(&zmd->unmap_nr_cache);  in dmz_nr_unmap_cache_zones()
293 unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_seq_zones()  argument
295 	return zmd->dev[idx].nr_seq;  in dmz_nr_seq_zones()
298 unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_seq_zones()  argument
300 	return atomic_read(&zmd->dev[idx].unmap_nr_seq);  in dmz_nr_unmap_seq_zones()
303 static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)  in dmz_get()  argument
305 	return xa_load(&zmd->zones, zone_id);  in dmz_get()
308 static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,  in dmz_insert()  argument
316 	if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {  in dmz_insert()
330 const char *dmz_metadata_label(struct dmz_metadata *zmd)  in dmz_metadata_label()  argument
332 	return (const char *)zmd->label;  in dmz_metadata_label()
335 bool dmz_check_dev(struct dmz_metadata *zmd)  in dmz_check_dev()  argument
339 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_check_dev()
340 		if (!dmz_check_bdev(&zmd->dev[i]))  in dmz_check_dev()
346 bool dmz_dev_is_dying(struct dmz_metadata *zmd)  in dmz_dev_is_dying()  argument
350 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_dev_is_dying()
351 		if (dmz_bdev_is_dying(&zmd->dev[i]))  in dmz_dev_is_dying()
361 void dmz_lock_map(struct dmz_metadata *zmd)  in dmz_lock_map()  argument
363 	mutex_lock(&zmd->map_lock);  in dmz_lock_map()
366 void dmz_unlock_map(struct dmz_metadata *zmd)  in dmz_unlock_map()  argument
368 	mutex_unlock(&zmd->map_lock);  in dmz_unlock_map()
378 void dmz_lock_metadata(struct dmz_metadata *zmd)  in dmz_lock_metadata()  argument
380 	down_read(&zmd->mblk_sem);  in dmz_lock_metadata()
383 void dmz_unlock_metadata(struct dmz_metadata *zmd)  in dmz_unlock_metadata()  argument
385 	up_read(&zmd->mblk_sem);  in dmz_unlock_metadata()
393 void dmz_lock_flush(struct dmz_metadata *zmd)  in dmz_lock_flush()  argument
395 	mutex_lock(&zmd->mblk_flush_lock);  in dmz_lock_flush()
398 void dmz_unlock_flush(struct dmz_metadata *zmd)  in dmz_unlock_flush()  argument
400 	mutex_unlock(&zmd->mblk_flush_lock);  in dmz_unlock_flush()
406 static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,  in dmz_alloc_mblock()  argument
412 	if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {  in dmz_alloc_mblock()
413 		spin_lock(&zmd->mblk_lock);  in dmz_alloc_mblock()
414 		mblk = list_first_entry_or_null(&zmd->mblk_lru_list,  in dmz_alloc_mblock()
418 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_alloc_mblock()
421 		spin_unlock(&zmd->mblk_lock);  in dmz_alloc_mblock()
444 	atomic_inc(&zmd->nr_mblks);  in dmz_alloc_mblock()
452 static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_free_mblock()  argument
457 	atomic_dec(&zmd->nr_mblks);  in dmz_free_mblock()
463 static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_insert_mblock()  argument
465 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_insert_mblock()
485 static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,  in dmz_get_mblock_fast()  argument
488 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_get_mblock_fast()
537 static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,  in dmz_get_mblock_slow()  argument
541 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;  in dmz_get_mblock_slow()
542 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock_slow()
549 	mblk = dmz_alloc_mblock(zmd, mblk_no);  in dmz_get_mblock_slow()
556 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
562 	m = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock_slow()
564 		spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
565 		dmz_free_mblock(zmd, mblk);  in dmz_get_mblock_slow()
572 	dmz_insert_mblock(zmd, mblk);  in dmz_get_mblock_slow()
574 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
589 static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,  in dmz_shrink_mblock_cache()  argument
595 	if (!zmd->max_nr_mblks)  in dmz_shrink_mblock_cache()
598 	while (!list_empty(&zmd->mblk_lru_list) &&  in dmz_shrink_mblock_cache()
599 	       atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&  in dmz_shrink_mblock_cache()
601 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_shrink_mblock_cache()
604 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_shrink_mblock_cache()
605 		dmz_free_mblock(zmd, mblk);  in dmz_shrink_mblock_cache()
618 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_count()  local
620 	return atomic_read(&zmd->nr_mblks);  in dmz_mblock_shrinker_count()
629 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_scan()  local
632 	spin_lock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
633 	count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);  in dmz_mblock_shrinker_scan()
634 	spin_unlock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
642 static void dmz_release_mblock(struct dmz_metadata *zmd,  in dmz_release_mblock()  argument
649 	spin_lock(&zmd->mblk_lock);  in dmz_release_mblock()
654 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_release_mblock()
655 			dmz_free_mblock(zmd, mblk);  in dmz_release_mblock()
657 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_release_mblock()
658 			dmz_shrink_mblock_cache(zmd, 1);  in dmz_release_mblock()
662 	spin_unlock(&zmd->mblk_lock);  in dmz_release_mblock()
669 static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,  in dmz_get_mblock()  argument
673 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock()
676 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock()
677 	mblk = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock()
678 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock()
682 		mblk = dmz_get_mblock_slow(zmd, mblk_no);  in dmz_get_mblock()
691 		dmz_release_mblock(zmd, mblk);  in dmz_get_mblock()
702 static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_dirty_mblock()  argument
704 	spin_lock(&zmd->mblk_lock);  in dmz_dirty_mblock()
706 		list_add_tail(&mblk->link, &zmd->mblk_dirty_list);  in dmz_dirty_mblock()
707 	spin_unlock(&zmd->mblk_lock);  in dmz_dirty_mblock()
713 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,  in dmz_write_mblock()  argument
716 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_mblock()
717 	sector_t block = zmd->sb[set].block + mblk->no;  in dmz_write_mblock()
767 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)  in dmz_write_sb()  argument
769 	struct dmz_mblock *mblk = zmd->sb[set].mblk;  in dmz_write_sb()
770 	struct dmz_super *sb = zmd->sb[set].sb;  in dmz_write_sb()
771 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_sb()
773 	u64 sb_gen = zmd->sb_gen + 1;  in dmz_write_sb()
778 	sb->version = cpu_to_le32(zmd->sb_version);  in dmz_write_sb()
779 	if (zmd->sb_version > 1) {  in dmz_write_sb()
781 		export_uuid(sb->dmz_uuid, &zmd->uuid);  in dmz_write_sb()
782 		memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);  in dmz_write_sb()
793 	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;  in dmz_write_sb()
795 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);  in dmz_write_sb()
796 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);  in dmz_write_sb()
797 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);  in dmz_write_sb()
799 	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);  in dmz_write_sb()
800 	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);  in dmz_write_sb()
805 	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,  in dmz_write_sb()
816 static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_write_dirty_mblocks()  argument
821 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_dirty_mblocks()
828 		ret = dmz_write_mblock(zmd, mblk, set);  in dmz_write_dirty_mblocks()
859 static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_log_dirty_mblocks()  argument
862 	unsigned int log_set = zmd->mblk_primary ^ 0x1;  in dmz_log_dirty_mblocks()
866 	ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);  in dmz_log_dirty_mblocks()
874 	ret = dmz_write_sb(zmd, log_set);  in dmz_log_dirty_mblocks()
884 int dmz_flush_metadata(struct dmz_metadata *zmd)  in dmz_flush_metadata()  argument
891 	if (WARN_ON(!zmd))  in dmz_flush_metadata()
901 	down_write(&zmd->mblk_sem);  in dmz_flush_metadata()
902 	dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_flush_metadata()
908 	dmz_lock_flush(zmd);  in dmz_flush_metadata()
916 	spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
917 	list_splice_init(&zmd->mblk_dirty_list, &write_list);  in dmz_flush_metadata()
918 	spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
931 	ret = dmz_log_dirty_mblocks(zmd, &write_list);  in dmz_flush_metadata()
939 	ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);  in dmz_flush_metadata()
943 	ret = dmz_write_sb(zmd, zmd->mblk_primary);  in dmz_flush_metadata()
951 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
954 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_flush_metadata()
955 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
958 	zmd->sb_gen++;  in dmz_flush_metadata()
960 	dmz_unlock_flush(zmd);  in dmz_flush_metadata()
961 	up_write(&zmd->mblk_sem);  in dmz_flush_metadata()
967 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
968 		list_splice(&write_list, &zmd->mblk_dirty_list);  in dmz_flush_metadata()
969 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
979 static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,  in dmz_check_sb()  argument
994 	zmd->sb_version = le32_to_cpu(sb->version);  in dmz_check_sb()
995 	if (zmd->sb_version > DMZ_META_VER) {  in dmz_check_sb()
997 			    DMZ_META_VER, zmd->sb_version);  in dmz_check_sb()
1000 	if (zmd->sb_version < 2 && tertiary) {  in dmz_check_sb()
1016 	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift) {  in dmz_check_sb()
1018 			    sb_block, (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);  in dmz_check_sb()
1021 	if (zmd->sb_version > 1) {  in dmz_check_sb()
1028 		} else if (uuid_is_null(&zmd->uuid)) {  in dmz_check_sb()
1029 			uuid_copy(&zmd->uuid, &sb_uuid);  in dmz_check_sb()
1030 		} else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {  in dmz_check_sb()
1032 				    &sb_uuid, &zmd->uuid);  in dmz_check_sb()
1035 		if (!strlen(zmd->label))  in dmz_check_sb()
1036 			memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);  in dmz_check_sb()
1037 		else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {  in dmz_check_sb()
1039 				    sb->dmz_label, zmd->label);  in dmz_check_sb()
1060 	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)  in dmz_check_sb()
1061 		>> zmd->zone_nr_blocks_shift;  in dmz_check_sb()
1063 	    (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||  in dmz_check_sb()
1064 	    (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {  in dmz_check_sb()
1070 	    le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {  in dmz_check_sb()
1075 	nr_data_zones = zmd->nr_useable_zones -  in dmz_check_sb()
1084 	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);  in dmz_check_sb()
1085 	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);  in dmz_check_sb()
1086 	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);  in dmz_check_sb()
1087 	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);  in dmz_check_sb()
1088 	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);  in dmz_check_sb()
1089 	zmd->nr_meta_zones = nr_meta_zones;  in dmz_check_sb()
1090 	zmd->nr_data_zones = nr_data_zones;  in dmz_check_sb()
1098 static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_read_sb()  argument
1100 	dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu",  in dmz_read_sb()
1112 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)  in dmz_lookup_secondary_sb()  argument
1114 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_lookup_secondary_sb()
1116 	unsigned int zone_id = zmd->sb[0].zone->id;  in dmz_lookup_secondary_sb()
1120 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_lookup_secondary_sb()
1124 	zmd->sb[1].mblk = mblk;  in dmz_lookup_secondary_sb()
1125 	zmd->sb[1].sb = mblk->data;  in dmz_lookup_secondary_sb()
1128 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;  in dmz_lookup_secondary_sb()
1129 	zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);  in dmz_lookup_secondary_sb()
1130 	zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_lookup_secondary_sb()
1131 	for (i = 1; i < zmd->nr_rnd_zones; i++) {  in dmz_lookup_secondary_sb()
1132 		if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)  in dmz_lookup_secondary_sb()
1134 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)  in dmz_lookup_secondary_sb()
1136 		zmd->sb[1].block += zone_nr_blocks;  in dmz_lookup_secondary_sb()
1137 		zmd->sb[1].zone = dmz_get(zmd, zone_id + i);  in dmz_lookup_secondary_sb()
1140 	dmz_free_mblock(zmd, mblk);  in dmz_lookup_secondary_sb()
1141 	zmd->sb[1].mblk = NULL;  in dmz_lookup_secondary_sb()
1142 	zmd->sb[1].zone = NULL;  in dmz_lookup_secondary_sb()
1143 	zmd->sb[1].dev = NULL;  in dmz_lookup_secondary_sb()
1151 static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_get_sb()  argument
1157 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_get_sb()
1165 	ret = dmz_read_sb(zmd, sb, set);  in dmz_get_sb()
1167 		dmz_free_mblock(zmd, mblk);  in dmz_get_sb()
1178 static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)  in dmz_recover_mblocks()  argument
1184 	dmz_dev_warn(zmd->sb[dst_set].dev,  in dmz_recover_mblocks()
1188 		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_recover_mblocks()
1190 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_recover_mblocks()
1197 	for (i = 1; i < zmd->nr_meta_blocks; i++) {  in dmz_recover_mblocks()
1198 		ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,  in dmz_recover_mblocks()
1199 				     zmd->sb[src_set].block + i, page);  in dmz_recover_mblocks()
1202 		ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,  in dmz_recover_mblocks()
1203 				     zmd->sb[dst_set].block + i, page);  in dmz_recover_mblocks()
1209 	if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1210 		zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);  in dmz_recover_mblocks()
1211 		if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1215 		zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;  in dmz_recover_mblocks()
1218 	ret = dmz_write_sb(zmd, dst_set);  in dmz_recover_mblocks()
1228 static int dmz_load_sb(struct dmz_metadata *zmd)  in dmz_load_sb()  argument
1234 	if (!zmd->sb[0].zone) {  in dmz_load_sb()
1235 		dmz_zmd_err(zmd, "Primary super block zone not set");  in dmz_load_sb()
1240 	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_load_sb()
1241 	zmd->sb[0].dev = zmd->sb[0].zone->dev;  in dmz_load_sb()
1242 	ret = dmz_get_sb(zmd, &zmd->sb[0], 0);  in dmz_load_sb()
1244 		dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");  in dmz_load_sb()
1248 	ret = dmz_check_sb(zmd, &zmd->sb[0], false);  in dmz_load_sb()
1253 		if (!zmd->sb[1].zone) {  in dmz_load_sb()
1255 				zmd->sb[0].zone->id + zmd->nr_meta_zones;  in dmz_load_sb()
1257 			zmd->sb[1].zone = dmz_get(zmd, zone_id);  in dmz_load_sb()
1259 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_load_sb()
1260 		zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_load_sb()
1261 		ret = dmz_get_sb(zmd, &zmd->sb[1], 1);  in dmz_load_sb()
1263 		ret = dmz_lookup_secondary_sb(zmd);  in dmz_load_sb()
1266 		dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");  in dmz_load_sb()
1270 	ret = dmz_check_sb(zmd, &zmd->sb[1], false);  in dmz_load_sb()
1276 		dmz_zmd_err(zmd, "No valid super block found");  in dmz_load_sb()
1281 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);  in dmz_load_sb()
1283 		ret = dmz_recover_mblocks(zmd, 0);  in dmz_load_sb()
1285 			dmz_dev_err(zmd->sb[0].dev,  in dmz_load_sb()
1292 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);  in dmz_load_sb()
1294 		ret = dmz_recover_mblocks(zmd, 1);  in dmz_load_sb()
1297 			dmz_dev_err(zmd->sb[1].dev,  in dmz_load_sb()
1304 		zmd->sb_gen = sb_gen[0];  in dmz_load_sb()
1305 		zmd->mblk_primary = 0;  in dmz_load_sb()
1307 		zmd->sb_gen = sb_gen[1];  in dmz_load_sb()
1308 		zmd->mblk_primary = 1;  in dmz_load_sb()
1311 	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,  in dmz_load_sb()
1313 		      zmd->mblk_primary, zmd->sb_gen);  in dmz_load_sb()
1315 	if (zmd->sb_version > 1) {  in dmz_load_sb()
1322 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_load_sb()
1324 			sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);  in dmz_load_sb()
1325 			sb->dev = &zmd->dev[i];  in dmz_load_sb()
1333 			ret = dmz_get_sb(zmd, sb, i + 1);  in dmz_load_sb()
1337 				dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1340 			ret = dmz_check_sb(zmd, sb, true);  in dmz_load_sb()
1341 			dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1357 	struct dmz_metadata *zmd = dev->metadata;  in dmz_init_zone()  local
1361 	zone = dmz_insert(zmd, idx, dev);  in dmz_init_zone()
1365 	if (blkz->len != zmd->zone_nr_sectors) {  in dmz_init_zone()
1366 		if (zmd->sb_version > 1) {  in dmz_init_zone()
1404 		zmd->nr_useable_zones++;  in dmz_init_zone()
1406 			zmd->nr_rnd_zones++;  in dmz_init_zone()
1407 			if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {  in dmz_init_zone()
1409 				zmd->sb[0].zone = zone;  in dmz_init_zone()
1412 		if (zmd->nr_devs > 1 && num == 0) {  in dmz_init_zone()
1424 static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)  in dmz_emulate_zones()  argument
1432 		zone = dmz_insert(zmd, idx, dev);  in dmz_emulate_zones()
1437 		zmd->nr_cache_zones++;  in dmz_emulate_zones()
1438 		zmd->nr_useable_zones++;  in dmz_emulate_zones()
1439 		if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {  in dmz_emulate_zones()
1444 		zone_offset += zmd->zone_nr_sectors;  in dmz_emulate_zones()
1452 static void dmz_drop_zones(struct dmz_metadata *zmd)  in dmz_drop_zones()  argument
1456 	for (idx = 0; idx < zmd->nr_zones; idx++) {  in dmz_drop_zones()
1457 		struct dm_zone *zone = xa_load(&zmd->zones, idx);  in dmz_drop_zones()
1460 		xa_erase(&zmd->zones, idx);  in dmz_drop_zones()
1462 	xa_destroy(&zmd->zones);  in dmz_drop_zones()
1469 static int dmz_init_zones(struct dmz_metadata *zmd)  in dmz_init_zones()  argument
1472 	struct dmz_dev *zoned_dev = &zmd->dev[0];  in dmz_init_zones()
1475 	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;  in dmz_init_zones()
1476 	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);  in dmz_init_zones()
1477 	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);  in dmz_init_zones()
1478 	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);  in dmz_init_zones()
1479 	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;  in dmz_init_zones()
1480 	zmd->zone_nr_bitmap_blocks =  in dmz_init_zones()
1481 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);  in dmz_init_zones()
1482 	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,  in dmz_init_zones()
1486 	zmd->nr_zones = 0;  in dmz_init_zones()
1487 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1488 		struct dmz_dev *dev = &zmd->dev[i];  in dmz_init_zones()
1490 		dev->metadata = zmd;  in dmz_init_zones()
1491 		zmd->nr_zones += dev->nr_zones;  in dmz_init_zones()
1502 	if (!zmd->nr_zones) {  in dmz_init_zones()
1503 		DMERR("(%s): No zones found", zmd->devname);  in dmz_init_zones()
1506 	xa_init(&zmd->zones);  in dmz_init_zones()
1509 		zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);  in dmz_init_zones()
1511 	if (zmd->nr_devs > 1) {  in dmz_init_zones()
1512 		ret = dmz_emulate_zones(zmd, &zmd->dev[0]);  in dmz_init_zones()
1515 				zmd->devname, ret);  in dmz_init_zones()
1516 			dmz_drop_zones(zmd);  in dmz_init_zones()
1524 		zmd->sb[0].zone = dmz_get(zmd, 0);  in dmz_init_zones()
1526 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1527 			zoned_dev = &zmd->dev[i];  in dmz_init_zones()
1534 					zmd->devname, ret);  in dmz_init_zones()
1535 				dmz_drop_zones(zmd);  in dmz_init_zones()
1551 			zmd->devname, ret);  in dmz_init_zones()
1552 		dmz_drop_zones(zmd);  in dmz_init_zones()
1581 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_update_zone()  argument
1597 	ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,  in dmz_update_zone()
1617 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,  in dmz_handle_seq_write_err()  argument
1625 	ret = dmz_update_zone(zmd, zone);  in dmz_handle_seq_write_err()
1633 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_handle_seq_write_err()
1643 static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_reset_zone()  argument
1660 				       dmz_start_sect(zmd, zone),  in dmz_reset_zone()
1661 				       zmd->zone_nr_sectors, GFP_NOIO);  in dmz_reset_zone()
1676 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1681 static int dmz_load_mapping(struct dmz_metadata *zmd)  in dmz_load_mapping()  argument
1691 	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,  in dmz_load_mapping()
1693 	if (!zmd->map_mblk)  in dmz_load_mapping()
1697 	while (chunk < zmd->nr_chunks) {  in dmz_load_mapping()
1700 			dmap_mblk = dmz_get_mblock(zmd, i + 1);  in dmz_load_mapping()
1703 			zmd->map_mblk[i] = dmap_mblk;  in dmz_load_mapping()
1714 		if (dzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1715 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",  in dmz_load_mapping()
1720 		dzone = dmz_get(zmd, dzone_id);  in dmz_load_mapping()
1722 			dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",  in dmz_load_mapping()
1728 		dmz_get_zone_weight(zmd, dzone);  in dmz_load_mapping()
1731 			list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1742 		if (bzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1743 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",  in dmz_load_mapping()
1748 		bzone = dmz_get(zmd, bzone_id);  in dmz_load_mapping()
1750 			dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",  in dmz_load_mapping()
1755 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",  in dmz_load_mapping()
1765 		dmz_get_zone_weight(zmd, bzone);  in dmz_load_mapping()
1767 			list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1782 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_load_mapping()
1783 		dzone = dmz_get(zmd, i);  in dmz_load_mapping()
1792 			zmd->nr_cache++;  in dmz_load_mapping()
1807 			list_add_tail(&dzone->link, &zmd->unmap_cache_list);  in dmz_load_mapping()
1808 			atomic_inc(&zmd->unmap_nr_cache);  in dmz_load_mapping()
1813 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {  in dmz_load_mapping()
1814 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);  in dmz_load_mapping()
1816 			atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_load_mapping()
1831 static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,  in dmz_set_chunk_mapping()  argument
1834 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_set_chunk_mapping()
1840 	dmz_dirty_mblock(zmd, dmap_mblk);  in dmz_set_chunk_mapping()
1847 static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in __dmz_lru_zone()  argument
1858 		list_add_tail(&zone->link, &zmd->map_cache_list);  in __dmz_lru_zone()
1869 static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_lru_zone()  argument
1871 	__dmz_lru_zone(zmd, zone);  in dmz_lru_zone()
1873 		__dmz_lru_zone(zmd, zone->bzone);  in dmz_lru_zone()
1879 static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)  in dmz_wait_for_free_zones()  argument
1883 	prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);  in dmz_wait_for_free_zones()
1884 	dmz_unlock_map(zmd);  in dmz_wait_for_free_zones()
1885 	dmz_unlock_metadata(zmd);  in dmz_wait_for_free_zones()
1889 	dmz_lock_metadata(zmd);  in dmz_wait_for_free_zones()
1890 	dmz_lock_map(zmd);  in dmz_wait_for_free_zones()
1891 	finish_wait(&zmd->free_wq, &wait);  in dmz_wait_for_free_zones()
1924 static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_wait_for_reclaim()  argument
1926 	dmz_unlock_map(zmd);  in dmz_wait_for_reclaim()
1927 	dmz_unlock_metadata(zmd);  in dmz_wait_for_reclaim()
1931 	dmz_lock_metadata(zmd);  in dmz_wait_for_reclaim()
1932 	dmz_lock_map(zmd);  in dmz_wait_for_reclaim()
1938 static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_rnd_zone_for_reclaim()  argument
1946 	if (zmd->nr_cache) {  in dmz_get_rnd_zone_for_reclaim()
1947 		zone_list = &zmd->map_cache_list;  in dmz_get_rnd_zone_for_reclaim()
1950 			zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
1952 		zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
1997 static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_seq_zone_for_reclaim()  argument
2002 	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {  in dmz_get_seq_zone_for_reclaim()
2015 struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_zone_for_reclaim()  argument
2028 	dmz_lock_map(zmd);  in dmz_get_zone_for_reclaim()
2029 	if (list_empty(&zmd->reserved_seq_zones_list))  in dmz_get_zone_for_reclaim()
2030 		zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);  in dmz_get_zone_for_reclaim()
2032 		zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);  in dmz_get_zone_for_reclaim()
2033 	dmz_unlock_map(zmd);  in dmz_get_zone_for_reclaim()
2044 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,  in dmz_get_chunk_mapping()  argument
2047 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_get_chunk_mapping()
2053 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_mapping()
2055 	dmz_lock_map(zmd);  in dmz_get_chunk_mapping()
2068 		dzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_mapping()
2070 			if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_mapping()
2074 			dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_mapping()
2078 		dmz_map_zone(zmd, dzone, chunk);  in dmz_get_chunk_mapping()
2082 		dzone = dmz_get(zmd, dzone_id);  in dmz_get_chunk_mapping()
2094 			ret = dmz_handle_seq_write_err(zmd, dzone);  in dmz_get_chunk_mapping()
2109 		dmz_wait_for_reclaim(zmd, dzone);  in dmz_get_chunk_mapping()
2113 	dmz_lru_zone(zmd, dzone);  in dmz_get_chunk_mapping()
2115 	dmz_unlock_map(zmd);  in dmz_get_chunk_mapping()
2126 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)  in dmz_put_chunk_mapping()  argument
2130 	dmz_lock_map(zmd);  in dmz_put_chunk_mapping()
2135 			dmz_lru_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2138 			dmz_unmap_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2139 			dmz_free_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2147 		dmz_lru_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2150 		dmz_unmap_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2151 		dmz_free_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2154 	dmz_unlock_map(zmd);  in dmz_put_chunk_mapping()
2161 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,  in dmz_get_chunk_buffer()  argument
2165 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_buffer()
2167 	dmz_lock_map(zmd);  in dmz_get_chunk_buffer()
2174 	bzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_buffer()
2176 		if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_buffer()
2180 		dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_buffer()
2185 	dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);  in dmz_get_chunk_buffer()
2192 		list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_get_chunk_buffer()
2196 	dmz_unlock_map(zmd);  in dmz_get_chunk_buffer()
2205 struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,  in dmz_alloc_zone()  argument
2214 		for (i = 0; i < zmd->nr_devs; i++)  in dmz_alloc_zone()
2215 			dmz_schedule_reclaim(zmd->dev[i].reclaim);  in dmz_alloc_zone()
2221 		list = &zmd->unmap_cache_list;  in dmz_alloc_zone()
2223 		list = &zmd->dev[dev_idx].unmap_rnd_list;  in dmz_alloc_zone()
2225 		list = &zmd->dev[dev_idx].unmap_seq_list;  in dmz_alloc_zone()
2236 		if (i < zmd->nr_devs) {  in dmz_alloc_zone()
2237 			dev_idx = (dev_idx + 1) % zmd->nr_devs;  in dmz_alloc_zone()
2245 		zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,  in dmz_alloc_zone()
2249 			atomic_dec(&zmd->nr_reserved_seq_zones);  in dmz_alloc_zone()
2258 		atomic_dec(&zmd->unmap_nr_cache);  in dmz_alloc_zone()
2265 		dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);  in dmz_alloc_zone()
2270 		dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);  in dmz_alloc_zone()
2281 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_free_zone()  argument
2285 		dmz_reset_zone(zmd, zone);  in dmz_free_zone()
2289 		list_add_tail(&zone->link, &zmd->unmap_cache_list);  in dmz_free_zone()
2290 		atomic_inc(&zmd->unmap_nr_cache);  in dmz_free_zone()
2295 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);  in dmz_free_zone()
2296 		atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_free_zone()
2302 	wake_up_all(&zmd->free_wq);  in dmz_free_zone()
2309 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,  in dmz_map_zone()  argument
2313 	dmz_set_chunk_mapping(zmd, chunk, dzone->id,  in dmz_map_zone()
2317 		list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_map_zone()
2328 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_unmap_zone()  argument
2359 	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);  in dmz_unmap_zone()
2400 static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,  in dmz_get_bitmap()  argument
2404 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +  in dmz_get_bitmap()
2405 		(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +  in dmz_get_bitmap()
2408 	return dmz_get_mblock(zmd, bitmap_block);  in dmz_get_bitmap()
2414 int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_copy_valid_blocks()  argument
2421 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_copy_valid_blocks()
2422 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);  in dmz_copy_valid_blocks()
2425 		to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);  in dmz_copy_valid_blocks()
2427 			dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2432 		dmz_dirty_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2434 		dmz_release_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2435 		dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2437 		chunk_block += zmd->zone_bits_per_mblk;  in dmz_copy_valid_blocks()
2449 int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_merge_valid_blocks()  argument
2456 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_merge_valid_blocks()
2458 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);  in dmz_merge_valid_blocks()
2463 		ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);  in dmz_merge_valid_blocks()
2476 int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_validate_blocks()  argument
2480 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_validate_blocks()
2484 	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",  in dmz_validate_blocks()
2492 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_validate_blocks()
2498 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_validate_blocks()
2502 			dmz_dirty_mblock(zmd, mblk);  in dmz_validate_blocks()
2505 		dmz_release_mblock(zmd, mblk);  in dmz_validate_blocks()
2514 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",  in dmz_validate_blocks()
2557 int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_invalidate_blocks()  argument
2564 	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",  in dmz_invalidate_blocks()
2567 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_invalidate_blocks()
2571 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_invalidate_blocks()
2577 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_invalidate_blocks()
2582 			dmz_dirty_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2585 		dmz_release_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2594 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",  in dmz_invalidate_blocks()
2605 static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_test_block()  argument
2611 	WARN_ON(chunk_block >= zmd->zone_nr_blocks);  in dmz_test_block()
2614 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_test_block()
2622 	dmz_release_mblock(zmd, mblk);  in dmz_test_block()
2631 static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_to_next_set_block()  argument
2637 	unsigned int zone_bits = zmd->zone_bits_per_mblk;  in dmz_to_next_set_block()
2641 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_to_next_set_block()
2645 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_to_next_set_block()
2657 		dmz_release_mblock(zmd, mblk);  in dmz_to_next_set_block()
2674 int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_block_valid()  argument
2679 	valid = dmz_test_block(zmd, zone, chunk_block);  in dmz_block_valid()
2684 	return dmz_to_next_set_block(zmd, zone, chunk_block,  in dmz_block_valid()
2685 				     zmd->zone_nr_blocks - chunk_block, 0);  in dmz_block_valid()
2694 int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_first_valid_block()  argument
2700 	ret = dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2701 				    zmd->zone_nr_blocks - start_block, 1);  in dmz_first_valid_block()
2708 	return dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2709 				     zmd->zone_nr_blocks - start_block, 0);  in dmz_first_valid_block()
2743 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_get_zone_weight()  argument
2748 	unsigned int nr_blocks = zmd->zone_nr_blocks;  in dmz_get_zone_weight()
2754 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_get_zone_weight()
2763 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_get_zone_weight()
2766 		dmz_release_mblock(zmd, mblk);  in dmz_get_zone_weight()
2778 static void dmz_cleanup_metadata(struct dmz_metadata *zmd)  in dmz_cleanup_metadata()  argument
2785 	if (zmd->map_mblk) {  in dmz_cleanup_metadata()
2786 		for (i = 0; i < zmd->nr_map_blocks; i++)  in dmz_cleanup_metadata()
2787 			dmz_release_mblock(zmd, zmd->map_mblk[i]);  in dmz_cleanup_metadata()
2788 		kfree(zmd->map_mblk);  in dmz_cleanup_metadata()
2789 		zmd->map_mblk = NULL;  in dmz_cleanup_metadata()
2794 		if (zmd->sb[i].mblk) {  in dmz_cleanup_metadata()
2795 			dmz_free_mblock(zmd, zmd->sb[i].mblk);  in dmz_cleanup_metadata()
2796 			zmd->sb[i].mblk = NULL;  in dmz_cleanup_metadata()
2801 	while (!list_empty(&zmd->mblk_dirty_list)) {  in dmz_cleanup_metadata()
2802 		mblk = list_first_entry(&zmd->mblk_dirty_list,  in dmz_cleanup_metadata()
2804 		dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",  in dmz_cleanup_metadata()
2807 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2808 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2811 	while (!list_empty(&zmd->mblk_lru_list)) {  in dmz_cleanup_metadata()
2812 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_cleanup_metadata()
2815 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2816 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2820 	root = &zmd->mblk_rbtree;  in dmz_cleanup_metadata()
2822 		dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",  in dmz_cleanup_metadata()
2825 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2829 	dmz_drop_zones(zmd);  in dmz_cleanup_metadata()
2831 	mutex_destroy(&zmd->mblk_flush_lock);  in dmz_cleanup_metadata()
2832 	mutex_destroy(&zmd->map_lock);  in dmz_cleanup_metadata()
2835 static void dmz_print_dev(struct dmz_metadata *zmd, int num)  in dmz_print_dev()  argument
2837 	struct dmz_dev *dev = &zmd->dev[num];  in dmz_print_dev()
2845 	if (zmd->sb_version > 1) {  in dmz_print_dev()
2847 			dev->zone_offset << zmd->zone_nr_sectors_shift;  in dmz_print_dev()
2852 			     dev->nr_zones, (u64)zmd->zone_nr_sectors,  in dmz_print_dev()
2858 			     dev->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_print_dev()
2869 	struct dmz_metadata *zmd;  in dmz_ctr_metadata()  local
2874 	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);  in dmz_ctr_metadata()
2875 	if (!zmd)  in dmz_ctr_metadata()
2878 	strcpy(zmd->devname, devname);  in dmz_ctr_metadata()
2879 	zmd->dev = dev;  in dmz_ctr_metadata()
2880 	zmd->nr_devs = num_dev;  in dmz_ctr_metadata()
2881 	zmd->mblk_rbtree = RB_ROOT;  in dmz_ctr_metadata()
2882 	init_rwsem(&zmd->mblk_sem);  in dmz_ctr_metadata()
2883 	mutex_init(&zmd->mblk_flush_lock);  in dmz_ctr_metadata()
2884 	spin_lock_init(&zmd->mblk_lock);  in dmz_ctr_metadata()
2885 	INIT_LIST_HEAD(&zmd->mblk_lru_list);  in dmz_ctr_metadata()
2886 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);  in dmz_ctr_metadata()
2888 	mutex_init(&zmd->map_lock);  in dmz_ctr_metadata()
2890 	atomic_set(&zmd->unmap_nr_cache, 0);  in dmz_ctr_metadata()
2891 	INIT_LIST_HEAD(&zmd->unmap_cache_list);  in dmz_ctr_metadata()
2892 	INIT_LIST_HEAD(&zmd->map_cache_list);  in dmz_ctr_metadata()
2894 	atomic_set(&zmd->nr_reserved_seq_zones, 0);  in dmz_ctr_metadata()
2895 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);  in dmz_ctr_metadata()
2897 	init_waitqueue_head(&zmd->free_wq);  in dmz_ctr_metadata()
2900 	ret = dmz_init_zones(zmd);  in dmz_ctr_metadata()
2905 	ret = dmz_load_sb(zmd);  in dmz_ctr_metadata()
2910 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {  in dmz_ctr_metadata()
2911 		zone = dmz_get(zmd, zmd->sb[0].zone->id + i);  in dmz_ctr_metadata()
2913 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2919 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2927 	ret = dmz_load_mapping(zmd);  in dmz_ctr_metadata()
2937 	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;  in dmz_ctr_metadata()
2938 	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;  in dmz_ctr_metadata()
2939 	zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;  in dmz_ctr_metadata()
2940 	zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;  in dmz_ctr_metadata()
2941 	zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;  in dmz_ctr_metadata()
2944 	ret = register_shrinker(&zmd->mblk_shrinker, "dm-zoned-meta:(%u:%u)",  in dmz_ctr_metadata()
2948 		dmz_zmd_err(zmd, "Register metadata cache shrinker failed");  in dmz_ctr_metadata()
2952 	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);  in dmz_ctr_metadata()
2953 	for (i = 0; i < zmd->nr_devs; i++)  in dmz_ctr_metadata()
2954 		dmz_print_dev(zmd, i);  in dmz_ctr_metadata()
2956 	dmz_zmd_info(zmd, "  %u zones of %llu 512-byte logical sectors",  in dmz_ctr_metadata()
2957 		     zmd->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_ctr_metadata()
2958 	dmz_zmd_debug(zmd, "  %u metadata zones",  in dmz_ctr_metadata()
2959 		      zmd->nr_meta_zones * 2);  in dmz_ctr_metadata()
2960 	dmz_zmd_debug(zmd, "  %u data zones for %u chunks",  in dmz_ctr_metadata()
2961 		      zmd->nr_data_zones, zmd->nr_chunks);  in dmz_ctr_metadata()
2962 	dmz_zmd_debug(zmd, "    %u cache zones (%u unmapped)",  in dmz_ctr_metadata()
2963 		      zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));  in dmz_ctr_metadata()
2964 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_ctr_metadata()
2965 		dmz_zmd_debug(zmd, "    %u random zones (%u unmapped)",  in dmz_ctr_metadata()
2966 			      dmz_nr_rnd_zones(zmd, i),  in dmz_ctr_metadata()
2967 			      dmz_nr_unmap_rnd_zones(zmd, i));  in dmz_ctr_metadata()
2968 		dmz_zmd_debug(zmd, "    %u sequential zones (%u unmapped)",  in dmz_ctr_metadata()
2969 			      dmz_nr_seq_zones(zmd, i),  in dmz_ctr_metadata()
2970 			      dmz_nr_unmap_seq_zones(zmd, i));  in dmz_ctr_metadata()
2972 	dmz_zmd_debug(zmd, "  %u reserved sequential data zones",  in dmz_ctr_metadata()
2973 		      zmd->nr_reserved_seq);  in dmz_ctr_metadata()
2974 	dmz_zmd_debug(zmd, "Format:");  in dmz_ctr_metadata()
2975 	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",  in dmz_ctr_metadata()
2976 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);  in dmz_ctr_metadata()
2977 	dmz_zmd_debug(zmd, "  %u data zone mapping blocks",  in dmz_ctr_metadata()
2978 		      zmd->nr_map_blocks);  in dmz_ctr_metadata()
2979 	dmz_zmd_debug(zmd, "  %u bitmap blocks",  in dmz_ctr_metadata()
2980 		      zmd->nr_bitmap_blocks);  in dmz_ctr_metadata()
2982 	*metadata = zmd;  in dmz_ctr_metadata()
2986 	dmz_cleanup_metadata(zmd);  in dmz_ctr_metadata()
2987 	kfree(zmd);  in dmz_ctr_metadata()
2996 void dmz_dtr_metadata(struct dmz_metadata *zmd)  in dmz_dtr_metadata()  argument
2998 	unregister_shrinker(&zmd->mblk_shrinker);  in dmz_dtr_metadata()
2999 	dmz_cleanup_metadata(zmd);  in dmz_dtr_metadata()
3000 	kfree(zmd);  in dmz_dtr_metadata()
3006 int dmz_resume_metadata(struct dmz_metadata *zmd)  in dmz_resume_metadata()  argument
3014 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_resume_metadata()
3015 		zone = dmz_get(zmd, i);  in dmz_resume_metadata()
3017 			dmz_zmd_err(zmd, "Unable to get zone %u", i);  in dmz_resume_metadata()
3022 		ret = dmz_update_zone(zmd, zone);  in dmz_resume_metadata()
3024 			dmz_zmd_err(zmd, "Broken zone %u", i);  in dmz_resume_metadata()
3029 			dmz_zmd_warn(zmd, "Zone %u is offline", i);  in dmz_resume_metadata()
3037 			dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",  in dmz_resume_metadata()
3040 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_resume_metadata()
3041 					      zmd->zone_nr_blocks - zone->wp_block);  in dmz_resume_metadata()