207 #define dmz_zmd_info(zmd, format, args...)	\  argument
208 	DMINFO("(%s): " format, (zmd)->label, ## args)
210 #define dmz_zmd_err(zmd, format, args...)	\  argument
211 	DMERR("(%s): " format, (zmd)->label, ## args)
213 #define dmz_zmd_warn(zmd, format, args...)	\  argument
214 	DMWARN("(%s): " format, (zmd)->label, ## args)
216 #define dmz_zmd_debug(zmd, format, args...)	\  argument
217 	DMDEBUG("(%s): " format, (zmd)->label, ## args)
221 static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_dev_zone_id()  argument
229 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_sect()  argument
231 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_sect()
233 	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;  in dmz_start_sect()
236 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_block()  argument
238 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_block()
240 	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;  in dmz_start_block()
243 unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks()  argument
245 	return zmd->zone_nr_blocks;  in dmz_zone_nr_blocks()
248 unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks_shift()  argument
250 	return zmd->zone_nr_blocks_shift;  in dmz_zone_nr_blocks_shift()
253 unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors()  argument
255 	return zmd->zone_nr_sectors;  in dmz_zone_nr_sectors()
258 unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors_shift()  argument
260 	return zmd->zone_nr_sectors_shift;  in dmz_zone_nr_sectors_shift()
263 unsigned int dmz_nr_zones(struct dmz_metadata *zmd)  in dmz_nr_zones()  argument
265 	return zmd->nr_zones;  in dmz_nr_zones()
268 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)  in dmz_nr_chunks()  argument
270 	return zmd->nr_chunks;  in dmz_nr_chunks()
273 unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_rnd_zones()  argument
275 	return zmd->dev[idx].nr_rnd;  in dmz_nr_rnd_zones()
278 unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_rnd_zones()  argument
280 	return atomic_read(&zmd->dev[idx].unmap_nr_rnd);  in dmz_nr_unmap_rnd_zones()
283 unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_cache_zones()  argument
285 	return zmd->nr_cache;  in dmz_nr_cache_zones()
288 unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_unmap_cache_zones()  argument
290 	return atomic_read(&zmd->unmap_nr_cache);  in dmz_nr_unmap_cache_zones()
293 unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_seq_zones()  argument
295 	return zmd->dev[idx].nr_seq;  in dmz_nr_seq_zones()
298 unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_seq_zones()  argument
300 	return atomic_read(&zmd->dev[idx].unmap_nr_seq);  in dmz_nr_unmap_seq_zones()
303 static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)  in dmz_get()  argument
305 	return xa_load(&zmd->zones, zone_id);  in dmz_get()
308 static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,  in dmz_insert()  argument
316 	if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {  in dmz_insert()
330 const char *dmz_metadata_label(struct dmz_metadata *zmd)  in dmz_metadata_label()  argument
332 	return (const char *)zmd->label;  in dmz_metadata_label()
335 bool dmz_check_dev(struct dmz_metadata *zmd)  in dmz_check_dev()  argument
339 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_check_dev()
340 		if (!dmz_check_bdev(&zmd->dev[i]))  in dmz_check_dev()
346 bool dmz_dev_is_dying(struct dmz_metadata *zmd)  in dmz_dev_is_dying()  argument
350 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_dev_is_dying()
351 		if (dmz_bdev_is_dying(&zmd->dev[i]))  in dmz_dev_is_dying()
361 void dmz_lock_map(struct dmz_metadata *zmd)  in dmz_lock_map()  argument
363 	mutex_lock(&zmd->map_lock);  in dmz_lock_map()
366 void dmz_unlock_map(struct dmz_metadata *zmd)  in dmz_unlock_map()  argument
368 	mutex_unlock(&zmd->map_lock);  in dmz_unlock_map()
378 void dmz_lock_metadata(struct dmz_metadata *zmd)  in dmz_lock_metadata()  argument
380 	down_read(&zmd->mblk_sem);  in dmz_lock_metadata()
383 void dmz_unlock_metadata(struct dmz_metadata *zmd)  in dmz_unlock_metadata()  argument
385 	up_read(&zmd->mblk_sem);  in dmz_unlock_metadata()
393 void dmz_lock_flush(struct dmz_metadata *zmd)  in dmz_lock_flush()  argument
395 	mutex_lock(&zmd->mblk_flush_lock);  in dmz_lock_flush()
398 void dmz_unlock_flush(struct dmz_metadata *zmd)  in dmz_unlock_flush()  argument
400 	mutex_unlock(&zmd->mblk_flush_lock);  in dmz_unlock_flush()
406 static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,  in dmz_alloc_mblock()  argument
412 	if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {  in dmz_alloc_mblock()
413 		spin_lock(&zmd->mblk_lock);  in dmz_alloc_mblock()
414 		mblk = list_first_entry_or_null(&zmd->mblk_lru_list,  in dmz_alloc_mblock()
418 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_alloc_mblock()
421 		spin_unlock(&zmd->mblk_lock);  in dmz_alloc_mblock()
444 	atomic_inc(&zmd->nr_mblks);  in dmz_alloc_mblock()
452 static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_free_mblock()  argument
457 	atomic_dec(&zmd->nr_mblks);  in dmz_free_mblock()
463 static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_insert_mblock()  argument
465 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_insert_mblock()
485 static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,  in dmz_get_mblock_fast()  argument
488 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_get_mblock_fast()
537 static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,  in dmz_get_mblock_slow()  argument
541 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;  in dmz_get_mblock_slow()
542 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock_slow()
549 	mblk = dmz_alloc_mblock(zmd, mblk_no);  in dmz_get_mblock_slow()
556 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
562 	m = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock_slow()
564 		spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
565 		dmz_free_mblock(zmd, mblk);  in dmz_get_mblock_slow()
572 	dmz_insert_mblock(zmd, mblk);  in dmz_get_mblock_slow()
574 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
589 static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,  in dmz_shrink_mblock_cache()  argument
595 	if (!zmd->max_nr_mblks)  in dmz_shrink_mblock_cache()
598 	while (!list_empty(&zmd->mblk_lru_list) &&  in dmz_shrink_mblock_cache()
599 	       atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&  in dmz_shrink_mblock_cache()
601 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_shrink_mblock_cache()
604 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_shrink_mblock_cache()
605 		dmz_free_mblock(zmd, mblk);  in dmz_shrink_mblock_cache()
618 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_count()  local
620 	return atomic_read(&zmd->nr_mblks);  in dmz_mblock_shrinker_count()
629 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_scan()  local
632 	spin_lock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
633 	count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);  in dmz_mblock_shrinker_scan()
634 	spin_unlock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
642 static void dmz_release_mblock(struct dmz_metadata *zmd,  in dmz_release_mblock()  argument
649 	spin_lock(&zmd->mblk_lock);  in dmz_release_mblock()
654 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_release_mblock()
655 			dmz_free_mblock(zmd, mblk);  in dmz_release_mblock()
657 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_release_mblock()
658 			dmz_shrink_mblock_cache(zmd, 1);  in dmz_release_mblock()
662 	spin_unlock(&zmd->mblk_lock);  in dmz_release_mblock()
669 static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,  in dmz_get_mblock()  argument
673 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock()
676 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock()
677 	mblk = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock()
678 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock()
682 		mblk = dmz_get_mblock_slow(zmd, mblk_no);  in dmz_get_mblock()
691 		dmz_release_mblock(zmd, mblk);  in dmz_get_mblock()
702 static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_dirty_mblock()  argument
704 	spin_lock(&zmd->mblk_lock);  in dmz_dirty_mblock()
706 		list_add_tail(&mblk->link, &zmd->mblk_dirty_list);  in dmz_dirty_mblock()
707 	spin_unlock(&zmd->mblk_lock);  in dmz_dirty_mblock()
713 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,  in dmz_write_mblock()  argument
716 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_mblock()
717 	sector_t block = zmd->sb[set].block + mblk->no;  in dmz_write_mblock()
767 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)  in dmz_write_sb()  argument
769 	struct dmz_mblock *mblk = zmd->sb[set].mblk;  in dmz_write_sb()
770 	struct dmz_super *sb = zmd->sb[set].sb;  in dmz_write_sb()
771 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_sb()
773 	u64 sb_gen = zmd->sb_gen + 1;  in dmz_write_sb()
778 	sb->version = cpu_to_le32(zmd->sb_version);  in dmz_write_sb()
779 	if (zmd->sb_version > 1) {  in dmz_write_sb()
781 		export_uuid(sb->dmz_uuid, &zmd->uuid);  in dmz_write_sb()
782 		memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);  in dmz_write_sb()
793 	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;  in dmz_write_sb()
795 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);  in dmz_write_sb()
796 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);  in dmz_write_sb()
797 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);  in dmz_write_sb()
799 	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);  in dmz_write_sb()
800 	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);  in dmz_write_sb()
805 	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,  in dmz_write_sb()
816 static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_write_dirty_mblocks()  argument
821 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_dirty_mblocks()
828 		ret = dmz_write_mblock(zmd, mblk, set);  in dmz_write_dirty_mblocks()
859 static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_log_dirty_mblocks()  argument
862 	unsigned int log_set = zmd->mblk_primary ^ 0x1;  in dmz_log_dirty_mblocks()
866 	ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);  in dmz_log_dirty_mblocks()
874 	ret = dmz_write_sb(zmd, log_set);  in dmz_log_dirty_mblocks()
884 int dmz_flush_metadata(struct dmz_metadata *zmd)  in dmz_flush_metadata()  argument
891 	if (WARN_ON(!zmd))  in dmz_flush_metadata()
901 	down_write(&zmd->mblk_sem);  in dmz_flush_metadata()
902 	dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_flush_metadata()
908 	dmz_lock_flush(zmd);  in dmz_flush_metadata()
916 	spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
917 	list_splice_init(&zmd->mblk_dirty_list, &write_list);  in dmz_flush_metadata()
918 	spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
931 	ret = dmz_log_dirty_mblocks(zmd, &write_list);  in dmz_flush_metadata()
939 	ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);  in dmz_flush_metadata()
943 	ret = dmz_write_sb(zmd, zmd->mblk_primary);  in dmz_flush_metadata()
951 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
954 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_flush_metadata()
955 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
958 	zmd->sb_gen++;  in dmz_flush_metadata()
960 	dmz_unlock_flush(zmd);  in dmz_flush_metadata()
961 	up_write(&zmd->mblk_sem);  in dmz_flush_metadata()
967 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
968 		list_splice(&write_list, &zmd->mblk_dirty_list);  in dmz_flush_metadata()
969 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
979 static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,  in dmz_check_sb()  argument
994 	zmd->sb_version = le32_to_cpu(sb->version);  in dmz_check_sb()
995 	if (zmd->sb_version > DMZ_META_VER) {  in dmz_check_sb()
997 			    DMZ_META_VER, zmd->sb_version);  in dmz_check_sb()
1000 	if (zmd->sb_version < 2 && tertiary) {  in dmz_check_sb()
1016 	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {  in dmz_check_sb()
1020 			    (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);  in dmz_check_sb()
1023 	if (zmd->sb_version > 1) {  in dmz_check_sb()
1030 		} else if (uuid_is_null(&zmd->uuid)) {  in dmz_check_sb()
1031 			uuid_copy(&zmd->uuid, &sb_uuid);  in dmz_check_sb()
1032 		} else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {  in dmz_check_sb()
1035 				    &sb_uuid, &zmd->uuid);  in dmz_check_sb()
1038 		if (!strlen(zmd->label))  in dmz_check_sb()
1039 			memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);  in dmz_check_sb()
1040 		else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {  in dmz_check_sb()
1043 				    sb->dmz_label, zmd->label);  in dmz_check_sb()
1064 	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)  in dmz_check_sb()
1065 		>> zmd->zone_nr_blocks_shift;  in dmz_check_sb()
1067 	    (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||  in dmz_check_sb()
1068 	    (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {  in dmz_check_sb()
1074 	    le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {  in dmz_check_sb()
1079 	nr_data_zones = zmd->nr_useable_zones -  in dmz_check_sb()
1088 	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);  in dmz_check_sb()
1089 	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);  in dmz_check_sb()
1090 	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);  in dmz_check_sb()
1091 	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);  in dmz_check_sb()
1092 	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);  in dmz_check_sb()
1093 	zmd->nr_meta_zones = nr_meta_zones;  in dmz_check_sb()
1094 	zmd->nr_data_zones = nr_data_zones;  in dmz_check_sb()
1102 static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_read_sb()  argument
1104 	dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu",  in dmz_read_sb()
1116 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)  in dmz_lookup_secondary_sb()  argument
1118 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_lookup_secondary_sb()
1120 	unsigned int zone_id = zmd->sb[0].zone->id;  in dmz_lookup_secondary_sb()
1124 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_lookup_secondary_sb()
1128 	zmd->sb[1].mblk = mblk;  in dmz_lookup_secondary_sb()
1129 	zmd->sb[1].sb = mblk->data;  in dmz_lookup_secondary_sb()
1132 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;  in dmz_lookup_secondary_sb()
1133 	zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);  in dmz_lookup_secondary_sb()
1134 	zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_lookup_secondary_sb()
1135 	for (i = 1; i < zmd->nr_rnd_zones; i++) {  in dmz_lookup_secondary_sb()
1136 		if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)  in dmz_lookup_secondary_sb()
1138 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)  in dmz_lookup_secondary_sb()
1140 		zmd->sb[1].block += zone_nr_blocks;  in dmz_lookup_secondary_sb()
1141 		zmd->sb[1].zone = dmz_get(zmd, zone_id + i);  in dmz_lookup_secondary_sb()
1144 	dmz_free_mblock(zmd, mblk);  in dmz_lookup_secondary_sb()
1145 	zmd->sb[1].mblk = NULL;  in dmz_lookup_secondary_sb()
1146 	zmd->sb[1].zone = NULL;  in dmz_lookup_secondary_sb()
1147 	zmd->sb[1].dev = NULL;  in dmz_lookup_secondary_sb()
1155 static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_get_sb()  argument
1161 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_get_sb()
1169 	ret = dmz_read_sb(zmd, sb, set);  in dmz_get_sb()
1171 		dmz_free_mblock(zmd, mblk);  in dmz_get_sb()
1182 static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)  in dmz_recover_mblocks()  argument
1188 	dmz_dev_warn(zmd->sb[dst_set].dev,  in dmz_recover_mblocks()
1192 		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_recover_mblocks()
1194 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_recover_mblocks()
1201 	for (i = 1; i < zmd->nr_meta_blocks; i++) {  in dmz_recover_mblocks()
1202 		ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,  in dmz_recover_mblocks()
1203 				     zmd->sb[src_set].block + i, page);  in dmz_recover_mblocks()
1206 		ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,  in dmz_recover_mblocks()
1207 				     zmd->sb[dst_set].block + i, page);  in dmz_recover_mblocks()
1213 	if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1214 		zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);  in dmz_recover_mblocks()
1215 		if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1219 		zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;  in dmz_recover_mblocks()
1222 	ret = dmz_write_sb(zmd, dst_set);  in dmz_recover_mblocks()
1232 static int dmz_load_sb(struct dmz_metadata *zmd)  in dmz_load_sb()  argument
1238 	if (!zmd->sb[0].zone) {  in dmz_load_sb()
1239 		dmz_zmd_err(zmd, "Primary super block zone not set");  in dmz_load_sb()
1244 	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_load_sb()
1245 	zmd->sb[0].dev = zmd->sb[0].zone->dev;  in dmz_load_sb()
1246 	ret = dmz_get_sb(zmd, &zmd->sb[0], 0);  in dmz_load_sb()
1248 		dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");  in dmz_load_sb()
1252 	ret = dmz_check_sb(zmd, &zmd->sb[0], false);  in dmz_load_sb()
1257 		if (!zmd->sb[1].zone) {  in dmz_load_sb()
1259 				zmd->sb[0].zone->id + zmd->nr_meta_zones;  in dmz_load_sb()
1261 			zmd->sb[1].zone = dmz_get(zmd, zone_id);  in dmz_load_sb()
1263 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_load_sb()
1264 		zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_load_sb()
1265 		ret = dmz_get_sb(zmd, &zmd->sb[1], 1);  in dmz_load_sb()
1267 		ret = dmz_lookup_secondary_sb(zmd);  in dmz_load_sb()
1270 		dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");  in dmz_load_sb()
1274 	ret = dmz_check_sb(zmd, &zmd->sb[1], false);  in dmz_load_sb()
1280 		dmz_zmd_err(zmd, "No valid super block found");  in dmz_load_sb()
1285 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);  in dmz_load_sb()
1287 		ret = dmz_recover_mblocks(zmd, 0);  in dmz_load_sb()
1289 			dmz_dev_err(zmd->sb[0].dev,  in dmz_load_sb()
1296 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);  in dmz_load_sb()
1298 		ret = dmz_recover_mblocks(zmd, 1);  in dmz_load_sb()
1301 			dmz_dev_err(zmd->sb[1].dev,  in dmz_load_sb()
1308 		zmd->sb_gen = sb_gen[0];  in dmz_load_sb()
1309 		zmd->mblk_primary = 0;  in dmz_load_sb()
1311 		zmd->sb_gen = sb_gen[1];  in dmz_load_sb()
1312 		zmd->mblk_primary = 1;  in dmz_load_sb()
1315 	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,  in dmz_load_sb()
1317 		      zmd->mblk_primary, zmd->sb_gen);  in dmz_load_sb()
1319 	if (zmd->sb_version > 1) {  in dmz_load_sb()
1326 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_load_sb()
1328 			sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);  in dmz_load_sb()
1329 			sb->dev = &zmd->dev[i];  in dmz_load_sb()
1337 			ret = dmz_get_sb(zmd, sb, i + 1);  in dmz_load_sb()
1341 				dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1344 			ret = dmz_check_sb(zmd, sb, true);  in dmz_load_sb()
1345 			dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1361 	struct dmz_metadata *zmd = dev->metadata;  in dmz_init_zone()  local
1365 	zone = dmz_insert(zmd, idx, dev);  in dmz_init_zone()
1369 	if (blkz->len != zmd->zone_nr_sectors) {  in dmz_init_zone()
1370 		if (zmd->sb_version > 1) {  in dmz_init_zone()
1408 		zmd->nr_useable_zones++;  in dmz_init_zone()
1410 			zmd->nr_rnd_zones++;  in dmz_init_zone()
1411 			if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {  in dmz_init_zone()
1413 				zmd->sb[0].zone = zone;  in dmz_init_zone()
1416 		if (zmd->nr_devs > 1 && num == 0) {  in dmz_init_zone()
1428 static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)  in dmz_emulate_zones()  argument
1436 		zone = dmz_insert(zmd, idx, dev);  in dmz_emulate_zones()
1441 		zmd->nr_cache_zones++;  in dmz_emulate_zones()
1442 		zmd->nr_useable_zones++;  in dmz_emulate_zones()
1443 		if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {  in dmz_emulate_zones()
1448 		zone_offset += zmd->zone_nr_sectors;  in dmz_emulate_zones()
1456 static void dmz_drop_zones(struct dmz_metadata *zmd)  in dmz_drop_zones()  argument
1460 	for(idx = 0; idx < zmd->nr_zones; idx++) {  in dmz_drop_zones()
1461 		struct dm_zone *zone = xa_load(&zmd->zones, idx);  in dmz_drop_zones()
1464 		xa_erase(&zmd->zones, idx);  in dmz_drop_zones()
1466 	xa_destroy(&zmd->zones);  in dmz_drop_zones()
1473 static int dmz_init_zones(struct dmz_metadata *zmd)  in dmz_init_zones()  argument
1476 	struct dmz_dev *zoned_dev = &zmd->dev[0];  in dmz_init_zones()
1479 	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;  in dmz_init_zones()
1480 	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);  in dmz_init_zones()
1481 	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);  in dmz_init_zones()
1482 	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);  in dmz_init_zones()
1483 	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;  in dmz_init_zones()
1484 	zmd->zone_nr_bitmap_blocks =  in dmz_init_zones()
1485 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);  in dmz_init_zones()
1486 	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,  in dmz_init_zones()
1490 	zmd->nr_zones = 0;  in dmz_init_zones()
1491 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1492 		struct dmz_dev *dev = &zmd->dev[i];  in dmz_init_zones()
1494 		dev->metadata = zmd;  in dmz_init_zones()
1495 		zmd->nr_zones += dev->nr_zones;  in dmz_init_zones()
1506 	if (!zmd->nr_zones) {  in dmz_init_zones()
1507 		DMERR("(%s): No zones found", zmd->devname);  in dmz_init_zones()
1510 	xa_init(&zmd->zones);  in dmz_init_zones()
1513 		zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);  in dmz_init_zones()
1515 	if (zmd->nr_devs > 1) {  in dmz_init_zones()
1516 		ret = dmz_emulate_zones(zmd, &zmd->dev[0]);  in dmz_init_zones()
1519 				zmd->devname, ret);  in dmz_init_zones()
1520 			dmz_drop_zones(zmd);  in dmz_init_zones()
1528 		zmd->sb[0].zone = dmz_get(zmd, 0);  in dmz_init_zones()
1530 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1531 			zoned_dev = &zmd->dev[i];  in dmz_init_zones()
1538 					zmd->devname, ret);  in dmz_init_zones()
1539 				dmz_drop_zones(zmd);  in dmz_init_zones()
1555 			zmd->devname, ret);  in dmz_init_zones()
1556 		dmz_drop_zones(zmd);  in dmz_init_zones()
1585 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_update_zone()  argument
1601 	ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,  in dmz_update_zone()
1621 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,  in dmz_handle_seq_write_err()  argument
1629 	ret = dmz_update_zone(zmd, zone);  in dmz_handle_seq_write_err()
1637 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_handle_seq_write_err()
1647 static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_reset_zone()  argument
1664 				       dmz_start_sect(zmd, zone),  in dmz_reset_zone()
1665 				       zmd->zone_nr_sectors, GFP_NOIO);  in dmz_reset_zone()
1680 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1685 static int dmz_load_mapping(struct dmz_metadata *zmd)  in dmz_load_mapping()  argument
1695 	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,  in dmz_load_mapping()
1697 	if (!zmd->map_mblk)  in dmz_load_mapping()
1701 	while (chunk < zmd->nr_chunks) {  in dmz_load_mapping()
1704 			dmap_mblk = dmz_get_mblock(zmd, i + 1);  in dmz_load_mapping()
1707 			zmd->map_mblk[i] = dmap_mblk;  in dmz_load_mapping()
1718 		if (dzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1719 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",  in dmz_load_mapping()
1724 		dzone = dmz_get(zmd, dzone_id);  in dmz_load_mapping()
1726 			dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",  in dmz_load_mapping()
1732 		dmz_get_zone_weight(zmd, dzone);  in dmz_load_mapping()
1735 			list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1746 		if (bzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1747 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",  in dmz_load_mapping()
1752 		bzone = dmz_get(zmd, bzone_id);  in dmz_load_mapping()
1754 			dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",  in dmz_load_mapping()
1759 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",  in dmz_load_mapping()
1769 		dmz_get_zone_weight(zmd, bzone);  in dmz_load_mapping()
1771 			list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1786 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_load_mapping()
1787 		dzone = dmz_get(zmd, i);  in dmz_load_mapping()
1796 			zmd->nr_cache++;  in dmz_load_mapping()
1811 			list_add_tail(&dzone->link, &zmd->unmap_cache_list);  in dmz_load_mapping()
1812 			atomic_inc(&zmd->unmap_nr_cache);  in dmz_load_mapping()
1817 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {  in dmz_load_mapping()
1818 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);  in dmz_load_mapping()
1820 			atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_load_mapping()
1835 static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,  in dmz_set_chunk_mapping()  argument
1838 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_set_chunk_mapping()
1844 	dmz_dirty_mblock(zmd, dmap_mblk);  in dmz_set_chunk_mapping()
1851 static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in __dmz_lru_zone()  argument
1862 		list_add_tail(&zone->link, &zmd->map_cache_list);  in __dmz_lru_zone()
1873 static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_lru_zone()  argument
1875 	__dmz_lru_zone(zmd, zone);  in dmz_lru_zone()
1877 		__dmz_lru_zone(zmd, zone->bzone);  in dmz_lru_zone()
1883 static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)  in dmz_wait_for_free_zones()  argument
1887 	prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);  in dmz_wait_for_free_zones()
1888 	dmz_unlock_map(zmd);  in dmz_wait_for_free_zones()
1889 	dmz_unlock_metadata(zmd);  in dmz_wait_for_free_zones()
1893 	dmz_lock_metadata(zmd);  in dmz_wait_for_free_zones()
1894 	dmz_lock_map(zmd);  in dmz_wait_for_free_zones()
1895 	finish_wait(&zmd->free_wq, &wait);  in dmz_wait_for_free_zones()
1928 static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_wait_for_reclaim()  argument
1930 	dmz_unlock_map(zmd);  in dmz_wait_for_reclaim()
1931 	dmz_unlock_metadata(zmd);  in dmz_wait_for_reclaim()
1935 	dmz_lock_metadata(zmd);  in dmz_wait_for_reclaim()
1936 	dmz_lock_map(zmd);  in dmz_wait_for_reclaim()
1942 static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_rnd_zone_for_reclaim()  argument
1950 	if (zmd->nr_cache) {  in dmz_get_rnd_zone_for_reclaim()
1951 		zone_list = &zmd->map_cache_list;  in dmz_get_rnd_zone_for_reclaim()
1954 			zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
1956 		zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
2001 static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_seq_zone_for_reclaim()  argument
2006 	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {  in dmz_get_seq_zone_for_reclaim()
2019 struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_zone_for_reclaim()  argument
2032 	dmz_lock_map(zmd);  in dmz_get_zone_for_reclaim()
2033 	if (list_empty(&zmd->reserved_seq_zones_list))  in dmz_get_zone_for_reclaim()
2034 		zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);  in dmz_get_zone_for_reclaim()
2036 		zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);  in dmz_get_zone_for_reclaim()
2037 	dmz_unlock_map(zmd);  in dmz_get_zone_for_reclaim()
2048 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,  in dmz_get_chunk_mapping()  argument
2051 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_get_chunk_mapping()
2057 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_mapping()
2059 	dmz_lock_map(zmd);  in dmz_get_chunk_mapping()
2072 		dzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_mapping()
2074 			if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_mapping()
2078 			dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_mapping()
2082 		dmz_map_zone(zmd, dzone, chunk);  in dmz_get_chunk_mapping()
2086 		dzone = dmz_get(zmd, dzone_id);  in dmz_get_chunk_mapping()
2098 			ret = dmz_handle_seq_write_err(zmd, dzone);  in dmz_get_chunk_mapping()
2113 		dmz_wait_for_reclaim(zmd, dzone);  in dmz_get_chunk_mapping()
2117 	dmz_lru_zone(zmd, dzone);  in dmz_get_chunk_mapping()
2119 	dmz_unlock_map(zmd);  in dmz_get_chunk_mapping()
2130 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)  in dmz_put_chunk_mapping()  argument
2134 	dmz_lock_map(zmd);  in dmz_put_chunk_mapping()
2139 			dmz_lru_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2142 			dmz_unmap_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2143 			dmz_free_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2151 		dmz_lru_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2154 		dmz_unmap_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2155 		dmz_free_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2158 	dmz_unlock_map(zmd);  in dmz_put_chunk_mapping()
2165 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,  in dmz_get_chunk_buffer()  argument
2169 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_buffer()
2171 	dmz_lock_map(zmd);  in dmz_get_chunk_buffer()
2178 	bzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_buffer()
2180 		if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_buffer()
2184 		dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_buffer()
2189 	dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);  in dmz_get_chunk_buffer()
2196 		list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_get_chunk_buffer()
2200 	dmz_unlock_map(zmd);  in dmz_get_chunk_buffer()
2209 struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,  in dmz_alloc_zone()  argument
2218 		for (i = 0; i < zmd->nr_devs; i++)  in dmz_alloc_zone()
2219 			dmz_schedule_reclaim(zmd->dev[i].reclaim);  in dmz_alloc_zone()
2225 		list = &zmd->unmap_cache_list;  in dmz_alloc_zone()
2227 		list = &zmd->dev[dev_idx].unmap_rnd_list;  in dmz_alloc_zone()
2229 		list = &zmd->dev[dev_idx].unmap_seq_list;  in dmz_alloc_zone()
2240 		if (i < zmd->nr_devs) {  in dmz_alloc_zone()
2241 			dev_idx = (dev_idx + 1) % zmd->nr_devs;  in dmz_alloc_zone()
2249 		zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,  in dmz_alloc_zone()
2253 			atomic_dec(&zmd->nr_reserved_seq_zones);  in dmz_alloc_zone()
2262 		atomic_dec(&zmd->unmap_nr_cache);  in dmz_alloc_zone()
2269 		dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);  in dmz_alloc_zone()
2274 		dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);  in dmz_alloc_zone()
2285 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_free_zone()  argument
2289 		dmz_reset_zone(zmd, zone);  in dmz_free_zone()
2293 		list_add_tail(&zone->link, &zmd->unmap_cache_list);  in dmz_free_zone()
2294 		atomic_inc(&zmd->unmap_nr_cache);  in dmz_free_zone()
2299 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);  in dmz_free_zone()
2300 		atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_free_zone()
2306 	wake_up_all(&zmd->free_wq);  in dmz_free_zone()
2313 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,  in dmz_map_zone()  argument
2317 	dmz_set_chunk_mapping(zmd, chunk, dzone->id,  in dmz_map_zone()
2321 		list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_map_zone()
2332 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_unmap_zone()  argument
2363 	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);  in dmz_unmap_zone()
2404 static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,  in dmz_get_bitmap()  argument
2408 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +  in dmz_get_bitmap()
2409 		(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +  in dmz_get_bitmap()
2412 	return dmz_get_mblock(zmd, bitmap_block);  in dmz_get_bitmap()
2418 int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_copy_valid_blocks()  argument
2425 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_copy_valid_blocks()
2426 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);  in dmz_copy_valid_blocks()
2429 		to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);  in dmz_copy_valid_blocks()
2431 			dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2436 		dmz_dirty_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2438 		dmz_release_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2439 		dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2441 		chunk_block += zmd->zone_bits_per_mblk;  in dmz_copy_valid_blocks()
2453 int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_merge_valid_blocks()  argument
2460 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_merge_valid_blocks()
2462 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);  in dmz_merge_valid_blocks()
2467 		ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);  in dmz_merge_valid_blocks()
2480 int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_validate_blocks()  argument
2484 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_validate_blocks()
2488 	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",  in dmz_validate_blocks()
2496 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_validate_blocks()
2502 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_validate_blocks()
2506 			dmz_dirty_mblock(zmd, mblk);  in dmz_validate_blocks()
2509 		dmz_release_mblock(zmd, mblk);  in dmz_validate_blocks()
2518 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",  in dmz_validate_blocks()
2561 int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_invalidate_blocks()  argument
2568 	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",  in dmz_invalidate_blocks()
2571 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_invalidate_blocks()
2575 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_invalidate_blocks()
2581 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_invalidate_blocks()
2586 			dmz_dirty_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2589 		dmz_release_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2598 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",  in dmz_invalidate_blocks()
2609 static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_test_block()  argument
2615 	WARN_ON(chunk_block >= zmd->zone_nr_blocks);  in dmz_test_block()
2618 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_test_block()
2626 	dmz_release_mblock(zmd, mblk);  in dmz_test_block()
2635 static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_to_next_set_block()  argument
2641 	unsigned int zone_bits = zmd->zone_bits_per_mblk;  in dmz_to_next_set_block()
2645 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_to_next_set_block()
2649 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_to_next_set_block()
2661 		dmz_release_mblock(zmd, mblk);  in dmz_to_next_set_block()
2678 int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_block_valid()  argument
2683 	valid = dmz_test_block(zmd, zone, chunk_block);  in dmz_block_valid()
2688 	return dmz_to_next_set_block(zmd, zone, chunk_block,  in dmz_block_valid()
2689 				     zmd->zone_nr_blocks - chunk_block, 0);  in dmz_block_valid()
2698 int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_first_valid_block()  argument
2704 	ret = dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2705 				    zmd->zone_nr_blocks - start_block, 1);  in dmz_first_valid_block()
2712 	return dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2713 				     zmd->zone_nr_blocks - start_block, 0);  in dmz_first_valid_block()
2747 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_get_zone_weight()  argument
2752 	unsigned int nr_blocks = zmd->zone_nr_blocks;  in dmz_get_zone_weight()
2758 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_get_zone_weight()
2767 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_get_zone_weight()
2770 		dmz_release_mblock(zmd, mblk);  in dmz_get_zone_weight()
2782 static void dmz_cleanup_metadata(struct dmz_metadata *zmd)  in dmz_cleanup_metadata()  argument
2789 	if (zmd->map_mblk) {  in dmz_cleanup_metadata()
2790 		for (i = 0; i < zmd->nr_map_blocks; i++)  in dmz_cleanup_metadata()
2791 			dmz_release_mblock(zmd, zmd->map_mblk[i]);  in dmz_cleanup_metadata()
2792 		kfree(zmd->map_mblk);  in dmz_cleanup_metadata()
2793 		zmd->map_mblk = NULL;  in dmz_cleanup_metadata()
2798 		if (zmd->sb[i].mblk) {  in dmz_cleanup_metadata()
2799 			dmz_free_mblock(zmd, zmd->sb[i].mblk);  in dmz_cleanup_metadata()
2800 			zmd->sb[i].mblk = NULL;  in dmz_cleanup_metadata()
2805 	while (!list_empty(&zmd->mblk_dirty_list)) {  in dmz_cleanup_metadata()
2806 		mblk = list_first_entry(&zmd->mblk_dirty_list,  in dmz_cleanup_metadata()
2808 		dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",  in dmz_cleanup_metadata()
2811 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2812 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2815 	while (!list_empty(&zmd->mblk_lru_list)) {  in dmz_cleanup_metadata()
2816 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_cleanup_metadata()
2819 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2820 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2824 	root = &zmd->mblk_rbtree;  in dmz_cleanup_metadata()
2826 		dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",  in dmz_cleanup_metadata()
2829 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2833 	dmz_drop_zones(zmd);  in dmz_cleanup_metadata()
2835 	mutex_destroy(&zmd->mblk_flush_lock);  in dmz_cleanup_metadata()
2836 	mutex_destroy(&zmd->map_lock);  in dmz_cleanup_metadata()
2839 static void dmz_print_dev(struct dmz_metadata *zmd, int num)  in dmz_print_dev()  argument
2841 	struct dmz_dev *dev = &zmd->dev[num];  in dmz_print_dev()
2849 	if (zmd->sb_version > 1) {  in dmz_print_dev()
2851 			dev->zone_offset << zmd->zone_nr_sectors_shift;  in dmz_print_dev()
2856 			     dev->nr_zones, (u64)zmd->zone_nr_sectors,  in dmz_print_dev()
2862 			     dev->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_print_dev()
2873 	struct dmz_metadata *zmd;  in dmz_ctr_metadata()  local
2878 	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);  in dmz_ctr_metadata()
2879 	if (!zmd)  in dmz_ctr_metadata()
2882 	strcpy(zmd->devname, devname);  in dmz_ctr_metadata()
2883 	zmd->dev = dev;  in dmz_ctr_metadata()
2884 	zmd->nr_devs = num_dev;  in dmz_ctr_metadata()
2885 	zmd->mblk_rbtree = RB_ROOT;  in dmz_ctr_metadata()
2886 	init_rwsem(&zmd->mblk_sem);  in dmz_ctr_metadata()
2887 	mutex_init(&zmd->mblk_flush_lock);  in dmz_ctr_metadata()
2888 	spin_lock_init(&zmd->mblk_lock);  in dmz_ctr_metadata()
2889 	INIT_LIST_HEAD(&zmd->mblk_lru_list);  in dmz_ctr_metadata()
2890 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);  in dmz_ctr_metadata()
2892 	mutex_init(&zmd->map_lock);  in dmz_ctr_metadata()
2894 	atomic_set(&zmd->unmap_nr_cache, 0);  in dmz_ctr_metadata()
2895 	INIT_LIST_HEAD(&zmd->unmap_cache_list);  in dmz_ctr_metadata()
2896 	INIT_LIST_HEAD(&zmd->map_cache_list);  in dmz_ctr_metadata()
2898 	atomic_set(&zmd->nr_reserved_seq_zones, 0);  in dmz_ctr_metadata()
2899 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);  in dmz_ctr_metadata()
2901 	init_waitqueue_head(&zmd->free_wq);  in dmz_ctr_metadata()
2904 	ret = dmz_init_zones(zmd);  in dmz_ctr_metadata()
2909 	ret = dmz_load_sb(zmd);  in dmz_ctr_metadata()
2914 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {  in dmz_ctr_metadata()
2915 		zone = dmz_get(zmd, zmd->sb[0].zone->id + i);  in dmz_ctr_metadata()
2917 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2923 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2931 	ret = dmz_load_mapping(zmd);  in dmz_ctr_metadata()
2941 	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;  in dmz_ctr_metadata()
2942 	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;  in dmz_ctr_metadata()
2943 	zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;  in dmz_ctr_metadata()
2944 	zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;  in dmz_ctr_metadata()
2945 	zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;  in dmz_ctr_metadata()
2948 	ret = register_shrinker(&zmd->mblk_shrinker, "md-meta:(%u:%u)",  in dmz_ctr_metadata()
2952 		dmz_zmd_err(zmd, "Register metadata cache shrinker failed");  in dmz_ctr_metadata()
2956 	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);  in dmz_ctr_metadata()
2957 	for (i = 0; i < zmd->nr_devs; i++)  in dmz_ctr_metadata()
2958 		dmz_print_dev(zmd, i);  in dmz_ctr_metadata()
2960 	dmz_zmd_info(zmd, "  %u zones of %llu 512-byte logical sectors",  in dmz_ctr_metadata()
2961 		     zmd->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_ctr_metadata()
2962 	dmz_zmd_debug(zmd, "  %u metadata zones",  in dmz_ctr_metadata()
2963 		      zmd->nr_meta_zones * 2);  in dmz_ctr_metadata()
2964 	dmz_zmd_debug(zmd, "  %u data zones for %u chunks",  in dmz_ctr_metadata()
2965 		      zmd->nr_data_zones, zmd->nr_chunks);  in dmz_ctr_metadata()
2966 	dmz_zmd_debug(zmd, "    %u cache zones (%u unmapped)",  in dmz_ctr_metadata()
2967 		      zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));  in dmz_ctr_metadata()
2968 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_ctr_metadata()
2969 		dmz_zmd_debug(zmd, "    %u random zones (%u unmapped)",  in dmz_ctr_metadata()
2970 			      dmz_nr_rnd_zones(zmd, i),  in dmz_ctr_metadata()
2971 			      dmz_nr_unmap_rnd_zones(zmd, i));  in dmz_ctr_metadata()
2972 		dmz_zmd_debug(zmd, "    %u sequential zones (%u unmapped)",  in dmz_ctr_metadata()
2973 			      dmz_nr_seq_zones(zmd, i),  in dmz_ctr_metadata()
2974 			      dmz_nr_unmap_seq_zones(zmd, i));  in dmz_ctr_metadata()
2976 	dmz_zmd_debug(zmd, "  %u reserved sequential data zones",  in dmz_ctr_metadata()
2977 		      zmd->nr_reserved_seq);  in dmz_ctr_metadata()
2978 	dmz_zmd_debug(zmd, "Format:");  in dmz_ctr_metadata()
2979 	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",  in dmz_ctr_metadata()
2980 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);  in dmz_ctr_metadata()
2981 	dmz_zmd_debug(zmd, "  %u data zone mapping blocks",  in dmz_ctr_metadata()
2982 		      zmd->nr_map_blocks);  in dmz_ctr_metadata()
2983 	dmz_zmd_debug(zmd, "  %u bitmap blocks",  in dmz_ctr_metadata()
2984 		      zmd->nr_bitmap_blocks);  in dmz_ctr_metadata()
2986 	*metadata = zmd;  in dmz_ctr_metadata()
2990 	dmz_cleanup_metadata(zmd);  in dmz_ctr_metadata()
2991 	kfree(zmd);  in dmz_ctr_metadata()
3000 void dmz_dtr_metadata(struct dmz_metadata *zmd)  in dmz_dtr_metadata()  argument
3002 	unregister_shrinker(&zmd->mblk_shrinker);  in dmz_dtr_metadata()
3003 	dmz_cleanup_metadata(zmd);  in dmz_dtr_metadata()
3004 	kfree(zmd);  in dmz_dtr_metadata()
3010 int dmz_resume_metadata(struct dmz_metadata *zmd)  in dmz_resume_metadata()  argument
3018 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_resume_metadata()
3019 		zone = dmz_get(zmd, i);  in dmz_resume_metadata()
3021 			dmz_zmd_err(zmd, "Unable to get zone %u", i);  in dmz_resume_metadata()
3026 		ret = dmz_update_zone(zmd, zone);  in dmz_resume_metadata()
3028 			dmz_zmd_err(zmd, "Broken zone %u", i);  in dmz_resume_metadata()
3033 			dmz_zmd_warn(zmd, "Zone %u is offline", i);  in dmz_resume_metadata()
3041 			dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",  in dmz_resume_metadata()
3044 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_resume_metadata()
3045 					      zmd->zone_nr_blocks - zone->wp_block);  in dmz_resume_metadata()