207 #define dmz_zmd_info(zmd, format, args...)	\  argument
208 	DMINFO("(%s): " format, (zmd)->label, ## args)
210 #define dmz_zmd_err(zmd, format, args...)	\  argument
211 	DMERR("(%s): " format, (zmd)->label, ## args)
213 #define dmz_zmd_warn(zmd, format, args...)	\  argument
214 	DMWARN("(%s): " format, (zmd)->label, ## args)
216 #define dmz_zmd_debug(zmd, format, args...)	\  argument
217 	DMDEBUG("(%s): " format, (zmd)->label, ## args)
221 static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_dev_zone_id()  argument
229 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_sect()  argument
231 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_sect()
233 	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;  in dmz_start_sect()
236 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_block()  argument
238 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_block()
240 	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;  in dmz_start_block()
243 unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks()  argument
245 	return zmd->zone_nr_blocks;  in dmz_zone_nr_blocks()
248 unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks_shift()  argument
250 	return zmd->zone_nr_blocks_shift;  in dmz_zone_nr_blocks_shift()
253 unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors()  argument
255 	return zmd->zone_nr_sectors;  in dmz_zone_nr_sectors()
258 unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors_shift()  argument
260 	return zmd->zone_nr_sectors_shift;  in dmz_zone_nr_sectors_shift()
263 unsigned int dmz_nr_zones(struct dmz_metadata *zmd)  in dmz_nr_zones()  argument
265 	return zmd->nr_zones;  in dmz_nr_zones()
268 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)  in dmz_nr_chunks()  argument
270 	return zmd->nr_chunks;  in dmz_nr_chunks()
273 unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_rnd_zones()  argument
275 	return zmd->dev[idx].nr_rnd;  in dmz_nr_rnd_zones()
278 unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_rnd_zones()  argument
280 	return atomic_read(&zmd->dev[idx].unmap_nr_rnd);  in dmz_nr_unmap_rnd_zones()
283 unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_cache_zones()  argument
285 	return zmd->nr_cache;  in dmz_nr_cache_zones()
288 unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_unmap_cache_zones()  argument
290 	return atomic_read(&zmd->unmap_nr_cache);  in dmz_nr_unmap_cache_zones()
293 unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_seq_zones()  argument
295 	return zmd->dev[idx].nr_seq;  in dmz_nr_seq_zones()
298 unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_seq_zones()  argument
300 	return atomic_read(&zmd->dev[idx].unmap_nr_seq);  in dmz_nr_unmap_seq_zones()
303 static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)  in dmz_get()  argument
305 	return xa_load(&zmd->zones, zone_id);  in dmz_get()
308 static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,  in dmz_insert()  argument
316 	if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {  in dmz_insert()
330 const char *dmz_metadata_label(struct dmz_metadata *zmd)  in dmz_metadata_label()  argument
332 	return (const char *)zmd->label;  in dmz_metadata_label()
335 bool dmz_check_dev(struct dmz_metadata *zmd)  in dmz_check_dev()  argument
339 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_check_dev()
340 		if (!dmz_check_bdev(&zmd->dev[i]))  in dmz_check_dev()
346 bool dmz_dev_is_dying(struct dmz_metadata *zmd)  in dmz_dev_is_dying()  argument
350 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_dev_is_dying()
351 		if (dmz_bdev_is_dying(&zmd->dev[i]))  in dmz_dev_is_dying()
361 void dmz_lock_map(struct dmz_metadata *zmd)  in dmz_lock_map()  argument
363 	mutex_lock(&zmd->map_lock);  in dmz_lock_map()
366 void dmz_unlock_map(struct dmz_metadata *zmd)  in dmz_unlock_map()  argument
368 	mutex_unlock(&zmd->map_lock);  in dmz_unlock_map()
378 void dmz_lock_metadata(struct dmz_metadata *zmd)  in dmz_lock_metadata()  argument
380 	down_read(&zmd->mblk_sem);  in dmz_lock_metadata()
383 void dmz_unlock_metadata(struct dmz_metadata *zmd)  in dmz_unlock_metadata()  argument
385 	up_read(&zmd->mblk_sem);  in dmz_unlock_metadata()
393 void dmz_lock_flush(struct dmz_metadata *zmd)  in dmz_lock_flush()  argument
395 	mutex_lock(&zmd->mblk_flush_lock);  in dmz_lock_flush()
398 void dmz_unlock_flush(struct dmz_metadata *zmd)  in dmz_unlock_flush()  argument
400 	mutex_unlock(&zmd->mblk_flush_lock);  in dmz_unlock_flush()
406 static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,  in dmz_alloc_mblock()  argument
412 	if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {  in dmz_alloc_mblock()
413 		spin_lock(&zmd->mblk_lock);  in dmz_alloc_mblock()
414 		mblk = list_first_entry_or_null(&zmd->mblk_lru_list,  in dmz_alloc_mblock()
418 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_alloc_mblock()
421 		spin_unlock(&zmd->mblk_lock);  in dmz_alloc_mblock()
444 	atomic_inc(&zmd->nr_mblks);  in dmz_alloc_mblock()
452 static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_free_mblock()  argument
457 	atomic_dec(&zmd->nr_mblks);  in dmz_free_mblock()
463 static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_insert_mblock()  argument
465 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_insert_mblock()
485 static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,  in dmz_get_mblock_fast()  argument
488 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_get_mblock_fast()
537 static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,  in dmz_get_mblock_slow()  argument
541 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;  in dmz_get_mblock_slow()
542 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock_slow()
549 	mblk = dmz_alloc_mblock(zmd, mblk_no);  in dmz_get_mblock_slow()
555 		dmz_free_mblock(zmd, mblk);  in dmz_get_mblock_slow()
559 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
565 	m = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock_slow()
567 		spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
568 		dmz_free_mblock(zmd, mblk);  in dmz_get_mblock_slow()
575 	dmz_insert_mblock(zmd, mblk);  in dmz_get_mblock_slow()
577 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
594 static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,  in dmz_shrink_mblock_cache()  argument
600 	if (!zmd->max_nr_mblks)  in dmz_shrink_mblock_cache()
603 	while (!list_empty(&zmd->mblk_lru_list) &&  in dmz_shrink_mblock_cache()
604 	       atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&  in dmz_shrink_mblock_cache()
606 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_shrink_mblock_cache()
609 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_shrink_mblock_cache()
610 		dmz_free_mblock(zmd, mblk);  in dmz_shrink_mblock_cache()
623 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_count()  local
625 	return atomic_read(&zmd->nr_mblks);  in dmz_mblock_shrinker_count()
634 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_scan()  local
637 	spin_lock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
638 	count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);  in dmz_mblock_shrinker_scan()
639 	spin_unlock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
647 static void dmz_release_mblock(struct dmz_metadata *zmd,  in dmz_release_mblock()  argument
654 	spin_lock(&zmd->mblk_lock);  in dmz_release_mblock()
659 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_release_mblock()
660 			dmz_free_mblock(zmd, mblk);  in dmz_release_mblock()
662 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_release_mblock()
663 			dmz_shrink_mblock_cache(zmd, 1);  in dmz_release_mblock()
667 	spin_unlock(&zmd->mblk_lock);  in dmz_release_mblock()
674 static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,  in dmz_get_mblock()  argument
678 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock()
681 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock()
682 	mblk = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock()
683 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock()
687 		mblk = dmz_get_mblock_slow(zmd, mblk_no);  in dmz_get_mblock()
696 		dmz_release_mblock(zmd, mblk);  in dmz_get_mblock()
707 static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_dirty_mblock()  argument
709 	spin_lock(&zmd->mblk_lock);  in dmz_dirty_mblock()
711 		list_add_tail(&mblk->link, &zmd->mblk_dirty_list);  in dmz_dirty_mblock()
712 	spin_unlock(&zmd->mblk_lock);  in dmz_dirty_mblock()
718 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,  in dmz_write_mblock()  argument
721 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_mblock()
722 	sector_t block = zmd->sb[set].block + mblk->no;  in dmz_write_mblock()
781 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)  in dmz_write_sb()  argument
783 	struct dmz_mblock *mblk = zmd->sb[set].mblk;  in dmz_write_sb()
784 	struct dmz_super *sb = zmd->sb[set].sb;  in dmz_write_sb()
785 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_sb()
787 	u64 sb_gen = zmd->sb_gen + 1;  in dmz_write_sb()
792 	sb->version = cpu_to_le32(zmd->sb_version);  in dmz_write_sb()
793 	if (zmd->sb_version > 1) {  in dmz_write_sb()
795 		export_uuid(sb->dmz_uuid, &zmd->uuid);  in dmz_write_sb()
796 		memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);  in dmz_write_sb()
807 	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;  in dmz_write_sb()
809 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);  in dmz_write_sb()
810 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);  in dmz_write_sb()
811 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);  in dmz_write_sb()
813 	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);  in dmz_write_sb()
814 	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);  in dmz_write_sb()
819 	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,  in dmz_write_sb()
830 static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_write_dirty_mblocks()  argument
835 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_dirty_mblocks()
842 		ret = dmz_write_mblock(zmd, mblk, set);  in dmz_write_dirty_mblocks()
873 static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_log_dirty_mblocks()  argument
876 	unsigned int log_set = zmd->mblk_primary ^ 0x1;  in dmz_log_dirty_mblocks()
880 	ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);  in dmz_log_dirty_mblocks()
888 	ret = dmz_write_sb(zmd, log_set);  in dmz_log_dirty_mblocks()
898 int dmz_flush_metadata(struct dmz_metadata *zmd)  in dmz_flush_metadata()  argument
905 	if (WARN_ON(!zmd))  in dmz_flush_metadata()
915 	down_write(&zmd->mblk_sem);  in dmz_flush_metadata()
916 	dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_flush_metadata()
922 	dmz_lock_flush(zmd);  in dmz_flush_metadata()
930 	spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
931 	list_splice_init(&zmd->mblk_dirty_list, &write_list);  in dmz_flush_metadata()
932 	spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
945 	ret = dmz_log_dirty_mblocks(zmd, &write_list);  in dmz_flush_metadata()
953 	ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);  in dmz_flush_metadata()
957 	ret = dmz_write_sb(zmd, zmd->mblk_primary);  in dmz_flush_metadata()
965 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
968 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_flush_metadata()
969 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
972 	zmd->sb_gen++;  in dmz_flush_metadata()
974 	dmz_unlock_flush(zmd);  in dmz_flush_metadata()
975 	up_write(&zmd->mblk_sem);  in dmz_flush_metadata()
981 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
982 		list_splice(&write_list, &zmd->mblk_dirty_list);  in dmz_flush_metadata()
983 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
993 static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,  in dmz_check_sb()  argument
1008 	zmd->sb_version = le32_to_cpu(sb->version);  in dmz_check_sb()
1009 	if (zmd->sb_version > DMZ_META_VER) {  in dmz_check_sb()
1011 			    DMZ_META_VER, zmd->sb_version);  in dmz_check_sb()
1014 	if (zmd->sb_version < 2 && tertiary) {  in dmz_check_sb()
1030 	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {  in dmz_check_sb()
1034 			    (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);  in dmz_check_sb()
1037 	if (zmd->sb_version > 1) {  in dmz_check_sb()
1044 		} else if (uuid_is_null(&zmd->uuid)) {  in dmz_check_sb()
1045 			uuid_copy(&zmd->uuid, &sb_uuid);  in dmz_check_sb()
1046 		} else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {  in dmz_check_sb()
1049 				    &sb_uuid, &zmd->uuid);  in dmz_check_sb()
1052 		if (!strlen(zmd->label))  in dmz_check_sb()
1053 			memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);  in dmz_check_sb()
1054 		else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {  in dmz_check_sb()
1057 				    sb->dmz_label, zmd->label);  in dmz_check_sb()
1078 	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)  in dmz_check_sb()
1079 		>> zmd->zone_nr_blocks_shift;  in dmz_check_sb()
1081 	    (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||  in dmz_check_sb()
1082 	    (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {  in dmz_check_sb()
1088 	    le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {  in dmz_check_sb()
1093 	nr_data_zones = zmd->nr_useable_zones -  in dmz_check_sb()
1102 	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);  in dmz_check_sb()
1103 	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);  in dmz_check_sb()
1104 	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);  in dmz_check_sb()
1105 	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);  in dmz_check_sb()
1106 	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);  in dmz_check_sb()
1107 	zmd->nr_meta_zones = nr_meta_zones;  in dmz_check_sb()
1108 	zmd->nr_data_zones = nr_data_zones;  in dmz_check_sb()
1116 static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_read_sb()  argument
1118 	dmz_zmd_debug(zmd, "read superblock set %d dev %s block %llu",  in dmz_read_sb()
1130 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)  in dmz_lookup_secondary_sb()  argument
1132 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_lookup_secondary_sb()
1134 	unsigned int zone_id = zmd->sb[0].zone->id;  in dmz_lookup_secondary_sb()
1138 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_lookup_secondary_sb()
1142 	zmd->sb[1].mblk = mblk;  in dmz_lookup_secondary_sb()
1143 	zmd->sb[1].sb = mblk->data;  in dmz_lookup_secondary_sb()
1146 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;  in dmz_lookup_secondary_sb()
1147 	zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);  in dmz_lookup_secondary_sb()
1148 	zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_lookup_secondary_sb()
1149 	for (i = 1; i < zmd->nr_rnd_zones; i++) {  in dmz_lookup_secondary_sb()
1150 		if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)  in dmz_lookup_secondary_sb()
1152 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)  in dmz_lookup_secondary_sb()
1154 		zmd->sb[1].block += zone_nr_blocks;  in dmz_lookup_secondary_sb()
1155 		zmd->sb[1].zone = dmz_get(zmd, zone_id + i);  in dmz_lookup_secondary_sb()
1158 	dmz_free_mblock(zmd, mblk);  in dmz_lookup_secondary_sb()
1159 	zmd->sb[1].mblk = NULL;  in dmz_lookup_secondary_sb()
1160 	zmd->sb[1].zone = NULL;  in dmz_lookup_secondary_sb()
1161 	zmd->sb[1].dev = NULL;  in dmz_lookup_secondary_sb()
1169 static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_get_sb()  argument
1175 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_get_sb()
1183 	ret = dmz_read_sb(zmd, sb, set);  in dmz_get_sb()
1185 		dmz_free_mblock(zmd, mblk);  in dmz_get_sb()
1196 static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)  in dmz_recover_mblocks()  argument
1202 	dmz_dev_warn(zmd->sb[dst_set].dev,  in dmz_recover_mblocks()
1206 		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_recover_mblocks()
1208 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_recover_mblocks()
1215 	for (i = 1; i < zmd->nr_meta_blocks; i++) {  in dmz_recover_mblocks()
1216 		ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,  in dmz_recover_mblocks()
1217 				     zmd->sb[src_set].block + i, page);  in dmz_recover_mblocks()
1220 		ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,  in dmz_recover_mblocks()
1221 				     zmd->sb[dst_set].block + i, page);  in dmz_recover_mblocks()
1227 	if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1228 		zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);  in dmz_recover_mblocks()
1229 		if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1233 		zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;  in dmz_recover_mblocks()
1236 	ret = dmz_write_sb(zmd, dst_set);  in dmz_recover_mblocks()
1246 static int dmz_load_sb(struct dmz_metadata *zmd)  in dmz_load_sb()  argument
1252 	if (!zmd->sb[0].zone) {  in dmz_load_sb()
1253 		dmz_zmd_err(zmd, "Primary super block zone not set");  in dmz_load_sb()
1258 	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_load_sb()
1259 	zmd->sb[0].dev = zmd->sb[0].zone->dev;  in dmz_load_sb()
1260 	ret = dmz_get_sb(zmd, &zmd->sb[0], 0);  in dmz_load_sb()
1262 		dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");  in dmz_load_sb()
1266 	ret = dmz_check_sb(zmd, &zmd->sb[0], false);  in dmz_load_sb()
1271 		if (!zmd->sb[1].zone) {  in dmz_load_sb()
1273 				zmd->sb[0].zone->id + zmd->nr_meta_zones;  in dmz_load_sb()
1275 			zmd->sb[1].zone = dmz_get(zmd, zone_id);  in dmz_load_sb()
1277 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_load_sb()
1278 		zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_load_sb()
1279 		ret = dmz_get_sb(zmd, &zmd->sb[1], 1);  in dmz_load_sb()
1281 		ret = dmz_lookup_secondary_sb(zmd);  in dmz_load_sb()
1284 		dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");  in dmz_load_sb()
1288 	ret = dmz_check_sb(zmd, &zmd->sb[1], false);  in dmz_load_sb()
1294 		dmz_zmd_err(zmd, "No valid super block found");  in dmz_load_sb()
1299 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);  in dmz_load_sb()
1301 		ret = dmz_recover_mblocks(zmd, 0);  in dmz_load_sb()
1303 			dmz_dev_err(zmd->sb[0].dev,  in dmz_load_sb()
1310 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);  in dmz_load_sb()
1312 		ret = dmz_recover_mblocks(zmd, 1);  in dmz_load_sb()
1315 			dmz_dev_err(zmd->sb[1].dev,  in dmz_load_sb()
1322 		zmd->sb_gen = sb_gen[0];  in dmz_load_sb()
1323 		zmd->mblk_primary = 0;  in dmz_load_sb()
1325 		zmd->sb_gen = sb_gen[1];  in dmz_load_sb()
1326 		zmd->mblk_primary = 1;  in dmz_load_sb()
1329 	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,  in dmz_load_sb()
1331 		      zmd->mblk_primary, zmd->sb_gen);  in dmz_load_sb()
1333 	if (zmd->sb_version > 1) {  in dmz_load_sb()
1340 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_load_sb()
1342 			sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);  in dmz_load_sb()
1343 			sb->dev = &zmd->dev[i];  in dmz_load_sb()
1351 			ret = dmz_get_sb(zmd, sb, i + 1);  in dmz_load_sb()
1355 				dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1358 			ret = dmz_check_sb(zmd, sb, true);  in dmz_load_sb()
1359 			dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1375 	struct dmz_metadata *zmd = dev->metadata;  in dmz_init_zone()  local
1379 	zone = dmz_insert(zmd, idx, dev);  in dmz_init_zone()
1383 	if (blkz->len != zmd->zone_nr_sectors) {  in dmz_init_zone()
1384 		if (zmd->sb_version > 1) {  in dmz_init_zone()
1415 		zmd->nr_useable_zones++;  in dmz_init_zone()
1417 			zmd->nr_rnd_zones++;  in dmz_init_zone()
1418 			if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {  in dmz_init_zone()
1420 				zmd->sb[0].zone = zone;  in dmz_init_zone()
1423 		if (zmd->nr_devs > 1 && num == 0) {  in dmz_init_zone()
1435 static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)  in dmz_emulate_zones()  argument
1443 		zone = dmz_insert(zmd, idx, dev);  in dmz_emulate_zones()
1448 		zmd->nr_cache_zones++;  in dmz_emulate_zones()
1449 		zmd->nr_useable_zones++;  in dmz_emulate_zones()
1450 		if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {  in dmz_emulate_zones()
1455 		zone_offset += zmd->zone_nr_sectors;  in dmz_emulate_zones()
1463 static void dmz_drop_zones(struct dmz_metadata *zmd)  in dmz_drop_zones()  argument
1467 	for(idx = 0; idx < zmd->nr_zones; idx++) {  in dmz_drop_zones()
1468 		struct dm_zone *zone = xa_load(&zmd->zones, idx);  in dmz_drop_zones()
1471 		xa_erase(&zmd->zones, idx);  in dmz_drop_zones()
1473 	xa_destroy(&zmd->zones);  in dmz_drop_zones()
1480 static int dmz_init_zones(struct dmz_metadata *zmd)  in dmz_init_zones()  argument
1483 	struct dmz_dev *zoned_dev = &zmd->dev[0];  in dmz_init_zones()
1486 	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;  in dmz_init_zones()
1487 	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);  in dmz_init_zones()
1488 	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);  in dmz_init_zones()
1489 	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);  in dmz_init_zones()
1490 	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;  in dmz_init_zones()
1491 	zmd->zone_nr_bitmap_blocks =  in dmz_init_zones()
1492 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);  in dmz_init_zones()
1493 	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,  in dmz_init_zones()
1497 	zmd->nr_zones = 0;  in dmz_init_zones()
1498 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1499 		struct dmz_dev *dev = &zmd->dev[i];  in dmz_init_zones()
1501 		dev->metadata = zmd;  in dmz_init_zones()
1502 		zmd->nr_zones += dev->nr_zones;  in dmz_init_zones()
1513 	if (!zmd->nr_zones) {  in dmz_init_zones()
1514 		DMERR("(%s): No zones found", zmd->devname);  in dmz_init_zones()
1517 	xa_init(&zmd->zones);  in dmz_init_zones()
1520 		zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);  in dmz_init_zones()
1522 	if (zmd->nr_devs > 1) {  in dmz_init_zones()
1523 		ret = dmz_emulate_zones(zmd, &zmd->dev[0]);  in dmz_init_zones()
1526 				zmd->devname, ret);  in dmz_init_zones()
1527 			dmz_drop_zones(zmd);  in dmz_init_zones()
1535 		zmd->sb[0].zone = dmz_get(zmd, 0);  in dmz_init_zones()
1537 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1538 			zoned_dev = &zmd->dev[i];  in dmz_init_zones()
1545 					zmd->devname, ret);  in dmz_init_zones()
1546 				dmz_drop_zones(zmd);  in dmz_init_zones()
1562 			zmd->devname, ret);  in dmz_init_zones()
1563 		dmz_drop_zones(zmd);  in dmz_init_zones()
1592 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_update_zone()  argument
1608 	ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,  in dmz_update_zone()
1628 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,  in dmz_handle_seq_write_err()  argument
1636 	ret = dmz_update_zone(zmd, zone);  in dmz_handle_seq_write_err()
1644 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_handle_seq_write_err()
1654 static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_reset_zone()  argument
1671 				       dmz_start_sect(zmd, zone),  in dmz_reset_zone()
1672 				       zmd->zone_nr_sectors, GFP_NOIO);  in dmz_reset_zone()
1687 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1692 static int dmz_load_mapping(struct dmz_metadata *zmd)  in dmz_load_mapping()  argument
1702 	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,  in dmz_load_mapping()
1704 	if (!zmd->map_mblk)  in dmz_load_mapping()
1708 	while (chunk < zmd->nr_chunks) {  in dmz_load_mapping()
1711 			dmap_mblk = dmz_get_mblock(zmd, i + 1);  in dmz_load_mapping()
1714 			zmd->map_mblk[i] = dmap_mblk;  in dmz_load_mapping()
1725 		if (dzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1726 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",  in dmz_load_mapping()
1731 		dzone = dmz_get(zmd, dzone_id);  in dmz_load_mapping()
1733 			dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",  in dmz_load_mapping()
1739 		dmz_get_zone_weight(zmd, dzone);  in dmz_load_mapping()
1742 			list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1753 		if (bzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1754 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",  in dmz_load_mapping()
1759 		bzone = dmz_get(zmd, bzone_id);  in dmz_load_mapping()
1761 			dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",  in dmz_load_mapping()
1766 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",  in dmz_load_mapping()
1776 		dmz_get_zone_weight(zmd, bzone);  in dmz_load_mapping()
1778 			list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1793 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_load_mapping()
1794 		dzone = dmz_get(zmd, i);  in dmz_load_mapping()
1803 			zmd->nr_cache++;  in dmz_load_mapping()
1818 			list_add_tail(&dzone->link, &zmd->unmap_cache_list);  in dmz_load_mapping()
1819 			atomic_inc(&zmd->unmap_nr_cache);  in dmz_load_mapping()
1824 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {  in dmz_load_mapping()
1825 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);  in dmz_load_mapping()
1827 			atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_load_mapping()
1842 static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,  in dmz_set_chunk_mapping()  argument
1845 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_set_chunk_mapping()
1851 	dmz_dirty_mblock(zmd, dmap_mblk);  in dmz_set_chunk_mapping()
1858 static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in __dmz_lru_zone()  argument
1869 		list_add_tail(&zone->link, &zmd->map_cache_list);  in __dmz_lru_zone()
1880 static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_lru_zone()  argument
1882 	__dmz_lru_zone(zmd, zone);  in dmz_lru_zone()
1884 		__dmz_lru_zone(zmd, zone->bzone);  in dmz_lru_zone()
1890 static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)  in dmz_wait_for_free_zones()  argument
1894 	prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);  in dmz_wait_for_free_zones()
1895 	dmz_unlock_map(zmd);  in dmz_wait_for_free_zones()
1896 	dmz_unlock_metadata(zmd);  in dmz_wait_for_free_zones()
1900 	dmz_lock_metadata(zmd);  in dmz_wait_for_free_zones()
1901 	dmz_lock_map(zmd);  in dmz_wait_for_free_zones()
1902 	finish_wait(&zmd->free_wq, &wait);  in dmz_wait_for_free_zones()
1935 static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_wait_for_reclaim()  argument
1937 	dmz_unlock_map(zmd);  in dmz_wait_for_reclaim()
1938 	dmz_unlock_metadata(zmd);  in dmz_wait_for_reclaim()
1942 	dmz_lock_metadata(zmd);  in dmz_wait_for_reclaim()
1943 	dmz_lock_map(zmd);  in dmz_wait_for_reclaim()
1949 static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_rnd_zone_for_reclaim()  argument
1957 	if (zmd->nr_cache) {  in dmz_get_rnd_zone_for_reclaim()
1958 		zone_list = &zmd->map_cache_list;  in dmz_get_rnd_zone_for_reclaim()
1961 			zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
1963 		zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
2008 static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_seq_zone_for_reclaim()  argument
2013 	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {  in dmz_get_seq_zone_for_reclaim()
2026 struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_zone_for_reclaim()  argument
2039 	dmz_lock_map(zmd);  in dmz_get_zone_for_reclaim()
2040 	if (list_empty(&zmd->reserved_seq_zones_list))  in dmz_get_zone_for_reclaim()
2041 		zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);  in dmz_get_zone_for_reclaim()
2043 		zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);  in dmz_get_zone_for_reclaim()
2044 	dmz_unlock_map(zmd);  in dmz_get_zone_for_reclaim()
2055 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)  in dmz_get_chunk_mapping()  argument
2057 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_get_chunk_mapping()
2063 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_mapping()
2065 	dmz_lock_map(zmd);  in dmz_get_chunk_mapping()
2078 		dzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_mapping()
2080 			if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_mapping()
2084 			dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_mapping()
2088 		dmz_map_zone(zmd, dzone, chunk);  in dmz_get_chunk_mapping()
2092 		dzone = dmz_get(zmd, dzone_id);  in dmz_get_chunk_mapping()
2104 			ret = dmz_handle_seq_write_err(zmd, dzone);  in dmz_get_chunk_mapping()
2119 		dmz_wait_for_reclaim(zmd, dzone);  in dmz_get_chunk_mapping()
2123 	dmz_lru_zone(zmd, dzone);  in dmz_get_chunk_mapping()
2125 	dmz_unlock_map(zmd);  in dmz_get_chunk_mapping()
2136 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)  in dmz_put_chunk_mapping()  argument
2140 	dmz_lock_map(zmd);  in dmz_put_chunk_mapping()
2145 			dmz_lru_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2148 			dmz_unmap_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2149 			dmz_free_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2157 		dmz_lru_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2160 		dmz_unmap_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2161 		dmz_free_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2164 	dmz_unlock_map(zmd);  in dmz_put_chunk_mapping()
2171 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,  in dmz_get_chunk_buffer()  argument
2175 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_buffer()
2177 	dmz_lock_map(zmd);  in dmz_get_chunk_buffer()
2184 	bzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_buffer()
2186 		if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_buffer()
2190 		dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_buffer()
2195 	dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);  in dmz_get_chunk_buffer()
2202 		list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_get_chunk_buffer()
2206 	dmz_unlock_map(zmd);  in dmz_get_chunk_buffer()
2215 struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,  in dmz_alloc_zone()  argument
2224 		for (i = 0; i < zmd->nr_devs; i++)  in dmz_alloc_zone()
2225 			dmz_schedule_reclaim(zmd->dev[i].reclaim);  in dmz_alloc_zone()
2231 		list = &zmd->unmap_cache_list;  in dmz_alloc_zone()
2233 		list = &zmd->dev[dev_idx].unmap_rnd_list;  in dmz_alloc_zone()
2235 		list = &zmd->dev[dev_idx].unmap_seq_list;  in dmz_alloc_zone()
2246 		if (i < zmd->nr_devs) {  in dmz_alloc_zone()
2247 			dev_idx = (dev_idx + 1) % zmd->nr_devs;  in dmz_alloc_zone()
2255 		zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,  in dmz_alloc_zone()
2259 			atomic_dec(&zmd->nr_reserved_seq_zones);  in dmz_alloc_zone()
2268 		atomic_dec(&zmd->unmap_nr_cache);  in dmz_alloc_zone()
2275 		dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);  in dmz_alloc_zone()
2280 		dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);  in dmz_alloc_zone()
2291 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_free_zone()  argument
2295 		dmz_reset_zone(zmd, zone);  in dmz_free_zone()
2299 		list_add_tail(&zone->link, &zmd->unmap_cache_list);  in dmz_free_zone()
2300 		atomic_inc(&zmd->unmap_nr_cache);  in dmz_free_zone()
2305 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);  in dmz_free_zone()
2306 		atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_free_zone()
2312 	wake_up_all(&zmd->free_wq);  in dmz_free_zone()
2319 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,  in dmz_map_zone()  argument
2323 	dmz_set_chunk_mapping(zmd, chunk, dzone->id,  in dmz_map_zone()
2327 		list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_map_zone()
2338 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_unmap_zone()  argument
2369 	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);  in dmz_unmap_zone()
2410 static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,  in dmz_get_bitmap()  argument
2414 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +  in dmz_get_bitmap()
2415 		(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +  in dmz_get_bitmap()
2418 	return dmz_get_mblock(zmd, bitmap_block);  in dmz_get_bitmap()
2424 int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_copy_valid_blocks()  argument
2431 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_copy_valid_blocks()
2432 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);  in dmz_copy_valid_blocks()
2435 		to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);  in dmz_copy_valid_blocks()
2437 			dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2442 		dmz_dirty_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2444 		dmz_release_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2445 		dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2447 		chunk_block += zmd->zone_bits_per_mblk;  in dmz_copy_valid_blocks()
2459 int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_merge_valid_blocks()  argument
2466 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_merge_valid_blocks()
2468 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);  in dmz_merge_valid_blocks()
2473 		ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);  in dmz_merge_valid_blocks()
2486 int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_validate_blocks()  argument
2490 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_validate_blocks()
2494 	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",  in dmz_validate_blocks()
2502 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_validate_blocks()
2508 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_validate_blocks()
2512 			dmz_dirty_mblock(zmd, mblk);  in dmz_validate_blocks()
2515 		dmz_release_mblock(zmd, mblk);  in dmz_validate_blocks()
2524 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",  in dmz_validate_blocks()
2567 int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_invalidate_blocks()  argument
2574 	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",  in dmz_invalidate_blocks()
2577 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_invalidate_blocks()
2581 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_invalidate_blocks()
2587 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_invalidate_blocks()
2592 			dmz_dirty_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2595 		dmz_release_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2604 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",  in dmz_invalidate_blocks()
2615 static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_test_block()  argument
2621 	WARN_ON(chunk_block >= zmd->zone_nr_blocks);  in dmz_test_block()
2624 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_test_block()
2632 	dmz_release_mblock(zmd, mblk);  in dmz_test_block()
2641 static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_to_next_set_block()  argument
2647 	unsigned int zone_bits = zmd->zone_bits_per_mblk;  in dmz_to_next_set_block()
2651 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_to_next_set_block()
2655 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_to_next_set_block()
2667 		dmz_release_mblock(zmd, mblk);  in dmz_to_next_set_block()
2684 int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_block_valid()  argument
2689 	valid = dmz_test_block(zmd, zone, chunk_block);  in dmz_block_valid()
2694 	return dmz_to_next_set_block(zmd, zone, chunk_block,  in dmz_block_valid()
2695 				     zmd->zone_nr_blocks - chunk_block, 0);  in dmz_block_valid()
2704 int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_first_valid_block()  argument
2710 	ret = dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2711 				    zmd->zone_nr_blocks - start_block, 1);  in dmz_first_valid_block()
2718 	return dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2719 				     zmd->zone_nr_blocks - start_block, 0);  in dmz_first_valid_block()
2753 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_get_zone_weight()  argument
2758 	unsigned int nr_blocks = zmd->zone_nr_blocks;  in dmz_get_zone_weight()
2764 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_get_zone_weight()
2773 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_get_zone_weight()
2776 		dmz_release_mblock(zmd, mblk);  in dmz_get_zone_weight()
2788 static void dmz_cleanup_metadata(struct dmz_metadata *zmd)  in dmz_cleanup_metadata()  argument
2795 	if (zmd->map_mblk) {  in dmz_cleanup_metadata()
2796 		for (i = 0; i < zmd->nr_map_blocks; i++)  in dmz_cleanup_metadata()
2797 			dmz_release_mblock(zmd, zmd->map_mblk[i]);  in dmz_cleanup_metadata()
2798 		kfree(zmd->map_mblk);  in dmz_cleanup_metadata()
2799 		zmd->map_mblk = NULL;  in dmz_cleanup_metadata()
2804 		if (zmd->sb[i].mblk) {  in dmz_cleanup_metadata()
2805 			dmz_free_mblock(zmd, zmd->sb[i].mblk);  in dmz_cleanup_metadata()
2806 			zmd->sb[i].mblk = NULL;  in dmz_cleanup_metadata()
2811 	while (!list_empty(&zmd->mblk_dirty_list)) {  in dmz_cleanup_metadata()
2812 		mblk = list_first_entry(&zmd->mblk_dirty_list,  in dmz_cleanup_metadata()
2814 		dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",  in dmz_cleanup_metadata()
2817 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2818 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2821 	while (!list_empty(&zmd->mblk_lru_list)) {  in dmz_cleanup_metadata()
2822 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_cleanup_metadata()
2825 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2826 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2830 	root = &zmd->mblk_rbtree;  in dmz_cleanup_metadata()
2832 		dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",  in dmz_cleanup_metadata()
2835 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2839 	dmz_drop_zones(zmd);  in dmz_cleanup_metadata()
2841 	mutex_destroy(&zmd->mblk_flush_lock);  in dmz_cleanup_metadata()
2842 	mutex_destroy(&zmd->map_lock);  in dmz_cleanup_metadata()
2845 static void dmz_print_dev(struct dmz_metadata *zmd, int num)  in dmz_print_dev()  argument
2847 	struct dmz_dev *dev = &zmd->dev[num];  in dmz_print_dev()
2855 	if (zmd->sb_version > 1) {  in dmz_print_dev()
2857 			dev->zone_offset << zmd->zone_nr_sectors_shift;  in dmz_print_dev()
2862 			     dev->nr_zones, (u64)zmd->zone_nr_sectors,  in dmz_print_dev()
2868 			     dev->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_print_dev()
2879 	struct dmz_metadata *zmd;  in dmz_ctr_metadata()  local
2884 	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);  in dmz_ctr_metadata()
2885 	if (!zmd)  in dmz_ctr_metadata()
2888 	strcpy(zmd->devname, devname);  in dmz_ctr_metadata()
2889 	zmd->dev = dev;  in dmz_ctr_metadata()
2890 	zmd->nr_devs = num_dev;  in dmz_ctr_metadata()
2891 	zmd->mblk_rbtree = RB_ROOT;  in dmz_ctr_metadata()
2892 	init_rwsem(&zmd->mblk_sem);  in dmz_ctr_metadata()
2893 	mutex_init(&zmd->mblk_flush_lock);  in dmz_ctr_metadata()
2894 	spin_lock_init(&zmd->mblk_lock);  in dmz_ctr_metadata()
2895 	INIT_LIST_HEAD(&zmd->mblk_lru_list);  in dmz_ctr_metadata()
2896 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);  in dmz_ctr_metadata()
2898 	mutex_init(&zmd->map_lock);  in dmz_ctr_metadata()
2900 	atomic_set(&zmd->unmap_nr_cache, 0);  in dmz_ctr_metadata()
2901 	INIT_LIST_HEAD(&zmd->unmap_cache_list);  in dmz_ctr_metadata()
2902 	INIT_LIST_HEAD(&zmd->map_cache_list);  in dmz_ctr_metadata()
2904 	atomic_set(&zmd->nr_reserved_seq_zones, 0);  in dmz_ctr_metadata()
2905 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);  in dmz_ctr_metadata()
2907 	init_waitqueue_head(&zmd->free_wq);  in dmz_ctr_metadata()
2910 	ret = dmz_init_zones(zmd);  in dmz_ctr_metadata()
2915 	ret = dmz_load_sb(zmd);  in dmz_ctr_metadata()
2920 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {  in dmz_ctr_metadata()
2921 		zone = dmz_get(zmd, zmd->sb[0].zone->id + i);  in dmz_ctr_metadata()
2923 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2929 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2937 	ret = dmz_load_mapping(zmd);  in dmz_ctr_metadata()
2947 	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;  in dmz_ctr_metadata()
2948 	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;  in dmz_ctr_metadata()
2949 	zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;  in dmz_ctr_metadata()
2950 	zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;  in dmz_ctr_metadata()
2951 	zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;  in dmz_ctr_metadata()
2954 	ret = register_shrinker(&zmd->mblk_shrinker);  in dmz_ctr_metadata()
2956 		dmz_zmd_err(zmd, "Register metadata cache shrinker failed");  in dmz_ctr_metadata()
2960 	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);  in dmz_ctr_metadata()
2961 	for (i = 0; i < zmd->nr_devs; i++)  in dmz_ctr_metadata()
2962 		dmz_print_dev(zmd, i);  in dmz_ctr_metadata()
2964 	dmz_zmd_info(zmd, "  %u zones of %llu 512-byte logical sectors",  in dmz_ctr_metadata()
2965 		     zmd->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_ctr_metadata()
2966 	dmz_zmd_debug(zmd, "  %u metadata zones",  in dmz_ctr_metadata()
2967 		      zmd->nr_meta_zones * 2);  in dmz_ctr_metadata()
2968 	dmz_zmd_debug(zmd, "  %u data zones for %u chunks",  in dmz_ctr_metadata()
2969 		      zmd->nr_data_zones, zmd->nr_chunks);  in dmz_ctr_metadata()
2970 	dmz_zmd_debug(zmd, "    %u cache zones (%u unmapped)",  in dmz_ctr_metadata()
2971 		      zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));  in dmz_ctr_metadata()
2972 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_ctr_metadata()
2973 		dmz_zmd_debug(zmd, "    %u random zones (%u unmapped)",  in dmz_ctr_metadata()
2974 			      dmz_nr_rnd_zones(zmd, i),  in dmz_ctr_metadata()
2975 			      dmz_nr_unmap_rnd_zones(zmd, i));  in dmz_ctr_metadata()
2976 		dmz_zmd_debug(zmd, "    %u sequential zones (%u unmapped)",  in dmz_ctr_metadata()
2977 			      dmz_nr_seq_zones(zmd, i),  in dmz_ctr_metadata()
2978 			      dmz_nr_unmap_seq_zones(zmd, i));  in dmz_ctr_metadata()
2980 	dmz_zmd_debug(zmd, "  %u reserved sequential data zones",  in dmz_ctr_metadata()
2981 		      zmd->nr_reserved_seq);  in dmz_ctr_metadata()
2982 	dmz_zmd_debug(zmd, "Format:");  in dmz_ctr_metadata()
2983 	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",  in dmz_ctr_metadata()
2984 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);  in dmz_ctr_metadata()
2985 	dmz_zmd_debug(zmd, "  %u data zone mapping blocks",  in dmz_ctr_metadata()
2986 		      zmd->nr_map_blocks);  in dmz_ctr_metadata()
2987 	dmz_zmd_debug(zmd, "  %u bitmap blocks",  in dmz_ctr_metadata()
2988 		      zmd->nr_bitmap_blocks);  in dmz_ctr_metadata()
2990 	*metadata = zmd;  in dmz_ctr_metadata()
2994 	dmz_cleanup_metadata(zmd);  in dmz_ctr_metadata()
2995 	kfree(zmd);  in dmz_ctr_metadata()
3004 void dmz_dtr_metadata(struct dmz_metadata *zmd)  in dmz_dtr_metadata()  argument
3006 	unregister_shrinker(&zmd->mblk_shrinker);  in dmz_dtr_metadata()
3007 	dmz_cleanup_metadata(zmd);  in dmz_dtr_metadata()
3008 	kfree(zmd);  in dmz_dtr_metadata()
3014 int dmz_resume_metadata(struct dmz_metadata *zmd)  in dmz_resume_metadata()  argument
3022 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_resume_metadata()
3023 		zone = dmz_get(zmd, i);  in dmz_resume_metadata()
3025 			dmz_zmd_err(zmd, "Unable to get zone %u", i);  in dmz_resume_metadata()
3030 		ret = dmz_update_zone(zmd, zone);  in dmz_resume_metadata()
3032 			dmz_zmd_err(zmd, "Broken zone %u", i);  in dmz_resume_metadata()
3037 			dmz_zmd_warn(zmd, "Zone %u is offline", i);  in dmz_resume_metadata()
3045 			dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",  in dmz_resume_metadata()
3048 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_resume_metadata()
3049 					      zmd->zone_nr_blocks - zone->wp_block);  in dmz_resume_metadata()