207 #define dmz_zmd_info(zmd, format, args...)	\  argument
208 	DMINFO("(%s): " format, (zmd)->label, ## args)
210 #define dmz_zmd_err(zmd, format, args...)	\  argument
211 	DMERR("(%s): " format, (zmd)->label, ## args)
213 #define dmz_zmd_warn(zmd, format, args...)	\  argument
214 	DMWARN("(%s): " format, (zmd)->label, ## args)
216 #define dmz_zmd_debug(zmd, format, args...)	\  argument
217 	DMDEBUG("(%s): " format, (zmd)->label, ## args)
221 static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_dev_zone_id()  argument
229 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_sect()  argument
231 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_sect()
233 	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;  in dmz_start_sect()
236 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_start_block()  argument
238 	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);  in dmz_start_block()
240 	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;  in dmz_start_block()
243 unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks()  argument
245 	return zmd->zone_nr_blocks;  in dmz_zone_nr_blocks()
248 unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_blocks_shift()  argument
250 	return zmd->zone_nr_blocks_shift;  in dmz_zone_nr_blocks_shift()
253 unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors()  argument
255 	return zmd->zone_nr_sectors;  in dmz_zone_nr_sectors()
258 unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)  in dmz_zone_nr_sectors_shift()  argument
260 	return zmd->zone_nr_sectors_shift;  in dmz_zone_nr_sectors_shift()
263 unsigned int dmz_nr_zones(struct dmz_metadata *zmd)  in dmz_nr_zones()  argument
265 	return zmd->nr_zones;  in dmz_nr_zones()
268 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)  in dmz_nr_chunks()  argument
270 	return zmd->nr_chunks;  in dmz_nr_chunks()
273 unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_rnd_zones()  argument
275 	return zmd->dev[idx].nr_rnd;  in dmz_nr_rnd_zones()
278 unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_rnd_zones()  argument
280 	return atomic_read(&zmd->dev[idx].unmap_nr_rnd);  in dmz_nr_unmap_rnd_zones()
283 unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_cache_zones()  argument
285 	return zmd->nr_cache;  in dmz_nr_cache_zones()
288 unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)  in dmz_nr_unmap_cache_zones()  argument
290 	return atomic_read(&zmd->unmap_nr_cache);  in dmz_nr_unmap_cache_zones()
293 unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_seq_zones()  argument
295 	return zmd->dev[idx].nr_seq;  in dmz_nr_seq_zones()
298 unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)  in dmz_nr_unmap_seq_zones()  argument
300 	return atomic_read(&zmd->dev[idx].unmap_nr_seq);  in dmz_nr_unmap_seq_zones()
303 static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)  in dmz_get()  argument
305 	return xa_load(&zmd->zones, zone_id);  in dmz_get()
308 static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,  in dmz_insert()  argument
316 	if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {  in dmz_insert()
330 const char *dmz_metadata_label(struct dmz_metadata *zmd)  in dmz_metadata_label()  argument
332 	return (const char *)zmd->label;  in dmz_metadata_label()
335 bool dmz_check_dev(struct dmz_metadata *zmd)  in dmz_check_dev()  argument
339 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_check_dev()
340 		if (!dmz_check_bdev(&zmd->dev[i]))  in dmz_check_dev()
346 bool dmz_dev_is_dying(struct dmz_metadata *zmd)  in dmz_dev_is_dying()  argument
350 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_dev_is_dying()
351 		if (dmz_bdev_is_dying(&zmd->dev[i]))  in dmz_dev_is_dying()
361 void dmz_lock_map(struct dmz_metadata *zmd)  in dmz_lock_map()  argument
363 	mutex_lock(&zmd->map_lock);  in dmz_lock_map()
366 void dmz_unlock_map(struct dmz_metadata *zmd)  in dmz_unlock_map()  argument
368 	mutex_unlock(&zmd->map_lock);  in dmz_unlock_map()
378 void dmz_lock_metadata(struct dmz_metadata *zmd)  in dmz_lock_metadata()  argument
380 	down_read(&zmd->mblk_sem);  in dmz_lock_metadata()
383 void dmz_unlock_metadata(struct dmz_metadata *zmd)  in dmz_unlock_metadata()  argument
385 	up_read(&zmd->mblk_sem);  in dmz_unlock_metadata()
393 void dmz_lock_flush(struct dmz_metadata *zmd)  in dmz_lock_flush()  argument
395 	mutex_lock(&zmd->mblk_flush_lock);  in dmz_lock_flush()
398 void dmz_unlock_flush(struct dmz_metadata *zmd)  in dmz_unlock_flush()  argument
400 	mutex_unlock(&zmd->mblk_flush_lock);  in dmz_unlock_flush()
406 static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,  in dmz_alloc_mblock()  argument
412 	if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {  in dmz_alloc_mblock()
413 		spin_lock(&zmd->mblk_lock);  in dmz_alloc_mblock()
414 		mblk = list_first_entry_or_null(&zmd->mblk_lru_list,  in dmz_alloc_mblock()
418 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_alloc_mblock()
421 		spin_unlock(&zmd->mblk_lock);  in dmz_alloc_mblock()
444 	atomic_inc(&zmd->nr_mblks);  in dmz_alloc_mblock()
452 static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_free_mblock()  argument
457 	atomic_dec(&zmd->nr_mblks);  in dmz_free_mblock()
463 static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_insert_mblock()  argument
465 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_insert_mblock()
485 static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,  in dmz_get_mblock_fast()  argument
488 	struct rb_root *root = &zmd->mblk_rbtree;  in dmz_get_mblock_fast()
537 static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,  in dmz_get_mblock_slow()  argument
541 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;  in dmz_get_mblock_slow()
542 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock_slow()
549 	mblk = dmz_alloc_mblock(zmd, mblk_no);  in dmz_get_mblock_slow()
556 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
562 	m = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock_slow()
564 		spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
565 		dmz_free_mblock(zmd, mblk);  in dmz_get_mblock_slow()
572 	dmz_insert_mblock(zmd, mblk);  in dmz_get_mblock_slow()
574 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock_slow()
589 static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,  in dmz_shrink_mblock_cache()  argument
595 	if (!zmd->max_nr_mblks)  in dmz_shrink_mblock_cache()
598 	while (!list_empty(&zmd->mblk_lru_list) &&  in dmz_shrink_mblock_cache()
599 	       atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&  in dmz_shrink_mblock_cache()
601 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_shrink_mblock_cache()
604 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_shrink_mblock_cache()
605 		dmz_free_mblock(zmd, mblk);  in dmz_shrink_mblock_cache()
618 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_count()  local
620 	return atomic_read(&zmd->nr_mblks);  in dmz_mblock_shrinker_count()
629 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);  in dmz_mblock_shrinker_scan()  local
632 	spin_lock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
633 	count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);  in dmz_mblock_shrinker_scan()
634 	spin_unlock(&zmd->mblk_lock);  in dmz_mblock_shrinker_scan()
642 static void dmz_release_mblock(struct dmz_metadata *zmd,  in dmz_release_mblock()  argument
649 	spin_lock(&zmd->mblk_lock);  in dmz_release_mblock()
654 			rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_release_mblock()
655 			dmz_free_mblock(zmd, mblk);  in dmz_release_mblock()
657 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_release_mblock()
658 			dmz_shrink_mblock_cache(zmd, 1);  in dmz_release_mblock()
662 	spin_unlock(&zmd->mblk_lock);  in dmz_release_mblock()
669 static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,  in dmz_get_mblock()  argument
673 	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_get_mblock()
676 	spin_lock(&zmd->mblk_lock);  in dmz_get_mblock()
677 	mblk = dmz_get_mblock_fast(zmd, mblk_no);  in dmz_get_mblock()
678 	spin_unlock(&zmd->mblk_lock);  in dmz_get_mblock()
682 		mblk = dmz_get_mblock_slow(zmd, mblk_no);  in dmz_get_mblock()
691 		dmz_release_mblock(zmd, mblk);  in dmz_get_mblock()
702 static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)  in dmz_dirty_mblock()  argument
704 	spin_lock(&zmd->mblk_lock);  in dmz_dirty_mblock()
706 		list_add_tail(&mblk->link, &zmd->mblk_dirty_list);  in dmz_dirty_mblock()
707 	spin_unlock(&zmd->mblk_lock);  in dmz_dirty_mblock()
713 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,  in dmz_write_mblock()  argument
716 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_mblock()
717 	sector_t block = zmd->sb[set].block + mblk->no;  in dmz_write_mblock()
767 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)  in dmz_write_sb()  argument
769 	struct dmz_mblock *mblk = zmd->sb[set].mblk;  in dmz_write_sb()
770 	struct dmz_super *sb = zmd->sb[set].sb;  in dmz_write_sb()
771 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_sb()
773 	u64 sb_gen = zmd->sb_gen + 1;  in dmz_write_sb()
778 	sb->version = cpu_to_le32(zmd->sb_version);  in dmz_write_sb()
779 	if (zmd->sb_version > 1) {  in dmz_write_sb()
781 		export_uuid(sb->dmz_uuid, &zmd->uuid);  in dmz_write_sb()
782 		memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);  in dmz_write_sb()
793 	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;  in dmz_write_sb()
795 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);  in dmz_write_sb()
796 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);  in dmz_write_sb()
797 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);  in dmz_write_sb()
799 	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);  in dmz_write_sb()
800 	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);  in dmz_write_sb()
805 	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,  in dmz_write_sb()
816 static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_write_dirty_mblocks()  argument
821 	struct dmz_dev *dev = zmd->sb[set].dev;  in dmz_write_dirty_mblocks()
828 		ret = dmz_write_mblock(zmd, mblk, set);  in dmz_write_dirty_mblocks()
859 static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,  in dmz_log_dirty_mblocks()  argument
862 	unsigned int log_set = zmd->mblk_primary ^ 0x1;  in dmz_log_dirty_mblocks()
866 	ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);  in dmz_log_dirty_mblocks()
874 	ret = dmz_write_sb(zmd, log_set);  in dmz_log_dirty_mblocks()
884 int dmz_flush_metadata(struct dmz_metadata *zmd)  in dmz_flush_metadata()  argument
891 	if (WARN_ON(!zmd))  in dmz_flush_metadata()
901 	down_write(&zmd->mblk_sem);  in dmz_flush_metadata()
902 	dev = zmd->sb[zmd->mblk_primary].dev;  in dmz_flush_metadata()
908 	dmz_lock_flush(zmd);  in dmz_flush_metadata()
916 	spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
917 	list_splice_init(&zmd->mblk_dirty_list, &write_list);  in dmz_flush_metadata()
918 	spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
931 	ret = dmz_log_dirty_mblocks(zmd, &write_list);  in dmz_flush_metadata()
939 	ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);  in dmz_flush_metadata()
943 	ret = dmz_write_sb(zmd, zmd->mblk_primary);  in dmz_flush_metadata()
951 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
954 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);  in dmz_flush_metadata()
955 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
958 	zmd->sb_gen++;  in dmz_flush_metadata()
960 	dmz_unlock_flush(zmd);  in dmz_flush_metadata()
961 	up_write(&zmd->mblk_sem);  in dmz_flush_metadata()
967 		spin_lock(&zmd->mblk_lock);  in dmz_flush_metadata()
968 		list_splice(&write_list, &zmd->mblk_dirty_list);  in dmz_flush_metadata()
969 		spin_unlock(&zmd->mblk_lock);  in dmz_flush_metadata()
979 static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,  in dmz_check_sb()  argument
994 	zmd->sb_version = le32_to_cpu(sb->version);  in dmz_check_sb()
995 	if (zmd->sb_version > DMZ_META_VER) {  in dmz_check_sb()
997 			    DMZ_META_VER, zmd->sb_version);  in dmz_check_sb()
1000 	if (zmd->sb_version < 2 && tertiary) {  in dmz_check_sb()
1016 	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {  in dmz_check_sb()
1020 			    (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);  in dmz_check_sb()
1023 	if (zmd->sb_version > 1) {  in dmz_check_sb()
1030 		} else if (uuid_is_null(&zmd->uuid)) {  in dmz_check_sb()
1031 			uuid_copy(&zmd->uuid, &sb_uuid);  in dmz_check_sb()
1032 		} else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {  in dmz_check_sb()
1035 				    &sb_uuid, &zmd->uuid);  in dmz_check_sb()
1038 		if (!strlen(zmd->label))  in dmz_check_sb()
1039 			memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);  in dmz_check_sb()
1040 		else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {  in dmz_check_sb()
1043 				    sb->dmz_label, zmd->label);  in dmz_check_sb()
1064 	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)  in dmz_check_sb()
1065 		>> zmd->zone_nr_blocks_shift;  in dmz_check_sb()
1067 	    (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||  in dmz_check_sb()
1068 	    (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {  in dmz_check_sb()
1074 	    le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {  in dmz_check_sb()
1079 	nr_data_zones = zmd->nr_useable_zones -  in dmz_check_sb()
1088 	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);  in dmz_check_sb()
1089 	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);  in dmz_check_sb()
1090 	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);  in dmz_check_sb()
1091 	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);  in dmz_check_sb()
1092 	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);  in dmz_check_sb()
1093 	zmd->nr_meta_zones = nr_meta_zones;  in dmz_check_sb()
1094 	zmd->nr_data_zones = nr_data_zones;  in dmz_check_sb()
1102 static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_read_sb()  argument
1104 	dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu",  in dmz_read_sb()
1116 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)  in dmz_lookup_secondary_sb()  argument
1118 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_lookup_secondary_sb()
1120 	unsigned int zone_id = zmd->sb[0].zone->id;  in dmz_lookup_secondary_sb()
1124 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_lookup_secondary_sb()
1128 	zmd->sb[1].mblk = mblk;  in dmz_lookup_secondary_sb()
1129 	zmd->sb[1].sb = mblk->data;  in dmz_lookup_secondary_sb()
1132 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;  in dmz_lookup_secondary_sb()
1133 	zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);  in dmz_lookup_secondary_sb()
1134 	zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_lookup_secondary_sb()
1135 	for (i = 1; i < zmd->nr_rnd_zones; i++) {  in dmz_lookup_secondary_sb()
1136 		if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)  in dmz_lookup_secondary_sb()
1138 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)  in dmz_lookup_secondary_sb()
1140 		zmd->sb[1].block += zone_nr_blocks;  in dmz_lookup_secondary_sb()
1141 		zmd->sb[1].zone = dmz_get(zmd, zone_id + i);  in dmz_lookup_secondary_sb()
1144 	dmz_free_mblock(zmd, mblk);  in dmz_lookup_secondary_sb()
1145 	zmd->sb[1].mblk = NULL;  in dmz_lookup_secondary_sb()
1146 	zmd->sb[1].zone = NULL;  in dmz_lookup_secondary_sb()
1147 	zmd->sb[1].dev = NULL;  in dmz_lookup_secondary_sb()
1155 static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)  in dmz_get_sb()  argument
1161 	mblk = dmz_alloc_mblock(zmd, 0);  in dmz_get_sb()
1169 	ret = dmz_read_sb(zmd, sb, set);  in dmz_get_sb()
1171 		dmz_free_mblock(zmd, mblk);  in dmz_get_sb()
1182 static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)  in dmz_recover_mblocks()  argument
1188 	dmz_dev_warn(zmd->sb[dst_set].dev,  in dmz_recover_mblocks()
1192 		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_recover_mblocks()
1194 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_recover_mblocks()
1201 	for (i = 1; i < zmd->nr_meta_blocks; i++) {  in dmz_recover_mblocks()
1202 		ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,  in dmz_recover_mblocks()
1203 				     zmd->sb[src_set].block + i, page);  in dmz_recover_mblocks()
1206 		ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,  in dmz_recover_mblocks()
1207 				     zmd->sb[dst_set].block + i, page);  in dmz_recover_mblocks()
1213 	if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1214 		zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);  in dmz_recover_mblocks()
1215 		if (!zmd->sb[dst_set].mblk) {  in dmz_recover_mblocks()
1219 		zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;  in dmz_recover_mblocks()
1222 	ret = dmz_write_sb(zmd, dst_set);  in dmz_recover_mblocks()
1232 static int dmz_load_sb(struct dmz_metadata *zmd)  in dmz_load_sb()  argument
1238 	if (!zmd->sb[0].zone) {  in dmz_load_sb()
1239 		dmz_zmd_err(zmd, "Primary super block zone not set");  in dmz_load_sb()
1244 	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);  in dmz_load_sb()
1245 	zmd->sb[0].dev = zmd->sb[0].zone->dev;  in dmz_load_sb()
1246 	ret = dmz_get_sb(zmd, &zmd->sb[0], 0);  in dmz_load_sb()
1248 		dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");  in dmz_load_sb()
1252 	ret = dmz_check_sb(zmd, &zmd->sb[0], false);  in dmz_load_sb()
1257 		if (!zmd->sb[1].zone) {  in dmz_load_sb()
1259 				zmd->sb[0].zone->id + zmd->nr_meta_zones;  in dmz_load_sb()
1261 			zmd->sb[1].zone = dmz_get(zmd, zone_id);  in dmz_load_sb()
1263 		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);  in dmz_load_sb()
1264 		zmd->sb[1].dev = zmd->sb[0].dev;  in dmz_load_sb()
1265 		ret = dmz_get_sb(zmd, &zmd->sb[1], 1);  in dmz_load_sb()
1267 		ret = dmz_lookup_secondary_sb(zmd);  in dmz_load_sb()
1270 		dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");  in dmz_load_sb()
1274 	ret = dmz_check_sb(zmd, &zmd->sb[1], false);  in dmz_load_sb()
1280 		dmz_zmd_err(zmd, "No valid super block found");  in dmz_load_sb()
1285 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);  in dmz_load_sb()
1287 		ret = dmz_recover_mblocks(zmd, 0);  in dmz_load_sb()
1289 			dmz_dev_err(zmd->sb[0].dev,  in dmz_load_sb()
1296 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);  in dmz_load_sb()
1298 		ret = dmz_recover_mblocks(zmd, 1);  in dmz_load_sb()
1301 			dmz_dev_err(zmd->sb[1].dev,  in dmz_load_sb()
1308 		zmd->sb_gen = sb_gen[0];  in dmz_load_sb()
1309 		zmd->mblk_primary = 0;  in dmz_load_sb()
1311 		zmd->sb_gen = sb_gen[1];  in dmz_load_sb()
1312 		zmd->mblk_primary = 1;  in dmz_load_sb()
1315 	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,  in dmz_load_sb()
1317 		      zmd->mblk_primary, zmd->sb_gen);  in dmz_load_sb()
1319 	if (zmd->sb_version > 1) {  in dmz_load_sb()
1326 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_load_sb()
1328 			sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);  in dmz_load_sb()
1329 			sb->dev = &zmd->dev[i];  in dmz_load_sb()
1337 			ret = dmz_get_sb(zmd, sb, i + 1);  in dmz_load_sb()
1341 				dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1344 			ret = dmz_check_sb(zmd, sb, true);  in dmz_load_sb()
1345 			dmz_free_mblock(zmd, sb->mblk);  in dmz_load_sb()
1361 	struct dmz_metadata *zmd = dev->metadata;  in dmz_init_zone()  local
1365 	zone = dmz_insert(zmd, idx, dev);  in dmz_init_zone()
1369 	if (blkz->len != zmd->zone_nr_sectors) {  in dmz_init_zone()
1370 		if (zmd->sb_version > 1) {  in dmz_init_zone()
1408 		zmd->nr_useable_zones++;  in dmz_init_zone()
1410 			zmd->nr_rnd_zones++;  in dmz_init_zone()
1411 			if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {  in dmz_init_zone()
1413 				zmd->sb[0].zone = zone;  in dmz_init_zone()
1416 		if (zmd->nr_devs > 1 && num == 0) {  in dmz_init_zone()
1428 static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)  in dmz_emulate_zones()  argument
1436 		zone = dmz_insert(zmd, idx, dev);  in dmz_emulate_zones()
1441 		zmd->nr_cache_zones++;  in dmz_emulate_zones()
1442 		zmd->nr_useable_zones++;  in dmz_emulate_zones()
1443 		if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {  in dmz_emulate_zones()
1448 		zone_offset += zmd->zone_nr_sectors;  in dmz_emulate_zones()
1456 static void dmz_drop_zones(struct dmz_metadata *zmd)  in dmz_drop_zones()  argument
1460 	for(idx = 0; idx < zmd->nr_zones; idx++) {  in dmz_drop_zones()
1461 		struct dm_zone *zone = xa_load(&zmd->zones, idx);  in dmz_drop_zones()
1464 		xa_erase(&zmd->zones, idx);  in dmz_drop_zones()
1466 	xa_destroy(&zmd->zones);  in dmz_drop_zones()
1473 static int dmz_init_zones(struct dmz_metadata *zmd)  in dmz_init_zones()  argument
1476 	struct dmz_dev *zoned_dev = &zmd->dev[0];  in dmz_init_zones()
1479 	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;  in dmz_init_zones()
1480 	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);  in dmz_init_zones()
1481 	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);  in dmz_init_zones()
1482 	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);  in dmz_init_zones()
1483 	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;  in dmz_init_zones()
1484 	zmd->zone_nr_bitmap_blocks =  in dmz_init_zones()
1485 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);  in dmz_init_zones()
1486 	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,  in dmz_init_zones()
1490 	zmd->nr_zones = 0;  in dmz_init_zones()
1491 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1492 		struct dmz_dev *dev = &zmd->dev[i];  in dmz_init_zones()
1494 		dev->metadata = zmd;  in dmz_init_zones()
1495 		zmd->nr_zones += dev->nr_zones;  in dmz_init_zones()
1506 	if (!zmd->nr_zones) {  in dmz_init_zones()
1507 		DMERR("(%s): No zones found", zmd->devname);  in dmz_init_zones()
1510 	xa_init(&zmd->zones);  in dmz_init_zones()
1513 		zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);  in dmz_init_zones()
1515 	if (zmd->nr_devs > 1) {  in dmz_init_zones()
1516 		ret = dmz_emulate_zones(zmd, &zmd->dev[0]);  in dmz_init_zones()
1519 				zmd->devname, ret);  in dmz_init_zones()
1520 			dmz_drop_zones(zmd);  in dmz_init_zones()
1528 		zmd->sb[0].zone = dmz_get(zmd, 0);  in dmz_init_zones()
1530 		for (i = 1; i < zmd->nr_devs; i++) {  in dmz_init_zones()
1531 			zoned_dev = &zmd->dev[i];  in dmz_init_zones()
1538 					zmd->devname, ret);  in dmz_init_zones()
1539 				dmz_drop_zones(zmd);  in dmz_init_zones()
1555 			zmd->devname, ret);  in dmz_init_zones()
1556 		dmz_drop_zones(zmd);  in dmz_init_zones()
1585 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_update_zone()  argument
1601 	ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,  in dmz_update_zone()
1621 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,  in dmz_handle_seq_write_err()  argument
1629 	ret = dmz_update_zone(zmd, zone);  in dmz_handle_seq_write_err()
1637 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_handle_seq_write_err()
1647 static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_reset_zone()  argument
1664 				       dmz_start_sect(zmd, zone),  in dmz_reset_zone()
1665 				       zmd->zone_nr_sectors, GFP_NOIO);  in dmz_reset_zone()
1680 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1685 static int dmz_load_mapping(struct dmz_metadata *zmd)  in dmz_load_mapping()  argument
1695 	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,  in dmz_load_mapping()
1697 	if (!zmd->map_mblk)  in dmz_load_mapping()
1701 	while (chunk < zmd->nr_chunks) {  in dmz_load_mapping()
1704 			dmap_mblk = dmz_get_mblock(zmd, i + 1);  in dmz_load_mapping()
1707 			zmd->map_mblk[i] = dmap_mblk;  in dmz_load_mapping()
1718 		if (dzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1719 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",  in dmz_load_mapping()
1724 		dzone = dmz_get(zmd, dzone_id);  in dmz_load_mapping()
1726 			dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",  in dmz_load_mapping()
1732 		dmz_get_zone_weight(zmd, dzone);  in dmz_load_mapping()
1735 			list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1746 		if (bzone_id >= zmd->nr_zones) {  in dmz_load_mapping()
1747 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",  in dmz_load_mapping()
1752 		bzone = dmz_get(zmd, bzone_id);  in dmz_load_mapping()
1754 			dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",  in dmz_load_mapping()
1759 			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",  in dmz_load_mapping()
1769 		dmz_get_zone_weight(zmd, bzone);  in dmz_load_mapping()
1771 			list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_load_mapping()
1786 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_load_mapping()
1787 		dzone = dmz_get(zmd, i);  in dmz_load_mapping()
1796 			zmd->nr_cache++;  in dmz_load_mapping()
1811 			list_add_tail(&dzone->link, &zmd->unmap_cache_list);  in dmz_load_mapping()
1812 			atomic_inc(&zmd->unmap_nr_cache);  in dmz_load_mapping()
1817 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {  in dmz_load_mapping()
1818 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);  in dmz_load_mapping()
1820 			atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_load_mapping()
1835 static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,  in dmz_set_chunk_mapping()  argument
1838 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_set_chunk_mapping()
1844 	dmz_dirty_mblock(zmd, dmap_mblk);  in dmz_set_chunk_mapping()
1851 static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in __dmz_lru_zone()  argument
1862 		list_add_tail(&zone->link, &zmd->map_cache_list);  in __dmz_lru_zone()
1873 static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_lru_zone()  argument
1875 	__dmz_lru_zone(zmd, zone);  in dmz_lru_zone()
1877 		__dmz_lru_zone(zmd, zone->bzone);  in dmz_lru_zone()
1883 static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)  in dmz_wait_for_free_zones()  argument
1887 	prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);  in dmz_wait_for_free_zones()
1888 	dmz_unlock_map(zmd);  in dmz_wait_for_free_zones()
1889 	dmz_unlock_metadata(zmd);  in dmz_wait_for_free_zones()
1893 	dmz_lock_metadata(zmd);  in dmz_wait_for_free_zones()
1894 	dmz_lock_map(zmd);  in dmz_wait_for_free_zones()
1895 	finish_wait(&zmd->free_wq, &wait);  in dmz_wait_for_free_zones()
1928 static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_wait_for_reclaim()  argument
1930 	dmz_unlock_map(zmd);  in dmz_wait_for_reclaim()
1931 	dmz_unlock_metadata(zmd);  in dmz_wait_for_reclaim()
1935 	dmz_lock_metadata(zmd);  in dmz_wait_for_reclaim()
1936 	dmz_lock_map(zmd);  in dmz_wait_for_reclaim()
1942 static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_rnd_zone_for_reclaim()  argument
1950 	if (zmd->nr_cache) {  in dmz_get_rnd_zone_for_reclaim()
1951 		zone_list = &zmd->map_cache_list;  in dmz_get_rnd_zone_for_reclaim()
1954 			zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
1956 		zone_list = &zmd->dev[idx].map_rnd_list;  in dmz_get_rnd_zone_for_reclaim()
2001 static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_seq_zone_for_reclaim()  argument
2006 	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {  in dmz_get_seq_zone_for_reclaim()
2019 struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,  in dmz_get_zone_for_reclaim()  argument
2032 	dmz_lock_map(zmd);  in dmz_get_zone_for_reclaim()
2033 	if (list_empty(&zmd->reserved_seq_zones_list))  in dmz_get_zone_for_reclaim()
2034 		zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);  in dmz_get_zone_for_reclaim()
2036 		zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);  in dmz_get_zone_for_reclaim()
2037 	dmz_unlock_map(zmd);  in dmz_get_zone_for_reclaim()
2048 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)  in dmz_get_chunk_mapping()  argument
2050 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];  in dmz_get_chunk_mapping()
2056 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_mapping()
2058 	dmz_lock_map(zmd);  in dmz_get_chunk_mapping()
2071 		dzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_mapping()
2073 			if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_mapping()
2077 			dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_mapping()
2081 		dmz_map_zone(zmd, dzone, chunk);  in dmz_get_chunk_mapping()
2085 		dzone = dmz_get(zmd, dzone_id);  in dmz_get_chunk_mapping()
2097 			ret = dmz_handle_seq_write_err(zmd, dzone);  in dmz_get_chunk_mapping()
2112 		dmz_wait_for_reclaim(zmd, dzone);  in dmz_get_chunk_mapping()
2116 	dmz_lru_zone(zmd, dzone);  in dmz_get_chunk_mapping()
2118 	dmz_unlock_map(zmd);  in dmz_get_chunk_mapping()
2129 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)  in dmz_put_chunk_mapping()  argument
2133 	dmz_lock_map(zmd);  in dmz_put_chunk_mapping()
2138 			dmz_lru_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2141 			dmz_unmap_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2142 			dmz_free_zone(zmd, bzone);  in dmz_put_chunk_mapping()
2150 		dmz_lru_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2153 		dmz_unmap_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2154 		dmz_free_zone(zmd, dzone);  in dmz_put_chunk_mapping()
2157 	dmz_unlock_map(zmd);  in dmz_put_chunk_mapping()
2164 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,  in dmz_get_chunk_buffer()  argument
2168 	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;  in dmz_get_chunk_buffer()
2170 	dmz_lock_map(zmd);  in dmz_get_chunk_buffer()
2177 	bzone = dmz_alloc_zone(zmd, 0, alloc_flags);  in dmz_get_chunk_buffer()
2179 		if (dmz_dev_is_dying(zmd)) {  in dmz_get_chunk_buffer()
2183 		dmz_wait_for_free_zones(zmd);  in dmz_get_chunk_buffer()
2188 	dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);  in dmz_get_chunk_buffer()
2195 		list_add_tail(&bzone->link, &zmd->map_cache_list);  in dmz_get_chunk_buffer()
2199 	dmz_unlock_map(zmd);  in dmz_get_chunk_buffer()
2208 struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,  in dmz_alloc_zone()  argument
2217 		for (i = 0; i < zmd->nr_devs; i++)  in dmz_alloc_zone()
2218 			dmz_schedule_reclaim(zmd->dev[i].reclaim);  in dmz_alloc_zone()
2224 		list = &zmd->unmap_cache_list;  in dmz_alloc_zone()
2226 		list = &zmd->dev[dev_idx].unmap_rnd_list;  in dmz_alloc_zone()
2228 		list = &zmd->dev[dev_idx].unmap_seq_list;  in dmz_alloc_zone()
2239 		if (i < zmd->nr_devs) {  in dmz_alloc_zone()
2240 			dev_idx = (dev_idx + 1) % zmd->nr_devs;  in dmz_alloc_zone()
2248 		zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,  in dmz_alloc_zone()
2252 			atomic_dec(&zmd->nr_reserved_seq_zones);  in dmz_alloc_zone()
2261 		atomic_dec(&zmd->unmap_nr_cache);  in dmz_alloc_zone()
2268 		dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);  in dmz_alloc_zone()
2273 		dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);  in dmz_alloc_zone()
2284 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_free_zone()  argument
2288 		dmz_reset_zone(zmd, zone);  in dmz_free_zone()
2292 		list_add_tail(&zone->link, &zmd->unmap_cache_list);  in dmz_free_zone()
2293 		atomic_inc(&zmd->unmap_nr_cache);  in dmz_free_zone()
2298 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);  in dmz_free_zone()
2299 		atomic_inc(&zmd->nr_reserved_seq_zones);  in dmz_free_zone()
2305 	wake_up_all(&zmd->free_wq);  in dmz_free_zone()
2312 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,  in dmz_map_zone()  argument
2316 	dmz_set_chunk_mapping(zmd, chunk, dzone->id,  in dmz_map_zone()
2320 		list_add_tail(&dzone->link, &zmd->map_cache_list);  in dmz_map_zone()
2331 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_unmap_zone()  argument
2362 	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);  in dmz_unmap_zone()
2403 static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,  in dmz_get_bitmap()  argument
2407 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +  in dmz_get_bitmap()
2408 		(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +  in dmz_get_bitmap()
2411 	return dmz_get_mblock(zmd, bitmap_block);  in dmz_get_bitmap()
2417 int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_copy_valid_blocks()  argument
2424 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_copy_valid_blocks()
2425 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);  in dmz_copy_valid_blocks()
2428 		to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);  in dmz_copy_valid_blocks()
2430 			dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2435 		dmz_dirty_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2437 		dmz_release_mblock(zmd, to_mblk);  in dmz_copy_valid_blocks()
2438 		dmz_release_mblock(zmd, from_mblk);  in dmz_copy_valid_blocks()
2440 		chunk_block += zmd->zone_bits_per_mblk;  in dmz_copy_valid_blocks()
2452 int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,  in dmz_merge_valid_blocks()  argument
2459 	while (chunk_block < zmd->zone_nr_blocks) {  in dmz_merge_valid_blocks()
2461 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);  in dmz_merge_valid_blocks()
2466 		ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);  in dmz_merge_valid_blocks()
2479 int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_validate_blocks()  argument
2483 	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;  in dmz_validate_blocks()
2487 	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",  in dmz_validate_blocks()
2495 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_validate_blocks()
2501 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_validate_blocks()
2505 			dmz_dirty_mblock(zmd, mblk);  in dmz_validate_blocks()
2508 		dmz_release_mblock(zmd, mblk);  in dmz_validate_blocks()
2517 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",  in dmz_validate_blocks()
2560 int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_invalidate_blocks()  argument
2567 	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",  in dmz_invalidate_blocks()
2570 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_invalidate_blocks()
2574 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_invalidate_blocks()
2580 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_invalidate_blocks()
2585 			dmz_dirty_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2588 		dmz_release_mblock(zmd, mblk);  in dmz_invalidate_blocks()
2597 		dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",  in dmz_invalidate_blocks()
2608 static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_test_block()  argument
2614 	WARN_ON(chunk_block >= zmd->zone_nr_blocks);  in dmz_test_block()
2617 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_test_block()
2625 	dmz_release_mblock(zmd, mblk);  in dmz_test_block()
2634 static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_to_next_set_block()  argument
2640 	unsigned int zone_bits = zmd->zone_bits_per_mblk;  in dmz_to_next_set_block()
2644 	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);  in dmz_to_next_set_block()
2648 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_to_next_set_block()
2660 		dmz_release_mblock(zmd, mblk);  in dmz_to_next_set_block()
2677 int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_block_valid()  argument
2682 	valid = dmz_test_block(zmd, zone, chunk_block);  in dmz_block_valid()
2687 	return dmz_to_next_set_block(zmd, zone, chunk_block,  in dmz_block_valid()
2688 				     zmd->zone_nr_blocks - chunk_block, 0);  in dmz_block_valid()
2697 int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,  in dmz_first_valid_block()  argument
2703 	ret = dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2704 				    zmd->zone_nr_blocks - start_block, 1);  in dmz_first_valid_block()
2711 	return dmz_to_next_set_block(zmd, zone, start_block,  in dmz_first_valid_block()
2712 				     zmd->zone_nr_blocks - start_block, 0);  in dmz_first_valid_block()
2746 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)  in dmz_get_zone_weight()  argument
2751 	unsigned int nr_blocks = zmd->zone_nr_blocks;  in dmz_get_zone_weight()
2757 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);  in dmz_get_zone_weight()
2766 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);  in dmz_get_zone_weight()
2769 		dmz_release_mblock(zmd, mblk);  in dmz_get_zone_weight()
2781 static void dmz_cleanup_metadata(struct dmz_metadata *zmd)  in dmz_cleanup_metadata()  argument
2788 	if (zmd->map_mblk) {  in dmz_cleanup_metadata()
2789 		for (i = 0; i < zmd->nr_map_blocks; i++)  in dmz_cleanup_metadata()
2790 			dmz_release_mblock(zmd, zmd->map_mblk[i]);  in dmz_cleanup_metadata()
2791 		kfree(zmd->map_mblk);  in dmz_cleanup_metadata()
2792 		zmd->map_mblk = NULL;  in dmz_cleanup_metadata()
2797 		if (zmd->sb[i].mblk) {  in dmz_cleanup_metadata()
2798 			dmz_free_mblock(zmd, zmd->sb[i].mblk);  in dmz_cleanup_metadata()
2799 			zmd->sb[i].mblk = NULL;  in dmz_cleanup_metadata()
2804 	while (!list_empty(&zmd->mblk_dirty_list)) {  in dmz_cleanup_metadata()
2805 		mblk = list_first_entry(&zmd->mblk_dirty_list,  in dmz_cleanup_metadata()
2807 		dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",  in dmz_cleanup_metadata()
2810 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2811 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2814 	while (!list_empty(&zmd->mblk_lru_list)) {  in dmz_cleanup_metadata()
2815 		mblk = list_first_entry(&zmd->mblk_lru_list,  in dmz_cleanup_metadata()
2818 		rb_erase(&mblk->node, &zmd->mblk_rbtree);  in dmz_cleanup_metadata()
2819 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2823 	root = &zmd->mblk_rbtree;  in dmz_cleanup_metadata()
2825 		dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",  in dmz_cleanup_metadata()
2828 		dmz_free_mblock(zmd, mblk);  in dmz_cleanup_metadata()
2832 	dmz_drop_zones(zmd);  in dmz_cleanup_metadata()
2834 	mutex_destroy(&zmd->mblk_flush_lock);  in dmz_cleanup_metadata()
2835 	mutex_destroy(&zmd->map_lock);  in dmz_cleanup_metadata()
2838 static void dmz_print_dev(struct dmz_metadata *zmd, int num)  in dmz_print_dev()  argument
2840 	struct dmz_dev *dev = &zmd->dev[num];  in dmz_print_dev()
2848 	if (zmd->sb_version > 1) {  in dmz_print_dev()
2850 			dev->zone_offset << zmd->zone_nr_sectors_shift;  in dmz_print_dev()
2855 			     dev->nr_zones, (u64)zmd->zone_nr_sectors,  in dmz_print_dev()
2861 			     dev->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_print_dev()
2872 	struct dmz_metadata *zmd;  in dmz_ctr_metadata()  local
2877 	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);  in dmz_ctr_metadata()
2878 	if (!zmd)  in dmz_ctr_metadata()
2881 	strcpy(zmd->devname, devname);  in dmz_ctr_metadata()
2882 	zmd->dev = dev;  in dmz_ctr_metadata()
2883 	zmd->nr_devs = num_dev;  in dmz_ctr_metadata()
2884 	zmd->mblk_rbtree = RB_ROOT;  in dmz_ctr_metadata()
2885 	init_rwsem(&zmd->mblk_sem);  in dmz_ctr_metadata()
2886 	mutex_init(&zmd->mblk_flush_lock);  in dmz_ctr_metadata()
2887 	spin_lock_init(&zmd->mblk_lock);  in dmz_ctr_metadata()
2888 	INIT_LIST_HEAD(&zmd->mblk_lru_list);  in dmz_ctr_metadata()
2889 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);  in dmz_ctr_metadata()
2891 	mutex_init(&zmd->map_lock);  in dmz_ctr_metadata()
2893 	atomic_set(&zmd->unmap_nr_cache, 0);  in dmz_ctr_metadata()
2894 	INIT_LIST_HEAD(&zmd->unmap_cache_list);  in dmz_ctr_metadata()
2895 	INIT_LIST_HEAD(&zmd->map_cache_list);  in dmz_ctr_metadata()
2897 	atomic_set(&zmd->nr_reserved_seq_zones, 0);  in dmz_ctr_metadata()
2898 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);  in dmz_ctr_metadata()
2900 	init_waitqueue_head(&zmd->free_wq);  in dmz_ctr_metadata()
2903 	ret = dmz_init_zones(zmd);  in dmz_ctr_metadata()
2908 	ret = dmz_load_sb(zmd);  in dmz_ctr_metadata()
2913 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {  in dmz_ctr_metadata()
2914 		zone = dmz_get(zmd, zmd->sb[0].zone->id + i);  in dmz_ctr_metadata()
2916 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2922 			dmz_zmd_err(zmd,  in dmz_ctr_metadata()
2930 	ret = dmz_load_mapping(zmd);  in dmz_ctr_metadata()
2940 	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;  in dmz_ctr_metadata()
2941 	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;  in dmz_ctr_metadata()
2942 	zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;  in dmz_ctr_metadata()
2943 	zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;  in dmz_ctr_metadata()
2944 	zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;  in dmz_ctr_metadata()
2947 	ret = register_shrinker(&zmd->mblk_shrinker);  in dmz_ctr_metadata()
2949 		dmz_zmd_err(zmd, "Register metadata cache shrinker failed");  in dmz_ctr_metadata()
2953 	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);  in dmz_ctr_metadata()
2954 	for (i = 0; i < zmd->nr_devs; i++)  in dmz_ctr_metadata()
2955 		dmz_print_dev(zmd, i);  in dmz_ctr_metadata()
2957 	dmz_zmd_info(zmd, "  %u zones of %llu 512-byte logical sectors",  in dmz_ctr_metadata()
2958 		     zmd->nr_zones, (u64)zmd->zone_nr_sectors);  in dmz_ctr_metadata()
2959 	dmz_zmd_debug(zmd, "  %u metadata zones",  in dmz_ctr_metadata()
2960 		      zmd->nr_meta_zones * 2);  in dmz_ctr_metadata()
2961 	dmz_zmd_debug(zmd, "  %u data zones for %u chunks",  in dmz_ctr_metadata()
2962 		      zmd->nr_data_zones, zmd->nr_chunks);  in dmz_ctr_metadata()
2963 	dmz_zmd_debug(zmd, "    %u cache zones (%u unmapped)",  in dmz_ctr_metadata()
2964 		      zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));  in dmz_ctr_metadata()
2965 	for (i = 0; i < zmd->nr_devs; i++) {  in dmz_ctr_metadata()
2966 		dmz_zmd_debug(zmd, "    %u random zones (%u unmapped)",  in dmz_ctr_metadata()
2967 			      dmz_nr_rnd_zones(zmd, i),  in dmz_ctr_metadata()
2968 			      dmz_nr_unmap_rnd_zones(zmd, i));  in dmz_ctr_metadata()
2969 		dmz_zmd_debug(zmd, "    %u sequential zones (%u unmapped)",  in dmz_ctr_metadata()
2970 			      dmz_nr_seq_zones(zmd, i),  in dmz_ctr_metadata()
2971 			      dmz_nr_unmap_seq_zones(zmd, i));  in dmz_ctr_metadata()
2973 	dmz_zmd_debug(zmd, "  %u reserved sequential data zones",  in dmz_ctr_metadata()
2974 		      zmd->nr_reserved_seq);  in dmz_ctr_metadata()
2975 	dmz_zmd_debug(zmd, "Format:");  in dmz_ctr_metadata()
2976 	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",  in dmz_ctr_metadata()
2977 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);  in dmz_ctr_metadata()
2978 	dmz_zmd_debug(zmd, "  %u data zone mapping blocks",  in dmz_ctr_metadata()
2979 		      zmd->nr_map_blocks);  in dmz_ctr_metadata()
2980 	dmz_zmd_debug(zmd, "  %u bitmap blocks",  in dmz_ctr_metadata()
2981 		      zmd->nr_bitmap_blocks);  in dmz_ctr_metadata()
2983 	*metadata = zmd;  in dmz_ctr_metadata()
2987 	dmz_cleanup_metadata(zmd);  in dmz_ctr_metadata()
2988 	kfree(zmd);  in dmz_ctr_metadata()
2997 void dmz_dtr_metadata(struct dmz_metadata *zmd)  in dmz_dtr_metadata()  argument
2999 	unregister_shrinker(&zmd->mblk_shrinker);  in dmz_dtr_metadata()
3000 	dmz_cleanup_metadata(zmd);  in dmz_dtr_metadata()
3001 	kfree(zmd);  in dmz_dtr_metadata()
3007 int dmz_resume_metadata(struct dmz_metadata *zmd)  in dmz_resume_metadata()  argument
3015 	for (i = 0; i < zmd->nr_zones; i++) {  in dmz_resume_metadata()
3016 		zone = dmz_get(zmd, i);  in dmz_resume_metadata()
3018 			dmz_zmd_err(zmd, "Unable to get zone %u", i);  in dmz_resume_metadata()
3023 		ret = dmz_update_zone(zmd, zone);  in dmz_resume_metadata()
3025 			dmz_zmd_err(zmd, "Broken zone %u", i);  in dmz_resume_metadata()
3030 			dmz_zmd_warn(zmd, "Zone %u is offline", i);  in dmz_resume_metadata()
3038 			dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",  in dmz_resume_metadata()
3041 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,  in dmz_resume_metadata()
3042 					      zmd->zone_nr_blocks - zone->wp_block);  in dmz_resume_metadata()