239 static int superblock_read_lock(struct dm_cache_metadata *cmd,  in superblock_read_lock()  argument
242 	return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,  in superblock_read_lock()
246 static int superblock_lock_zero(struct dm_cache_metadata *cmd,  in superblock_lock_zero()  argument
249 	return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,  in superblock_lock_zero()
253 static int superblock_lock(struct dm_cache_metadata *cmd,  in superblock_lock()  argument
256 	return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,  in superblock_lock()
291 static void __setup_mapping_info(struct dm_cache_metadata *cmd)  in __setup_mapping_info()  argument
300 	dm_array_info_init(&cmd->info, cmd->tm, &vt);  in __setup_mapping_info()
302 	if (cmd->policy_hint_size) {  in __setup_mapping_info()
304 		dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);  in __setup_mapping_info()
308 static int __save_sm_root(struct dm_cache_metadata *cmd)  in __save_sm_root()  argument
313 	r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);  in __save_sm_root()
317 	return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,  in __save_sm_root()
321 static void __copy_sm_root(struct dm_cache_metadata *cmd,  in __copy_sm_root()  argument
325 	       &cmd->metadata_space_map_root,  in __copy_sm_root()
326 	       sizeof(cmd->metadata_space_map_root));  in __copy_sm_root()
329 static bool separate_dirty_bits(struct dm_cache_metadata *cmd)  in separate_dirty_bits()  argument
331 	return cmd->version >= 2;  in separate_dirty_bits()
334 static int __write_initial_superblock(struct dm_cache_metadata *cmd)  in __write_initial_superblock()  argument
339 	sector_t bdev_size = bdev_nr_sectors(cmd->bdev);  in __write_initial_superblock()
345 	r = dm_tm_pre_commit(cmd->tm);  in __write_initial_superblock()
353 	r = __save_sm_root(cmd);  in __write_initial_superblock()
357 	r = superblock_lock_zero(cmd, &sblock);  in __write_initial_superblock()
365 	disk_super->version = cpu_to_le32(cmd->version);  in __write_initial_superblock()
370 	__copy_sm_root(cmd, disk_super);  in __write_initial_superblock()
372 	disk_super->mapping_root = cpu_to_le64(cmd->root);  in __write_initial_superblock()
373 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);  in __write_initial_superblock()
374 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);  in __write_initial_superblock()
375 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);  in __write_initial_superblock()
376 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));  in __write_initial_superblock()
378 	disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);  in __write_initial_superblock()
386 	if (separate_dirty_bits(cmd))  in __write_initial_superblock()
387 		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);  in __write_initial_superblock()
389 	return dm_tm_commit(cmd->tm, sblock);  in __write_initial_superblock()
392 static int __format_metadata(struct dm_cache_metadata *cmd)  in __format_metadata()  argument
396 	r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,  in __format_metadata()
397 				 &cmd->tm, &cmd->metadata_sm);  in __format_metadata()
403 	__setup_mapping_info(cmd);  in __format_metadata()
405 	r = dm_array_empty(&cmd->info, &cmd->root);  in __format_metadata()
409 	if (separate_dirty_bits(cmd)) {  in __format_metadata()
410 		dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);  in __format_metadata()
411 		r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);  in __format_metadata()
416 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);  in __format_metadata()
417 	r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);  in __format_metadata()
421 	cmd->discard_block_size = 0;  in __format_metadata()
422 	cmd->discard_nr_blocks = 0;  in __format_metadata()
424 	r = __write_initial_superblock(cmd);  in __format_metadata()
428 	cmd->clean_when_opened = true;  in __format_metadata()
432 	dm_tm_destroy(cmd->tm);  in __format_metadata()
433 	dm_sm_destroy(cmd->metadata_sm);  in __format_metadata()
439 				     struct dm_cache_metadata *cmd)  in __check_incompat_features()  argument
454 	if (bdev_read_only(cmd->bdev))  in __check_incompat_features()
467 static int __open_metadata(struct dm_cache_metadata *cmd)  in __open_metadata()  argument
474 	r = superblock_read_lock(cmd, &sblock);  in __open_metadata()
483 	if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {  in __open_metadata()
486 		      (unsigned long long)cmd->data_block_size);  in __open_metadata()
491 	r = __check_incompat_features(disk_super, cmd);  in __open_metadata()
495 	r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,  in __open_metadata()
498 			       &cmd->tm, &cmd->metadata_sm);  in __open_metadata()
504 	__setup_mapping_info(cmd);  in __open_metadata()
505 	dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);  in __open_metadata()
506 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);  in __open_metadata()
508 	cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);  in __open_metadata()
518 static int __open_or_format_metadata(struct dm_cache_metadata *cmd,  in __open_or_format_metadata()  argument
524 	r = __superblock_all_zeroes(cmd->bm, &unformatted);  in __open_or_format_metadata()
529 		return format_device ? __format_metadata(cmd) : -EPERM;  in __open_or_format_metadata()
531 	return __open_metadata(cmd);  in __open_or_format_metadata()
534 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,  in __create_persistent_data_objects()  argument
539 	cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,  in __create_persistent_data_objects()
541 	if (IS_ERR(cmd->bm)) {  in __create_persistent_data_objects()
543 		r = PTR_ERR(cmd->bm);  in __create_persistent_data_objects()
544 		cmd->bm = NULL;  in __create_persistent_data_objects()
548 	r = __open_or_format_metadata(cmd, may_format_device);  in __create_persistent_data_objects()
550 		dm_block_manager_destroy(cmd->bm);  in __create_persistent_data_objects()
551 		cmd->bm = NULL;  in __create_persistent_data_objects()
557 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd,  in __destroy_persistent_data_objects()  argument
560 	dm_sm_destroy(cmd->metadata_sm);  in __destroy_persistent_data_objects()
561 	dm_tm_destroy(cmd->tm);  in __destroy_persistent_data_objects()
563 		dm_block_manager_destroy(cmd->bm);  in __destroy_persistent_data_objects()
588 static void read_superblock_fields(struct dm_cache_metadata *cmd,  in read_superblock_fields()  argument
591 	cmd->version = le32_to_cpu(disk_super->version);  in read_superblock_fields()
592 	cmd->flags = le32_to_cpu(disk_super->flags);  in read_superblock_fields()
593 	cmd->root = le64_to_cpu(disk_super->mapping_root);  in read_superblock_fields()
594 	cmd->hint_root = le64_to_cpu(disk_super->hint_root);  in read_superblock_fields()
595 	cmd->discard_root = le64_to_cpu(disk_super->discard_root);  in read_superblock_fields()
596 	cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);  in read_superblock_fields()
597 	cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));  in read_superblock_fields()
598 	cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);  in read_superblock_fields()
599 	cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));  in read_superblock_fields()
600 	strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));  in read_superblock_fields()
601 	cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);  in read_superblock_fields()
602 	cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);  in read_superblock_fields()
603 	cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);  in read_superblock_fields()
604 	cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);  in read_superblock_fields()
606 	cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);  in read_superblock_fields()
607 	cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);  in read_superblock_fields()
608 	cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);  in read_superblock_fields()
609 	cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);  in read_superblock_fields()
611 	if (separate_dirty_bits(cmd))  in read_superblock_fields()
612 		cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);  in read_superblock_fields()
614 	cmd->changed = false;  in read_superblock_fields()
620 static int __begin_transaction_flags(struct dm_cache_metadata *cmd,  in __begin_transaction_flags()  argument
627 	r = superblock_lock(cmd, &sblock);  in __begin_transaction_flags()
633 	read_superblock_fields(cmd, disk_super);  in __begin_transaction_flags()
636 	return dm_bm_flush(cmd->bm);  in __begin_transaction_flags()
639 static int __begin_transaction(struct dm_cache_metadata *cmd)  in __begin_transaction()  argument
649 	r = superblock_read_lock(cmd, &sblock);  in __begin_transaction()
654 	read_superblock_fields(cmd, disk_super);  in __begin_transaction()
660 static int __commit_transaction(struct dm_cache_metadata *cmd,  in __commit_transaction()  argument
672 	if (separate_dirty_bits(cmd)) {  in __commit_transaction()
673 		r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,  in __commit_transaction()
674 				    &cmd->dirty_root);  in __commit_transaction()
679 	r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,  in __commit_transaction()
680 			    &cmd->discard_root);  in __commit_transaction()
684 	r = dm_tm_pre_commit(cmd->tm);  in __commit_transaction()
688 	r = __save_sm_root(cmd);  in __commit_transaction()
692 	r = superblock_lock(cmd, &sblock);  in __commit_transaction()
698 	disk_super->flags = cpu_to_le32(cmd->flags);  in __commit_transaction()
702 	disk_super->mapping_root = cpu_to_le64(cmd->root);  in __commit_transaction()
703 	if (separate_dirty_bits(cmd))  in __commit_transaction()
704 		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);  in __commit_transaction()
705 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);  in __commit_transaction()
706 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);  in __commit_transaction()
707 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);  in __commit_transaction()
708 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));  in __commit_transaction()
709 	disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));  in __commit_transaction()
710 	strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));  in __commit_transaction()
711 	disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);  in __commit_transaction()
712 	disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);  in __commit_transaction()
713 	disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);  in __commit_transaction()
714 	disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);  in __commit_transaction()
716 	disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);  in __commit_transaction()
717 	disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);  in __commit_transaction()
718 	disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);  in __commit_transaction()
719 	disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);  in __commit_transaction()
720 	__copy_sm_root(cmd, disk_super);  in __commit_transaction()
722 	return dm_tm_commit(cmd->tm, sblock);  in __commit_transaction()
761 	struct dm_cache_metadata *cmd;  in metadata_open()  local
763 	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);  in metadata_open()
764 	if (!cmd) {  in metadata_open()
769 	cmd->version = metadata_version;  in metadata_open()
770 	refcount_set(&cmd->ref_count, 1);  in metadata_open()
771 	init_rwsem(&cmd->root_lock);  in metadata_open()
772 	cmd->bdev = bdev;  in metadata_open()
773 	cmd->data_block_size = data_block_size;  in metadata_open()
774 	cmd->cache_blocks = 0;  in metadata_open()
775 	cmd->policy_hint_size = policy_hint_size;  in metadata_open()
776 	cmd->changed = true;  in metadata_open()
777 	cmd->fail_io = false;  in metadata_open()
779 	r = __create_persistent_data_objects(cmd, may_format_device);  in metadata_open()
781 		kfree(cmd);  in metadata_open()
785 	r = __begin_transaction_flags(cmd, clear_clean_shutdown);  in metadata_open()
787 		dm_cache_metadata_close(cmd);  in metadata_open()
791 	return cmd;  in metadata_open()
804 	struct dm_cache_metadata *cmd;  in lookup()  local
806 	list_for_each_entry(cmd, &table, list)  in lookup()
807 		if (cmd->bdev == bdev) {  in lookup()
808 			refcount_inc(&cmd->ref_count);  in lookup()
809 			return cmd;  in lookup()
821 	struct dm_cache_metadata *cmd, *cmd2;  in lookup_or_open()  local
824 	cmd = lookup(bdev);  in lookup_or_open()
827 	if (cmd)  in lookup_or_open()
828 		return cmd;  in lookup_or_open()
830 	cmd = metadata_open(bdev, data_block_size, may_format_device,  in lookup_or_open()
832 	if (!IS_ERR(cmd)) {  in lookup_or_open()
837 			__destroy_persistent_data_objects(cmd, true);  in lookup_or_open()
838 			kfree(cmd);  in lookup_or_open()
841 		list_add(&cmd->list, &table);  in lookup_or_open()
845 	return cmd;  in lookup_or_open()
848 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)  in same_params()  argument
850 	if (cmd->data_block_size != data_block_size) {  in same_params()
853 		      (unsigned long long) cmd->data_block_size);  in same_params()
866 	struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,  in dm_cache_metadata_open()  local
869 	if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {  in dm_cache_metadata_open()
870 		dm_cache_metadata_close(cmd);  in dm_cache_metadata_open()
874 	return cmd;  in dm_cache_metadata_open()
877 void dm_cache_metadata_close(struct dm_cache_metadata *cmd)  in dm_cache_metadata_close()  argument
879 	if (refcount_dec_and_test(&cmd->ref_count)) {  in dm_cache_metadata_close()
881 		list_del(&cmd->list);  in dm_cache_metadata_close()
884 		if (!cmd->fail_io)  in dm_cache_metadata_close()
885 			__destroy_persistent_data_objects(cmd, true);  in dm_cache_metadata_close()
886 		kfree(cmd);  in dm_cache_metadata_close()
893 static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,  in block_clean_combined_dirty()  argument
901 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);  in block_clean_combined_dirty()
911 static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,  in blocks_are_clean_combined_dirty()  argument
919 		r = block_clean_combined_dirty(cmd, begin, result);  in blocks_are_clean_combined_dirty()
937 static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,  in blocks_are_clean_separate_dirty()  argument
945 	if (from_cblock(cmd->cache_blocks) == 0)  in blocks_are_clean_separate_dirty()
949 	r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,  in blocks_are_clean_separate_dirty()
950 				   from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
956 	r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));  in blocks_are_clean_separate_dirty()
959 		dm_bitset_cursor_end(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
968 		dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
972 			dm_bitset_cursor_end(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
981 		r = dm_bitset_cursor_next(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
984 			dm_bitset_cursor_end(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
989 	dm_bitset_cursor_end(&cmd->dirty_cursor);  in blocks_are_clean_separate_dirty()
994 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,  in blocks_are_unmapped_or_clean()  argument
998 	if (separate_dirty_bits(cmd))  in blocks_are_unmapped_or_clean()
999 		return blocks_are_clean_separate_dirty(cmd, begin, end, result);  in blocks_are_unmapped_or_clean()
1001 		return blocks_are_clean_combined_dirty(cmd, begin, end, result);  in blocks_are_unmapped_or_clean()
1004 static bool cmd_write_lock(struct dm_cache_metadata *cmd)  in cmd_write_lock()  argument
1006 	down_write(&cmd->root_lock);  in cmd_write_lock()
1007 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {  in cmd_write_lock()
1008 		up_write(&cmd->root_lock);  in cmd_write_lock()
1014 #define WRITE_LOCK(cmd)				\  argument
1016 		if (!cmd_write_lock((cmd)))	\
1020 #define WRITE_LOCK_VOID(cmd)			\  argument
1022 		if (!cmd_write_lock((cmd)))	\
1026 #define WRITE_UNLOCK(cmd) \  argument
1027 	up_write(&(cmd)->root_lock)
1029 static bool cmd_read_lock(struct dm_cache_metadata *cmd)  in cmd_read_lock()  argument
1031 	down_read(&cmd->root_lock);  in cmd_read_lock()
1032 	if (cmd->fail_io) {  in cmd_read_lock()
1033 		up_read(&cmd->root_lock);  in cmd_read_lock()
1039 #define READ_LOCK(cmd)				\  argument
1041 		if (!cmd_read_lock((cmd)))	\
1045 #define READ_LOCK_VOID(cmd)			\  argument
1047 		if (!cmd_read_lock((cmd)))	\
1051 #define READ_UNLOCK(cmd) \  argument
1052 	up_read(&(cmd)->root_lock)
1054 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)  in dm_cache_resize()  argument
1060 	WRITE_LOCK(cmd);  in dm_cache_resize()
1063 	if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {  in dm_cache_resize()
1064 		r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);  in dm_cache_resize()
1078 	r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),  in dm_cache_resize()
1080 			    &null_mapping, &cmd->root);  in dm_cache_resize()
1084 	if (separate_dirty_bits(cmd)) {  in dm_cache_resize()
1085 		r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,  in dm_cache_resize()
1086 				     from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),  in dm_cache_resize()
1087 				     false, &cmd->dirty_root);  in dm_cache_resize()
1092 	cmd->cache_blocks = new_cache_size;  in dm_cache_resize()
1093 	cmd->changed = true;  in dm_cache_resize()
1096 	WRITE_UNLOCK(cmd);  in dm_cache_resize()
1101 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,  in dm_cache_discard_bitset_resize()  argument
1107 	WRITE_LOCK(cmd);  in dm_cache_discard_bitset_resize()
1108 	r = dm_bitset_resize(&cmd->discard_info,  in dm_cache_discard_bitset_resize()
1109 			     cmd->discard_root,  in dm_cache_discard_bitset_resize()
1110 			     from_dblock(cmd->discard_nr_blocks),  in dm_cache_discard_bitset_resize()
1112 			     false, &cmd->discard_root);  in dm_cache_discard_bitset_resize()
1114 		cmd->discard_block_size = discard_block_size;  in dm_cache_discard_bitset_resize()
1115 		cmd->discard_nr_blocks = new_nr_entries;  in dm_cache_discard_bitset_resize()
1118 	cmd->changed = true;  in dm_cache_discard_bitset_resize()
1119 	WRITE_UNLOCK(cmd);  in dm_cache_discard_bitset_resize()
1124 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)  in __set_discard()  argument
1126 	return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,  in __set_discard()
1127 				 from_dblock(b), &cmd->discard_root);  in __set_discard()
1130 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)  in __clear_discard()  argument
1132 	return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,  in __clear_discard()
1133 				   from_dblock(b), &cmd->discard_root);  in __clear_discard()
1136 static int __discard(struct dm_cache_metadata *cmd,  in __discard()  argument
1141 	r = (discard ? __set_discard : __clear_discard)(cmd, dblock);  in __discard()
1145 	cmd->changed = true;  in __discard()
1149 int dm_cache_set_discard(struct dm_cache_metadata *cmd,  in dm_cache_set_discard()  argument
1154 	WRITE_LOCK(cmd);  in dm_cache_set_discard()
1155 	r = __discard(cmd, dblock, discard);  in dm_cache_set_discard()
1156 	WRITE_UNLOCK(cmd);  in dm_cache_set_discard()
1161 static int __load_discards(struct dm_cache_metadata *cmd,  in __load_discards()  argument
1168 	if (from_dblock(cmd->discard_nr_blocks) == 0)  in __load_discards()
1172 	if (cmd->clean_when_opened) {  in __load_discards()
1173 		r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);  in __load_discards()
1177 		r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,  in __load_discards()
1178 					   from_dblock(cmd->discard_nr_blocks), &c);  in __load_discards()
1183 			r = fn(context, cmd->discard_block_size, to_dblock(b),  in __load_discards()
1188 			if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))  in __load_discards()
1199 		for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {  in __load_discards()
1200 			r = fn(context, cmd->discard_block_size, to_dblock(b), false);  in __load_discards()
1209 int dm_cache_load_discards(struct dm_cache_metadata *cmd,  in dm_cache_load_discards()  argument
1214 	READ_LOCK(cmd);  in dm_cache_load_discards()
1215 	r = __load_discards(cmd, fn, context);  in dm_cache_load_discards()
1216 	READ_UNLOCK(cmd);  in dm_cache_load_discards()
1221 int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)  in dm_cache_size()  argument
1223 	READ_LOCK(cmd);  in dm_cache_size()
1224 	*result = cmd->cache_blocks;  in dm_cache_size()
1225 	READ_UNLOCK(cmd);  in dm_cache_size()
1230 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)  in __remove()  argument
1236 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),  in __remove()
1237 			       &value, &cmd->root);  in __remove()
1241 	cmd->changed = true;  in __remove()
1245 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)  in dm_cache_remove_mapping()  argument
1249 	WRITE_LOCK(cmd);  in dm_cache_remove_mapping()
1250 	r = __remove(cmd, cblock);  in dm_cache_remove_mapping()
1251 	WRITE_UNLOCK(cmd);  in dm_cache_remove_mapping()
1256 static int __insert(struct dm_cache_metadata *cmd,  in __insert()  argument
1264 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),  in __insert()
1265 			       &value, &cmd->root);  in __insert()
1269 	cmd->changed = true;  in __insert()
1273 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,  in dm_cache_insert_mapping()  argument
1278 	WRITE_LOCK(cmd);  in dm_cache_insert_mapping()
1279 	r = __insert(cmd, cblock, oblock);  in dm_cache_insert_mapping()
1280 	WRITE_UNLOCK(cmd);  in dm_cache_insert_mapping()
1289 	struct dm_cache_metadata *cmd;  member
1294 static bool policy_unchanged(struct dm_cache_metadata *cmd,  in policy_unchanged()  argument
1304 	if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))  in policy_unchanged()
1310 	if (cmd->policy_version[0] != policy_version[0])  in policy_unchanged()
1316 	if (cmd->policy_hint_size != policy_hint_size)  in policy_unchanged()
1322 static bool hints_array_initialized(struct dm_cache_metadata *cmd)  in hints_array_initialized()  argument
1324 	return cmd->hint_root && cmd->policy_hint_size;  in hints_array_initialized()
1327 static bool hints_array_available(struct dm_cache_metadata *cmd,  in hints_array_available()  argument
1330 	return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&  in hints_array_available()
1331 		hints_array_initialized(cmd);  in hints_array_available()
1334 static int __load_mapping_v1(struct dm_cache_metadata *cmd,  in __load_mapping_v1()  argument
1361 		if (cmd->clean_when_opened)  in __load_mapping_v1()
1375 static int __load_mapping_v2(struct dm_cache_metadata *cmd,  in __load_mapping_v2()  argument
1403 		if (cmd->clean_when_opened)  in __load_mapping_v2()
1417 static int __load_mappings(struct dm_cache_metadata *cmd,  in __load_mappings()  argument
1424 	bool hints_valid = hints_array_available(cmd, policy);  in __load_mappings()
1426 	if (from_cblock(cmd->cache_blocks) == 0)  in __load_mappings()
1430 	r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);  in __load_mappings()
1435 		r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);  in __load_mappings()
1437 			dm_array_cursor_end(&cmd->mapping_cursor);  in __load_mappings()
1442 	if (separate_dirty_bits(cmd)) {  in __load_mappings()
1443 		r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,  in __load_mappings()
1444 					   from_cblock(cmd->cache_blocks),  in __load_mappings()
1445 					   &cmd->dirty_cursor);  in __load_mappings()
1447 			dm_array_cursor_end(&cmd->hint_cursor);  in __load_mappings()
1448 			dm_array_cursor_end(&cmd->mapping_cursor);  in __load_mappings()
1454 		if (separate_dirty_bits(cmd))  in __load_mappings()
1455 			r = __load_mapping_v2(cmd, cb, hints_valid,  in __load_mappings()
1456 					      &cmd->mapping_cursor,  in __load_mappings()
1457 					      &cmd->hint_cursor,  in __load_mappings()
1458 					      &cmd->dirty_cursor,  in __load_mappings()
1461 			r = __load_mapping_v1(cmd, cb, hints_valid,  in __load_mappings()
1462 					      &cmd->mapping_cursor, &cmd->hint_cursor,  in __load_mappings()
1470 		if (cb >= (from_cblock(cmd->cache_blocks) - 1))  in __load_mappings()
1473 		r = dm_array_cursor_next(&cmd->mapping_cursor);  in __load_mappings()
1480 			r = dm_array_cursor_next(&cmd->hint_cursor);  in __load_mappings()
1482 				dm_array_cursor_end(&cmd->hint_cursor);  in __load_mappings()
1487 		if (separate_dirty_bits(cmd)) {  in __load_mappings()
1488 			r = dm_bitset_cursor_next(&cmd->dirty_cursor);  in __load_mappings()
1496 	dm_array_cursor_end(&cmd->mapping_cursor);  in __load_mappings()
1498 		dm_array_cursor_end(&cmd->hint_cursor);  in __load_mappings()
1500 	if (separate_dirty_bits(cmd))  in __load_mappings()
1501 		dm_bitset_cursor_end(&cmd->dirty_cursor);  in __load_mappings()
1506 int dm_cache_load_mappings(struct dm_cache_metadata *cmd,  in dm_cache_load_mappings()  argument
1512 	READ_LOCK(cmd);  in dm_cache_load_mappings()
1513 	r = __load_mappings(cmd, policy, fn, context);  in dm_cache_load_mappings()
1514 	READ_UNLOCK(cmd);  in dm_cache_load_mappings()
1531 static int __dump_mappings(struct dm_cache_metadata *cmd)  in __dump_mappings()  argument
1533 	return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);  in __dump_mappings()
1536 void dm_cache_dump(struct dm_cache_metadata *cmd)  in dm_cache_dump()  argument
1538 	READ_LOCK_VOID(cmd);  in dm_cache_dump()
1539 	__dump_mappings(cmd);  in dm_cache_dump()
1540 	READ_UNLOCK(cmd);  in dm_cache_dump()
1543 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)  in dm_cache_changed_this_transaction()  argument
1547 	READ_LOCK(cmd);  in dm_cache_changed_this_transaction()
1548 	r = cmd->changed;  in dm_cache_changed_this_transaction()
1549 	READ_UNLOCK(cmd);  in dm_cache_changed_this_transaction()
1554 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)  in __dirty()  argument
1561 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);  in __dirty()
1574 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),  in __dirty()
1575 			       &value, &cmd->root);  in __dirty()
1579 	cmd->changed = true;  in __dirty()
1584 static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *…  in __set_dirty_bits_v1()  argument
1590 		r = __dirty(cmd, to_cblock(i), test_bit(i, bits));  in __set_dirty_bits_v1()
1605 static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *…  in __set_dirty_bits_v2()  argument
1610 	if (nr_bits != from_cblock(cmd->cache_blocks)) {  in __set_dirty_bits_v2()
1615 	r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);  in __set_dirty_bits_v2()
1619 	cmd->changed = true;  in __set_dirty_bits_v2()
1620 	return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);  in __set_dirty_bits_v2()
1623 int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,  in dm_cache_set_dirty_bits()  argument
1629 	WRITE_LOCK(cmd);  in dm_cache_set_dirty_bits()
1630 	if (separate_dirty_bits(cmd))  in dm_cache_set_dirty_bits()
1631 		r = __set_dirty_bits_v2(cmd, nr_bits, bits);  in dm_cache_set_dirty_bits()
1633 		r = __set_dirty_bits_v1(cmd, nr_bits, bits);  in dm_cache_set_dirty_bits()
1634 	WRITE_UNLOCK(cmd);  in dm_cache_set_dirty_bits()
1639 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,  in dm_cache_metadata_get_stats()  argument
1642 	READ_LOCK_VOID(cmd);  in dm_cache_metadata_get_stats()
1643 	*stats = cmd->stats;  in dm_cache_metadata_get_stats()
1644 	READ_UNLOCK(cmd);  in dm_cache_metadata_get_stats()
1647 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,  in dm_cache_metadata_set_stats()  argument
1650 	WRITE_LOCK_VOID(cmd);  in dm_cache_metadata_set_stats()
1651 	cmd->stats = *stats;  in dm_cache_metadata_set_stats()
1652 	WRITE_UNLOCK(cmd);  in dm_cache_metadata_set_stats()
1655 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)  in dm_cache_commit()  argument
1661 	WRITE_LOCK(cmd);  in dm_cache_commit()
1662 	if (cmd->fail_io)  in dm_cache_commit()
1665 	r = __commit_transaction(cmd, mutator);  in dm_cache_commit()
1669 	r = __begin_transaction(cmd);  in dm_cache_commit()
1671 	WRITE_UNLOCK(cmd);  in dm_cache_commit()
1675 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,  in dm_cache_get_free_metadata_block_count()  argument
1680 	READ_LOCK(cmd);  in dm_cache_get_free_metadata_block_count()
1681 	if (!cmd->fail_io)  in dm_cache_get_free_metadata_block_count()
1682 		r = dm_sm_get_nr_free(cmd->metadata_sm, result);  in dm_cache_get_free_metadata_block_count()
1683 	READ_UNLOCK(cmd);  in dm_cache_get_free_metadata_block_count()
1688 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,  in dm_cache_get_metadata_dev_size()  argument
1693 	READ_LOCK(cmd);  in dm_cache_get_metadata_dev_size()
1694 	if (!cmd->fail_io)  in dm_cache_get_metadata_dev_size()
1695 		r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);  in dm_cache_get_metadata_dev_size()
1696 	READ_UNLOCK(cmd);  in dm_cache_get_metadata_dev_size()
1718 static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)  in write_hints()  argument
1726 	    (strlen(policy_name) > sizeof(cmd->policy_name) - 1))  in write_hints()
1729 	strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));  in write_hints()
1730 	memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));  in write_hints()
1735 	cmd->policy_hint_size = hint_size;  in write_hints()
1737 	if (cmd->hint_root) {  in write_hints()
1738 		r = dm_array_del(&cmd->hint_info, cmd->hint_root);  in write_hints()
1743 	return dm_array_new(&cmd->hint_info, &cmd->hint_root,  in write_hints()
1744 			    from_cblock(cmd->cache_blocks),  in write_hints()
1748 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)  in dm_cache_write_hints()  argument
1752 	WRITE_LOCK(cmd);  in dm_cache_write_hints()
1753 	r = write_hints(cmd, policy);  in dm_cache_write_hints()
1754 	WRITE_UNLOCK(cmd);  in dm_cache_write_hints()
1759 int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)  in dm_cache_metadata_all_clean()  argument
1763 	READ_LOCK(cmd);  in dm_cache_metadata_all_clean()
1764 	r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);  in dm_cache_metadata_all_clean()
1765 	READ_UNLOCK(cmd);  in dm_cache_metadata_all_clean()
1770 void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)  in dm_cache_metadata_set_read_only()  argument
1772 	WRITE_LOCK_VOID(cmd);  in dm_cache_metadata_set_read_only()
1773 	dm_bm_set_read_only(cmd->bm);  in dm_cache_metadata_set_read_only()
1774 	WRITE_UNLOCK(cmd);  in dm_cache_metadata_set_read_only()
1777 void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)  in dm_cache_metadata_set_read_write()  argument
1779 	WRITE_LOCK_VOID(cmd);  in dm_cache_metadata_set_read_write()
1780 	dm_bm_set_read_write(cmd->bm);  in dm_cache_metadata_set_read_write()
1781 	WRITE_UNLOCK(cmd);  in dm_cache_metadata_set_read_write()
1784 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)  in dm_cache_metadata_set_needs_check()  argument
1790 	WRITE_LOCK(cmd);  in dm_cache_metadata_set_needs_check()
1791 	set_bit(NEEDS_CHECK, &cmd->flags);  in dm_cache_metadata_set_needs_check()
1793 	r = superblock_lock(cmd, &sblock);  in dm_cache_metadata_set_needs_check()
1800 	disk_super->flags = cpu_to_le32(cmd->flags);  in dm_cache_metadata_set_needs_check()
1805 	WRITE_UNLOCK(cmd);  in dm_cache_metadata_set_needs_check()
1809 int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)  in dm_cache_metadata_needs_check()  argument
1811 	READ_LOCK(cmd);  in dm_cache_metadata_needs_check()
1812 	*result = !!test_bit(NEEDS_CHECK, &cmd->flags);  in dm_cache_metadata_needs_check()
1813 	READ_UNLOCK(cmd);  in dm_cache_metadata_needs_check()
1818 int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)  in dm_cache_metadata_abort()  argument
1824 	if (unlikely(cmd->fail_io))  in dm_cache_metadata_abort()
1833 	new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,  in dm_cache_metadata_abort()
1836 	WRITE_LOCK(cmd);  in dm_cache_metadata_abort()
1837 	if (cmd->fail_io) {  in dm_cache_metadata_abort()
1838 		WRITE_UNLOCK(cmd);  in dm_cache_metadata_abort()
1842 	__destroy_persistent_data_objects(cmd, false);  in dm_cache_metadata_abort()
1843 	old_bm = cmd->bm;  in dm_cache_metadata_abort()
1846 		cmd->bm = NULL;  in dm_cache_metadata_abort()
1851 	cmd->bm = new_bm;  in dm_cache_metadata_abort()
1852 	r = __open_or_format_metadata(cmd, false);  in dm_cache_metadata_abort()
1854 		cmd->bm = NULL;  in dm_cache_metadata_abort()
1860 		cmd->fail_io = true;  in dm_cache_metadata_abort()
1861 	WRITE_UNLOCK(cmd);  in dm_cache_metadata_abort()