161 	struct btrfs_fs_info	*fs_info;  member
329 	sblock->len += sblock->sctx->fs_info->sectorsize;  in alloc_scrub_sector()
385 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
412 static void scrub_find_good_copy(struct btrfs_fs_info *fs_info,
443 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)  in __scrub_blocked_if_needed()  argument
445 	while (atomic_read(&fs_info->scrub_pause_req)) {  in __scrub_blocked_if_needed()
446 		mutex_unlock(&fs_info->scrub_lock);  in __scrub_blocked_if_needed()
447 		wait_event(fs_info->scrub_pause_wait,  in __scrub_blocked_if_needed()
448 		   atomic_read(&fs_info->scrub_pause_req) == 0);  in __scrub_blocked_if_needed()
449 		mutex_lock(&fs_info->scrub_lock);  in __scrub_blocked_if_needed()
453 static void scrub_pause_on(struct btrfs_fs_info *fs_info)  in scrub_pause_on()  argument
455 	atomic_inc(&fs_info->scrubs_paused);  in scrub_pause_on()
456 	wake_up(&fs_info->scrub_pause_wait);  in scrub_pause_on()
459 static void scrub_pause_off(struct btrfs_fs_info *fs_info)  in scrub_pause_off()  argument
461 	mutex_lock(&fs_info->scrub_lock);  in scrub_pause_off()
462 	__scrub_blocked_if_needed(fs_info);  in scrub_pause_off()
463 	atomic_dec(&fs_info->scrubs_paused);  in scrub_pause_off()
464 	mutex_unlock(&fs_info->scrub_lock);  in scrub_pause_off()
466 	wake_up(&fs_info->scrub_pause_wait);  in scrub_pause_off()
469 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)  in scrub_blocked_if_needed()  argument
471 	scrub_pause_on(fs_info);  in scrub_blocked_if_needed()
472 	scrub_pause_off(fs_info);  in scrub_blocked_if_needed()
588 static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,  in lock_full_stripe()  argument
598 	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);  in lock_full_stripe()
635 static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,  in unlock_full_stripe()  argument
649 	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);  in unlock_full_stripe()
672 		btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",  in unlock_full_stripe()
739 		struct btrfs_fs_info *fs_info, int is_dev_replace)  in scrub_setup_ctx()  argument
751 	sctx->fs_info = fs_info;  in scrub_setup_ctx()
785 		WARN_ON(!fs_info->dev_replace.tgtdev);  in scrub_setup_ctx()
786 		sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;  in scrub_setup_ctx()
807 	struct btrfs_fs_info *fs_info = swarn->dev->fs_info;  in scrub_print_warning_inode()  local
812 	local_root = btrfs_get_fs_root(fs_info, root, true);  in scrub_print_warning_inode()
862 		btrfs_warn_in_rcu(fs_info,  in scrub_print_warning_inode()
868 				  fs_info->sectorsize, nlink,  in scrub_print_warning_inode()
876 	btrfs_warn_in_rcu(fs_info,  in scrub_print_warning_inode()
890 	struct btrfs_fs_info *fs_info;  in scrub_print_warning()  local
906 	fs_info = sblock->sctx->fs_info;  in scrub_print_warning()
910 		btrfs_warn_in_rcu(fs_info, "%s on device %s, physical %llu",  in scrub_print_warning()
924 	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,  in scrub_print_warning()
941 			btrfs_warn_in_rcu(fs_info,  in scrub_print_warning()
955 		iterate_extent_inodes(fs_info, found_key.objectid,  in scrub_print_warning()
969 static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,  in scrub_put_recover()  argument
973 		btrfs_bio_counter_dec(fs_info);  in scrub_put_recover()
991 	struct btrfs_fs_info *fs_info;  in scrub_handle_errored_block()  local
1009 	fs_info = sctx->fs_info;  in scrub_handle_errored_block()
1030 	if (!sctx->is_dev_replace && btrfs_repair_one_zone(fs_info, logical))  in scrub_handle_errored_block()
1050 	ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);  in scrub_handle_errored_block()
1128 	scrub_recheck_block(fs_info, sblock_bad, 1);  in scrub_handle_errored_block()
1229 		scrub_recheck_block(fs_info, sblock_other, 0);  in scrub_handle_errored_block()
1321 					&fs_info->dev_replace.num_write_errors);  in scrub_handle_errored_block()
1346 			scrub_recheck_block(fs_info, sblock_bad, 1);  in scrub_handle_errored_block()
1359 			btrfs_err_rl_in_rcu(fs_info,  in scrub_handle_errored_block()
1368 		btrfs_err_rl_in_rcu(fs_info,  in scrub_handle_errored_block()
1391 				scrub_put_recover(fs_info, recover);  in scrub_handle_errored_block()
1398 	ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);  in scrub_handle_errored_block()
1448 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_setup_recheck_block()  local
1450 	u64 length = original_sblock->sector_count << fs_info->sectorsize_bits;  in scrub_setup_recheck_block()
1466 		sublen = min_t(u64, length, fs_info->sectorsize);  in scrub_setup_recheck_block()
1474 		btrfs_bio_counter_inc_blocked(fs_info);  in scrub_setup_recheck_block()
1475 		ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,  in scrub_setup_recheck_block()
1479 			btrfs_bio_counter_dec(fs_info);  in scrub_setup_recheck_block()
1486 			btrfs_bio_counter_dec(fs_info);  in scrub_setup_recheck_block()
1511 				scrub_put_recover(fs_info, recover);  in scrub_setup_recheck_block()
1520 				       sctx->fs_info->csum_size);  in scrub_setup_recheck_block()
1547 		scrub_put_recover(fs_info, recover);  in scrub_setup_recheck_block()
1561 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,  in scrub_submit_raid56_bio_wait()  argument
1577 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,  in scrub_recheck_block_on_raid56()  argument
1594 		bio_add_scrub_sector(bio, sector, fs_info->sectorsize);  in scrub_recheck_block_on_raid56()
1597 	if (scrub_submit_raid56_bio_wait(fs_info, bio, first_sector)) {  in scrub_recheck_block_on_raid56()
1621 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,  in scrub_recheck_block()  argument
1631 		return scrub_recheck_block_on_raid56(fs_info, sblock);  in scrub_recheck_block()
1645 		bio_add_scrub_sector(&bio, sector, fs_info->sectorsize);  in scrub_recheck_block()
1707 	struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;  in scrub_repair_sector_from_good_copy()  local
1708 	const u32 sectorsize = fs_info->sectorsize;  in scrub_repair_sector_from_good_copy()
1717 			btrfs_warn_rl(fs_info,  in scrub_repair_sector_from_good_copy()
1734 			atomic64_inc(&fs_info->dev_replace.num_write_errors);  in scrub_repair_sector_from_good_copy()
1744 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_write_block_to_dev_replace()  local
1759 			atomic64_inc(&fs_info->dev_replace.num_write_errors);  in scrub_write_block_to_dev_replace()
1765 	const u32 sectorsize = sblock->sctx->fs_info->sectorsize;  in scrub_write_sector_to_dev_replace()
1779 	if (!btrfs_is_zoned(sctx->fs_info))  in fill_writer_pointer_gap()
1807 	const u32 sectorsize = sctx->fs_info->sectorsize;  in scrub_add_sector_to_wr_bio()
1895 	if (btrfs_is_zoned(sctx->fs_info))  in scrub_wr_submit()
1897 			sctx->fs_info->sectorsize;  in scrub_wr_submit()
1903 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_wr_bio_end_io()  local
1909 	queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1921 			&sbio->sctx->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1982 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_checksum_data()  local
1983 	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);  in scrub_checksum_data()
1995 	shash->tfm = fs_info->csum_shash;  in scrub_checksum_data()
1998 	crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);  in scrub_checksum_data()
2000 	if (memcmp(csum, sector->csum, fs_info->csum_size))  in scrub_checksum_data()
2009 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_checksum_tree_block()  local
2010 	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);  in scrub_checksum_tree_block()
2018 	const u32 sectorsize = sctx->fs_info->sectorsize;  in scrub_checksum_tree_block()
2019 	const int num_sectors = fs_info->nodesize >> fs_info->sectorsize_bits;  in scrub_checksum_tree_block()
2032 	memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size);  in scrub_checksum_tree_block()
2050 	if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,  in scrub_checksum_tree_block()
2054 	shash->tfm = fs_info->csum_shash;  in scrub_checksum_tree_block()
2065 	if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size))  in scrub_checksum_tree_block()
2075 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_checksum_super()  local
2076 	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);  in scrub_checksum_super()
2097 	shash->tfm = fs_info->csum_shash;  in scrub_checksum_super()
2102 	if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size))  in scrub_checksum_super()
2219 	const u32 sectorsize = sctx->fs_info->sectorsize;  in scrub_add_sector_to_rd_bio()
2285 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_missing_raid56_end_io()  local
2287 	btrfs_bio_counter_dec(fs_info);  in scrub_missing_raid56_end_io()
2293 	queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2300 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_missing_raid56_worker()  local
2314 		btrfs_err_rl_in_rcu(fs_info,  in scrub_missing_raid56_worker()
2321 		btrfs_err_rl_in_rcu(fs_info,  in scrub_missing_raid56_worker()
2341 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_missing_raid56_pages()  local
2342 	u64 length = sblock->sector_count << fs_info->sectorsize_bits;  in scrub_missing_raid56_pages()
2350 	btrfs_bio_counter_inc_blocked(fs_info);  in scrub_missing_raid56_pages()
2351 	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,  in scrub_missing_raid56_pages()
2394 	btrfs_bio_counter_dec(fs_info);  in scrub_missing_raid56_pages()
2407 	const u32 sectorsize = sctx->fs_info->sectorsize;  in scrub_sectors()
2440 			memcpy(sector->csum, csum, sctx->fs_info->csum_size);  in scrub_sectors()
2481 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_bio_end_io()  local
2486 	queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2537 	u32 sectorsize_bits = sparity->sctx->fs_info->sectorsize_bits;  in __scrub_mark_bitmap()
2592 			  sblock->sctx->fs_info->sectorsize;  in scrub_block_complete()
2602 	sctx->stat.csum_discards += sum->len >> sctx->fs_info->sectorsize_bits;  in drop_csum_range()
2646 		index = (logical - sum->bytenr) >> sctx->fs_info->sectorsize_bits;  in scrub_find_csum()
2647 		num_sectors = sum->len >> sctx->fs_info->sectorsize_bits;  in scrub_find_csum()
2649 		memcpy(csum, sum->sums + index * sctx->fs_info->csum_size,  in scrub_find_csum()
2650 		       sctx->fs_info->csum_size);  in scrub_find_csum()
2679 			blocksize = sctx->fs_info->sectorsize;  in scrub_extent()
2688 			blocksize = sctx->fs_info->nodesize;  in scrub_extent()
2694 		blocksize = sctx->fs_info->sectorsize;  in scrub_extent()
2708 		scrub_find_good_copy(sctx->fs_info, logical, len, &src_physical,  in scrub_extent()
2740 	const u32 sectorsize = sctx->fs_info->sectorsize;  in scrub_sectors_for_parity()
2775 			memcpy(sector->csum, csum, sctx->fs_info->csum_size);  in scrub_sectors_for_parity()
2823 		blocksize = sctx->fs_info->sectorsize;  in scrub_extent_for_parity()
2922 	btrfs_bio_counter_dec(sctx->fs_info);  in scrub_parity_bio_endio_worker()
2930 	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;  in scrub_parity_bio_endio()  local
2939 	queue_work(fs_info->scrub_parity_workers, &sparity->work);  in scrub_parity_bio_endio()
2945 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_parity_check_and_repair()  local
2958 	btrfs_bio_counter_inc_blocked(fs_info);  in scrub_parity_check_and_repair()
2959 	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,  in scrub_parity_check_and_repair()
2984 	btrfs_bio_counter_dec(fs_info);  in scrub_parity_check_and_repair()
3015 	struct btrfs_fs_info *fs_info = path->nodes[0]->fs_info;  in compare_extent_item_range()  local
3023 		len = fs_info->nodesize;  in compare_extent_item_range()
3054 	struct btrfs_fs_info *fs_info = extent_root->fs_info;  in find_first_extent_item()  local
3062 	if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))  in find_first_extent_item()
3125 		*size_ret = path->nodes[0]->fs_info->nodesize;  in get_extent_info()
3149 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_raid56_data_stripe_for_parity()  local
3150 	struct btrfs_root *extent_root = btrfs_extent_root(fs_info, logical);  in scrub_raid56_data_stripe_for_parity()
3151 	struct btrfs_root *csum_root = btrfs_csum_root(fs_info, logical);  in scrub_raid56_data_stripe_for_parity()
3187 			btrfs_err(fs_info,  in scrub_raid56_data_stripe_for_parity()
3209 		ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_start,  in scrub_raid56_data_stripe_for_parity()
3258 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_raid56_parity()  local
3276 	nsectors = map->stripe_len >> fs_info->sectorsize_bits;  in scrub_raid56_parity()
3318 	if (!btrfs_is_zoned(sctx->fs_info))  in sync_replace_for_zoned()
3333 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in sync_write_pointer_for_zoned()  local
3336 	if (!btrfs_is_zoned(fs_info))  in sync_write_pointer_for_zoned()
3347 			btrfs_err(fs_info,  in sync_write_pointer_for_zoned()
3373 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_simple_mirror()  local
3395 		if (atomic_read(&fs_info->scrub_cancel_req) ||  in scrub_simple_mirror()
3401 		if (atomic_read(&fs_info->scrub_pause_req)) {  in scrub_simple_mirror()
3411 			scrub_blocked_if_needed(fs_info);  in scrub_simple_mirror()
3459 			btrfs_err(fs_info,  in scrub_simple_mirror()
3563 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_stripe()  local
3599 	scrub_blocked_if_needed(fs_info);  in scrub_stripe()
3601 	root = btrfs_extent_root(fs_info, bg->start);  in scrub_stripe()
3602 	csum_root = btrfs_csum_root(fs_info, bg->start);  in scrub_stripe()
3738 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_chunk()  local
3739 	struct extent_map_tree *map_tree = &fs_info->mapping_tree;  in scrub_chunk()
3784 	struct btrfs_fs_info *fs_info = cache->fs_info;  in finish_extent_writes_for_zoned()  local
3787 	if (!btrfs_is_zoned(fs_info))  in finish_extent_writes_for_zoned()
3792 	btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, cache->length);  in finish_extent_writes_for_zoned()
3806 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_enumerate_chunks()  local
3807 	struct btrfs_root *root = fs_info->dev_root;  in scrub_enumerate_chunks()
3816 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in scrub_enumerate_chunks()
3880 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);  in scrub_enumerate_chunks()
3912 		if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) {  in scrub_enumerate_chunks()
3944 		scrub_pause_on(fs_info);  in scrub_enumerate_chunks()
3981 				scrub_pause_off(fs_info);  in scrub_enumerate_chunks()
3999 			btrfs_warn(fs_info,  in scrub_enumerate_chunks()
4002 			scrub_pause_off(fs_info);  in scrub_enumerate_chunks()
4006 			btrfs_warn(fs_info,  in scrub_enumerate_chunks()
4010 			scrub_pause_off(fs_info);  in scrub_enumerate_chunks()
4021 			btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start,  in scrub_enumerate_chunks()
4025 		scrub_pause_off(fs_info);  in scrub_enumerate_chunks()
4054 		scrub_pause_on(fs_info);  in scrub_enumerate_chunks()
4065 		scrub_pause_off(fs_info);  in scrub_enumerate_chunks()
4091 			if (btrfs_test_opt(fs_info, DISCARD_ASYNC))  in scrub_enumerate_chunks()
4092 				btrfs_discard_queue_work(&fs_info->discard_ctl,  in scrub_enumerate_chunks()
4130 	struct btrfs_fs_info *fs_info = sctx->fs_info;  in scrub_supers()  local
4132 	if (BTRFS_FS_ERROR(fs_info))  in scrub_supers()
4136 	if (scrub_dev->fs_devices != fs_info->fs_devices)  in scrub_supers()
4139 		gen = fs_info->last_trans_committed;  in scrub_supers()
4160 static void scrub_workers_put(struct btrfs_fs_info *fs_info)  in scrub_workers_put()  argument
4162 	if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt,  in scrub_workers_put()
4163 					&fs_info->scrub_lock)) {  in scrub_workers_put()
4164 		struct workqueue_struct *scrub_workers = fs_info->scrub_workers;  in scrub_workers_put()
4166 						fs_info->scrub_wr_completion_workers;  in scrub_workers_put()
4168 						fs_info->scrub_parity_workers;  in scrub_workers_put()
4170 		fs_info->scrub_workers = NULL;  in scrub_workers_put()
4171 		fs_info->scrub_wr_completion_workers = NULL;  in scrub_workers_put()
4172 		fs_info->scrub_parity_workers = NULL;  in scrub_workers_put()
4173 		mutex_unlock(&fs_info->scrub_lock);  in scrub_workers_put()
4187 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,  in scrub_workers_get()  argument
4194 	int max_active = fs_info->thread_pool_size;  in scrub_workers_get()
4197 	if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt))  in scrub_workers_get()
4213 	mutex_lock(&fs_info->scrub_lock);  in scrub_workers_get()
4214 	if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {  in scrub_workers_get()
4215 		ASSERT(fs_info->scrub_workers == NULL &&  in scrub_workers_get()
4216 		       fs_info->scrub_wr_completion_workers == NULL &&  in scrub_workers_get()
4217 		       fs_info->scrub_parity_workers == NULL);  in scrub_workers_get()
4218 		fs_info->scrub_workers = scrub_workers;  in scrub_workers_get()
4219 		fs_info->scrub_wr_completion_workers = scrub_wr_comp;  in scrub_workers_get()
4220 		fs_info->scrub_parity_workers = scrub_parity;  in scrub_workers_get()
4221 		refcount_set(&fs_info->scrub_workers_refcnt, 1);  in scrub_workers_get()
4222 		mutex_unlock(&fs_info->scrub_lock);  in scrub_workers_get()
4226 	refcount_inc(&fs_info->scrub_workers_refcnt);  in scrub_workers_get()
4227 	mutex_unlock(&fs_info->scrub_lock);  in scrub_workers_get()
4239 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,  in btrfs_scrub_dev()  argument
4250 	if (btrfs_fs_closing(fs_info))  in btrfs_scrub_dev()
4254 	ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN);  in btrfs_scrub_dev()
4261 	ASSERT(fs_info->nodesize <=  in btrfs_scrub_dev()
4262 	       SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits);  in btrfs_scrub_dev()
4265 	sctx = scrub_setup_ctx(fs_info, is_dev_replace);  in btrfs_scrub_dev()
4269 	ret = scrub_workers_get(fs_info, is_dev_replace);  in btrfs_scrub_dev()
4273 	mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4274 	dev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_scrub_dev()
4277 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4284 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4285 		btrfs_err_in_rcu(fs_info,  in btrfs_scrub_dev()
4292 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4295 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4296 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4301 	down_read(&fs_info->dev_replace.rwsem);  in btrfs_scrub_dev()
4304 	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {  in btrfs_scrub_dev()
4305 		up_read(&fs_info->dev_replace.rwsem);  in btrfs_scrub_dev()
4306 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4307 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4311 	up_read(&fs_info->dev_replace.rwsem);  in btrfs_scrub_dev()
4315 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4321 	__scrub_blocked_if_needed(fs_info);  in btrfs_scrub_dev()
4322 	atomic_inc(&fs_info->scrubs_running);  in btrfs_scrub_dev()
4323 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4342 		btrfs_info(fs_info, "scrub: started on devid %llu", devid);  in btrfs_scrub_dev()
4347 		mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4349 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
4367 	atomic_dec(&fs_info->scrubs_running);  in btrfs_scrub_dev()
4368 	wake_up(&fs_info->scrub_pause_wait);  in btrfs_scrub_dev()
4376 		btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",  in btrfs_scrub_dev()
4379 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4381 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
4383 	scrub_workers_put(fs_info);  in btrfs_scrub_dev()
4393 		trans = btrfs_start_transaction(fs_info->tree_root, 0);  in btrfs_scrub_dev()
4396 			btrfs_err(fs_info,  in btrfs_scrub_dev()
4402 			btrfs_err(fs_info,  in btrfs_scrub_dev()
4407 	scrub_workers_put(fs_info);  in btrfs_scrub_dev()
4414 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)  in btrfs_scrub_pause()  argument
4416 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
4417 	atomic_inc(&fs_info->scrub_pause_req);  in btrfs_scrub_pause()
4418 	while (atomic_read(&fs_info->scrubs_paused) !=  in btrfs_scrub_pause()
4419 	       atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_pause()
4420 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
4421 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_pause()
4422 			   atomic_read(&fs_info->scrubs_paused) ==  in btrfs_scrub_pause()
4423 			   atomic_read(&fs_info->scrubs_running));  in btrfs_scrub_pause()
4424 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
4426 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
4429 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)  in btrfs_scrub_continue()  argument
4431 	atomic_dec(&fs_info->scrub_pause_req);  in btrfs_scrub_continue()
4432 	wake_up(&fs_info->scrub_pause_wait);  in btrfs_scrub_continue()
4435 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)  in btrfs_scrub_cancel()  argument
4437 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
4438 	if (!atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_cancel()
4439 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
4443 	atomic_inc(&fs_info->scrub_cancel_req);  in btrfs_scrub_cancel()
4444 	while (atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_cancel()
4445 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
4446 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_cancel()
4447 			   atomic_read(&fs_info->scrubs_running) == 0);  in btrfs_scrub_cancel()
4448 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
4450 	atomic_dec(&fs_info->scrub_cancel_req);  in btrfs_scrub_cancel()
4451 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
4458 	struct btrfs_fs_info *fs_info = dev->fs_info;  in btrfs_scrub_cancel_dev()  local
4461 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
4464 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
4469 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
4470 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_cancel_dev()
4472 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
4474 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
4479 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,  in btrfs_scrub_progress()  argument
4486 	mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_progress()
4487 	dev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_scrub_progress()
4492 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_progress()
4497 static void scrub_find_good_copy(struct btrfs_fs_info *fs_info,  in scrub_find_good_copy()  argument
4508 	ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,  in scrub_find_good_copy()