1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) STRATO AG 2012.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/blkdev.h>
10 #include <linux/kthread.h>
11 #include <linux/math64.h>
12 #include "misc.h"
13 #include "ctree.h"
14 #include "extent_map.h"
15 #include "disk-io.h"
16 #include "transaction.h"
17 #include "print-tree.h"
18 #include "volumes.h"
19 #include "async-thread.h"
20 #include "check-integrity.h"
21 #include "rcu-string.h"
22 #include "dev-replace.h"
23 #include "sysfs.h"
24 #include "zoned.h"
25 #include "block-group.h"
26 
27 /*
28  * Device replace overview
29  *
30  * [Objective]
31  * To copy all extents (both new and on-disk) from source device to target
32  * device, while still keeping the filesystem read-write.
33  *
34  * [Method]
35  * There are two main methods involved:
36  *
37  * - Write duplication
38  *
39  *   All new writes will be written to both target and source devices, so even
40  *   if replace gets canceled, sources device still contains up-to-date data.
41  *
42  *   Location:		handle_ops_on_dev_replace() from __btrfs_map_block()
43  *   Start:		btrfs_dev_replace_start()
44  *   End:		btrfs_dev_replace_finishing()
45  *   Content:		Latest data/metadata
46  *
47  * - Copy existing extents
48  *
49  *   This happens by re-using scrub facility, as scrub also iterates through
50  *   existing extents from commit root.
51  *
52  *   Location:		scrub_write_block_to_dev_replace() from
53  *   			scrub_block_complete()
54  *   Content:		Data/meta from commit root.
55  *
56  * Due to the content difference, we need to avoid nocow write when dev-replace
57  * is happening.  This is done by marking the block group read-only and waiting
58  * for NOCOW writes.
59  *
60  * After replace is done, the finishing part is done by swapping the target and
61  * source devices.
62  *
63  *   Location:		btrfs_dev_replace_update_device_in_mapping_tree() from
64  *   			btrfs_dev_replace_finishing()
65  */
66 
67 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
68 				       int scrub_ret);
69 static int btrfs_dev_replace_kthread(void *data);
70 
btrfs_init_dev_replace(struct btrfs_fs_info * fs_info)71 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
72 {
73 	struct btrfs_dev_lookup_args args = { .devid = BTRFS_DEV_REPLACE_DEVID };
74 	struct btrfs_key key;
75 	struct btrfs_root *dev_root = fs_info->dev_root;
76 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
77 	struct extent_buffer *eb;
78 	int slot;
79 	int ret = 0;
80 	struct btrfs_path *path = NULL;
81 	int item_size;
82 	struct btrfs_dev_replace_item *ptr;
83 	u64 src_devid;
84 
85 	if (!dev_root)
86 		return 0;
87 
88 	path = btrfs_alloc_path();
89 	if (!path) {
90 		ret = -ENOMEM;
91 		goto out;
92 	}
93 
94 	key.objectid = 0;
95 	key.type = BTRFS_DEV_REPLACE_KEY;
96 	key.offset = 0;
97 	ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
98 	if (ret) {
99 no_valid_dev_replace_entry_found:
100 		/*
101 		 * We don't have a replace item or it's corrupted.  If there is
102 		 * a replace target, fail the mount.
103 		 */
104 		if (btrfs_find_device(fs_info->fs_devices, &args)) {
105 			btrfs_err(fs_info,
106 			"found replace target device without a valid replace item");
107 			ret = -EUCLEAN;
108 			goto out;
109 		}
110 		ret = 0;
111 		dev_replace->replace_state =
112 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
113 		dev_replace->cont_reading_from_srcdev_mode =
114 		    BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
115 		dev_replace->time_started = 0;
116 		dev_replace->time_stopped = 0;
117 		atomic64_set(&dev_replace->num_write_errors, 0);
118 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
119 		dev_replace->cursor_left = 0;
120 		dev_replace->committed_cursor_left = 0;
121 		dev_replace->cursor_left_last_write_of_item = 0;
122 		dev_replace->cursor_right = 0;
123 		dev_replace->srcdev = NULL;
124 		dev_replace->tgtdev = NULL;
125 		dev_replace->is_valid = 0;
126 		dev_replace->item_needs_writeback = 0;
127 		goto out;
128 	}
129 	slot = path->slots[0];
130 	eb = path->nodes[0];
131 	item_size = btrfs_item_size(eb, slot);
132 	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);
133 
134 	if (item_size != sizeof(struct btrfs_dev_replace_item)) {
135 		btrfs_warn(fs_info,
136 			"dev_replace entry found has unexpected size, ignore entry");
137 		goto no_valid_dev_replace_entry_found;
138 	}
139 
140 	src_devid = btrfs_dev_replace_src_devid(eb, ptr);
141 	dev_replace->cont_reading_from_srcdev_mode =
142 		btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
143 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
144 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
145 	dev_replace->time_stopped =
146 		btrfs_dev_replace_time_stopped(eb, ptr);
147 	atomic64_set(&dev_replace->num_write_errors,
148 		     btrfs_dev_replace_num_write_errors(eb, ptr));
149 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,
150 		     btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
151 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
152 	dev_replace->committed_cursor_left = dev_replace->cursor_left;
153 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
154 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
155 	dev_replace->is_valid = 1;
156 
157 	dev_replace->item_needs_writeback = 0;
158 	switch (dev_replace->replace_state) {
159 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
160 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
161 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
162 		/*
163 		 * We don't have an active replace item but if there is a
164 		 * replace target, fail the mount.
165 		 */
166 		if (btrfs_find_device(fs_info->fs_devices, &args)) {
167 			btrfs_err(fs_info,
168 "replace without active item, run 'device scan --forget' on the target device");
169 			ret = -EUCLEAN;
170 		} else {
171 			dev_replace->srcdev = NULL;
172 			dev_replace->tgtdev = NULL;
173 		}
174 		break;
175 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
176 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
177 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, &args);
178 		args.devid = src_devid;
179 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, &args);
180 
181 		/*
182 		 * allow 'btrfs dev replace_cancel' if src/tgt device is
183 		 * missing
184 		 */
185 		if (!dev_replace->srcdev &&
186 		    !btrfs_test_opt(fs_info, DEGRADED)) {
187 			ret = -EIO;
188 			btrfs_warn(fs_info,
189 			   "cannot mount because device replace operation is ongoing and");
190 			btrfs_warn(fs_info,
191 			   "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
192 			   src_devid);
193 		}
194 		if (!dev_replace->tgtdev &&
195 		    !btrfs_test_opt(fs_info, DEGRADED)) {
196 			ret = -EIO;
197 			btrfs_warn(fs_info,
198 			   "cannot mount because device replace operation is ongoing and");
199 			btrfs_warn(fs_info,
200 			   "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
201 				BTRFS_DEV_REPLACE_DEVID);
202 		}
203 		if (dev_replace->tgtdev) {
204 			if (dev_replace->srcdev) {
205 				dev_replace->tgtdev->total_bytes =
206 					dev_replace->srcdev->total_bytes;
207 				dev_replace->tgtdev->disk_total_bytes =
208 					dev_replace->srcdev->disk_total_bytes;
209 				dev_replace->tgtdev->commit_total_bytes =
210 					dev_replace->srcdev->commit_total_bytes;
211 				dev_replace->tgtdev->bytes_used =
212 					dev_replace->srcdev->bytes_used;
213 				dev_replace->tgtdev->commit_bytes_used =
214 					dev_replace->srcdev->commit_bytes_used;
215 			}
216 			set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
217 				&dev_replace->tgtdev->dev_state);
218 
219 			WARN_ON(fs_info->fs_devices->rw_devices == 0);
220 			dev_replace->tgtdev->io_width = fs_info->sectorsize;
221 			dev_replace->tgtdev->io_align = fs_info->sectorsize;
222 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;
223 			dev_replace->tgtdev->fs_info = fs_info;
224 			set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
225 				&dev_replace->tgtdev->dev_state);
226 		}
227 		break;
228 	}
229 
230 out:
231 	btrfs_free_path(path);
232 	return ret;
233 }
234 
235 /*
236  * Initialize a new device for device replace target from a given source dev
237  * and path.
238  *
239  * Return 0 and new device in @device_out, otherwise return < 0
240  */
btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info * fs_info,const char * device_path,struct btrfs_device * srcdev,struct btrfs_device ** device_out)241 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
242 				  const char *device_path,
243 				  struct btrfs_device *srcdev,
244 				  struct btrfs_device **device_out)
245 {
246 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
247 	struct btrfs_device *device;
248 	struct block_device *bdev;
249 	struct rcu_string *name;
250 	u64 devid = BTRFS_DEV_REPLACE_DEVID;
251 	int ret = 0;
252 
253 	*device_out = NULL;
254 	if (srcdev->fs_devices->seeding) {
255 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
256 		return -EINVAL;
257 	}
258 
259 	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
260 				  fs_info->bdev_holder);
261 	if (IS_ERR(bdev)) {
262 		btrfs_err(fs_info, "target device %s is invalid!", device_path);
263 		return PTR_ERR(bdev);
264 	}
265 
266 	if (!btrfs_check_device_zone_type(fs_info, bdev)) {
267 		btrfs_err(fs_info,
268 		"dev-replace: zoned type of target device mismatch with filesystem");
269 		ret = -EINVAL;
270 		goto error;
271 	}
272 
273 	sync_blockdev(bdev);
274 
275 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
276 		if (device->bdev == bdev) {
277 			btrfs_err(fs_info,
278 				  "target device is in the filesystem!");
279 			ret = -EEXIST;
280 			goto error;
281 		}
282 	}
283 
284 
285 	if (bdev_nr_bytes(bdev) < btrfs_device_get_total_bytes(srcdev)) {
286 		btrfs_err(fs_info,
287 			  "target device is smaller than source device!");
288 		ret = -EINVAL;
289 		goto error;
290 	}
291 
292 
293 	device = btrfs_alloc_device(NULL, &devid, NULL);
294 	if (IS_ERR(device)) {
295 		ret = PTR_ERR(device);
296 		goto error;
297 	}
298 
299 	name = rcu_string_strdup(device_path, GFP_KERNEL);
300 	if (!name) {
301 		btrfs_free_device(device);
302 		ret = -ENOMEM;
303 		goto error;
304 	}
305 	rcu_assign_pointer(device->name, name);
306 	ret = lookup_bdev(device_path, &device->devt);
307 	if (ret)
308 		goto error;
309 
310 	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
311 	device->generation = 0;
312 	device->io_width = fs_info->sectorsize;
313 	device->io_align = fs_info->sectorsize;
314 	device->sector_size = fs_info->sectorsize;
315 	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
316 	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
317 	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
318 	device->commit_total_bytes = srcdev->commit_total_bytes;
319 	device->commit_bytes_used = device->bytes_used;
320 	device->fs_info = fs_info;
321 	device->bdev = bdev;
322 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
323 	set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
324 	device->mode = FMODE_EXCL;
325 	device->dev_stats_valid = 1;
326 	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
327 	device->fs_devices = fs_devices;
328 
329 	ret = btrfs_get_dev_zone_info(device, false);
330 	if (ret)
331 		goto error;
332 
333 	mutex_lock(&fs_devices->device_list_mutex);
334 	list_add(&device->dev_list, &fs_devices->devices);
335 	fs_devices->num_devices++;
336 	fs_devices->open_devices++;
337 	mutex_unlock(&fs_devices->device_list_mutex);
338 
339 	*device_out = device;
340 	return 0;
341 
342 error:
343 	blkdev_put(bdev, FMODE_EXCL);
344 	return ret;
345 }
346 
347 /*
348  * called from commit_transaction. Writes changed device replace state to
349  * disk.
350  */
btrfs_run_dev_replace(struct btrfs_trans_handle * trans)351 int btrfs_run_dev_replace(struct btrfs_trans_handle *trans)
352 {
353 	struct btrfs_fs_info *fs_info = trans->fs_info;
354 	int ret;
355 	struct btrfs_root *dev_root = fs_info->dev_root;
356 	struct btrfs_path *path;
357 	struct btrfs_key key;
358 	struct extent_buffer *eb;
359 	struct btrfs_dev_replace_item *ptr;
360 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
361 
362 	down_read(&dev_replace->rwsem);
363 	if (!dev_replace->is_valid ||
364 	    !dev_replace->item_needs_writeback) {
365 		up_read(&dev_replace->rwsem);
366 		return 0;
367 	}
368 	up_read(&dev_replace->rwsem);
369 
370 	key.objectid = 0;
371 	key.type = BTRFS_DEV_REPLACE_KEY;
372 	key.offset = 0;
373 
374 	path = btrfs_alloc_path();
375 	if (!path) {
376 		ret = -ENOMEM;
377 		goto out;
378 	}
379 	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
380 	if (ret < 0) {
381 		btrfs_warn(fs_info,
382 			   "error %d while searching for dev_replace item!",
383 			   ret);
384 		goto out;
385 	}
386 
387 	if (ret == 0 &&
388 	    btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
389 		/*
390 		 * need to delete old one and insert a new one.
391 		 * Since no attempt is made to recover any old state, if the
392 		 * dev_replace state is 'running', the data on the target
393 		 * drive is lost.
394 		 * It would be possible to recover the state: just make sure
395 		 * that the beginning of the item is never changed and always
396 		 * contains all the essential information. Then read this
397 		 * minimal set of information and use it as a base for the
398 		 * new state.
399 		 */
400 		ret = btrfs_del_item(trans, dev_root, path);
401 		if (ret != 0) {
402 			btrfs_warn(fs_info,
403 				   "delete too small dev_replace item failed %d!",
404 				   ret);
405 			goto out;
406 		}
407 		ret = 1;
408 	}
409 
410 	if (ret == 1) {
411 		/* need to insert a new item */
412 		btrfs_release_path(path);
413 		ret = btrfs_insert_empty_item(trans, dev_root, path,
414 					      &key, sizeof(*ptr));
415 		if (ret < 0) {
416 			btrfs_warn(fs_info,
417 				   "insert dev_replace item failed %d!", ret);
418 			goto out;
419 		}
420 	}
421 
422 	eb = path->nodes[0];
423 	ptr = btrfs_item_ptr(eb, path->slots[0],
424 			     struct btrfs_dev_replace_item);
425 
426 	down_write(&dev_replace->rwsem);
427 	if (dev_replace->srcdev)
428 		btrfs_set_dev_replace_src_devid(eb, ptr,
429 			dev_replace->srcdev->devid);
430 	else
431 		btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
432 	btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
433 		dev_replace->cont_reading_from_srcdev_mode);
434 	btrfs_set_dev_replace_replace_state(eb, ptr,
435 		dev_replace->replace_state);
436 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
437 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
438 	btrfs_set_dev_replace_num_write_errors(eb, ptr,
439 		atomic64_read(&dev_replace->num_write_errors));
440 	btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
441 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));
442 	dev_replace->cursor_left_last_write_of_item =
443 		dev_replace->cursor_left;
444 	btrfs_set_dev_replace_cursor_left(eb, ptr,
445 		dev_replace->cursor_left_last_write_of_item);
446 	btrfs_set_dev_replace_cursor_right(eb, ptr,
447 		dev_replace->cursor_right);
448 	dev_replace->item_needs_writeback = 0;
449 	up_write(&dev_replace->rwsem);
450 
451 	btrfs_mark_buffer_dirty(eb);
452 
453 out:
454 	btrfs_free_path(path);
455 
456 	return ret;
457 }
458 
btrfs_dev_name(struct btrfs_device * device)459 static char* btrfs_dev_name(struct btrfs_device *device)
460 {
461 	if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
462 		return "<missing disk>";
463 	else
464 		return rcu_str_deref(device->name);
465 }
466 
mark_block_group_to_copy(struct btrfs_fs_info * fs_info,struct btrfs_device * src_dev)467 static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info,
468 				    struct btrfs_device *src_dev)
469 {
470 	struct btrfs_path *path;
471 	struct btrfs_key key;
472 	struct btrfs_key found_key;
473 	struct btrfs_root *root = fs_info->dev_root;
474 	struct btrfs_dev_extent *dev_extent = NULL;
475 	struct btrfs_block_group *cache;
476 	struct btrfs_trans_handle *trans;
477 	int iter_ret = 0;
478 	int ret = 0;
479 	u64 chunk_offset;
480 
481 	/* Do not use "to_copy" on non zoned filesystem for now */
482 	if (!btrfs_is_zoned(fs_info))
483 		return 0;
484 
485 	mutex_lock(&fs_info->chunk_mutex);
486 
487 	/* Ensure we don't have pending new block group */
488 	spin_lock(&fs_info->trans_lock);
489 	while (fs_info->running_transaction &&
490 	       !list_empty(&fs_info->running_transaction->dev_update_list)) {
491 		spin_unlock(&fs_info->trans_lock);
492 		mutex_unlock(&fs_info->chunk_mutex);
493 		trans = btrfs_attach_transaction(root);
494 		if (IS_ERR(trans)) {
495 			ret = PTR_ERR(trans);
496 			mutex_lock(&fs_info->chunk_mutex);
497 			if (ret == -ENOENT) {
498 				spin_lock(&fs_info->trans_lock);
499 				continue;
500 			} else {
501 				goto unlock;
502 			}
503 		}
504 
505 		ret = btrfs_commit_transaction(trans);
506 		mutex_lock(&fs_info->chunk_mutex);
507 		if (ret)
508 			goto unlock;
509 
510 		spin_lock(&fs_info->trans_lock);
511 	}
512 	spin_unlock(&fs_info->trans_lock);
513 
514 	path = btrfs_alloc_path();
515 	if (!path) {
516 		ret = -ENOMEM;
517 		goto unlock;
518 	}
519 
520 	path->reada = READA_FORWARD;
521 	path->search_commit_root = 1;
522 	path->skip_locking = 1;
523 
524 	key.objectid = src_dev->devid;
525 	key.type = BTRFS_DEV_EXTENT_KEY;
526 	key.offset = 0;
527 
528 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
529 		struct extent_buffer *leaf = path->nodes[0];
530 
531 		if (found_key.objectid != src_dev->devid)
532 			break;
533 
534 		if (found_key.type != BTRFS_DEV_EXTENT_KEY)
535 			break;
536 
537 		if (found_key.offset < key.offset)
538 			break;
539 
540 		dev_extent = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent);
541 
542 		chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dev_extent);
543 
544 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
545 		if (!cache)
546 			continue;
547 
548 		set_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
549 		btrfs_put_block_group(cache);
550 	}
551 	if (iter_ret < 0)
552 		ret = iter_ret;
553 
554 	btrfs_free_path(path);
555 unlock:
556 	mutex_unlock(&fs_info->chunk_mutex);
557 
558 	return ret;
559 }
560 
btrfs_finish_block_group_to_copy(struct btrfs_device * srcdev,struct btrfs_block_group * cache,u64 physical)561 bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev,
562 				      struct btrfs_block_group *cache,
563 				      u64 physical)
564 {
565 	struct btrfs_fs_info *fs_info = cache->fs_info;
566 	struct extent_map *em;
567 	struct map_lookup *map;
568 	u64 chunk_offset = cache->start;
569 	int num_extents, cur_extent;
570 	int i;
571 
572 	/* Do not use "to_copy" on non zoned filesystem for now */
573 	if (!btrfs_is_zoned(fs_info))
574 		return true;
575 
576 	spin_lock(&cache->lock);
577 	if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) {
578 		spin_unlock(&cache->lock);
579 		return true;
580 	}
581 	spin_unlock(&cache->lock);
582 
583 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
584 	ASSERT(!IS_ERR(em));
585 	map = em->map_lookup;
586 
587 	num_extents = 0;
588 	cur_extent = 0;
589 	for (i = 0; i < map->num_stripes; i++) {
590 		/* We have more device extent to copy */
591 		if (srcdev != map->stripes[i].dev)
592 			continue;
593 
594 		num_extents++;
595 		if (physical == map->stripes[i].physical)
596 			cur_extent = i;
597 	}
598 
599 	free_extent_map(em);
600 
601 	if (num_extents > 1 && cur_extent < num_extents - 1) {
602 		/*
603 		 * Has more stripes on this device. Keep this block group
604 		 * readonly until we finish all the stripes.
605 		 */
606 		return false;
607 	}
608 
609 	/* Last stripe on this device */
610 	clear_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
611 
612 	return true;
613 }
614 
btrfs_dev_replace_start(struct btrfs_fs_info * fs_info,const char * tgtdev_name,u64 srcdevid,const char * srcdev_name,int read_src)615 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
616 		const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
617 		int read_src)
618 {
619 	struct btrfs_root *root = fs_info->dev_root;
620 	struct btrfs_trans_handle *trans;
621 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
622 	int ret;
623 	struct btrfs_device *tgt_device = NULL;
624 	struct btrfs_device *src_device = NULL;
625 
626 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,
627 						  srcdev_name);
628 	if (IS_ERR(src_device))
629 		return PTR_ERR(src_device);
630 
631 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {
632 		btrfs_warn_in_rcu(fs_info,
633 	  "cannot replace device %s (devid %llu) due to active swapfile",
634 			btrfs_dev_name(src_device), src_device->devid);
635 		return -ETXTBSY;
636 	}
637 
638 	/*
639 	 * Here we commit the transaction to make sure commit_total_bytes
640 	 * of all the devices are updated.
641 	 */
642 	trans = btrfs_attach_transaction(root);
643 	if (!IS_ERR(trans)) {
644 		ret = btrfs_commit_transaction(trans);
645 		if (ret)
646 			return ret;
647 	} else if (PTR_ERR(trans) != -ENOENT) {
648 		return PTR_ERR(trans);
649 	}
650 
651 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
652 					    src_device, &tgt_device);
653 	if (ret)
654 		return ret;
655 
656 	ret = mark_block_group_to_copy(fs_info, src_device);
657 	if (ret)
658 		return ret;
659 
660 	down_write(&dev_replace->rwsem);
661 	switch (dev_replace->replace_state) {
662 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
663 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
664 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
665 		break;
666 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
667 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
668 		ASSERT(0);
669 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
670 		up_write(&dev_replace->rwsem);
671 		goto leave;
672 	}
673 
674 	dev_replace->cont_reading_from_srcdev_mode = read_src;
675 	dev_replace->srcdev = src_device;
676 	dev_replace->tgtdev = tgt_device;
677 
678 	btrfs_info_in_rcu(fs_info,
679 		      "dev_replace from %s (devid %llu) to %s started",
680 		      btrfs_dev_name(src_device),
681 		      src_device->devid,
682 		      rcu_str_deref(tgt_device->name));
683 
684 	/*
685 	 * from now on, the writes to the srcdev are all duplicated to
686 	 * go to the tgtdev as well (refer to btrfs_map_block()).
687 	 */
688 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
689 	dev_replace->time_started = ktime_get_real_seconds();
690 	dev_replace->cursor_left = 0;
691 	dev_replace->committed_cursor_left = 0;
692 	dev_replace->cursor_left_last_write_of_item = 0;
693 	dev_replace->cursor_right = 0;
694 	dev_replace->is_valid = 1;
695 	dev_replace->item_needs_writeback = 1;
696 	atomic64_set(&dev_replace->num_write_errors, 0);
697 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
698 	up_write(&dev_replace->rwsem);
699 
700 	ret = btrfs_sysfs_add_device(tgt_device);
701 	if (ret)
702 		btrfs_err(fs_info, "kobj add dev failed %d", ret);
703 
704 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
705 
706 	/*
707 	 * Commit dev_replace state and reserve 1 item for it.
708 	 * This is crucial to ensure we won't miss copying extents for new block
709 	 * groups that are allocated after we started the device replace, and
710 	 * must be done after setting up the device replace state.
711 	 */
712 	trans = btrfs_start_transaction(root, 1);
713 	if (IS_ERR(trans)) {
714 		ret = PTR_ERR(trans);
715 		down_write(&dev_replace->rwsem);
716 		dev_replace->replace_state =
717 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
718 		dev_replace->srcdev = NULL;
719 		dev_replace->tgtdev = NULL;
720 		up_write(&dev_replace->rwsem);
721 		goto leave;
722 	}
723 
724 	ret = btrfs_commit_transaction(trans);
725 	WARN_ON(ret);
726 
727 	/* the disk copy procedure reuses the scrub code */
728 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
729 			      btrfs_device_get_total_bytes(src_device),
730 			      &dev_replace->scrub_progress, 0, 1);
731 
732 	ret = btrfs_dev_replace_finishing(fs_info, ret);
733 	if (ret == -EINPROGRESS)
734 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
735 
736 	return ret;
737 
738 leave:
739 	btrfs_destroy_dev_replace_tgtdev(tgt_device);
740 	return ret;
741 }
742 
btrfs_dev_replace_by_ioctl(struct btrfs_fs_info * fs_info,struct btrfs_ioctl_dev_replace_args * args)743 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
744 			    struct btrfs_ioctl_dev_replace_args *args)
745 {
746 	int ret;
747 
748 	switch (args->start.cont_reading_from_srcdev_mode) {
749 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
750 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
751 		break;
752 	default:
753 		return -EINVAL;
754 	}
755 
756 	if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') ||
757 	    args->start.tgtdev_name[0] == '\0')
758 		return -EINVAL;
759 
760 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
761 					args->start.srcdevid,
762 					args->start.srcdev_name,
763 					args->start.cont_reading_from_srcdev_mode);
764 	args->result = ret;
765 	/* don't warn if EINPROGRESS, someone else might be running scrub */
766 	if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS ||
767 	    ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR)
768 		return 0;
769 
770 	return ret;
771 }
772 
773 /*
774  * blocked until all in-flight bios operations are finished.
775  */
btrfs_rm_dev_replace_blocked(struct btrfs_fs_info * fs_info)776 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
777 {
778 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
779 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(
780 		   &fs_info->dev_replace.bio_counter));
781 }
782 
783 /*
784  * we have removed target device, it is safe to allow new bios request.
785  */
btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info * fs_info)786 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
787 {
788 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
789 	wake_up(&fs_info->dev_replace.replace_wait);
790 }
791 
792 /*
793  * When finishing the device replace, before swapping the source device with the
794  * target device we must update the chunk allocation state in the target device,
795  * as it is empty because replace works by directly copying the chunks and not
796  * through the normal chunk allocation path.
797  */
btrfs_set_target_alloc_state(struct btrfs_device * srcdev,struct btrfs_device * tgtdev)798 static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev,
799 					struct btrfs_device *tgtdev)
800 {
801 	struct extent_state *cached_state = NULL;
802 	u64 start = 0;
803 	u64 found_start;
804 	u64 found_end;
805 	int ret = 0;
806 
807 	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);
808 
809 	while (!find_first_extent_bit(&srcdev->alloc_state, start,
810 				      &found_start, &found_end,
811 				      CHUNK_ALLOCATED, &cached_state)) {
812 		ret = set_extent_bits(&tgtdev->alloc_state, found_start,
813 				      found_end, CHUNK_ALLOCATED);
814 		if (ret)
815 			break;
816 		start = found_end + 1;
817 	}
818 
819 	free_extent_state(cached_state);
820 	return ret;
821 }
822 
btrfs_dev_replace_update_device_in_mapping_tree(struct btrfs_fs_info * fs_info,struct btrfs_device * srcdev,struct btrfs_device * tgtdev)823 static void btrfs_dev_replace_update_device_in_mapping_tree(
824 						struct btrfs_fs_info *fs_info,
825 						struct btrfs_device *srcdev,
826 						struct btrfs_device *tgtdev)
827 {
828 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
829 	struct extent_map *em;
830 	struct map_lookup *map;
831 	u64 start = 0;
832 	int i;
833 
834 	write_lock(&em_tree->lock);
835 	do {
836 		em = lookup_extent_mapping(em_tree, start, (u64)-1);
837 		if (!em)
838 			break;
839 		map = em->map_lookup;
840 		for (i = 0; i < map->num_stripes; i++)
841 			if (srcdev == map->stripes[i].dev)
842 				map->stripes[i].dev = tgtdev;
843 		start = em->start + em->len;
844 		free_extent_map(em);
845 	} while (start);
846 	write_unlock(&em_tree->lock);
847 }
848 
btrfs_dev_replace_finishing(struct btrfs_fs_info * fs_info,int scrub_ret)849 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
850 				       int scrub_ret)
851 {
852 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
853 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
854 	struct btrfs_device *tgt_device;
855 	struct btrfs_device *src_device;
856 	struct btrfs_root *root = fs_info->tree_root;
857 	u8 uuid_tmp[BTRFS_UUID_SIZE];
858 	struct btrfs_trans_handle *trans;
859 	int ret = 0;
860 
861 	/* don't allow cancel or unmount to disturb the finishing procedure */
862 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
863 
864 	down_read(&dev_replace->rwsem);
865 	/* was the operation canceled, or is it finished? */
866 	if (dev_replace->replace_state !=
867 	    BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
868 		up_read(&dev_replace->rwsem);
869 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
870 		return 0;
871 	}
872 
873 	tgt_device = dev_replace->tgtdev;
874 	src_device = dev_replace->srcdev;
875 	up_read(&dev_replace->rwsem);
876 
877 	/*
878 	 * flush all outstanding I/O and inode extent mappings before the
879 	 * copy operation is declared as being finished
880 	 */
881 	ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
882 	if (ret) {
883 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
884 		return ret;
885 	}
886 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
887 
888 	/*
889 	 * We have to use this loop approach because at this point src_device
890 	 * has to be available for transaction commit to complete, yet new
891 	 * chunks shouldn't be allocated on the device.
892 	 */
893 	while (1) {
894 		trans = btrfs_start_transaction(root, 0);
895 		if (IS_ERR(trans)) {
896 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
897 			return PTR_ERR(trans);
898 		}
899 		ret = btrfs_commit_transaction(trans);
900 		WARN_ON(ret);
901 
902 		/* Prevent write_all_supers() during the finishing procedure */
903 		mutex_lock(&fs_devices->device_list_mutex);
904 		/* Prevent new chunks being allocated on the source device */
905 		mutex_lock(&fs_info->chunk_mutex);
906 
907 		if (!list_empty(&src_device->post_commit_list)) {
908 			mutex_unlock(&fs_devices->device_list_mutex);
909 			mutex_unlock(&fs_info->chunk_mutex);
910 		} else {
911 			break;
912 		}
913 	}
914 
915 	down_write(&dev_replace->rwsem);
916 	dev_replace->replace_state =
917 		scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
918 			  : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
919 	dev_replace->tgtdev = NULL;
920 	dev_replace->srcdev = NULL;
921 	dev_replace->time_stopped = ktime_get_real_seconds();
922 	dev_replace->item_needs_writeback = 1;
923 
924 	/*
925 	 * Update allocation state in the new device and replace the old device
926 	 * with the new one in the mapping tree.
927 	 */
928 	if (!scrub_ret) {
929 		scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device);
930 		if (scrub_ret)
931 			goto error;
932 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
933 								src_device,
934 								tgt_device);
935 	} else {
936 		if (scrub_ret != -ECANCELED)
937 			btrfs_err_in_rcu(fs_info,
938 				 "btrfs_scrub_dev(%s, %llu, %s) failed %d",
939 				 btrfs_dev_name(src_device),
940 				 src_device->devid,
941 				 rcu_str_deref(tgt_device->name), scrub_ret);
942 error:
943 		up_write(&dev_replace->rwsem);
944 		mutex_unlock(&fs_info->chunk_mutex);
945 		mutex_unlock(&fs_devices->device_list_mutex);
946 		btrfs_rm_dev_replace_blocked(fs_info);
947 		if (tgt_device)
948 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
949 		btrfs_rm_dev_replace_unblocked(fs_info);
950 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
951 
952 		return scrub_ret;
953 	}
954 
955 	btrfs_info_in_rcu(fs_info,
956 			  "dev_replace from %s (devid %llu) to %s finished",
957 			  btrfs_dev_name(src_device),
958 			  src_device->devid,
959 			  rcu_str_deref(tgt_device->name));
960 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
961 	tgt_device->devid = src_device->devid;
962 	src_device->devid = BTRFS_DEV_REPLACE_DEVID;
963 	memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
964 	memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
965 	memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
966 	btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
967 	btrfs_device_set_disk_total_bytes(tgt_device,
968 					  src_device->disk_total_bytes);
969 	btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
970 	tgt_device->commit_bytes_used = src_device->bytes_used;
971 
972 	btrfs_assign_next_active_device(src_device, tgt_device);
973 
974 	list_add(&tgt_device->dev_alloc_list, &fs_devices->alloc_list);
975 	fs_devices->rw_devices++;
976 
977 	up_write(&dev_replace->rwsem);
978 	btrfs_rm_dev_replace_blocked(fs_info);
979 
980 	btrfs_rm_dev_replace_remove_srcdev(src_device);
981 
982 	btrfs_rm_dev_replace_unblocked(fs_info);
983 
984 	/*
985 	 * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will
986 	 * update on-disk dev stats value during commit transaction
987 	 */
988 	atomic_inc(&tgt_device->dev_stats_ccnt);
989 
990 	/*
991 	 * this is again a consistent state where no dev_replace procedure
992 	 * is running, the target device is part of the filesystem, the
993 	 * source device is not part of the filesystem anymore and its 1st
994 	 * superblock is scratched out so that it is no longer marked to
995 	 * belong to this filesystem.
996 	 */
997 	mutex_unlock(&fs_info->chunk_mutex);
998 	mutex_unlock(&fs_devices->device_list_mutex);
999 
1000 	/* replace the sysfs entry */
1001 	btrfs_sysfs_remove_device(src_device);
1002 	btrfs_sysfs_update_devid(tgt_device);
1003 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state))
1004 		btrfs_scratch_superblocks(fs_info, src_device->bdev,
1005 					  src_device->name->str);
1006 
1007 	/* write back the superblocks */
1008 	trans = btrfs_start_transaction(root, 0);
1009 	if (!IS_ERR(trans))
1010 		btrfs_commit_transaction(trans);
1011 
1012 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1013 
1014 	btrfs_rm_dev_replace_free_srcdev(src_device);
1015 
1016 	return 0;
1017 }
1018 
1019 /*
1020  * Read progress of device replace status according to the state and last
1021  * stored position. The value format is the same as for
1022  * btrfs_dev_replace::progress_1000
1023  */
btrfs_dev_replace_progress(struct btrfs_fs_info * fs_info)1024 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
1025 {
1026 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1027 	u64 ret = 0;
1028 
1029 	switch (dev_replace->replace_state) {
1030 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1031 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1032 		ret = 0;
1033 		break;
1034 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1035 		ret = 1000;
1036 		break;
1037 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1038 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1039 		ret = div64_u64(dev_replace->cursor_left,
1040 				div_u64(btrfs_device_get_total_bytes(
1041 						dev_replace->srcdev), 1000));
1042 		break;
1043 	}
1044 
1045 	return ret;
1046 }
1047 
btrfs_dev_replace_status(struct btrfs_fs_info * fs_info,struct btrfs_ioctl_dev_replace_args * args)1048 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
1049 			      struct btrfs_ioctl_dev_replace_args *args)
1050 {
1051 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1052 
1053 	down_read(&dev_replace->rwsem);
1054 	/* even if !dev_replace_is_valid, the values are good enough for
1055 	 * the replace_status ioctl */
1056 	args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1057 	args->status.replace_state = dev_replace->replace_state;
1058 	args->status.time_started = dev_replace->time_started;
1059 	args->status.time_stopped = dev_replace->time_stopped;
1060 	args->status.num_write_errors =
1061 		atomic64_read(&dev_replace->num_write_errors);
1062 	args->status.num_uncorrectable_read_errors =
1063 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);
1064 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
1065 	up_read(&dev_replace->rwsem);
1066 }
1067 
btrfs_dev_replace_cancel(struct btrfs_fs_info * fs_info)1068 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
1069 {
1070 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1071 	struct btrfs_device *tgt_device = NULL;
1072 	struct btrfs_device *src_device = NULL;
1073 	struct btrfs_trans_handle *trans;
1074 	struct btrfs_root *root = fs_info->tree_root;
1075 	int result;
1076 	int ret;
1077 
1078 	if (sb_rdonly(fs_info->sb))
1079 		return -EROFS;
1080 
1081 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
1082 	down_write(&dev_replace->rwsem);
1083 	switch (dev_replace->replace_state) {
1084 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1085 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1086 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1087 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
1088 		up_write(&dev_replace->rwsem);
1089 		break;
1090 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1091 		tgt_device = dev_replace->tgtdev;
1092 		src_device = dev_replace->srcdev;
1093 		up_write(&dev_replace->rwsem);
1094 		ret = btrfs_scrub_cancel(fs_info);
1095 		if (ret < 0) {
1096 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
1097 		} else {
1098 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1099 			/*
1100 			 * btrfs_dev_replace_finishing() will handle the
1101 			 * cleanup part
1102 			 */
1103 			btrfs_info_in_rcu(fs_info,
1104 				"dev_replace from %s (devid %llu) to %s canceled",
1105 				btrfs_dev_name(src_device), src_device->devid,
1106 				btrfs_dev_name(tgt_device));
1107 		}
1108 		break;
1109 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1110 		/*
1111 		 * Scrub doing the replace isn't running so we need to do the
1112 		 * cleanup step of btrfs_dev_replace_finishing() here
1113 		 */
1114 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1115 		tgt_device = dev_replace->tgtdev;
1116 		src_device = dev_replace->srcdev;
1117 		dev_replace->tgtdev = NULL;
1118 		dev_replace->srcdev = NULL;
1119 		dev_replace->replace_state =
1120 				BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
1121 		dev_replace->time_stopped = ktime_get_real_seconds();
1122 		dev_replace->item_needs_writeback = 1;
1123 
1124 		up_write(&dev_replace->rwsem);
1125 
1126 		/* Scrub for replace must not be running in suspended state */
1127 		btrfs_scrub_cancel(fs_info);
1128 
1129 		trans = btrfs_start_transaction(root, 0);
1130 		if (IS_ERR(trans)) {
1131 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1132 			return PTR_ERR(trans);
1133 		}
1134 		ret = btrfs_commit_transaction(trans);
1135 		WARN_ON(ret);
1136 
1137 		btrfs_info_in_rcu(fs_info,
1138 		"suspended dev_replace from %s (devid %llu) to %s canceled",
1139 			btrfs_dev_name(src_device), src_device->devid,
1140 			btrfs_dev_name(tgt_device));
1141 
1142 		if (tgt_device)
1143 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
1144 		break;
1145 	default:
1146 		up_write(&dev_replace->rwsem);
1147 		result = -EINVAL;
1148 	}
1149 
1150 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1151 	return result;
1152 }
1153 
btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info * fs_info)1154 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
1155 {
1156 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1157 
1158 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
1159 	down_write(&dev_replace->rwsem);
1160 
1161 	switch (dev_replace->replace_state) {
1162 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1163 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1164 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1165 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1166 		break;
1167 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1168 		dev_replace->replace_state =
1169 			BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1170 		dev_replace->time_stopped = ktime_get_real_seconds();
1171 		dev_replace->item_needs_writeback = 1;
1172 		btrfs_info(fs_info, "suspending dev_replace for unmount");
1173 		break;
1174 	}
1175 
1176 	up_write(&dev_replace->rwsem);
1177 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1178 }
1179 
1180 /* resume dev_replace procedure that was interrupted by unmount */
btrfs_resume_dev_replace_async(struct btrfs_fs_info * fs_info)1181 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
1182 {
1183 	struct task_struct *task;
1184 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1185 
1186 	down_write(&dev_replace->rwsem);
1187 
1188 	switch (dev_replace->replace_state) {
1189 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1190 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1191 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1192 		up_write(&dev_replace->rwsem);
1193 		return 0;
1194 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1195 		break;
1196 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1197 		dev_replace->replace_state =
1198 			BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
1199 		break;
1200 	}
1201 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
1202 		btrfs_info(fs_info,
1203 			   "cannot continue dev_replace, tgtdev is missing");
1204 		btrfs_info(fs_info,
1205 			   "you may cancel the operation after 'mount -o degraded'");
1206 		dev_replace->replace_state =
1207 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1208 		up_write(&dev_replace->rwsem);
1209 		return 0;
1210 	}
1211 	up_write(&dev_replace->rwsem);
1212 
1213 	/*
1214 	 * This could collide with a paused balance, but the exclusive op logic
1215 	 * should never allow both to start and pause. We don't want to allow
1216 	 * dev-replace to start anyway.
1217 	 */
1218 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {
1219 		down_write(&dev_replace->rwsem);
1220 		dev_replace->replace_state =
1221 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1222 		up_write(&dev_replace->rwsem);
1223 		btrfs_info(fs_info,
1224 		"cannot resume dev-replace, other exclusive operation running");
1225 		return 0;
1226 	}
1227 
1228 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
1229 	return PTR_ERR_OR_ZERO(task);
1230 }
1231 
btrfs_dev_replace_kthread(void * data)1232 static int btrfs_dev_replace_kthread(void *data)
1233 {
1234 	struct btrfs_fs_info *fs_info = data;
1235 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1236 	u64 progress;
1237 	int ret;
1238 
1239 	progress = btrfs_dev_replace_progress(fs_info);
1240 	progress = div_u64(progress, 10);
1241 	btrfs_info_in_rcu(fs_info,
1242 		"continuing dev_replace from %s (devid %llu) to target %s @%u%%",
1243 		btrfs_dev_name(dev_replace->srcdev),
1244 		dev_replace->srcdev->devid,
1245 		btrfs_dev_name(dev_replace->tgtdev),
1246 		(unsigned int)progress);
1247 
1248 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
1249 			      dev_replace->committed_cursor_left,
1250 			      btrfs_device_get_total_bytes(dev_replace->srcdev),
1251 			      &dev_replace->scrub_progress, 0, 1);
1252 	ret = btrfs_dev_replace_finishing(fs_info, ret);
1253 	WARN_ON(ret && ret != -ECANCELED);
1254 
1255 	btrfs_exclop_finish(fs_info);
1256 	return 0;
1257 }
1258 
btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace * dev_replace)1259 int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
1260 {
1261 	if (!dev_replace->is_valid)
1262 		return 0;
1263 
1264 	switch (dev_replace->replace_state) {
1265 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1266 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1267 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1268 		return 0;
1269 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1270 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1271 		/*
1272 		 * return true even if tgtdev is missing (this is
1273 		 * something that can happen if the dev_replace
1274 		 * procedure is suspended by an umount and then
1275 		 * the tgtdev is missing (or "btrfs dev scan") was
1276 		 * not called and the filesystem is remounted
1277 		 * in degraded state. This does not stop the
1278 		 * dev_replace procedure. It needs to be canceled
1279 		 * manually if the cancellation is wanted.
1280 		 */
1281 		break;
1282 	}
1283 	return 1;
1284 }
1285 
btrfs_bio_counter_sub(struct btrfs_fs_info * fs_info,s64 amount)1286 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1287 {
1288 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);
1289 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);
1290 }
1291 
btrfs_bio_counter_inc_blocked(struct btrfs_fs_info * fs_info)1292 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
1293 {
1294 	while (1) {
1295 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1296 		if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1297 				     &fs_info->fs_state)))
1298 			break;
1299 
1300 		btrfs_bio_counter_dec(fs_info);
1301 		wait_event(fs_info->dev_replace.replace_wait,
1302 			   !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1303 				     &fs_info->fs_state));
1304 	}
1305 }
1306