1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2012 Alexander Block.  All rights reserved.
4  */
5 
6 #include <linux/bsearch.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/sort.h>
10 #include <linux/mount.h>
11 #include <linux/xattr.h>
12 #include <linux/posix_acl_xattr.h>
13 #include <linux/radix-tree.h>
14 #include <linux/vmalloc.h>
15 #include <linux/string.h>
16 #include <linux/compat.h>
17 #include <linux/crc32c.h>
18 #include <linux/fsverity.h>
19 
20 #include "send.h"
21 #include "ctree.h"
22 #include "backref.h"
23 #include "locking.h"
24 #include "disk-io.h"
25 #include "btrfs_inode.h"
26 #include "transaction.h"
27 #include "compression.h"
28 #include "xattr.h"
29 #include "print-tree.h"
30 
31 /*
32  * Maximum number of references an extent can have in order for us to attempt to
33  * issue clone operations instead of write operations. This currently exists to
34  * avoid hitting limitations of the backreference walking code (taking a lot of
35  * time and using too much memory for extents with large number of references).
36  */
37 #define SEND_MAX_EXTENT_REFS	64
38 
39 /*
40  * A fs_path is a helper to dynamically build path names with unknown size.
41  * It reallocates the internal buffer on demand.
42  * It allows fast adding of path elements on the right side (normal path) and
43  * fast adding to the left side (reversed path). A reversed path can also be
44  * unreversed if needed.
45  */
46 struct fs_path {
47 	union {
48 		struct {
49 			char *start;
50 			char *end;
51 
52 			char *buf;
53 			unsigned short buf_len:15;
54 			unsigned short reversed:1;
55 			char inline_buf[];
56 		};
57 		/*
58 		 * Average path length does not exceed 200 bytes, we'll have
59 		 * better packing in the slab and higher chance to satisfy
60 		 * a allocation later during send.
61 		 */
62 		char pad[256];
63 	};
64 };
65 #define FS_PATH_INLINE_SIZE \
66 	(sizeof(struct fs_path) - offsetof(struct fs_path, inline_buf))
67 
68 
69 /* reused for each extent */
70 struct clone_root {
71 	struct btrfs_root *root;
72 	u64 ino;
73 	u64 offset;
74 
75 	u64 found_refs;
76 };
77 
78 #define SEND_CTX_MAX_NAME_CACHE_SIZE 128
79 #define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2)
80 
81 struct send_ctx {
82 	struct file *send_filp;
83 	loff_t send_off;
84 	char *send_buf;
85 	u32 send_size;
86 	u32 send_max_size;
87 	/*
88 	 * Whether BTRFS_SEND_A_DATA attribute was already added to current
89 	 * command (since protocol v2, data must be the last attribute).
90 	 */
91 	bool put_data;
92 	struct page **send_buf_pages;
93 	u64 flags;	/* 'flags' member of btrfs_ioctl_send_args is u64 */
94 	/* Protocol version compatibility requested */
95 	u32 proto;
96 
97 	struct btrfs_root *send_root;
98 	struct btrfs_root *parent_root;
99 	struct clone_root *clone_roots;
100 	int clone_roots_cnt;
101 
102 	/* current state of the compare_tree call */
103 	struct btrfs_path *left_path;
104 	struct btrfs_path *right_path;
105 	struct btrfs_key *cmp_key;
106 
107 	/*
108 	 * Keep track of the generation of the last transaction that was used
109 	 * for relocating a block group. This is periodically checked in order
110 	 * to detect if a relocation happened since the last check, so that we
111 	 * don't operate on stale extent buffers for nodes (level >= 1) or on
112 	 * stale disk_bytenr values of file extent items.
113 	 */
114 	u64 last_reloc_trans;
115 
116 	/*
117 	 * infos of the currently processed inode. In case of deleted inodes,
118 	 * these are the values from the deleted inode.
119 	 */
120 	u64 cur_ino;
121 	u64 cur_inode_gen;
122 	u64 cur_inode_size;
123 	u64 cur_inode_mode;
124 	u64 cur_inode_rdev;
125 	u64 cur_inode_last_extent;
126 	u64 cur_inode_next_write_offset;
127 	bool cur_inode_new;
128 	bool cur_inode_new_gen;
129 	bool cur_inode_deleted;
130 	bool ignore_cur_inode;
131 	bool cur_inode_needs_verity;
132 	void *verity_descriptor;
133 
134 	u64 send_progress;
135 
136 	struct list_head new_refs;
137 	struct list_head deleted_refs;
138 
139 	struct radix_tree_root name_cache;
140 	struct list_head name_cache_list;
141 	int name_cache_size;
142 
143 	/*
144 	 * The inode we are currently processing. It's not NULL only when we
145 	 * need to issue write commands for data extents from this inode.
146 	 */
147 	struct inode *cur_inode;
148 	struct file_ra_state ra;
149 	u64 page_cache_clear_start;
150 	bool clean_page_cache;
151 
152 	/*
153 	 * We process inodes by their increasing order, so if before an
154 	 * incremental send we reverse the parent/child relationship of
155 	 * directories such that a directory with a lower inode number was
156 	 * the parent of a directory with a higher inode number, and the one
157 	 * becoming the new parent got renamed too, we can't rename/move the
158 	 * directory with lower inode number when we finish processing it - we
159 	 * must process the directory with higher inode number first, then
160 	 * rename/move it and then rename/move the directory with lower inode
161 	 * number. Example follows.
162 	 *
163 	 * Tree state when the first send was performed:
164 	 *
165 	 * .
166 	 * |-- a                   (ino 257)
167 	 *     |-- b               (ino 258)
168 	 *         |
169 	 *         |
170 	 *         |-- c           (ino 259)
171 	 *         |   |-- d       (ino 260)
172 	 *         |
173 	 *         |-- c2          (ino 261)
174 	 *
175 	 * Tree state when the second (incremental) send is performed:
176 	 *
177 	 * .
178 	 * |-- a                   (ino 257)
179 	 *     |-- b               (ino 258)
180 	 *         |-- c2          (ino 261)
181 	 *             |-- d2      (ino 260)
182 	 *                 |-- cc  (ino 259)
183 	 *
184 	 * The sequence of steps that lead to the second state was:
185 	 *
186 	 * mv /a/b/c/d /a/b/c2/d2
187 	 * mv /a/b/c /a/b/c2/d2/cc
188 	 *
189 	 * "c" has lower inode number, but we can't move it (2nd mv operation)
190 	 * before we move "d", which has higher inode number.
191 	 *
192 	 * So we just memorize which move/rename operations must be performed
193 	 * later when their respective parent is processed and moved/renamed.
194 	 */
195 
196 	/* Indexed by parent directory inode number. */
197 	struct rb_root pending_dir_moves;
198 
199 	/*
200 	 * Reverse index, indexed by the inode number of a directory that
201 	 * is waiting for the move/rename of its immediate parent before its
202 	 * own move/rename can be performed.
203 	 */
204 	struct rb_root waiting_dir_moves;
205 
206 	/*
207 	 * A directory that is going to be rm'ed might have a child directory
208 	 * which is in the pending directory moves index above. In this case,
209 	 * the directory can only be removed after the move/rename of its child
210 	 * is performed. Example:
211 	 *
212 	 * Parent snapshot:
213 	 *
214 	 * .                        (ino 256)
215 	 * |-- a/                   (ino 257)
216 	 *     |-- b/               (ino 258)
217 	 *         |-- c/           (ino 259)
218 	 *         |   |-- x/       (ino 260)
219 	 *         |
220 	 *         |-- y/           (ino 261)
221 	 *
222 	 * Send snapshot:
223 	 *
224 	 * .                        (ino 256)
225 	 * |-- a/                   (ino 257)
226 	 *     |-- b/               (ino 258)
227 	 *         |-- YY/          (ino 261)
228 	 *              |-- x/      (ino 260)
229 	 *
230 	 * Sequence of steps that lead to the send snapshot:
231 	 * rm -f /a/b/c/foo.txt
232 	 * mv /a/b/y /a/b/YY
233 	 * mv /a/b/c/x /a/b/YY
234 	 * rmdir /a/b/c
235 	 *
236 	 * When the child is processed, its move/rename is delayed until its
237 	 * parent is processed (as explained above), but all other operations
238 	 * like update utimes, chown, chgrp, etc, are performed and the paths
239 	 * that it uses for those operations must use the orphanized name of
240 	 * its parent (the directory we're going to rm later), so we need to
241 	 * memorize that name.
242 	 *
243 	 * Indexed by the inode number of the directory to be deleted.
244 	 */
245 	struct rb_root orphan_dirs;
246 
247 	struct rb_root rbtree_new_refs;
248 	struct rb_root rbtree_deleted_refs;
249 };
250 
251 struct pending_dir_move {
252 	struct rb_node node;
253 	struct list_head list;
254 	u64 parent_ino;
255 	u64 ino;
256 	u64 gen;
257 	struct list_head update_refs;
258 };
259 
260 struct waiting_dir_move {
261 	struct rb_node node;
262 	u64 ino;
263 	/*
264 	 * There might be some directory that could not be removed because it
265 	 * was waiting for this directory inode to be moved first. Therefore
266 	 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
267 	 */
268 	u64 rmdir_ino;
269 	u64 rmdir_gen;
270 	bool orphanized;
271 };
272 
273 struct orphan_dir_info {
274 	struct rb_node node;
275 	u64 ino;
276 	u64 gen;
277 	u64 last_dir_index_offset;
278 };
279 
280 struct name_cache_entry {
281 	struct list_head list;
282 	/*
283 	 * radix_tree has only 32bit entries but we need to handle 64bit inums.
284 	 * We use the lower 32bit of the 64bit inum to store it in the tree. If
285 	 * more then one inum would fall into the same entry, we use radix_list
286 	 * to store the additional entries. radix_list is also used to store
287 	 * entries where two entries have the same inum but different
288 	 * generations.
289 	 */
290 	struct list_head radix_list;
291 	u64 ino;
292 	u64 gen;
293 	u64 parent_ino;
294 	u64 parent_gen;
295 	int ret;
296 	int need_later_update;
297 	int name_len;
298 	char name[];
299 };
300 
301 #define ADVANCE							1
302 #define ADVANCE_ONLY_NEXT					-1
303 
304 enum btrfs_compare_tree_result {
305 	BTRFS_COMPARE_TREE_NEW,
306 	BTRFS_COMPARE_TREE_DELETED,
307 	BTRFS_COMPARE_TREE_CHANGED,
308 	BTRFS_COMPARE_TREE_SAME,
309 };
310 
311 __cold
inconsistent_snapshot_error(struct send_ctx * sctx,enum btrfs_compare_tree_result result,const char * what)312 static void inconsistent_snapshot_error(struct send_ctx *sctx,
313 					enum btrfs_compare_tree_result result,
314 					const char *what)
315 {
316 	const char *result_string;
317 
318 	switch (result) {
319 	case BTRFS_COMPARE_TREE_NEW:
320 		result_string = "new";
321 		break;
322 	case BTRFS_COMPARE_TREE_DELETED:
323 		result_string = "deleted";
324 		break;
325 	case BTRFS_COMPARE_TREE_CHANGED:
326 		result_string = "updated";
327 		break;
328 	case BTRFS_COMPARE_TREE_SAME:
329 		ASSERT(0);
330 		result_string = "unchanged";
331 		break;
332 	default:
333 		ASSERT(0);
334 		result_string = "unexpected";
335 	}
336 
337 	btrfs_err(sctx->send_root->fs_info,
338 		  "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
339 		  result_string, what, sctx->cmp_key->objectid,
340 		  sctx->send_root->root_key.objectid,
341 		  (sctx->parent_root ?
342 		   sctx->parent_root->root_key.objectid : 0));
343 }
344 
345 __maybe_unused
proto_cmd_ok(const struct send_ctx * sctx,int cmd)346 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
347 {
348 	switch (sctx->proto) {
349 	case 1:	 return cmd <= BTRFS_SEND_C_MAX_V1;
350 	case 2:	 return cmd <= BTRFS_SEND_C_MAX_V2;
351 	case 3:	 return cmd <= BTRFS_SEND_C_MAX_V3;
352 	default: return false;
353 	}
354 }
355 
356 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
357 
358 static struct waiting_dir_move *
359 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
360 
361 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
362 
need_send_hole(struct send_ctx * sctx)363 static int need_send_hole(struct send_ctx *sctx)
364 {
365 	return (sctx->parent_root && !sctx->cur_inode_new &&
366 		!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
367 		S_ISREG(sctx->cur_inode_mode));
368 }
369 
fs_path_reset(struct fs_path * p)370 static void fs_path_reset(struct fs_path *p)
371 {
372 	if (p->reversed) {
373 		p->start = p->buf + p->buf_len - 1;
374 		p->end = p->start;
375 		*p->start = 0;
376 	} else {
377 		p->start = p->buf;
378 		p->end = p->start;
379 		*p->start = 0;
380 	}
381 }
382 
fs_path_alloc(void)383 static struct fs_path *fs_path_alloc(void)
384 {
385 	struct fs_path *p;
386 
387 	p = kmalloc(sizeof(*p), GFP_KERNEL);
388 	if (!p)
389 		return NULL;
390 	p->reversed = 0;
391 	p->buf = p->inline_buf;
392 	p->buf_len = FS_PATH_INLINE_SIZE;
393 	fs_path_reset(p);
394 	return p;
395 }
396 
fs_path_alloc_reversed(void)397 static struct fs_path *fs_path_alloc_reversed(void)
398 {
399 	struct fs_path *p;
400 
401 	p = fs_path_alloc();
402 	if (!p)
403 		return NULL;
404 	p->reversed = 1;
405 	fs_path_reset(p);
406 	return p;
407 }
408 
fs_path_free(struct fs_path * p)409 static void fs_path_free(struct fs_path *p)
410 {
411 	if (!p)
412 		return;
413 	if (p->buf != p->inline_buf)
414 		kfree(p->buf);
415 	kfree(p);
416 }
417 
fs_path_len(struct fs_path * p)418 static int fs_path_len(struct fs_path *p)
419 {
420 	return p->end - p->start;
421 }
422 
fs_path_ensure_buf(struct fs_path * p,int len)423 static int fs_path_ensure_buf(struct fs_path *p, int len)
424 {
425 	char *tmp_buf;
426 	int path_len;
427 	int old_buf_len;
428 
429 	len++;
430 
431 	if (p->buf_len >= len)
432 		return 0;
433 
434 	if (len > PATH_MAX) {
435 		WARN_ON(1);
436 		return -ENOMEM;
437 	}
438 
439 	path_len = p->end - p->start;
440 	old_buf_len = p->buf_len;
441 
442 	/*
443 	 * First time the inline_buf does not suffice
444 	 */
445 	if (p->buf == p->inline_buf) {
446 		tmp_buf = kmalloc(len, GFP_KERNEL);
447 		if (tmp_buf)
448 			memcpy(tmp_buf, p->buf, old_buf_len);
449 	} else {
450 		tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
451 	}
452 	if (!tmp_buf)
453 		return -ENOMEM;
454 	p->buf = tmp_buf;
455 	/*
456 	 * The real size of the buffer is bigger, this will let the fast path
457 	 * happen most of the time
458 	 */
459 	p->buf_len = ksize(p->buf);
460 
461 	if (p->reversed) {
462 		tmp_buf = p->buf + old_buf_len - path_len - 1;
463 		p->end = p->buf + p->buf_len - 1;
464 		p->start = p->end - path_len;
465 		memmove(p->start, tmp_buf, path_len + 1);
466 	} else {
467 		p->start = p->buf;
468 		p->end = p->start + path_len;
469 	}
470 	return 0;
471 }
472 
fs_path_prepare_for_add(struct fs_path * p,int name_len,char ** prepared)473 static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
474 				   char **prepared)
475 {
476 	int ret;
477 	int new_len;
478 
479 	new_len = p->end - p->start + name_len;
480 	if (p->start != p->end)
481 		new_len++;
482 	ret = fs_path_ensure_buf(p, new_len);
483 	if (ret < 0)
484 		goto out;
485 
486 	if (p->reversed) {
487 		if (p->start != p->end)
488 			*--p->start = '/';
489 		p->start -= name_len;
490 		*prepared = p->start;
491 	} else {
492 		if (p->start != p->end)
493 			*p->end++ = '/';
494 		*prepared = p->end;
495 		p->end += name_len;
496 		*p->end = 0;
497 	}
498 
499 out:
500 	return ret;
501 }
502 
fs_path_add(struct fs_path * p,const char * name,int name_len)503 static int fs_path_add(struct fs_path *p, const char *name, int name_len)
504 {
505 	int ret;
506 	char *prepared;
507 
508 	ret = fs_path_prepare_for_add(p, name_len, &prepared);
509 	if (ret < 0)
510 		goto out;
511 	memcpy(prepared, name, name_len);
512 
513 out:
514 	return ret;
515 }
516 
fs_path_add_path(struct fs_path * p,struct fs_path * p2)517 static int fs_path_add_path(struct fs_path *p, struct fs_path *p2)
518 {
519 	int ret;
520 	char *prepared;
521 
522 	ret = fs_path_prepare_for_add(p, p2->end - p2->start, &prepared);
523 	if (ret < 0)
524 		goto out;
525 	memcpy(prepared, p2->start, p2->end - p2->start);
526 
527 out:
528 	return ret;
529 }
530 
fs_path_add_from_extent_buffer(struct fs_path * p,struct extent_buffer * eb,unsigned long off,int len)531 static int fs_path_add_from_extent_buffer(struct fs_path *p,
532 					  struct extent_buffer *eb,
533 					  unsigned long off, int len)
534 {
535 	int ret;
536 	char *prepared;
537 
538 	ret = fs_path_prepare_for_add(p, len, &prepared);
539 	if (ret < 0)
540 		goto out;
541 
542 	read_extent_buffer(eb, prepared, off, len);
543 
544 out:
545 	return ret;
546 }
547 
fs_path_copy(struct fs_path * p,struct fs_path * from)548 static int fs_path_copy(struct fs_path *p, struct fs_path *from)
549 {
550 	p->reversed = from->reversed;
551 	fs_path_reset(p);
552 
553 	return fs_path_add_path(p, from);
554 }
555 
fs_path_unreverse(struct fs_path * p)556 static void fs_path_unreverse(struct fs_path *p)
557 {
558 	char *tmp;
559 	int len;
560 
561 	if (!p->reversed)
562 		return;
563 
564 	tmp = p->start;
565 	len = p->end - p->start;
566 	p->start = p->buf;
567 	p->end = p->start + len;
568 	memmove(p->start, tmp, len + 1);
569 	p->reversed = 0;
570 }
571 
alloc_path_for_send(void)572 static struct btrfs_path *alloc_path_for_send(void)
573 {
574 	struct btrfs_path *path;
575 
576 	path = btrfs_alloc_path();
577 	if (!path)
578 		return NULL;
579 	path->search_commit_root = 1;
580 	path->skip_locking = 1;
581 	path->need_commit_sem = 1;
582 	return path;
583 }
584 
write_buf(struct file * filp,const void * buf,u32 len,loff_t * off)585 static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
586 {
587 	int ret;
588 	u32 pos = 0;
589 
590 	while (pos < len) {
591 		ret = kernel_write(filp, buf + pos, len - pos, off);
592 		if (ret < 0)
593 			return ret;
594 		if (ret == 0)
595 			return -EIO;
596 		pos += ret;
597 	}
598 
599 	return 0;
600 }
601 
tlv_put(struct send_ctx * sctx,u16 attr,const void * data,int len)602 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
603 {
604 	struct btrfs_tlv_header *hdr;
605 	int total_len = sizeof(*hdr) + len;
606 	int left = sctx->send_max_size - sctx->send_size;
607 
608 	if (WARN_ON_ONCE(sctx->put_data))
609 		return -EINVAL;
610 
611 	if (unlikely(left < total_len))
612 		return -EOVERFLOW;
613 
614 	hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
615 	put_unaligned_le16(attr, &hdr->tlv_type);
616 	put_unaligned_le16(len, &hdr->tlv_len);
617 	memcpy(hdr + 1, data, len);
618 	sctx->send_size += total_len;
619 
620 	return 0;
621 }
622 
623 #define TLV_PUT_DEFINE_INT(bits) \
624 	static int tlv_put_u##bits(struct send_ctx *sctx,	 	\
625 			u##bits attr, u##bits value)			\
626 	{								\
627 		__le##bits __tmp = cpu_to_le##bits(value);		\
628 		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
629 	}
630 
631 TLV_PUT_DEFINE_INT(8)
632 TLV_PUT_DEFINE_INT(32)
633 TLV_PUT_DEFINE_INT(64)
634 
tlv_put_string(struct send_ctx * sctx,u16 attr,const char * str,int len)635 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
636 			  const char *str, int len)
637 {
638 	if (len == -1)
639 		len = strlen(str);
640 	return tlv_put(sctx, attr, str, len);
641 }
642 
tlv_put_uuid(struct send_ctx * sctx,u16 attr,const u8 * uuid)643 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
644 			const u8 *uuid)
645 {
646 	return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
647 }
648 
tlv_put_btrfs_timespec(struct send_ctx * sctx,u16 attr,struct extent_buffer * eb,struct btrfs_timespec * ts)649 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
650 				  struct extent_buffer *eb,
651 				  struct btrfs_timespec *ts)
652 {
653 	struct btrfs_timespec bts;
654 	read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
655 	return tlv_put(sctx, attr, &bts, sizeof(bts));
656 }
657 
658 
659 #define TLV_PUT(sctx, attrtype, data, attrlen) \
660 	do { \
661 		ret = tlv_put(sctx, attrtype, data, attrlen); \
662 		if (ret < 0) \
663 			goto tlv_put_failure; \
664 	} while (0)
665 
666 #define TLV_PUT_INT(sctx, attrtype, bits, value) \
667 	do { \
668 		ret = tlv_put_u##bits(sctx, attrtype, value); \
669 		if (ret < 0) \
670 			goto tlv_put_failure; \
671 	} while (0)
672 
673 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
674 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
675 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
676 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
677 #define TLV_PUT_STRING(sctx, attrtype, str, len) \
678 	do { \
679 		ret = tlv_put_string(sctx, attrtype, str, len); \
680 		if (ret < 0) \
681 			goto tlv_put_failure; \
682 	} while (0)
683 #define TLV_PUT_PATH(sctx, attrtype, p) \
684 	do { \
685 		ret = tlv_put_string(sctx, attrtype, p->start, \
686 			p->end - p->start); \
687 		if (ret < 0) \
688 			goto tlv_put_failure; \
689 	} while(0)
690 #define TLV_PUT_UUID(sctx, attrtype, uuid) \
691 	do { \
692 		ret = tlv_put_uuid(sctx, attrtype, uuid); \
693 		if (ret < 0) \
694 			goto tlv_put_failure; \
695 	} while (0)
696 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
697 	do { \
698 		ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
699 		if (ret < 0) \
700 			goto tlv_put_failure; \
701 	} while (0)
702 
send_header(struct send_ctx * sctx)703 static int send_header(struct send_ctx *sctx)
704 {
705 	struct btrfs_stream_header hdr;
706 
707 	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
708 	hdr.version = cpu_to_le32(sctx->proto);
709 	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
710 					&sctx->send_off);
711 }
712 
713 /*
714  * For each command/item we want to send to userspace, we call this function.
715  */
begin_cmd(struct send_ctx * sctx,int cmd)716 static int begin_cmd(struct send_ctx *sctx, int cmd)
717 {
718 	struct btrfs_cmd_header *hdr;
719 
720 	if (WARN_ON(!sctx->send_buf))
721 		return -EINVAL;
722 
723 	BUG_ON(sctx->send_size);
724 
725 	sctx->send_size += sizeof(*hdr);
726 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
727 	put_unaligned_le16(cmd, &hdr->cmd);
728 
729 	return 0;
730 }
731 
send_cmd(struct send_ctx * sctx)732 static int send_cmd(struct send_ctx *sctx)
733 {
734 	int ret;
735 	struct btrfs_cmd_header *hdr;
736 	u32 crc;
737 
738 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
739 	put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
740 	put_unaligned_le32(0, &hdr->crc);
741 
742 	crc = btrfs_crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
743 	put_unaligned_le32(crc, &hdr->crc);
744 
745 	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
746 					&sctx->send_off);
747 
748 	sctx->send_size = 0;
749 	sctx->put_data = false;
750 
751 	return ret;
752 }
753 
754 /*
755  * Sends a move instruction to user space
756  */
send_rename(struct send_ctx * sctx,struct fs_path * from,struct fs_path * to)757 static int send_rename(struct send_ctx *sctx,
758 		     struct fs_path *from, struct fs_path *to)
759 {
760 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
761 	int ret;
762 
763 	btrfs_debug(fs_info, "send_rename %s -> %s", from->start, to->start);
764 
765 	ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
766 	if (ret < 0)
767 		goto out;
768 
769 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
770 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
771 
772 	ret = send_cmd(sctx);
773 
774 tlv_put_failure:
775 out:
776 	return ret;
777 }
778 
779 /*
780  * Sends a link instruction to user space
781  */
send_link(struct send_ctx * sctx,struct fs_path * path,struct fs_path * lnk)782 static int send_link(struct send_ctx *sctx,
783 		     struct fs_path *path, struct fs_path *lnk)
784 {
785 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
786 	int ret;
787 
788 	btrfs_debug(fs_info, "send_link %s -> %s", path->start, lnk->start);
789 
790 	ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
791 	if (ret < 0)
792 		goto out;
793 
794 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
795 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
796 
797 	ret = send_cmd(sctx);
798 
799 tlv_put_failure:
800 out:
801 	return ret;
802 }
803 
804 /*
805  * Sends an unlink instruction to user space
806  */
send_unlink(struct send_ctx * sctx,struct fs_path * path)807 static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
808 {
809 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
810 	int ret;
811 
812 	btrfs_debug(fs_info, "send_unlink %s", path->start);
813 
814 	ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
815 	if (ret < 0)
816 		goto out;
817 
818 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
819 
820 	ret = send_cmd(sctx);
821 
822 tlv_put_failure:
823 out:
824 	return ret;
825 }
826 
827 /*
828  * Sends a rmdir instruction to user space
829  */
send_rmdir(struct send_ctx * sctx,struct fs_path * path)830 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
831 {
832 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
833 	int ret;
834 
835 	btrfs_debug(fs_info, "send_rmdir %s", path->start);
836 
837 	ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
838 	if (ret < 0)
839 		goto out;
840 
841 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
842 
843 	ret = send_cmd(sctx);
844 
845 tlv_put_failure:
846 out:
847 	return ret;
848 }
849 
850 struct btrfs_inode_info {
851 	u64 size;
852 	u64 gen;
853 	u64 mode;
854 	u64 uid;
855 	u64 gid;
856 	u64 rdev;
857 	u64 fileattr;
858 	u64 nlink;
859 };
860 
861 /*
862  * Helper function to retrieve some fields from an inode item.
863  */
get_inode_info(struct btrfs_root * root,u64 ino,struct btrfs_inode_info * info)864 static int get_inode_info(struct btrfs_root *root, u64 ino,
865 			  struct btrfs_inode_info *info)
866 {
867 	int ret;
868 	struct btrfs_path *path;
869 	struct btrfs_inode_item *ii;
870 	struct btrfs_key key;
871 
872 	path = alloc_path_for_send();
873 	if (!path)
874 		return -ENOMEM;
875 
876 	key.objectid = ino;
877 	key.type = BTRFS_INODE_ITEM_KEY;
878 	key.offset = 0;
879 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
880 	if (ret) {
881 		if (ret > 0)
882 			ret = -ENOENT;
883 		goto out;
884 	}
885 
886 	if (!info)
887 		goto out;
888 
889 	ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
890 			struct btrfs_inode_item);
891 	info->size = btrfs_inode_size(path->nodes[0], ii);
892 	info->gen = btrfs_inode_generation(path->nodes[0], ii);
893 	info->mode = btrfs_inode_mode(path->nodes[0], ii);
894 	info->uid = btrfs_inode_uid(path->nodes[0], ii);
895 	info->gid = btrfs_inode_gid(path->nodes[0], ii);
896 	info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
897 	info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
898 	/*
899 	 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
900 	 * otherwise logically split to 32/32 parts.
901 	 */
902 	info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
903 
904 out:
905 	btrfs_free_path(path);
906 	return ret;
907 }
908 
get_inode_gen(struct btrfs_root * root,u64 ino,u64 * gen)909 static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
910 {
911 	int ret;
912 	struct btrfs_inode_info info;
913 
914 	if (!gen)
915 		return -EPERM;
916 
917 	ret = get_inode_info(root, ino, &info);
918 	if (!ret)
919 		*gen = info.gen;
920 	return ret;
921 }
922 
923 typedef int (*iterate_inode_ref_t)(int num, u64 dir, int index,
924 				   struct fs_path *p,
925 				   void *ctx);
926 
927 /*
928  * Helper function to iterate the entries in ONE btrfs_inode_ref or
929  * btrfs_inode_extref.
930  * The iterate callback may return a non zero value to stop iteration. This can
931  * be a negative value for error codes or 1 to simply stop it.
932  *
933  * path must point to the INODE_REF or INODE_EXTREF when called.
934  */
iterate_inode_ref(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * found_key,int resolve,iterate_inode_ref_t iterate,void * ctx)935 static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
936 			     struct btrfs_key *found_key, int resolve,
937 			     iterate_inode_ref_t iterate, void *ctx)
938 {
939 	struct extent_buffer *eb = path->nodes[0];
940 	struct btrfs_inode_ref *iref;
941 	struct btrfs_inode_extref *extref;
942 	struct btrfs_path *tmp_path;
943 	struct fs_path *p;
944 	u32 cur = 0;
945 	u32 total;
946 	int slot = path->slots[0];
947 	u32 name_len;
948 	char *start;
949 	int ret = 0;
950 	int num = 0;
951 	int index;
952 	u64 dir;
953 	unsigned long name_off;
954 	unsigned long elem_size;
955 	unsigned long ptr;
956 
957 	p = fs_path_alloc_reversed();
958 	if (!p)
959 		return -ENOMEM;
960 
961 	tmp_path = alloc_path_for_send();
962 	if (!tmp_path) {
963 		fs_path_free(p);
964 		return -ENOMEM;
965 	}
966 
967 
968 	if (found_key->type == BTRFS_INODE_REF_KEY) {
969 		ptr = (unsigned long)btrfs_item_ptr(eb, slot,
970 						    struct btrfs_inode_ref);
971 		total = btrfs_item_size(eb, slot);
972 		elem_size = sizeof(*iref);
973 	} else {
974 		ptr = btrfs_item_ptr_offset(eb, slot);
975 		total = btrfs_item_size(eb, slot);
976 		elem_size = sizeof(*extref);
977 	}
978 
979 	while (cur < total) {
980 		fs_path_reset(p);
981 
982 		if (found_key->type == BTRFS_INODE_REF_KEY) {
983 			iref = (struct btrfs_inode_ref *)(ptr + cur);
984 			name_len = btrfs_inode_ref_name_len(eb, iref);
985 			name_off = (unsigned long)(iref + 1);
986 			index = btrfs_inode_ref_index(eb, iref);
987 			dir = found_key->offset;
988 		} else {
989 			extref = (struct btrfs_inode_extref *)(ptr + cur);
990 			name_len = btrfs_inode_extref_name_len(eb, extref);
991 			name_off = (unsigned long)&extref->name;
992 			index = btrfs_inode_extref_index(eb, extref);
993 			dir = btrfs_inode_extref_parent(eb, extref);
994 		}
995 
996 		if (resolve) {
997 			start = btrfs_ref_to_path(root, tmp_path, name_len,
998 						  name_off, eb, dir,
999 						  p->buf, p->buf_len);
1000 			if (IS_ERR(start)) {
1001 				ret = PTR_ERR(start);
1002 				goto out;
1003 			}
1004 			if (start < p->buf) {
1005 				/* overflow , try again with larger buffer */
1006 				ret = fs_path_ensure_buf(p,
1007 						p->buf_len + p->buf - start);
1008 				if (ret < 0)
1009 					goto out;
1010 				start = btrfs_ref_to_path(root, tmp_path,
1011 							  name_len, name_off,
1012 							  eb, dir,
1013 							  p->buf, p->buf_len);
1014 				if (IS_ERR(start)) {
1015 					ret = PTR_ERR(start);
1016 					goto out;
1017 				}
1018 				BUG_ON(start < p->buf);
1019 			}
1020 			p->start = start;
1021 		} else {
1022 			ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1023 							     name_len);
1024 			if (ret < 0)
1025 				goto out;
1026 		}
1027 
1028 		cur += elem_size + name_len;
1029 		ret = iterate(num, dir, index, p, ctx);
1030 		if (ret)
1031 			goto out;
1032 		num++;
1033 	}
1034 
1035 out:
1036 	btrfs_free_path(tmp_path);
1037 	fs_path_free(p);
1038 	return ret;
1039 }
1040 
1041 typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1042 				  const char *name, int name_len,
1043 				  const char *data, int data_len,
1044 				  void *ctx);
1045 
1046 /*
1047  * Helper function to iterate the entries in ONE btrfs_dir_item.
1048  * The iterate callback may return a non zero value to stop iteration. This can
1049  * be a negative value for error codes or 1 to simply stop it.
1050  *
1051  * path must point to the dir item when called.
1052  */
iterate_dir_item(struct btrfs_root * root,struct btrfs_path * path,iterate_dir_item_t iterate,void * ctx)1053 static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1054 			    iterate_dir_item_t iterate, void *ctx)
1055 {
1056 	int ret = 0;
1057 	struct extent_buffer *eb;
1058 	struct btrfs_dir_item *di;
1059 	struct btrfs_key di_key;
1060 	char *buf = NULL;
1061 	int buf_len;
1062 	u32 name_len;
1063 	u32 data_len;
1064 	u32 cur;
1065 	u32 len;
1066 	u32 total;
1067 	int slot;
1068 	int num;
1069 
1070 	/*
1071 	 * Start with a small buffer (1 page). If later we end up needing more
1072 	 * space, which can happen for xattrs on a fs with a leaf size greater
1073 	 * then the page size, attempt to increase the buffer. Typically xattr
1074 	 * values are small.
1075 	 */
1076 	buf_len = PATH_MAX;
1077 	buf = kmalloc(buf_len, GFP_KERNEL);
1078 	if (!buf) {
1079 		ret = -ENOMEM;
1080 		goto out;
1081 	}
1082 
1083 	eb = path->nodes[0];
1084 	slot = path->slots[0];
1085 	di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1086 	cur = 0;
1087 	len = 0;
1088 	total = btrfs_item_size(eb, slot);
1089 
1090 	num = 0;
1091 	while (cur < total) {
1092 		name_len = btrfs_dir_name_len(eb, di);
1093 		data_len = btrfs_dir_data_len(eb, di);
1094 		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1095 
1096 		if (btrfs_dir_type(eb, di) == BTRFS_FT_XATTR) {
1097 			if (name_len > XATTR_NAME_MAX) {
1098 				ret = -ENAMETOOLONG;
1099 				goto out;
1100 			}
1101 			if (name_len + data_len >
1102 					BTRFS_MAX_XATTR_SIZE(root->fs_info)) {
1103 				ret = -E2BIG;
1104 				goto out;
1105 			}
1106 		} else {
1107 			/*
1108 			 * Path too long
1109 			 */
1110 			if (name_len + data_len > PATH_MAX) {
1111 				ret = -ENAMETOOLONG;
1112 				goto out;
1113 			}
1114 		}
1115 
1116 		if (name_len + data_len > buf_len) {
1117 			buf_len = name_len + data_len;
1118 			if (is_vmalloc_addr(buf)) {
1119 				vfree(buf);
1120 				buf = NULL;
1121 			} else {
1122 				char *tmp = krealloc(buf, buf_len,
1123 						GFP_KERNEL | __GFP_NOWARN);
1124 
1125 				if (!tmp)
1126 					kfree(buf);
1127 				buf = tmp;
1128 			}
1129 			if (!buf) {
1130 				buf = kvmalloc(buf_len, GFP_KERNEL);
1131 				if (!buf) {
1132 					ret = -ENOMEM;
1133 					goto out;
1134 				}
1135 			}
1136 		}
1137 
1138 		read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1139 				name_len + data_len);
1140 
1141 		len = sizeof(*di) + name_len + data_len;
1142 		di = (struct btrfs_dir_item *)((char *)di + len);
1143 		cur += len;
1144 
1145 		ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1146 			      data_len, ctx);
1147 		if (ret < 0)
1148 			goto out;
1149 		if (ret) {
1150 			ret = 0;
1151 			goto out;
1152 		}
1153 
1154 		num++;
1155 	}
1156 
1157 out:
1158 	kvfree(buf);
1159 	return ret;
1160 }
1161 
__copy_first_ref(int num,u64 dir,int index,struct fs_path * p,void * ctx)1162 static int __copy_first_ref(int num, u64 dir, int index,
1163 			    struct fs_path *p, void *ctx)
1164 {
1165 	int ret;
1166 	struct fs_path *pt = ctx;
1167 
1168 	ret = fs_path_copy(pt, p);
1169 	if (ret < 0)
1170 		return ret;
1171 
1172 	/* we want the first only */
1173 	return 1;
1174 }
1175 
1176 /*
1177  * Retrieve the first path of an inode. If an inode has more then one
1178  * ref/hardlink, this is ignored.
1179  */
get_inode_path(struct btrfs_root * root,u64 ino,struct fs_path * path)1180 static int get_inode_path(struct btrfs_root *root,
1181 			  u64 ino, struct fs_path *path)
1182 {
1183 	int ret;
1184 	struct btrfs_key key, found_key;
1185 	struct btrfs_path *p;
1186 
1187 	p = alloc_path_for_send();
1188 	if (!p)
1189 		return -ENOMEM;
1190 
1191 	fs_path_reset(path);
1192 
1193 	key.objectid = ino;
1194 	key.type = BTRFS_INODE_REF_KEY;
1195 	key.offset = 0;
1196 
1197 	ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1198 	if (ret < 0)
1199 		goto out;
1200 	if (ret) {
1201 		ret = 1;
1202 		goto out;
1203 	}
1204 	btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1205 	if (found_key.objectid != ino ||
1206 	    (found_key.type != BTRFS_INODE_REF_KEY &&
1207 	     found_key.type != BTRFS_INODE_EXTREF_KEY)) {
1208 		ret = -ENOENT;
1209 		goto out;
1210 	}
1211 
1212 	ret = iterate_inode_ref(root, p, &found_key, 1,
1213 				__copy_first_ref, path);
1214 	if (ret < 0)
1215 		goto out;
1216 	ret = 0;
1217 
1218 out:
1219 	btrfs_free_path(p);
1220 	return ret;
1221 }
1222 
1223 struct backref_ctx {
1224 	struct send_ctx *sctx;
1225 
1226 	/* number of total found references */
1227 	u64 found;
1228 
1229 	/*
1230 	 * used for clones found in send_root. clones found behind cur_objectid
1231 	 * and cur_offset are not considered as allowed clones.
1232 	 */
1233 	u64 cur_objectid;
1234 	u64 cur_offset;
1235 
1236 	/* may be truncated in case it's the last extent in a file */
1237 	u64 extent_len;
1238 
1239 	/* Just to check for bugs in backref resolving */
1240 	int found_itself;
1241 };
1242 
__clone_root_cmp_bsearch(const void * key,const void * elt)1243 static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1244 {
1245 	u64 root = (u64)(uintptr_t)key;
1246 	const struct clone_root *cr = elt;
1247 
1248 	if (root < cr->root->root_key.objectid)
1249 		return -1;
1250 	if (root > cr->root->root_key.objectid)
1251 		return 1;
1252 	return 0;
1253 }
1254 
__clone_root_cmp_sort(const void * e1,const void * e2)1255 static int __clone_root_cmp_sort(const void *e1, const void *e2)
1256 {
1257 	const struct clone_root *cr1 = e1;
1258 	const struct clone_root *cr2 = e2;
1259 
1260 	if (cr1->root->root_key.objectid < cr2->root->root_key.objectid)
1261 		return -1;
1262 	if (cr1->root->root_key.objectid > cr2->root->root_key.objectid)
1263 		return 1;
1264 	return 0;
1265 }
1266 
1267 /*
1268  * Called for every backref that is found for the current extent.
1269  * Results are collected in sctx->clone_roots->ino/offset/found_refs
1270  */
__iterate_backrefs(u64 ino,u64 offset,u64 root,void * ctx_)1271 static int __iterate_backrefs(u64 ino, u64 offset, u64 root, void *ctx_)
1272 {
1273 	struct backref_ctx *bctx = ctx_;
1274 	struct clone_root *found;
1275 
1276 	/* First check if the root is in the list of accepted clone sources */
1277 	found = bsearch((void *)(uintptr_t)root, bctx->sctx->clone_roots,
1278 			bctx->sctx->clone_roots_cnt,
1279 			sizeof(struct clone_root),
1280 			__clone_root_cmp_bsearch);
1281 	if (!found)
1282 		return 0;
1283 
1284 	if (found->root == bctx->sctx->send_root &&
1285 	    ino == bctx->cur_objectid &&
1286 	    offset == bctx->cur_offset) {
1287 		bctx->found_itself = 1;
1288 	}
1289 
1290 	/*
1291 	 * Make sure we don't consider clones from send_root that are
1292 	 * behind the current inode/offset.
1293 	 */
1294 	if (found->root == bctx->sctx->send_root) {
1295 		/*
1296 		 * If the source inode was not yet processed we can't issue a
1297 		 * clone operation, as the source extent does not exist yet at
1298 		 * the destination of the stream.
1299 		 */
1300 		if (ino > bctx->cur_objectid)
1301 			return 0;
1302 		/*
1303 		 * We clone from the inode currently being sent as long as the
1304 		 * source extent is already processed, otherwise we could try
1305 		 * to clone from an extent that does not exist yet at the
1306 		 * destination of the stream.
1307 		 */
1308 		if (ino == bctx->cur_objectid &&
1309 		    offset + bctx->extent_len >
1310 		    bctx->sctx->cur_inode_next_write_offset)
1311 			return 0;
1312 	}
1313 
1314 	bctx->found++;
1315 	found->found_refs++;
1316 	if (ino < found->ino) {
1317 		found->ino = ino;
1318 		found->offset = offset;
1319 	} else if (found->ino == ino) {
1320 		/*
1321 		 * same extent found more then once in the same file.
1322 		 */
1323 		if (found->offset > offset + bctx->extent_len)
1324 			found->offset = offset;
1325 	}
1326 
1327 	return 0;
1328 }
1329 
1330 /*
1331  * Given an inode, offset and extent item, it finds a good clone for a clone
1332  * instruction. Returns -ENOENT when none could be found. The function makes
1333  * sure that the returned clone is usable at the point where sending is at the
1334  * moment. This means, that no clones are accepted which lie behind the current
1335  * inode+offset.
1336  *
1337  * path must point to the extent item when called.
1338  */
find_extent_clone(struct send_ctx * sctx,struct btrfs_path * path,u64 ino,u64 data_offset,u64 ino_size,struct clone_root ** found)1339 static int find_extent_clone(struct send_ctx *sctx,
1340 			     struct btrfs_path *path,
1341 			     u64 ino, u64 data_offset,
1342 			     u64 ino_size,
1343 			     struct clone_root **found)
1344 {
1345 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1346 	int ret;
1347 	int extent_type;
1348 	u64 logical;
1349 	u64 disk_byte;
1350 	u64 num_bytes;
1351 	u64 extent_item_pos;
1352 	u64 flags = 0;
1353 	struct btrfs_file_extent_item *fi;
1354 	struct extent_buffer *eb = path->nodes[0];
1355 	struct backref_ctx backref_ctx = {0};
1356 	struct clone_root *cur_clone_root;
1357 	struct btrfs_key found_key;
1358 	struct btrfs_path *tmp_path;
1359 	struct btrfs_extent_item *ei;
1360 	int compressed;
1361 	u32 i;
1362 
1363 	tmp_path = alloc_path_for_send();
1364 	if (!tmp_path)
1365 		return -ENOMEM;
1366 
1367 	/* We only use this path under the commit sem */
1368 	tmp_path->need_commit_sem = 0;
1369 
1370 	if (data_offset >= ino_size) {
1371 		/*
1372 		 * There may be extents that lie behind the file's size.
1373 		 * I at least had this in combination with snapshotting while
1374 		 * writing large files.
1375 		 */
1376 		ret = 0;
1377 		goto out;
1378 	}
1379 
1380 	fi = btrfs_item_ptr(eb, path->slots[0],
1381 			struct btrfs_file_extent_item);
1382 	extent_type = btrfs_file_extent_type(eb, fi);
1383 	if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1384 		ret = -ENOENT;
1385 		goto out;
1386 	}
1387 	compressed = btrfs_file_extent_compression(eb, fi);
1388 
1389 	num_bytes = btrfs_file_extent_num_bytes(eb, fi);
1390 	disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1391 	if (disk_byte == 0) {
1392 		ret = -ENOENT;
1393 		goto out;
1394 	}
1395 	logical = disk_byte + btrfs_file_extent_offset(eb, fi);
1396 
1397 	down_read(&fs_info->commit_root_sem);
1398 	ret = extent_from_logical(fs_info, disk_byte, tmp_path,
1399 				  &found_key, &flags);
1400 	up_read(&fs_info->commit_root_sem);
1401 
1402 	if (ret < 0)
1403 		goto out;
1404 	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
1405 		ret = -EIO;
1406 		goto out;
1407 	}
1408 
1409 	ei = btrfs_item_ptr(tmp_path->nodes[0], tmp_path->slots[0],
1410 			    struct btrfs_extent_item);
1411 	/*
1412 	 * Backreference walking (iterate_extent_inodes() below) is currently
1413 	 * too expensive when an extent has a large number of references, both
1414 	 * in time spent and used memory. So for now just fallback to write
1415 	 * operations instead of clone operations when an extent has more than
1416 	 * a certain amount of references.
1417 	 */
1418 	if (btrfs_extent_refs(tmp_path->nodes[0], ei) > SEND_MAX_EXTENT_REFS) {
1419 		ret = -ENOENT;
1420 		goto out;
1421 	}
1422 	btrfs_release_path(tmp_path);
1423 
1424 	/*
1425 	 * Setup the clone roots.
1426 	 */
1427 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1428 		cur_clone_root = sctx->clone_roots + i;
1429 		cur_clone_root->ino = (u64)-1;
1430 		cur_clone_root->offset = 0;
1431 		cur_clone_root->found_refs = 0;
1432 	}
1433 
1434 	backref_ctx.sctx = sctx;
1435 	backref_ctx.found = 0;
1436 	backref_ctx.cur_objectid = ino;
1437 	backref_ctx.cur_offset = data_offset;
1438 	backref_ctx.found_itself = 0;
1439 	backref_ctx.extent_len = num_bytes;
1440 
1441 	/*
1442 	 * The last extent of a file may be too large due to page alignment.
1443 	 * We need to adjust extent_len in this case so that the checks in
1444 	 * __iterate_backrefs work.
1445 	 */
1446 	if (data_offset + num_bytes >= ino_size)
1447 		backref_ctx.extent_len = ino_size - data_offset;
1448 
1449 	/*
1450 	 * Now collect all backrefs.
1451 	 */
1452 	if (compressed == BTRFS_COMPRESS_NONE)
1453 		extent_item_pos = logical - found_key.objectid;
1454 	else
1455 		extent_item_pos = 0;
1456 	ret = iterate_extent_inodes(fs_info, found_key.objectid,
1457 				    extent_item_pos, 1, __iterate_backrefs,
1458 				    &backref_ctx, false);
1459 
1460 	if (ret < 0)
1461 		goto out;
1462 
1463 	down_read(&fs_info->commit_root_sem);
1464 	if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1465 		/*
1466 		 * A transaction commit for a transaction in which block group
1467 		 * relocation was done just happened.
1468 		 * The disk_bytenr of the file extent item we processed is
1469 		 * possibly stale, referring to the extent's location before
1470 		 * relocation. So act as if we haven't found any clone sources
1471 		 * and fallback to write commands, which will read the correct
1472 		 * data from the new extent location. Otherwise we will fail
1473 		 * below because we haven't found our own back reference or we
1474 		 * could be getting incorrect sources in case the old extent
1475 		 * was already reallocated after the relocation.
1476 		 */
1477 		up_read(&fs_info->commit_root_sem);
1478 		ret = -ENOENT;
1479 		goto out;
1480 	}
1481 	up_read(&fs_info->commit_root_sem);
1482 
1483 	if (!backref_ctx.found_itself) {
1484 		/* found a bug in backref code? */
1485 		ret = -EIO;
1486 		btrfs_err(fs_info,
1487 			  "did not find backref in send_root. inode=%llu, offset=%llu, disk_byte=%llu found extent=%llu",
1488 			  ino, data_offset, disk_byte, found_key.objectid);
1489 		goto out;
1490 	}
1491 
1492 	btrfs_debug(fs_info,
1493 		    "find_extent_clone: data_offset=%llu, ino=%llu, num_bytes=%llu, logical=%llu",
1494 		    data_offset, ino, num_bytes, logical);
1495 
1496 	if (!backref_ctx.found)
1497 		btrfs_debug(fs_info, "no clones found");
1498 
1499 	cur_clone_root = NULL;
1500 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1501 		if (sctx->clone_roots[i].found_refs) {
1502 			if (!cur_clone_root)
1503 				cur_clone_root = sctx->clone_roots + i;
1504 			else if (sctx->clone_roots[i].root == sctx->send_root)
1505 				/* prefer clones from send_root over others */
1506 				cur_clone_root = sctx->clone_roots + i;
1507 		}
1508 
1509 	}
1510 
1511 	if (cur_clone_root) {
1512 		*found = cur_clone_root;
1513 		ret = 0;
1514 	} else {
1515 		ret = -ENOENT;
1516 	}
1517 
1518 out:
1519 	btrfs_free_path(tmp_path);
1520 	return ret;
1521 }
1522 
read_symlink(struct btrfs_root * root,u64 ino,struct fs_path * dest)1523 static int read_symlink(struct btrfs_root *root,
1524 			u64 ino,
1525 			struct fs_path *dest)
1526 {
1527 	int ret;
1528 	struct btrfs_path *path;
1529 	struct btrfs_key key;
1530 	struct btrfs_file_extent_item *ei;
1531 	u8 type;
1532 	u8 compression;
1533 	unsigned long off;
1534 	int len;
1535 
1536 	path = alloc_path_for_send();
1537 	if (!path)
1538 		return -ENOMEM;
1539 
1540 	key.objectid = ino;
1541 	key.type = BTRFS_EXTENT_DATA_KEY;
1542 	key.offset = 0;
1543 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1544 	if (ret < 0)
1545 		goto out;
1546 	if (ret) {
1547 		/*
1548 		 * An empty symlink inode. Can happen in rare error paths when
1549 		 * creating a symlink (transaction committed before the inode
1550 		 * eviction handler removed the symlink inode items and a crash
1551 		 * happened in between or the subvol was snapshoted in between).
1552 		 * Print an informative message to dmesg/syslog so that the user
1553 		 * can delete the symlink.
1554 		 */
1555 		btrfs_err(root->fs_info,
1556 			  "Found empty symlink inode %llu at root %llu",
1557 			  ino, root->root_key.objectid);
1558 		ret = -EIO;
1559 		goto out;
1560 	}
1561 
1562 	ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1563 			struct btrfs_file_extent_item);
1564 	type = btrfs_file_extent_type(path->nodes[0], ei);
1565 	compression = btrfs_file_extent_compression(path->nodes[0], ei);
1566 	BUG_ON(type != BTRFS_FILE_EXTENT_INLINE);
1567 	BUG_ON(compression);
1568 
1569 	off = btrfs_file_extent_inline_start(ei);
1570 	len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1571 
1572 	ret = fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1573 
1574 out:
1575 	btrfs_free_path(path);
1576 	return ret;
1577 }
1578 
1579 /*
1580  * Helper function to generate a file name that is unique in the root of
1581  * send_root and parent_root. This is used to generate names for orphan inodes.
1582  */
gen_unique_name(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)1583 static int gen_unique_name(struct send_ctx *sctx,
1584 			   u64 ino, u64 gen,
1585 			   struct fs_path *dest)
1586 {
1587 	int ret = 0;
1588 	struct btrfs_path *path;
1589 	struct btrfs_dir_item *di;
1590 	char tmp[64];
1591 	int len;
1592 	u64 idx = 0;
1593 
1594 	path = alloc_path_for_send();
1595 	if (!path)
1596 		return -ENOMEM;
1597 
1598 	while (1) {
1599 		len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1600 				ino, gen, idx);
1601 		ASSERT(len < sizeof(tmp));
1602 
1603 		di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1604 				path, BTRFS_FIRST_FREE_OBJECTID,
1605 				tmp, strlen(tmp), 0);
1606 		btrfs_release_path(path);
1607 		if (IS_ERR(di)) {
1608 			ret = PTR_ERR(di);
1609 			goto out;
1610 		}
1611 		if (di) {
1612 			/* not unique, try again */
1613 			idx++;
1614 			continue;
1615 		}
1616 
1617 		if (!sctx->parent_root) {
1618 			/* unique */
1619 			ret = 0;
1620 			break;
1621 		}
1622 
1623 		di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1624 				path, BTRFS_FIRST_FREE_OBJECTID,
1625 				tmp, strlen(tmp), 0);
1626 		btrfs_release_path(path);
1627 		if (IS_ERR(di)) {
1628 			ret = PTR_ERR(di);
1629 			goto out;
1630 		}
1631 		if (di) {
1632 			/* not unique, try again */
1633 			idx++;
1634 			continue;
1635 		}
1636 		/* unique */
1637 		break;
1638 	}
1639 
1640 	ret = fs_path_add(dest, tmp, strlen(tmp));
1641 
1642 out:
1643 	btrfs_free_path(path);
1644 	return ret;
1645 }
1646 
1647 enum inode_state {
1648 	inode_state_no_change,
1649 	inode_state_will_create,
1650 	inode_state_did_create,
1651 	inode_state_will_delete,
1652 	inode_state_did_delete,
1653 };
1654 
get_cur_inode_state(struct send_ctx * sctx,u64 ino,u64 gen)1655 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen)
1656 {
1657 	int ret;
1658 	int left_ret;
1659 	int right_ret;
1660 	u64 left_gen;
1661 	u64 right_gen;
1662 	struct btrfs_inode_info info;
1663 
1664 	ret = get_inode_info(sctx->send_root, ino, &info);
1665 	if (ret < 0 && ret != -ENOENT)
1666 		goto out;
1667 	left_ret = (info.nlink == 0) ? -ENOENT : ret;
1668 	left_gen = info.gen;
1669 
1670 	if (!sctx->parent_root) {
1671 		right_ret = -ENOENT;
1672 	} else {
1673 		ret = get_inode_info(sctx->parent_root, ino, &info);
1674 		if (ret < 0 && ret != -ENOENT)
1675 			goto out;
1676 		right_ret = (info.nlink == 0) ? -ENOENT : ret;
1677 		right_gen = info.gen;
1678 	}
1679 
1680 	if (!left_ret && !right_ret) {
1681 		if (left_gen == gen && right_gen == gen) {
1682 			ret = inode_state_no_change;
1683 		} else if (left_gen == gen) {
1684 			if (ino < sctx->send_progress)
1685 				ret = inode_state_did_create;
1686 			else
1687 				ret = inode_state_will_create;
1688 		} else if (right_gen == gen) {
1689 			if (ino < sctx->send_progress)
1690 				ret = inode_state_did_delete;
1691 			else
1692 				ret = inode_state_will_delete;
1693 		} else  {
1694 			ret = -ENOENT;
1695 		}
1696 	} else if (!left_ret) {
1697 		if (left_gen == gen) {
1698 			if (ino < sctx->send_progress)
1699 				ret = inode_state_did_create;
1700 			else
1701 				ret = inode_state_will_create;
1702 		} else {
1703 			ret = -ENOENT;
1704 		}
1705 	} else if (!right_ret) {
1706 		if (right_gen == gen) {
1707 			if (ino < sctx->send_progress)
1708 				ret = inode_state_did_delete;
1709 			else
1710 				ret = inode_state_will_delete;
1711 		} else {
1712 			ret = -ENOENT;
1713 		}
1714 	} else {
1715 		ret = -ENOENT;
1716 	}
1717 
1718 out:
1719 	return ret;
1720 }
1721 
is_inode_existent(struct send_ctx * sctx,u64 ino,u64 gen)1722 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen)
1723 {
1724 	int ret;
1725 
1726 	if (ino == BTRFS_FIRST_FREE_OBJECTID)
1727 		return 1;
1728 
1729 	ret = get_cur_inode_state(sctx, ino, gen);
1730 	if (ret < 0)
1731 		goto out;
1732 
1733 	if (ret == inode_state_no_change ||
1734 	    ret == inode_state_did_create ||
1735 	    ret == inode_state_will_delete)
1736 		ret = 1;
1737 	else
1738 		ret = 0;
1739 
1740 out:
1741 	return ret;
1742 }
1743 
1744 /*
1745  * Helper function to lookup a dir item in a dir.
1746  */
lookup_dir_item_inode(struct btrfs_root * root,u64 dir,const char * name,int name_len,u64 * found_inode)1747 static int lookup_dir_item_inode(struct btrfs_root *root,
1748 				 u64 dir, const char *name, int name_len,
1749 				 u64 *found_inode)
1750 {
1751 	int ret = 0;
1752 	struct btrfs_dir_item *di;
1753 	struct btrfs_key key;
1754 	struct btrfs_path *path;
1755 
1756 	path = alloc_path_for_send();
1757 	if (!path)
1758 		return -ENOMEM;
1759 
1760 	di = btrfs_lookup_dir_item(NULL, root, path,
1761 			dir, name, name_len, 0);
1762 	if (IS_ERR_OR_NULL(di)) {
1763 		ret = di ? PTR_ERR(di) : -ENOENT;
1764 		goto out;
1765 	}
1766 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
1767 	if (key.type == BTRFS_ROOT_ITEM_KEY) {
1768 		ret = -ENOENT;
1769 		goto out;
1770 	}
1771 	*found_inode = key.objectid;
1772 
1773 out:
1774 	btrfs_free_path(path);
1775 	return ret;
1776 }
1777 
1778 /*
1779  * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
1780  * generation of the parent dir and the name of the dir entry.
1781  */
get_first_ref(struct btrfs_root * root,u64 ino,u64 * dir,u64 * dir_gen,struct fs_path * name)1782 static int get_first_ref(struct btrfs_root *root, u64 ino,
1783 			 u64 *dir, u64 *dir_gen, struct fs_path *name)
1784 {
1785 	int ret;
1786 	struct btrfs_key key;
1787 	struct btrfs_key found_key;
1788 	struct btrfs_path *path;
1789 	int len;
1790 	u64 parent_dir;
1791 
1792 	path = alloc_path_for_send();
1793 	if (!path)
1794 		return -ENOMEM;
1795 
1796 	key.objectid = ino;
1797 	key.type = BTRFS_INODE_REF_KEY;
1798 	key.offset = 0;
1799 
1800 	ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
1801 	if (ret < 0)
1802 		goto out;
1803 	if (!ret)
1804 		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1805 				path->slots[0]);
1806 	if (ret || found_key.objectid != ino ||
1807 	    (found_key.type != BTRFS_INODE_REF_KEY &&
1808 	     found_key.type != BTRFS_INODE_EXTREF_KEY)) {
1809 		ret = -ENOENT;
1810 		goto out;
1811 	}
1812 
1813 	if (found_key.type == BTRFS_INODE_REF_KEY) {
1814 		struct btrfs_inode_ref *iref;
1815 		iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1816 				      struct btrfs_inode_ref);
1817 		len = btrfs_inode_ref_name_len(path->nodes[0], iref);
1818 		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
1819 						     (unsigned long)(iref + 1),
1820 						     len);
1821 		parent_dir = found_key.offset;
1822 	} else {
1823 		struct btrfs_inode_extref *extref;
1824 		extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1825 					struct btrfs_inode_extref);
1826 		len = btrfs_inode_extref_name_len(path->nodes[0], extref);
1827 		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
1828 					(unsigned long)&extref->name, len);
1829 		parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
1830 	}
1831 	if (ret < 0)
1832 		goto out;
1833 	btrfs_release_path(path);
1834 
1835 	if (dir_gen) {
1836 		ret = get_inode_gen(root, parent_dir, dir_gen);
1837 		if (ret < 0)
1838 			goto out;
1839 	}
1840 
1841 	*dir = parent_dir;
1842 
1843 out:
1844 	btrfs_free_path(path);
1845 	return ret;
1846 }
1847 
is_first_ref(struct btrfs_root * root,u64 ino,u64 dir,const char * name,int name_len)1848 static int is_first_ref(struct btrfs_root *root,
1849 			u64 ino, u64 dir,
1850 			const char *name, int name_len)
1851 {
1852 	int ret;
1853 	struct fs_path *tmp_name;
1854 	u64 tmp_dir;
1855 
1856 	tmp_name = fs_path_alloc();
1857 	if (!tmp_name)
1858 		return -ENOMEM;
1859 
1860 	ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
1861 	if (ret < 0)
1862 		goto out;
1863 
1864 	if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) {
1865 		ret = 0;
1866 		goto out;
1867 	}
1868 
1869 	ret = !memcmp(tmp_name->start, name, name_len);
1870 
1871 out:
1872 	fs_path_free(tmp_name);
1873 	return ret;
1874 }
1875 
1876 /*
1877  * Used by process_recorded_refs to determine if a new ref would overwrite an
1878  * already existing ref. In case it detects an overwrite, it returns the
1879  * inode/gen in who_ino/who_gen.
1880  * When an overwrite is detected, process_recorded_refs does proper orphanizing
1881  * to make sure later references to the overwritten inode are possible.
1882  * Orphanizing is however only required for the first ref of an inode.
1883  * process_recorded_refs does an additional is_first_ref check to see if
1884  * orphanizing is really required.
1885  */
will_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,const char * name,int name_len,u64 * who_ino,u64 * who_gen,u64 * who_mode)1886 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
1887 			      const char *name, int name_len,
1888 			      u64 *who_ino, u64 *who_gen, u64 *who_mode)
1889 {
1890 	int ret = 0;
1891 	u64 gen;
1892 	u64 other_inode = 0;
1893 	struct btrfs_inode_info info;
1894 
1895 	if (!sctx->parent_root)
1896 		goto out;
1897 
1898 	ret = is_inode_existent(sctx, dir, dir_gen);
1899 	if (ret <= 0)
1900 		goto out;
1901 
1902 	/*
1903 	 * If we have a parent root we need to verify that the parent dir was
1904 	 * not deleted and then re-created, if it was then we have no overwrite
1905 	 * and we can just unlink this entry.
1906 	 */
1907 	if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) {
1908 		ret = get_inode_gen(sctx->parent_root, dir, &gen);
1909 		if (ret < 0 && ret != -ENOENT)
1910 			goto out;
1911 		if (ret) {
1912 			ret = 0;
1913 			goto out;
1914 		}
1915 		if (gen != dir_gen)
1916 			goto out;
1917 	}
1918 
1919 	ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
1920 				    &other_inode);
1921 	if (ret < 0 && ret != -ENOENT)
1922 		goto out;
1923 	if (ret) {
1924 		ret = 0;
1925 		goto out;
1926 	}
1927 
1928 	/*
1929 	 * Check if the overwritten ref was already processed. If yes, the ref
1930 	 * was already unlinked/moved, so we can safely assume that we will not
1931 	 * overwrite anything at this point in time.
1932 	 */
1933 	if (other_inode > sctx->send_progress ||
1934 	    is_waiting_for_move(sctx, other_inode)) {
1935 		ret = get_inode_info(sctx->parent_root, other_inode, &info);
1936 		if (ret < 0)
1937 			goto out;
1938 
1939 		ret = 1;
1940 		*who_ino = other_inode;
1941 		*who_gen = info.gen;
1942 		*who_mode = info.mode;
1943 	} else {
1944 		ret = 0;
1945 	}
1946 
1947 out:
1948 	return ret;
1949 }
1950 
1951 /*
1952  * Checks if the ref was overwritten by an already processed inode. This is
1953  * used by __get_cur_name_and_parent to find out if the ref was orphanized and
1954  * thus the orphan name needs be used.
1955  * process_recorded_refs also uses it to avoid unlinking of refs that were
1956  * overwritten.
1957  */
did_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,u64 ino,u64 ino_gen,const char * name,int name_len)1958 static int did_overwrite_ref(struct send_ctx *sctx,
1959 			    u64 dir, u64 dir_gen,
1960 			    u64 ino, u64 ino_gen,
1961 			    const char *name, int name_len)
1962 {
1963 	int ret = 0;
1964 	u64 gen;
1965 	u64 ow_inode;
1966 
1967 	if (!sctx->parent_root)
1968 		goto out;
1969 
1970 	ret = is_inode_existent(sctx, dir, dir_gen);
1971 	if (ret <= 0)
1972 		goto out;
1973 
1974 	if (dir != BTRFS_FIRST_FREE_OBJECTID) {
1975 		ret = get_inode_gen(sctx->send_root, dir, &gen);
1976 		if (ret < 0 && ret != -ENOENT)
1977 			goto out;
1978 		if (ret) {
1979 			ret = 0;
1980 			goto out;
1981 		}
1982 		if (gen != dir_gen)
1983 			goto out;
1984 	}
1985 
1986 	/* check if the ref was overwritten by another ref */
1987 	ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
1988 				    &ow_inode);
1989 	if (ret < 0 && ret != -ENOENT)
1990 		goto out;
1991 	if (ret) {
1992 		/* was never and will never be overwritten */
1993 		ret = 0;
1994 		goto out;
1995 	}
1996 
1997 	ret = get_inode_gen(sctx->send_root, ow_inode, &gen);
1998 	if (ret < 0)
1999 		goto out;
2000 
2001 	if (ow_inode == ino && gen == ino_gen) {
2002 		ret = 0;
2003 		goto out;
2004 	}
2005 
2006 	/*
2007 	 * We know that it is or will be overwritten. Check this now.
2008 	 * The current inode being processed might have been the one that caused
2009 	 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2010 	 * the current inode being processed.
2011 	 */
2012 	if ((ow_inode < sctx->send_progress) ||
2013 	    (ino != sctx->cur_ino && ow_inode == sctx->cur_ino &&
2014 	     gen == sctx->cur_inode_gen))
2015 		ret = 1;
2016 	else
2017 		ret = 0;
2018 
2019 out:
2020 	return ret;
2021 }
2022 
2023 /*
2024  * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2025  * that got overwritten. This is used by process_recorded_refs to determine
2026  * if it has to use the path as returned by get_cur_path or the orphan name.
2027  */
did_overwrite_first_ref(struct send_ctx * sctx,u64 ino,u64 gen)2028 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2029 {
2030 	int ret = 0;
2031 	struct fs_path *name = NULL;
2032 	u64 dir;
2033 	u64 dir_gen;
2034 
2035 	if (!sctx->parent_root)
2036 		goto out;
2037 
2038 	name = fs_path_alloc();
2039 	if (!name)
2040 		return -ENOMEM;
2041 
2042 	ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2043 	if (ret < 0)
2044 		goto out;
2045 
2046 	ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2047 			name->start, fs_path_len(name));
2048 
2049 out:
2050 	fs_path_free(name);
2051 	return ret;
2052 }
2053 
2054 /*
2055  * Insert a name cache entry. On 32bit kernels the radix tree index is 32bit,
2056  * so we need to do some special handling in case we have clashes. This function
2057  * takes care of this with the help of name_cache_entry::radix_list.
2058  * In case of error, nce is kfreed.
2059  */
name_cache_insert(struct send_ctx * sctx,struct name_cache_entry * nce)2060 static int name_cache_insert(struct send_ctx *sctx,
2061 			     struct name_cache_entry *nce)
2062 {
2063 	int ret = 0;
2064 	struct list_head *nce_head;
2065 
2066 	nce_head = radix_tree_lookup(&sctx->name_cache,
2067 			(unsigned long)nce->ino);
2068 	if (!nce_head) {
2069 		nce_head = kmalloc(sizeof(*nce_head), GFP_KERNEL);
2070 		if (!nce_head) {
2071 			kfree(nce);
2072 			return -ENOMEM;
2073 		}
2074 		INIT_LIST_HEAD(nce_head);
2075 
2076 		ret = radix_tree_insert(&sctx->name_cache, nce->ino, nce_head);
2077 		if (ret < 0) {
2078 			kfree(nce_head);
2079 			kfree(nce);
2080 			return ret;
2081 		}
2082 	}
2083 	list_add_tail(&nce->radix_list, nce_head);
2084 	list_add_tail(&nce->list, &sctx->name_cache_list);
2085 	sctx->name_cache_size++;
2086 
2087 	return ret;
2088 }
2089 
name_cache_delete(struct send_ctx * sctx,struct name_cache_entry * nce)2090 static void name_cache_delete(struct send_ctx *sctx,
2091 			      struct name_cache_entry *nce)
2092 {
2093 	struct list_head *nce_head;
2094 
2095 	nce_head = radix_tree_lookup(&sctx->name_cache,
2096 			(unsigned long)nce->ino);
2097 	if (!nce_head) {
2098 		btrfs_err(sctx->send_root->fs_info,
2099 	      "name_cache_delete lookup failed ino %llu cache size %d, leaking memory",
2100 			nce->ino, sctx->name_cache_size);
2101 	}
2102 
2103 	list_del(&nce->radix_list);
2104 	list_del(&nce->list);
2105 	sctx->name_cache_size--;
2106 
2107 	/*
2108 	 * We may not get to the final release of nce_head if the lookup fails
2109 	 */
2110 	if (nce_head && list_empty(nce_head)) {
2111 		radix_tree_delete(&sctx->name_cache, (unsigned long)nce->ino);
2112 		kfree(nce_head);
2113 	}
2114 }
2115 
name_cache_search(struct send_ctx * sctx,u64 ino,u64 gen)2116 static struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2117 						    u64 ino, u64 gen)
2118 {
2119 	struct list_head *nce_head;
2120 	struct name_cache_entry *cur;
2121 
2122 	nce_head = radix_tree_lookup(&sctx->name_cache, (unsigned long)ino);
2123 	if (!nce_head)
2124 		return NULL;
2125 
2126 	list_for_each_entry(cur, nce_head, radix_list) {
2127 		if (cur->ino == ino && cur->gen == gen)
2128 			return cur;
2129 	}
2130 	return NULL;
2131 }
2132 
2133 /*
2134  * Remove some entries from the beginning of name_cache_list.
2135  */
name_cache_clean_unused(struct send_ctx * sctx)2136 static void name_cache_clean_unused(struct send_ctx *sctx)
2137 {
2138 	struct name_cache_entry *nce;
2139 
2140 	if (sctx->name_cache_size < SEND_CTX_NAME_CACHE_CLEAN_SIZE)
2141 		return;
2142 
2143 	while (sctx->name_cache_size > SEND_CTX_MAX_NAME_CACHE_SIZE) {
2144 		nce = list_entry(sctx->name_cache_list.next,
2145 				struct name_cache_entry, list);
2146 		name_cache_delete(sctx, nce);
2147 		kfree(nce);
2148 	}
2149 }
2150 
name_cache_free(struct send_ctx * sctx)2151 static void name_cache_free(struct send_ctx *sctx)
2152 {
2153 	struct name_cache_entry *nce;
2154 
2155 	while (!list_empty(&sctx->name_cache_list)) {
2156 		nce = list_entry(sctx->name_cache_list.next,
2157 				struct name_cache_entry, list);
2158 		name_cache_delete(sctx, nce);
2159 		kfree(nce);
2160 	}
2161 }
2162 
2163 /*
2164  * Used by get_cur_path for each ref up to the root.
2165  * Returns 0 if it succeeded.
2166  * Returns 1 if the inode is not existent or got overwritten. In that case, the
2167  * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2168  * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2169  * Returns <0 in case of error.
2170  */
__get_cur_name_and_parent(struct send_ctx * sctx,u64 ino,u64 gen,u64 * parent_ino,u64 * parent_gen,struct fs_path * dest)2171 static int __get_cur_name_and_parent(struct send_ctx *sctx,
2172 				     u64 ino, u64 gen,
2173 				     u64 *parent_ino,
2174 				     u64 *parent_gen,
2175 				     struct fs_path *dest)
2176 {
2177 	int ret;
2178 	int nce_ret;
2179 	struct name_cache_entry *nce = NULL;
2180 
2181 	/*
2182 	 * First check if we already did a call to this function with the same
2183 	 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2184 	 * return the cached result.
2185 	 */
2186 	nce = name_cache_search(sctx, ino, gen);
2187 	if (nce) {
2188 		if (ino < sctx->send_progress && nce->need_later_update) {
2189 			name_cache_delete(sctx, nce);
2190 			kfree(nce);
2191 			nce = NULL;
2192 		} else {
2193 			/*
2194 			 * Removes the entry from the list and adds it back to
2195 			 * the end.  This marks the entry as recently used so
2196 			 * that name_cache_clean_unused does not remove it.
2197 			 */
2198 			list_move_tail(&nce->list, &sctx->name_cache_list);
2199 
2200 			*parent_ino = nce->parent_ino;
2201 			*parent_gen = nce->parent_gen;
2202 			ret = fs_path_add(dest, nce->name, nce->name_len);
2203 			if (ret < 0)
2204 				goto out;
2205 			ret = nce->ret;
2206 			goto out;
2207 		}
2208 	}
2209 
2210 	/*
2211 	 * If the inode is not existent yet, add the orphan name and return 1.
2212 	 * This should only happen for the parent dir that we determine in
2213 	 * record_new_ref_if_needed().
2214 	 */
2215 	ret = is_inode_existent(sctx, ino, gen);
2216 	if (ret < 0)
2217 		goto out;
2218 
2219 	if (!ret) {
2220 		ret = gen_unique_name(sctx, ino, gen, dest);
2221 		if (ret < 0)
2222 			goto out;
2223 		ret = 1;
2224 		goto out_cache;
2225 	}
2226 
2227 	/*
2228 	 * Depending on whether the inode was already processed or not, use
2229 	 * send_root or parent_root for ref lookup.
2230 	 */
2231 	if (ino < sctx->send_progress)
2232 		ret = get_first_ref(sctx->send_root, ino,
2233 				    parent_ino, parent_gen, dest);
2234 	else
2235 		ret = get_first_ref(sctx->parent_root, ino,
2236 				    parent_ino, parent_gen, dest);
2237 	if (ret < 0)
2238 		goto out;
2239 
2240 	/*
2241 	 * Check if the ref was overwritten by an inode's ref that was processed
2242 	 * earlier. If yes, treat as orphan and return 1.
2243 	 */
2244 	ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2245 			dest->start, dest->end - dest->start);
2246 	if (ret < 0)
2247 		goto out;
2248 	if (ret) {
2249 		fs_path_reset(dest);
2250 		ret = gen_unique_name(sctx, ino, gen, dest);
2251 		if (ret < 0)
2252 			goto out;
2253 		ret = 1;
2254 	}
2255 
2256 out_cache:
2257 	/*
2258 	 * Store the result of the lookup in the name cache.
2259 	 */
2260 	nce = kmalloc(sizeof(*nce) + fs_path_len(dest) + 1, GFP_KERNEL);
2261 	if (!nce) {
2262 		ret = -ENOMEM;
2263 		goto out;
2264 	}
2265 
2266 	nce->ino = ino;
2267 	nce->gen = gen;
2268 	nce->parent_ino = *parent_ino;
2269 	nce->parent_gen = *parent_gen;
2270 	nce->name_len = fs_path_len(dest);
2271 	nce->ret = ret;
2272 	strcpy(nce->name, dest->start);
2273 
2274 	if (ino < sctx->send_progress)
2275 		nce->need_later_update = 0;
2276 	else
2277 		nce->need_later_update = 1;
2278 
2279 	nce_ret = name_cache_insert(sctx, nce);
2280 	if (nce_ret < 0)
2281 		ret = nce_ret;
2282 	name_cache_clean_unused(sctx);
2283 
2284 out:
2285 	return ret;
2286 }
2287 
2288 /*
2289  * Magic happens here. This function returns the first ref to an inode as it
2290  * would look like while receiving the stream at this point in time.
2291  * We walk the path up to the root. For every inode in between, we check if it
2292  * was already processed/sent. If yes, we continue with the parent as found
2293  * in send_root. If not, we continue with the parent as found in parent_root.
2294  * If we encounter an inode that was deleted at this point in time, we use the
2295  * inodes "orphan" name instead of the real name and stop. Same with new inodes
2296  * that were not created yet and overwritten inodes/refs.
2297  *
2298  * When do we have orphan inodes:
2299  * 1. When an inode is freshly created and thus no valid refs are available yet
2300  * 2. When a directory lost all it's refs (deleted) but still has dir items
2301  *    inside which were not processed yet (pending for move/delete). If anyone
2302  *    tried to get the path to the dir items, it would get a path inside that
2303  *    orphan directory.
2304  * 3. When an inode is moved around or gets new links, it may overwrite the ref
2305  *    of an unprocessed inode. If in that case the first ref would be
2306  *    overwritten, the overwritten inode gets "orphanized". Later when we
2307  *    process this overwritten inode, it is restored at a new place by moving
2308  *    the orphan inode.
2309  *
2310  * sctx->send_progress tells this function at which point in time receiving
2311  * would be.
2312  */
get_cur_path(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)2313 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2314 			struct fs_path *dest)
2315 {
2316 	int ret = 0;
2317 	struct fs_path *name = NULL;
2318 	u64 parent_inode = 0;
2319 	u64 parent_gen = 0;
2320 	int stop = 0;
2321 
2322 	name = fs_path_alloc();
2323 	if (!name) {
2324 		ret = -ENOMEM;
2325 		goto out;
2326 	}
2327 
2328 	dest->reversed = 1;
2329 	fs_path_reset(dest);
2330 
2331 	while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2332 		struct waiting_dir_move *wdm;
2333 
2334 		fs_path_reset(name);
2335 
2336 		if (is_waiting_for_rm(sctx, ino, gen)) {
2337 			ret = gen_unique_name(sctx, ino, gen, name);
2338 			if (ret < 0)
2339 				goto out;
2340 			ret = fs_path_add_path(dest, name);
2341 			break;
2342 		}
2343 
2344 		wdm = get_waiting_dir_move(sctx, ino);
2345 		if (wdm && wdm->orphanized) {
2346 			ret = gen_unique_name(sctx, ino, gen, name);
2347 			stop = 1;
2348 		} else if (wdm) {
2349 			ret = get_first_ref(sctx->parent_root, ino,
2350 					    &parent_inode, &parent_gen, name);
2351 		} else {
2352 			ret = __get_cur_name_and_parent(sctx, ino, gen,
2353 							&parent_inode,
2354 							&parent_gen, name);
2355 			if (ret)
2356 				stop = 1;
2357 		}
2358 
2359 		if (ret < 0)
2360 			goto out;
2361 
2362 		ret = fs_path_add_path(dest, name);
2363 		if (ret < 0)
2364 			goto out;
2365 
2366 		ino = parent_inode;
2367 		gen = parent_gen;
2368 	}
2369 
2370 out:
2371 	fs_path_free(name);
2372 	if (!ret)
2373 		fs_path_unreverse(dest);
2374 	return ret;
2375 }
2376 
2377 /*
2378  * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2379  */
send_subvol_begin(struct send_ctx * sctx)2380 static int send_subvol_begin(struct send_ctx *sctx)
2381 {
2382 	int ret;
2383 	struct btrfs_root *send_root = sctx->send_root;
2384 	struct btrfs_root *parent_root = sctx->parent_root;
2385 	struct btrfs_path *path;
2386 	struct btrfs_key key;
2387 	struct btrfs_root_ref *ref;
2388 	struct extent_buffer *leaf;
2389 	char *name = NULL;
2390 	int namelen;
2391 
2392 	path = btrfs_alloc_path();
2393 	if (!path)
2394 		return -ENOMEM;
2395 
2396 	name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2397 	if (!name) {
2398 		btrfs_free_path(path);
2399 		return -ENOMEM;
2400 	}
2401 
2402 	key.objectid = send_root->root_key.objectid;
2403 	key.type = BTRFS_ROOT_BACKREF_KEY;
2404 	key.offset = 0;
2405 
2406 	ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2407 				&key, path, 1, 0);
2408 	if (ret < 0)
2409 		goto out;
2410 	if (ret) {
2411 		ret = -ENOENT;
2412 		goto out;
2413 	}
2414 
2415 	leaf = path->nodes[0];
2416 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2417 	if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2418 	    key.objectid != send_root->root_key.objectid) {
2419 		ret = -ENOENT;
2420 		goto out;
2421 	}
2422 	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2423 	namelen = btrfs_root_ref_name_len(leaf, ref);
2424 	read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2425 	btrfs_release_path(path);
2426 
2427 	if (parent_root) {
2428 		ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2429 		if (ret < 0)
2430 			goto out;
2431 	} else {
2432 		ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2433 		if (ret < 0)
2434 			goto out;
2435 	}
2436 
2437 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2438 
2439 	if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2440 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2441 			    sctx->send_root->root_item.received_uuid);
2442 	else
2443 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2444 			    sctx->send_root->root_item.uuid);
2445 
2446 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2447 		    btrfs_root_ctransid(&sctx->send_root->root_item));
2448 	if (parent_root) {
2449 		if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2450 			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2451 				     parent_root->root_item.received_uuid);
2452 		else
2453 			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2454 				     parent_root->root_item.uuid);
2455 		TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2456 			    btrfs_root_ctransid(&sctx->parent_root->root_item));
2457 	}
2458 
2459 	ret = send_cmd(sctx);
2460 
2461 tlv_put_failure:
2462 out:
2463 	btrfs_free_path(path);
2464 	kfree(name);
2465 	return ret;
2466 }
2467 
send_truncate(struct send_ctx * sctx,u64 ino,u64 gen,u64 size)2468 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2469 {
2470 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2471 	int ret = 0;
2472 	struct fs_path *p;
2473 
2474 	btrfs_debug(fs_info, "send_truncate %llu size=%llu", ino, size);
2475 
2476 	p = fs_path_alloc();
2477 	if (!p)
2478 		return -ENOMEM;
2479 
2480 	ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2481 	if (ret < 0)
2482 		goto out;
2483 
2484 	ret = get_cur_path(sctx, ino, gen, p);
2485 	if (ret < 0)
2486 		goto out;
2487 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2488 	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2489 
2490 	ret = send_cmd(sctx);
2491 
2492 tlv_put_failure:
2493 out:
2494 	fs_path_free(p);
2495 	return ret;
2496 }
2497 
send_chmod(struct send_ctx * sctx,u64 ino,u64 gen,u64 mode)2498 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2499 {
2500 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2501 	int ret = 0;
2502 	struct fs_path *p;
2503 
2504 	btrfs_debug(fs_info, "send_chmod %llu mode=%llu", ino, mode);
2505 
2506 	p = fs_path_alloc();
2507 	if (!p)
2508 		return -ENOMEM;
2509 
2510 	ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2511 	if (ret < 0)
2512 		goto out;
2513 
2514 	ret = get_cur_path(sctx, ino, gen, p);
2515 	if (ret < 0)
2516 		goto out;
2517 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2518 	TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2519 
2520 	ret = send_cmd(sctx);
2521 
2522 tlv_put_failure:
2523 out:
2524 	fs_path_free(p);
2525 	return ret;
2526 }
2527 
send_fileattr(struct send_ctx * sctx,u64 ino,u64 gen,u64 fileattr)2528 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2529 {
2530 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2531 	int ret = 0;
2532 	struct fs_path *p;
2533 
2534 	if (sctx->proto < 2)
2535 		return 0;
2536 
2537 	btrfs_debug(fs_info, "send_fileattr %llu fileattr=%llu", ino, fileattr);
2538 
2539 	p = fs_path_alloc();
2540 	if (!p)
2541 		return -ENOMEM;
2542 
2543 	ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2544 	if (ret < 0)
2545 		goto out;
2546 
2547 	ret = get_cur_path(sctx, ino, gen, p);
2548 	if (ret < 0)
2549 		goto out;
2550 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2551 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2552 
2553 	ret = send_cmd(sctx);
2554 
2555 tlv_put_failure:
2556 out:
2557 	fs_path_free(p);
2558 	return ret;
2559 }
2560 
send_chown(struct send_ctx * sctx,u64 ino,u64 gen,u64 uid,u64 gid)2561 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2562 {
2563 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2564 	int ret = 0;
2565 	struct fs_path *p;
2566 
2567 	btrfs_debug(fs_info, "send_chown %llu uid=%llu, gid=%llu",
2568 		    ino, uid, gid);
2569 
2570 	p = fs_path_alloc();
2571 	if (!p)
2572 		return -ENOMEM;
2573 
2574 	ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2575 	if (ret < 0)
2576 		goto out;
2577 
2578 	ret = get_cur_path(sctx, ino, gen, p);
2579 	if (ret < 0)
2580 		goto out;
2581 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2582 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2583 	TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2584 
2585 	ret = send_cmd(sctx);
2586 
2587 tlv_put_failure:
2588 out:
2589 	fs_path_free(p);
2590 	return ret;
2591 }
2592 
send_utimes(struct send_ctx * sctx,u64 ino,u64 gen)2593 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2594 {
2595 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2596 	int ret = 0;
2597 	struct fs_path *p = NULL;
2598 	struct btrfs_inode_item *ii;
2599 	struct btrfs_path *path = NULL;
2600 	struct extent_buffer *eb;
2601 	struct btrfs_key key;
2602 	int slot;
2603 
2604 	btrfs_debug(fs_info, "send_utimes %llu", ino);
2605 
2606 	p = fs_path_alloc();
2607 	if (!p)
2608 		return -ENOMEM;
2609 
2610 	path = alloc_path_for_send();
2611 	if (!path) {
2612 		ret = -ENOMEM;
2613 		goto out;
2614 	}
2615 
2616 	key.objectid = ino;
2617 	key.type = BTRFS_INODE_ITEM_KEY;
2618 	key.offset = 0;
2619 	ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2620 	if (ret > 0)
2621 		ret = -ENOENT;
2622 	if (ret < 0)
2623 		goto out;
2624 
2625 	eb = path->nodes[0];
2626 	slot = path->slots[0];
2627 	ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2628 
2629 	ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2630 	if (ret < 0)
2631 		goto out;
2632 
2633 	ret = get_cur_path(sctx, ino, gen, p);
2634 	if (ret < 0)
2635 		goto out;
2636 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2637 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2638 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2639 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2640 	if (sctx->proto >= 2)
2641 		TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2642 
2643 	ret = send_cmd(sctx);
2644 
2645 tlv_put_failure:
2646 out:
2647 	fs_path_free(p);
2648 	btrfs_free_path(path);
2649 	return ret;
2650 }
2651 
2652 /*
2653  * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2654  * a valid path yet because we did not process the refs yet. So, the inode
2655  * is created as orphan.
2656  */
send_create_inode(struct send_ctx * sctx,u64 ino)2657 static int send_create_inode(struct send_ctx *sctx, u64 ino)
2658 {
2659 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
2660 	int ret = 0;
2661 	struct fs_path *p;
2662 	int cmd;
2663 	struct btrfs_inode_info info;
2664 	u64 gen;
2665 	u64 mode;
2666 	u64 rdev;
2667 
2668 	btrfs_debug(fs_info, "send_create_inode %llu", ino);
2669 
2670 	p = fs_path_alloc();
2671 	if (!p)
2672 		return -ENOMEM;
2673 
2674 	if (ino != sctx->cur_ino) {
2675 		ret = get_inode_info(sctx->send_root, ino, &info);
2676 		if (ret < 0)
2677 			goto out;
2678 		gen = info.gen;
2679 		mode = info.mode;
2680 		rdev = info.rdev;
2681 	} else {
2682 		gen = sctx->cur_inode_gen;
2683 		mode = sctx->cur_inode_mode;
2684 		rdev = sctx->cur_inode_rdev;
2685 	}
2686 
2687 	if (S_ISREG(mode)) {
2688 		cmd = BTRFS_SEND_C_MKFILE;
2689 	} else if (S_ISDIR(mode)) {
2690 		cmd = BTRFS_SEND_C_MKDIR;
2691 	} else if (S_ISLNK(mode)) {
2692 		cmd = BTRFS_SEND_C_SYMLINK;
2693 	} else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2694 		cmd = BTRFS_SEND_C_MKNOD;
2695 	} else if (S_ISFIFO(mode)) {
2696 		cmd = BTRFS_SEND_C_MKFIFO;
2697 	} else if (S_ISSOCK(mode)) {
2698 		cmd = BTRFS_SEND_C_MKSOCK;
2699 	} else {
2700 		btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2701 				(int)(mode & S_IFMT));
2702 		ret = -EOPNOTSUPP;
2703 		goto out;
2704 	}
2705 
2706 	ret = begin_cmd(sctx, cmd);
2707 	if (ret < 0)
2708 		goto out;
2709 
2710 	ret = gen_unique_name(sctx, ino, gen, p);
2711 	if (ret < 0)
2712 		goto out;
2713 
2714 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2715 	TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2716 
2717 	if (S_ISLNK(mode)) {
2718 		fs_path_reset(p);
2719 		ret = read_symlink(sctx->send_root, ino, p);
2720 		if (ret < 0)
2721 			goto out;
2722 		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2723 	} else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2724 		   S_ISFIFO(mode) || S_ISSOCK(mode)) {
2725 		TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2726 		TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2727 	}
2728 
2729 	ret = send_cmd(sctx);
2730 	if (ret < 0)
2731 		goto out;
2732 
2733 
2734 tlv_put_failure:
2735 out:
2736 	fs_path_free(p);
2737 	return ret;
2738 }
2739 
2740 /*
2741  * We need some special handling for inodes that get processed before the parent
2742  * directory got created. See process_recorded_refs for details.
2743  * This function does the check if we already created the dir out of order.
2744  */
did_create_dir(struct send_ctx * sctx,u64 dir)2745 static int did_create_dir(struct send_ctx *sctx, u64 dir)
2746 {
2747 	int ret = 0;
2748 	int iter_ret = 0;
2749 	struct btrfs_path *path = NULL;
2750 	struct btrfs_key key;
2751 	struct btrfs_key found_key;
2752 	struct btrfs_key di_key;
2753 	struct btrfs_dir_item *di;
2754 
2755 	path = alloc_path_for_send();
2756 	if (!path)
2757 		return -ENOMEM;
2758 
2759 	key.objectid = dir;
2760 	key.type = BTRFS_DIR_INDEX_KEY;
2761 	key.offset = 0;
2762 
2763 	btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2764 		struct extent_buffer *eb = path->nodes[0];
2765 
2766 		if (found_key.objectid != key.objectid ||
2767 		    found_key.type != key.type) {
2768 			ret = 0;
2769 			break;
2770 		}
2771 
2772 		di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2773 		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2774 
2775 		if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2776 		    di_key.objectid < sctx->send_progress) {
2777 			ret = 1;
2778 			break;
2779 		}
2780 	}
2781 	/* Catch error found during iteration */
2782 	if (iter_ret < 0)
2783 		ret = iter_ret;
2784 
2785 	btrfs_free_path(path);
2786 	return ret;
2787 }
2788 
2789 /*
2790  * Only creates the inode if it is:
2791  * 1. Not a directory
2792  * 2. Or a directory which was not created already due to out of order
2793  *    directories. See did_create_dir and process_recorded_refs for details.
2794  */
send_create_inode_if_needed(struct send_ctx * sctx)2795 static int send_create_inode_if_needed(struct send_ctx *sctx)
2796 {
2797 	int ret;
2798 
2799 	if (S_ISDIR(sctx->cur_inode_mode)) {
2800 		ret = did_create_dir(sctx, sctx->cur_ino);
2801 		if (ret < 0)
2802 			return ret;
2803 		else if (ret > 0)
2804 			return 0;
2805 	}
2806 
2807 	return send_create_inode(sctx, sctx->cur_ino);
2808 }
2809 
2810 struct recorded_ref {
2811 	struct list_head list;
2812 	char *name;
2813 	struct fs_path *full_path;
2814 	u64 dir;
2815 	u64 dir_gen;
2816 	int name_len;
2817 	struct rb_node node;
2818 	struct rb_root *root;
2819 };
2820 
recorded_ref_alloc(void)2821 static struct recorded_ref *recorded_ref_alloc(void)
2822 {
2823 	struct recorded_ref *ref;
2824 
2825 	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
2826 	if (!ref)
2827 		return NULL;
2828 	RB_CLEAR_NODE(&ref->node);
2829 	INIT_LIST_HEAD(&ref->list);
2830 	return ref;
2831 }
2832 
recorded_ref_free(struct recorded_ref * ref)2833 static void recorded_ref_free(struct recorded_ref *ref)
2834 {
2835 	if (!ref)
2836 		return;
2837 	if (!RB_EMPTY_NODE(&ref->node))
2838 		rb_erase(&ref->node, ref->root);
2839 	list_del(&ref->list);
2840 	fs_path_free(ref->full_path);
2841 	kfree(ref);
2842 }
2843 
set_ref_path(struct recorded_ref * ref,struct fs_path * path)2844 static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
2845 {
2846 	ref->full_path = path;
2847 	ref->name = (char *)kbasename(ref->full_path->start);
2848 	ref->name_len = ref->full_path->end - ref->name;
2849 }
2850 
dup_ref(struct recorded_ref * ref,struct list_head * list)2851 static int dup_ref(struct recorded_ref *ref, struct list_head *list)
2852 {
2853 	struct recorded_ref *new;
2854 
2855 	new = recorded_ref_alloc();
2856 	if (!new)
2857 		return -ENOMEM;
2858 
2859 	new->dir = ref->dir;
2860 	new->dir_gen = ref->dir_gen;
2861 	list_add_tail(&new->list, list);
2862 	return 0;
2863 }
2864 
__free_recorded_refs(struct list_head * head)2865 static void __free_recorded_refs(struct list_head *head)
2866 {
2867 	struct recorded_ref *cur;
2868 
2869 	while (!list_empty(head)) {
2870 		cur = list_entry(head->next, struct recorded_ref, list);
2871 		recorded_ref_free(cur);
2872 	}
2873 }
2874 
free_recorded_refs(struct send_ctx * sctx)2875 static void free_recorded_refs(struct send_ctx *sctx)
2876 {
2877 	__free_recorded_refs(&sctx->new_refs);
2878 	__free_recorded_refs(&sctx->deleted_refs);
2879 }
2880 
2881 /*
2882  * Renames/moves a file/dir to its orphan name. Used when the first
2883  * ref of an unprocessed inode gets overwritten and for all non empty
2884  * directories.
2885  */
orphanize_inode(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * path)2886 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
2887 			  struct fs_path *path)
2888 {
2889 	int ret;
2890 	struct fs_path *orphan;
2891 
2892 	orphan = fs_path_alloc();
2893 	if (!orphan)
2894 		return -ENOMEM;
2895 
2896 	ret = gen_unique_name(sctx, ino, gen, orphan);
2897 	if (ret < 0)
2898 		goto out;
2899 
2900 	ret = send_rename(sctx, path, orphan);
2901 
2902 out:
2903 	fs_path_free(orphan);
2904 	return ret;
2905 }
2906 
add_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 dir_gen)2907 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
2908 						   u64 dir_ino, u64 dir_gen)
2909 {
2910 	struct rb_node **p = &sctx->orphan_dirs.rb_node;
2911 	struct rb_node *parent = NULL;
2912 	struct orphan_dir_info *entry, *odi;
2913 
2914 	while (*p) {
2915 		parent = *p;
2916 		entry = rb_entry(parent, struct orphan_dir_info, node);
2917 		if (dir_ino < entry->ino)
2918 			p = &(*p)->rb_left;
2919 		else if (dir_ino > entry->ino)
2920 			p = &(*p)->rb_right;
2921 		else if (dir_gen < entry->gen)
2922 			p = &(*p)->rb_left;
2923 		else if (dir_gen > entry->gen)
2924 			p = &(*p)->rb_right;
2925 		else
2926 			return entry;
2927 	}
2928 
2929 	odi = kmalloc(sizeof(*odi), GFP_KERNEL);
2930 	if (!odi)
2931 		return ERR_PTR(-ENOMEM);
2932 	odi->ino = dir_ino;
2933 	odi->gen = dir_gen;
2934 	odi->last_dir_index_offset = 0;
2935 
2936 	rb_link_node(&odi->node, parent, p);
2937 	rb_insert_color(&odi->node, &sctx->orphan_dirs);
2938 	return odi;
2939 }
2940 
get_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 gen)2941 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
2942 						   u64 dir_ino, u64 gen)
2943 {
2944 	struct rb_node *n = sctx->orphan_dirs.rb_node;
2945 	struct orphan_dir_info *entry;
2946 
2947 	while (n) {
2948 		entry = rb_entry(n, struct orphan_dir_info, node);
2949 		if (dir_ino < entry->ino)
2950 			n = n->rb_left;
2951 		else if (dir_ino > entry->ino)
2952 			n = n->rb_right;
2953 		else if (gen < entry->gen)
2954 			n = n->rb_left;
2955 		else if (gen > entry->gen)
2956 			n = n->rb_right;
2957 		else
2958 			return entry;
2959 	}
2960 	return NULL;
2961 }
2962 
is_waiting_for_rm(struct send_ctx * sctx,u64 dir_ino,u64 gen)2963 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
2964 {
2965 	struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
2966 
2967 	return odi != NULL;
2968 }
2969 
free_orphan_dir_info(struct send_ctx * sctx,struct orphan_dir_info * odi)2970 static void free_orphan_dir_info(struct send_ctx *sctx,
2971 				 struct orphan_dir_info *odi)
2972 {
2973 	if (!odi)
2974 		return;
2975 	rb_erase(&odi->node, &sctx->orphan_dirs);
2976 	kfree(odi);
2977 }
2978 
2979 /*
2980  * Returns 1 if a directory can be removed at this point in time.
2981  * We check this by iterating all dir items and checking if the inode behind
2982  * the dir item was already processed.
2983  */
can_rmdir(struct send_ctx * sctx,u64 dir,u64 dir_gen,u64 send_progress)2984 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2985 		     u64 send_progress)
2986 {
2987 	int ret = 0;
2988 	int iter_ret = 0;
2989 	struct btrfs_root *root = sctx->parent_root;
2990 	struct btrfs_path *path;
2991 	struct btrfs_key key;
2992 	struct btrfs_key found_key;
2993 	struct btrfs_key loc;
2994 	struct btrfs_dir_item *di;
2995 	struct orphan_dir_info *odi = NULL;
2996 
2997 	/*
2998 	 * Don't try to rmdir the top/root subvolume dir.
2999 	 */
3000 	if (dir == BTRFS_FIRST_FREE_OBJECTID)
3001 		return 0;
3002 
3003 	path = alloc_path_for_send();
3004 	if (!path)
3005 		return -ENOMEM;
3006 
3007 	key.objectid = dir;
3008 	key.type = BTRFS_DIR_INDEX_KEY;
3009 	key.offset = 0;
3010 
3011 	odi = get_orphan_dir_info(sctx, dir, dir_gen);
3012 	if (odi)
3013 		key.offset = odi->last_dir_index_offset;
3014 
3015 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3016 		struct waiting_dir_move *dm;
3017 
3018 		if (found_key.objectid != key.objectid ||
3019 		    found_key.type != key.type)
3020 			break;
3021 
3022 		di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3023 				struct btrfs_dir_item);
3024 		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3025 
3026 		dm = get_waiting_dir_move(sctx, loc.objectid);
3027 		if (dm) {
3028 			odi = add_orphan_dir_info(sctx, dir, dir_gen);
3029 			if (IS_ERR(odi)) {
3030 				ret = PTR_ERR(odi);
3031 				goto out;
3032 			}
3033 			odi->gen = dir_gen;
3034 			odi->last_dir_index_offset = found_key.offset;
3035 			dm->rmdir_ino = dir;
3036 			dm->rmdir_gen = dir_gen;
3037 			ret = 0;
3038 			goto out;
3039 		}
3040 
3041 		if (loc.objectid > send_progress) {
3042 			odi = add_orphan_dir_info(sctx, dir, dir_gen);
3043 			if (IS_ERR(odi)) {
3044 				ret = PTR_ERR(odi);
3045 				goto out;
3046 			}
3047 			odi->gen = dir_gen;
3048 			odi->last_dir_index_offset = found_key.offset;
3049 			ret = 0;
3050 			goto out;
3051 		}
3052 	}
3053 	if (iter_ret < 0) {
3054 		ret = iter_ret;
3055 		goto out;
3056 	}
3057 	free_orphan_dir_info(sctx, odi);
3058 
3059 	ret = 1;
3060 
3061 out:
3062 	btrfs_free_path(path);
3063 	return ret;
3064 }
3065 
is_waiting_for_move(struct send_ctx * sctx,u64 ino)3066 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3067 {
3068 	struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3069 
3070 	return entry != NULL;
3071 }
3072 
add_waiting_dir_move(struct send_ctx * sctx,u64 ino,bool orphanized)3073 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3074 {
3075 	struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3076 	struct rb_node *parent = NULL;
3077 	struct waiting_dir_move *entry, *dm;
3078 
3079 	dm = kmalloc(sizeof(*dm), GFP_KERNEL);
3080 	if (!dm)
3081 		return -ENOMEM;
3082 	dm->ino = ino;
3083 	dm->rmdir_ino = 0;
3084 	dm->rmdir_gen = 0;
3085 	dm->orphanized = orphanized;
3086 
3087 	while (*p) {
3088 		parent = *p;
3089 		entry = rb_entry(parent, struct waiting_dir_move, node);
3090 		if (ino < entry->ino) {
3091 			p = &(*p)->rb_left;
3092 		} else if (ino > entry->ino) {
3093 			p = &(*p)->rb_right;
3094 		} else {
3095 			kfree(dm);
3096 			return -EEXIST;
3097 		}
3098 	}
3099 
3100 	rb_link_node(&dm->node, parent, p);
3101 	rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3102 	return 0;
3103 }
3104 
3105 static struct waiting_dir_move *
get_waiting_dir_move(struct send_ctx * sctx,u64 ino)3106 get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3107 {
3108 	struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3109 	struct waiting_dir_move *entry;
3110 
3111 	while (n) {
3112 		entry = rb_entry(n, struct waiting_dir_move, node);
3113 		if (ino < entry->ino)
3114 			n = n->rb_left;
3115 		else if (ino > entry->ino)
3116 			n = n->rb_right;
3117 		else
3118 			return entry;
3119 	}
3120 	return NULL;
3121 }
3122 
free_waiting_dir_move(struct send_ctx * sctx,struct waiting_dir_move * dm)3123 static void free_waiting_dir_move(struct send_ctx *sctx,
3124 				  struct waiting_dir_move *dm)
3125 {
3126 	if (!dm)
3127 		return;
3128 	rb_erase(&dm->node, &sctx->waiting_dir_moves);
3129 	kfree(dm);
3130 }
3131 
add_pending_dir_move(struct send_ctx * sctx,u64 ino,u64 ino_gen,u64 parent_ino,struct list_head * new_refs,struct list_head * deleted_refs,const bool is_orphan)3132 static int add_pending_dir_move(struct send_ctx *sctx,
3133 				u64 ino,
3134 				u64 ino_gen,
3135 				u64 parent_ino,
3136 				struct list_head *new_refs,
3137 				struct list_head *deleted_refs,
3138 				const bool is_orphan)
3139 {
3140 	struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3141 	struct rb_node *parent = NULL;
3142 	struct pending_dir_move *entry = NULL, *pm;
3143 	struct recorded_ref *cur;
3144 	int exists = 0;
3145 	int ret;
3146 
3147 	pm = kmalloc(sizeof(*pm), GFP_KERNEL);
3148 	if (!pm)
3149 		return -ENOMEM;
3150 	pm->parent_ino = parent_ino;
3151 	pm->ino = ino;
3152 	pm->gen = ino_gen;
3153 	INIT_LIST_HEAD(&pm->list);
3154 	INIT_LIST_HEAD(&pm->update_refs);
3155 	RB_CLEAR_NODE(&pm->node);
3156 
3157 	while (*p) {
3158 		parent = *p;
3159 		entry = rb_entry(parent, struct pending_dir_move, node);
3160 		if (parent_ino < entry->parent_ino) {
3161 			p = &(*p)->rb_left;
3162 		} else if (parent_ino > entry->parent_ino) {
3163 			p = &(*p)->rb_right;
3164 		} else {
3165 			exists = 1;
3166 			break;
3167 		}
3168 	}
3169 
3170 	list_for_each_entry(cur, deleted_refs, list) {
3171 		ret = dup_ref(cur, &pm->update_refs);
3172 		if (ret < 0)
3173 			goto out;
3174 	}
3175 	list_for_each_entry(cur, new_refs, list) {
3176 		ret = dup_ref(cur, &pm->update_refs);
3177 		if (ret < 0)
3178 			goto out;
3179 	}
3180 
3181 	ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3182 	if (ret)
3183 		goto out;
3184 
3185 	if (exists) {
3186 		list_add_tail(&pm->list, &entry->list);
3187 	} else {
3188 		rb_link_node(&pm->node, parent, p);
3189 		rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3190 	}
3191 	ret = 0;
3192 out:
3193 	if (ret) {
3194 		__free_recorded_refs(&pm->update_refs);
3195 		kfree(pm);
3196 	}
3197 	return ret;
3198 }
3199 
get_pending_dir_moves(struct send_ctx * sctx,u64 parent_ino)3200 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3201 						      u64 parent_ino)
3202 {
3203 	struct rb_node *n = sctx->pending_dir_moves.rb_node;
3204 	struct pending_dir_move *entry;
3205 
3206 	while (n) {
3207 		entry = rb_entry(n, struct pending_dir_move, node);
3208 		if (parent_ino < entry->parent_ino)
3209 			n = n->rb_left;
3210 		else if (parent_ino > entry->parent_ino)
3211 			n = n->rb_right;
3212 		else
3213 			return entry;
3214 	}
3215 	return NULL;
3216 }
3217 
path_loop(struct send_ctx * sctx,struct fs_path * name,u64 ino,u64 gen,u64 * ancestor_ino)3218 static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3219 		     u64 ino, u64 gen, u64 *ancestor_ino)
3220 {
3221 	int ret = 0;
3222 	u64 parent_inode = 0;
3223 	u64 parent_gen = 0;
3224 	u64 start_ino = ino;
3225 
3226 	*ancestor_ino = 0;
3227 	while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3228 		fs_path_reset(name);
3229 
3230 		if (is_waiting_for_rm(sctx, ino, gen))
3231 			break;
3232 		if (is_waiting_for_move(sctx, ino)) {
3233 			if (*ancestor_ino == 0)
3234 				*ancestor_ino = ino;
3235 			ret = get_first_ref(sctx->parent_root, ino,
3236 					    &parent_inode, &parent_gen, name);
3237 		} else {
3238 			ret = __get_cur_name_and_parent(sctx, ino, gen,
3239 							&parent_inode,
3240 							&parent_gen, name);
3241 			if (ret > 0) {
3242 				ret = 0;
3243 				break;
3244 			}
3245 		}
3246 		if (ret < 0)
3247 			break;
3248 		if (parent_inode == start_ino) {
3249 			ret = 1;
3250 			if (*ancestor_ino == 0)
3251 				*ancestor_ino = ino;
3252 			break;
3253 		}
3254 		ino = parent_inode;
3255 		gen = parent_gen;
3256 	}
3257 	return ret;
3258 }
3259 
apply_dir_move(struct send_ctx * sctx,struct pending_dir_move * pm)3260 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3261 {
3262 	struct fs_path *from_path = NULL;
3263 	struct fs_path *to_path = NULL;
3264 	struct fs_path *name = NULL;
3265 	u64 orig_progress = sctx->send_progress;
3266 	struct recorded_ref *cur;
3267 	u64 parent_ino, parent_gen;
3268 	struct waiting_dir_move *dm = NULL;
3269 	u64 rmdir_ino = 0;
3270 	u64 rmdir_gen;
3271 	u64 ancestor;
3272 	bool is_orphan;
3273 	int ret;
3274 
3275 	name = fs_path_alloc();
3276 	from_path = fs_path_alloc();
3277 	if (!name || !from_path) {
3278 		ret = -ENOMEM;
3279 		goto out;
3280 	}
3281 
3282 	dm = get_waiting_dir_move(sctx, pm->ino);
3283 	ASSERT(dm);
3284 	rmdir_ino = dm->rmdir_ino;
3285 	rmdir_gen = dm->rmdir_gen;
3286 	is_orphan = dm->orphanized;
3287 	free_waiting_dir_move(sctx, dm);
3288 
3289 	if (is_orphan) {
3290 		ret = gen_unique_name(sctx, pm->ino,
3291 				      pm->gen, from_path);
3292 	} else {
3293 		ret = get_first_ref(sctx->parent_root, pm->ino,
3294 				    &parent_ino, &parent_gen, name);
3295 		if (ret < 0)
3296 			goto out;
3297 		ret = get_cur_path(sctx, parent_ino, parent_gen,
3298 				   from_path);
3299 		if (ret < 0)
3300 			goto out;
3301 		ret = fs_path_add_path(from_path, name);
3302 	}
3303 	if (ret < 0)
3304 		goto out;
3305 
3306 	sctx->send_progress = sctx->cur_ino + 1;
3307 	ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3308 	if (ret < 0)
3309 		goto out;
3310 	if (ret) {
3311 		LIST_HEAD(deleted_refs);
3312 		ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3313 		ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3314 					   &pm->update_refs, &deleted_refs,
3315 					   is_orphan);
3316 		if (ret < 0)
3317 			goto out;
3318 		if (rmdir_ino) {
3319 			dm = get_waiting_dir_move(sctx, pm->ino);
3320 			ASSERT(dm);
3321 			dm->rmdir_ino = rmdir_ino;
3322 			dm->rmdir_gen = rmdir_gen;
3323 		}
3324 		goto out;
3325 	}
3326 	fs_path_reset(name);
3327 	to_path = name;
3328 	name = NULL;
3329 	ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3330 	if (ret < 0)
3331 		goto out;
3332 
3333 	ret = send_rename(sctx, from_path, to_path);
3334 	if (ret < 0)
3335 		goto out;
3336 
3337 	if (rmdir_ino) {
3338 		struct orphan_dir_info *odi;
3339 		u64 gen;
3340 
3341 		odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3342 		if (!odi) {
3343 			/* already deleted */
3344 			goto finish;
3345 		}
3346 		gen = odi->gen;
3347 
3348 		ret = can_rmdir(sctx, rmdir_ino, gen, sctx->cur_ino);
3349 		if (ret < 0)
3350 			goto out;
3351 		if (!ret)
3352 			goto finish;
3353 
3354 		name = fs_path_alloc();
3355 		if (!name) {
3356 			ret = -ENOMEM;
3357 			goto out;
3358 		}
3359 		ret = get_cur_path(sctx, rmdir_ino, gen, name);
3360 		if (ret < 0)
3361 			goto out;
3362 		ret = send_rmdir(sctx, name);
3363 		if (ret < 0)
3364 			goto out;
3365 	}
3366 
3367 finish:
3368 	ret = send_utimes(sctx, pm->ino, pm->gen);
3369 	if (ret < 0)
3370 		goto out;
3371 
3372 	/*
3373 	 * After rename/move, need to update the utimes of both new parent(s)
3374 	 * and old parent(s).
3375 	 */
3376 	list_for_each_entry(cur, &pm->update_refs, list) {
3377 		/*
3378 		 * The parent inode might have been deleted in the send snapshot
3379 		 */
3380 		ret = get_inode_info(sctx->send_root, cur->dir, NULL);
3381 		if (ret == -ENOENT) {
3382 			ret = 0;
3383 			continue;
3384 		}
3385 		if (ret < 0)
3386 			goto out;
3387 
3388 		ret = send_utimes(sctx, cur->dir, cur->dir_gen);
3389 		if (ret < 0)
3390 			goto out;
3391 	}
3392 
3393 out:
3394 	fs_path_free(name);
3395 	fs_path_free(from_path);
3396 	fs_path_free(to_path);
3397 	sctx->send_progress = orig_progress;
3398 
3399 	return ret;
3400 }
3401 
free_pending_move(struct send_ctx * sctx,struct pending_dir_move * m)3402 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3403 {
3404 	if (!list_empty(&m->list))
3405 		list_del(&m->list);
3406 	if (!RB_EMPTY_NODE(&m->node))
3407 		rb_erase(&m->node, &sctx->pending_dir_moves);
3408 	__free_recorded_refs(&m->update_refs);
3409 	kfree(m);
3410 }
3411 
tail_append_pending_moves(struct send_ctx * sctx,struct pending_dir_move * moves,struct list_head * stack)3412 static void tail_append_pending_moves(struct send_ctx *sctx,
3413 				      struct pending_dir_move *moves,
3414 				      struct list_head *stack)
3415 {
3416 	if (list_empty(&moves->list)) {
3417 		list_add_tail(&moves->list, stack);
3418 	} else {
3419 		LIST_HEAD(list);
3420 		list_splice_init(&moves->list, &list);
3421 		list_add_tail(&moves->list, stack);
3422 		list_splice_tail(&list, stack);
3423 	}
3424 	if (!RB_EMPTY_NODE(&moves->node)) {
3425 		rb_erase(&moves->node, &sctx->pending_dir_moves);
3426 		RB_CLEAR_NODE(&moves->node);
3427 	}
3428 }
3429 
apply_children_dir_moves(struct send_ctx * sctx)3430 static int apply_children_dir_moves(struct send_ctx *sctx)
3431 {
3432 	struct pending_dir_move *pm;
3433 	struct list_head stack;
3434 	u64 parent_ino = sctx->cur_ino;
3435 	int ret = 0;
3436 
3437 	pm = get_pending_dir_moves(sctx, parent_ino);
3438 	if (!pm)
3439 		return 0;
3440 
3441 	INIT_LIST_HEAD(&stack);
3442 	tail_append_pending_moves(sctx, pm, &stack);
3443 
3444 	while (!list_empty(&stack)) {
3445 		pm = list_first_entry(&stack, struct pending_dir_move, list);
3446 		parent_ino = pm->ino;
3447 		ret = apply_dir_move(sctx, pm);
3448 		free_pending_move(sctx, pm);
3449 		if (ret)
3450 			goto out;
3451 		pm = get_pending_dir_moves(sctx, parent_ino);
3452 		if (pm)
3453 			tail_append_pending_moves(sctx, pm, &stack);
3454 	}
3455 	return 0;
3456 
3457 out:
3458 	while (!list_empty(&stack)) {
3459 		pm = list_first_entry(&stack, struct pending_dir_move, list);
3460 		free_pending_move(sctx, pm);
3461 	}
3462 	return ret;
3463 }
3464 
3465 /*
3466  * We might need to delay a directory rename even when no ancestor directory
3467  * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3468  * renamed. This happens when we rename a directory to the old name (the name
3469  * in the parent root) of some other unrelated directory that got its rename
3470  * delayed due to some ancestor with higher number that got renamed.
3471  *
3472  * Example:
3473  *
3474  * Parent snapshot:
3475  * .                                       (ino 256)
3476  * |---- a/                                (ino 257)
3477  * |     |---- file                        (ino 260)
3478  * |
3479  * |---- b/                                (ino 258)
3480  * |---- c/                                (ino 259)
3481  *
3482  * Send snapshot:
3483  * .                                       (ino 256)
3484  * |---- a/                                (ino 258)
3485  * |---- x/                                (ino 259)
3486  *       |---- y/                          (ino 257)
3487  *             |----- file                 (ino 260)
3488  *
3489  * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3490  * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3491  * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3492  * must issue is:
3493  *
3494  * 1 - rename 259 from 'c' to 'x'
3495  * 2 - rename 257 from 'a' to 'x/y'
3496  * 3 - rename 258 from 'b' to 'a'
3497  *
3498  * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3499  * be done right away and < 0 on error.
3500  */
wait_for_dest_dir_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3501 static int wait_for_dest_dir_move(struct send_ctx *sctx,
3502 				  struct recorded_ref *parent_ref,
3503 				  const bool is_orphan)
3504 {
3505 	struct btrfs_fs_info *fs_info = sctx->parent_root->fs_info;
3506 	struct btrfs_path *path;
3507 	struct btrfs_key key;
3508 	struct btrfs_key di_key;
3509 	struct btrfs_dir_item *di;
3510 	u64 left_gen;
3511 	u64 right_gen;
3512 	int ret = 0;
3513 	struct waiting_dir_move *wdm;
3514 
3515 	if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3516 		return 0;
3517 
3518 	path = alloc_path_for_send();
3519 	if (!path)
3520 		return -ENOMEM;
3521 
3522 	key.objectid = parent_ref->dir;
3523 	key.type = BTRFS_DIR_ITEM_KEY;
3524 	key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3525 
3526 	ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3527 	if (ret < 0) {
3528 		goto out;
3529 	} else if (ret > 0) {
3530 		ret = 0;
3531 		goto out;
3532 	}
3533 
3534 	di = btrfs_match_dir_item_name(fs_info, path, parent_ref->name,
3535 				       parent_ref->name_len);
3536 	if (!di) {
3537 		ret = 0;
3538 		goto out;
3539 	}
3540 	/*
3541 	 * di_key.objectid has the number of the inode that has a dentry in the
3542 	 * parent directory with the same name that sctx->cur_ino is being
3543 	 * renamed to. We need to check if that inode is in the send root as
3544 	 * well and if it is currently marked as an inode with a pending rename,
3545 	 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3546 	 * that it happens after that other inode is renamed.
3547 	 */
3548 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3549 	if (di_key.type != BTRFS_INODE_ITEM_KEY) {
3550 		ret = 0;
3551 		goto out;
3552 	}
3553 
3554 	ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
3555 	if (ret < 0)
3556 		goto out;
3557 	ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
3558 	if (ret < 0) {
3559 		if (ret == -ENOENT)
3560 			ret = 0;
3561 		goto out;
3562 	}
3563 
3564 	/* Different inode, no need to delay the rename of sctx->cur_ino */
3565 	if (right_gen != left_gen) {
3566 		ret = 0;
3567 		goto out;
3568 	}
3569 
3570 	wdm = get_waiting_dir_move(sctx, di_key.objectid);
3571 	if (wdm && !wdm->orphanized) {
3572 		ret = add_pending_dir_move(sctx,
3573 					   sctx->cur_ino,
3574 					   sctx->cur_inode_gen,
3575 					   di_key.objectid,
3576 					   &sctx->new_refs,
3577 					   &sctx->deleted_refs,
3578 					   is_orphan);
3579 		if (!ret)
3580 			ret = 1;
3581 	}
3582 out:
3583 	btrfs_free_path(path);
3584 	return ret;
3585 }
3586 
3587 /*
3588  * Check if inode ino2, or any of its ancestors, is inode ino1.
3589  * Return 1 if true, 0 if false and < 0 on error.
3590  */
check_ino_in_path(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,const u64 ino2_gen,struct fs_path * fs_path)3591 static int check_ino_in_path(struct btrfs_root *root,
3592 			     const u64 ino1,
3593 			     const u64 ino1_gen,
3594 			     const u64 ino2,
3595 			     const u64 ino2_gen,
3596 			     struct fs_path *fs_path)
3597 {
3598 	u64 ino = ino2;
3599 
3600 	if (ino1 == ino2)
3601 		return ino1_gen == ino2_gen;
3602 
3603 	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3604 		u64 parent;
3605 		u64 parent_gen;
3606 		int ret;
3607 
3608 		fs_path_reset(fs_path);
3609 		ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3610 		if (ret < 0)
3611 			return ret;
3612 		if (parent == ino1)
3613 			return parent_gen == ino1_gen;
3614 		ino = parent;
3615 	}
3616 	return 0;
3617 }
3618 
3619 /*
3620  * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3621  * possible path (in case ino2 is not a directory and has multiple hard links).
3622  * Return 1 if true, 0 if false and < 0 on error.
3623  */
is_ancestor(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,struct fs_path * fs_path)3624 static int is_ancestor(struct btrfs_root *root,
3625 		       const u64 ino1,
3626 		       const u64 ino1_gen,
3627 		       const u64 ino2,
3628 		       struct fs_path *fs_path)
3629 {
3630 	bool free_fs_path = false;
3631 	int ret = 0;
3632 	int iter_ret = 0;
3633 	struct btrfs_path *path = NULL;
3634 	struct btrfs_key key;
3635 
3636 	if (!fs_path) {
3637 		fs_path = fs_path_alloc();
3638 		if (!fs_path)
3639 			return -ENOMEM;
3640 		free_fs_path = true;
3641 	}
3642 
3643 	path = alloc_path_for_send();
3644 	if (!path) {
3645 		ret = -ENOMEM;
3646 		goto out;
3647 	}
3648 
3649 	key.objectid = ino2;
3650 	key.type = BTRFS_INODE_REF_KEY;
3651 	key.offset = 0;
3652 
3653 	btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
3654 		struct extent_buffer *leaf = path->nodes[0];
3655 		int slot = path->slots[0];
3656 		u32 cur_offset = 0;
3657 		u32 item_size;
3658 
3659 		if (key.objectid != ino2)
3660 			break;
3661 		if (key.type != BTRFS_INODE_REF_KEY &&
3662 		    key.type != BTRFS_INODE_EXTREF_KEY)
3663 			break;
3664 
3665 		item_size = btrfs_item_size(leaf, slot);
3666 		while (cur_offset < item_size) {
3667 			u64 parent;
3668 			u64 parent_gen;
3669 
3670 			if (key.type == BTRFS_INODE_EXTREF_KEY) {
3671 				unsigned long ptr;
3672 				struct btrfs_inode_extref *extref;
3673 
3674 				ptr = btrfs_item_ptr_offset(leaf, slot);
3675 				extref = (struct btrfs_inode_extref *)
3676 					(ptr + cur_offset);
3677 				parent = btrfs_inode_extref_parent(leaf,
3678 								   extref);
3679 				cur_offset += sizeof(*extref);
3680 				cur_offset += btrfs_inode_extref_name_len(leaf,
3681 								  extref);
3682 			} else {
3683 				parent = key.offset;
3684 				cur_offset = item_size;
3685 			}
3686 
3687 			ret = get_inode_gen(root, parent, &parent_gen);
3688 			if (ret < 0)
3689 				goto out;
3690 			ret = check_ino_in_path(root, ino1, ino1_gen,
3691 						parent, parent_gen, fs_path);
3692 			if (ret)
3693 				goto out;
3694 		}
3695 	}
3696 	ret = 0;
3697 	if (iter_ret < 0)
3698 		ret = iter_ret;
3699 
3700 out:
3701 	btrfs_free_path(path);
3702 	if (free_fs_path)
3703 		fs_path_free(fs_path);
3704 	return ret;
3705 }
3706 
wait_for_parent_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3707 static int wait_for_parent_move(struct send_ctx *sctx,
3708 				struct recorded_ref *parent_ref,
3709 				const bool is_orphan)
3710 {
3711 	int ret = 0;
3712 	u64 ino = parent_ref->dir;
3713 	u64 ino_gen = parent_ref->dir_gen;
3714 	u64 parent_ino_before, parent_ino_after;
3715 	struct fs_path *path_before = NULL;
3716 	struct fs_path *path_after = NULL;
3717 	int len1, len2;
3718 
3719 	path_after = fs_path_alloc();
3720 	path_before = fs_path_alloc();
3721 	if (!path_after || !path_before) {
3722 		ret = -ENOMEM;
3723 		goto out;
3724 	}
3725 
3726 	/*
3727 	 * Our current directory inode may not yet be renamed/moved because some
3728 	 * ancestor (immediate or not) has to be renamed/moved first. So find if
3729 	 * such ancestor exists and make sure our own rename/move happens after
3730 	 * that ancestor is processed to avoid path build infinite loops (done
3731 	 * at get_cur_path()).
3732 	 */
3733 	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3734 		u64 parent_ino_after_gen;
3735 
3736 		if (is_waiting_for_move(sctx, ino)) {
3737 			/*
3738 			 * If the current inode is an ancestor of ino in the
3739 			 * parent root, we need to delay the rename of the
3740 			 * current inode, otherwise don't delayed the rename
3741 			 * because we can end up with a circular dependency
3742 			 * of renames, resulting in some directories never
3743 			 * getting the respective rename operations issued in
3744 			 * the send stream or getting into infinite path build
3745 			 * loops.
3746 			 */
3747 			ret = is_ancestor(sctx->parent_root,
3748 					  sctx->cur_ino, sctx->cur_inode_gen,
3749 					  ino, path_before);
3750 			if (ret)
3751 				break;
3752 		}
3753 
3754 		fs_path_reset(path_before);
3755 		fs_path_reset(path_after);
3756 
3757 		ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
3758 				    &parent_ino_after_gen, path_after);
3759 		if (ret < 0)
3760 			goto out;
3761 		ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
3762 				    NULL, path_before);
3763 		if (ret < 0 && ret != -ENOENT) {
3764 			goto out;
3765 		} else if (ret == -ENOENT) {
3766 			ret = 0;
3767 			break;
3768 		}
3769 
3770 		len1 = fs_path_len(path_before);
3771 		len2 = fs_path_len(path_after);
3772 		if (ino > sctx->cur_ino &&
3773 		    (parent_ino_before != parent_ino_after || len1 != len2 ||
3774 		     memcmp(path_before->start, path_after->start, len1))) {
3775 			u64 parent_ino_gen;
3776 
3777 			ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
3778 			if (ret < 0)
3779 				goto out;
3780 			if (ino_gen == parent_ino_gen) {
3781 				ret = 1;
3782 				break;
3783 			}
3784 		}
3785 		ino = parent_ino_after;
3786 		ino_gen = parent_ino_after_gen;
3787 	}
3788 
3789 out:
3790 	fs_path_free(path_before);
3791 	fs_path_free(path_after);
3792 
3793 	if (ret == 1) {
3794 		ret = add_pending_dir_move(sctx,
3795 					   sctx->cur_ino,
3796 					   sctx->cur_inode_gen,
3797 					   ino,
3798 					   &sctx->new_refs,
3799 					   &sctx->deleted_refs,
3800 					   is_orphan);
3801 		if (!ret)
3802 			ret = 1;
3803 	}
3804 
3805 	return ret;
3806 }
3807 
update_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)3808 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
3809 {
3810 	int ret;
3811 	struct fs_path *new_path;
3812 
3813 	/*
3814 	 * Our reference's name member points to its full_path member string, so
3815 	 * we use here a new path.
3816 	 */
3817 	new_path = fs_path_alloc();
3818 	if (!new_path)
3819 		return -ENOMEM;
3820 
3821 	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
3822 	if (ret < 0) {
3823 		fs_path_free(new_path);
3824 		return ret;
3825 	}
3826 	ret = fs_path_add(new_path, ref->name, ref->name_len);
3827 	if (ret < 0) {
3828 		fs_path_free(new_path);
3829 		return ret;
3830 	}
3831 
3832 	fs_path_free(ref->full_path);
3833 	set_ref_path(ref, new_path);
3834 
3835 	return 0;
3836 }
3837 
3838 /*
3839  * When processing the new references for an inode we may orphanize an existing
3840  * directory inode because its old name conflicts with one of the new references
3841  * of the current inode. Later, when processing another new reference of our
3842  * inode, we might need to orphanize another inode, but the path we have in the
3843  * reference reflects the pre-orphanization name of the directory we previously
3844  * orphanized. For example:
3845  *
3846  * parent snapshot looks like:
3847  *
3848  * .                                     (ino 256)
3849  * |----- f1                             (ino 257)
3850  * |----- f2                             (ino 258)
3851  * |----- d1/                            (ino 259)
3852  *        |----- d2/                     (ino 260)
3853  *
3854  * send snapshot looks like:
3855  *
3856  * .                                     (ino 256)
3857  * |----- d1                             (ino 258)
3858  * |----- f2/                            (ino 259)
3859  *        |----- f2_link/                (ino 260)
3860  *        |       |----- f1              (ino 257)
3861  *        |
3862  *        |----- d2                      (ino 258)
3863  *
3864  * When processing inode 257 we compute the name for inode 259 as "d1", and we
3865  * cache it in the name cache. Later when we start processing inode 258, when
3866  * collecting all its new references we set a full path of "d1/d2" for its new
3867  * reference with name "d2". When we start processing the new references we
3868  * start by processing the new reference with name "d1", and this results in
3869  * orphanizing inode 259, since its old reference causes a conflict. Then we
3870  * move on the next new reference, with name "d2", and we find out we must
3871  * orphanize inode 260, as its old reference conflicts with ours - but for the
3872  * orphanization we use a source path corresponding to the path we stored in the
3873  * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
3874  * receiver fail since the path component "d1/" no longer exists, it was renamed
3875  * to "o259-6-0/" when processing the previous new reference. So in this case we
3876  * must recompute the path in the new reference and use it for the new
3877  * orphanization operation.
3878  */
refresh_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)3879 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
3880 {
3881 	char *name;
3882 	int ret;
3883 
3884 	name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
3885 	if (!name)
3886 		return -ENOMEM;
3887 
3888 	fs_path_reset(ref->full_path);
3889 	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
3890 	if (ret < 0)
3891 		goto out;
3892 
3893 	ret = fs_path_add(ref->full_path, name, ref->name_len);
3894 	if (ret < 0)
3895 		goto out;
3896 
3897 	/* Update the reference's base name pointer. */
3898 	set_ref_path(ref, ref->full_path);
3899 out:
3900 	kfree(name);
3901 	return ret;
3902 }
3903 
3904 /*
3905  * This does all the move/link/unlink/rmdir magic.
3906  */
process_recorded_refs(struct send_ctx * sctx,int * pending_move)3907 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
3908 {
3909 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
3910 	int ret = 0;
3911 	struct recorded_ref *cur;
3912 	struct recorded_ref *cur2;
3913 	struct list_head check_dirs;
3914 	struct fs_path *valid_path = NULL;
3915 	u64 ow_inode = 0;
3916 	u64 ow_gen;
3917 	u64 ow_mode;
3918 	int did_overwrite = 0;
3919 	int is_orphan = 0;
3920 	u64 last_dir_ino_rm = 0;
3921 	bool can_rename = true;
3922 	bool orphanized_dir = false;
3923 	bool orphanized_ancestor = false;
3924 
3925 	btrfs_debug(fs_info, "process_recorded_refs %llu", sctx->cur_ino);
3926 
3927 	/*
3928 	 * This should never happen as the root dir always has the same ref
3929 	 * which is always '..'
3930 	 */
3931 	BUG_ON(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID);
3932 	INIT_LIST_HEAD(&check_dirs);
3933 
3934 	valid_path = fs_path_alloc();
3935 	if (!valid_path) {
3936 		ret = -ENOMEM;
3937 		goto out;
3938 	}
3939 
3940 	/*
3941 	 * First, check if the first ref of the current inode was overwritten
3942 	 * before. If yes, we know that the current inode was already orphanized
3943 	 * and thus use the orphan name. If not, we can use get_cur_path to
3944 	 * get the path of the first ref as it would like while receiving at
3945 	 * this point in time.
3946 	 * New inodes are always orphan at the beginning, so force to use the
3947 	 * orphan name in this case.
3948 	 * The first ref is stored in valid_path and will be updated if it
3949 	 * gets moved around.
3950 	 */
3951 	if (!sctx->cur_inode_new) {
3952 		ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
3953 				sctx->cur_inode_gen);
3954 		if (ret < 0)
3955 			goto out;
3956 		if (ret)
3957 			did_overwrite = 1;
3958 	}
3959 	if (sctx->cur_inode_new || did_overwrite) {
3960 		ret = gen_unique_name(sctx, sctx->cur_ino,
3961 				sctx->cur_inode_gen, valid_path);
3962 		if (ret < 0)
3963 			goto out;
3964 		is_orphan = 1;
3965 	} else {
3966 		ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
3967 				valid_path);
3968 		if (ret < 0)
3969 			goto out;
3970 	}
3971 
3972 	/*
3973 	 * Before doing any rename and link operations, do a first pass on the
3974 	 * new references to orphanize any unprocessed inodes that may have a
3975 	 * reference that conflicts with one of the new references of the current
3976 	 * inode. This needs to happen first because a new reference may conflict
3977 	 * with the old reference of a parent directory, so we must make sure
3978 	 * that the path used for link and rename commands don't use an
3979 	 * orphanized name when an ancestor was not yet orphanized.
3980 	 *
3981 	 * Example:
3982 	 *
3983 	 * Parent snapshot:
3984 	 *
3985 	 * .                                                      (ino 256)
3986 	 * |----- testdir/                                        (ino 259)
3987 	 * |          |----- a                                    (ino 257)
3988 	 * |
3989 	 * |----- b                                               (ino 258)
3990 	 *
3991 	 * Send snapshot:
3992 	 *
3993 	 * .                                                      (ino 256)
3994 	 * |----- testdir_2/                                      (ino 259)
3995 	 * |          |----- a                                    (ino 260)
3996 	 * |
3997 	 * |----- testdir                                         (ino 257)
3998 	 * |----- b                                               (ino 257)
3999 	 * |----- b2                                              (ino 258)
4000 	 *
4001 	 * Processing the new reference for inode 257 with name "b" may happen
4002 	 * before processing the new reference with name "testdir". If so, we
4003 	 * must make sure that by the time we send a link command to create the
4004 	 * hard link "b", inode 259 was already orphanized, since the generated
4005 	 * path in "valid_path" already contains the orphanized name for 259.
4006 	 * We are processing inode 257, so only later when processing 259 we do
4007 	 * the rename operation to change its temporary (orphanized) name to
4008 	 * "testdir_2".
4009 	 */
4010 	list_for_each_entry(cur, &sctx->new_refs, list) {
4011 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen);
4012 		if (ret < 0)
4013 			goto out;
4014 		if (ret == inode_state_will_create)
4015 			continue;
4016 
4017 		/*
4018 		 * Check if this new ref would overwrite the first ref of another
4019 		 * unprocessed inode. If yes, orphanize the overwritten inode.
4020 		 * If we find an overwritten ref that is not the first ref,
4021 		 * simply unlink it.
4022 		 */
4023 		ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4024 				cur->name, cur->name_len,
4025 				&ow_inode, &ow_gen, &ow_mode);
4026 		if (ret < 0)
4027 			goto out;
4028 		if (ret) {
4029 			ret = is_first_ref(sctx->parent_root,
4030 					   ow_inode, cur->dir, cur->name,
4031 					   cur->name_len);
4032 			if (ret < 0)
4033 				goto out;
4034 			if (ret) {
4035 				struct name_cache_entry *nce;
4036 				struct waiting_dir_move *wdm;
4037 
4038 				if (orphanized_dir) {
4039 					ret = refresh_ref_path(sctx, cur);
4040 					if (ret < 0)
4041 						goto out;
4042 				}
4043 
4044 				ret = orphanize_inode(sctx, ow_inode, ow_gen,
4045 						cur->full_path);
4046 				if (ret < 0)
4047 					goto out;
4048 				if (S_ISDIR(ow_mode))
4049 					orphanized_dir = true;
4050 
4051 				/*
4052 				 * If ow_inode has its rename operation delayed
4053 				 * make sure that its orphanized name is used in
4054 				 * the source path when performing its rename
4055 				 * operation.
4056 				 */
4057 				if (is_waiting_for_move(sctx, ow_inode)) {
4058 					wdm = get_waiting_dir_move(sctx,
4059 								   ow_inode);
4060 					ASSERT(wdm);
4061 					wdm->orphanized = true;
4062 				}
4063 
4064 				/*
4065 				 * Make sure we clear our orphanized inode's
4066 				 * name from the name cache. This is because the
4067 				 * inode ow_inode might be an ancestor of some
4068 				 * other inode that will be orphanized as well
4069 				 * later and has an inode number greater than
4070 				 * sctx->send_progress. We need to prevent
4071 				 * future name lookups from using the old name
4072 				 * and get instead the orphan name.
4073 				 */
4074 				nce = name_cache_search(sctx, ow_inode, ow_gen);
4075 				if (nce) {
4076 					name_cache_delete(sctx, nce);
4077 					kfree(nce);
4078 				}
4079 
4080 				/*
4081 				 * ow_inode might currently be an ancestor of
4082 				 * cur_ino, therefore compute valid_path (the
4083 				 * current path of cur_ino) again because it
4084 				 * might contain the pre-orphanization name of
4085 				 * ow_inode, which is no longer valid.
4086 				 */
4087 				ret = is_ancestor(sctx->parent_root,
4088 						  ow_inode, ow_gen,
4089 						  sctx->cur_ino, NULL);
4090 				if (ret > 0) {
4091 					orphanized_ancestor = true;
4092 					fs_path_reset(valid_path);
4093 					ret = get_cur_path(sctx, sctx->cur_ino,
4094 							   sctx->cur_inode_gen,
4095 							   valid_path);
4096 				}
4097 				if (ret < 0)
4098 					goto out;
4099 			} else {
4100 				/*
4101 				 * If we previously orphanized a directory that
4102 				 * collided with a new reference that we already
4103 				 * processed, recompute the current path because
4104 				 * that directory may be part of the path.
4105 				 */
4106 				if (orphanized_dir) {
4107 					ret = refresh_ref_path(sctx, cur);
4108 					if (ret < 0)
4109 						goto out;
4110 				}
4111 				ret = send_unlink(sctx, cur->full_path);
4112 				if (ret < 0)
4113 					goto out;
4114 			}
4115 		}
4116 
4117 	}
4118 
4119 	list_for_each_entry(cur, &sctx->new_refs, list) {
4120 		/*
4121 		 * We may have refs where the parent directory does not exist
4122 		 * yet. This happens if the parent directories inum is higher
4123 		 * than the current inum. To handle this case, we create the
4124 		 * parent directory out of order. But we need to check if this
4125 		 * did already happen before due to other refs in the same dir.
4126 		 */
4127 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen);
4128 		if (ret < 0)
4129 			goto out;
4130 		if (ret == inode_state_will_create) {
4131 			ret = 0;
4132 			/*
4133 			 * First check if any of the current inodes refs did
4134 			 * already create the dir.
4135 			 */
4136 			list_for_each_entry(cur2, &sctx->new_refs, list) {
4137 				if (cur == cur2)
4138 					break;
4139 				if (cur2->dir == cur->dir) {
4140 					ret = 1;
4141 					break;
4142 				}
4143 			}
4144 
4145 			/*
4146 			 * If that did not happen, check if a previous inode
4147 			 * did already create the dir.
4148 			 */
4149 			if (!ret)
4150 				ret = did_create_dir(sctx, cur->dir);
4151 			if (ret < 0)
4152 				goto out;
4153 			if (!ret) {
4154 				ret = send_create_inode(sctx, cur->dir);
4155 				if (ret < 0)
4156 					goto out;
4157 			}
4158 		}
4159 
4160 		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4161 			ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4162 			if (ret < 0)
4163 				goto out;
4164 			if (ret == 1) {
4165 				can_rename = false;
4166 				*pending_move = 1;
4167 			}
4168 		}
4169 
4170 		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4171 		    can_rename) {
4172 			ret = wait_for_parent_move(sctx, cur, is_orphan);
4173 			if (ret < 0)
4174 				goto out;
4175 			if (ret == 1) {
4176 				can_rename = false;
4177 				*pending_move = 1;
4178 			}
4179 		}
4180 
4181 		/*
4182 		 * link/move the ref to the new place. If we have an orphan
4183 		 * inode, move it and update valid_path. If not, link or move
4184 		 * it depending on the inode mode.
4185 		 */
4186 		if (is_orphan && can_rename) {
4187 			ret = send_rename(sctx, valid_path, cur->full_path);
4188 			if (ret < 0)
4189 				goto out;
4190 			is_orphan = 0;
4191 			ret = fs_path_copy(valid_path, cur->full_path);
4192 			if (ret < 0)
4193 				goto out;
4194 		} else if (can_rename) {
4195 			if (S_ISDIR(sctx->cur_inode_mode)) {
4196 				/*
4197 				 * Dirs can't be linked, so move it. For moved
4198 				 * dirs, we always have one new and one deleted
4199 				 * ref. The deleted ref is ignored later.
4200 				 */
4201 				ret = send_rename(sctx, valid_path,
4202 						  cur->full_path);
4203 				if (!ret)
4204 					ret = fs_path_copy(valid_path,
4205 							   cur->full_path);
4206 				if (ret < 0)
4207 					goto out;
4208 			} else {
4209 				/*
4210 				 * We might have previously orphanized an inode
4211 				 * which is an ancestor of our current inode,
4212 				 * so our reference's full path, which was
4213 				 * computed before any such orphanizations, must
4214 				 * be updated.
4215 				 */
4216 				if (orphanized_dir) {
4217 					ret = update_ref_path(sctx, cur);
4218 					if (ret < 0)
4219 						goto out;
4220 				}
4221 				ret = send_link(sctx, cur->full_path,
4222 						valid_path);
4223 				if (ret < 0)
4224 					goto out;
4225 			}
4226 		}
4227 		ret = dup_ref(cur, &check_dirs);
4228 		if (ret < 0)
4229 			goto out;
4230 	}
4231 
4232 	if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4233 		/*
4234 		 * Check if we can already rmdir the directory. If not,
4235 		 * orphanize it. For every dir item inside that gets deleted
4236 		 * later, we do this check again and rmdir it then if possible.
4237 		 * See the use of check_dirs for more details.
4238 		 */
4239 		ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4240 				sctx->cur_ino);
4241 		if (ret < 0)
4242 			goto out;
4243 		if (ret) {
4244 			ret = send_rmdir(sctx, valid_path);
4245 			if (ret < 0)
4246 				goto out;
4247 		} else if (!is_orphan) {
4248 			ret = orphanize_inode(sctx, sctx->cur_ino,
4249 					sctx->cur_inode_gen, valid_path);
4250 			if (ret < 0)
4251 				goto out;
4252 			is_orphan = 1;
4253 		}
4254 
4255 		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4256 			ret = dup_ref(cur, &check_dirs);
4257 			if (ret < 0)
4258 				goto out;
4259 		}
4260 	} else if (S_ISDIR(sctx->cur_inode_mode) &&
4261 		   !list_empty(&sctx->deleted_refs)) {
4262 		/*
4263 		 * We have a moved dir. Add the old parent to check_dirs
4264 		 */
4265 		cur = list_entry(sctx->deleted_refs.next, struct recorded_ref,
4266 				list);
4267 		ret = dup_ref(cur, &check_dirs);
4268 		if (ret < 0)
4269 			goto out;
4270 	} else if (!S_ISDIR(sctx->cur_inode_mode)) {
4271 		/*
4272 		 * We have a non dir inode. Go through all deleted refs and
4273 		 * unlink them if they were not already overwritten by other
4274 		 * inodes.
4275 		 */
4276 		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4277 			ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4278 					sctx->cur_ino, sctx->cur_inode_gen,
4279 					cur->name, cur->name_len);
4280 			if (ret < 0)
4281 				goto out;
4282 			if (!ret) {
4283 				/*
4284 				 * If we orphanized any ancestor before, we need
4285 				 * to recompute the full path for deleted names,
4286 				 * since any such path was computed before we
4287 				 * processed any references and orphanized any
4288 				 * ancestor inode.
4289 				 */
4290 				if (orphanized_ancestor) {
4291 					ret = update_ref_path(sctx, cur);
4292 					if (ret < 0)
4293 						goto out;
4294 				}
4295 				ret = send_unlink(sctx, cur->full_path);
4296 				if (ret < 0)
4297 					goto out;
4298 			}
4299 			ret = dup_ref(cur, &check_dirs);
4300 			if (ret < 0)
4301 				goto out;
4302 		}
4303 		/*
4304 		 * If the inode is still orphan, unlink the orphan. This may
4305 		 * happen when a previous inode did overwrite the first ref
4306 		 * of this inode and no new refs were added for the current
4307 		 * inode. Unlinking does not mean that the inode is deleted in
4308 		 * all cases. There may still be links to this inode in other
4309 		 * places.
4310 		 */
4311 		if (is_orphan) {
4312 			ret = send_unlink(sctx, valid_path);
4313 			if (ret < 0)
4314 				goto out;
4315 		}
4316 	}
4317 
4318 	/*
4319 	 * We did collect all parent dirs where cur_inode was once located. We
4320 	 * now go through all these dirs and check if they are pending for
4321 	 * deletion and if it's finally possible to perform the rmdir now.
4322 	 * We also update the inode stats of the parent dirs here.
4323 	 */
4324 	list_for_each_entry(cur, &check_dirs, list) {
4325 		/*
4326 		 * In case we had refs into dirs that were not processed yet,
4327 		 * we don't need to do the utime and rmdir logic for these dirs.
4328 		 * The dir will be processed later.
4329 		 */
4330 		if (cur->dir > sctx->cur_ino)
4331 			continue;
4332 
4333 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen);
4334 		if (ret < 0)
4335 			goto out;
4336 
4337 		if (ret == inode_state_did_create ||
4338 		    ret == inode_state_no_change) {
4339 			/* TODO delayed utimes */
4340 			ret = send_utimes(sctx, cur->dir, cur->dir_gen);
4341 			if (ret < 0)
4342 				goto out;
4343 		} else if (ret == inode_state_did_delete &&
4344 			   cur->dir != last_dir_ino_rm) {
4345 			ret = can_rmdir(sctx, cur->dir, cur->dir_gen,
4346 					sctx->cur_ino);
4347 			if (ret < 0)
4348 				goto out;
4349 			if (ret) {
4350 				ret = get_cur_path(sctx, cur->dir,
4351 						   cur->dir_gen, valid_path);
4352 				if (ret < 0)
4353 					goto out;
4354 				ret = send_rmdir(sctx, valid_path);
4355 				if (ret < 0)
4356 					goto out;
4357 				last_dir_ino_rm = cur->dir;
4358 			}
4359 		}
4360 	}
4361 
4362 	ret = 0;
4363 
4364 out:
4365 	__free_recorded_refs(&check_dirs);
4366 	free_recorded_refs(sctx);
4367 	fs_path_free(valid_path);
4368 	return ret;
4369 }
4370 
rbtree_ref_comp(const void * k,const struct rb_node * node)4371 static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4372 {
4373 	const struct recorded_ref *data = k;
4374 	const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4375 	int result;
4376 
4377 	if (data->dir > ref->dir)
4378 		return 1;
4379 	if (data->dir < ref->dir)
4380 		return -1;
4381 	if (data->dir_gen > ref->dir_gen)
4382 		return 1;
4383 	if (data->dir_gen < ref->dir_gen)
4384 		return -1;
4385 	if (data->name_len > ref->name_len)
4386 		return 1;
4387 	if (data->name_len < ref->name_len)
4388 		return -1;
4389 	result = strcmp(data->name, ref->name);
4390 	if (result > 0)
4391 		return 1;
4392 	if (result < 0)
4393 		return -1;
4394 	return 0;
4395 }
4396 
rbtree_ref_less(struct rb_node * node,const struct rb_node * parent)4397 static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4398 {
4399 	const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4400 
4401 	return rbtree_ref_comp(entry, parent) < 0;
4402 }
4403 
record_ref_in_tree(struct rb_root * root,struct list_head * refs,struct fs_path * name,u64 dir,u64 dir_gen,struct send_ctx * sctx)4404 static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4405 			      struct fs_path *name, u64 dir, u64 dir_gen,
4406 			      struct send_ctx *sctx)
4407 {
4408 	int ret = 0;
4409 	struct fs_path *path = NULL;
4410 	struct recorded_ref *ref = NULL;
4411 
4412 	path = fs_path_alloc();
4413 	if (!path) {
4414 		ret = -ENOMEM;
4415 		goto out;
4416 	}
4417 
4418 	ref = recorded_ref_alloc();
4419 	if (!ref) {
4420 		ret = -ENOMEM;
4421 		goto out;
4422 	}
4423 
4424 	ret = get_cur_path(sctx, dir, dir_gen, path);
4425 	if (ret < 0)
4426 		goto out;
4427 	ret = fs_path_add_path(path, name);
4428 	if (ret < 0)
4429 		goto out;
4430 
4431 	ref->dir = dir;
4432 	ref->dir_gen = dir_gen;
4433 	set_ref_path(ref, path);
4434 	list_add_tail(&ref->list, refs);
4435 	rb_add(&ref->node, root, rbtree_ref_less);
4436 	ref->root = root;
4437 out:
4438 	if (ret) {
4439 		if (path && (!ref || !ref->full_path))
4440 			fs_path_free(path);
4441 		recorded_ref_free(ref);
4442 	}
4443 	return ret;
4444 }
4445 
record_new_ref_if_needed(int num,u64 dir,int index,struct fs_path * name,void * ctx)4446 static int record_new_ref_if_needed(int num, u64 dir, int index,
4447 				    struct fs_path *name, void *ctx)
4448 {
4449 	int ret = 0;
4450 	struct send_ctx *sctx = ctx;
4451 	struct rb_node *node = NULL;
4452 	struct recorded_ref data;
4453 	struct recorded_ref *ref;
4454 	u64 dir_gen;
4455 
4456 	ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4457 	if (ret < 0)
4458 		goto out;
4459 
4460 	data.dir = dir;
4461 	data.dir_gen = dir_gen;
4462 	set_ref_path(&data, name);
4463 	node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4464 	if (node) {
4465 		ref = rb_entry(node, struct recorded_ref, node);
4466 		recorded_ref_free(ref);
4467 	} else {
4468 		ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4469 					 &sctx->new_refs, name, dir, dir_gen,
4470 					 sctx);
4471 	}
4472 out:
4473 	return ret;
4474 }
4475 
record_deleted_ref_if_needed(int num,u64 dir,int index,struct fs_path * name,void * ctx)4476 static int record_deleted_ref_if_needed(int num, u64 dir, int index,
4477 					struct fs_path *name, void *ctx)
4478 {
4479 	int ret = 0;
4480 	struct send_ctx *sctx = ctx;
4481 	struct rb_node *node = NULL;
4482 	struct recorded_ref data;
4483 	struct recorded_ref *ref;
4484 	u64 dir_gen;
4485 
4486 	ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4487 	if (ret < 0)
4488 		goto out;
4489 
4490 	data.dir = dir;
4491 	data.dir_gen = dir_gen;
4492 	set_ref_path(&data, name);
4493 	node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4494 	if (node) {
4495 		ref = rb_entry(node, struct recorded_ref, node);
4496 		recorded_ref_free(ref);
4497 	} else {
4498 		ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4499 					 &sctx->deleted_refs, name, dir,
4500 					 dir_gen, sctx);
4501 	}
4502 out:
4503 	return ret;
4504 }
4505 
record_new_ref(struct send_ctx * sctx)4506 static int record_new_ref(struct send_ctx *sctx)
4507 {
4508 	int ret;
4509 
4510 	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4511 				sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4512 	if (ret < 0)
4513 		goto out;
4514 	ret = 0;
4515 
4516 out:
4517 	return ret;
4518 }
4519 
record_deleted_ref(struct send_ctx * sctx)4520 static int record_deleted_ref(struct send_ctx *sctx)
4521 {
4522 	int ret;
4523 
4524 	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4525 				sctx->cmp_key, 0, record_deleted_ref_if_needed,
4526 				sctx);
4527 	if (ret < 0)
4528 		goto out;
4529 	ret = 0;
4530 
4531 out:
4532 	return ret;
4533 }
4534 
record_changed_ref(struct send_ctx * sctx)4535 static int record_changed_ref(struct send_ctx *sctx)
4536 {
4537 	int ret = 0;
4538 
4539 	ret = iterate_inode_ref(sctx->send_root, sctx->left_path,
4540 			sctx->cmp_key, 0, record_new_ref_if_needed, sctx);
4541 	if (ret < 0)
4542 		goto out;
4543 	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path,
4544 			sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx);
4545 	if (ret < 0)
4546 		goto out;
4547 	ret = 0;
4548 
4549 out:
4550 	return ret;
4551 }
4552 
4553 /*
4554  * Record and process all refs at once. Needed when an inode changes the
4555  * generation number, which means that it was deleted and recreated.
4556  */
process_all_refs(struct send_ctx * sctx,enum btrfs_compare_tree_result cmd)4557 static int process_all_refs(struct send_ctx *sctx,
4558 			    enum btrfs_compare_tree_result cmd)
4559 {
4560 	int ret = 0;
4561 	int iter_ret = 0;
4562 	struct btrfs_root *root;
4563 	struct btrfs_path *path;
4564 	struct btrfs_key key;
4565 	struct btrfs_key found_key;
4566 	iterate_inode_ref_t cb;
4567 	int pending_move = 0;
4568 
4569 	path = alloc_path_for_send();
4570 	if (!path)
4571 		return -ENOMEM;
4572 
4573 	if (cmd == BTRFS_COMPARE_TREE_NEW) {
4574 		root = sctx->send_root;
4575 		cb = record_new_ref_if_needed;
4576 	} else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4577 		root = sctx->parent_root;
4578 		cb = record_deleted_ref_if_needed;
4579 	} else {
4580 		btrfs_err(sctx->send_root->fs_info,
4581 				"Wrong command %d in process_all_refs", cmd);
4582 		ret = -EINVAL;
4583 		goto out;
4584 	}
4585 
4586 	key.objectid = sctx->cmp_key->objectid;
4587 	key.type = BTRFS_INODE_REF_KEY;
4588 	key.offset = 0;
4589 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4590 		if (found_key.objectid != key.objectid ||
4591 		    (found_key.type != BTRFS_INODE_REF_KEY &&
4592 		     found_key.type != BTRFS_INODE_EXTREF_KEY))
4593 			break;
4594 
4595 		ret = iterate_inode_ref(root, path, &found_key, 0, cb, sctx);
4596 		if (ret < 0)
4597 			goto out;
4598 	}
4599 	/* Catch error found during iteration */
4600 	if (iter_ret < 0) {
4601 		ret = iter_ret;
4602 		goto out;
4603 	}
4604 	btrfs_release_path(path);
4605 
4606 	/*
4607 	 * We don't actually care about pending_move as we are simply
4608 	 * re-creating this inode and will be rename'ing it into place once we
4609 	 * rename the parent directory.
4610 	 */
4611 	ret = process_recorded_refs(sctx, &pending_move);
4612 out:
4613 	btrfs_free_path(path);
4614 	return ret;
4615 }
4616 
send_set_xattr(struct send_ctx * sctx,struct fs_path * path,const char * name,int name_len,const char * data,int data_len)4617 static int send_set_xattr(struct send_ctx *sctx,
4618 			  struct fs_path *path,
4619 			  const char *name, int name_len,
4620 			  const char *data, int data_len)
4621 {
4622 	int ret = 0;
4623 
4624 	ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4625 	if (ret < 0)
4626 		goto out;
4627 
4628 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4629 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4630 	TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4631 
4632 	ret = send_cmd(sctx);
4633 
4634 tlv_put_failure:
4635 out:
4636 	return ret;
4637 }
4638 
send_remove_xattr(struct send_ctx * sctx,struct fs_path * path,const char * name,int name_len)4639 static int send_remove_xattr(struct send_ctx *sctx,
4640 			  struct fs_path *path,
4641 			  const char *name, int name_len)
4642 {
4643 	int ret = 0;
4644 
4645 	ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4646 	if (ret < 0)
4647 		goto out;
4648 
4649 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4650 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4651 
4652 	ret = send_cmd(sctx);
4653 
4654 tlv_put_failure:
4655 out:
4656 	return ret;
4657 }
4658 
__process_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4659 static int __process_new_xattr(int num, struct btrfs_key *di_key,
4660 			       const char *name, int name_len, const char *data,
4661 			       int data_len, void *ctx)
4662 {
4663 	int ret;
4664 	struct send_ctx *sctx = ctx;
4665 	struct fs_path *p;
4666 	struct posix_acl_xattr_header dummy_acl;
4667 
4668 	/* Capabilities are emitted by finish_inode_if_needed */
4669 	if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4670 		return 0;
4671 
4672 	p = fs_path_alloc();
4673 	if (!p)
4674 		return -ENOMEM;
4675 
4676 	/*
4677 	 * This hack is needed because empty acls are stored as zero byte
4678 	 * data in xattrs. Problem with that is, that receiving these zero byte
4679 	 * acls will fail later. To fix this, we send a dummy acl list that
4680 	 * only contains the version number and no entries.
4681 	 */
4682 	if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4683 	    !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4684 		if (data_len == 0) {
4685 			dummy_acl.a_version =
4686 					cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4687 			data = (char *)&dummy_acl;
4688 			data_len = sizeof(dummy_acl);
4689 		}
4690 	}
4691 
4692 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4693 	if (ret < 0)
4694 		goto out;
4695 
4696 	ret = send_set_xattr(sctx, p, name, name_len, data, data_len);
4697 
4698 out:
4699 	fs_path_free(p);
4700 	return ret;
4701 }
4702 
__process_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4703 static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4704 				   const char *name, int name_len,
4705 				   const char *data, int data_len, void *ctx)
4706 {
4707 	int ret;
4708 	struct send_ctx *sctx = ctx;
4709 	struct fs_path *p;
4710 
4711 	p = fs_path_alloc();
4712 	if (!p)
4713 		return -ENOMEM;
4714 
4715 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4716 	if (ret < 0)
4717 		goto out;
4718 
4719 	ret = send_remove_xattr(sctx, p, name, name_len);
4720 
4721 out:
4722 	fs_path_free(p);
4723 	return ret;
4724 }
4725 
process_new_xattr(struct send_ctx * sctx)4726 static int process_new_xattr(struct send_ctx *sctx)
4727 {
4728 	int ret = 0;
4729 
4730 	ret = iterate_dir_item(sctx->send_root, sctx->left_path,
4731 			       __process_new_xattr, sctx);
4732 
4733 	return ret;
4734 }
4735 
process_deleted_xattr(struct send_ctx * sctx)4736 static int process_deleted_xattr(struct send_ctx *sctx)
4737 {
4738 	return iterate_dir_item(sctx->parent_root, sctx->right_path,
4739 				__process_deleted_xattr, sctx);
4740 }
4741 
4742 struct find_xattr_ctx {
4743 	const char *name;
4744 	int name_len;
4745 	int found_idx;
4746 	char *found_data;
4747 	int found_data_len;
4748 };
4749 
__find_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * vctx)4750 static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
4751 			int name_len, const char *data, int data_len, void *vctx)
4752 {
4753 	struct find_xattr_ctx *ctx = vctx;
4754 
4755 	if (name_len == ctx->name_len &&
4756 	    strncmp(name, ctx->name, name_len) == 0) {
4757 		ctx->found_idx = num;
4758 		ctx->found_data_len = data_len;
4759 		ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
4760 		if (!ctx->found_data)
4761 			return -ENOMEM;
4762 		return 1;
4763 	}
4764 	return 0;
4765 }
4766 
find_xattr(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * key,const char * name,int name_len,char ** data,int * data_len)4767 static int find_xattr(struct btrfs_root *root,
4768 		      struct btrfs_path *path,
4769 		      struct btrfs_key *key,
4770 		      const char *name, int name_len,
4771 		      char **data, int *data_len)
4772 {
4773 	int ret;
4774 	struct find_xattr_ctx ctx;
4775 
4776 	ctx.name = name;
4777 	ctx.name_len = name_len;
4778 	ctx.found_idx = -1;
4779 	ctx.found_data = NULL;
4780 	ctx.found_data_len = 0;
4781 
4782 	ret = iterate_dir_item(root, path, __find_xattr, &ctx);
4783 	if (ret < 0)
4784 		return ret;
4785 
4786 	if (ctx.found_idx == -1)
4787 		return -ENOENT;
4788 	if (data) {
4789 		*data = ctx.found_data;
4790 		*data_len = ctx.found_data_len;
4791 	} else {
4792 		kfree(ctx.found_data);
4793 	}
4794 	return ctx.found_idx;
4795 }
4796 
4797 
__process_changed_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4798 static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
4799 				       const char *name, int name_len,
4800 				       const char *data, int data_len,
4801 				       void *ctx)
4802 {
4803 	int ret;
4804 	struct send_ctx *sctx = ctx;
4805 	char *found_data = NULL;
4806 	int found_data_len  = 0;
4807 
4808 	ret = find_xattr(sctx->parent_root, sctx->right_path,
4809 			 sctx->cmp_key, name, name_len, &found_data,
4810 			 &found_data_len);
4811 	if (ret == -ENOENT) {
4812 		ret = __process_new_xattr(num, di_key, name, name_len, data,
4813 					  data_len, ctx);
4814 	} else if (ret >= 0) {
4815 		if (data_len != found_data_len ||
4816 		    memcmp(data, found_data, data_len)) {
4817 			ret = __process_new_xattr(num, di_key, name, name_len,
4818 						  data, data_len, ctx);
4819 		} else {
4820 			ret = 0;
4821 		}
4822 	}
4823 
4824 	kfree(found_data);
4825 	return ret;
4826 }
4827 
__process_changed_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4828 static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
4829 					   const char *name, int name_len,
4830 					   const char *data, int data_len,
4831 					   void *ctx)
4832 {
4833 	int ret;
4834 	struct send_ctx *sctx = ctx;
4835 
4836 	ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
4837 			 name, name_len, NULL, NULL);
4838 	if (ret == -ENOENT)
4839 		ret = __process_deleted_xattr(num, di_key, name, name_len, data,
4840 					      data_len, ctx);
4841 	else if (ret >= 0)
4842 		ret = 0;
4843 
4844 	return ret;
4845 }
4846 
process_changed_xattr(struct send_ctx * sctx)4847 static int process_changed_xattr(struct send_ctx *sctx)
4848 {
4849 	int ret = 0;
4850 
4851 	ret = iterate_dir_item(sctx->send_root, sctx->left_path,
4852 			__process_changed_new_xattr, sctx);
4853 	if (ret < 0)
4854 		goto out;
4855 	ret = iterate_dir_item(sctx->parent_root, sctx->right_path,
4856 			__process_changed_deleted_xattr, sctx);
4857 
4858 out:
4859 	return ret;
4860 }
4861 
process_all_new_xattrs(struct send_ctx * sctx)4862 static int process_all_new_xattrs(struct send_ctx *sctx)
4863 {
4864 	int ret = 0;
4865 	int iter_ret = 0;
4866 	struct btrfs_root *root;
4867 	struct btrfs_path *path;
4868 	struct btrfs_key key;
4869 	struct btrfs_key found_key;
4870 
4871 	path = alloc_path_for_send();
4872 	if (!path)
4873 		return -ENOMEM;
4874 
4875 	root = sctx->send_root;
4876 
4877 	key.objectid = sctx->cmp_key->objectid;
4878 	key.type = BTRFS_XATTR_ITEM_KEY;
4879 	key.offset = 0;
4880 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4881 		if (found_key.objectid != key.objectid ||
4882 		    found_key.type != key.type) {
4883 			ret = 0;
4884 			break;
4885 		}
4886 
4887 		ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
4888 		if (ret < 0)
4889 			break;
4890 	}
4891 	/* Catch error found during iteration */
4892 	if (iter_ret < 0)
4893 		ret = iter_ret;
4894 
4895 	btrfs_free_path(path);
4896 	return ret;
4897 }
4898 
send_verity(struct send_ctx * sctx,struct fs_path * path,struct fsverity_descriptor * desc)4899 static int send_verity(struct send_ctx *sctx, struct fs_path *path,
4900 		       struct fsverity_descriptor *desc)
4901 {
4902 	int ret;
4903 
4904 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
4905 	if (ret < 0)
4906 		goto out;
4907 
4908 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4909 	TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
4910 			le8_to_cpu(desc->hash_algorithm));
4911 	TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
4912 			1U << le8_to_cpu(desc->log_blocksize));
4913 	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
4914 			le8_to_cpu(desc->salt_size));
4915 	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
4916 			le32_to_cpu(desc->sig_size));
4917 
4918 	ret = send_cmd(sctx);
4919 
4920 tlv_put_failure:
4921 out:
4922 	return ret;
4923 }
4924 
process_verity(struct send_ctx * sctx)4925 static int process_verity(struct send_ctx *sctx)
4926 {
4927 	int ret = 0;
4928 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4929 	struct inode *inode;
4930 	struct fs_path *p;
4931 
4932 	inode = btrfs_iget(fs_info->sb, sctx->cur_ino, sctx->send_root);
4933 	if (IS_ERR(inode))
4934 		return PTR_ERR(inode);
4935 
4936 	ret = btrfs_get_verity_descriptor(inode, NULL, 0);
4937 	if (ret < 0)
4938 		goto iput;
4939 
4940 	if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) {
4941 		ret = -EMSGSIZE;
4942 		goto iput;
4943 	}
4944 	if (!sctx->verity_descriptor) {
4945 		sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
4946 						   GFP_KERNEL);
4947 		if (!sctx->verity_descriptor) {
4948 			ret = -ENOMEM;
4949 			goto iput;
4950 		}
4951 	}
4952 
4953 	ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret);
4954 	if (ret < 0)
4955 		goto iput;
4956 
4957 	p = fs_path_alloc();
4958 	if (!p) {
4959 		ret = -ENOMEM;
4960 		goto iput;
4961 	}
4962 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
4963 	if (ret < 0)
4964 		goto free_path;
4965 
4966 	ret = send_verity(sctx, p, sctx->verity_descriptor);
4967 	if (ret < 0)
4968 		goto free_path;
4969 
4970 free_path:
4971 	fs_path_free(p);
4972 iput:
4973 	iput(inode);
4974 	return ret;
4975 }
4976 
max_send_read_size(const struct send_ctx * sctx)4977 static inline u64 max_send_read_size(const struct send_ctx *sctx)
4978 {
4979 	return sctx->send_max_size - SZ_16K;
4980 }
4981 
put_data_header(struct send_ctx * sctx,u32 len)4982 static int put_data_header(struct send_ctx *sctx, u32 len)
4983 {
4984 	if (WARN_ON_ONCE(sctx->put_data))
4985 		return -EINVAL;
4986 	sctx->put_data = true;
4987 	if (sctx->proto >= 2) {
4988 		/*
4989 		 * Since v2, the data attribute header doesn't include a length,
4990 		 * it is implicitly to the end of the command.
4991 		 */
4992 		if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
4993 			return -EOVERFLOW;
4994 		put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
4995 		sctx->send_size += sizeof(__le16);
4996 	} else {
4997 		struct btrfs_tlv_header *hdr;
4998 
4999 		if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
5000 			return -EOVERFLOW;
5001 		hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5002 		put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5003 		put_unaligned_le16(len, &hdr->tlv_len);
5004 		sctx->send_size += sizeof(*hdr);
5005 	}
5006 	return 0;
5007 }
5008 
put_file_data(struct send_ctx * sctx,u64 offset,u32 len)5009 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5010 {
5011 	struct btrfs_root *root = sctx->send_root;
5012 	struct btrfs_fs_info *fs_info = root->fs_info;
5013 	struct page *page;
5014 	pgoff_t index = offset >> PAGE_SHIFT;
5015 	pgoff_t last_index;
5016 	unsigned pg_offset = offset_in_page(offset);
5017 	int ret;
5018 
5019 	ret = put_data_header(sctx, len);
5020 	if (ret)
5021 		return ret;
5022 
5023 	last_index = (offset + len - 1) >> PAGE_SHIFT;
5024 
5025 	while (index <= last_index) {
5026 		unsigned cur_len = min_t(unsigned, len,
5027 					 PAGE_SIZE - pg_offset);
5028 
5029 		page = find_lock_page(sctx->cur_inode->i_mapping, index);
5030 		if (!page) {
5031 			page_cache_sync_readahead(sctx->cur_inode->i_mapping,
5032 						  &sctx->ra, NULL, index,
5033 						  last_index + 1 - index);
5034 
5035 			page = find_or_create_page(sctx->cur_inode->i_mapping,
5036 						   index, GFP_KERNEL);
5037 			if (!page) {
5038 				ret = -ENOMEM;
5039 				break;
5040 			}
5041 		}
5042 
5043 		if (PageReadahead(page))
5044 			page_cache_async_readahead(sctx->cur_inode->i_mapping,
5045 						   &sctx->ra, NULL, page_folio(page),
5046 						   index, last_index + 1 - index);
5047 
5048 		if (!PageUptodate(page)) {
5049 			btrfs_read_folio(NULL, page_folio(page));
5050 			lock_page(page);
5051 			if (!PageUptodate(page)) {
5052 				unlock_page(page);
5053 				btrfs_err(fs_info,
5054 			"send: IO error at offset %llu for inode %llu root %llu",
5055 					page_offset(page), sctx->cur_ino,
5056 					sctx->send_root->root_key.objectid);
5057 				put_page(page);
5058 				ret = -EIO;
5059 				break;
5060 			}
5061 		}
5062 
5063 		memcpy_from_page(sctx->send_buf + sctx->send_size, page,
5064 				 pg_offset, cur_len);
5065 		unlock_page(page);
5066 		put_page(page);
5067 		index++;
5068 		pg_offset = 0;
5069 		len -= cur_len;
5070 		sctx->send_size += cur_len;
5071 	}
5072 
5073 	return ret;
5074 }
5075 
5076 /*
5077  * Read some bytes from the current inode/file and send a write command to
5078  * user space.
5079  */
send_write(struct send_ctx * sctx,u64 offset,u32 len)5080 static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5081 {
5082 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5083 	int ret = 0;
5084 	struct fs_path *p;
5085 
5086 	p = fs_path_alloc();
5087 	if (!p)
5088 		return -ENOMEM;
5089 
5090 	btrfs_debug(fs_info, "send_write offset=%llu, len=%d", offset, len);
5091 
5092 	ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5093 	if (ret < 0)
5094 		goto out;
5095 
5096 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5097 	if (ret < 0)
5098 		goto out;
5099 
5100 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5101 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5102 	ret = put_file_data(sctx, offset, len);
5103 	if (ret < 0)
5104 		goto out;
5105 
5106 	ret = send_cmd(sctx);
5107 
5108 tlv_put_failure:
5109 out:
5110 	fs_path_free(p);
5111 	return ret;
5112 }
5113 
5114 /*
5115  * Send a clone command to user space.
5116  */
send_clone(struct send_ctx * sctx,u64 offset,u32 len,struct clone_root * clone_root)5117 static int send_clone(struct send_ctx *sctx,
5118 		      u64 offset, u32 len,
5119 		      struct clone_root *clone_root)
5120 {
5121 	int ret = 0;
5122 	struct fs_path *p;
5123 	u64 gen;
5124 
5125 	btrfs_debug(sctx->send_root->fs_info,
5126 		    "send_clone offset=%llu, len=%d, clone_root=%llu, clone_inode=%llu, clone_offset=%llu",
5127 		    offset, len, clone_root->root->root_key.objectid,
5128 		    clone_root->ino, clone_root->offset);
5129 
5130 	p = fs_path_alloc();
5131 	if (!p)
5132 		return -ENOMEM;
5133 
5134 	ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5135 	if (ret < 0)
5136 		goto out;
5137 
5138 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5139 	if (ret < 0)
5140 		goto out;
5141 
5142 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5143 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5144 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5145 
5146 	if (clone_root->root == sctx->send_root) {
5147 		ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
5148 		if (ret < 0)
5149 			goto out;
5150 		ret = get_cur_path(sctx, clone_root->ino, gen, p);
5151 	} else {
5152 		ret = get_inode_path(clone_root->root, clone_root->ino, p);
5153 	}
5154 	if (ret < 0)
5155 		goto out;
5156 
5157 	/*
5158 	 * If the parent we're using has a received_uuid set then use that as
5159 	 * our clone source as that is what we will look for when doing a
5160 	 * receive.
5161 	 *
5162 	 * This covers the case that we create a snapshot off of a received
5163 	 * subvolume and then use that as the parent and try to receive on a
5164 	 * different host.
5165 	 */
5166 	if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5167 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5168 			     clone_root->root->root_item.received_uuid);
5169 	else
5170 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5171 			     clone_root->root->root_item.uuid);
5172 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5173 		    btrfs_root_ctransid(&clone_root->root->root_item));
5174 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5175 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5176 			clone_root->offset);
5177 
5178 	ret = send_cmd(sctx);
5179 
5180 tlv_put_failure:
5181 out:
5182 	fs_path_free(p);
5183 	return ret;
5184 }
5185 
5186 /*
5187  * Send an update extent command to user space.
5188  */
send_update_extent(struct send_ctx * sctx,u64 offset,u32 len)5189 static int send_update_extent(struct send_ctx *sctx,
5190 			      u64 offset, u32 len)
5191 {
5192 	int ret = 0;
5193 	struct fs_path *p;
5194 
5195 	p = fs_path_alloc();
5196 	if (!p)
5197 		return -ENOMEM;
5198 
5199 	ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5200 	if (ret < 0)
5201 		goto out;
5202 
5203 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5204 	if (ret < 0)
5205 		goto out;
5206 
5207 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5208 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5209 	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5210 
5211 	ret = send_cmd(sctx);
5212 
5213 tlv_put_failure:
5214 out:
5215 	fs_path_free(p);
5216 	return ret;
5217 }
5218 
send_hole(struct send_ctx * sctx,u64 end)5219 static int send_hole(struct send_ctx *sctx, u64 end)
5220 {
5221 	struct fs_path *p = NULL;
5222 	u64 read_size = max_send_read_size(sctx);
5223 	u64 offset = sctx->cur_inode_last_extent;
5224 	int ret = 0;
5225 
5226 	/*
5227 	 * A hole that starts at EOF or beyond it. Since we do not yet support
5228 	 * fallocate (for extent preallocation and hole punching), sending a
5229 	 * write of zeroes starting at EOF or beyond would later require issuing
5230 	 * a truncate operation which would undo the write and achieve nothing.
5231 	 */
5232 	if (offset >= sctx->cur_inode_size)
5233 		return 0;
5234 
5235 	/*
5236 	 * Don't go beyond the inode's i_size due to prealloc extents that start
5237 	 * after the i_size.
5238 	 */
5239 	end = min_t(u64, end, sctx->cur_inode_size);
5240 
5241 	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5242 		return send_update_extent(sctx, offset, end - offset);
5243 
5244 	p = fs_path_alloc();
5245 	if (!p)
5246 		return -ENOMEM;
5247 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
5248 	if (ret < 0)
5249 		goto tlv_put_failure;
5250 	while (offset < end) {
5251 		u64 len = min(end - offset, read_size);
5252 
5253 		ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5254 		if (ret < 0)
5255 			break;
5256 		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5257 		TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5258 		ret = put_data_header(sctx, len);
5259 		if (ret < 0)
5260 			break;
5261 		memset(sctx->send_buf + sctx->send_size, 0, len);
5262 		sctx->send_size += len;
5263 		ret = send_cmd(sctx);
5264 		if (ret < 0)
5265 			break;
5266 		offset += len;
5267 	}
5268 	sctx->cur_inode_next_write_offset = offset;
5269 tlv_put_failure:
5270 	fs_path_free(p);
5271 	return ret;
5272 }
5273 
send_encoded_inline_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5274 static int send_encoded_inline_extent(struct send_ctx *sctx,
5275 				      struct btrfs_path *path, u64 offset,
5276 				      u64 len)
5277 {
5278 	struct btrfs_root *root = sctx->send_root;
5279 	struct btrfs_fs_info *fs_info = root->fs_info;
5280 	struct inode *inode;
5281 	struct fs_path *fspath;
5282 	struct extent_buffer *leaf = path->nodes[0];
5283 	struct btrfs_key key;
5284 	struct btrfs_file_extent_item *ei;
5285 	u64 ram_bytes;
5286 	size_t inline_size;
5287 	int ret;
5288 
5289 	inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
5290 	if (IS_ERR(inode))
5291 		return PTR_ERR(inode);
5292 
5293 	fspath = fs_path_alloc();
5294 	if (!fspath) {
5295 		ret = -ENOMEM;
5296 		goto out;
5297 	}
5298 
5299 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5300 	if (ret < 0)
5301 		goto out;
5302 
5303 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5304 	if (ret < 0)
5305 		goto out;
5306 
5307 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5308 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5309 	ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5310 	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5311 
5312 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5313 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5314 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5315 		    min(key.offset + ram_bytes - offset, len));
5316 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5317 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5318 	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5319 				btrfs_file_extent_compression(leaf, ei));
5320 	if (ret < 0)
5321 		goto out;
5322 	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5323 
5324 	ret = put_data_header(sctx, inline_size);
5325 	if (ret < 0)
5326 		goto out;
5327 	read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5328 			   btrfs_file_extent_inline_start(ei), inline_size);
5329 	sctx->send_size += inline_size;
5330 
5331 	ret = send_cmd(sctx);
5332 
5333 tlv_put_failure:
5334 out:
5335 	fs_path_free(fspath);
5336 	iput(inode);
5337 	return ret;
5338 }
5339 
send_encoded_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5340 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5341 			       u64 offset, u64 len)
5342 {
5343 	struct btrfs_root *root = sctx->send_root;
5344 	struct btrfs_fs_info *fs_info = root->fs_info;
5345 	struct inode *inode;
5346 	struct fs_path *fspath;
5347 	struct extent_buffer *leaf = path->nodes[0];
5348 	struct btrfs_key key;
5349 	struct btrfs_file_extent_item *ei;
5350 	u64 disk_bytenr, disk_num_bytes;
5351 	u32 data_offset;
5352 	struct btrfs_cmd_header *hdr;
5353 	u32 crc;
5354 	int ret;
5355 
5356 	inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
5357 	if (IS_ERR(inode))
5358 		return PTR_ERR(inode);
5359 
5360 	fspath = fs_path_alloc();
5361 	if (!fspath) {
5362 		ret = -ENOMEM;
5363 		goto out;
5364 	}
5365 
5366 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5367 	if (ret < 0)
5368 		goto out;
5369 
5370 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5371 	if (ret < 0)
5372 		goto out;
5373 
5374 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5375 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5376 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5377 	disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5378 
5379 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5380 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5381 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5382 		    min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5383 			len));
5384 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5385 		    btrfs_file_extent_ram_bytes(leaf, ei));
5386 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5387 		    offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5388 	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5389 				btrfs_file_extent_compression(leaf, ei));
5390 	if (ret < 0)
5391 		goto out;
5392 	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5393 	TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5394 
5395 	ret = put_data_header(sctx, disk_num_bytes);
5396 	if (ret < 0)
5397 		goto out;
5398 
5399 	/*
5400 	 * We want to do I/O directly into the send buffer, so get the next page
5401 	 * boundary in the send buffer. This means that there may be a gap
5402 	 * between the beginning of the command and the file data.
5403 	 */
5404 	data_offset = ALIGN(sctx->send_size, PAGE_SIZE);
5405 	if (data_offset > sctx->send_max_size ||
5406 	    sctx->send_max_size - data_offset < disk_num_bytes) {
5407 		ret = -EOVERFLOW;
5408 		goto out;
5409 	}
5410 
5411 	/*
5412 	 * Note that send_buf is a mapping of send_buf_pages, so this is really
5413 	 * reading into send_buf.
5414 	 */
5415 	ret = btrfs_encoded_read_regular_fill_pages(BTRFS_I(inode), offset,
5416 						    disk_bytenr, disk_num_bytes,
5417 						    sctx->send_buf_pages +
5418 						    (data_offset >> PAGE_SHIFT));
5419 	if (ret)
5420 		goto out;
5421 
5422 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5423 	hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5424 	hdr->crc = 0;
5425 	crc = btrfs_crc32c(0, sctx->send_buf, sctx->send_size);
5426 	crc = btrfs_crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5427 	hdr->crc = cpu_to_le32(crc);
5428 
5429 	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5430 			&sctx->send_off);
5431 	if (!ret) {
5432 		ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5433 				disk_num_bytes, &sctx->send_off);
5434 	}
5435 	sctx->send_size = 0;
5436 	sctx->put_data = false;
5437 
5438 tlv_put_failure:
5439 out:
5440 	fs_path_free(fspath);
5441 	iput(inode);
5442 	return ret;
5443 }
5444 
send_extent_data(struct send_ctx * sctx,struct btrfs_path * path,const u64 offset,const u64 len)5445 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5446 			    const u64 offset, const u64 len)
5447 {
5448 	const u64 end = offset + len;
5449 	struct extent_buffer *leaf = path->nodes[0];
5450 	struct btrfs_file_extent_item *ei;
5451 	u64 read_size = max_send_read_size(sctx);
5452 	u64 sent = 0;
5453 
5454 	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5455 		return send_update_extent(sctx, offset, len);
5456 
5457 	ei = btrfs_item_ptr(leaf, path->slots[0],
5458 			    struct btrfs_file_extent_item);
5459 	if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5460 	    btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5461 		bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5462 				  BTRFS_FILE_EXTENT_INLINE);
5463 
5464 		/*
5465 		 * Send the compressed extent unless the compressed data is
5466 		 * larger than the decompressed data. This can happen if we're
5467 		 * not sending the entire extent, either because it has been
5468 		 * partially overwritten/truncated or because this is a part of
5469 		 * the extent that we couldn't clone in clone_range().
5470 		 */
5471 		if (is_inline &&
5472 		    btrfs_file_extent_inline_item_len(leaf,
5473 						      path->slots[0]) <= len) {
5474 			return send_encoded_inline_extent(sctx, path, offset,
5475 							  len);
5476 		} else if (!is_inline &&
5477 			   btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5478 			return send_encoded_extent(sctx, path, offset, len);
5479 		}
5480 	}
5481 
5482 	if (sctx->cur_inode == NULL) {
5483 		struct btrfs_root *root = sctx->send_root;
5484 
5485 		sctx->cur_inode = btrfs_iget(root->fs_info->sb, sctx->cur_ino, root);
5486 		if (IS_ERR(sctx->cur_inode)) {
5487 			int err = PTR_ERR(sctx->cur_inode);
5488 
5489 			sctx->cur_inode = NULL;
5490 			return err;
5491 		}
5492 		memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5493 		file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5494 
5495 		/*
5496 		 * It's very likely there are no pages from this inode in the page
5497 		 * cache, so after reading extents and sending their data, we clean
5498 		 * the page cache to avoid trashing the page cache (adding pressure
5499 		 * to the page cache and forcing eviction of other data more useful
5500 		 * for applications).
5501 		 *
5502 		 * We decide if we should clean the page cache simply by checking
5503 		 * if the inode's mapping nrpages is 0 when we first open it, and
5504 		 * not by using something like filemap_range_has_page() before
5505 		 * reading an extent because when we ask the readahead code to
5506 		 * read a given file range, it may (and almost always does) read
5507 		 * pages from beyond that range (see the documentation for
5508 		 * page_cache_sync_readahead()), so it would not be reliable,
5509 		 * because after reading the first extent future calls to
5510 		 * filemap_range_has_page() would return true because the readahead
5511 		 * on the previous extent resulted in reading pages of the current
5512 		 * extent as well.
5513 		 */
5514 		sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5515 		sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5516 	}
5517 
5518 	while (sent < len) {
5519 		u64 size = min(len - sent, read_size);
5520 		int ret;
5521 
5522 		ret = send_write(sctx, offset + sent, size);
5523 		if (ret < 0)
5524 			return ret;
5525 		sent += size;
5526 	}
5527 
5528 	if (sctx->clean_page_cache && IS_ALIGNED(end, PAGE_SIZE)) {
5529 		/*
5530 		 * Always operate only on ranges that are a multiple of the page
5531 		 * size. This is not only to prevent zeroing parts of a page in
5532 		 * the case of subpage sector size, but also to guarantee we evict
5533 		 * pages, as passing a range that is smaller than page size does
5534 		 * not evict the respective page (only zeroes part of its content).
5535 		 *
5536 		 * Always start from the end offset of the last range cleared.
5537 		 * This is because the readahead code may (and very often does)
5538 		 * reads pages beyond the range we request for readahead. So if
5539 		 * we have an extent layout like this:
5540 		 *
5541 		 *            [ extent A ] [ extent B ] [ extent C ]
5542 		 *
5543 		 * When we ask page_cache_sync_readahead() to read extent A, it
5544 		 * may also trigger reads for pages of extent B. If we are doing
5545 		 * an incremental send and extent B has not changed between the
5546 		 * parent and send snapshots, some or all of its pages may end
5547 		 * up being read and placed in the page cache. So when truncating
5548 		 * the page cache we always start from the end offset of the
5549 		 * previously processed extent up to the end of the current
5550 		 * extent.
5551 		 */
5552 		truncate_inode_pages_range(&sctx->cur_inode->i_data,
5553 					   sctx->page_cache_clear_start,
5554 					   end - 1);
5555 		sctx->page_cache_clear_start = end;
5556 	}
5557 
5558 	return 0;
5559 }
5560 
5561 /*
5562  * Search for a capability xattr related to sctx->cur_ino. If the capability is
5563  * found, call send_set_xattr function to emit it.
5564  *
5565  * Return 0 if there isn't a capability, or when the capability was emitted
5566  * successfully, or < 0 if an error occurred.
5567  */
send_capabilities(struct send_ctx * sctx)5568 static int send_capabilities(struct send_ctx *sctx)
5569 {
5570 	struct fs_path *fspath = NULL;
5571 	struct btrfs_path *path;
5572 	struct btrfs_dir_item *di;
5573 	struct extent_buffer *leaf;
5574 	unsigned long data_ptr;
5575 	char *buf = NULL;
5576 	int buf_len;
5577 	int ret = 0;
5578 
5579 	path = alloc_path_for_send();
5580 	if (!path)
5581 		return -ENOMEM;
5582 
5583 	di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5584 				XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5585 	if (!di) {
5586 		/* There is no xattr for this inode */
5587 		goto out;
5588 	} else if (IS_ERR(di)) {
5589 		ret = PTR_ERR(di);
5590 		goto out;
5591 	}
5592 
5593 	leaf = path->nodes[0];
5594 	buf_len = btrfs_dir_data_len(leaf, di);
5595 
5596 	fspath = fs_path_alloc();
5597 	buf = kmalloc(buf_len, GFP_KERNEL);
5598 	if (!fspath || !buf) {
5599 		ret = -ENOMEM;
5600 		goto out;
5601 	}
5602 
5603 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath);
5604 	if (ret < 0)
5605 		goto out;
5606 
5607 	data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5608 	read_extent_buffer(leaf, buf, data_ptr, buf_len);
5609 
5610 	ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS,
5611 			strlen(XATTR_NAME_CAPS), buf, buf_len);
5612 out:
5613 	kfree(buf);
5614 	fs_path_free(fspath);
5615 	btrfs_free_path(path);
5616 	return ret;
5617 }
5618 
clone_range(struct send_ctx * sctx,struct btrfs_path * dst_path,struct clone_root * clone_root,const u64 disk_byte,u64 data_offset,u64 offset,u64 len)5619 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5620 		       struct clone_root *clone_root, const u64 disk_byte,
5621 		       u64 data_offset, u64 offset, u64 len)
5622 {
5623 	struct btrfs_path *path;
5624 	struct btrfs_key key;
5625 	int ret;
5626 	struct btrfs_inode_info info;
5627 	u64 clone_src_i_size = 0;
5628 
5629 	/*
5630 	 * Prevent cloning from a zero offset with a length matching the sector
5631 	 * size because in some scenarios this will make the receiver fail.
5632 	 *
5633 	 * For example, if in the source filesystem the extent at offset 0
5634 	 * has a length of sectorsize and it was written using direct IO, then
5635 	 * it can never be an inline extent (even if compression is enabled).
5636 	 * Then this extent can be cloned in the original filesystem to a non
5637 	 * zero file offset, but it may not be possible to clone in the
5638 	 * destination filesystem because it can be inlined due to compression
5639 	 * on the destination filesystem (as the receiver's write operations are
5640 	 * always done using buffered IO). The same happens when the original
5641 	 * filesystem does not have compression enabled but the destination
5642 	 * filesystem has.
5643 	 */
5644 	if (clone_root->offset == 0 &&
5645 	    len == sctx->send_root->fs_info->sectorsize)
5646 		return send_extent_data(sctx, dst_path, offset, len);
5647 
5648 	path = alloc_path_for_send();
5649 	if (!path)
5650 		return -ENOMEM;
5651 
5652 	/*
5653 	 * There are inodes that have extents that lie behind its i_size. Don't
5654 	 * accept clones from these extents.
5655 	 */
5656 	ret = get_inode_info(clone_root->root, clone_root->ino, &info);
5657 	btrfs_release_path(path);
5658 	if (ret < 0)
5659 		goto out;
5660 	clone_src_i_size = info.size;
5661 
5662 	/*
5663 	 * We can't send a clone operation for the entire range if we find
5664 	 * extent items in the respective range in the source file that
5665 	 * refer to different extents or if we find holes.
5666 	 * So check for that and do a mix of clone and regular write/copy
5667 	 * operations if needed.
5668 	 *
5669 	 * Example:
5670 	 *
5671 	 * mkfs.btrfs -f /dev/sda
5672 	 * mount /dev/sda /mnt
5673 	 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5674 	 * cp --reflink=always /mnt/foo /mnt/bar
5675 	 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5676 	 * btrfs subvolume snapshot -r /mnt /mnt/snap
5677 	 *
5678 	 * If when we send the snapshot and we are processing file bar (which
5679 	 * has a higher inode number than foo) we blindly send a clone operation
5680 	 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5681 	 * a file bar that matches the content of file foo - iow, doesn't match
5682 	 * the content from bar in the original filesystem.
5683 	 */
5684 	key.objectid = clone_root->ino;
5685 	key.type = BTRFS_EXTENT_DATA_KEY;
5686 	key.offset = clone_root->offset;
5687 	ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5688 	if (ret < 0)
5689 		goto out;
5690 	if (ret > 0 && path->slots[0] > 0) {
5691 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5692 		if (key.objectid == clone_root->ino &&
5693 		    key.type == BTRFS_EXTENT_DATA_KEY)
5694 			path->slots[0]--;
5695 	}
5696 
5697 	while (true) {
5698 		struct extent_buffer *leaf = path->nodes[0];
5699 		int slot = path->slots[0];
5700 		struct btrfs_file_extent_item *ei;
5701 		u8 type;
5702 		u64 ext_len;
5703 		u64 clone_len;
5704 		u64 clone_data_offset;
5705 		bool crossed_src_i_size = false;
5706 
5707 		if (slot >= btrfs_header_nritems(leaf)) {
5708 			ret = btrfs_next_leaf(clone_root->root, path);
5709 			if (ret < 0)
5710 				goto out;
5711 			else if (ret > 0)
5712 				break;
5713 			continue;
5714 		}
5715 
5716 		btrfs_item_key_to_cpu(leaf, &key, slot);
5717 
5718 		/*
5719 		 * We might have an implicit trailing hole (NO_HOLES feature
5720 		 * enabled). We deal with it after leaving this loop.
5721 		 */
5722 		if (key.objectid != clone_root->ino ||
5723 		    key.type != BTRFS_EXTENT_DATA_KEY)
5724 			break;
5725 
5726 		ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5727 		type = btrfs_file_extent_type(leaf, ei);
5728 		if (type == BTRFS_FILE_EXTENT_INLINE) {
5729 			ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5730 			ext_len = PAGE_ALIGN(ext_len);
5731 		} else {
5732 			ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5733 		}
5734 
5735 		if (key.offset + ext_len <= clone_root->offset)
5736 			goto next;
5737 
5738 		if (key.offset > clone_root->offset) {
5739 			/* Implicit hole, NO_HOLES feature enabled. */
5740 			u64 hole_len = key.offset - clone_root->offset;
5741 
5742 			if (hole_len > len)
5743 				hole_len = len;
5744 			ret = send_extent_data(sctx, dst_path, offset,
5745 					       hole_len);
5746 			if (ret < 0)
5747 				goto out;
5748 
5749 			len -= hole_len;
5750 			if (len == 0)
5751 				break;
5752 			offset += hole_len;
5753 			clone_root->offset += hole_len;
5754 			data_offset += hole_len;
5755 		}
5756 
5757 		if (key.offset >= clone_root->offset + len)
5758 			break;
5759 
5760 		if (key.offset >= clone_src_i_size)
5761 			break;
5762 
5763 		if (key.offset + ext_len > clone_src_i_size) {
5764 			ext_len = clone_src_i_size - key.offset;
5765 			crossed_src_i_size = true;
5766 		}
5767 
5768 		clone_data_offset = btrfs_file_extent_offset(leaf, ei);
5769 		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
5770 			clone_root->offset = key.offset;
5771 			if (clone_data_offset < data_offset &&
5772 				clone_data_offset + ext_len > data_offset) {
5773 				u64 extent_offset;
5774 
5775 				extent_offset = data_offset - clone_data_offset;
5776 				ext_len -= extent_offset;
5777 				clone_data_offset += extent_offset;
5778 				clone_root->offset += extent_offset;
5779 			}
5780 		}
5781 
5782 		clone_len = min_t(u64, ext_len, len);
5783 
5784 		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
5785 		    clone_data_offset == data_offset) {
5786 			const u64 src_end = clone_root->offset + clone_len;
5787 			const u64 sectorsize = SZ_64K;
5788 
5789 			/*
5790 			 * We can't clone the last block, when its size is not
5791 			 * sector size aligned, into the middle of a file. If we
5792 			 * do so, the receiver will get a failure (-EINVAL) when
5793 			 * trying to clone or will silently corrupt the data in
5794 			 * the destination file if it's on a kernel without the
5795 			 * fix introduced by commit ac765f83f1397646
5796 			 * ("Btrfs: fix data corruption due to cloning of eof
5797 			 * block).
5798 			 *
5799 			 * So issue a clone of the aligned down range plus a
5800 			 * regular write for the eof block, if we hit that case.
5801 			 *
5802 			 * Also, we use the maximum possible sector size, 64K,
5803 			 * because we don't know what's the sector size of the
5804 			 * filesystem that receives the stream, so we have to
5805 			 * assume the largest possible sector size.
5806 			 */
5807 			if (src_end == clone_src_i_size &&
5808 			    !IS_ALIGNED(src_end, sectorsize) &&
5809 			    offset + clone_len < sctx->cur_inode_size) {
5810 				u64 slen;
5811 
5812 				slen = ALIGN_DOWN(src_end - clone_root->offset,
5813 						  sectorsize);
5814 				if (slen > 0) {
5815 					ret = send_clone(sctx, offset, slen,
5816 							 clone_root);
5817 					if (ret < 0)
5818 						goto out;
5819 				}
5820 				ret = send_extent_data(sctx, dst_path,
5821 						       offset + slen,
5822 						       clone_len - slen);
5823 			} else {
5824 				ret = send_clone(sctx, offset, clone_len,
5825 						 clone_root);
5826 			}
5827 		} else if (crossed_src_i_size && clone_len < len) {
5828 			/*
5829 			 * If we are at i_size of the clone source inode and we
5830 			 * can not clone from it, terminate the loop. This is
5831 			 * to avoid sending two write operations, one with a
5832 			 * length matching clone_len and the final one after
5833 			 * this loop with a length of len - clone_len.
5834 			 *
5835 			 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
5836 			 * was passed to the send ioctl), this helps avoid
5837 			 * sending an encoded write for an offset that is not
5838 			 * sector size aligned, in case the i_size of the source
5839 			 * inode is not sector size aligned. That will make the
5840 			 * receiver fallback to decompression of the data and
5841 			 * writing it using regular buffered IO, therefore while
5842 			 * not incorrect, it's not optimal due decompression and
5843 			 * possible re-compression at the receiver.
5844 			 */
5845 			break;
5846 		} else {
5847 			ret = send_extent_data(sctx, dst_path, offset,
5848 					       clone_len);
5849 		}
5850 
5851 		if (ret < 0)
5852 			goto out;
5853 
5854 		len -= clone_len;
5855 		if (len == 0)
5856 			break;
5857 		offset += clone_len;
5858 		clone_root->offset += clone_len;
5859 
5860 		/*
5861 		 * If we are cloning from the file we are currently processing,
5862 		 * and using the send root as the clone root, we must stop once
5863 		 * the current clone offset reaches the current eof of the file
5864 		 * at the receiver, otherwise we would issue an invalid clone
5865 		 * operation (source range going beyond eof) and cause the
5866 		 * receiver to fail. So if we reach the current eof, bail out
5867 		 * and fallback to a regular write.
5868 		 */
5869 		if (clone_root->root == sctx->send_root &&
5870 		    clone_root->ino == sctx->cur_ino &&
5871 		    clone_root->offset >= sctx->cur_inode_next_write_offset)
5872 			break;
5873 
5874 		data_offset += clone_len;
5875 next:
5876 		path->slots[0]++;
5877 	}
5878 
5879 	if (len > 0)
5880 		ret = send_extent_data(sctx, dst_path, offset, len);
5881 	else
5882 		ret = 0;
5883 out:
5884 	btrfs_free_path(path);
5885 	return ret;
5886 }
5887 
send_write_or_clone(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key,struct clone_root * clone_root)5888 static int send_write_or_clone(struct send_ctx *sctx,
5889 			       struct btrfs_path *path,
5890 			       struct btrfs_key *key,
5891 			       struct clone_root *clone_root)
5892 {
5893 	int ret = 0;
5894 	u64 offset = key->offset;
5895 	u64 end;
5896 	u64 bs = sctx->send_root->fs_info->sb->s_blocksize;
5897 
5898 	end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
5899 	if (offset >= end)
5900 		return 0;
5901 
5902 	if (clone_root && IS_ALIGNED(end, bs)) {
5903 		struct btrfs_file_extent_item *ei;
5904 		u64 disk_byte;
5905 		u64 data_offset;
5906 
5907 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
5908 				    struct btrfs_file_extent_item);
5909 		disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
5910 		data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
5911 		ret = clone_range(sctx, path, clone_root, disk_byte,
5912 				  data_offset, offset, end - offset);
5913 	} else {
5914 		ret = send_extent_data(sctx, path, offset, end - offset);
5915 	}
5916 	sctx->cur_inode_next_write_offset = end;
5917 	return ret;
5918 }
5919 
is_extent_unchanged(struct send_ctx * sctx,struct btrfs_path * left_path,struct btrfs_key * ekey)5920 static int is_extent_unchanged(struct send_ctx *sctx,
5921 			       struct btrfs_path *left_path,
5922 			       struct btrfs_key *ekey)
5923 {
5924 	int ret = 0;
5925 	struct btrfs_key key;
5926 	struct btrfs_path *path = NULL;
5927 	struct extent_buffer *eb;
5928 	int slot;
5929 	struct btrfs_key found_key;
5930 	struct btrfs_file_extent_item *ei;
5931 	u64 left_disknr;
5932 	u64 right_disknr;
5933 	u64 left_offset;
5934 	u64 right_offset;
5935 	u64 left_offset_fixed;
5936 	u64 left_len;
5937 	u64 right_len;
5938 	u64 left_gen;
5939 	u64 right_gen;
5940 	u8 left_type;
5941 	u8 right_type;
5942 
5943 	path = alloc_path_for_send();
5944 	if (!path)
5945 		return -ENOMEM;
5946 
5947 	eb = left_path->nodes[0];
5948 	slot = left_path->slots[0];
5949 	ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
5950 	left_type = btrfs_file_extent_type(eb, ei);
5951 
5952 	if (left_type != BTRFS_FILE_EXTENT_REG) {
5953 		ret = 0;
5954 		goto out;
5955 	}
5956 	left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
5957 	left_len = btrfs_file_extent_num_bytes(eb, ei);
5958 	left_offset = btrfs_file_extent_offset(eb, ei);
5959 	left_gen = btrfs_file_extent_generation(eb, ei);
5960 
5961 	/*
5962 	 * Following comments will refer to these graphics. L is the left
5963 	 * extents which we are checking at the moment. 1-8 are the right
5964 	 * extents that we iterate.
5965 	 *
5966 	 *       |-----L-----|
5967 	 * |-1-|-2a-|-3-|-4-|-5-|-6-|
5968 	 *
5969 	 *       |-----L-----|
5970 	 * |--1--|-2b-|...(same as above)
5971 	 *
5972 	 * Alternative situation. Happens on files where extents got split.
5973 	 *       |-----L-----|
5974 	 * |-----------7-----------|-6-|
5975 	 *
5976 	 * Alternative situation. Happens on files which got larger.
5977 	 *       |-----L-----|
5978 	 * |-8-|
5979 	 * Nothing follows after 8.
5980 	 */
5981 
5982 	key.objectid = ekey->objectid;
5983 	key.type = BTRFS_EXTENT_DATA_KEY;
5984 	key.offset = ekey->offset;
5985 	ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
5986 	if (ret < 0)
5987 		goto out;
5988 	if (ret) {
5989 		ret = 0;
5990 		goto out;
5991 	}
5992 
5993 	/*
5994 	 * Handle special case where the right side has no extents at all.
5995 	 */
5996 	eb = path->nodes[0];
5997 	slot = path->slots[0];
5998 	btrfs_item_key_to_cpu(eb, &found_key, slot);
5999 	if (found_key.objectid != key.objectid ||
6000 	    found_key.type != key.type) {
6001 		/* If we're a hole then just pretend nothing changed */
6002 		ret = (left_disknr) ? 0 : 1;
6003 		goto out;
6004 	}
6005 
6006 	/*
6007 	 * We're now on 2a, 2b or 7.
6008 	 */
6009 	key = found_key;
6010 	while (key.offset < ekey->offset + left_len) {
6011 		ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6012 		right_type = btrfs_file_extent_type(eb, ei);
6013 		if (right_type != BTRFS_FILE_EXTENT_REG &&
6014 		    right_type != BTRFS_FILE_EXTENT_INLINE) {
6015 			ret = 0;
6016 			goto out;
6017 		}
6018 
6019 		if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6020 			right_len = btrfs_file_extent_ram_bytes(eb, ei);
6021 			right_len = PAGE_ALIGN(right_len);
6022 		} else {
6023 			right_len = btrfs_file_extent_num_bytes(eb, ei);
6024 		}
6025 
6026 		/*
6027 		 * Are we at extent 8? If yes, we know the extent is changed.
6028 		 * This may only happen on the first iteration.
6029 		 */
6030 		if (found_key.offset + right_len <= ekey->offset) {
6031 			/* If we're a hole just pretend nothing changed */
6032 			ret = (left_disknr) ? 0 : 1;
6033 			goto out;
6034 		}
6035 
6036 		/*
6037 		 * We just wanted to see if when we have an inline extent, what
6038 		 * follows it is a regular extent (wanted to check the above
6039 		 * condition for inline extents too). This should normally not
6040 		 * happen but it's possible for example when we have an inline
6041 		 * compressed extent representing data with a size matching
6042 		 * the page size (currently the same as sector size).
6043 		 */
6044 		if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6045 			ret = 0;
6046 			goto out;
6047 		}
6048 
6049 		right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6050 		right_offset = btrfs_file_extent_offset(eb, ei);
6051 		right_gen = btrfs_file_extent_generation(eb, ei);
6052 
6053 		left_offset_fixed = left_offset;
6054 		if (key.offset < ekey->offset) {
6055 			/* Fix the right offset for 2a and 7. */
6056 			right_offset += ekey->offset - key.offset;
6057 		} else {
6058 			/* Fix the left offset for all behind 2a and 2b */
6059 			left_offset_fixed += key.offset - ekey->offset;
6060 		}
6061 
6062 		/*
6063 		 * Check if we have the same extent.
6064 		 */
6065 		if (left_disknr != right_disknr ||
6066 		    left_offset_fixed != right_offset ||
6067 		    left_gen != right_gen) {
6068 			ret = 0;
6069 			goto out;
6070 		}
6071 
6072 		/*
6073 		 * Go to the next extent.
6074 		 */
6075 		ret = btrfs_next_item(sctx->parent_root, path);
6076 		if (ret < 0)
6077 			goto out;
6078 		if (!ret) {
6079 			eb = path->nodes[0];
6080 			slot = path->slots[0];
6081 			btrfs_item_key_to_cpu(eb, &found_key, slot);
6082 		}
6083 		if (ret || found_key.objectid != key.objectid ||
6084 		    found_key.type != key.type) {
6085 			key.offset += right_len;
6086 			break;
6087 		}
6088 		if (found_key.offset != key.offset + right_len) {
6089 			ret = 0;
6090 			goto out;
6091 		}
6092 		key = found_key;
6093 	}
6094 
6095 	/*
6096 	 * We're now behind the left extent (treat as unchanged) or at the end
6097 	 * of the right side (treat as changed).
6098 	 */
6099 	if (key.offset >= ekey->offset + left_len)
6100 		ret = 1;
6101 	else
6102 		ret = 0;
6103 
6104 
6105 out:
6106 	btrfs_free_path(path);
6107 	return ret;
6108 }
6109 
get_last_extent(struct send_ctx * sctx,u64 offset)6110 static int get_last_extent(struct send_ctx *sctx, u64 offset)
6111 {
6112 	struct btrfs_path *path;
6113 	struct btrfs_root *root = sctx->send_root;
6114 	struct btrfs_key key;
6115 	int ret;
6116 
6117 	path = alloc_path_for_send();
6118 	if (!path)
6119 		return -ENOMEM;
6120 
6121 	sctx->cur_inode_last_extent = 0;
6122 
6123 	key.objectid = sctx->cur_ino;
6124 	key.type = BTRFS_EXTENT_DATA_KEY;
6125 	key.offset = offset;
6126 	ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6127 	if (ret < 0)
6128 		goto out;
6129 	ret = 0;
6130 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6131 	if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6132 		goto out;
6133 
6134 	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6135 out:
6136 	btrfs_free_path(path);
6137 	return ret;
6138 }
6139 
range_is_hole_in_parent(struct send_ctx * sctx,const u64 start,const u64 end)6140 static int range_is_hole_in_parent(struct send_ctx *sctx,
6141 				   const u64 start,
6142 				   const u64 end)
6143 {
6144 	struct btrfs_path *path;
6145 	struct btrfs_key key;
6146 	struct btrfs_root *root = sctx->parent_root;
6147 	u64 search_start = start;
6148 	int ret;
6149 
6150 	path = alloc_path_for_send();
6151 	if (!path)
6152 		return -ENOMEM;
6153 
6154 	key.objectid = sctx->cur_ino;
6155 	key.type = BTRFS_EXTENT_DATA_KEY;
6156 	key.offset = search_start;
6157 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6158 	if (ret < 0)
6159 		goto out;
6160 	if (ret > 0 && path->slots[0] > 0)
6161 		path->slots[0]--;
6162 
6163 	while (search_start < end) {
6164 		struct extent_buffer *leaf = path->nodes[0];
6165 		int slot = path->slots[0];
6166 		struct btrfs_file_extent_item *fi;
6167 		u64 extent_end;
6168 
6169 		if (slot >= btrfs_header_nritems(leaf)) {
6170 			ret = btrfs_next_leaf(root, path);
6171 			if (ret < 0)
6172 				goto out;
6173 			else if (ret > 0)
6174 				break;
6175 			continue;
6176 		}
6177 
6178 		btrfs_item_key_to_cpu(leaf, &key, slot);
6179 		if (key.objectid < sctx->cur_ino ||
6180 		    key.type < BTRFS_EXTENT_DATA_KEY)
6181 			goto next;
6182 		if (key.objectid > sctx->cur_ino ||
6183 		    key.type > BTRFS_EXTENT_DATA_KEY ||
6184 		    key.offset >= end)
6185 			break;
6186 
6187 		fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6188 		extent_end = btrfs_file_extent_end(path);
6189 		if (extent_end <= start)
6190 			goto next;
6191 		if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6192 			search_start = extent_end;
6193 			goto next;
6194 		}
6195 		ret = 0;
6196 		goto out;
6197 next:
6198 		path->slots[0]++;
6199 	}
6200 	ret = 1;
6201 out:
6202 	btrfs_free_path(path);
6203 	return ret;
6204 }
6205 
maybe_send_hole(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6206 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6207 			   struct btrfs_key *key)
6208 {
6209 	int ret = 0;
6210 
6211 	if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6212 		return 0;
6213 
6214 	if (sctx->cur_inode_last_extent == (u64)-1) {
6215 		ret = get_last_extent(sctx, key->offset - 1);
6216 		if (ret)
6217 			return ret;
6218 	}
6219 
6220 	if (path->slots[0] == 0 &&
6221 	    sctx->cur_inode_last_extent < key->offset) {
6222 		/*
6223 		 * We might have skipped entire leafs that contained only
6224 		 * file extent items for our current inode. These leafs have
6225 		 * a generation number smaller (older) than the one in the
6226 		 * current leaf and the leaf our last extent came from, and
6227 		 * are located between these 2 leafs.
6228 		 */
6229 		ret = get_last_extent(sctx, key->offset - 1);
6230 		if (ret)
6231 			return ret;
6232 	}
6233 
6234 	if (sctx->cur_inode_last_extent < key->offset) {
6235 		ret = range_is_hole_in_parent(sctx,
6236 					      sctx->cur_inode_last_extent,
6237 					      key->offset);
6238 		if (ret < 0)
6239 			return ret;
6240 		else if (ret == 0)
6241 			ret = send_hole(sctx, key->offset);
6242 		else
6243 			ret = 0;
6244 	}
6245 	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6246 	return ret;
6247 }
6248 
process_extent(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6249 static int process_extent(struct send_ctx *sctx,
6250 			  struct btrfs_path *path,
6251 			  struct btrfs_key *key)
6252 {
6253 	struct clone_root *found_clone = NULL;
6254 	int ret = 0;
6255 
6256 	if (S_ISLNK(sctx->cur_inode_mode))
6257 		return 0;
6258 
6259 	if (sctx->parent_root && !sctx->cur_inode_new) {
6260 		ret = is_extent_unchanged(sctx, path, key);
6261 		if (ret < 0)
6262 			goto out;
6263 		if (ret) {
6264 			ret = 0;
6265 			goto out_hole;
6266 		}
6267 	} else {
6268 		struct btrfs_file_extent_item *ei;
6269 		u8 type;
6270 
6271 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6272 				    struct btrfs_file_extent_item);
6273 		type = btrfs_file_extent_type(path->nodes[0], ei);
6274 		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6275 		    type == BTRFS_FILE_EXTENT_REG) {
6276 			/*
6277 			 * The send spec does not have a prealloc command yet,
6278 			 * so just leave a hole for prealloc'ed extents until
6279 			 * we have enough commands queued up to justify rev'ing
6280 			 * the send spec.
6281 			 */
6282 			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
6283 				ret = 0;
6284 				goto out;
6285 			}
6286 
6287 			/* Have a hole, just skip it. */
6288 			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
6289 				ret = 0;
6290 				goto out;
6291 			}
6292 		}
6293 	}
6294 
6295 	ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6296 			sctx->cur_inode_size, &found_clone);
6297 	if (ret != -ENOENT && ret < 0)
6298 		goto out;
6299 
6300 	ret = send_write_or_clone(sctx, path, key, found_clone);
6301 	if (ret)
6302 		goto out;
6303 out_hole:
6304 	ret = maybe_send_hole(sctx, path, key);
6305 out:
6306 	return ret;
6307 }
6308 
process_all_extents(struct send_ctx * sctx)6309 static int process_all_extents(struct send_ctx *sctx)
6310 {
6311 	int ret = 0;
6312 	int iter_ret = 0;
6313 	struct btrfs_root *root;
6314 	struct btrfs_path *path;
6315 	struct btrfs_key key;
6316 	struct btrfs_key found_key;
6317 
6318 	root = sctx->send_root;
6319 	path = alloc_path_for_send();
6320 	if (!path)
6321 		return -ENOMEM;
6322 
6323 	key.objectid = sctx->cmp_key->objectid;
6324 	key.type = BTRFS_EXTENT_DATA_KEY;
6325 	key.offset = 0;
6326 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
6327 		if (found_key.objectid != key.objectid ||
6328 		    found_key.type != key.type) {
6329 			ret = 0;
6330 			break;
6331 		}
6332 
6333 		ret = process_extent(sctx, path, &found_key);
6334 		if (ret < 0)
6335 			break;
6336 	}
6337 	/* Catch error found during iteration */
6338 	if (iter_ret < 0)
6339 		ret = iter_ret;
6340 
6341 	btrfs_free_path(path);
6342 	return ret;
6343 }
6344 
process_recorded_refs_if_needed(struct send_ctx * sctx,int at_end,int * pending_move,int * refs_processed)6345 static int process_recorded_refs_if_needed(struct send_ctx *sctx, int at_end,
6346 					   int *pending_move,
6347 					   int *refs_processed)
6348 {
6349 	int ret = 0;
6350 
6351 	if (sctx->cur_ino == 0)
6352 		goto out;
6353 	if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6354 	    sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6355 		goto out;
6356 	if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6357 		goto out;
6358 
6359 	ret = process_recorded_refs(sctx, pending_move);
6360 	if (ret < 0)
6361 		goto out;
6362 
6363 	*refs_processed = 1;
6364 out:
6365 	return ret;
6366 }
6367 
finish_inode_if_needed(struct send_ctx * sctx,int at_end)6368 static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
6369 {
6370 	int ret = 0;
6371 	struct btrfs_inode_info info;
6372 	u64 left_mode;
6373 	u64 left_uid;
6374 	u64 left_gid;
6375 	u64 left_fileattr;
6376 	u64 right_mode;
6377 	u64 right_uid;
6378 	u64 right_gid;
6379 	u64 right_fileattr;
6380 	int need_chmod = 0;
6381 	int need_chown = 0;
6382 	bool need_fileattr = false;
6383 	int need_truncate = 1;
6384 	int pending_move = 0;
6385 	int refs_processed = 0;
6386 
6387 	if (sctx->ignore_cur_inode)
6388 		return 0;
6389 
6390 	ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6391 					      &refs_processed);
6392 	if (ret < 0)
6393 		goto out;
6394 
6395 	/*
6396 	 * We have processed the refs and thus need to advance send_progress.
6397 	 * Now, calls to get_cur_xxx will take the updated refs of the current
6398 	 * inode into account.
6399 	 *
6400 	 * On the other hand, if our current inode is a directory and couldn't
6401 	 * be moved/renamed because its parent was renamed/moved too and it has
6402 	 * a higher inode number, we can only move/rename our current inode
6403 	 * after we moved/renamed its parent. Therefore in this case operate on
6404 	 * the old path (pre move/rename) of our current inode, and the
6405 	 * move/rename will be performed later.
6406 	 */
6407 	if (refs_processed && !pending_move)
6408 		sctx->send_progress = sctx->cur_ino + 1;
6409 
6410 	if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6411 		goto out;
6412 	if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6413 		goto out;
6414 	ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
6415 	if (ret < 0)
6416 		goto out;
6417 	left_mode = info.mode;
6418 	left_uid = info.uid;
6419 	left_gid = info.gid;
6420 	left_fileattr = info.fileattr;
6421 
6422 	if (!sctx->parent_root || sctx->cur_inode_new) {
6423 		need_chown = 1;
6424 		if (!S_ISLNK(sctx->cur_inode_mode))
6425 			need_chmod = 1;
6426 		if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6427 			need_truncate = 0;
6428 	} else {
6429 		u64 old_size;
6430 
6431 		ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
6432 		if (ret < 0)
6433 			goto out;
6434 		old_size = info.size;
6435 		right_mode = info.mode;
6436 		right_uid = info.uid;
6437 		right_gid = info.gid;
6438 		right_fileattr = info.fileattr;
6439 
6440 		if (left_uid != right_uid || left_gid != right_gid)
6441 			need_chown = 1;
6442 		if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6443 			need_chmod = 1;
6444 		if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6445 			need_fileattr = true;
6446 		if ((old_size == sctx->cur_inode_size) ||
6447 		    (sctx->cur_inode_size > old_size &&
6448 		     sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6449 			need_truncate = 0;
6450 	}
6451 
6452 	if (S_ISREG(sctx->cur_inode_mode)) {
6453 		if (need_send_hole(sctx)) {
6454 			if (sctx->cur_inode_last_extent == (u64)-1 ||
6455 			    sctx->cur_inode_last_extent <
6456 			    sctx->cur_inode_size) {
6457 				ret = get_last_extent(sctx, (u64)-1);
6458 				if (ret)
6459 					goto out;
6460 			}
6461 			if (sctx->cur_inode_last_extent <
6462 			    sctx->cur_inode_size) {
6463 				ret = send_hole(sctx, sctx->cur_inode_size);
6464 				if (ret)
6465 					goto out;
6466 			}
6467 		}
6468 		if (need_truncate) {
6469 			ret = send_truncate(sctx, sctx->cur_ino,
6470 					    sctx->cur_inode_gen,
6471 					    sctx->cur_inode_size);
6472 			if (ret < 0)
6473 				goto out;
6474 		}
6475 	}
6476 
6477 	if (need_chown) {
6478 		ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6479 				left_uid, left_gid);
6480 		if (ret < 0)
6481 			goto out;
6482 	}
6483 	if (need_chmod) {
6484 		ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6485 				left_mode);
6486 		if (ret < 0)
6487 			goto out;
6488 	}
6489 	if (need_fileattr) {
6490 		ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6491 				    left_fileattr);
6492 		if (ret < 0)
6493 			goto out;
6494 	}
6495 
6496 	if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6497 	    && sctx->cur_inode_needs_verity) {
6498 		ret = process_verity(sctx);
6499 		if (ret < 0)
6500 			goto out;
6501 	}
6502 
6503 	ret = send_capabilities(sctx);
6504 	if (ret < 0)
6505 		goto out;
6506 
6507 	/*
6508 	 * If other directory inodes depended on our current directory
6509 	 * inode's move/rename, now do their move/rename operations.
6510 	 */
6511 	if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6512 		ret = apply_children_dir_moves(sctx);
6513 		if (ret)
6514 			goto out;
6515 		/*
6516 		 * Need to send that every time, no matter if it actually
6517 		 * changed between the two trees as we have done changes to
6518 		 * the inode before. If our inode is a directory and it's
6519 		 * waiting to be moved/renamed, we will send its utimes when
6520 		 * it's moved/renamed, therefore we don't need to do it here.
6521 		 */
6522 		sctx->send_progress = sctx->cur_ino + 1;
6523 		ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6524 		if (ret < 0)
6525 			goto out;
6526 	}
6527 
6528 out:
6529 	return ret;
6530 }
6531 
close_current_inode(struct send_ctx * sctx)6532 static void close_current_inode(struct send_ctx *sctx)
6533 {
6534 	u64 i_size;
6535 
6536 	if (sctx->cur_inode == NULL)
6537 		return;
6538 
6539 	i_size = i_size_read(sctx->cur_inode);
6540 
6541 	/*
6542 	 * If we are doing an incremental send, we may have extents between the
6543 	 * last processed extent and the i_size that have not been processed
6544 	 * because they haven't changed but we may have read some of their pages
6545 	 * through readahead, see the comments at send_extent_data().
6546 	 */
6547 	if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6548 		truncate_inode_pages_range(&sctx->cur_inode->i_data,
6549 					   sctx->page_cache_clear_start,
6550 					   round_up(i_size, PAGE_SIZE) - 1);
6551 
6552 	iput(sctx->cur_inode);
6553 	sctx->cur_inode = NULL;
6554 }
6555 
changed_inode(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6556 static int changed_inode(struct send_ctx *sctx,
6557 			 enum btrfs_compare_tree_result result)
6558 {
6559 	int ret = 0;
6560 	struct btrfs_key *key = sctx->cmp_key;
6561 	struct btrfs_inode_item *left_ii = NULL;
6562 	struct btrfs_inode_item *right_ii = NULL;
6563 	u64 left_gen = 0;
6564 	u64 right_gen = 0;
6565 
6566 	close_current_inode(sctx);
6567 
6568 	sctx->cur_ino = key->objectid;
6569 	sctx->cur_inode_new_gen = false;
6570 	sctx->cur_inode_last_extent = (u64)-1;
6571 	sctx->cur_inode_next_write_offset = 0;
6572 	sctx->ignore_cur_inode = false;
6573 
6574 	/*
6575 	 * Set send_progress to current inode. This will tell all get_cur_xxx
6576 	 * functions that the current inode's refs are not updated yet. Later,
6577 	 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6578 	 */
6579 	sctx->send_progress = sctx->cur_ino;
6580 
6581 	if (result == BTRFS_COMPARE_TREE_NEW ||
6582 	    result == BTRFS_COMPARE_TREE_CHANGED) {
6583 		left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6584 				sctx->left_path->slots[0],
6585 				struct btrfs_inode_item);
6586 		left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6587 				left_ii);
6588 	} else {
6589 		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6590 				sctx->right_path->slots[0],
6591 				struct btrfs_inode_item);
6592 		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6593 				right_ii);
6594 	}
6595 	if (result == BTRFS_COMPARE_TREE_CHANGED) {
6596 		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6597 				sctx->right_path->slots[0],
6598 				struct btrfs_inode_item);
6599 
6600 		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6601 				right_ii);
6602 
6603 		/*
6604 		 * The cur_ino = root dir case is special here. We can't treat
6605 		 * the inode as deleted+reused because it would generate a
6606 		 * stream that tries to delete/mkdir the root dir.
6607 		 */
6608 		if (left_gen != right_gen &&
6609 		    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6610 			sctx->cur_inode_new_gen = true;
6611 	}
6612 
6613 	/*
6614 	 * Normally we do not find inodes with a link count of zero (orphans)
6615 	 * because the most common case is to create a snapshot and use it
6616 	 * for a send operation. However other less common use cases involve
6617 	 * using a subvolume and send it after turning it to RO mode just
6618 	 * after deleting all hard links of a file while holding an open
6619 	 * file descriptor against it or turning a RO snapshot into RW mode,
6620 	 * keep an open file descriptor against a file, delete it and then
6621 	 * turn the snapshot back to RO mode before using it for a send
6622 	 * operation. The former is what the receiver operation does.
6623 	 * Therefore, if we want to send these snapshots soon after they're
6624 	 * received, we need to handle orphan inodes as well. Moreover, orphans
6625 	 * can appear not only in the send snapshot but also in the parent
6626 	 * snapshot. Here are several cases:
6627 	 *
6628 	 * Case 1: BTRFS_COMPARE_TREE_NEW
6629 	 *       |  send snapshot  | action
6630 	 * --------------------------------
6631 	 * nlink |        0        | ignore
6632 	 *
6633 	 * Case 2: BTRFS_COMPARE_TREE_DELETED
6634 	 *       | parent snapshot | action
6635 	 * ----------------------------------
6636 	 * nlink |        0        | as usual
6637 	 * Note: No unlinks will be sent because there're no paths for it.
6638 	 *
6639 	 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6640 	 *           |       | parent snapshot | send snapshot | action
6641 	 * -----------------------------------------------------------------------
6642 	 * subcase 1 | nlink |        0        |       0       | ignore
6643 	 * subcase 2 | nlink |       >0        |       0       | new_gen(deletion)
6644 	 * subcase 3 | nlink |        0        |      >0       | new_gen(creation)
6645 	 *
6646 	 */
6647 	if (result == BTRFS_COMPARE_TREE_NEW) {
6648 		if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
6649 			sctx->ignore_cur_inode = true;
6650 			goto out;
6651 		}
6652 		sctx->cur_inode_gen = left_gen;
6653 		sctx->cur_inode_new = true;
6654 		sctx->cur_inode_deleted = false;
6655 		sctx->cur_inode_size = btrfs_inode_size(
6656 				sctx->left_path->nodes[0], left_ii);
6657 		sctx->cur_inode_mode = btrfs_inode_mode(
6658 				sctx->left_path->nodes[0], left_ii);
6659 		sctx->cur_inode_rdev = btrfs_inode_rdev(
6660 				sctx->left_path->nodes[0], left_ii);
6661 		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6662 			ret = send_create_inode_if_needed(sctx);
6663 	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
6664 		sctx->cur_inode_gen = right_gen;
6665 		sctx->cur_inode_new = false;
6666 		sctx->cur_inode_deleted = true;
6667 		sctx->cur_inode_size = btrfs_inode_size(
6668 				sctx->right_path->nodes[0], right_ii);
6669 		sctx->cur_inode_mode = btrfs_inode_mode(
6670 				sctx->right_path->nodes[0], right_ii);
6671 	} else if (result == BTRFS_COMPARE_TREE_CHANGED) {
6672 		u32 new_nlinks, old_nlinks;
6673 
6674 		new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
6675 		old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
6676 		if (new_nlinks == 0 && old_nlinks == 0) {
6677 			sctx->ignore_cur_inode = true;
6678 			goto out;
6679 		} else if (new_nlinks == 0 || old_nlinks == 0) {
6680 			sctx->cur_inode_new_gen = 1;
6681 		}
6682 		/*
6683 		 * We need to do some special handling in case the inode was
6684 		 * reported as changed with a changed generation number. This
6685 		 * means that the original inode was deleted and new inode
6686 		 * reused the same inum. So we have to treat the old inode as
6687 		 * deleted and the new one as new.
6688 		 */
6689 		if (sctx->cur_inode_new_gen) {
6690 			/*
6691 			 * First, process the inode as if it was deleted.
6692 			 */
6693 			if (old_nlinks > 0) {
6694 				sctx->cur_inode_gen = right_gen;
6695 				sctx->cur_inode_new = false;
6696 				sctx->cur_inode_deleted = true;
6697 				sctx->cur_inode_size = btrfs_inode_size(
6698 						sctx->right_path->nodes[0], right_ii);
6699 				sctx->cur_inode_mode = btrfs_inode_mode(
6700 						sctx->right_path->nodes[0], right_ii);
6701 				ret = process_all_refs(sctx,
6702 						BTRFS_COMPARE_TREE_DELETED);
6703 				if (ret < 0)
6704 					goto out;
6705 			}
6706 
6707 			/*
6708 			 * Now process the inode as if it was new.
6709 			 */
6710 			if (new_nlinks > 0) {
6711 				sctx->cur_inode_gen = left_gen;
6712 				sctx->cur_inode_new = true;
6713 				sctx->cur_inode_deleted = false;
6714 				sctx->cur_inode_size = btrfs_inode_size(
6715 						sctx->left_path->nodes[0],
6716 						left_ii);
6717 				sctx->cur_inode_mode = btrfs_inode_mode(
6718 						sctx->left_path->nodes[0],
6719 						left_ii);
6720 				sctx->cur_inode_rdev = btrfs_inode_rdev(
6721 						sctx->left_path->nodes[0],
6722 						left_ii);
6723 				ret = send_create_inode_if_needed(sctx);
6724 				if (ret < 0)
6725 					goto out;
6726 
6727 				ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
6728 				if (ret < 0)
6729 					goto out;
6730 				/*
6731 				 * Advance send_progress now as we did not get
6732 				 * into process_recorded_refs_if_needed in the
6733 				 * new_gen case.
6734 				 */
6735 				sctx->send_progress = sctx->cur_ino + 1;
6736 
6737 				/*
6738 				 * Now process all extents and xattrs of the
6739 				 * inode as if they were all new.
6740 				 */
6741 				ret = process_all_extents(sctx);
6742 				if (ret < 0)
6743 					goto out;
6744 				ret = process_all_new_xattrs(sctx);
6745 				if (ret < 0)
6746 					goto out;
6747 			}
6748 		} else {
6749 			sctx->cur_inode_gen = left_gen;
6750 			sctx->cur_inode_new = false;
6751 			sctx->cur_inode_new_gen = false;
6752 			sctx->cur_inode_deleted = false;
6753 			sctx->cur_inode_size = btrfs_inode_size(
6754 					sctx->left_path->nodes[0], left_ii);
6755 			sctx->cur_inode_mode = btrfs_inode_mode(
6756 					sctx->left_path->nodes[0], left_ii);
6757 		}
6758 	}
6759 
6760 out:
6761 	return ret;
6762 }
6763 
6764 /*
6765  * We have to process new refs before deleted refs, but compare_trees gives us
6766  * the new and deleted refs mixed. To fix this, we record the new/deleted refs
6767  * first and later process them in process_recorded_refs.
6768  * For the cur_inode_new_gen case, we skip recording completely because
6769  * changed_inode did already initiate processing of refs. The reason for this is
6770  * that in this case, compare_tree actually compares the refs of 2 different
6771  * inodes. To fix this, process_all_refs is used in changed_inode to handle all
6772  * refs of the right tree as deleted and all refs of the left tree as new.
6773  */
changed_ref(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6774 static int changed_ref(struct send_ctx *sctx,
6775 		       enum btrfs_compare_tree_result result)
6776 {
6777 	int ret = 0;
6778 
6779 	if (sctx->cur_ino != sctx->cmp_key->objectid) {
6780 		inconsistent_snapshot_error(sctx, result, "reference");
6781 		return -EIO;
6782 	}
6783 
6784 	if (!sctx->cur_inode_new_gen &&
6785 	    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
6786 		if (result == BTRFS_COMPARE_TREE_NEW)
6787 			ret = record_new_ref(sctx);
6788 		else if (result == BTRFS_COMPARE_TREE_DELETED)
6789 			ret = record_deleted_ref(sctx);
6790 		else if (result == BTRFS_COMPARE_TREE_CHANGED)
6791 			ret = record_changed_ref(sctx);
6792 	}
6793 
6794 	return ret;
6795 }
6796 
6797 /*
6798  * Process new/deleted/changed xattrs. We skip processing in the
6799  * cur_inode_new_gen case because changed_inode did already initiate processing
6800  * of xattrs. The reason is the same as in changed_ref
6801  */
changed_xattr(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6802 static int changed_xattr(struct send_ctx *sctx,
6803 			 enum btrfs_compare_tree_result result)
6804 {
6805 	int ret = 0;
6806 
6807 	if (sctx->cur_ino != sctx->cmp_key->objectid) {
6808 		inconsistent_snapshot_error(sctx, result, "xattr");
6809 		return -EIO;
6810 	}
6811 
6812 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
6813 		if (result == BTRFS_COMPARE_TREE_NEW)
6814 			ret = process_new_xattr(sctx);
6815 		else if (result == BTRFS_COMPARE_TREE_DELETED)
6816 			ret = process_deleted_xattr(sctx);
6817 		else if (result == BTRFS_COMPARE_TREE_CHANGED)
6818 			ret = process_changed_xattr(sctx);
6819 	}
6820 
6821 	return ret;
6822 }
6823 
6824 /*
6825  * Process new/deleted/changed extents. We skip processing in the
6826  * cur_inode_new_gen case because changed_inode did already initiate processing
6827  * of extents. The reason is the same as in changed_ref
6828  */
changed_extent(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6829 static int changed_extent(struct send_ctx *sctx,
6830 			  enum btrfs_compare_tree_result result)
6831 {
6832 	int ret = 0;
6833 
6834 	/*
6835 	 * We have found an extent item that changed without the inode item
6836 	 * having changed. This can happen either after relocation (where the
6837 	 * disk_bytenr of an extent item is replaced at
6838 	 * relocation.c:replace_file_extents()) or after deduplication into a
6839 	 * file in both the parent and send snapshots (where an extent item can
6840 	 * get modified or replaced with a new one). Note that deduplication
6841 	 * updates the inode item, but it only changes the iversion (sequence
6842 	 * field in the inode item) of the inode, so if a file is deduplicated
6843 	 * the same amount of times in both the parent and send snapshots, its
6844 	 * iversion becomes the same in both snapshots, whence the inode item is
6845 	 * the same on both snapshots.
6846 	 */
6847 	if (sctx->cur_ino != sctx->cmp_key->objectid)
6848 		return 0;
6849 
6850 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
6851 		if (result != BTRFS_COMPARE_TREE_DELETED)
6852 			ret = process_extent(sctx, sctx->left_path,
6853 					sctx->cmp_key);
6854 	}
6855 
6856 	return ret;
6857 }
6858 
changed_verity(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6859 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
6860 {
6861 	int ret = 0;
6862 
6863 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
6864 		if (result == BTRFS_COMPARE_TREE_NEW)
6865 			sctx->cur_inode_needs_verity = true;
6866 	}
6867 	return ret;
6868 }
6869 
dir_changed(struct send_ctx * sctx,u64 dir)6870 static int dir_changed(struct send_ctx *sctx, u64 dir)
6871 {
6872 	u64 orig_gen, new_gen;
6873 	int ret;
6874 
6875 	ret = get_inode_gen(sctx->send_root, dir, &new_gen);
6876 	if (ret)
6877 		return ret;
6878 
6879 	ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
6880 	if (ret)
6881 		return ret;
6882 
6883 	return (orig_gen != new_gen) ? 1 : 0;
6884 }
6885 
compare_refs(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6886 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
6887 			struct btrfs_key *key)
6888 {
6889 	struct btrfs_inode_extref *extref;
6890 	struct extent_buffer *leaf;
6891 	u64 dirid = 0, last_dirid = 0;
6892 	unsigned long ptr;
6893 	u32 item_size;
6894 	u32 cur_offset = 0;
6895 	int ref_name_len;
6896 	int ret = 0;
6897 
6898 	/* Easy case, just check this one dirid */
6899 	if (key->type == BTRFS_INODE_REF_KEY) {
6900 		dirid = key->offset;
6901 
6902 		ret = dir_changed(sctx, dirid);
6903 		goto out;
6904 	}
6905 
6906 	leaf = path->nodes[0];
6907 	item_size = btrfs_item_size(leaf, path->slots[0]);
6908 	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
6909 	while (cur_offset < item_size) {
6910 		extref = (struct btrfs_inode_extref *)(ptr +
6911 						       cur_offset);
6912 		dirid = btrfs_inode_extref_parent(leaf, extref);
6913 		ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
6914 		cur_offset += ref_name_len + sizeof(*extref);
6915 		if (dirid == last_dirid)
6916 			continue;
6917 		ret = dir_changed(sctx, dirid);
6918 		if (ret)
6919 			break;
6920 		last_dirid = dirid;
6921 	}
6922 out:
6923 	return ret;
6924 }
6925 
6926 /*
6927  * Updates compare related fields in sctx and simply forwards to the actual
6928  * changed_xxx functions.
6929  */
changed_cb(struct btrfs_path * left_path,struct btrfs_path * right_path,struct btrfs_key * key,enum btrfs_compare_tree_result result,struct send_ctx * sctx)6930 static int changed_cb(struct btrfs_path *left_path,
6931 		      struct btrfs_path *right_path,
6932 		      struct btrfs_key *key,
6933 		      enum btrfs_compare_tree_result result,
6934 		      struct send_ctx *sctx)
6935 {
6936 	int ret = 0;
6937 
6938 	/*
6939 	 * We can not hold the commit root semaphore here. This is because in
6940 	 * the case of sending and receiving to the same filesystem, using a
6941 	 * pipe, could result in a deadlock:
6942 	 *
6943 	 * 1) The task running send blocks on the pipe because it's full;
6944 	 *
6945 	 * 2) The task running receive, which is the only consumer of the pipe,
6946 	 *    is waiting for a transaction commit (for example due to a space
6947 	 *    reservation when doing a write or triggering a transaction commit
6948 	 *    when creating a subvolume);
6949 	 *
6950 	 * 3) The transaction is waiting to write lock the commit root semaphore,
6951 	 *    but can not acquire it since it's being held at 1).
6952 	 *
6953 	 * Down this call chain we write to the pipe through kernel_write().
6954 	 * The same type of problem can also happen when sending to a file that
6955 	 * is stored in the same filesystem - when reserving space for a write
6956 	 * into the file, we can trigger a transaction commit.
6957 	 *
6958 	 * Our caller has supplied us with clones of leaves from the send and
6959 	 * parent roots, so we're safe here from a concurrent relocation and
6960 	 * further reallocation of metadata extents while we are here. Below we
6961 	 * also assert that the leaves are clones.
6962 	 */
6963 	lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
6964 
6965 	/*
6966 	 * We always have a send root, so left_path is never NULL. We will not
6967 	 * have a leaf when we have reached the end of the send root but have
6968 	 * not yet reached the end of the parent root.
6969 	 */
6970 	if (left_path->nodes[0])
6971 		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
6972 				&left_path->nodes[0]->bflags));
6973 	/*
6974 	 * When doing a full send we don't have a parent root, so right_path is
6975 	 * NULL. When doing an incremental send, we may have reached the end of
6976 	 * the parent root already, so we don't have a leaf at right_path.
6977 	 */
6978 	if (right_path && right_path->nodes[0])
6979 		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
6980 				&right_path->nodes[0]->bflags));
6981 
6982 	if (result == BTRFS_COMPARE_TREE_SAME) {
6983 		if (key->type == BTRFS_INODE_REF_KEY ||
6984 		    key->type == BTRFS_INODE_EXTREF_KEY) {
6985 			ret = compare_refs(sctx, left_path, key);
6986 			if (!ret)
6987 				return 0;
6988 			if (ret < 0)
6989 				return ret;
6990 		} else if (key->type == BTRFS_EXTENT_DATA_KEY) {
6991 			return maybe_send_hole(sctx, left_path, key);
6992 		} else {
6993 			return 0;
6994 		}
6995 		result = BTRFS_COMPARE_TREE_CHANGED;
6996 		ret = 0;
6997 	}
6998 
6999 	sctx->left_path = left_path;
7000 	sctx->right_path = right_path;
7001 	sctx->cmp_key = key;
7002 
7003 	ret = finish_inode_if_needed(sctx, 0);
7004 	if (ret < 0)
7005 		goto out;
7006 
7007 	/* Ignore non-FS objects */
7008 	if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7009 	    key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7010 		goto out;
7011 
7012 	if (key->type == BTRFS_INODE_ITEM_KEY) {
7013 		ret = changed_inode(sctx, result);
7014 	} else if (!sctx->ignore_cur_inode) {
7015 		if (key->type == BTRFS_INODE_REF_KEY ||
7016 		    key->type == BTRFS_INODE_EXTREF_KEY)
7017 			ret = changed_ref(sctx, result);
7018 		else if (key->type == BTRFS_XATTR_ITEM_KEY)
7019 			ret = changed_xattr(sctx, result);
7020 		else if (key->type == BTRFS_EXTENT_DATA_KEY)
7021 			ret = changed_extent(sctx, result);
7022 		else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7023 			 key->offset == 0)
7024 			ret = changed_verity(sctx, result);
7025 	}
7026 
7027 out:
7028 	return ret;
7029 }
7030 
search_key_again(const struct send_ctx * sctx,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * key)7031 static int search_key_again(const struct send_ctx *sctx,
7032 			    struct btrfs_root *root,
7033 			    struct btrfs_path *path,
7034 			    const struct btrfs_key *key)
7035 {
7036 	int ret;
7037 
7038 	if (!path->need_commit_sem)
7039 		lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7040 
7041 	/*
7042 	 * Roots used for send operations are readonly and no one can add,
7043 	 * update or remove keys from them, so we should be able to find our
7044 	 * key again. The only exception is deduplication, which can operate on
7045 	 * readonly roots and add, update or remove keys to/from them - but at
7046 	 * the moment we don't allow it to run in parallel with send.
7047 	 */
7048 	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7049 	ASSERT(ret <= 0);
7050 	if (ret > 0) {
7051 		btrfs_print_tree(path->nodes[path->lowest_level], false);
7052 		btrfs_err(root->fs_info,
7053 "send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
7054 			  key->objectid, key->type, key->offset,
7055 			  (root == sctx->parent_root ? "parent" : "send"),
7056 			  root->root_key.objectid, path->lowest_level,
7057 			  path->slots[path->lowest_level]);
7058 		return -EUCLEAN;
7059 	}
7060 
7061 	return ret;
7062 }
7063 
full_send_tree(struct send_ctx * sctx)7064 static int full_send_tree(struct send_ctx *sctx)
7065 {
7066 	int ret;
7067 	struct btrfs_root *send_root = sctx->send_root;
7068 	struct btrfs_key key;
7069 	struct btrfs_fs_info *fs_info = send_root->fs_info;
7070 	struct btrfs_path *path;
7071 
7072 	path = alloc_path_for_send();
7073 	if (!path)
7074 		return -ENOMEM;
7075 	path->reada = READA_FORWARD_ALWAYS;
7076 
7077 	key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7078 	key.type = BTRFS_INODE_ITEM_KEY;
7079 	key.offset = 0;
7080 
7081 	down_read(&fs_info->commit_root_sem);
7082 	sctx->last_reloc_trans = fs_info->last_reloc_trans;
7083 	up_read(&fs_info->commit_root_sem);
7084 
7085 	ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7086 	if (ret < 0)
7087 		goto out;
7088 	if (ret)
7089 		goto out_finish;
7090 
7091 	while (1) {
7092 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
7093 
7094 		ret = changed_cb(path, NULL, &key,
7095 				 BTRFS_COMPARE_TREE_NEW, sctx);
7096 		if (ret < 0)
7097 			goto out;
7098 
7099 		down_read(&fs_info->commit_root_sem);
7100 		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7101 			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7102 			up_read(&fs_info->commit_root_sem);
7103 			/*
7104 			 * A transaction used for relocating a block group was
7105 			 * committed or is about to finish its commit. Release
7106 			 * our path (leaf) and restart the search, so that we
7107 			 * avoid operating on any file extent items that are
7108 			 * stale, with a disk_bytenr that reflects a pre
7109 			 * relocation value. This way we avoid as much as
7110 			 * possible to fallback to regular writes when checking
7111 			 * if we can clone file ranges.
7112 			 */
7113 			btrfs_release_path(path);
7114 			ret = search_key_again(sctx, send_root, path, &key);
7115 			if (ret < 0)
7116 				goto out;
7117 		} else {
7118 			up_read(&fs_info->commit_root_sem);
7119 		}
7120 
7121 		ret = btrfs_next_item(send_root, path);
7122 		if (ret < 0)
7123 			goto out;
7124 		if (ret) {
7125 			ret  = 0;
7126 			break;
7127 		}
7128 	}
7129 
7130 out_finish:
7131 	ret = finish_inode_if_needed(sctx, 1);
7132 
7133 out:
7134 	btrfs_free_path(path);
7135 	return ret;
7136 }
7137 
replace_node_with_clone(struct btrfs_path * path,int level)7138 static int replace_node_with_clone(struct btrfs_path *path, int level)
7139 {
7140 	struct extent_buffer *clone;
7141 
7142 	clone = btrfs_clone_extent_buffer(path->nodes[level]);
7143 	if (!clone)
7144 		return -ENOMEM;
7145 
7146 	free_extent_buffer(path->nodes[level]);
7147 	path->nodes[level] = clone;
7148 
7149 	return 0;
7150 }
7151 
tree_move_down(struct btrfs_path * path,int * level,u64 reada_min_gen)7152 static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7153 {
7154 	struct extent_buffer *eb;
7155 	struct extent_buffer *parent = path->nodes[*level];
7156 	int slot = path->slots[*level];
7157 	const int nritems = btrfs_header_nritems(parent);
7158 	u64 reada_max;
7159 	u64 reada_done = 0;
7160 
7161 	lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7162 
7163 	BUG_ON(*level == 0);
7164 	eb = btrfs_read_node_slot(parent, slot);
7165 	if (IS_ERR(eb))
7166 		return PTR_ERR(eb);
7167 
7168 	/*
7169 	 * Trigger readahead for the next leaves we will process, so that it is
7170 	 * very likely that when we need them they are already in memory and we
7171 	 * will not block on disk IO. For nodes we only do readahead for one,
7172 	 * since the time window between processing nodes is typically larger.
7173 	 */
7174 	reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7175 
7176 	for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7177 		if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7178 			btrfs_readahead_node_child(parent, slot);
7179 			reada_done += eb->fs_info->nodesize;
7180 		}
7181 	}
7182 
7183 	path->nodes[*level - 1] = eb;
7184 	path->slots[*level - 1] = 0;
7185 	(*level)--;
7186 
7187 	if (*level == 0)
7188 		return replace_node_with_clone(path, 0);
7189 
7190 	return 0;
7191 }
7192 
tree_move_next_or_upnext(struct btrfs_path * path,int * level,int root_level)7193 static int tree_move_next_or_upnext(struct btrfs_path *path,
7194 				    int *level, int root_level)
7195 {
7196 	int ret = 0;
7197 	int nritems;
7198 	nritems = btrfs_header_nritems(path->nodes[*level]);
7199 
7200 	path->slots[*level]++;
7201 
7202 	while (path->slots[*level] >= nritems) {
7203 		if (*level == root_level) {
7204 			path->slots[*level] = nritems - 1;
7205 			return -1;
7206 		}
7207 
7208 		/* move upnext */
7209 		path->slots[*level] = 0;
7210 		free_extent_buffer(path->nodes[*level]);
7211 		path->nodes[*level] = NULL;
7212 		(*level)++;
7213 		path->slots[*level]++;
7214 
7215 		nritems = btrfs_header_nritems(path->nodes[*level]);
7216 		ret = 1;
7217 	}
7218 	return ret;
7219 }
7220 
7221 /*
7222  * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7223  * or down.
7224  */
tree_advance(struct btrfs_path * path,int * level,int root_level,int allow_down,struct btrfs_key * key,u64 reada_min_gen)7225 static int tree_advance(struct btrfs_path *path,
7226 			int *level, int root_level,
7227 			int allow_down,
7228 			struct btrfs_key *key,
7229 			u64 reada_min_gen)
7230 {
7231 	int ret;
7232 
7233 	if (*level == 0 || !allow_down) {
7234 		ret = tree_move_next_or_upnext(path, level, root_level);
7235 	} else {
7236 		ret = tree_move_down(path, level, reada_min_gen);
7237 	}
7238 
7239 	/*
7240 	 * Even if we have reached the end of a tree, ret is -1, update the key
7241 	 * anyway, so that in case we need to restart due to a block group
7242 	 * relocation, we can assert that the last key of the root node still
7243 	 * exists in the tree.
7244 	 */
7245 	if (*level == 0)
7246 		btrfs_item_key_to_cpu(path->nodes[*level], key,
7247 				      path->slots[*level]);
7248 	else
7249 		btrfs_node_key_to_cpu(path->nodes[*level], key,
7250 				      path->slots[*level]);
7251 
7252 	return ret;
7253 }
7254 
tree_compare_item(struct btrfs_path * left_path,struct btrfs_path * right_path,char * tmp_buf)7255 static int tree_compare_item(struct btrfs_path *left_path,
7256 			     struct btrfs_path *right_path,
7257 			     char *tmp_buf)
7258 {
7259 	int cmp;
7260 	int len1, len2;
7261 	unsigned long off1, off2;
7262 
7263 	len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7264 	len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7265 	if (len1 != len2)
7266 		return 1;
7267 
7268 	off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7269 	off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7270 				right_path->slots[0]);
7271 
7272 	read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7273 
7274 	cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7275 	if (cmp)
7276 		return 1;
7277 	return 0;
7278 }
7279 
7280 /*
7281  * A transaction used for relocating a block group was committed or is about to
7282  * finish its commit. Release our paths and restart the search, so that we are
7283  * not using stale extent buffers:
7284  *
7285  * 1) For levels > 0, we are only holding references of extent buffers, without
7286  *    any locks on them, which does not prevent them from having been relocated
7287  *    and reallocated after the last time we released the commit root semaphore.
7288  *    The exception are the root nodes, for which we always have a clone, see
7289  *    the comment at btrfs_compare_trees();
7290  *
7291  * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7292  *    we are safe from the concurrent relocation and reallocation. However they
7293  *    can have file extent items with a pre relocation disk_bytenr value, so we
7294  *    restart the start from the current commit roots and clone the new leaves so
7295  *    that we get the post relocation disk_bytenr values. Not doing so, could
7296  *    make us clone the wrong data in case there are new extents using the old
7297  *    disk_bytenr that happen to be shared.
7298  */
restart_after_relocation(struct btrfs_path * left_path,struct btrfs_path * right_path,const struct btrfs_key * left_key,const struct btrfs_key * right_key,int left_level,int right_level,const struct send_ctx * sctx)7299 static int restart_after_relocation(struct btrfs_path *left_path,
7300 				    struct btrfs_path *right_path,
7301 				    const struct btrfs_key *left_key,
7302 				    const struct btrfs_key *right_key,
7303 				    int left_level,
7304 				    int right_level,
7305 				    const struct send_ctx *sctx)
7306 {
7307 	int root_level;
7308 	int ret;
7309 
7310 	lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7311 
7312 	btrfs_release_path(left_path);
7313 	btrfs_release_path(right_path);
7314 
7315 	/*
7316 	 * Since keys can not be added or removed to/from our roots because they
7317 	 * are readonly and we do not allow deduplication to run in parallel
7318 	 * (which can add, remove or change keys), the layout of the trees should
7319 	 * not change.
7320 	 */
7321 	left_path->lowest_level = left_level;
7322 	ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7323 	if (ret < 0)
7324 		return ret;
7325 
7326 	right_path->lowest_level = right_level;
7327 	ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7328 	if (ret < 0)
7329 		return ret;
7330 
7331 	/*
7332 	 * If the lowest level nodes are leaves, clone them so that they can be
7333 	 * safely used by changed_cb() while not under the protection of the
7334 	 * commit root semaphore, even if relocation and reallocation happens in
7335 	 * parallel.
7336 	 */
7337 	if (left_level == 0) {
7338 		ret = replace_node_with_clone(left_path, 0);
7339 		if (ret < 0)
7340 			return ret;
7341 	}
7342 
7343 	if (right_level == 0) {
7344 		ret = replace_node_with_clone(right_path, 0);
7345 		if (ret < 0)
7346 			return ret;
7347 	}
7348 
7349 	/*
7350 	 * Now clone the root nodes (unless they happen to be the leaves we have
7351 	 * already cloned). This is to protect against concurrent snapshotting of
7352 	 * the send and parent roots (see the comment at btrfs_compare_trees()).
7353 	 */
7354 	root_level = btrfs_header_level(sctx->send_root->commit_root);
7355 	if (root_level > 0) {
7356 		ret = replace_node_with_clone(left_path, root_level);
7357 		if (ret < 0)
7358 			return ret;
7359 	}
7360 
7361 	root_level = btrfs_header_level(sctx->parent_root->commit_root);
7362 	if (root_level > 0) {
7363 		ret = replace_node_with_clone(right_path, root_level);
7364 		if (ret < 0)
7365 			return ret;
7366 	}
7367 
7368 	return 0;
7369 }
7370 
7371 /*
7372  * This function compares two trees and calls the provided callback for
7373  * every changed/new/deleted item it finds.
7374  * If shared tree blocks are encountered, whole subtrees are skipped, making
7375  * the compare pretty fast on snapshotted subvolumes.
7376  *
7377  * This currently works on commit roots only. As commit roots are read only,
7378  * we don't do any locking. The commit roots are protected with transactions.
7379  * Transactions are ended and rejoined when a commit is tried in between.
7380  *
7381  * This function checks for modifications done to the trees while comparing.
7382  * If it detects a change, it aborts immediately.
7383  */
btrfs_compare_trees(struct btrfs_root * left_root,struct btrfs_root * right_root,struct send_ctx * sctx)7384 static int btrfs_compare_trees(struct btrfs_root *left_root,
7385 			struct btrfs_root *right_root, struct send_ctx *sctx)
7386 {
7387 	struct btrfs_fs_info *fs_info = left_root->fs_info;
7388 	int ret;
7389 	int cmp;
7390 	struct btrfs_path *left_path = NULL;
7391 	struct btrfs_path *right_path = NULL;
7392 	struct btrfs_key left_key;
7393 	struct btrfs_key right_key;
7394 	char *tmp_buf = NULL;
7395 	int left_root_level;
7396 	int right_root_level;
7397 	int left_level;
7398 	int right_level;
7399 	int left_end_reached = 0;
7400 	int right_end_reached = 0;
7401 	int advance_left = 0;
7402 	int advance_right = 0;
7403 	u64 left_blockptr;
7404 	u64 right_blockptr;
7405 	u64 left_gen;
7406 	u64 right_gen;
7407 	u64 reada_min_gen;
7408 
7409 	left_path = btrfs_alloc_path();
7410 	if (!left_path) {
7411 		ret = -ENOMEM;
7412 		goto out;
7413 	}
7414 	right_path = btrfs_alloc_path();
7415 	if (!right_path) {
7416 		ret = -ENOMEM;
7417 		goto out;
7418 	}
7419 
7420 	tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7421 	if (!tmp_buf) {
7422 		ret = -ENOMEM;
7423 		goto out;
7424 	}
7425 
7426 	left_path->search_commit_root = 1;
7427 	left_path->skip_locking = 1;
7428 	right_path->search_commit_root = 1;
7429 	right_path->skip_locking = 1;
7430 
7431 	/*
7432 	 * Strategy: Go to the first items of both trees. Then do
7433 	 *
7434 	 * If both trees are at level 0
7435 	 *   Compare keys of current items
7436 	 *     If left < right treat left item as new, advance left tree
7437 	 *       and repeat
7438 	 *     If left > right treat right item as deleted, advance right tree
7439 	 *       and repeat
7440 	 *     If left == right do deep compare of items, treat as changed if
7441 	 *       needed, advance both trees and repeat
7442 	 * If both trees are at the same level but not at level 0
7443 	 *   Compare keys of current nodes/leafs
7444 	 *     If left < right advance left tree and repeat
7445 	 *     If left > right advance right tree and repeat
7446 	 *     If left == right compare blockptrs of the next nodes/leafs
7447 	 *       If they match advance both trees but stay at the same level
7448 	 *         and repeat
7449 	 *       If they don't match advance both trees while allowing to go
7450 	 *         deeper and repeat
7451 	 * If tree levels are different
7452 	 *   Advance the tree that needs it and repeat
7453 	 *
7454 	 * Advancing a tree means:
7455 	 *   If we are at level 0, try to go to the next slot. If that's not
7456 	 *   possible, go one level up and repeat. Stop when we found a level
7457 	 *   where we could go to the next slot. We may at this point be on a
7458 	 *   node or a leaf.
7459 	 *
7460 	 *   If we are not at level 0 and not on shared tree blocks, go one
7461 	 *   level deeper.
7462 	 *
7463 	 *   If we are not at level 0 and on shared tree blocks, go one slot to
7464 	 *   the right if possible or go up and right.
7465 	 */
7466 
7467 	down_read(&fs_info->commit_root_sem);
7468 	left_level = btrfs_header_level(left_root->commit_root);
7469 	left_root_level = left_level;
7470 	/*
7471 	 * We clone the root node of the send and parent roots to prevent races
7472 	 * with snapshot creation of these roots. Snapshot creation COWs the
7473 	 * root node of a tree, so after the transaction is committed the old
7474 	 * extent can be reallocated while this send operation is still ongoing.
7475 	 * So we clone them, under the commit root semaphore, to be race free.
7476 	 */
7477 	left_path->nodes[left_level] =
7478 			btrfs_clone_extent_buffer(left_root->commit_root);
7479 	if (!left_path->nodes[left_level]) {
7480 		ret = -ENOMEM;
7481 		goto out_unlock;
7482 	}
7483 
7484 	right_level = btrfs_header_level(right_root->commit_root);
7485 	right_root_level = right_level;
7486 	right_path->nodes[right_level] =
7487 			btrfs_clone_extent_buffer(right_root->commit_root);
7488 	if (!right_path->nodes[right_level]) {
7489 		ret = -ENOMEM;
7490 		goto out_unlock;
7491 	}
7492 	/*
7493 	 * Our right root is the parent root, while the left root is the "send"
7494 	 * root. We know that all new nodes/leaves in the left root must have
7495 	 * a generation greater than the right root's generation, so we trigger
7496 	 * readahead for those nodes and leaves of the left root, as we know we
7497 	 * will need to read them at some point.
7498 	 */
7499 	reada_min_gen = btrfs_header_generation(right_root->commit_root);
7500 
7501 	if (left_level == 0)
7502 		btrfs_item_key_to_cpu(left_path->nodes[left_level],
7503 				&left_key, left_path->slots[left_level]);
7504 	else
7505 		btrfs_node_key_to_cpu(left_path->nodes[left_level],
7506 				&left_key, left_path->slots[left_level]);
7507 	if (right_level == 0)
7508 		btrfs_item_key_to_cpu(right_path->nodes[right_level],
7509 				&right_key, right_path->slots[right_level]);
7510 	else
7511 		btrfs_node_key_to_cpu(right_path->nodes[right_level],
7512 				&right_key, right_path->slots[right_level]);
7513 
7514 	sctx->last_reloc_trans = fs_info->last_reloc_trans;
7515 
7516 	while (1) {
7517 		if (need_resched() ||
7518 		    rwsem_is_contended(&fs_info->commit_root_sem)) {
7519 			up_read(&fs_info->commit_root_sem);
7520 			cond_resched();
7521 			down_read(&fs_info->commit_root_sem);
7522 		}
7523 
7524 		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7525 			ret = restart_after_relocation(left_path, right_path,
7526 						       &left_key, &right_key,
7527 						       left_level, right_level,
7528 						       sctx);
7529 			if (ret < 0)
7530 				goto out_unlock;
7531 			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7532 		}
7533 
7534 		if (advance_left && !left_end_reached) {
7535 			ret = tree_advance(left_path, &left_level,
7536 					left_root_level,
7537 					advance_left != ADVANCE_ONLY_NEXT,
7538 					&left_key, reada_min_gen);
7539 			if (ret == -1)
7540 				left_end_reached = ADVANCE;
7541 			else if (ret < 0)
7542 				goto out_unlock;
7543 			advance_left = 0;
7544 		}
7545 		if (advance_right && !right_end_reached) {
7546 			ret = tree_advance(right_path, &right_level,
7547 					right_root_level,
7548 					advance_right != ADVANCE_ONLY_NEXT,
7549 					&right_key, reada_min_gen);
7550 			if (ret == -1)
7551 				right_end_reached = ADVANCE;
7552 			else if (ret < 0)
7553 				goto out_unlock;
7554 			advance_right = 0;
7555 		}
7556 
7557 		if (left_end_reached && right_end_reached) {
7558 			ret = 0;
7559 			goto out_unlock;
7560 		} else if (left_end_reached) {
7561 			if (right_level == 0) {
7562 				up_read(&fs_info->commit_root_sem);
7563 				ret = changed_cb(left_path, right_path,
7564 						&right_key,
7565 						BTRFS_COMPARE_TREE_DELETED,
7566 						sctx);
7567 				if (ret < 0)
7568 					goto out;
7569 				down_read(&fs_info->commit_root_sem);
7570 			}
7571 			advance_right = ADVANCE;
7572 			continue;
7573 		} else if (right_end_reached) {
7574 			if (left_level == 0) {
7575 				up_read(&fs_info->commit_root_sem);
7576 				ret = changed_cb(left_path, right_path,
7577 						&left_key,
7578 						BTRFS_COMPARE_TREE_NEW,
7579 						sctx);
7580 				if (ret < 0)
7581 					goto out;
7582 				down_read(&fs_info->commit_root_sem);
7583 			}
7584 			advance_left = ADVANCE;
7585 			continue;
7586 		}
7587 
7588 		if (left_level == 0 && right_level == 0) {
7589 			up_read(&fs_info->commit_root_sem);
7590 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7591 			if (cmp < 0) {
7592 				ret = changed_cb(left_path, right_path,
7593 						&left_key,
7594 						BTRFS_COMPARE_TREE_NEW,
7595 						sctx);
7596 				advance_left = ADVANCE;
7597 			} else if (cmp > 0) {
7598 				ret = changed_cb(left_path, right_path,
7599 						&right_key,
7600 						BTRFS_COMPARE_TREE_DELETED,
7601 						sctx);
7602 				advance_right = ADVANCE;
7603 			} else {
7604 				enum btrfs_compare_tree_result result;
7605 
7606 				WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7607 				ret = tree_compare_item(left_path, right_path,
7608 							tmp_buf);
7609 				if (ret)
7610 					result = BTRFS_COMPARE_TREE_CHANGED;
7611 				else
7612 					result = BTRFS_COMPARE_TREE_SAME;
7613 				ret = changed_cb(left_path, right_path,
7614 						 &left_key, result, sctx);
7615 				advance_left = ADVANCE;
7616 				advance_right = ADVANCE;
7617 			}
7618 
7619 			if (ret < 0)
7620 				goto out;
7621 			down_read(&fs_info->commit_root_sem);
7622 		} else if (left_level == right_level) {
7623 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7624 			if (cmp < 0) {
7625 				advance_left = ADVANCE;
7626 			} else if (cmp > 0) {
7627 				advance_right = ADVANCE;
7628 			} else {
7629 				left_blockptr = btrfs_node_blockptr(
7630 						left_path->nodes[left_level],
7631 						left_path->slots[left_level]);
7632 				right_blockptr = btrfs_node_blockptr(
7633 						right_path->nodes[right_level],
7634 						right_path->slots[right_level]);
7635 				left_gen = btrfs_node_ptr_generation(
7636 						left_path->nodes[left_level],
7637 						left_path->slots[left_level]);
7638 				right_gen = btrfs_node_ptr_generation(
7639 						right_path->nodes[right_level],
7640 						right_path->slots[right_level]);
7641 				if (left_blockptr == right_blockptr &&
7642 				    left_gen == right_gen) {
7643 					/*
7644 					 * As we're on a shared block, don't
7645 					 * allow to go deeper.
7646 					 */
7647 					advance_left = ADVANCE_ONLY_NEXT;
7648 					advance_right = ADVANCE_ONLY_NEXT;
7649 				} else {
7650 					advance_left = ADVANCE;
7651 					advance_right = ADVANCE;
7652 				}
7653 			}
7654 		} else if (left_level < right_level) {
7655 			advance_right = ADVANCE;
7656 		} else {
7657 			advance_left = ADVANCE;
7658 		}
7659 	}
7660 
7661 out_unlock:
7662 	up_read(&fs_info->commit_root_sem);
7663 out:
7664 	btrfs_free_path(left_path);
7665 	btrfs_free_path(right_path);
7666 	kvfree(tmp_buf);
7667 	return ret;
7668 }
7669 
send_subvol(struct send_ctx * sctx)7670 static int send_subvol(struct send_ctx *sctx)
7671 {
7672 	int ret;
7673 
7674 	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
7675 		ret = send_header(sctx);
7676 		if (ret < 0)
7677 			goto out;
7678 	}
7679 
7680 	ret = send_subvol_begin(sctx);
7681 	if (ret < 0)
7682 		goto out;
7683 
7684 	if (sctx->parent_root) {
7685 		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
7686 		if (ret < 0)
7687 			goto out;
7688 		ret = finish_inode_if_needed(sctx, 1);
7689 		if (ret < 0)
7690 			goto out;
7691 	} else {
7692 		ret = full_send_tree(sctx);
7693 		if (ret < 0)
7694 			goto out;
7695 	}
7696 
7697 out:
7698 	free_recorded_refs(sctx);
7699 	return ret;
7700 }
7701 
7702 /*
7703  * If orphan cleanup did remove any orphans from a root, it means the tree
7704  * was modified and therefore the commit root is not the same as the current
7705  * root anymore. This is a problem, because send uses the commit root and
7706  * therefore can see inode items that don't exist in the current root anymore,
7707  * and for example make calls to btrfs_iget, which will do tree lookups based
7708  * on the current root and not on the commit root. Those lookups will fail,
7709  * returning a -ESTALE error, and making send fail with that error. So make
7710  * sure a send does not see any orphans we have just removed, and that it will
7711  * see the same inodes regardless of whether a transaction commit happened
7712  * before it started (meaning that the commit root will be the same as the
7713  * current root) or not.
7714  */
ensure_commit_roots_uptodate(struct send_ctx * sctx)7715 static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
7716 {
7717 	int i;
7718 	struct btrfs_trans_handle *trans = NULL;
7719 
7720 again:
7721 	if (sctx->parent_root &&
7722 	    sctx->parent_root->node != sctx->parent_root->commit_root)
7723 		goto commit_trans;
7724 
7725 	for (i = 0; i < sctx->clone_roots_cnt; i++)
7726 		if (sctx->clone_roots[i].root->node !=
7727 		    sctx->clone_roots[i].root->commit_root)
7728 			goto commit_trans;
7729 
7730 	if (trans)
7731 		return btrfs_end_transaction(trans);
7732 
7733 	return 0;
7734 
7735 commit_trans:
7736 	/* Use any root, all fs roots will get their commit roots updated. */
7737 	if (!trans) {
7738 		trans = btrfs_join_transaction(sctx->send_root);
7739 		if (IS_ERR(trans))
7740 			return PTR_ERR(trans);
7741 		goto again;
7742 	}
7743 
7744 	return btrfs_commit_transaction(trans);
7745 }
7746 
7747 /*
7748  * Make sure any existing dellaloc is flushed for any root used by a send
7749  * operation so that we do not miss any data and we do not race with writeback
7750  * finishing and changing a tree while send is using the tree. This could
7751  * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
7752  * a send operation then uses the subvolume.
7753  * After flushing delalloc ensure_commit_roots_uptodate() must be called.
7754  */
flush_delalloc_roots(struct send_ctx * sctx)7755 static int flush_delalloc_roots(struct send_ctx *sctx)
7756 {
7757 	struct btrfs_root *root = sctx->parent_root;
7758 	int ret;
7759 	int i;
7760 
7761 	if (root) {
7762 		ret = btrfs_start_delalloc_snapshot(root, false);
7763 		if (ret)
7764 			return ret;
7765 		btrfs_wait_ordered_extents(root, U64_MAX, 0, U64_MAX);
7766 	}
7767 
7768 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
7769 		root = sctx->clone_roots[i].root;
7770 		ret = btrfs_start_delalloc_snapshot(root, false);
7771 		if (ret)
7772 			return ret;
7773 		btrfs_wait_ordered_extents(root, U64_MAX, 0, U64_MAX);
7774 	}
7775 
7776 	return 0;
7777 }
7778 
btrfs_root_dec_send_in_progress(struct btrfs_root * root)7779 static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
7780 {
7781 	spin_lock(&root->root_item_lock);
7782 	root->send_in_progress--;
7783 	/*
7784 	 * Not much left to do, we don't know why it's unbalanced and
7785 	 * can't blindly reset it to 0.
7786 	 */
7787 	if (root->send_in_progress < 0)
7788 		btrfs_err(root->fs_info,
7789 			  "send_in_progress unbalanced %d root %llu",
7790 			  root->send_in_progress, root->root_key.objectid);
7791 	spin_unlock(&root->root_item_lock);
7792 }
7793 
dedupe_in_progress_warn(const struct btrfs_root * root)7794 static void dedupe_in_progress_warn(const struct btrfs_root *root)
7795 {
7796 	btrfs_warn_rl(root->fs_info,
7797 "cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
7798 		      root->root_key.objectid, root->dedupe_in_progress);
7799 }
7800 
btrfs_ioctl_send(struct inode * inode,struct btrfs_ioctl_send_args * arg)7801 long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg)
7802 {
7803 	int ret = 0;
7804 	struct btrfs_root *send_root = BTRFS_I(inode)->root;
7805 	struct btrfs_fs_info *fs_info = send_root->fs_info;
7806 	struct btrfs_root *clone_root;
7807 	struct send_ctx *sctx = NULL;
7808 	u32 i;
7809 	u64 *clone_sources_tmp = NULL;
7810 	int clone_sources_to_rollback = 0;
7811 	size_t alloc_size;
7812 	int sort_clone_roots = 0;
7813 
7814 	if (!capable(CAP_SYS_ADMIN))
7815 		return -EPERM;
7816 
7817 	/*
7818 	 * The subvolume must remain read-only during send, protect against
7819 	 * making it RW. This also protects against deletion.
7820 	 */
7821 	spin_lock(&send_root->root_item_lock);
7822 	if (btrfs_root_readonly(send_root) && send_root->dedupe_in_progress) {
7823 		dedupe_in_progress_warn(send_root);
7824 		spin_unlock(&send_root->root_item_lock);
7825 		return -EAGAIN;
7826 	}
7827 	send_root->send_in_progress++;
7828 	spin_unlock(&send_root->root_item_lock);
7829 
7830 	/*
7831 	 * Userspace tools do the checks and warn the user if it's
7832 	 * not RO.
7833 	 */
7834 	if (!btrfs_root_readonly(send_root)) {
7835 		ret = -EPERM;
7836 		goto out;
7837 	}
7838 
7839 	/*
7840 	 * Check that we don't overflow at later allocations, we request
7841 	 * clone_sources_count + 1 items, and compare to unsigned long inside
7842 	 * access_ok.
7843 	 */
7844 	if (arg->clone_sources_count >
7845 	    ULONG_MAX / sizeof(struct clone_root) - 1) {
7846 		ret = -EINVAL;
7847 		goto out;
7848 	}
7849 
7850 	if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
7851 		ret = -EINVAL;
7852 		goto out;
7853 	}
7854 
7855 	sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL);
7856 	if (!sctx) {
7857 		ret = -ENOMEM;
7858 		goto out;
7859 	}
7860 
7861 	INIT_LIST_HEAD(&sctx->new_refs);
7862 	INIT_LIST_HEAD(&sctx->deleted_refs);
7863 	INIT_RADIX_TREE(&sctx->name_cache, GFP_KERNEL);
7864 	INIT_LIST_HEAD(&sctx->name_cache_list);
7865 
7866 	sctx->flags = arg->flags;
7867 
7868 	if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
7869 		if (arg->version > BTRFS_SEND_STREAM_VERSION) {
7870 			ret = -EPROTO;
7871 			goto out;
7872 		}
7873 		/* Zero means "use the highest version" */
7874 		sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
7875 	} else {
7876 		sctx->proto = 1;
7877 	}
7878 	if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
7879 		ret = -EINVAL;
7880 		goto out;
7881 	}
7882 
7883 	sctx->send_filp = fget(arg->send_fd);
7884 	if (!sctx->send_filp) {
7885 		ret = -EBADF;
7886 		goto out;
7887 	}
7888 
7889 	sctx->send_root = send_root;
7890 	/*
7891 	 * Unlikely but possible, if the subvolume is marked for deletion but
7892 	 * is slow to remove the directory entry, send can still be started
7893 	 */
7894 	if (btrfs_root_dead(sctx->send_root)) {
7895 		ret = -EPERM;
7896 		goto out;
7897 	}
7898 
7899 	sctx->clone_roots_cnt = arg->clone_sources_count;
7900 
7901 	if (sctx->proto >= 2) {
7902 		u32 send_buf_num_pages;
7903 
7904 		sctx->send_max_size = ALIGN(SZ_16K + BTRFS_MAX_COMPRESSED, PAGE_SIZE);
7905 		sctx->send_buf = vmalloc(sctx->send_max_size);
7906 		if (!sctx->send_buf) {
7907 			ret = -ENOMEM;
7908 			goto out;
7909 		}
7910 		send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
7911 		sctx->send_buf_pages = kcalloc(send_buf_num_pages,
7912 					       sizeof(*sctx->send_buf_pages),
7913 					       GFP_KERNEL);
7914 		if (!sctx->send_buf_pages) {
7915 			ret = -ENOMEM;
7916 			goto out;
7917 		}
7918 		for (i = 0; i < send_buf_num_pages; i++) {
7919 			sctx->send_buf_pages[i] =
7920 				vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
7921 		}
7922 	} else {
7923 		sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
7924 		sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
7925 	}
7926 	if (!sctx->send_buf) {
7927 		ret = -ENOMEM;
7928 		goto out;
7929 	}
7930 
7931 	sctx->pending_dir_moves = RB_ROOT;
7932 	sctx->waiting_dir_moves = RB_ROOT;
7933 	sctx->orphan_dirs = RB_ROOT;
7934 	sctx->rbtree_new_refs = RB_ROOT;
7935 	sctx->rbtree_deleted_refs = RB_ROOT;
7936 
7937 	sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots),
7938 				     arg->clone_sources_count + 1,
7939 				     GFP_KERNEL);
7940 	if (!sctx->clone_roots) {
7941 		ret = -ENOMEM;
7942 		goto out;
7943 	}
7944 
7945 	alloc_size = array_size(sizeof(*arg->clone_sources),
7946 				arg->clone_sources_count);
7947 
7948 	if (arg->clone_sources_count) {
7949 		clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
7950 		if (!clone_sources_tmp) {
7951 			ret = -ENOMEM;
7952 			goto out;
7953 		}
7954 
7955 		ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
7956 				alloc_size);
7957 		if (ret) {
7958 			ret = -EFAULT;
7959 			goto out;
7960 		}
7961 
7962 		for (i = 0; i < arg->clone_sources_count; i++) {
7963 			clone_root = btrfs_get_fs_root(fs_info,
7964 						clone_sources_tmp[i], true);
7965 			if (IS_ERR(clone_root)) {
7966 				ret = PTR_ERR(clone_root);
7967 				goto out;
7968 			}
7969 			spin_lock(&clone_root->root_item_lock);
7970 			if (!btrfs_root_readonly(clone_root) ||
7971 			    btrfs_root_dead(clone_root)) {
7972 				spin_unlock(&clone_root->root_item_lock);
7973 				btrfs_put_root(clone_root);
7974 				ret = -EPERM;
7975 				goto out;
7976 			}
7977 			if (clone_root->dedupe_in_progress) {
7978 				dedupe_in_progress_warn(clone_root);
7979 				spin_unlock(&clone_root->root_item_lock);
7980 				btrfs_put_root(clone_root);
7981 				ret = -EAGAIN;
7982 				goto out;
7983 			}
7984 			clone_root->send_in_progress++;
7985 			spin_unlock(&clone_root->root_item_lock);
7986 
7987 			sctx->clone_roots[i].root = clone_root;
7988 			clone_sources_to_rollback = i + 1;
7989 		}
7990 		kvfree(clone_sources_tmp);
7991 		clone_sources_tmp = NULL;
7992 	}
7993 
7994 	if (arg->parent_root) {
7995 		sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
7996 						      true);
7997 		if (IS_ERR(sctx->parent_root)) {
7998 			ret = PTR_ERR(sctx->parent_root);
7999 			goto out;
8000 		}
8001 
8002 		spin_lock(&sctx->parent_root->root_item_lock);
8003 		sctx->parent_root->send_in_progress++;
8004 		if (!btrfs_root_readonly(sctx->parent_root) ||
8005 				btrfs_root_dead(sctx->parent_root)) {
8006 			spin_unlock(&sctx->parent_root->root_item_lock);
8007 			ret = -EPERM;
8008 			goto out;
8009 		}
8010 		if (sctx->parent_root->dedupe_in_progress) {
8011 			dedupe_in_progress_warn(sctx->parent_root);
8012 			spin_unlock(&sctx->parent_root->root_item_lock);
8013 			ret = -EAGAIN;
8014 			goto out;
8015 		}
8016 		spin_unlock(&sctx->parent_root->root_item_lock);
8017 	}
8018 
8019 	/*
8020 	 * Clones from send_root are allowed, but only if the clone source
8021 	 * is behind the current send position. This is checked while searching
8022 	 * for possible clone sources.
8023 	 */
8024 	sctx->clone_roots[sctx->clone_roots_cnt++].root =
8025 		btrfs_grab_root(sctx->send_root);
8026 
8027 	/* We do a bsearch later */
8028 	sort(sctx->clone_roots, sctx->clone_roots_cnt,
8029 			sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8030 			NULL);
8031 	sort_clone_roots = 1;
8032 
8033 	ret = flush_delalloc_roots(sctx);
8034 	if (ret)
8035 		goto out;
8036 
8037 	ret = ensure_commit_roots_uptodate(sctx);
8038 	if (ret)
8039 		goto out;
8040 
8041 	ret = send_subvol(sctx);
8042 	if (ret < 0)
8043 		goto out;
8044 
8045 	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8046 		ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8047 		if (ret < 0)
8048 			goto out;
8049 		ret = send_cmd(sctx);
8050 		if (ret < 0)
8051 			goto out;
8052 	}
8053 
8054 out:
8055 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8056 	while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8057 		struct rb_node *n;
8058 		struct pending_dir_move *pm;
8059 
8060 		n = rb_first(&sctx->pending_dir_moves);
8061 		pm = rb_entry(n, struct pending_dir_move, node);
8062 		while (!list_empty(&pm->list)) {
8063 			struct pending_dir_move *pm2;
8064 
8065 			pm2 = list_first_entry(&pm->list,
8066 					       struct pending_dir_move, list);
8067 			free_pending_move(sctx, pm2);
8068 		}
8069 		free_pending_move(sctx, pm);
8070 	}
8071 
8072 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8073 	while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8074 		struct rb_node *n;
8075 		struct waiting_dir_move *dm;
8076 
8077 		n = rb_first(&sctx->waiting_dir_moves);
8078 		dm = rb_entry(n, struct waiting_dir_move, node);
8079 		rb_erase(&dm->node, &sctx->waiting_dir_moves);
8080 		kfree(dm);
8081 	}
8082 
8083 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8084 	while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8085 		struct rb_node *n;
8086 		struct orphan_dir_info *odi;
8087 
8088 		n = rb_first(&sctx->orphan_dirs);
8089 		odi = rb_entry(n, struct orphan_dir_info, node);
8090 		free_orphan_dir_info(sctx, odi);
8091 	}
8092 
8093 	if (sort_clone_roots) {
8094 		for (i = 0; i < sctx->clone_roots_cnt; i++) {
8095 			btrfs_root_dec_send_in_progress(
8096 					sctx->clone_roots[i].root);
8097 			btrfs_put_root(sctx->clone_roots[i].root);
8098 		}
8099 	} else {
8100 		for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8101 			btrfs_root_dec_send_in_progress(
8102 					sctx->clone_roots[i].root);
8103 			btrfs_put_root(sctx->clone_roots[i].root);
8104 		}
8105 
8106 		btrfs_root_dec_send_in_progress(send_root);
8107 	}
8108 	if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8109 		btrfs_root_dec_send_in_progress(sctx->parent_root);
8110 		btrfs_put_root(sctx->parent_root);
8111 	}
8112 
8113 	kvfree(clone_sources_tmp);
8114 
8115 	if (sctx) {
8116 		if (sctx->send_filp)
8117 			fput(sctx->send_filp);
8118 
8119 		kvfree(sctx->clone_roots);
8120 		kfree(sctx->send_buf_pages);
8121 		kvfree(sctx->send_buf);
8122 		kvfree(sctx->verity_descriptor);
8123 
8124 		name_cache_free(sctx);
8125 
8126 		close_current_inode(sctx);
8127 
8128 		kfree(sctx);
8129 	}
8130 
8131 	return ret;
8132 }
8133