1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/sched/mm.h>
10 #include <linux/spinlock.h>
11 #include <linux/blkdev.h>
12 #include <linux/swap.h>
13 #include <linux/writeback.h>
14 #include <linux/pagevec.h>
15 #include <linux/prefetch.h>
16 #include <linux/fsverity.h>
17 #include "misc.h"
18 #include "extent_io.h"
19 #include "extent-io-tree.h"
20 #include "extent_map.h"
21 #include "ctree.h"
22 #include "btrfs_inode.h"
23 #include "volumes.h"
24 #include "check-integrity.h"
25 #include "locking.h"
26 #include "rcu-string.h"
27 #include "backref.h"
28 #include "disk-io.h"
29 #include "subpage.h"
30 #include "zoned.h"
31 #include "block-group.h"
32 #include "compression.h"
33 
34 static struct kmem_cache *extent_buffer_cache;
35 
36 #ifdef CONFIG_BTRFS_DEBUG
btrfs_leak_debug_add_eb(struct extent_buffer * eb)37 static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
38 {
39 	struct btrfs_fs_info *fs_info = eb->fs_info;
40 	unsigned long flags;
41 
42 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
43 	list_add(&eb->leak_list, &fs_info->allocated_ebs);
44 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
45 }
46 
btrfs_leak_debug_del_eb(struct extent_buffer * eb)47 static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
48 {
49 	struct btrfs_fs_info *fs_info = eb->fs_info;
50 	unsigned long flags;
51 
52 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
53 	list_del(&eb->leak_list);
54 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
55 }
56 
btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info * fs_info)57 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
58 {
59 	struct extent_buffer *eb;
60 	unsigned long flags;
61 
62 	/*
63 	 * If we didn't get into open_ctree our allocated_ebs will not be
64 	 * initialized, so just skip this.
65 	 */
66 	if (!fs_info->allocated_ebs.next)
67 		return;
68 
69 	WARN_ON(!list_empty(&fs_info->allocated_ebs));
70 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
71 	while (!list_empty(&fs_info->allocated_ebs)) {
72 		eb = list_first_entry(&fs_info->allocated_ebs,
73 				      struct extent_buffer, leak_list);
74 		pr_err(
75 	"BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
76 		       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
77 		       btrfs_header_owner(eb));
78 		list_del(&eb->leak_list);
79 		kmem_cache_free(extent_buffer_cache, eb);
80 	}
81 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
82 }
83 #else
84 #define btrfs_leak_debug_add_eb(eb)			do {} while (0)
85 #define btrfs_leak_debug_del_eb(eb)			do {} while (0)
86 #endif
87 
88 /*
89  * Structure to record info about the bio being assembled, and other info like
90  * how many bytes are there before stripe/ordered extent boundary.
91  */
92 struct btrfs_bio_ctrl {
93 	struct bio *bio;
94 	int mirror_num;
95 	enum btrfs_compression_type compress_type;
96 	u32 len_to_stripe_boundary;
97 	u32 len_to_oe_boundary;
98 	btrfs_bio_end_io_t end_io_func;
99 };
100 
101 struct extent_page_data {
102 	struct btrfs_bio_ctrl bio_ctrl;
103 	/* tells writepage not to lock the state bits for this range
104 	 * it still does the unlocking
105 	 */
106 	unsigned int extent_locked:1;
107 
108 	/* tells the submit_bio code to use REQ_SYNC */
109 	unsigned int sync_io:1;
110 };
111 
submit_one_bio(struct btrfs_bio_ctrl * bio_ctrl)112 static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
113 {
114 	struct bio *bio;
115 	struct bio_vec *bv;
116 	struct inode *inode;
117 	int mirror_num;
118 
119 	if (!bio_ctrl->bio)
120 		return;
121 
122 	bio = bio_ctrl->bio;
123 	bv = bio_first_bvec_all(bio);
124 	inode = bv->bv_page->mapping->host;
125 	mirror_num = bio_ctrl->mirror_num;
126 
127 	/* Caller should ensure the bio has at least some range added */
128 	ASSERT(bio->bi_iter.bi_size);
129 
130 	btrfs_bio(bio)->file_offset = page_offset(bv->bv_page) + bv->bv_offset;
131 
132 	if (!is_data_inode(inode))
133 		btrfs_submit_metadata_bio(inode, bio, mirror_num);
134 	else if (btrfs_op(bio) == BTRFS_MAP_WRITE)
135 		btrfs_submit_data_write_bio(inode, bio, mirror_num);
136 	else
137 		btrfs_submit_data_read_bio(inode, bio, mirror_num,
138 					   bio_ctrl->compress_type);
139 
140 	/* The bio is owned by the end_io handler now */
141 	bio_ctrl->bio = NULL;
142 }
143 
144 /*
145  * Submit or fail the current bio in an extent_page_data structure.
146  */
submit_write_bio(struct extent_page_data * epd,int ret)147 static void submit_write_bio(struct extent_page_data *epd, int ret)
148 {
149 	struct bio *bio = epd->bio_ctrl.bio;
150 
151 	if (!bio)
152 		return;
153 
154 	if (ret) {
155 		ASSERT(ret < 0);
156 		btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret));
157 		/* The bio is owned by the end_io handler now */
158 		epd->bio_ctrl.bio = NULL;
159 	} else {
160 		submit_one_bio(&epd->bio_ctrl);
161 	}
162 }
163 
extent_buffer_init_cachep(void)164 int __init extent_buffer_init_cachep(void)
165 {
166 	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
167 			sizeof(struct extent_buffer), 0,
168 			SLAB_MEM_SPREAD, NULL);
169 	if (!extent_buffer_cache)
170 		return -ENOMEM;
171 
172 	return 0;
173 }
174 
extent_buffer_free_cachep(void)175 void __cold extent_buffer_free_cachep(void)
176 {
177 	/*
178 	 * Make sure all delayed rcu free are flushed before we
179 	 * destroy caches.
180 	 */
181 	rcu_barrier();
182 	kmem_cache_destroy(extent_buffer_cache);
183 }
184 
extent_range_clear_dirty_for_io(struct inode * inode,u64 start,u64 end)185 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
186 {
187 	unsigned long index = start >> PAGE_SHIFT;
188 	unsigned long end_index = end >> PAGE_SHIFT;
189 	struct page *page;
190 
191 	while (index <= end_index) {
192 		page = find_get_page(inode->i_mapping, index);
193 		BUG_ON(!page); /* Pages should be in the extent_io_tree */
194 		clear_page_dirty_for_io(page);
195 		put_page(page);
196 		index++;
197 	}
198 }
199 
extent_range_redirty_for_io(struct inode * inode,u64 start,u64 end)200 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
201 {
202 	struct address_space *mapping = inode->i_mapping;
203 	unsigned long index = start >> PAGE_SHIFT;
204 	unsigned long end_index = end >> PAGE_SHIFT;
205 	struct folio *folio;
206 
207 	while (index <= end_index) {
208 		folio = filemap_get_folio(mapping, index);
209 		filemap_dirty_folio(mapping, folio);
210 		folio_account_redirty(folio);
211 		index += folio_nr_pages(folio);
212 		folio_put(folio);
213 	}
214 }
215 
216 /*
217  * Process one page for __process_pages_contig().
218  *
219  * Return >0 if we hit @page == @locked_page.
220  * Return 0 if we updated the page status.
221  * Return -EGAIN if the we need to try again.
222  * (For PAGE_LOCK case but got dirty page or page not belong to mapping)
223  */
process_one_page(struct btrfs_fs_info * fs_info,struct address_space * mapping,struct page * page,struct page * locked_page,unsigned long page_ops,u64 start,u64 end)224 static int process_one_page(struct btrfs_fs_info *fs_info,
225 			    struct address_space *mapping,
226 			    struct page *page, struct page *locked_page,
227 			    unsigned long page_ops, u64 start, u64 end)
228 {
229 	u32 len;
230 
231 	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
232 	len = end + 1 - start;
233 
234 	if (page_ops & PAGE_SET_ORDERED)
235 		btrfs_page_clamp_set_ordered(fs_info, page, start, len);
236 	if (page_ops & PAGE_SET_ERROR)
237 		btrfs_page_clamp_set_error(fs_info, page, start, len);
238 	if (page_ops & PAGE_START_WRITEBACK) {
239 		btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
240 		btrfs_page_clamp_set_writeback(fs_info, page, start, len);
241 	}
242 	if (page_ops & PAGE_END_WRITEBACK)
243 		btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
244 
245 	if (page == locked_page)
246 		return 1;
247 
248 	if (page_ops & PAGE_LOCK) {
249 		int ret;
250 
251 		ret = btrfs_page_start_writer_lock(fs_info, page, start, len);
252 		if (ret)
253 			return ret;
254 		if (!PageDirty(page) || page->mapping != mapping) {
255 			btrfs_page_end_writer_lock(fs_info, page, start, len);
256 			return -EAGAIN;
257 		}
258 	}
259 	if (page_ops & PAGE_UNLOCK)
260 		btrfs_page_end_writer_lock(fs_info, page, start, len);
261 	return 0;
262 }
263 
__process_pages_contig(struct address_space * mapping,struct page * locked_page,u64 start,u64 end,unsigned long page_ops,u64 * processed_end)264 static int __process_pages_contig(struct address_space *mapping,
265 				  struct page *locked_page,
266 				  u64 start, u64 end, unsigned long page_ops,
267 				  u64 *processed_end)
268 {
269 	struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
270 	pgoff_t start_index = start >> PAGE_SHIFT;
271 	pgoff_t end_index = end >> PAGE_SHIFT;
272 	pgoff_t index = start_index;
273 	unsigned long pages_processed = 0;
274 	struct folio_batch fbatch;
275 	int err = 0;
276 	int i;
277 
278 	if (page_ops & PAGE_LOCK) {
279 		ASSERT(page_ops == PAGE_LOCK);
280 		ASSERT(processed_end && *processed_end == start);
281 	}
282 
283 	if ((page_ops & PAGE_SET_ERROR) && start_index <= end_index)
284 		mapping_set_error(mapping, -EIO);
285 
286 	folio_batch_init(&fbatch);
287 	while (index <= end_index) {
288 		int found_folios;
289 
290 		found_folios = filemap_get_folios_contig(mapping, &index,
291 				end_index, &fbatch);
292 
293 		if (found_folios == 0) {
294 			/*
295 			 * Only if we're going to lock these pages, we can find
296 			 * nothing at @index.
297 			 */
298 			ASSERT(page_ops & PAGE_LOCK);
299 			err = -EAGAIN;
300 			goto out;
301 		}
302 
303 		for (i = 0; i < found_folios; i++) {
304 			int process_ret;
305 			struct folio *folio = fbatch.folios[i];
306 			process_ret = process_one_page(fs_info, mapping,
307 					&folio->page, locked_page, page_ops,
308 					start, end);
309 			if (process_ret < 0) {
310 				err = -EAGAIN;
311 				folio_batch_release(&fbatch);
312 				goto out;
313 			}
314 			pages_processed += folio_nr_pages(folio);
315 		}
316 		folio_batch_release(&fbatch);
317 		cond_resched();
318 	}
319 out:
320 	if (err && processed_end) {
321 		/*
322 		 * Update @processed_end. I know this is awful since it has
323 		 * two different return value patterns (inclusive vs exclusive).
324 		 *
325 		 * But the exclusive pattern is necessary if @start is 0, or we
326 		 * underflow and check against processed_end won't work as
327 		 * expected.
328 		 */
329 		if (pages_processed)
330 			*processed_end = min(end,
331 			((u64)(start_index + pages_processed) << PAGE_SHIFT) - 1);
332 		else
333 			*processed_end = start;
334 	}
335 	return err;
336 }
337 
__unlock_for_delalloc(struct inode * inode,struct page * locked_page,u64 start,u64 end)338 static noinline void __unlock_for_delalloc(struct inode *inode,
339 					   struct page *locked_page,
340 					   u64 start, u64 end)
341 {
342 	unsigned long index = start >> PAGE_SHIFT;
343 	unsigned long end_index = end >> PAGE_SHIFT;
344 
345 	ASSERT(locked_page);
346 	if (index == locked_page->index && end_index == index)
347 		return;
348 
349 	__process_pages_contig(inode->i_mapping, locked_page, start, end,
350 			       PAGE_UNLOCK, NULL);
351 }
352 
lock_delalloc_pages(struct inode * inode,struct page * locked_page,u64 delalloc_start,u64 delalloc_end)353 static noinline int lock_delalloc_pages(struct inode *inode,
354 					struct page *locked_page,
355 					u64 delalloc_start,
356 					u64 delalloc_end)
357 {
358 	unsigned long index = delalloc_start >> PAGE_SHIFT;
359 	unsigned long end_index = delalloc_end >> PAGE_SHIFT;
360 	u64 processed_end = delalloc_start;
361 	int ret;
362 
363 	ASSERT(locked_page);
364 	if (index == locked_page->index && index == end_index)
365 		return 0;
366 
367 	ret = __process_pages_contig(inode->i_mapping, locked_page, delalloc_start,
368 				     delalloc_end, PAGE_LOCK, &processed_end);
369 	if (ret == -EAGAIN && processed_end > delalloc_start)
370 		__unlock_for_delalloc(inode, locked_page, delalloc_start,
371 				      processed_end);
372 	return ret;
373 }
374 
375 /*
376  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
377  * more than @max_bytes.
378  *
379  * @start:	The original start bytenr to search.
380  *		Will store the extent range start bytenr.
381  * @end:	The original end bytenr of the search range
382  *		Will store the extent range end bytenr.
383  *
384  * Return true if we find a delalloc range which starts inside the original
385  * range, and @start/@end will store the delalloc range start/end.
386  *
387  * Return false if we can't find any delalloc range which starts inside the
388  * original range, and @start/@end will be the non-delalloc range start/end.
389  */
390 EXPORT_FOR_TESTS
find_lock_delalloc_range(struct inode * inode,struct page * locked_page,u64 * start,u64 * end)391 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
392 				    struct page *locked_page, u64 *start,
393 				    u64 *end)
394 {
395 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
396 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
397 	const u64 orig_start = *start;
398 	const u64 orig_end = *end;
399 	/* The sanity tests may not set a valid fs_info. */
400 	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
401 	u64 delalloc_start;
402 	u64 delalloc_end;
403 	bool found;
404 	struct extent_state *cached_state = NULL;
405 	int ret;
406 	int loops = 0;
407 
408 	/* Caller should pass a valid @end to indicate the search range end */
409 	ASSERT(orig_end > orig_start);
410 
411 	/* The range should at least cover part of the page */
412 	ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
413 		 orig_end <= page_offset(locked_page)));
414 again:
415 	/* step one, find a bunch of delalloc bytes starting at start */
416 	delalloc_start = *start;
417 	delalloc_end = 0;
418 	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
419 					  max_bytes, &cached_state);
420 	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
421 		*start = delalloc_start;
422 
423 		/* @delalloc_end can be -1, never go beyond @orig_end */
424 		*end = min(delalloc_end, orig_end);
425 		free_extent_state(cached_state);
426 		return false;
427 	}
428 
429 	/*
430 	 * start comes from the offset of locked_page.  We have to lock
431 	 * pages in order, so we can't process delalloc bytes before
432 	 * locked_page
433 	 */
434 	if (delalloc_start < *start)
435 		delalloc_start = *start;
436 
437 	/*
438 	 * make sure to limit the number of pages we try to lock down
439 	 */
440 	if (delalloc_end + 1 - delalloc_start > max_bytes)
441 		delalloc_end = delalloc_start + max_bytes - 1;
442 
443 	/* step two, lock all the pages after the page that has start */
444 	ret = lock_delalloc_pages(inode, locked_page,
445 				  delalloc_start, delalloc_end);
446 	ASSERT(!ret || ret == -EAGAIN);
447 	if (ret == -EAGAIN) {
448 		/* some of the pages are gone, lets avoid looping by
449 		 * shortening the size of the delalloc range we're searching
450 		 */
451 		free_extent_state(cached_state);
452 		cached_state = NULL;
453 		if (!loops) {
454 			max_bytes = PAGE_SIZE;
455 			loops = 1;
456 			goto again;
457 		} else {
458 			found = false;
459 			goto out_failed;
460 		}
461 	}
462 
463 	/* step three, lock the state bits for the whole range */
464 	lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
465 
466 	/* then test to make sure it is all still delalloc */
467 	ret = test_range_bit(tree, delalloc_start, delalloc_end,
468 			     EXTENT_DELALLOC, 1, cached_state);
469 	if (!ret) {
470 		unlock_extent(tree, delalloc_start, delalloc_end,
471 			      &cached_state);
472 		__unlock_for_delalloc(inode, locked_page,
473 			      delalloc_start, delalloc_end);
474 		cond_resched();
475 		goto again;
476 	}
477 	free_extent_state(cached_state);
478 	*start = delalloc_start;
479 	*end = delalloc_end;
480 out_failed:
481 	return found;
482 }
483 
extent_clear_unlock_delalloc(struct btrfs_inode * inode,u64 start,u64 end,struct page * locked_page,u32 clear_bits,unsigned long page_ops)484 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
485 				  struct page *locked_page,
486 				  u32 clear_bits, unsigned long page_ops)
487 {
488 	clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL);
489 
490 	__process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
491 			       start, end, page_ops, NULL);
492 }
493 
insert_failrec(struct btrfs_inode * inode,struct io_failure_record * failrec)494 static int insert_failrec(struct btrfs_inode *inode,
495 			  struct io_failure_record *failrec)
496 {
497 	struct rb_node *exist;
498 
499 	spin_lock(&inode->io_failure_lock);
500 	exist = rb_simple_insert(&inode->io_failure_tree, failrec->bytenr,
501 				 &failrec->rb_node);
502 	spin_unlock(&inode->io_failure_lock);
503 
504 	return (exist == NULL) ? 0 : -EEXIST;
505 }
506 
get_failrec(struct btrfs_inode * inode,u64 start)507 static struct io_failure_record *get_failrec(struct btrfs_inode *inode, u64 start)
508 {
509 	struct rb_node *node;
510 	struct io_failure_record *failrec = ERR_PTR(-ENOENT);
511 
512 	spin_lock(&inode->io_failure_lock);
513 	node = rb_simple_search(&inode->io_failure_tree, start);
514 	if (node)
515 		failrec = rb_entry(node, struct io_failure_record, rb_node);
516 	spin_unlock(&inode->io_failure_lock);
517 	return failrec;
518 }
519 
free_io_failure(struct btrfs_inode * inode,struct io_failure_record * rec)520 static void free_io_failure(struct btrfs_inode *inode,
521 			    struct io_failure_record *rec)
522 {
523 	spin_lock(&inode->io_failure_lock);
524 	rb_erase(&rec->rb_node, &inode->io_failure_tree);
525 	spin_unlock(&inode->io_failure_lock);
526 
527 	kfree(rec);
528 }
529 
530 /*
531  * this bypasses the standard btrfs submit functions deliberately, as
532  * the standard behavior is to write all copies in a raid setup. here we only
533  * want to write the one bad copy. so we do the mapping for ourselves and issue
534  * submit_bio directly.
535  * to avoid any synchronization issues, wait for the data after writing, which
536  * actually prevents the read that triggered the error from finishing.
537  * currently, there can be no more than two copies of every data bit. thus,
538  * exactly one rewrite is required.
539  */
repair_io_failure(struct btrfs_fs_info * fs_info,u64 ino,u64 start,u64 length,u64 logical,struct page * page,unsigned int pg_offset,int mirror_num)540 static int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
541 			     u64 length, u64 logical, struct page *page,
542 			     unsigned int pg_offset, int mirror_num)
543 {
544 	struct btrfs_device *dev;
545 	struct bio_vec bvec;
546 	struct bio bio;
547 	u64 map_length = 0;
548 	u64 sector;
549 	struct btrfs_io_context *bioc = NULL;
550 	int ret = 0;
551 
552 	ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
553 	BUG_ON(!mirror_num);
554 
555 	if (btrfs_repair_one_zone(fs_info, logical))
556 		return 0;
557 
558 	map_length = length;
559 
560 	/*
561 	 * Avoid races with device replace and make sure our bioc has devices
562 	 * associated to its stripes that don't go away while we are doing the
563 	 * read repair operation.
564 	 */
565 	btrfs_bio_counter_inc_blocked(fs_info);
566 	if (btrfs_is_parity_mirror(fs_info, logical, length)) {
567 		/*
568 		 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
569 		 * to update all raid stripes, but here we just want to correct
570 		 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
571 		 * stripe's dev and sector.
572 		 */
573 		ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
574 				      &map_length, &bioc, 0);
575 		if (ret)
576 			goto out_counter_dec;
577 		ASSERT(bioc->mirror_num == 1);
578 	} else {
579 		ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
580 				      &map_length, &bioc, mirror_num);
581 		if (ret)
582 			goto out_counter_dec;
583 		/*
584 		 * This happens when dev-replace is also running, and the
585 		 * mirror_num indicates the dev-replace target.
586 		 *
587 		 * In this case, we don't need to do anything, as the read
588 		 * error just means the replace progress hasn't reached our
589 		 * read range, and later replace routine would handle it well.
590 		 */
591 		if (mirror_num != bioc->mirror_num)
592 			goto out_counter_dec;
593 	}
594 
595 	sector = bioc->stripes[bioc->mirror_num - 1].physical >> 9;
596 	dev = bioc->stripes[bioc->mirror_num - 1].dev;
597 	btrfs_put_bioc(bioc);
598 
599 	if (!dev || !dev->bdev ||
600 	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
601 		ret = -EIO;
602 		goto out_counter_dec;
603 	}
604 
605 	bio_init(&bio, dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC);
606 	bio.bi_iter.bi_sector = sector;
607 	__bio_add_page(&bio, page, length, pg_offset);
608 
609 	btrfsic_check_bio(&bio);
610 	ret = submit_bio_wait(&bio);
611 	if (ret) {
612 		/* try to remap that extent elsewhere? */
613 		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
614 		goto out_bio_uninit;
615 	}
616 
617 	btrfs_info_rl_in_rcu(fs_info,
618 		"read error corrected: ino %llu off %llu (dev %s sector %llu)",
619 				  ino, start,
620 				  rcu_str_deref(dev->name), sector);
621 	ret = 0;
622 
623 out_bio_uninit:
624 	bio_uninit(&bio);
625 out_counter_dec:
626 	btrfs_bio_counter_dec(fs_info);
627 	return ret;
628 }
629 
btrfs_repair_eb_io_failure(const struct extent_buffer * eb,int mirror_num)630 int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num)
631 {
632 	struct btrfs_fs_info *fs_info = eb->fs_info;
633 	u64 start = eb->start;
634 	int i, num_pages = num_extent_pages(eb);
635 	int ret = 0;
636 
637 	if (sb_rdonly(fs_info->sb))
638 		return -EROFS;
639 
640 	for (i = 0; i < num_pages; i++) {
641 		struct page *p = eb->pages[i];
642 
643 		ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
644 					start - page_offset(p), mirror_num);
645 		if (ret)
646 			break;
647 		start += PAGE_SIZE;
648 	}
649 
650 	return ret;
651 }
652 
next_mirror(const struct io_failure_record * failrec,int cur_mirror)653 static int next_mirror(const struct io_failure_record *failrec, int cur_mirror)
654 {
655 	if (cur_mirror == failrec->num_copies)
656 		return cur_mirror + 1 - failrec->num_copies;
657 	return cur_mirror + 1;
658 }
659 
prev_mirror(const struct io_failure_record * failrec,int cur_mirror)660 static int prev_mirror(const struct io_failure_record *failrec, int cur_mirror)
661 {
662 	if (cur_mirror == 1)
663 		return failrec->num_copies;
664 	return cur_mirror - 1;
665 }
666 
667 /*
668  * each time an IO finishes, we do a fast check in the IO failure tree
669  * to see if we need to process or clean up an io_failure_record
670  */
btrfs_clean_io_failure(struct btrfs_inode * inode,u64 start,struct page * page,unsigned int pg_offset)671 int btrfs_clean_io_failure(struct btrfs_inode *inode, u64 start,
672 			   struct page *page, unsigned int pg_offset)
673 {
674 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
675 	struct extent_io_tree *io_tree = &inode->io_tree;
676 	u64 ino = btrfs_ino(inode);
677 	u64 locked_start, locked_end;
678 	struct io_failure_record *failrec;
679 	int mirror;
680 	int ret;
681 
682 	failrec = get_failrec(inode, start);
683 	if (IS_ERR(failrec))
684 		return 0;
685 
686 	BUG_ON(!failrec->this_mirror);
687 
688 	if (sb_rdonly(fs_info->sb))
689 		goto out;
690 
691 	ret = find_first_extent_bit(io_tree, failrec->bytenr, &locked_start,
692 				    &locked_end, EXTENT_LOCKED, NULL);
693 	if (ret || locked_start > failrec->bytenr ||
694 	    locked_end < failrec->bytenr + failrec->len - 1)
695 		goto out;
696 
697 	mirror = failrec->this_mirror;
698 	do {
699 		mirror = prev_mirror(failrec, mirror);
700 		repair_io_failure(fs_info, ino, start, failrec->len,
701 				  failrec->logical, page, pg_offset, mirror);
702 	} while (mirror != failrec->failed_mirror);
703 
704 out:
705 	free_io_failure(inode, failrec);
706 	return 0;
707 }
708 
709 /*
710  * Can be called when
711  * - hold extent lock
712  * - under ordered extent
713  * - the inode is freeing
714  */
btrfs_free_io_failure_record(struct btrfs_inode * inode,u64 start,u64 end)715 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
716 {
717 	struct io_failure_record *failrec;
718 	struct rb_node *node, *next;
719 
720 	if (RB_EMPTY_ROOT(&inode->io_failure_tree))
721 		return;
722 
723 	spin_lock(&inode->io_failure_lock);
724 	node = rb_simple_search_first(&inode->io_failure_tree, start);
725 	while (node) {
726 		failrec = rb_entry(node, struct io_failure_record, rb_node);
727 		if (failrec->bytenr > end)
728 			break;
729 
730 		next = rb_next(node);
731 		rb_erase(&failrec->rb_node, &inode->io_failure_tree);
732 		kfree(failrec);
733 
734 		node = next;
735 	}
736 	spin_unlock(&inode->io_failure_lock);
737 }
738 
btrfs_get_io_failure_record(struct inode * inode,struct btrfs_bio * bbio,unsigned int bio_offset)739 static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode,
740 							     struct btrfs_bio *bbio,
741 							     unsigned int bio_offset)
742 {
743 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
744 	u64 start = bbio->file_offset + bio_offset;
745 	struct io_failure_record *failrec;
746 	const u32 sectorsize = fs_info->sectorsize;
747 	int ret;
748 
749 	failrec = get_failrec(BTRFS_I(inode), start);
750 	if (!IS_ERR(failrec)) {
751 		btrfs_debug(fs_info,
752 	"Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu",
753 			failrec->logical, failrec->bytenr, failrec->len);
754 		/*
755 		 * when data can be on disk more than twice, add to failrec here
756 		 * (e.g. with a list for failed_mirror) to make
757 		 * clean_io_failure() clean all those errors at once.
758 		 */
759 		ASSERT(failrec->this_mirror == bbio->mirror_num);
760 		ASSERT(failrec->len == fs_info->sectorsize);
761 		return failrec;
762 	}
763 
764 	failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
765 	if (!failrec)
766 		return ERR_PTR(-ENOMEM);
767 
768 	RB_CLEAR_NODE(&failrec->rb_node);
769 	failrec->bytenr = start;
770 	failrec->len = sectorsize;
771 	failrec->failed_mirror = bbio->mirror_num;
772 	failrec->this_mirror = bbio->mirror_num;
773 	failrec->logical = (bbio->iter.bi_sector << SECTOR_SHIFT) + bio_offset;
774 
775 	btrfs_debug(fs_info,
776 		    "new io failure record logical %llu start %llu",
777 		    failrec->logical, start);
778 
779 	failrec->num_copies = btrfs_num_copies(fs_info, failrec->logical, sectorsize);
780 	if (failrec->num_copies == 1) {
781 		/*
782 		 * We only have a single copy of the data, so don't bother with
783 		 * all the retry and error correction code that follows. No
784 		 * matter what the error is, it is very likely to persist.
785 		 */
786 		btrfs_debug(fs_info,
787 			"cannot repair logical %llu num_copies %d",
788 			failrec->logical, failrec->num_copies);
789 		kfree(failrec);
790 		return ERR_PTR(-EIO);
791 	}
792 
793 	/* Set the bits in the private failure tree */
794 	ret = insert_failrec(BTRFS_I(inode), failrec);
795 	if (ret) {
796 		kfree(failrec);
797 		return ERR_PTR(ret);
798 	}
799 
800 	return failrec;
801 }
802 
btrfs_repair_one_sector(struct inode * inode,struct btrfs_bio * failed_bbio,u32 bio_offset,struct page * page,unsigned int pgoff,submit_bio_hook_t * submit_bio_hook)803 int btrfs_repair_one_sector(struct inode *inode, struct btrfs_bio *failed_bbio,
804 			    u32 bio_offset, struct page *page, unsigned int pgoff,
805 			    submit_bio_hook_t *submit_bio_hook)
806 {
807 	u64 start = failed_bbio->file_offset + bio_offset;
808 	struct io_failure_record *failrec;
809 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
810 	struct bio *failed_bio = &failed_bbio->bio;
811 	const int icsum = bio_offset >> fs_info->sectorsize_bits;
812 	struct bio *repair_bio;
813 	struct btrfs_bio *repair_bbio;
814 
815 	btrfs_debug(fs_info,
816 		   "repair read error: read error at %llu", start);
817 
818 	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
819 
820 	failrec = btrfs_get_io_failure_record(inode, failed_bbio, bio_offset);
821 	if (IS_ERR(failrec))
822 		return PTR_ERR(failrec);
823 
824 	/*
825 	 * There are two premises:
826 	 * a) deliver good data to the caller
827 	 * b) correct the bad sectors on disk
828 	 *
829 	 * Since we're only doing repair for one sector, we only need to get
830 	 * a good copy of the failed sector and if we succeed, we have setup
831 	 * everything for repair_io_failure to do the rest for us.
832 	 */
833 	failrec->this_mirror = next_mirror(failrec, failrec->this_mirror);
834 	if (failrec->this_mirror == failrec->failed_mirror) {
835 		btrfs_debug(fs_info,
836 			"failed to repair num_copies %d this_mirror %d failed_mirror %d",
837 			failrec->num_copies, failrec->this_mirror, failrec->failed_mirror);
838 		free_io_failure(BTRFS_I(inode), failrec);
839 		return -EIO;
840 	}
841 
842 	repair_bio = btrfs_bio_alloc(1, REQ_OP_READ, failed_bbio->end_io,
843 				     failed_bbio->private);
844 	repair_bbio = btrfs_bio(repair_bio);
845 	repair_bbio->file_offset = start;
846 	repair_bio->bi_iter.bi_sector = failrec->logical >> 9;
847 
848 	if (failed_bbio->csum) {
849 		const u32 csum_size = fs_info->csum_size;
850 
851 		repair_bbio->csum = repair_bbio->csum_inline;
852 		memcpy(repair_bbio->csum,
853 		       failed_bbio->csum + csum_size * icsum, csum_size);
854 	}
855 
856 	bio_add_page(repair_bio, page, failrec->len, pgoff);
857 	repair_bbio->iter = repair_bio->bi_iter;
858 
859 	btrfs_debug(btrfs_sb(inode->i_sb),
860 		    "repair read error: submitting new read to mirror %d",
861 		    failrec->this_mirror);
862 
863 	/*
864 	 * At this point we have a bio, so any errors from submit_bio_hook()
865 	 * will be handled by the endio on the repair_bio, so we can't return an
866 	 * error here.
867 	 */
868 	submit_bio_hook(inode, repair_bio, failrec->this_mirror, 0);
869 	return BLK_STS_OK;
870 }
871 
end_page_read(struct page * page,bool uptodate,u64 start,u32 len)872 static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
873 {
874 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
875 
876 	ASSERT(page_offset(page) <= start &&
877 	       start + len <= page_offset(page) + PAGE_SIZE);
878 
879 	if (uptodate) {
880 		if (fsverity_active(page->mapping->host) &&
881 		    !PageError(page) &&
882 		    !PageUptodate(page) &&
883 		    start < i_size_read(page->mapping->host) &&
884 		    !fsverity_verify_page(page)) {
885 			btrfs_page_set_error(fs_info, page, start, len);
886 		} else {
887 			btrfs_page_set_uptodate(fs_info, page, start, len);
888 		}
889 	} else {
890 		btrfs_page_clear_uptodate(fs_info, page, start, len);
891 		btrfs_page_set_error(fs_info, page, start, len);
892 	}
893 
894 	if (!btrfs_is_subpage(fs_info, page))
895 		unlock_page(page);
896 	else
897 		btrfs_subpage_end_reader(fs_info, page, start, len);
898 }
899 
end_sector_io(struct page * page,u64 offset,bool uptodate)900 static void end_sector_io(struct page *page, u64 offset, bool uptodate)
901 {
902 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
903 	const u32 sectorsize = inode->root->fs_info->sectorsize;
904 	struct extent_state *cached = NULL;
905 
906 	end_page_read(page, uptodate, offset, sectorsize);
907 	if (uptodate)
908 		set_extent_uptodate(&inode->io_tree, offset,
909 				    offset + sectorsize - 1, &cached, GFP_ATOMIC);
910 	unlock_extent_atomic(&inode->io_tree, offset, offset + sectorsize - 1,
911 			     &cached);
912 }
913 
submit_data_read_repair(struct inode * inode,struct btrfs_bio * failed_bbio,u32 bio_offset,const struct bio_vec * bvec,unsigned int error_bitmap)914 static void submit_data_read_repair(struct inode *inode,
915 				    struct btrfs_bio *failed_bbio,
916 				    u32 bio_offset, const struct bio_vec *bvec,
917 				    unsigned int error_bitmap)
918 {
919 	const unsigned int pgoff = bvec->bv_offset;
920 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
921 	struct page *page = bvec->bv_page;
922 	const u64 start = page_offset(bvec->bv_page) + bvec->bv_offset;
923 	const u64 end = start + bvec->bv_len - 1;
924 	const u32 sectorsize = fs_info->sectorsize;
925 	const int nr_bits = (end + 1 - start) >> fs_info->sectorsize_bits;
926 	int i;
927 
928 	BUG_ON(bio_op(&failed_bbio->bio) == REQ_OP_WRITE);
929 
930 	/* This repair is only for data */
931 	ASSERT(is_data_inode(inode));
932 
933 	/* We're here because we had some read errors or csum mismatch */
934 	ASSERT(error_bitmap);
935 
936 	/*
937 	 * We only get called on buffered IO, thus page must be mapped and bio
938 	 * must not be cloned.
939 	 */
940 	ASSERT(page->mapping && !bio_flagged(&failed_bbio->bio, BIO_CLONED));
941 
942 	/* Iterate through all the sectors in the range */
943 	for (i = 0; i < nr_bits; i++) {
944 		const unsigned int offset = i * sectorsize;
945 		bool uptodate = false;
946 		int ret;
947 
948 		if (!(error_bitmap & (1U << i))) {
949 			/*
950 			 * This sector has no error, just end the page read
951 			 * and unlock the range.
952 			 */
953 			uptodate = true;
954 			goto next;
955 		}
956 
957 		ret = btrfs_repair_one_sector(inode, failed_bbio,
958 				bio_offset + offset, page, pgoff + offset,
959 				btrfs_submit_data_read_bio);
960 		if (!ret) {
961 			/*
962 			 * We have submitted the read repair, the page release
963 			 * will be handled by the endio function of the
964 			 * submitted repair bio.
965 			 * Thus we don't need to do any thing here.
966 			 */
967 			continue;
968 		}
969 		/*
970 		 * Continue on failed repair, otherwise the remaining sectors
971 		 * will not be properly unlocked.
972 		 */
973 next:
974 		end_sector_io(page, start + offset, uptodate);
975 	}
976 }
977 
978 /* lots and lots of room for performance fixes in the end_bio funcs */
979 
end_extent_writepage(struct page * page,int err,u64 start,u64 end)980 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
981 {
982 	struct btrfs_inode *inode;
983 	const bool uptodate = (err == 0);
984 	int ret = 0;
985 
986 	ASSERT(page && page->mapping);
987 	inode = BTRFS_I(page->mapping->host);
988 	btrfs_writepage_endio_finish_ordered(inode, page, start, end, uptodate);
989 
990 	if (!uptodate) {
991 		const struct btrfs_fs_info *fs_info = inode->root->fs_info;
992 		u32 len;
993 
994 		ASSERT(end + 1 - start <= U32_MAX);
995 		len = end + 1 - start;
996 
997 		btrfs_page_clear_uptodate(fs_info, page, start, len);
998 		btrfs_page_set_error(fs_info, page, start, len);
999 		ret = err < 0 ? err : -EIO;
1000 		mapping_set_error(page->mapping, ret);
1001 	}
1002 }
1003 
1004 /*
1005  * after a writepage IO is done, we need to:
1006  * clear the uptodate bits on error
1007  * clear the writeback bits in the extent tree for this IO
1008  * end_page_writeback if the page has no more pending IO
1009  *
1010  * Scheduling is not allowed, so the extent state tree is expected
1011  * to have one and only one object corresponding to this IO.
1012  */
end_bio_extent_writepage(struct btrfs_bio * bbio)1013 static void end_bio_extent_writepage(struct btrfs_bio *bbio)
1014 {
1015 	struct bio *bio = &bbio->bio;
1016 	int error = blk_status_to_errno(bio->bi_status);
1017 	struct bio_vec *bvec;
1018 	u64 start;
1019 	u64 end;
1020 	struct bvec_iter_all iter_all;
1021 	bool first_bvec = true;
1022 
1023 	ASSERT(!bio_flagged(bio, BIO_CLONED));
1024 	bio_for_each_segment_all(bvec, bio, iter_all) {
1025 		struct page *page = bvec->bv_page;
1026 		struct inode *inode = page->mapping->host;
1027 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1028 		const u32 sectorsize = fs_info->sectorsize;
1029 
1030 		/* Our read/write should always be sector aligned. */
1031 		if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
1032 			btrfs_err(fs_info,
1033 		"partial page write in btrfs with offset %u and length %u",
1034 				  bvec->bv_offset, bvec->bv_len);
1035 		else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
1036 			btrfs_info(fs_info,
1037 		"incomplete page write with offset %u and length %u",
1038 				   bvec->bv_offset, bvec->bv_len);
1039 
1040 		start = page_offset(page) + bvec->bv_offset;
1041 		end = start + bvec->bv_len - 1;
1042 
1043 		if (first_bvec) {
1044 			btrfs_record_physical_zoned(inode, start, bio);
1045 			first_bvec = false;
1046 		}
1047 
1048 		end_extent_writepage(page, error, start, end);
1049 
1050 		btrfs_page_clear_writeback(fs_info, page, start, bvec->bv_len);
1051 	}
1052 
1053 	bio_put(bio);
1054 }
1055 
1056 /*
1057  * Record previously processed extent range
1058  *
1059  * For endio_readpage_release_extent() to handle a full extent range, reducing
1060  * the extent io operations.
1061  */
1062 struct processed_extent {
1063 	struct btrfs_inode *inode;
1064 	/* Start of the range in @inode */
1065 	u64 start;
1066 	/* End of the range in @inode */
1067 	u64 end;
1068 	bool uptodate;
1069 };
1070 
1071 /*
1072  * Try to release processed extent range
1073  *
1074  * May not release the extent range right now if the current range is
1075  * contiguous to processed extent.
1076  *
1077  * Will release processed extent when any of @inode, @uptodate, the range is
1078  * no longer contiguous to the processed range.
1079  *
1080  * Passing @inode == NULL will force processed extent to be released.
1081  */
endio_readpage_release_extent(struct processed_extent * processed,struct btrfs_inode * inode,u64 start,u64 end,bool uptodate)1082 static void endio_readpage_release_extent(struct processed_extent *processed,
1083 			      struct btrfs_inode *inode, u64 start, u64 end,
1084 			      bool uptodate)
1085 {
1086 	struct extent_state *cached = NULL;
1087 	struct extent_io_tree *tree;
1088 
1089 	/* The first extent, initialize @processed */
1090 	if (!processed->inode)
1091 		goto update;
1092 
1093 	/*
1094 	 * Contiguous to processed extent, just uptodate the end.
1095 	 *
1096 	 * Several things to notice:
1097 	 *
1098 	 * - bio can be merged as long as on-disk bytenr is contiguous
1099 	 *   This means we can have page belonging to other inodes, thus need to
1100 	 *   check if the inode still matches.
1101 	 * - bvec can contain range beyond current page for multi-page bvec
1102 	 *   Thus we need to do processed->end + 1 >= start check
1103 	 */
1104 	if (processed->inode == inode && processed->uptodate == uptodate &&
1105 	    processed->end + 1 >= start && end >= processed->end) {
1106 		processed->end = end;
1107 		return;
1108 	}
1109 
1110 	tree = &processed->inode->io_tree;
1111 	/*
1112 	 * Now we don't have range contiguous to the processed range, release
1113 	 * the processed range now.
1114 	 */
1115 	unlock_extent_atomic(tree, processed->start, processed->end, &cached);
1116 
1117 update:
1118 	/* Update processed to current range */
1119 	processed->inode = inode;
1120 	processed->start = start;
1121 	processed->end = end;
1122 	processed->uptodate = uptodate;
1123 }
1124 
begin_page_read(struct btrfs_fs_info * fs_info,struct page * page)1125 static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
1126 {
1127 	ASSERT(PageLocked(page));
1128 	if (!btrfs_is_subpage(fs_info, page))
1129 		return;
1130 
1131 	ASSERT(PagePrivate(page));
1132 	btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
1133 }
1134 
1135 /*
1136  * Find extent buffer for a givne bytenr.
1137  *
1138  * This is for end_bio_extent_readpage(), thus we can't do any unsafe locking
1139  * in endio context.
1140  */
find_extent_buffer_readpage(struct btrfs_fs_info * fs_info,struct page * page,u64 bytenr)1141 static struct extent_buffer *find_extent_buffer_readpage(
1142 		struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
1143 {
1144 	struct extent_buffer *eb;
1145 
1146 	/*
1147 	 * For regular sectorsize, we can use page->private to grab extent
1148 	 * buffer
1149 	 */
1150 	if (fs_info->nodesize >= PAGE_SIZE) {
1151 		ASSERT(PagePrivate(page) && page->private);
1152 		return (struct extent_buffer *)page->private;
1153 	}
1154 
1155 	/* For subpage case, we need to lookup buffer radix tree */
1156 	rcu_read_lock();
1157 	eb = radix_tree_lookup(&fs_info->buffer_radix,
1158 			       bytenr >> fs_info->sectorsize_bits);
1159 	rcu_read_unlock();
1160 	ASSERT(eb);
1161 	return eb;
1162 }
1163 
1164 /*
1165  * after a readpage IO is done, we need to:
1166  * clear the uptodate bits on error
1167  * set the uptodate bits if things worked
1168  * set the page up to date if all extents in the tree are uptodate
1169  * clear the lock bit in the extent tree
1170  * unlock the page if there are no other extents locked for it
1171  *
1172  * Scheduling is not allowed, so the extent state tree is expected
1173  * to have one and only one object corresponding to this IO.
1174  */
end_bio_extent_readpage(struct btrfs_bio * bbio)1175 static void end_bio_extent_readpage(struct btrfs_bio *bbio)
1176 {
1177 	struct bio *bio = &bbio->bio;
1178 	struct bio_vec *bvec;
1179 	struct processed_extent processed = { 0 };
1180 	/*
1181 	 * The offset to the beginning of a bio, since one bio can never be
1182 	 * larger than UINT_MAX, u32 here is enough.
1183 	 */
1184 	u32 bio_offset = 0;
1185 	int mirror;
1186 	struct bvec_iter_all iter_all;
1187 
1188 	ASSERT(!bio_flagged(bio, BIO_CLONED));
1189 	bio_for_each_segment_all(bvec, bio, iter_all) {
1190 		bool uptodate = !bio->bi_status;
1191 		struct page *page = bvec->bv_page;
1192 		struct inode *inode = page->mapping->host;
1193 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1194 		const u32 sectorsize = fs_info->sectorsize;
1195 		unsigned int error_bitmap = (unsigned int)-1;
1196 		bool repair = false;
1197 		u64 start;
1198 		u64 end;
1199 		u32 len;
1200 
1201 		btrfs_debug(fs_info,
1202 			"end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
1203 			bio->bi_iter.bi_sector, bio->bi_status,
1204 			bbio->mirror_num);
1205 
1206 		/*
1207 		 * We always issue full-sector reads, but if some block in a
1208 		 * page fails to read, blk_update_request() will advance
1209 		 * bv_offset and adjust bv_len to compensate.  Print a warning
1210 		 * for unaligned offsets, and an error if they don't add up to
1211 		 * a full sector.
1212 		 */
1213 		if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
1214 			btrfs_err(fs_info,
1215 		"partial page read in btrfs with offset %u and length %u",
1216 				  bvec->bv_offset, bvec->bv_len);
1217 		else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
1218 				     sectorsize))
1219 			btrfs_info(fs_info,
1220 		"incomplete page read with offset %u and length %u",
1221 				   bvec->bv_offset, bvec->bv_len);
1222 
1223 		start = page_offset(page) + bvec->bv_offset;
1224 		end = start + bvec->bv_len - 1;
1225 		len = bvec->bv_len;
1226 
1227 		mirror = bbio->mirror_num;
1228 		if (likely(uptodate)) {
1229 			if (is_data_inode(inode)) {
1230 				error_bitmap = btrfs_verify_data_csum(bbio,
1231 						bio_offset, page, start, end);
1232 				if (error_bitmap)
1233 					uptodate = false;
1234 			} else {
1235 				if (btrfs_validate_metadata_buffer(bbio,
1236 						page, start, end, mirror))
1237 					uptodate = false;
1238 			}
1239 		}
1240 
1241 		if (likely(uptodate)) {
1242 			loff_t i_size = i_size_read(inode);
1243 			pgoff_t end_index = i_size >> PAGE_SHIFT;
1244 
1245 			btrfs_clean_io_failure(BTRFS_I(inode), start, page, 0);
1246 
1247 			/*
1248 			 * Zero out the remaining part if this range straddles
1249 			 * i_size.
1250 			 *
1251 			 * Here we should only zero the range inside the bvec,
1252 			 * not touch anything else.
1253 			 *
1254 			 * NOTE: i_size is exclusive while end is inclusive.
1255 			 */
1256 			if (page->index == end_index && i_size <= end) {
1257 				u32 zero_start = max(offset_in_page(i_size),
1258 						     offset_in_page(start));
1259 
1260 				zero_user_segment(page, zero_start,
1261 						  offset_in_page(end) + 1);
1262 			}
1263 		} else if (is_data_inode(inode)) {
1264 			/*
1265 			 * Only try to repair bios that actually made it to a
1266 			 * device.  If the bio failed to be submitted mirror
1267 			 * is 0 and we need to fail it without retrying.
1268 			 *
1269 			 * This also includes the high level bios for compressed
1270 			 * extents - these never make it to a device and repair
1271 			 * is already handled on the lower compressed bio.
1272 			 */
1273 			if (mirror > 0)
1274 				repair = true;
1275 		} else {
1276 			struct extent_buffer *eb;
1277 
1278 			eb = find_extent_buffer_readpage(fs_info, page, start);
1279 			set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
1280 			eb->read_mirror = mirror;
1281 			atomic_dec(&eb->io_pages);
1282 		}
1283 
1284 		if (repair) {
1285 			/*
1286 			 * submit_data_read_repair() will handle all the good
1287 			 * and bad sectors, we just continue to the next bvec.
1288 			 */
1289 			submit_data_read_repair(inode, bbio, bio_offset, bvec,
1290 						error_bitmap);
1291 		} else {
1292 			/* Update page status and unlock */
1293 			end_page_read(page, uptodate, start, len);
1294 			endio_readpage_release_extent(&processed, BTRFS_I(inode),
1295 					start, end, PageUptodate(page));
1296 		}
1297 
1298 		ASSERT(bio_offset + len > bio_offset);
1299 		bio_offset += len;
1300 
1301 	}
1302 	/* Release the last extent */
1303 	endio_readpage_release_extent(&processed, NULL, 0, 0, false);
1304 	btrfs_bio_free_csum(bbio);
1305 	bio_put(bio);
1306 }
1307 
1308 /**
1309  * Populate every free slot in a provided array with pages.
1310  *
1311  * @nr_pages:   number of pages to allocate
1312  * @page_array: the array to fill with pages; any existing non-null entries in
1313  * 		the array will be skipped
1314  *
1315  * Return: 0        if all pages were able to be allocated;
1316  *         -ENOMEM  otherwise, and the caller is responsible for freeing all
1317  *                  non-null page pointers in the array.
1318  */
btrfs_alloc_page_array(unsigned int nr_pages,struct page ** page_array)1319 int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
1320 {
1321 	unsigned int allocated;
1322 
1323 	for (allocated = 0; allocated < nr_pages;) {
1324 		unsigned int last = allocated;
1325 
1326 		allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
1327 
1328 		if (allocated == nr_pages)
1329 			return 0;
1330 
1331 		/*
1332 		 * During this iteration, no page could be allocated, even
1333 		 * though alloc_pages_bulk_array() falls back to alloc_page()
1334 		 * if  it could not bulk-allocate. So we must be out of memory.
1335 		 */
1336 		if (allocated == last)
1337 			return -ENOMEM;
1338 
1339 		memalloc_retry_wait(GFP_NOFS);
1340 	}
1341 	return 0;
1342 }
1343 
1344 /**
1345  * Attempt to add a page to bio
1346  *
1347  * @bio_ctrl:	record both the bio, and its bio_flags
1348  * @page:	page to add to the bio
1349  * @disk_bytenr:  offset of the new bio or to check whether we are adding
1350  *                a contiguous page to the previous one
1351  * @size:	portion of page that we want to write
1352  * @pg_offset:	starting offset in the page
1353  * @compress_type:   compression type of the current bio to see if we can merge them
1354  *
1355  * Attempt to add a page to bio considering stripe alignment etc.
1356  *
1357  * Return >= 0 for the number of bytes added to the bio.
1358  * Can return 0 if the current bio is already at stripe/zone boundary.
1359  * Return <0 for error.
1360  */
btrfs_bio_add_page(struct btrfs_bio_ctrl * bio_ctrl,struct page * page,u64 disk_bytenr,unsigned int size,unsigned int pg_offset,enum btrfs_compression_type compress_type)1361 static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl,
1362 			      struct page *page,
1363 			      u64 disk_bytenr, unsigned int size,
1364 			      unsigned int pg_offset,
1365 			      enum btrfs_compression_type compress_type)
1366 {
1367 	struct bio *bio = bio_ctrl->bio;
1368 	u32 bio_size = bio->bi_iter.bi_size;
1369 	u32 real_size;
1370 	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
1371 	bool contig = false;
1372 	int ret;
1373 
1374 	ASSERT(bio);
1375 	/* The limit should be calculated when bio_ctrl->bio is allocated */
1376 	ASSERT(bio_ctrl->len_to_oe_boundary && bio_ctrl->len_to_stripe_boundary);
1377 	if (bio_ctrl->compress_type != compress_type)
1378 		return 0;
1379 
1380 
1381 	if (bio->bi_iter.bi_size == 0) {
1382 		/* We can always add a page into an empty bio. */
1383 		contig = true;
1384 	} else if (bio_ctrl->compress_type == BTRFS_COMPRESS_NONE) {
1385 		struct bio_vec *bvec = bio_last_bvec_all(bio);
1386 
1387 		/*
1388 		 * The contig check requires the following conditions to be met:
1389 		 * 1) The pages are belonging to the same inode
1390 		 *    This is implied by the call chain.
1391 		 *
1392 		 * 2) The range has adjacent logical bytenr
1393 		 *
1394 		 * 3) The range has adjacent file offset
1395 		 *    This is required for the usage of btrfs_bio->file_offset.
1396 		 */
1397 		if (bio_end_sector(bio) == sector &&
1398 		    page_offset(bvec->bv_page) + bvec->bv_offset +
1399 		    bvec->bv_len == page_offset(page) + pg_offset)
1400 			contig = true;
1401 	} else {
1402 		/*
1403 		 * For compression, all IO should have its logical bytenr
1404 		 * set to the starting bytenr of the compressed extent.
1405 		 */
1406 		contig = bio->bi_iter.bi_sector == sector;
1407 	}
1408 
1409 	if (!contig)
1410 		return 0;
1411 
1412 	real_size = min(bio_ctrl->len_to_oe_boundary,
1413 			bio_ctrl->len_to_stripe_boundary) - bio_size;
1414 	real_size = min(real_size, size);
1415 
1416 	/*
1417 	 * If real_size is 0, never call bio_add_*_page(), as even size is 0,
1418 	 * bio will still execute its endio function on the page!
1419 	 */
1420 	if (real_size == 0)
1421 		return 0;
1422 
1423 	if (bio_op(bio) == REQ_OP_ZONE_APPEND)
1424 		ret = bio_add_zone_append_page(bio, page, real_size, pg_offset);
1425 	else
1426 		ret = bio_add_page(bio, page, real_size, pg_offset);
1427 
1428 	return ret;
1429 }
1430 
calc_bio_boundaries(struct btrfs_bio_ctrl * bio_ctrl,struct btrfs_inode * inode,u64 file_offset)1431 static int calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl,
1432 			       struct btrfs_inode *inode, u64 file_offset)
1433 {
1434 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1435 	struct btrfs_io_geometry geom;
1436 	struct btrfs_ordered_extent *ordered;
1437 	struct extent_map *em;
1438 	u64 logical = (bio_ctrl->bio->bi_iter.bi_sector << SECTOR_SHIFT);
1439 	int ret;
1440 
1441 	/*
1442 	 * Pages for compressed extent are never submitted to disk directly,
1443 	 * thus it has no real boundary, just set them to U32_MAX.
1444 	 *
1445 	 * The split happens for real compressed bio, which happens in
1446 	 * btrfs_submit_compressed_read/write().
1447 	 */
1448 	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
1449 		bio_ctrl->len_to_oe_boundary = U32_MAX;
1450 		bio_ctrl->len_to_stripe_boundary = U32_MAX;
1451 		return 0;
1452 	}
1453 	em = btrfs_get_chunk_map(fs_info, logical, fs_info->sectorsize);
1454 	if (IS_ERR(em))
1455 		return PTR_ERR(em);
1456 	ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio_ctrl->bio),
1457 				    logical, &geom);
1458 	free_extent_map(em);
1459 	if (ret < 0) {
1460 		return ret;
1461 	}
1462 	if (geom.len > U32_MAX)
1463 		bio_ctrl->len_to_stripe_boundary = U32_MAX;
1464 	else
1465 		bio_ctrl->len_to_stripe_boundary = (u32)geom.len;
1466 
1467 	if (bio_op(bio_ctrl->bio) != REQ_OP_ZONE_APPEND) {
1468 		bio_ctrl->len_to_oe_boundary = U32_MAX;
1469 		return 0;
1470 	}
1471 
1472 	/* Ordered extent not yet created, so we're good */
1473 	ordered = btrfs_lookup_ordered_extent(inode, file_offset);
1474 	if (!ordered) {
1475 		bio_ctrl->len_to_oe_boundary = U32_MAX;
1476 		return 0;
1477 	}
1478 
1479 	bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
1480 		ordered->disk_bytenr + ordered->disk_num_bytes - logical);
1481 	btrfs_put_ordered_extent(ordered);
1482 	return 0;
1483 }
1484 
alloc_new_bio(struct btrfs_inode * inode,struct btrfs_bio_ctrl * bio_ctrl,struct writeback_control * wbc,blk_opf_t opf,u64 disk_bytenr,u32 offset,u64 file_offset,enum btrfs_compression_type compress_type)1485 static int alloc_new_bio(struct btrfs_inode *inode,
1486 			 struct btrfs_bio_ctrl *bio_ctrl,
1487 			 struct writeback_control *wbc,
1488 			 blk_opf_t opf,
1489 			 u64 disk_bytenr, u32 offset, u64 file_offset,
1490 			 enum btrfs_compression_type compress_type)
1491 {
1492 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1493 	struct bio *bio;
1494 	int ret;
1495 
1496 	ASSERT(bio_ctrl->end_io_func);
1497 
1498 	bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, bio_ctrl->end_io_func, NULL);
1499 	/*
1500 	 * For compressed page range, its disk_bytenr is always @disk_bytenr
1501 	 * passed in, no matter if we have added any range into previous bio.
1502 	 */
1503 	if (compress_type != BTRFS_COMPRESS_NONE)
1504 		bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
1505 	else
1506 		bio->bi_iter.bi_sector = (disk_bytenr + offset) >> SECTOR_SHIFT;
1507 	bio_ctrl->bio = bio;
1508 	bio_ctrl->compress_type = compress_type;
1509 	ret = calc_bio_boundaries(bio_ctrl, inode, file_offset);
1510 	if (ret < 0)
1511 		goto error;
1512 
1513 	if (wbc) {
1514 		/*
1515 		 * For Zone append we need the correct block_device that we are
1516 		 * going to write to set in the bio to be able to respect the
1517 		 * hardware limitation.  Look it up here:
1518 		 */
1519 		if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
1520 			struct btrfs_device *dev;
1521 
1522 			dev = btrfs_zoned_get_device(fs_info, disk_bytenr,
1523 						     fs_info->sectorsize);
1524 			if (IS_ERR(dev)) {
1525 				ret = PTR_ERR(dev);
1526 				goto error;
1527 			}
1528 
1529 			bio_set_dev(bio, dev->bdev);
1530 		} else {
1531 			/*
1532 			 * Otherwise pick the last added device to support
1533 			 * cgroup writeback.  For multi-device file systems this
1534 			 * means blk-cgroup policies have to always be set on the
1535 			 * last added/replaced device.  This is a bit odd but has
1536 			 * been like that for a long time.
1537 			 */
1538 			bio_set_dev(bio, fs_info->fs_devices->latest_dev->bdev);
1539 		}
1540 		wbc_init_bio(wbc, bio);
1541 	} else {
1542 		ASSERT(bio_op(bio) != REQ_OP_ZONE_APPEND);
1543 	}
1544 	return 0;
1545 error:
1546 	bio_ctrl->bio = NULL;
1547 	btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret));
1548 	return ret;
1549 }
1550 
1551 /*
1552  * @opf:	bio REQ_OP_* and REQ_* flags as one value
1553  * @wbc:	optional writeback control for io accounting
1554  * @disk_bytenr: logical bytenr where the write will be
1555  * @page:	page to add to the bio
1556  * @size:	portion of page that we want to write to
1557  * @pg_offset:	offset of the new bio or to check whether we are adding
1558  *              a contiguous page to the previous one
1559  * @compress_type:   compress type for current bio
1560  *
1561  * The will either add the page into the existing @bio_ctrl->bio, or allocate a
1562  * new one in @bio_ctrl->bio.
1563  * The mirror number for this IO should already be initizlied in
1564  * @bio_ctrl->mirror_num.
1565  */
submit_extent_page(blk_opf_t opf,struct writeback_control * wbc,struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,struct page * page,size_t size,unsigned long pg_offset,enum btrfs_compression_type compress_type,bool force_bio_submit)1566 static int submit_extent_page(blk_opf_t opf,
1567 			      struct writeback_control *wbc,
1568 			      struct btrfs_bio_ctrl *bio_ctrl,
1569 			      u64 disk_bytenr, struct page *page,
1570 			      size_t size, unsigned long pg_offset,
1571 			      enum btrfs_compression_type compress_type,
1572 			      bool force_bio_submit)
1573 {
1574 	int ret = 0;
1575 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
1576 	unsigned int cur = pg_offset;
1577 
1578 	ASSERT(bio_ctrl);
1579 
1580 	ASSERT(pg_offset < PAGE_SIZE && size <= PAGE_SIZE &&
1581 	       pg_offset + size <= PAGE_SIZE);
1582 
1583 	ASSERT(bio_ctrl->end_io_func);
1584 
1585 	if (force_bio_submit)
1586 		submit_one_bio(bio_ctrl);
1587 
1588 	while (cur < pg_offset + size) {
1589 		u32 offset = cur - pg_offset;
1590 		int added;
1591 
1592 		/* Allocate new bio if needed */
1593 		if (!bio_ctrl->bio) {
1594 			ret = alloc_new_bio(inode, bio_ctrl, wbc, opf,
1595 					    disk_bytenr, offset,
1596 					    page_offset(page) + cur,
1597 					    compress_type);
1598 			if (ret < 0)
1599 				return ret;
1600 		}
1601 		/*
1602 		 * We must go through btrfs_bio_add_page() to ensure each
1603 		 * page range won't cross various boundaries.
1604 		 */
1605 		if (compress_type != BTRFS_COMPRESS_NONE)
1606 			added = btrfs_bio_add_page(bio_ctrl, page, disk_bytenr,
1607 					size - offset, pg_offset + offset,
1608 					compress_type);
1609 		else
1610 			added = btrfs_bio_add_page(bio_ctrl, page,
1611 					disk_bytenr + offset, size - offset,
1612 					pg_offset + offset, compress_type);
1613 
1614 		/* Metadata page range should never be split */
1615 		if (!is_data_inode(&inode->vfs_inode))
1616 			ASSERT(added == 0 || added == size - offset);
1617 
1618 		/* At least we added some page, update the account */
1619 		if (wbc && added)
1620 			wbc_account_cgroup_owner(wbc, page, added);
1621 
1622 		/* We have reached boundary, submit right now */
1623 		if (added < size - offset) {
1624 			/* The bio should contain some page(s) */
1625 			ASSERT(bio_ctrl->bio->bi_iter.bi_size);
1626 			submit_one_bio(bio_ctrl);
1627 		}
1628 		cur += added;
1629 	}
1630 	return 0;
1631 }
1632 
attach_extent_buffer_page(struct extent_buffer * eb,struct page * page,struct btrfs_subpage * prealloc)1633 static int attach_extent_buffer_page(struct extent_buffer *eb,
1634 				     struct page *page,
1635 				     struct btrfs_subpage *prealloc)
1636 {
1637 	struct btrfs_fs_info *fs_info = eb->fs_info;
1638 	int ret = 0;
1639 
1640 	/*
1641 	 * If the page is mapped to btree inode, we should hold the private
1642 	 * lock to prevent race.
1643 	 * For cloned or dummy extent buffers, their pages are not mapped and
1644 	 * will not race with any other ebs.
1645 	 */
1646 	if (page->mapping)
1647 		lockdep_assert_held(&page->mapping->private_lock);
1648 
1649 	if (fs_info->nodesize >= PAGE_SIZE) {
1650 		if (!PagePrivate(page))
1651 			attach_page_private(page, eb);
1652 		else
1653 			WARN_ON(page->private != (unsigned long)eb);
1654 		return 0;
1655 	}
1656 
1657 	/* Already mapped, just free prealloc */
1658 	if (PagePrivate(page)) {
1659 		btrfs_free_subpage(prealloc);
1660 		return 0;
1661 	}
1662 
1663 	if (prealloc)
1664 		/* Has preallocated memory for subpage */
1665 		attach_page_private(page, prealloc);
1666 	else
1667 		/* Do new allocation to attach subpage */
1668 		ret = btrfs_attach_subpage(fs_info, page,
1669 					   BTRFS_SUBPAGE_METADATA);
1670 	return ret;
1671 }
1672 
set_page_extent_mapped(struct page * page)1673 int set_page_extent_mapped(struct page *page)
1674 {
1675 	struct btrfs_fs_info *fs_info;
1676 
1677 	ASSERT(page->mapping);
1678 
1679 	if (PagePrivate(page))
1680 		return 0;
1681 
1682 	fs_info = btrfs_sb(page->mapping->host->i_sb);
1683 
1684 	if (btrfs_is_subpage(fs_info, page))
1685 		return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
1686 
1687 	attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
1688 	return 0;
1689 }
1690 
clear_page_extent_mapped(struct page * page)1691 void clear_page_extent_mapped(struct page *page)
1692 {
1693 	struct btrfs_fs_info *fs_info;
1694 
1695 	ASSERT(page->mapping);
1696 
1697 	if (!PagePrivate(page))
1698 		return;
1699 
1700 	fs_info = btrfs_sb(page->mapping->host->i_sb);
1701 	if (btrfs_is_subpage(fs_info, page))
1702 		return btrfs_detach_subpage(fs_info, page);
1703 
1704 	detach_page_private(page);
1705 }
1706 
1707 static struct extent_map *
__get_extent_map(struct inode * inode,struct page * page,size_t pg_offset,u64 start,u64 len,struct extent_map ** em_cached)1708 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
1709 		 u64 start, u64 len, struct extent_map **em_cached)
1710 {
1711 	struct extent_map *em;
1712 
1713 	if (em_cached && *em_cached) {
1714 		em = *em_cached;
1715 		if (extent_map_in_tree(em) && start >= em->start &&
1716 		    start < extent_map_end(em)) {
1717 			refcount_inc(&em->refs);
1718 			return em;
1719 		}
1720 
1721 		free_extent_map(em);
1722 		*em_cached = NULL;
1723 	}
1724 
1725 	em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
1726 	if (em_cached && !IS_ERR(em)) {
1727 		BUG_ON(*em_cached);
1728 		refcount_inc(&em->refs);
1729 		*em_cached = em;
1730 	}
1731 	return em;
1732 }
1733 /*
1734  * basic readpage implementation.  Locked extent state structs are inserted
1735  * into the tree that are removed when the IO is done (by the end_io
1736  * handlers)
1737  * XXX JDM: This needs looking at to ensure proper page locking
1738  * return 0 on success, otherwise return error
1739  */
btrfs_do_readpage(struct page * page,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,blk_opf_t read_flags,u64 * prev_em_start)1740 static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
1741 		      struct btrfs_bio_ctrl *bio_ctrl,
1742 		      blk_opf_t read_flags, u64 *prev_em_start)
1743 {
1744 	struct inode *inode = page->mapping->host;
1745 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1746 	u64 start = page_offset(page);
1747 	const u64 end = start + PAGE_SIZE - 1;
1748 	u64 cur = start;
1749 	u64 extent_offset;
1750 	u64 last_byte = i_size_read(inode);
1751 	u64 block_start;
1752 	struct extent_map *em;
1753 	int ret = 0;
1754 	size_t pg_offset = 0;
1755 	size_t iosize;
1756 	size_t blocksize = inode->i_sb->s_blocksize;
1757 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1758 
1759 	ret = set_page_extent_mapped(page);
1760 	if (ret < 0) {
1761 		unlock_extent(tree, start, end, NULL);
1762 		btrfs_page_set_error(fs_info, page, start, PAGE_SIZE);
1763 		unlock_page(page);
1764 		goto out;
1765 	}
1766 
1767 	if (page->index == last_byte >> PAGE_SHIFT) {
1768 		size_t zero_offset = offset_in_page(last_byte);
1769 
1770 		if (zero_offset) {
1771 			iosize = PAGE_SIZE - zero_offset;
1772 			memzero_page(page, zero_offset, iosize);
1773 		}
1774 	}
1775 	bio_ctrl->end_io_func = end_bio_extent_readpage;
1776 	begin_page_read(fs_info, page);
1777 	while (cur <= end) {
1778 		unsigned long this_bio_flag = 0;
1779 		bool force_bio_submit = false;
1780 		u64 disk_bytenr;
1781 
1782 		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1783 		if (cur >= last_byte) {
1784 			struct extent_state *cached = NULL;
1785 
1786 			iosize = PAGE_SIZE - pg_offset;
1787 			memzero_page(page, pg_offset, iosize);
1788 			set_extent_uptodate(tree, cur, cur + iosize - 1,
1789 					    &cached, GFP_NOFS);
1790 			unlock_extent(tree, cur, cur + iosize - 1, &cached);
1791 			end_page_read(page, true, cur, iosize);
1792 			break;
1793 		}
1794 		em = __get_extent_map(inode, page, pg_offset, cur,
1795 				      end - cur + 1, em_cached);
1796 		if (IS_ERR(em)) {
1797 			unlock_extent(tree, cur, end, NULL);
1798 			end_page_read(page, false, cur, end + 1 - cur);
1799 			ret = PTR_ERR(em);
1800 			break;
1801 		}
1802 		extent_offset = cur - em->start;
1803 		BUG_ON(extent_map_end(em) <= cur);
1804 		BUG_ON(end < cur);
1805 
1806 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1807 			this_bio_flag = em->compress_type;
1808 
1809 		iosize = min(extent_map_end(em) - cur, end - cur + 1);
1810 		iosize = ALIGN(iosize, blocksize);
1811 		if (this_bio_flag != BTRFS_COMPRESS_NONE)
1812 			disk_bytenr = em->block_start;
1813 		else
1814 			disk_bytenr = em->block_start + extent_offset;
1815 		block_start = em->block_start;
1816 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1817 			block_start = EXTENT_MAP_HOLE;
1818 
1819 		/*
1820 		 * If we have a file range that points to a compressed extent
1821 		 * and it's followed by a consecutive file range that points
1822 		 * to the same compressed extent (possibly with a different
1823 		 * offset and/or length, so it either points to the whole extent
1824 		 * or only part of it), we must make sure we do not submit a
1825 		 * single bio to populate the pages for the 2 ranges because
1826 		 * this makes the compressed extent read zero out the pages
1827 		 * belonging to the 2nd range. Imagine the following scenario:
1828 		 *
1829 		 *  File layout
1830 		 *  [0 - 8K]                     [8K - 24K]
1831 		 *    |                               |
1832 		 *    |                               |
1833 		 * points to extent X,         points to extent X,
1834 		 * offset 4K, length of 8K     offset 0, length 16K
1835 		 *
1836 		 * [extent X, compressed length = 4K uncompressed length = 16K]
1837 		 *
1838 		 * If the bio to read the compressed extent covers both ranges,
1839 		 * it will decompress extent X into the pages belonging to the
1840 		 * first range and then it will stop, zeroing out the remaining
1841 		 * pages that belong to the other range that points to extent X.
1842 		 * So here we make sure we submit 2 bios, one for the first
1843 		 * range and another one for the third range. Both will target
1844 		 * the same physical extent from disk, but we can't currently
1845 		 * make the compressed bio endio callback populate the pages
1846 		 * for both ranges because each compressed bio is tightly
1847 		 * coupled with a single extent map, and each range can have
1848 		 * an extent map with a different offset value relative to the
1849 		 * uncompressed data of our extent and different lengths. This
1850 		 * is a corner case so we prioritize correctness over
1851 		 * non-optimal behavior (submitting 2 bios for the same extent).
1852 		 */
1853 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1854 		    prev_em_start && *prev_em_start != (u64)-1 &&
1855 		    *prev_em_start != em->start)
1856 			force_bio_submit = true;
1857 
1858 		if (prev_em_start)
1859 			*prev_em_start = em->start;
1860 
1861 		free_extent_map(em);
1862 		em = NULL;
1863 
1864 		/* we've found a hole, just zero and go on */
1865 		if (block_start == EXTENT_MAP_HOLE) {
1866 			struct extent_state *cached = NULL;
1867 
1868 			memzero_page(page, pg_offset, iosize);
1869 
1870 			set_extent_uptodate(tree, cur, cur + iosize - 1,
1871 					    &cached, GFP_NOFS);
1872 			unlock_extent(tree, cur, cur + iosize - 1, &cached);
1873 			end_page_read(page, true, cur, iosize);
1874 			cur = cur + iosize;
1875 			pg_offset += iosize;
1876 			continue;
1877 		}
1878 		/* the get_extent function already copied into the page */
1879 		if (block_start == EXTENT_MAP_INLINE) {
1880 			unlock_extent(tree, cur, cur + iosize - 1, NULL);
1881 			end_page_read(page, true, cur, iosize);
1882 			cur = cur + iosize;
1883 			pg_offset += iosize;
1884 			continue;
1885 		}
1886 
1887 		ret = submit_extent_page(REQ_OP_READ | read_flags, NULL,
1888 					 bio_ctrl, disk_bytenr, page, iosize,
1889 					 pg_offset, this_bio_flag,
1890 					 force_bio_submit);
1891 		if (ret) {
1892 			/*
1893 			 * We have to unlock the remaining range, or the page
1894 			 * will never be unlocked.
1895 			 */
1896 			unlock_extent(tree, cur, end, NULL);
1897 			end_page_read(page, false, cur, end + 1 - cur);
1898 			goto out;
1899 		}
1900 		cur = cur + iosize;
1901 		pg_offset += iosize;
1902 	}
1903 out:
1904 	return ret;
1905 }
1906 
btrfs_read_folio(struct file * file,struct folio * folio)1907 int btrfs_read_folio(struct file *file, struct folio *folio)
1908 {
1909 	struct page *page = &folio->page;
1910 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
1911 	u64 start = page_offset(page);
1912 	u64 end = start + PAGE_SIZE - 1;
1913 	struct btrfs_bio_ctrl bio_ctrl = { 0 };
1914 	int ret;
1915 
1916 	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1917 
1918 	ret = btrfs_do_readpage(page, NULL, &bio_ctrl, 0, NULL);
1919 	/*
1920 	 * If btrfs_do_readpage() failed we will want to submit the assembled
1921 	 * bio to do the cleanup.
1922 	 */
1923 	submit_one_bio(&bio_ctrl);
1924 	return ret;
1925 }
1926 
contiguous_readpages(struct page * pages[],int nr_pages,u64 start,u64 end,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)1927 static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1928 					u64 start, u64 end,
1929 					struct extent_map **em_cached,
1930 					struct btrfs_bio_ctrl *bio_ctrl,
1931 					u64 *prev_em_start)
1932 {
1933 	struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
1934 	int index;
1935 
1936 	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1937 
1938 	for (index = 0; index < nr_pages; index++) {
1939 		btrfs_do_readpage(pages[index], em_cached, bio_ctrl,
1940 				  REQ_RAHEAD, prev_em_start);
1941 		put_page(pages[index]);
1942 	}
1943 }
1944 
1945 /*
1946  * helper for __extent_writepage, doing all of the delayed allocation setup.
1947  *
1948  * This returns 1 if btrfs_run_delalloc_range function did all the work required
1949  * to write the page (copy into inline extent).  In this case the IO has
1950  * been started and the page is already unlocked.
1951  *
1952  * This returns 0 if all went well (page still locked)
1953  * This returns < 0 if there were errors (page still locked)
1954  */
writepage_delalloc(struct btrfs_inode * inode,struct page * page,struct writeback_control * wbc)1955 static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1956 		struct page *page, struct writeback_control *wbc)
1957 {
1958 	const u64 page_end = page_offset(page) + PAGE_SIZE - 1;
1959 	u64 delalloc_start = page_offset(page);
1960 	u64 delalloc_to_write = 0;
1961 	/* How many pages are started by btrfs_run_delalloc_range() */
1962 	unsigned long nr_written = 0;
1963 	int ret;
1964 	int page_started = 0;
1965 
1966 	while (delalloc_start < page_end) {
1967 		u64 delalloc_end = page_end;
1968 		bool found;
1969 
1970 		found = find_lock_delalloc_range(&inode->vfs_inode, page,
1971 					       &delalloc_start,
1972 					       &delalloc_end);
1973 		if (!found) {
1974 			delalloc_start = delalloc_end + 1;
1975 			continue;
1976 		}
1977 		ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
1978 				delalloc_end, &page_started, &nr_written, wbc);
1979 		if (ret) {
1980 			btrfs_page_set_error(inode->root->fs_info, page,
1981 					     page_offset(page), PAGE_SIZE);
1982 			return ret;
1983 		}
1984 		/*
1985 		 * delalloc_end is already one less than the total length, so
1986 		 * we don't subtract one from PAGE_SIZE
1987 		 */
1988 		delalloc_to_write += (delalloc_end - delalloc_start +
1989 				      PAGE_SIZE) >> PAGE_SHIFT;
1990 		delalloc_start = delalloc_end + 1;
1991 	}
1992 	if (wbc->nr_to_write < delalloc_to_write) {
1993 		int thresh = 8192;
1994 
1995 		if (delalloc_to_write < thresh * 2)
1996 			thresh = delalloc_to_write;
1997 		wbc->nr_to_write = min_t(u64, delalloc_to_write,
1998 					 thresh);
1999 	}
2000 
2001 	/* Did btrfs_run_dealloc_range() already unlock and start the IO? */
2002 	if (page_started) {
2003 		/*
2004 		 * We've unlocked the page, so we can't update the mapping's
2005 		 * writeback index, just update nr_to_write.
2006 		 */
2007 		wbc->nr_to_write -= nr_written;
2008 		return 1;
2009 	}
2010 
2011 	return 0;
2012 }
2013 
2014 /*
2015  * Find the first byte we need to write.
2016  *
2017  * For subpage, one page can contain several sectors, and
2018  * __extent_writepage_io() will just grab all extent maps in the page
2019  * range and try to submit all non-inline/non-compressed extents.
2020  *
2021  * This is a big problem for subpage, we shouldn't re-submit already written
2022  * data at all.
2023  * This function will lookup subpage dirty bit to find which range we really
2024  * need to submit.
2025  *
2026  * Return the next dirty range in [@start, @end).
2027  * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
2028  */
find_next_dirty_byte(struct btrfs_fs_info * fs_info,struct page * page,u64 * start,u64 * end)2029 static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
2030 				 struct page *page, u64 *start, u64 *end)
2031 {
2032 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
2033 	struct btrfs_subpage_info *spi = fs_info->subpage_info;
2034 	u64 orig_start = *start;
2035 	/* Declare as unsigned long so we can use bitmap ops */
2036 	unsigned long flags;
2037 	int range_start_bit;
2038 	int range_end_bit;
2039 
2040 	/*
2041 	 * For regular sector size == page size case, since one page only
2042 	 * contains one sector, we return the page offset directly.
2043 	 */
2044 	if (!btrfs_is_subpage(fs_info, page)) {
2045 		*start = page_offset(page);
2046 		*end = page_offset(page) + PAGE_SIZE;
2047 		return;
2048 	}
2049 
2050 	range_start_bit = spi->dirty_offset +
2051 			  (offset_in_page(orig_start) >> fs_info->sectorsize_bits);
2052 
2053 	/* We should have the page locked, but just in case */
2054 	spin_lock_irqsave(&subpage->lock, flags);
2055 	bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
2056 			       spi->dirty_offset + spi->bitmap_nr_bits);
2057 	spin_unlock_irqrestore(&subpage->lock, flags);
2058 
2059 	range_start_bit -= spi->dirty_offset;
2060 	range_end_bit -= spi->dirty_offset;
2061 
2062 	*start = page_offset(page) + range_start_bit * fs_info->sectorsize;
2063 	*end = page_offset(page) + range_end_bit * fs_info->sectorsize;
2064 }
2065 
2066 /*
2067  * helper for __extent_writepage.  This calls the writepage start hooks,
2068  * and does the loop to map the page into extents and bios.
2069  *
2070  * We return 1 if the IO is started and the page is unlocked,
2071  * 0 if all went well (page still locked)
2072  * < 0 if there were errors (page still locked)
2073  */
__extent_writepage_io(struct btrfs_inode * inode,struct page * page,struct writeback_control * wbc,struct extent_page_data * epd,loff_t i_size,int * nr_ret)2074 static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
2075 				 struct page *page,
2076 				 struct writeback_control *wbc,
2077 				 struct extent_page_data *epd,
2078 				 loff_t i_size,
2079 				 int *nr_ret)
2080 {
2081 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2082 	u64 cur = page_offset(page);
2083 	u64 end = cur + PAGE_SIZE - 1;
2084 	u64 extent_offset;
2085 	u64 block_start;
2086 	struct extent_map *em;
2087 	int saved_ret = 0;
2088 	int ret = 0;
2089 	int nr = 0;
2090 	enum req_op op = REQ_OP_WRITE;
2091 	const blk_opf_t write_flags = wbc_to_write_flags(wbc);
2092 	bool has_error = false;
2093 	bool compressed;
2094 
2095 	ret = btrfs_writepage_cow_fixup(page);
2096 	if (ret) {
2097 		/* Fixup worker will requeue */
2098 		redirty_page_for_writepage(wbc, page);
2099 		unlock_page(page);
2100 		return 1;
2101 	}
2102 
2103 	/*
2104 	 * we don't want to touch the inode after unlocking the page,
2105 	 * so we update the mapping writeback index now
2106 	 */
2107 	wbc->nr_to_write--;
2108 
2109 	epd->bio_ctrl.end_io_func = end_bio_extent_writepage;
2110 	while (cur <= end) {
2111 		u64 disk_bytenr;
2112 		u64 em_end;
2113 		u64 dirty_range_start = cur;
2114 		u64 dirty_range_end;
2115 		u32 iosize;
2116 
2117 		if (cur >= i_size) {
2118 			btrfs_writepage_endio_finish_ordered(inode, page, cur,
2119 							     end, true);
2120 			/*
2121 			 * This range is beyond i_size, thus we don't need to
2122 			 * bother writing back.
2123 			 * But we still need to clear the dirty subpage bit, or
2124 			 * the next time the page gets dirtied, we will try to
2125 			 * writeback the sectors with subpage dirty bits,
2126 			 * causing writeback without ordered extent.
2127 			 */
2128 			btrfs_page_clear_dirty(fs_info, page, cur, end + 1 - cur);
2129 			break;
2130 		}
2131 
2132 		find_next_dirty_byte(fs_info, page, &dirty_range_start,
2133 				     &dirty_range_end);
2134 		if (cur < dirty_range_start) {
2135 			cur = dirty_range_start;
2136 			continue;
2137 		}
2138 
2139 		em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1);
2140 		if (IS_ERR(em)) {
2141 			btrfs_page_set_error(fs_info, page, cur, end - cur + 1);
2142 			ret = PTR_ERR_OR_ZERO(em);
2143 			has_error = true;
2144 			if (!saved_ret)
2145 				saved_ret = ret;
2146 			break;
2147 		}
2148 
2149 		extent_offset = cur - em->start;
2150 		em_end = extent_map_end(em);
2151 		ASSERT(cur <= em_end);
2152 		ASSERT(cur < end);
2153 		ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
2154 		ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
2155 		block_start = em->block_start;
2156 		compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
2157 		disk_bytenr = em->block_start + extent_offset;
2158 
2159 		/*
2160 		 * Note that em_end from extent_map_end() and dirty_range_end from
2161 		 * find_next_dirty_byte() are all exclusive
2162 		 */
2163 		iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
2164 
2165 		if (btrfs_use_zone_append(inode, em->block_start))
2166 			op = REQ_OP_ZONE_APPEND;
2167 
2168 		free_extent_map(em);
2169 		em = NULL;
2170 
2171 		/*
2172 		 * compressed and inline extents are written through other
2173 		 * paths in the FS
2174 		 */
2175 		if (compressed || block_start == EXTENT_MAP_HOLE ||
2176 		    block_start == EXTENT_MAP_INLINE) {
2177 			if (compressed)
2178 				nr++;
2179 			else
2180 				btrfs_writepage_endio_finish_ordered(inode,
2181 						page, cur, cur + iosize - 1, true);
2182 			btrfs_page_clear_dirty(fs_info, page, cur, iosize);
2183 			cur += iosize;
2184 			continue;
2185 		}
2186 
2187 		btrfs_set_range_writeback(inode, cur, cur + iosize - 1);
2188 		if (!PageWriteback(page)) {
2189 			btrfs_err(inode->root->fs_info,
2190 				   "page %lu not writeback, cur %llu end %llu",
2191 			       page->index, cur, end);
2192 		}
2193 
2194 		/*
2195 		 * Although the PageDirty bit is cleared before entering this
2196 		 * function, subpage dirty bit is not cleared.
2197 		 * So clear subpage dirty bit here so next time we won't submit
2198 		 * page for range already written to disk.
2199 		 */
2200 		btrfs_page_clear_dirty(fs_info, page, cur, iosize);
2201 
2202 		ret = submit_extent_page(op | write_flags, wbc,
2203 					 &epd->bio_ctrl, disk_bytenr,
2204 					 page, iosize,
2205 					 cur - page_offset(page),
2206 					 0, false);
2207 		if (ret) {
2208 			has_error = true;
2209 			if (!saved_ret)
2210 				saved_ret = ret;
2211 
2212 			btrfs_page_set_error(fs_info, page, cur, iosize);
2213 			if (PageWriteback(page))
2214 				btrfs_page_clear_writeback(fs_info, page, cur,
2215 							   iosize);
2216 		}
2217 
2218 		cur += iosize;
2219 		nr++;
2220 	}
2221 	/*
2222 	 * If we finish without problem, we should not only clear page dirty,
2223 	 * but also empty subpage dirty bits
2224 	 */
2225 	if (!has_error)
2226 		btrfs_page_assert_not_dirty(fs_info, page);
2227 	else
2228 		ret = saved_ret;
2229 	*nr_ret = nr;
2230 	return ret;
2231 }
2232 
2233 /*
2234  * the writepage semantics are similar to regular writepage.  extent
2235  * records are inserted to lock ranges in the tree, and as dirty areas
2236  * are found, they are marked writeback.  Then the lock bits are removed
2237  * and the end_io handler clears the writeback ranges
2238  *
2239  * Return 0 if everything goes well.
2240  * Return <0 for error.
2241  */
__extent_writepage(struct page * page,struct writeback_control * wbc,struct extent_page_data * epd)2242 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2243 			      struct extent_page_data *epd)
2244 {
2245 	struct folio *folio = page_folio(page);
2246 	struct inode *inode = page->mapping->host;
2247 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2248 	const u64 page_start = page_offset(page);
2249 	const u64 page_end = page_start + PAGE_SIZE - 1;
2250 	int ret;
2251 	int nr = 0;
2252 	size_t pg_offset;
2253 	loff_t i_size = i_size_read(inode);
2254 	unsigned long end_index = i_size >> PAGE_SHIFT;
2255 
2256 	trace___extent_writepage(page, inode, wbc);
2257 
2258 	WARN_ON(!PageLocked(page));
2259 
2260 	btrfs_page_clear_error(btrfs_sb(inode->i_sb), page,
2261 			       page_offset(page), PAGE_SIZE);
2262 
2263 	pg_offset = offset_in_page(i_size);
2264 	if (page->index > end_index ||
2265 	   (page->index == end_index && !pg_offset)) {
2266 		folio_invalidate(folio, 0, folio_size(folio));
2267 		folio_unlock(folio);
2268 		return 0;
2269 	}
2270 
2271 	if (page->index == end_index)
2272 		memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
2273 
2274 	ret = set_page_extent_mapped(page);
2275 	if (ret < 0) {
2276 		SetPageError(page);
2277 		goto done;
2278 	}
2279 
2280 	if (!epd->extent_locked) {
2281 		ret = writepage_delalloc(BTRFS_I(inode), page, wbc);
2282 		if (ret == 1)
2283 			return 0;
2284 		if (ret)
2285 			goto done;
2286 	}
2287 
2288 	ret = __extent_writepage_io(BTRFS_I(inode), page, wbc, epd, i_size,
2289 				    &nr);
2290 	if (ret == 1)
2291 		return 0;
2292 
2293 done:
2294 	if (nr == 0) {
2295 		/* make sure the mapping tag for page dirty gets cleared */
2296 		set_page_writeback(page);
2297 		end_page_writeback(page);
2298 	}
2299 	/*
2300 	 * Here we used to have a check for PageError() and then set @ret and
2301 	 * call end_extent_writepage().
2302 	 *
2303 	 * But in fact setting @ret here will cause different error paths
2304 	 * between subpage and regular sectorsize.
2305 	 *
2306 	 * For regular page size, we never submit current page, but only add
2307 	 * current page to current bio.
2308 	 * The bio submission can only happen in next page.
2309 	 * Thus if we hit the PageError() branch, @ret is already set to
2310 	 * non-zero value and will not get updated for regular sectorsize.
2311 	 *
2312 	 * But for subpage case, it's possible we submit part of current page,
2313 	 * thus can get PageError() set by submitted bio of the same page,
2314 	 * while our @ret is still 0.
2315 	 *
2316 	 * So here we unify the behavior and don't set @ret.
2317 	 * Error can still be properly passed to higher layer as page will
2318 	 * be set error, here we just don't handle the IO failure.
2319 	 *
2320 	 * NOTE: This is just a hotfix for subpage.
2321 	 * The root fix will be properly ending ordered extent when we hit
2322 	 * an error during writeback.
2323 	 *
2324 	 * But that needs a bigger refactoring, as we not only need to grab the
2325 	 * submitted OE, but also need to know exactly at which bytenr we hit
2326 	 * the error.
2327 	 * Currently the full page based __extent_writepage_io() is not
2328 	 * capable of that.
2329 	 */
2330 	if (PageError(page))
2331 		end_extent_writepage(page, ret, page_start, page_end);
2332 	if (epd->extent_locked) {
2333 		/*
2334 		 * If epd->extent_locked, it's from extent_write_locked_range(),
2335 		 * the page can either be locked by lock_page() or
2336 		 * process_one_page().
2337 		 * Let btrfs_page_unlock_writer() handle both cases.
2338 		 */
2339 		ASSERT(wbc);
2340 		btrfs_page_unlock_writer(fs_info, page, wbc->range_start,
2341 					 wbc->range_end + 1 - wbc->range_start);
2342 	} else {
2343 		unlock_page(page);
2344 	}
2345 	ASSERT(ret <= 0);
2346 	return ret;
2347 }
2348 
wait_on_extent_buffer_writeback(struct extent_buffer * eb)2349 void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
2350 {
2351 	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
2352 		       TASK_UNINTERRUPTIBLE);
2353 }
2354 
end_extent_buffer_writeback(struct extent_buffer * eb)2355 static void end_extent_buffer_writeback(struct extent_buffer *eb)
2356 {
2357 	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
2358 	smp_mb__after_atomic();
2359 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
2360 }
2361 
2362 /*
2363  * Lock extent buffer status and pages for writeback.
2364  *
2365  * May try to flush write bio if we can't get the lock.
2366  *
2367  * Return  0 if the extent buffer doesn't need to be submitted.
2368  *           (E.g. the extent buffer is not dirty)
2369  * Return >0 is the extent buffer is submitted to bio.
2370  * Return <0 if something went wrong, no page is locked.
2371  */
lock_extent_buffer_for_io(struct extent_buffer * eb,struct extent_page_data * epd)2372 static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb,
2373 			  struct extent_page_data *epd)
2374 {
2375 	struct btrfs_fs_info *fs_info = eb->fs_info;
2376 	int i, num_pages;
2377 	int flush = 0;
2378 	int ret = 0;
2379 
2380 	if (!btrfs_try_tree_write_lock(eb)) {
2381 		submit_write_bio(epd, 0);
2382 		flush = 1;
2383 		btrfs_tree_lock(eb);
2384 	}
2385 
2386 	if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
2387 		btrfs_tree_unlock(eb);
2388 		if (!epd->sync_io)
2389 			return 0;
2390 		if (!flush) {
2391 			submit_write_bio(epd, 0);
2392 			flush = 1;
2393 		}
2394 		while (1) {
2395 			wait_on_extent_buffer_writeback(eb);
2396 			btrfs_tree_lock(eb);
2397 			if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
2398 				break;
2399 			btrfs_tree_unlock(eb);
2400 		}
2401 	}
2402 
2403 	/*
2404 	 * We need to do this to prevent races in people who check if the eb is
2405 	 * under IO since we can end up having no IO bits set for a short period
2406 	 * of time.
2407 	 */
2408 	spin_lock(&eb->refs_lock);
2409 	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
2410 		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
2411 		spin_unlock(&eb->refs_lock);
2412 		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
2413 		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
2414 					 -eb->len,
2415 					 fs_info->dirty_metadata_batch);
2416 		ret = 1;
2417 	} else {
2418 		spin_unlock(&eb->refs_lock);
2419 	}
2420 
2421 	btrfs_tree_unlock(eb);
2422 
2423 	/*
2424 	 * Either we don't need to submit any tree block, or we're submitting
2425 	 * subpage eb.
2426 	 * Subpage metadata doesn't use page locking at all, so we can skip
2427 	 * the page locking.
2428 	 */
2429 	if (!ret || fs_info->nodesize < PAGE_SIZE)
2430 		return ret;
2431 
2432 	num_pages = num_extent_pages(eb);
2433 	for (i = 0; i < num_pages; i++) {
2434 		struct page *p = eb->pages[i];
2435 
2436 		if (!trylock_page(p)) {
2437 			if (!flush) {
2438 				submit_write_bio(epd, 0);
2439 				flush = 1;
2440 			}
2441 			lock_page(p);
2442 		}
2443 	}
2444 
2445 	return ret;
2446 }
2447 
set_btree_ioerr(struct page * page,struct extent_buffer * eb)2448 static void set_btree_ioerr(struct page *page, struct extent_buffer *eb)
2449 {
2450 	struct btrfs_fs_info *fs_info = eb->fs_info;
2451 
2452 	btrfs_page_set_error(fs_info, page, eb->start, eb->len);
2453 	if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
2454 		return;
2455 
2456 	/*
2457 	 * A read may stumble upon this buffer later, make sure that it gets an
2458 	 * error and knows there was an error.
2459 	 */
2460 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
2461 
2462 	/*
2463 	 * We need to set the mapping with the io error as well because a write
2464 	 * error will flip the file system readonly, and then syncfs() will
2465 	 * return a 0 because we are readonly if we don't modify the err seq for
2466 	 * the superblock.
2467 	 */
2468 	mapping_set_error(page->mapping, -EIO);
2469 
2470 	/*
2471 	 * If we error out, we should add back the dirty_metadata_bytes
2472 	 * to make it consistent.
2473 	 */
2474 	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
2475 				 eb->len, fs_info->dirty_metadata_batch);
2476 
2477 	/*
2478 	 * If writeback for a btree extent that doesn't belong to a log tree
2479 	 * failed, increment the counter transaction->eb_write_errors.
2480 	 * We do this because while the transaction is running and before it's
2481 	 * committing (when we call filemap_fdata[write|wait]_range against
2482 	 * the btree inode), we might have
2483 	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
2484 	 * returns an error or an error happens during writeback, when we're
2485 	 * committing the transaction we wouldn't know about it, since the pages
2486 	 * can be no longer dirty nor marked anymore for writeback (if a
2487 	 * subsequent modification to the extent buffer didn't happen before the
2488 	 * transaction commit), which makes filemap_fdata[write|wait]_range not
2489 	 * able to find the pages tagged with SetPageError at transaction
2490 	 * commit time. So if this happens we must abort the transaction,
2491 	 * otherwise we commit a super block with btree roots that point to
2492 	 * btree nodes/leafs whose content on disk is invalid - either garbage
2493 	 * or the content of some node/leaf from a past generation that got
2494 	 * cowed or deleted and is no longer valid.
2495 	 *
2496 	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
2497 	 * not be enough - we need to distinguish between log tree extents vs
2498 	 * non-log tree extents, and the next filemap_fdatawait_range() call
2499 	 * will catch and clear such errors in the mapping - and that call might
2500 	 * be from a log sync and not from a transaction commit. Also, checking
2501 	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
2502 	 * not done and would not be reliable - the eb might have been released
2503 	 * from memory and reading it back again means that flag would not be
2504 	 * set (since it's a runtime flag, not persisted on disk).
2505 	 *
2506 	 * Using the flags below in the btree inode also makes us achieve the
2507 	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
2508 	 * writeback for all dirty pages and before filemap_fdatawait_range()
2509 	 * is called, the writeback for all dirty pages had already finished
2510 	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
2511 	 * filemap_fdatawait_range() would return success, as it could not know
2512 	 * that writeback errors happened (the pages were no longer tagged for
2513 	 * writeback).
2514 	 */
2515 	switch (eb->log_index) {
2516 	case -1:
2517 		set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
2518 		break;
2519 	case 0:
2520 		set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
2521 		break;
2522 	case 1:
2523 		set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
2524 		break;
2525 	default:
2526 		BUG(); /* unexpected, logic error */
2527 	}
2528 }
2529 
2530 /*
2531  * The endio specific version which won't touch any unsafe spinlock in endio
2532  * context.
2533  */
find_extent_buffer_nolock(struct btrfs_fs_info * fs_info,u64 start)2534 static struct extent_buffer *find_extent_buffer_nolock(
2535 		struct btrfs_fs_info *fs_info, u64 start)
2536 {
2537 	struct extent_buffer *eb;
2538 
2539 	rcu_read_lock();
2540 	eb = radix_tree_lookup(&fs_info->buffer_radix,
2541 			       start >> fs_info->sectorsize_bits);
2542 	if (eb && atomic_inc_not_zero(&eb->refs)) {
2543 		rcu_read_unlock();
2544 		return eb;
2545 	}
2546 	rcu_read_unlock();
2547 	return NULL;
2548 }
2549 
2550 /*
2551  * The endio function for subpage extent buffer write.
2552  *
2553  * Unlike end_bio_extent_buffer_writepage(), we only call end_page_writeback()
2554  * after all extent buffers in the page has finished their writeback.
2555  */
end_bio_subpage_eb_writepage(struct btrfs_bio * bbio)2556 static void end_bio_subpage_eb_writepage(struct btrfs_bio *bbio)
2557 {
2558 	struct bio *bio = &bbio->bio;
2559 	struct btrfs_fs_info *fs_info;
2560 	struct bio_vec *bvec;
2561 	struct bvec_iter_all iter_all;
2562 
2563 	fs_info = btrfs_sb(bio_first_page_all(bio)->mapping->host->i_sb);
2564 	ASSERT(fs_info->nodesize < PAGE_SIZE);
2565 
2566 	ASSERT(!bio_flagged(bio, BIO_CLONED));
2567 	bio_for_each_segment_all(bvec, bio, iter_all) {
2568 		struct page *page = bvec->bv_page;
2569 		u64 bvec_start = page_offset(page) + bvec->bv_offset;
2570 		u64 bvec_end = bvec_start + bvec->bv_len - 1;
2571 		u64 cur_bytenr = bvec_start;
2572 
2573 		ASSERT(IS_ALIGNED(bvec->bv_len, fs_info->nodesize));
2574 
2575 		/* Iterate through all extent buffers in the range */
2576 		while (cur_bytenr <= bvec_end) {
2577 			struct extent_buffer *eb;
2578 			int done;
2579 
2580 			/*
2581 			 * Here we can't use find_extent_buffer(), as it may
2582 			 * try to lock eb->refs_lock, which is not safe in endio
2583 			 * context.
2584 			 */
2585 			eb = find_extent_buffer_nolock(fs_info, cur_bytenr);
2586 			ASSERT(eb);
2587 
2588 			cur_bytenr = eb->start + eb->len;
2589 
2590 			ASSERT(test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags));
2591 			done = atomic_dec_and_test(&eb->io_pages);
2592 			ASSERT(done);
2593 
2594 			if (bio->bi_status ||
2595 			    test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
2596 				ClearPageUptodate(page);
2597 				set_btree_ioerr(page, eb);
2598 			}
2599 
2600 			btrfs_subpage_clear_writeback(fs_info, page, eb->start,
2601 						      eb->len);
2602 			end_extent_buffer_writeback(eb);
2603 			/*
2604 			 * free_extent_buffer() will grab spinlock which is not
2605 			 * safe in endio context. Thus here we manually dec
2606 			 * the ref.
2607 			 */
2608 			atomic_dec(&eb->refs);
2609 		}
2610 	}
2611 	bio_put(bio);
2612 }
2613 
end_bio_extent_buffer_writepage(struct btrfs_bio * bbio)2614 static void end_bio_extent_buffer_writepage(struct btrfs_bio *bbio)
2615 {
2616 	struct bio *bio = &bbio->bio;
2617 	struct bio_vec *bvec;
2618 	struct extent_buffer *eb;
2619 	int done;
2620 	struct bvec_iter_all iter_all;
2621 
2622 	ASSERT(!bio_flagged(bio, BIO_CLONED));
2623 	bio_for_each_segment_all(bvec, bio, iter_all) {
2624 		struct page *page = bvec->bv_page;
2625 
2626 		eb = (struct extent_buffer *)page->private;
2627 		BUG_ON(!eb);
2628 		done = atomic_dec_and_test(&eb->io_pages);
2629 
2630 		if (bio->bi_status ||
2631 		    test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
2632 			ClearPageUptodate(page);
2633 			set_btree_ioerr(page, eb);
2634 		}
2635 
2636 		end_page_writeback(page);
2637 
2638 		if (!done)
2639 			continue;
2640 
2641 		end_extent_buffer_writeback(eb);
2642 	}
2643 
2644 	bio_put(bio);
2645 }
2646 
prepare_eb_write(struct extent_buffer * eb)2647 static void prepare_eb_write(struct extent_buffer *eb)
2648 {
2649 	u32 nritems;
2650 	unsigned long start;
2651 	unsigned long end;
2652 
2653 	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
2654 	atomic_set(&eb->io_pages, num_extent_pages(eb));
2655 
2656 	/* Set btree blocks beyond nritems with 0 to avoid stale content */
2657 	nritems = btrfs_header_nritems(eb);
2658 	if (btrfs_header_level(eb) > 0) {
2659 		end = btrfs_node_key_ptr_offset(nritems);
2660 		memzero_extent_buffer(eb, end, eb->len - end);
2661 	} else {
2662 		/*
2663 		 * Leaf:
2664 		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
2665 		 */
2666 		start = btrfs_item_nr_offset(nritems);
2667 		end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb);
2668 		memzero_extent_buffer(eb, start, end - start);
2669 	}
2670 }
2671 
2672 /*
2673  * Unlike the work in write_one_eb(), we rely completely on extent locking.
2674  * Page locking is only utilized at minimum to keep the VMM code happy.
2675  */
write_one_subpage_eb(struct extent_buffer * eb,struct writeback_control * wbc,struct extent_page_data * epd)2676 static int write_one_subpage_eb(struct extent_buffer *eb,
2677 				struct writeback_control *wbc,
2678 				struct extent_page_data *epd)
2679 {
2680 	struct btrfs_fs_info *fs_info = eb->fs_info;
2681 	struct page *page = eb->pages[0];
2682 	blk_opf_t write_flags = wbc_to_write_flags(wbc);
2683 	bool no_dirty_ebs = false;
2684 	int ret;
2685 
2686 	prepare_eb_write(eb);
2687 
2688 	/* clear_page_dirty_for_io() in subpage helper needs page locked */
2689 	lock_page(page);
2690 	btrfs_subpage_set_writeback(fs_info, page, eb->start, eb->len);
2691 
2692 	/* Check if this is the last dirty bit to update nr_written */
2693 	no_dirty_ebs = btrfs_subpage_clear_and_test_dirty(fs_info, page,
2694 							  eb->start, eb->len);
2695 	if (no_dirty_ebs)
2696 		clear_page_dirty_for_io(page);
2697 
2698 	epd->bio_ctrl.end_io_func = end_bio_subpage_eb_writepage;
2699 
2700 	ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc,
2701 			&epd->bio_ctrl, eb->start, page, eb->len,
2702 			eb->start - page_offset(page), 0, false);
2703 	if (ret) {
2704 		btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len);
2705 		set_btree_ioerr(page, eb);
2706 		unlock_page(page);
2707 
2708 		if (atomic_dec_and_test(&eb->io_pages))
2709 			end_extent_buffer_writeback(eb);
2710 		return -EIO;
2711 	}
2712 	unlock_page(page);
2713 	/*
2714 	 * Submission finished without problem, if no range of the page is
2715 	 * dirty anymore, we have submitted a page.  Update nr_written in wbc.
2716 	 */
2717 	if (no_dirty_ebs)
2718 		wbc->nr_to_write--;
2719 	return ret;
2720 }
2721 
write_one_eb(struct extent_buffer * eb,struct writeback_control * wbc,struct extent_page_data * epd)2722 static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
2723 			struct writeback_control *wbc,
2724 			struct extent_page_data *epd)
2725 {
2726 	u64 disk_bytenr = eb->start;
2727 	int i, num_pages;
2728 	blk_opf_t write_flags = wbc_to_write_flags(wbc);
2729 	int ret = 0;
2730 
2731 	prepare_eb_write(eb);
2732 
2733 	epd->bio_ctrl.end_io_func = end_bio_extent_buffer_writepage;
2734 
2735 	num_pages = num_extent_pages(eb);
2736 	for (i = 0; i < num_pages; i++) {
2737 		struct page *p = eb->pages[i];
2738 
2739 		clear_page_dirty_for_io(p);
2740 		set_page_writeback(p);
2741 		ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc,
2742 					 &epd->bio_ctrl, disk_bytenr, p,
2743 					 PAGE_SIZE, 0, 0, false);
2744 		if (ret) {
2745 			set_btree_ioerr(p, eb);
2746 			if (PageWriteback(p))
2747 				end_page_writeback(p);
2748 			if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
2749 				end_extent_buffer_writeback(eb);
2750 			ret = -EIO;
2751 			break;
2752 		}
2753 		disk_bytenr += PAGE_SIZE;
2754 		wbc->nr_to_write--;
2755 		unlock_page(p);
2756 	}
2757 
2758 	if (unlikely(ret)) {
2759 		for (; i < num_pages; i++) {
2760 			struct page *p = eb->pages[i];
2761 			clear_page_dirty_for_io(p);
2762 			unlock_page(p);
2763 		}
2764 	}
2765 
2766 	return ret;
2767 }
2768 
2769 /*
2770  * Submit one subpage btree page.
2771  *
2772  * The main difference to submit_eb_page() is:
2773  * - Page locking
2774  *   For subpage, we don't rely on page locking at all.
2775  *
2776  * - Flush write bio
2777  *   We only flush bio if we may be unable to fit current extent buffers into
2778  *   current bio.
2779  *
2780  * Return >=0 for the number of submitted extent buffers.
2781  * Return <0 for fatal error.
2782  */
submit_eb_subpage(struct page * page,struct writeback_control * wbc,struct extent_page_data * epd)2783 static int submit_eb_subpage(struct page *page,
2784 			     struct writeback_control *wbc,
2785 			     struct extent_page_data *epd)
2786 {
2787 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
2788 	int submitted = 0;
2789 	u64 page_start = page_offset(page);
2790 	int bit_start = 0;
2791 	int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
2792 	int ret;
2793 
2794 	/* Lock and write each dirty extent buffers in the range */
2795 	while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
2796 		struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
2797 		struct extent_buffer *eb;
2798 		unsigned long flags;
2799 		u64 start;
2800 
2801 		/*
2802 		 * Take private lock to ensure the subpage won't be detached
2803 		 * in the meantime.
2804 		 */
2805 		spin_lock(&page->mapping->private_lock);
2806 		if (!PagePrivate(page)) {
2807 			spin_unlock(&page->mapping->private_lock);
2808 			break;
2809 		}
2810 		spin_lock_irqsave(&subpage->lock, flags);
2811 		if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
2812 			      subpage->bitmaps)) {
2813 			spin_unlock_irqrestore(&subpage->lock, flags);
2814 			spin_unlock(&page->mapping->private_lock);
2815 			bit_start++;
2816 			continue;
2817 		}
2818 
2819 		start = page_start + bit_start * fs_info->sectorsize;
2820 		bit_start += sectors_per_node;
2821 
2822 		/*
2823 		 * Here we just want to grab the eb without touching extra
2824 		 * spin locks, so call find_extent_buffer_nolock().
2825 		 */
2826 		eb = find_extent_buffer_nolock(fs_info, start);
2827 		spin_unlock_irqrestore(&subpage->lock, flags);
2828 		spin_unlock(&page->mapping->private_lock);
2829 
2830 		/*
2831 		 * The eb has already reached 0 refs thus find_extent_buffer()
2832 		 * doesn't return it. We don't need to write back such eb
2833 		 * anyway.
2834 		 */
2835 		if (!eb)
2836 			continue;
2837 
2838 		ret = lock_extent_buffer_for_io(eb, epd);
2839 		if (ret == 0) {
2840 			free_extent_buffer(eb);
2841 			continue;
2842 		}
2843 		if (ret < 0) {
2844 			free_extent_buffer(eb);
2845 			goto cleanup;
2846 		}
2847 		ret = write_one_subpage_eb(eb, wbc, epd);
2848 		free_extent_buffer(eb);
2849 		if (ret < 0)
2850 			goto cleanup;
2851 		submitted++;
2852 	}
2853 	return submitted;
2854 
2855 cleanup:
2856 	/* We hit error, end bio for the submitted extent buffers */
2857 	submit_write_bio(epd, ret);
2858 	return ret;
2859 }
2860 
2861 /*
2862  * Submit all page(s) of one extent buffer.
2863  *
2864  * @page:	the page of one extent buffer
2865  * @eb_context:	to determine if we need to submit this page, if current page
2866  *		belongs to this eb, we don't need to submit
2867  *
2868  * The caller should pass each page in their bytenr order, and here we use
2869  * @eb_context to determine if we have submitted pages of one extent buffer.
2870  *
2871  * If we have, we just skip until we hit a new page that doesn't belong to
2872  * current @eb_context.
2873  *
2874  * If not, we submit all the page(s) of the extent buffer.
2875  *
2876  * Return >0 if we have submitted the extent buffer successfully.
2877  * Return 0 if we don't need to submit the page, as it's already submitted by
2878  * previous call.
2879  * Return <0 for fatal error.
2880  */
submit_eb_page(struct page * page,struct writeback_control * wbc,struct extent_page_data * epd,struct extent_buffer ** eb_context)2881 static int submit_eb_page(struct page *page, struct writeback_control *wbc,
2882 			  struct extent_page_data *epd,
2883 			  struct extent_buffer **eb_context)
2884 {
2885 	struct address_space *mapping = page->mapping;
2886 	struct btrfs_block_group *cache = NULL;
2887 	struct extent_buffer *eb;
2888 	int ret;
2889 
2890 	if (!PagePrivate(page))
2891 		return 0;
2892 
2893 	if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
2894 		return submit_eb_subpage(page, wbc, epd);
2895 
2896 	spin_lock(&mapping->private_lock);
2897 	if (!PagePrivate(page)) {
2898 		spin_unlock(&mapping->private_lock);
2899 		return 0;
2900 	}
2901 
2902 	eb = (struct extent_buffer *)page->private;
2903 
2904 	/*
2905 	 * Shouldn't happen and normally this would be a BUG_ON but no point
2906 	 * crashing the machine for something we can survive anyway.
2907 	 */
2908 	if (WARN_ON(!eb)) {
2909 		spin_unlock(&mapping->private_lock);
2910 		return 0;
2911 	}
2912 
2913 	if (eb == *eb_context) {
2914 		spin_unlock(&mapping->private_lock);
2915 		return 0;
2916 	}
2917 	ret = atomic_inc_not_zero(&eb->refs);
2918 	spin_unlock(&mapping->private_lock);
2919 	if (!ret)
2920 		return 0;
2921 
2922 	if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) {
2923 		/*
2924 		 * If for_sync, this hole will be filled with
2925 		 * trasnsaction commit.
2926 		 */
2927 		if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
2928 			ret = -EAGAIN;
2929 		else
2930 			ret = 0;
2931 		free_extent_buffer(eb);
2932 		return ret;
2933 	}
2934 
2935 	*eb_context = eb;
2936 
2937 	ret = lock_extent_buffer_for_io(eb, epd);
2938 	if (ret <= 0) {
2939 		btrfs_revert_meta_write_pointer(cache, eb);
2940 		if (cache)
2941 			btrfs_put_block_group(cache);
2942 		free_extent_buffer(eb);
2943 		return ret;
2944 	}
2945 	if (cache) {
2946 		/*
2947 		 * Implies write in zoned mode. Mark the last eb in a block group.
2948 		 */
2949 		btrfs_schedule_zone_finish_bg(cache, eb);
2950 		btrfs_put_block_group(cache);
2951 	}
2952 	ret = write_one_eb(eb, wbc, epd);
2953 	free_extent_buffer(eb);
2954 	if (ret < 0)
2955 		return ret;
2956 	return 1;
2957 }
2958 
btree_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc)2959 int btree_write_cache_pages(struct address_space *mapping,
2960 				   struct writeback_control *wbc)
2961 {
2962 	struct extent_buffer *eb_context = NULL;
2963 	struct extent_page_data epd = {
2964 		.bio_ctrl = { 0 },
2965 		.extent_locked = 0,
2966 		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
2967 	};
2968 	struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
2969 	int ret = 0;
2970 	int done = 0;
2971 	int nr_to_write_done = 0;
2972 	struct pagevec pvec;
2973 	int nr_pages;
2974 	pgoff_t index;
2975 	pgoff_t end;		/* Inclusive */
2976 	int scanned = 0;
2977 	xa_mark_t tag;
2978 
2979 	pagevec_init(&pvec);
2980 	if (wbc->range_cyclic) {
2981 		index = mapping->writeback_index; /* Start from prev offset */
2982 		end = -1;
2983 		/*
2984 		 * Start from the beginning does not need to cycle over the
2985 		 * range, mark it as scanned.
2986 		 */
2987 		scanned = (index == 0);
2988 	} else {
2989 		index = wbc->range_start >> PAGE_SHIFT;
2990 		end = wbc->range_end >> PAGE_SHIFT;
2991 		scanned = 1;
2992 	}
2993 	if (wbc->sync_mode == WB_SYNC_ALL)
2994 		tag = PAGECACHE_TAG_TOWRITE;
2995 	else
2996 		tag = PAGECACHE_TAG_DIRTY;
2997 	btrfs_zoned_meta_io_lock(fs_info);
2998 retry:
2999 	if (wbc->sync_mode == WB_SYNC_ALL)
3000 		tag_pages_for_writeback(mapping, index, end);
3001 	while (!done && !nr_to_write_done && (index <= end) &&
3002 	       (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
3003 			tag))) {
3004 		unsigned i;
3005 
3006 		for (i = 0; i < nr_pages; i++) {
3007 			struct page *page = pvec.pages[i];
3008 
3009 			ret = submit_eb_page(page, wbc, &epd, &eb_context);
3010 			if (ret == 0)
3011 				continue;
3012 			if (ret < 0) {
3013 				done = 1;
3014 				break;
3015 			}
3016 
3017 			/*
3018 			 * the filesystem may choose to bump up nr_to_write.
3019 			 * We have to make sure to honor the new nr_to_write
3020 			 * at any time
3021 			 */
3022 			nr_to_write_done = wbc->nr_to_write <= 0;
3023 		}
3024 		pagevec_release(&pvec);
3025 		cond_resched();
3026 	}
3027 	if (!scanned && !done) {
3028 		/*
3029 		 * We hit the last page and there is more work to be done: wrap
3030 		 * back to the start of the file
3031 		 */
3032 		scanned = 1;
3033 		index = 0;
3034 		goto retry;
3035 	}
3036 	/*
3037 	 * If something went wrong, don't allow any metadata write bio to be
3038 	 * submitted.
3039 	 *
3040 	 * This would prevent use-after-free if we had dirty pages not
3041 	 * cleaned up, which can still happen by fuzzed images.
3042 	 *
3043 	 * - Bad extent tree
3044 	 *   Allowing existing tree block to be allocated for other trees.
3045 	 *
3046 	 * - Log tree operations
3047 	 *   Exiting tree blocks get allocated to log tree, bumps its
3048 	 *   generation, then get cleaned in tree re-balance.
3049 	 *   Such tree block will not be written back, since it's clean,
3050 	 *   thus no WRITTEN flag set.
3051 	 *   And after log writes back, this tree block is not traced by
3052 	 *   any dirty extent_io_tree.
3053 	 *
3054 	 * - Offending tree block gets re-dirtied from its original owner
3055 	 *   Since it has bumped generation, no WRITTEN flag, it can be
3056 	 *   reused without COWing. This tree block will not be traced
3057 	 *   by btrfs_transaction::dirty_pages.
3058 	 *
3059 	 *   Now such dirty tree block will not be cleaned by any dirty
3060 	 *   extent io tree. Thus we don't want to submit such wild eb
3061 	 *   if the fs already has error.
3062 	 *
3063 	 * We can get ret > 0 from submit_extent_page() indicating how many ebs
3064 	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
3065 	 */
3066 	if (ret > 0)
3067 		ret = 0;
3068 	if (!ret && BTRFS_FS_ERROR(fs_info))
3069 		ret = -EROFS;
3070 	submit_write_bio(&epd, ret);
3071 
3072 	btrfs_zoned_meta_io_unlock(fs_info);
3073 	return ret;
3074 }
3075 
3076 /**
3077  * Walk the list of dirty pages of the given address space and write all of them.
3078  *
3079  * @mapping: address space structure to write
3080  * @wbc:     subtract the number of written pages from *@wbc->nr_to_write
3081  * @epd:     holds context for the write, namely the bio
3082  *
3083  * If a page is already under I/O, write_cache_pages() skips it, even
3084  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
3085  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
3086  * and msync() need to guarantee that all the data which was dirty at the time
3087  * the call was made get new I/O started against them.  If wbc->sync_mode is
3088  * WB_SYNC_ALL then we were called for data integrity and we must wait for
3089  * existing IO to complete.
3090  */
extent_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc,struct extent_page_data * epd)3091 static int extent_write_cache_pages(struct address_space *mapping,
3092 			     struct writeback_control *wbc,
3093 			     struct extent_page_data *epd)
3094 {
3095 	struct inode *inode = mapping->host;
3096 	int ret = 0;
3097 	int done = 0;
3098 	int nr_to_write_done = 0;
3099 	struct pagevec pvec;
3100 	int nr_pages;
3101 	pgoff_t index;
3102 	pgoff_t end;		/* Inclusive */
3103 	pgoff_t done_index;
3104 	int range_whole = 0;
3105 	int scanned = 0;
3106 	xa_mark_t tag;
3107 
3108 	/*
3109 	 * We have to hold onto the inode so that ordered extents can do their
3110 	 * work when the IO finishes.  The alternative to this is failing to add
3111 	 * an ordered extent if the igrab() fails there and that is a huge pain
3112 	 * to deal with, so instead just hold onto the inode throughout the
3113 	 * writepages operation.  If it fails here we are freeing up the inode
3114 	 * anyway and we'd rather not waste our time writing out stuff that is
3115 	 * going to be truncated anyway.
3116 	 */
3117 	if (!igrab(inode))
3118 		return 0;
3119 
3120 	pagevec_init(&pvec);
3121 	if (wbc->range_cyclic) {
3122 		index = mapping->writeback_index; /* Start from prev offset */
3123 		end = -1;
3124 		/*
3125 		 * Start from the beginning does not need to cycle over the
3126 		 * range, mark it as scanned.
3127 		 */
3128 		scanned = (index == 0);
3129 	} else {
3130 		index = wbc->range_start >> PAGE_SHIFT;
3131 		end = wbc->range_end >> PAGE_SHIFT;
3132 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
3133 			range_whole = 1;
3134 		scanned = 1;
3135 	}
3136 
3137 	/*
3138 	 * We do the tagged writepage as long as the snapshot flush bit is set
3139 	 * and we are the first one who do the filemap_flush() on this inode.
3140 	 *
3141 	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
3142 	 * not race in and drop the bit.
3143 	 */
3144 	if (range_whole && wbc->nr_to_write == LONG_MAX &&
3145 	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
3146 			       &BTRFS_I(inode)->runtime_flags))
3147 		wbc->tagged_writepages = 1;
3148 
3149 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3150 		tag = PAGECACHE_TAG_TOWRITE;
3151 	else
3152 		tag = PAGECACHE_TAG_DIRTY;
3153 retry:
3154 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3155 		tag_pages_for_writeback(mapping, index, end);
3156 	done_index = index;
3157 	while (!done && !nr_to_write_done && (index <= end) &&
3158 			(nr_pages = pagevec_lookup_range_tag(&pvec, mapping,
3159 						&index, end, tag))) {
3160 		unsigned i;
3161 
3162 		for (i = 0; i < nr_pages; i++) {
3163 			struct page *page = pvec.pages[i];
3164 
3165 			done_index = page->index + 1;
3166 			/*
3167 			 * At this point we hold neither the i_pages lock nor
3168 			 * the page lock: the page may be truncated or
3169 			 * invalidated (changing page->mapping to NULL),
3170 			 * or even swizzled back from swapper_space to
3171 			 * tmpfs file mapping
3172 			 */
3173 			if (!trylock_page(page)) {
3174 				submit_write_bio(epd, 0);
3175 				lock_page(page);
3176 			}
3177 
3178 			if (unlikely(page->mapping != mapping)) {
3179 				unlock_page(page);
3180 				continue;
3181 			}
3182 
3183 			if (wbc->sync_mode != WB_SYNC_NONE) {
3184 				if (PageWriteback(page))
3185 					submit_write_bio(epd, 0);
3186 				wait_on_page_writeback(page);
3187 			}
3188 
3189 			if (PageWriteback(page) ||
3190 			    !clear_page_dirty_for_io(page)) {
3191 				unlock_page(page);
3192 				continue;
3193 			}
3194 
3195 			ret = __extent_writepage(page, wbc, epd);
3196 			if (ret < 0) {
3197 				done = 1;
3198 				break;
3199 			}
3200 
3201 			/*
3202 			 * the filesystem may choose to bump up nr_to_write.
3203 			 * We have to make sure to honor the new nr_to_write
3204 			 * at any time
3205 			 */
3206 			nr_to_write_done = wbc->nr_to_write <= 0;
3207 		}
3208 		pagevec_release(&pvec);
3209 		cond_resched();
3210 	}
3211 	if (!scanned && !done) {
3212 		/*
3213 		 * We hit the last page and there is more work to be done: wrap
3214 		 * back to the start of the file
3215 		 */
3216 		scanned = 1;
3217 		index = 0;
3218 
3219 		/*
3220 		 * If we're looping we could run into a page that is locked by a
3221 		 * writer and that writer could be waiting on writeback for a
3222 		 * page in our current bio, and thus deadlock, so flush the
3223 		 * write bio here.
3224 		 */
3225 		submit_write_bio(epd, 0);
3226 		goto retry;
3227 	}
3228 
3229 	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
3230 		mapping->writeback_index = done_index;
3231 
3232 	btrfs_add_delayed_iput(inode);
3233 	return ret;
3234 }
3235 
3236 /*
3237  * Submit the pages in the range to bio for call sites which delalloc range has
3238  * already been ran (aka, ordered extent inserted) and all pages are still
3239  * locked.
3240  */
extent_write_locked_range(struct inode * inode,u64 start,u64 end)3241 int extent_write_locked_range(struct inode *inode, u64 start, u64 end)
3242 {
3243 	bool found_error = false;
3244 	int first_error = 0;
3245 	int ret = 0;
3246 	struct address_space *mapping = inode->i_mapping;
3247 	struct page *page;
3248 	u64 cur = start;
3249 	unsigned long nr_pages;
3250 	const u32 sectorsize = btrfs_sb(inode->i_sb)->sectorsize;
3251 	struct extent_page_data epd = {
3252 		.bio_ctrl = { 0 },
3253 		.extent_locked = 1,
3254 		.sync_io = 1,
3255 	};
3256 	struct writeback_control wbc_writepages = {
3257 		.sync_mode	= WB_SYNC_ALL,
3258 		.range_start	= start,
3259 		.range_end	= end + 1,
3260 		/* We're called from an async helper function */
3261 		.punt_to_cgroup	= 1,
3262 		.no_cgroup_owner = 1,
3263 	};
3264 
3265 	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
3266 	nr_pages = (round_up(end, PAGE_SIZE) - round_down(start, PAGE_SIZE)) >>
3267 		   PAGE_SHIFT;
3268 	wbc_writepages.nr_to_write = nr_pages * 2;
3269 
3270 	wbc_attach_fdatawrite_inode(&wbc_writepages, inode);
3271 	while (cur <= end) {
3272 		u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
3273 
3274 		page = find_get_page(mapping, cur >> PAGE_SHIFT);
3275 		/*
3276 		 * All pages in the range are locked since
3277 		 * btrfs_run_delalloc_range(), thus there is no way to clear
3278 		 * the page dirty flag.
3279 		 */
3280 		ASSERT(PageLocked(page));
3281 		ASSERT(PageDirty(page));
3282 		clear_page_dirty_for_io(page);
3283 		ret = __extent_writepage(page, &wbc_writepages, &epd);
3284 		ASSERT(ret <= 0);
3285 		if (ret < 0) {
3286 			found_error = true;
3287 			first_error = ret;
3288 		}
3289 		put_page(page);
3290 		cur = cur_end + 1;
3291 	}
3292 
3293 	submit_write_bio(&epd, found_error ? ret : 0);
3294 
3295 	wbc_detach_inode(&wbc_writepages);
3296 	if (found_error)
3297 		return first_error;
3298 	return ret;
3299 }
3300 
extent_writepages(struct address_space * mapping,struct writeback_control * wbc)3301 int extent_writepages(struct address_space *mapping,
3302 		      struct writeback_control *wbc)
3303 {
3304 	struct inode *inode = mapping->host;
3305 	int ret = 0;
3306 	struct extent_page_data epd = {
3307 		.bio_ctrl = { 0 },
3308 		.extent_locked = 0,
3309 		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
3310 	};
3311 
3312 	/*
3313 	 * Allow only a single thread to do the reloc work in zoned mode to
3314 	 * protect the write pointer updates.
3315 	 */
3316 	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
3317 	ret = extent_write_cache_pages(mapping, wbc, &epd);
3318 	submit_write_bio(&epd, ret);
3319 	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
3320 	return ret;
3321 }
3322 
extent_readahead(struct readahead_control * rac)3323 void extent_readahead(struct readahead_control *rac)
3324 {
3325 	struct btrfs_bio_ctrl bio_ctrl = { 0 };
3326 	struct page *pagepool[16];
3327 	struct extent_map *em_cached = NULL;
3328 	u64 prev_em_start = (u64)-1;
3329 	int nr;
3330 
3331 	while ((nr = readahead_page_batch(rac, pagepool))) {
3332 		u64 contig_start = readahead_pos(rac);
3333 		u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
3334 
3335 		contiguous_readpages(pagepool, nr, contig_start, contig_end,
3336 				&em_cached, &bio_ctrl, &prev_em_start);
3337 	}
3338 
3339 	if (em_cached)
3340 		free_extent_map(em_cached);
3341 	submit_one_bio(&bio_ctrl);
3342 }
3343 
3344 /*
3345  * basic invalidate_folio code, this waits on any locked or writeback
3346  * ranges corresponding to the folio, and then deletes any extent state
3347  * records from the tree
3348  */
extent_invalidate_folio(struct extent_io_tree * tree,struct folio * folio,size_t offset)3349 int extent_invalidate_folio(struct extent_io_tree *tree,
3350 			  struct folio *folio, size_t offset)
3351 {
3352 	struct extent_state *cached_state = NULL;
3353 	u64 start = folio_pos(folio);
3354 	u64 end = start + folio_size(folio) - 1;
3355 	size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
3356 
3357 	/* This function is only called for the btree inode */
3358 	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
3359 
3360 	start += ALIGN(offset, blocksize);
3361 	if (start > end)
3362 		return 0;
3363 
3364 	lock_extent(tree, start, end, &cached_state);
3365 	folio_wait_writeback(folio);
3366 
3367 	/*
3368 	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
3369 	 * so here we only need to unlock the extent range to free any
3370 	 * existing extent state.
3371 	 */
3372 	unlock_extent(tree, start, end, &cached_state);
3373 	return 0;
3374 }
3375 
3376 /*
3377  * a helper for release_folio, this tests for areas of the page that
3378  * are locked or under IO and drops the related state bits if it is safe
3379  * to drop the page.
3380  */
try_release_extent_state(struct extent_io_tree * tree,struct page * page,gfp_t mask)3381 static int try_release_extent_state(struct extent_io_tree *tree,
3382 				    struct page *page, gfp_t mask)
3383 {
3384 	u64 start = page_offset(page);
3385 	u64 end = start + PAGE_SIZE - 1;
3386 	int ret = 1;
3387 
3388 	if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
3389 		ret = 0;
3390 	} else {
3391 		u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
3392 				   EXTENT_DELALLOC_NEW | EXTENT_CTLBITS);
3393 
3394 		/*
3395 		 * At this point we can safely clear everything except the
3396 		 * locked bit, the nodatasum bit and the delalloc new bit.
3397 		 * The delalloc new bit will be cleared by ordered extent
3398 		 * completion.
3399 		 */
3400 		ret = __clear_extent_bit(tree, start, end, clear_bits, NULL,
3401 					 mask, NULL);
3402 
3403 		/* if clear_extent_bit failed for enomem reasons,
3404 		 * we can't allow the release to continue.
3405 		 */
3406 		if (ret < 0)
3407 			ret = 0;
3408 		else
3409 			ret = 1;
3410 	}
3411 	return ret;
3412 }
3413 
3414 /*
3415  * a helper for release_folio.  As long as there are no locked extents
3416  * in the range corresponding to the page, both state records and extent
3417  * map records are removed
3418  */
try_release_extent_mapping(struct page * page,gfp_t mask)3419 int try_release_extent_mapping(struct page *page, gfp_t mask)
3420 {
3421 	struct extent_map *em;
3422 	u64 start = page_offset(page);
3423 	u64 end = start + PAGE_SIZE - 1;
3424 	struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
3425 	struct extent_io_tree *tree = &btrfs_inode->io_tree;
3426 	struct extent_map_tree *map = &btrfs_inode->extent_tree;
3427 
3428 	if (gfpflags_allow_blocking(mask) &&
3429 	    page->mapping->host->i_size > SZ_16M) {
3430 		u64 len;
3431 		while (start <= end) {
3432 			struct btrfs_fs_info *fs_info;
3433 			u64 cur_gen;
3434 
3435 			len = end - start + 1;
3436 			write_lock(&map->lock);
3437 			em = lookup_extent_mapping(map, start, len);
3438 			if (!em) {
3439 				write_unlock(&map->lock);
3440 				break;
3441 			}
3442 			if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3443 			    em->start != start) {
3444 				write_unlock(&map->lock);
3445 				free_extent_map(em);
3446 				break;
3447 			}
3448 			if (test_range_bit(tree, em->start,
3449 					   extent_map_end(em) - 1,
3450 					   EXTENT_LOCKED, 0, NULL))
3451 				goto next;
3452 			/*
3453 			 * If it's not in the list of modified extents, used
3454 			 * by a fast fsync, we can remove it. If it's being
3455 			 * logged we can safely remove it since fsync took an
3456 			 * extra reference on the em.
3457 			 */
3458 			if (list_empty(&em->list) ||
3459 			    test_bit(EXTENT_FLAG_LOGGING, &em->flags))
3460 				goto remove_em;
3461 			/*
3462 			 * If it's in the list of modified extents, remove it
3463 			 * only if its generation is older then the current one,
3464 			 * in which case we don't need it for a fast fsync.
3465 			 * Otherwise don't remove it, we could be racing with an
3466 			 * ongoing fast fsync that could miss the new extent.
3467 			 */
3468 			fs_info = btrfs_inode->root->fs_info;
3469 			spin_lock(&fs_info->trans_lock);
3470 			cur_gen = fs_info->generation;
3471 			spin_unlock(&fs_info->trans_lock);
3472 			if (em->generation >= cur_gen)
3473 				goto next;
3474 remove_em:
3475 			/*
3476 			 * We only remove extent maps that are not in the list of
3477 			 * modified extents or that are in the list but with a
3478 			 * generation lower then the current generation, so there
3479 			 * is no need to set the full fsync flag on the inode (it
3480 			 * hurts the fsync performance for workloads with a data
3481 			 * size that exceeds or is close to the system's memory).
3482 			 */
3483 			remove_extent_mapping(map, em);
3484 			/* once for the rb tree */
3485 			free_extent_map(em);
3486 next:
3487 			start = extent_map_end(em);
3488 			write_unlock(&map->lock);
3489 
3490 			/* once for us */
3491 			free_extent_map(em);
3492 
3493 			cond_resched(); /* Allow large-extent preemption. */
3494 		}
3495 	}
3496 	return try_release_extent_state(tree, page, mask);
3497 }
3498 
3499 /*
3500  * To cache previous fiemap extent
3501  *
3502  * Will be used for merging fiemap extent
3503  */
3504 struct fiemap_cache {
3505 	u64 offset;
3506 	u64 phys;
3507 	u64 len;
3508 	u32 flags;
3509 	bool cached;
3510 };
3511 
3512 /*
3513  * Helper to submit fiemap extent.
3514  *
3515  * Will try to merge current fiemap extent specified by @offset, @phys,
3516  * @len and @flags with cached one.
3517  * And only when we fails to merge, cached one will be submitted as
3518  * fiemap extent.
3519  *
3520  * Return value is the same as fiemap_fill_next_extent().
3521  */
emit_fiemap_extent(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,u64 offset,u64 phys,u64 len,u32 flags)3522 static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
3523 				struct fiemap_cache *cache,
3524 				u64 offset, u64 phys, u64 len, u32 flags)
3525 {
3526 	int ret = 0;
3527 
3528 	/* Set at the end of extent_fiemap(). */
3529 	ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
3530 
3531 	if (!cache->cached)
3532 		goto assign;
3533 
3534 	/*
3535 	 * Sanity check, extent_fiemap() should have ensured that new
3536 	 * fiemap extent won't overlap with cached one.
3537 	 * Not recoverable.
3538 	 *
3539 	 * NOTE: Physical address can overlap, due to compression
3540 	 */
3541 	if (cache->offset + cache->len > offset) {
3542 		WARN_ON(1);
3543 		return -EINVAL;
3544 	}
3545 
3546 	/*
3547 	 * Only merges fiemap extents if
3548 	 * 1) Their logical addresses are continuous
3549 	 *
3550 	 * 2) Their physical addresses are continuous
3551 	 *    So truly compressed (physical size smaller than logical size)
3552 	 *    extents won't get merged with each other
3553 	 *
3554 	 * 3) Share same flags
3555 	 */
3556 	if (cache->offset + cache->len  == offset &&
3557 	    cache->phys + cache->len == phys  &&
3558 	    cache->flags == flags) {
3559 		cache->len += len;
3560 		return 0;
3561 	}
3562 
3563 	/* Not mergeable, need to submit cached one */
3564 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
3565 				      cache->len, cache->flags);
3566 	cache->cached = false;
3567 	if (ret)
3568 		return ret;
3569 assign:
3570 	cache->cached = true;
3571 	cache->offset = offset;
3572 	cache->phys = phys;
3573 	cache->len = len;
3574 	cache->flags = flags;
3575 
3576 	return 0;
3577 }
3578 
3579 /*
3580  * Emit last fiemap cache
3581  *
3582  * The last fiemap cache may still be cached in the following case:
3583  * 0		      4k		    8k
3584  * |<- Fiemap range ->|
3585  * |<------------  First extent ----------->|
3586  *
3587  * In this case, the first extent range will be cached but not emitted.
3588  * So we must emit it before ending extent_fiemap().
3589  */
emit_last_fiemap_cache(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache)3590 static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
3591 				  struct fiemap_cache *cache)
3592 {
3593 	int ret;
3594 
3595 	if (!cache->cached)
3596 		return 0;
3597 
3598 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
3599 				      cache->len, cache->flags);
3600 	cache->cached = false;
3601 	if (ret > 0)
3602 		ret = 0;
3603 	return ret;
3604 }
3605 
fiemap_next_leaf_item(struct btrfs_inode * inode,struct btrfs_path * path)3606 static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
3607 {
3608 	struct extent_buffer *clone;
3609 	struct btrfs_key key;
3610 	int slot;
3611 	int ret;
3612 
3613 	path->slots[0]++;
3614 	if (path->slots[0] < btrfs_header_nritems(path->nodes[0]))
3615 		return 0;
3616 
3617 	ret = btrfs_next_leaf(inode->root, path);
3618 	if (ret != 0)
3619 		return ret;
3620 
3621 	/*
3622 	 * Don't bother with cloning if there are no more file extent items for
3623 	 * our inode.
3624 	 */
3625 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3626 	if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
3627 		return 1;
3628 
3629 	/* See the comment at fiemap_search_slot() about why we clone. */
3630 	clone = btrfs_clone_extent_buffer(path->nodes[0]);
3631 	if (!clone)
3632 		return -ENOMEM;
3633 
3634 	slot = path->slots[0];
3635 	btrfs_release_path(path);
3636 	path->nodes[0] = clone;
3637 	path->slots[0] = slot;
3638 
3639 	return 0;
3640 }
3641 
3642 /*
3643  * Search for the first file extent item that starts at a given file offset or
3644  * the one that starts immediately before that offset.
3645  * Returns: 0 on success, < 0 on error, 1 if not found.
3646  */
fiemap_search_slot(struct btrfs_inode * inode,struct btrfs_path * path,u64 file_offset)3647 static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
3648 			      u64 file_offset)
3649 {
3650 	const u64 ino = btrfs_ino(inode);
3651 	struct btrfs_root *root = inode->root;
3652 	struct extent_buffer *clone;
3653 	struct btrfs_key key;
3654 	int slot;
3655 	int ret;
3656 
3657 	key.objectid = ino;
3658 	key.type = BTRFS_EXTENT_DATA_KEY;
3659 	key.offset = file_offset;
3660 
3661 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3662 	if (ret < 0)
3663 		return ret;
3664 
3665 	if (ret > 0 && path->slots[0] > 0) {
3666 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
3667 		if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
3668 			path->slots[0]--;
3669 	}
3670 
3671 	if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
3672 		ret = btrfs_next_leaf(root, path);
3673 		if (ret != 0)
3674 			return ret;
3675 
3676 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3677 		if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
3678 			return 1;
3679 	}
3680 
3681 	/*
3682 	 * We clone the leaf and use it during fiemap. This is because while
3683 	 * using the leaf we do expensive things like checking if an extent is
3684 	 * shared, which can take a long time. In order to prevent blocking
3685 	 * other tasks for too long, we use a clone of the leaf. We have locked
3686 	 * the file range in the inode's io tree, so we know none of our file
3687 	 * extent items can change. This way we avoid blocking other tasks that
3688 	 * want to insert items for other inodes in the same leaf or b+tree
3689 	 * rebalance operations (triggered for example when someone is trying
3690 	 * to push items into this leaf when trying to insert an item in a
3691 	 * neighbour leaf).
3692 	 * We also need the private clone because holding a read lock on an
3693 	 * extent buffer of the subvolume's b+tree will make lockdep unhappy
3694 	 * when we call fiemap_fill_next_extent(), because that may cause a page
3695 	 * fault when filling the user space buffer with fiemap data.
3696 	 */
3697 	clone = btrfs_clone_extent_buffer(path->nodes[0]);
3698 	if (!clone)
3699 		return -ENOMEM;
3700 
3701 	slot = path->slots[0];
3702 	btrfs_release_path(path);
3703 	path->nodes[0] = clone;
3704 	path->slots[0] = slot;
3705 
3706 	return 0;
3707 }
3708 
3709 /*
3710  * Process a range which is a hole or a prealloc extent in the inode's subvolume
3711  * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
3712  * extent. The end offset (@end) is inclusive.
3713  */
fiemap_process_hole(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,struct btrfs_backref_shared_cache * backref_cache,u64 disk_bytenr,u64 extent_offset,u64 extent_gen,struct ulist * roots,struct ulist * tmp_ulist,u64 start,u64 end)3714 static int fiemap_process_hole(struct btrfs_inode *inode,
3715 			       struct fiemap_extent_info *fieinfo,
3716 			       struct fiemap_cache *cache,
3717 			       struct btrfs_backref_shared_cache *backref_cache,
3718 			       u64 disk_bytenr, u64 extent_offset,
3719 			       u64 extent_gen,
3720 			       struct ulist *roots, struct ulist *tmp_ulist,
3721 			       u64 start, u64 end)
3722 {
3723 	const u64 i_size = i_size_read(&inode->vfs_inode);
3724 	const u64 ino = btrfs_ino(inode);
3725 	u64 cur_offset = start;
3726 	u64 last_delalloc_end = 0;
3727 	u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
3728 	bool checked_extent_shared = false;
3729 	int ret;
3730 
3731 	/*
3732 	 * There can be no delalloc past i_size, so don't waste time looking for
3733 	 * it beyond i_size.
3734 	 */
3735 	while (cur_offset < end && cur_offset < i_size) {
3736 		u64 delalloc_start;
3737 		u64 delalloc_end;
3738 		u64 prealloc_start;
3739 		u64 prealloc_len = 0;
3740 		bool delalloc;
3741 
3742 		delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end,
3743 							&delalloc_start,
3744 							&delalloc_end);
3745 		if (!delalloc)
3746 			break;
3747 
3748 		/*
3749 		 * If this is a prealloc extent we have to report every section
3750 		 * of it that has no delalloc.
3751 		 */
3752 		if (disk_bytenr != 0) {
3753 			if (last_delalloc_end == 0) {
3754 				prealloc_start = start;
3755 				prealloc_len = delalloc_start - start;
3756 			} else {
3757 				prealloc_start = last_delalloc_end + 1;
3758 				prealloc_len = delalloc_start - prealloc_start;
3759 			}
3760 		}
3761 
3762 		if (prealloc_len > 0) {
3763 			if (!checked_extent_shared && fieinfo->fi_extents_max) {
3764 				ret = btrfs_is_data_extent_shared(inode->root,
3765 							  ino, disk_bytenr,
3766 							  extent_gen, roots,
3767 							  tmp_ulist,
3768 							  backref_cache);
3769 				if (ret < 0)
3770 					return ret;
3771 				else if (ret > 0)
3772 					prealloc_flags |= FIEMAP_EXTENT_SHARED;
3773 
3774 				checked_extent_shared = true;
3775 			}
3776 			ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
3777 						 disk_bytenr + extent_offset,
3778 						 prealloc_len, prealloc_flags);
3779 			if (ret)
3780 				return ret;
3781 			extent_offset += prealloc_len;
3782 		}
3783 
3784 		ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0,
3785 					 delalloc_end + 1 - delalloc_start,
3786 					 FIEMAP_EXTENT_DELALLOC |
3787 					 FIEMAP_EXTENT_UNKNOWN);
3788 		if (ret)
3789 			return ret;
3790 
3791 		last_delalloc_end = delalloc_end;
3792 		cur_offset = delalloc_end + 1;
3793 		extent_offset += cur_offset - delalloc_start;
3794 		cond_resched();
3795 	}
3796 
3797 	/*
3798 	 * Either we found no delalloc for the whole prealloc extent or we have
3799 	 * a prealloc extent that spans i_size or starts at or after i_size.
3800 	 */
3801 	if (disk_bytenr != 0 && last_delalloc_end < end) {
3802 		u64 prealloc_start;
3803 		u64 prealloc_len;
3804 
3805 		if (last_delalloc_end == 0) {
3806 			prealloc_start = start;
3807 			prealloc_len = end + 1 - start;
3808 		} else {
3809 			prealloc_start = last_delalloc_end + 1;
3810 			prealloc_len = end + 1 - prealloc_start;
3811 		}
3812 
3813 		if (!checked_extent_shared && fieinfo->fi_extents_max) {
3814 			ret = btrfs_is_data_extent_shared(inode->root,
3815 							  ino, disk_bytenr,
3816 							  extent_gen, roots,
3817 							  tmp_ulist,
3818 							  backref_cache);
3819 			if (ret < 0)
3820 				return ret;
3821 			else if (ret > 0)
3822 				prealloc_flags |= FIEMAP_EXTENT_SHARED;
3823 		}
3824 		ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
3825 					 disk_bytenr + extent_offset,
3826 					 prealloc_len, prealloc_flags);
3827 		if (ret)
3828 			return ret;
3829 	}
3830 
3831 	return 0;
3832 }
3833 
fiemap_find_last_extent_offset(struct btrfs_inode * inode,struct btrfs_path * path,u64 * last_extent_end_ret)3834 static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
3835 					  struct btrfs_path *path,
3836 					  u64 *last_extent_end_ret)
3837 {
3838 	const u64 ino = btrfs_ino(inode);
3839 	struct btrfs_root *root = inode->root;
3840 	struct extent_buffer *leaf;
3841 	struct btrfs_file_extent_item *ei;
3842 	struct btrfs_key key;
3843 	u64 disk_bytenr;
3844 	int ret;
3845 
3846 	/*
3847 	 * Lookup the last file extent. We're not using i_size here because
3848 	 * there might be preallocation past i_size.
3849 	 */
3850 	ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0);
3851 	/* There can't be a file extent item at offset (u64)-1 */
3852 	ASSERT(ret != 0);
3853 	if (ret < 0)
3854 		return ret;
3855 
3856 	/*
3857 	 * For a non-existing key, btrfs_search_slot() always leaves us at a
3858 	 * slot > 0, except if the btree is empty, which is impossible because
3859 	 * at least it has the inode item for this inode and all the items for
3860 	 * the root inode 256.
3861 	 */
3862 	ASSERT(path->slots[0] > 0);
3863 	path->slots[0]--;
3864 	leaf = path->nodes[0];
3865 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3866 	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
3867 		/* No file extent items in the subvolume tree. */
3868 		*last_extent_end_ret = 0;
3869 		return 0;
3870 	}
3871 
3872 	/*
3873 	 * For an inline extent, the disk_bytenr is where inline data starts at,
3874 	 * so first check if we have an inline extent item before checking if we
3875 	 * have an implicit hole (disk_bytenr == 0).
3876 	 */
3877 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
3878 	if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
3879 		*last_extent_end_ret = btrfs_file_extent_end(path);
3880 		return 0;
3881 	}
3882 
3883 	/*
3884 	 * Find the last file extent item that is not a hole (when NO_HOLES is
3885 	 * not enabled). This should take at most 2 iterations in the worst
3886 	 * case: we have one hole file extent item at slot 0 of a leaf and
3887 	 * another hole file extent item as the last item in the previous leaf.
3888 	 * This is because we merge file extent items that represent holes.
3889 	 */
3890 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
3891 	while (disk_bytenr == 0) {
3892 		ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY);
3893 		if (ret < 0) {
3894 			return ret;
3895 		} else if (ret > 0) {
3896 			/* No file extent items that are not holes. */
3897 			*last_extent_end_ret = 0;
3898 			return 0;
3899 		}
3900 		leaf = path->nodes[0];
3901 		ei = btrfs_item_ptr(leaf, path->slots[0],
3902 				    struct btrfs_file_extent_item);
3903 		disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
3904 	}
3905 
3906 	*last_extent_end_ret = btrfs_file_extent_end(path);
3907 	return 0;
3908 }
3909 
extent_fiemap(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,u64 start,u64 len)3910 int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
3911 		  u64 start, u64 len)
3912 {
3913 	const u64 ino = btrfs_ino(inode);
3914 	struct extent_state *cached_state = NULL;
3915 	struct btrfs_path *path;
3916 	struct btrfs_root *root = inode->root;
3917 	struct fiemap_cache cache = { 0 };
3918 	struct btrfs_backref_shared_cache *backref_cache;
3919 	struct ulist *roots;
3920 	struct ulist *tmp_ulist;
3921 	u64 last_extent_end;
3922 	u64 prev_extent_end;
3923 	u64 lockstart;
3924 	u64 lockend;
3925 	bool stopped = false;
3926 	int ret;
3927 
3928 	backref_cache = kzalloc(sizeof(*backref_cache), GFP_KERNEL);
3929 	path = btrfs_alloc_path();
3930 	roots = ulist_alloc(GFP_KERNEL);
3931 	tmp_ulist = ulist_alloc(GFP_KERNEL);
3932 	if (!backref_cache || !path || !roots || !tmp_ulist) {
3933 		ret = -ENOMEM;
3934 		goto out;
3935 	}
3936 
3937 	lockstart = round_down(start, root->fs_info->sectorsize);
3938 	lockend = round_up(start + len, root->fs_info->sectorsize);
3939 	prev_extent_end = lockstart;
3940 
3941 	lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3942 
3943 	ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end);
3944 	if (ret < 0)
3945 		goto out_unlock;
3946 	btrfs_release_path(path);
3947 
3948 	path->reada = READA_FORWARD;
3949 	ret = fiemap_search_slot(inode, path, lockstart);
3950 	if (ret < 0) {
3951 		goto out_unlock;
3952 	} else if (ret > 0) {
3953 		/*
3954 		 * No file extent item found, but we may have delalloc between
3955 		 * the current offset and i_size. So check for that.
3956 		 */
3957 		ret = 0;
3958 		goto check_eof_delalloc;
3959 	}
3960 
3961 	while (prev_extent_end < lockend) {
3962 		struct extent_buffer *leaf = path->nodes[0];
3963 		struct btrfs_file_extent_item *ei;
3964 		struct btrfs_key key;
3965 		u64 extent_end;
3966 		u64 extent_len;
3967 		u64 extent_offset = 0;
3968 		u64 extent_gen;
3969 		u64 disk_bytenr = 0;
3970 		u64 flags = 0;
3971 		int extent_type;
3972 		u8 compression;
3973 
3974 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3975 		if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
3976 			break;
3977 
3978 		extent_end = btrfs_file_extent_end(path);
3979 
3980 		/*
3981 		 * The first iteration can leave us at an extent item that ends
3982 		 * before our range's start. Move to the next item.
3983 		 */
3984 		if (extent_end <= lockstart)
3985 			goto next_item;
3986 
3987 		/* We have in implicit hole (NO_HOLES feature enabled). */
3988 		if (prev_extent_end < key.offset) {
3989 			const u64 range_end = min(key.offset, lockend) - 1;
3990 
3991 			ret = fiemap_process_hole(inode, fieinfo, &cache,
3992 						  backref_cache, 0, 0, 0,
3993 						  roots, tmp_ulist,
3994 						  prev_extent_end, range_end);
3995 			if (ret < 0) {
3996 				goto out_unlock;
3997 			} else if (ret > 0) {
3998 				/* fiemap_fill_next_extent() told us to stop. */
3999 				stopped = true;
4000 				break;
4001 			}
4002 
4003 			/* We've reached the end of the fiemap range, stop. */
4004 			if (key.offset >= lockend) {
4005 				stopped = true;
4006 				break;
4007 			}
4008 		}
4009 
4010 		extent_len = extent_end - key.offset;
4011 		ei = btrfs_item_ptr(leaf, path->slots[0],
4012 				    struct btrfs_file_extent_item);
4013 		compression = btrfs_file_extent_compression(leaf, ei);
4014 		extent_type = btrfs_file_extent_type(leaf, ei);
4015 		extent_gen = btrfs_file_extent_generation(leaf, ei);
4016 
4017 		if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4018 			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
4019 			if (compression == BTRFS_COMPRESS_NONE)
4020 				extent_offset = btrfs_file_extent_offset(leaf, ei);
4021 		}
4022 
4023 		if (compression != BTRFS_COMPRESS_NONE)
4024 			flags |= FIEMAP_EXTENT_ENCODED;
4025 
4026 		if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4027 			flags |= FIEMAP_EXTENT_DATA_INLINE;
4028 			flags |= FIEMAP_EXTENT_NOT_ALIGNED;
4029 			ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0,
4030 						 extent_len, flags);
4031 		} else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
4032 			ret = fiemap_process_hole(inode, fieinfo, &cache,
4033 						  backref_cache,
4034 						  disk_bytenr, extent_offset,
4035 						  extent_gen, roots, tmp_ulist,
4036 						  key.offset, extent_end - 1);
4037 		} else if (disk_bytenr == 0) {
4038 			/* We have an explicit hole. */
4039 			ret = fiemap_process_hole(inode, fieinfo, &cache,
4040 						  backref_cache, 0, 0, 0,
4041 						  roots, tmp_ulist,
4042 						  key.offset, extent_end - 1);
4043 		} else {
4044 			/* We have a regular extent. */
4045 			if (fieinfo->fi_extents_max) {
4046 				ret = btrfs_is_data_extent_shared(root, ino,
4047 								  disk_bytenr,
4048 								  extent_gen,
4049 								  roots,
4050 								  tmp_ulist,
4051 								  backref_cache);
4052 				if (ret < 0)
4053 					goto out_unlock;
4054 				else if (ret > 0)
4055 					flags |= FIEMAP_EXTENT_SHARED;
4056 			}
4057 
4058 			ret = emit_fiemap_extent(fieinfo, &cache, key.offset,
4059 						 disk_bytenr + extent_offset,
4060 						 extent_len, flags);
4061 		}
4062 
4063 		if (ret < 0) {
4064 			goto out_unlock;
4065 		} else if (ret > 0) {
4066 			/* fiemap_fill_next_extent() told us to stop. */
4067 			stopped = true;
4068 			break;
4069 		}
4070 
4071 		prev_extent_end = extent_end;
4072 next_item:
4073 		if (fatal_signal_pending(current)) {
4074 			ret = -EINTR;
4075 			goto out_unlock;
4076 		}
4077 
4078 		ret = fiemap_next_leaf_item(inode, path);
4079 		if (ret < 0) {
4080 			goto out_unlock;
4081 		} else if (ret > 0) {
4082 			/* No more file extent items for this inode. */
4083 			break;
4084 		}
4085 		cond_resched();
4086 	}
4087 
4088 check_eof_delalloc:
4089 	/*
4090 	 * Release (and free) the path before emitting any final entries to
4091 	 * fiemap_fill_next_extent() to keep lockdep happy. This is because
4092 	 * once we find no more file extent items exist, we may have a
4093 	 * non-cloned leaf, and fiemap_fill_next_extent() can trigger page
4094 	 * faults when copying data to the user space buffer.
4095 	 */
4096 	btrfs_free_path(path);
4097 	path = NULL;
4098 
4099 	if (!stopped && prev_extent_end < lockend) {
4100 		ret = fiemap_process_hole(inode, fieinfo, &cache, backref_cache,
4101 					  0, 0, 0, roots, tmp_ulist,
4102 					  prev_extent_end, lockend - 1);
4103 		if (ret < 0)
4104 			goto out_unlock;
4105 		prev_extent_end = lockend;
4106 	}
4107 
4108 	if (cache.cached && cache.offset + cache.len >= last_extent_end) {
4109 		const u64 i_size = i_size_read(&inode->vfs_inode);
4110 
4111 		if (prev_extent_end < i_size) {
4112 			u64 delalloc_start;
4113 			u64 delalloc_end;
4114 			bool delalloc;
4115 
4116 			delalloc = btrfs_find_delalloc_in_range(inode,
4117 								prev_extent_end,
4118 								i_size - 1,
4119 								&delalloc_start,
4120 								&delalloc_end);
4121 			if (!delalloc)
4122 				cache.flags |= FIEMAP_EXTENT_LAST;
4123 		} else {
4124 			cache.flags |= FIEMAP_EXTENT_LAST;
4125 		}
4126 	}
4127 
4128 	ret = emit_last_fiemap_cache(fieinfo, &cache);
4129 
4130 out_unlock:
4131 	unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
4132 out:
4133 	kfree(backref_cache);
4134 	btrfs_free_path(path);
4135 	ulist_free(roots);
4136 	ulist_free(tmp_ulist);
4137 	return ret;
4138 }
4139 
__free_extent_buffer(struct extent_buffer * eb)4140 static void __free_extent_buffer(struct extent_buffer *eb)
4141 {
4142 	kmem_cache_free(extent_buffer_cache, eb);
4143 }
4144 
extent_buffer_under_io(const struct extent_buffer * eb)4145 int extent_buffer_under_io(const struct extent_buffer *eb)
4146 {
4147 	return (atomic_read(&eb->io_pages) ||
4148 		test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4149 		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4150 }
4151 
page_range_has_eb(struct btrfs_fs_info * fs_info,struct page * page)4152 static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
4153 {
4154 	struct btrfs_subpage *subpage;
4155 
4156 	lockdep_assert_held(&page->mapping->private_lock);
4157 
4158 	if (PagePrivate(page)) {
4159 		subpage = (struct btrfs_subpage *)page->private;
4160 		if (atomic_read(&subpage->eb_refs))
4161 			return true;
4162 		/*
4163 		 * Even there is no eb refs here, we may still have
4164 		 * end_page_read() call relying on page::private.
4165 		 */
4166 		if (atomic_read(&subpage->readers))
4167 			return true;
4168 	}
4169 	return false;
4170 }
4171 
detach_extent_buffer_page(struct extent_buffer * eb,struct page * page)4172 static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
4173 {
4174 	struct btrfs_fs_info *fs_info = eb->fs_info;
4175 	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4176 
4177 	/*
4178 	 * For mapped eb, we're going to change the page private, which should
4179 	 * be done under the private_lock.
4180 	 */
4181 	if (mapped)
4182 		spin_lock(&page->mapping->private_lock);
4183 
4184 	if (!PagePrivate(page)) {
4185 		if (mapped)
4186 			spin_unlock(&page->mapping->private_lock);
4187 		return;
4188 	}
4189 
4190 	if (fs_info->nodesize >= PAGE_SIZE) {
4191 		/*
4192 		 * We do this since we'll remove the pages after we've
4193 		 * removed the eb from the radix tree, so we could race
4194 		 * and have this page now attached to the new eb.  So
4195 		 * only clear page_private if it's still connected to
4196 		 * this eb.
4197 		 */
4198 		if (PagePrivate(page) &&
4199 		    page->private == (unsigned long)eb) {
4200 			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4201 			BUG_ON(PageDirty(page));
4202 			BUG_ON(PageWriteback(page));
4203 			/*
4204 			 * We need to make sure we haven't be attached
4205 			 * to a new eb.
4206 			 */
4207 			detach_page_private(page);
4208 		}
4209 		if (mapped)
4210 			spin_unlock(&page->mapping->private_lock);
4211 		return;
4212 	}
4213 
4214 	/*
4215 	 * For subpage, we can have dummy eb with page private.  In this case,
4216 	 * we can directly detach the private as such page is only attached to
4217 	 * one dummy eb, no sharing.
4218 	 */
4219 	if (!mapped) {
4220 		btrfs_detach_subpage(fs_info, page);
4221 		return;
4222 	}
4223 
4224 	btrfs_page_dec_eb_refs(fs_info, page);
4225 
4226 	/*
4227 	 * We can only detach the page private if there are no other ebs in the
4228 	 * page range and no unfinished IO.
4229 	 */
4230 	if (!page_range_has_eb(fs_info, page))
4231 		btrfs_detach_subpage(fs_info, page);
4232 
4233 	spin_unlock(&page->mapping->private_lock);
4234 }
4235 
4236 /* Release all pages attached to the extent buffer */
btrfs_release_extent_buffer_pages(struct extent_buffer * eb)4237 static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
4238 {
4239 	int i;
4240 	int num_pages;
4241 
4242 	ASSERT(!extent_buffer_under_io(eb));
4243 
4244 	num_pages = num_extent_pages(eb);
4245 	for (i = 0; i < num_pages; i++) {
4246 		struct page *page = eb->pages[i];
4247 
4248 		if (!page)
4249 			continue;
4250 
4251 		detach_extent_buffer_page(eb, page);
4252 
4253 		/* One for when we allocated the page */
4254 		put_page(page);
4255 	}
4256 }
4257 
4258 /*
4259  * Helper for releasing the extent buffer.
4260  */
btrfs_release_extent_buffer(struct extent_buffer * eb)4261 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4262 {
4263 	btrfs_release_extent_buffer_pages(eb);
4264 	btrfs_leak_debug_del_eb(eb);
4265 	__free_extent_buffer(eb);
4266 }
4267 
4268 static struct extent_buffer *
__alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)4269 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
4270 		      unsigned long len)
4271 {
4272 	struct extent_buffer *eb = NULL;
4273 
4274 	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
4275 	eb->start = start;
4276 	eb->len = len;
4277 	eb->fs_info = fs_info;
4278 	eb->bflags = 0;
4279 	init_rwsem(&eb->lock);
4280 
4281 	btrfs_leak_debug_add_eb(eb);
4282 	INIT_LIST_HEAD(&eb->release_list);
4283 
4284 	spin_lock_init(&eb->refs_lock);
4285 	atomic_set(&eb->refs, 1);
4286 	atomic_set(&eb->io_pages, 0);
4287 
4288 	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
4289 
4290 	return eb;
4291 }
4292 
btrfs_clone_extent_buffer(const struct extent_buffer * src)4293 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
4294 {
4295 	int i;
4296 	struct extent_buffer *new;
4297 	int num_pages = num_extent_pages(src);
4298 	int ret;
4299 
4300 	new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
4301 	if (new == NULL)
4302 		return NULL;
4303 
4304 	/*
4305 	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
4306 	 * btrfs_release_extent_buffer() have different behavior for
4307 	 * UNMAPPED subpage extent buffer.
4308 	 */
4309 	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
4310 
4311 	memset(new->pages, 0, sizeof(*new->pages) * num_pages);
4312 	ret = btrfs_alloc_page_array(num_pages, new->pages);
4313 	if (ret) {
4314 		btrfs_release_extent_buffer(new);
4315 		return NULL;
4316 	}
4317 
4318 	for (i = 0; i < num_pages; i++) {
4319 		int ret;
4320 		struct page *p = new->pages[i];
4321 
4322 		ret = attach_extent_buffer_page(new, p, NULL);
4323 		if (ret < 0) {
4324 			btrfs_release_extent_buffer(new);
4325 			return NULL;
4326 		}
4327 		WARN_ON(PageDirty(p));
4328 		copy_page(page_address(p), page_address(src->pages[i]));
4329 	}
4330 	set_extent_buffer_uptodate(new);
4331 
4332 	return new;
4333 }
4334 
__alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)4335 struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4336 						  u64 start, unsigned long len)
4337 {
4338 	struct extent_buffer *eb;
4339 	int num_pages;
4340 	int i;
4341 	int ret;
4342 
4343 	eb = __alloc_extent_buffer(fs_info, start, len);
4344 	if (!eb)
4345 		return NULL;
4346 
4347 	num_pages = num_extent_pages(eb);
4348 	ret = btrfs_alloc_page_array(num_pages, eb->pages);
4349 	if (ret)
4350 		goto err;
4351 
4352 	for (i = 0; i < num_pages; i++) {
4353 		struct page *p = eb->pages[i];
4354 
4355 		ret = attach_extent_buffer_page(eb, p, NULL);
4356 		if (ret < 0)
4357 			goto err;
4358 	}
4359 
4360 	set_extent_buffer_uptodate(eb);
4361 	btrfs_set_header_nritems(eb, 0);
4362 	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4363 
4364 	return eb;
4365 err:
4366 	for (i = 0; i < num_pages; i++) {
4367 		if (eb->pages[i]) {
4368 			detach_extent_buffer_page(eb, eb->pages[i]);
4369 			__free_page(eb->pages[i]);
4370 		}
4371 	}
4372 	__free_extent_buffer(eb);
4373 	return NULL;
4374 }
4375 
alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)4376 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4377 						u64 start)
4378 {
4379 	return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
4380 }
4381 
check_buffer_tree_ref(struct extent_buffer * eb)4382 static void check_buffer_tree_ref(struct extent_buffer *eb)
4383 {
4384 	int refs;
4385 	/*
4386 	 * The TREE_REF bit is first set when the extent_buffer is added
4387 	 * to the radix tree. It is also reset, if unset, when a new reference
4388 	 * is created by find_extent_buffer.
4389 	 *
4390 	 * It is only cleared in two cases: freeing the last non-tree
4391 	 * reference to the extent_buffer when its STALE bit is set or
4392 	 * calling release_folio when the tree reference is the only reference.
4393 	 *
4394 	 * In both cases, care is taken to ensure that the extent_buffer's
4395 	 * pages are not under io. However, release_folio can be concurrently
4396 	 * called with creating new references, which is prone to race
4397 	 * conditions between the calls to check_buffer_tree_ref in those
4398 	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
4399 	 *
4400 	 * The actual lifetime of the extent_buffer in the radix tree is
4401 	 * adequately protected by the refcount, but the TREE_REF bit and
4402 	 * its corresponding reference are not. To protect against this
4403 	 * class of races, we call check_buffer_tree_ref from the codepaths
4404 	 * which trigger io after they set eb->io_pages. Note that once io is
4405 	 * initiated, TREE_REF can no longer be cleared, so that is the
4406 	 * moment at which any such race is best fixed.
4407 	 */
4408 	refs = atomic_read(&eb->refs);
4409 	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4410 		return;
4411 
4412 	spin_lock(&eb->refs_lock);
4413 	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4414 		atomic_inc(&eb->refs);
4415 	spin_unlock(&eb->refs_lock);
4416 }
4417 
mark_extent_buffer_accessed(struct extent_buffer * eb,struct page * accessed)4418 static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4419 		struct page *accessed)
4420 {
4421 	int num_pages, i;
4422 
4423 	check_buffer_tree_ref(eb);
4424 
4425 	num_pages = num_extent_pages(eb);
4426 	for (i = 0; i < num_pages; i++) {
4427 		struct page *p = eb->pages[i];
4428 
4429 		if (p != accessed)
4430 			mark_page_accessed(p);
4431 	}
4432 }
4433 
find_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)4434 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4435 					 u64 start)
4436 {
4437 	struct extent_buffer *eb;
4438 
4439 	eb = find_extent_buffer_nolock(fs_info, start);
4440 	if (!eb)
4441 		return NULL;
4442 	/*
4443 	 * Lock our eb's refs_lock to avoid races with free_extent_buffer().
4444 	 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
4445 	 * another task running free_extent_buffer() might have seen that flag
4446 	 * set, eb->refs == 2, that the buffer isn't under IO (dirty and
4447 	 * writeback flags not set) and it's still in the tree (flag
4448 	 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
4449 	 * decrementing the extent buffer's reference count twice.  So here we
4450 	 * could race and increment the eb's reference count, clear its stale
4451 	 * flag, mark it as dirty and drop our reference before the other task
4452 	 * finishes executing free_extent_buffer, which would later result in
4453 	 * an attempt to free an extent buffer that is dirty.
4454 	 */
4455 	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4456 		spin_lock(&eb->refs_lock);
4457 		spin_unlock(&eb->refs_lock);
4458 	}
4459 	mark_extent_buffer_accessed(eb, NULL);
4460 	return eb;
4461 }
4462 
4463 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
alloc_test_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)4464 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
4465 					u64 start)
4466 {
4467 	struct extent_buffer *eb, *exists = NULL;
4468 	int ret;
4469 
4470 	eb = find_extent_buffer(fs_info, start);
4471 	if (eb)
4472 		return eb;
4473 	eb = alloc_dummy_extent_buffer(fs_info, start);
4474 	if (!eb)
4475 		return ERR_PTR(-ENOMEM);
4476 	eb->fs_info = fs_info;
4477 again:
4478 	ret = radix_tree_preload(GFP_NOFS);
4479 	if (ret) {
4480 		exists = ERR_PTR(ret);
4481 		goto free_eb;
4482 	}
4483 	spin_lock(&fs_info->buffer_lock);
4484 	ret = radix_tree_insert(&fs_info->buffer_radix,
4485 				start >> fs_info->sectorsize_bits, eb);
4486 	spin_unlock(&fs_info->buffer_lock);
4487 	radix_tree_preload_end();
4488 	if (ret == -EEXIST) {
4489 		exists = find_extent_buffer(fs_info, start);
4490 		if (exists)
4491 			goto free_eb;
4492 		else
4493 			goto again;
4494 	}
4495 	check_buffer_tree_ref(eb);
4496 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4497 
4498 	return eb;
4499 free_eb:
4500 	btrfs_release_extent_buffer(eb);
4501 	return exists;
4502 }
4503 #endif
4504 
grab_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page)4505 static struct extent_buffer *grab_extent_buffer(
4506 		struct btrfs_fs_info *fs_info, struct page *page)
4507 {
4508 	struct extent_buffer *exists;
4509 
4510 	/*
4511 	 * For subpage case, we completely rely on radix tree to ensure we
4512 	 * don't try to insert two ebs for the same bytenr.  So here we always
4513 	 * return NULL and just continue.
4514 	 */
4515 	if (fs_info->nodesize < PAGE_SIZE)
4516 		return NULL;
4517 
4518 	/* Page not yet attached to an extent buffer */
4519 	if (!PagePrivate(page))
4520 		return NULL;
4521 
4522 	/*
4523 	 * We could have already allocated an eb for this page and attached one
4524 	 * so lets see if we can get a ref on the existing eb, and if we can we
4525 	 * know it's good and we can just return that one, else we know we can
4526 	 * just overwrite page->private.
4527 	 */
4528 	exists = (struct extent_buffer *)page->private;
4529 	if (atomic_inc_not_zero(&exists->refs))
4530 		return exists;
4531 
4532 	WARN_ON(PageDirty(page));
4533 	detach_page_private(page);
4534 	return NULL;
4535 }
4536 
check_eb_alignment(struct btrfs_fs_info * fs_info,u64 start)4537 static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
4538 {
4539 	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
4540 		btrfs_err(fs_info, "bad tree block start %llu", start);
4541 		return -EINVAL;
4542 	}
4543 
4544 	if (fs_info->nodesize < PAGE_SIZE &&
4545 	    offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
4546 		btrfs_err(fs_info,
4547 		"tree block crosses page boundary, start %llu nodesize %u",
4548 			  start, fs_info->nodesize);
4549 		return -EINVAL;
4550 	}
4551 	if (fs_info->nodesize >= PAGE_SIZE &&
4552 	    !PAGE_ALIGNED(start)) {
4553 		btrfs_err(fs_info,
4554 		"tree block is not page aligned, start %llu nodesize %u",
4555 			  start, fs_info->nodesize);
4556 		return -EINVAL;
4557 	}
4558 	return 0;
4559 }
4560 
alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,u64 owner_root,int level)4561 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
4562 					  u64 start, u64 owner_root, int level)
4563 {
4564 	unsigned long len = fs_info->nodesize;
4565 	int num_pages;
4566 	int i;
4567 	unsigned long index = start >> PAGE_SHIFT;
4568 	struct extent_buffer *eb;
4569 	struct extent_buffer *exists = NULL;
4570 	struct page *p;
4571 	struct address_space *mapping = fs_info->btree_inode->i_mapping;
4572 	u64 lockdep_owner = owner_root;
4573 	int uptodate = 1;
4574 	int ret;
4575 
4576 	if (check_eb_alignment(fs_info, start))
4577 		return ERR_PTR(-EINVAL);
4578 
4579 #if BITS_PER_LONG == 32
4580 	if (start >= MAX_LFS_FILESIZE) {
4581 		btrfs_err_rl(fs_info,
4582 		"extent buffer %llu is beyond 32bit page cache limit", start);
4583 		btrfs_err_32bit_limit(fs_info);
4584 		return ERR_PTR(-EOVERFLOW);
4585 	}
4586 	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
4587 		btrfs_warn_32bit_limit(fs_info);
4588 #endif
4589 
4590 	eb = find_extent_buffer(fs_info, start);
4591 	if (eb)
4592 		return eb;
4593 
4594 	eb = __alloc_extent_buffer(fs_info, start, len);
4595 	if (!eb)
4596 		return ERR_PTR(-ENOMEM);
4597 
4598 	/*
4599 	 * The reloc trees are just snapshots, so we need them to appear to be
4600 	 * just like any other fs tree WRT lockdep.
4601 	 */
4602 	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
4603 		lockdep_owner = BTRFS_FS_TREE_OBJECTID;
4604 
4605 	btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
4606 
4607 	num_pages = num_extent_pages(eb);
4608 	for (i = 0; i < num_pages; i++, index++) {
4609 		struct btrfs_subpage *prealloc = NULL;
4610 
4611 		p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
4612 		if (!p) {
4613 			exists = ERR_PTR(-ENOMEM);
4614 			goto free_eb;
4615 		}
4616 
4617 		/*
4618 		 * Preallocate page->private for subpage case, so that we won't
4619 		 * allocate memory with private_lock hold.  The memory will be
4620 		 * freed by attach_extent_buffer_page() or freed manually if
4621 		 * we exit earlier.
4622 		 *
4623 		 * Although we have ensured one subpage eb can only have one
4624 		 * page, but it may change in the future for 16K page size
4625 		 * support, so we still preallocate the memory in the loop.
4626 		 */
4627 		if (fs_info->nodesize < PAGE_SIZE) {
4628 			prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
4629 			if (IS_ERR(prealloc)) {
4630 				ret = PTR_ERR(prealloc);
4631 				unlock_page(p);
4632 				put_page(p);
4633 				exists = ERR_PTR(ret);
4634 				goto free_eb;
4635 			}
4636 		}
4637 
4638 		spin_lock(&mapping->private_lock);
4639 		exists = grab_extent_buffer(fs_info, p);
4640 		if (exists) {
4641 			spin_unlock(&mapping->private_lock);
4642 			unlock_page(p);
4643 			put_page(p);
4644 			mark_extent_buffer_accessed(exists, p);
4645 			btrfs_free_subpage(prealloc);
4646 			goto free_eb;
4647 		}
4648 		/* Should not fail, as we have preallocated the memory */
4649 		ret = attach_extent_buffer_page(eb, p, prealloc);
4650 		ASSERT(!ret);
4651 		/*
4652 		 * To inform we have extra eb under allocation, so that
4653 		 * detach_extent_buffer_page() won't release the page private
4654 		 * when the eb hasn't yet been inserted into radix tree.
4655 		 *
4656 		 * The ref will be decreased when the eb released the page, in
4657 		 * detach_extent_buffer_page().
4658 		 * Thus needs no special handling in error path.
4659 		 */
4660 		btrfs_page_inc_eb_refs(fs_info, p);
4661 		spin_unlock(&mapping->private_lock);
4662 
4663 		WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
4664 		eb->pages[i] = p;
4665 		if (!PageUptodate(p))
4666 			uptodate = 0;
4667 
4668 		/*
4669 		 * We can't unlock the pages just yet since the extent buffer
4670 		 * hasn't been properly inserted in the radix tree, this
4671 		 * opens a race with btree_release_folio which can free a page
4672 		 * while we are still filling in all pages for the buffer and
4673 		 * we could crash.
4674 		 */
4675 	}
4676 	if (uptodate)
4677 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4678 again:
4679 	ret = radix_tree_preload(GFP_NOFS);
4680 	if (ret) {
4681 		exists = ERR_PTR(ret);
4682 		goto free_eb;
4683 	}
4684 
4685 	spin_lock(&fs_info->buffer_lock);
4686 	ret = radix_tree_insert(&fs_info->buffer_radix,
4687 				start >> fs_info->sectorsize_bits, eb);
4688 	spin_unlock(&fs_info->buffer_lock);
4689 	radix_tree_preload_end();
4690 	if (ret == -EEXIST) {
4691 		exists = find_extent_buffer(fs_info, start);
4692 		if (exists)
4693 			goto free_eb;
4694 		else
4695 			goto again;
4696 	}
4697 	/* add one reference for the tree */
4698 	check_buffer_tree_ref(eb);
4699 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4700 
4701 	/*
4702 	 * Now it's safe to unlock the pages because any calls to
4703 	 * btree_release_folio will correctly detect that a page belongs to a
4704 	 * live buffer and won't free them prematurely.
4705 	 */
4706 	for (i = 0; i < num_pages; i++)
4707 		unlock_page(eb->pages[i]);
4708 	return eb;
4709 
4710 free_eb:
4711 	WARN_ON(!atomic_dec_and_test(&eb->refs));
4712 	for (i = 0; i < num_pages; i++) {
4713 		if (eb->pages[i])
4714 			unlock_page(eb->pages[i]);
4715 	}
4716 
4717 	btrfs_release_extent_buffer(eb);
4718 	return exists;
4719 }
4720 
btrfs_release_extent_buffer_rcu(struct rcu_head * head)4721 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4722 {
4723 	struct extent_buffer *eb =
4724 			container_of(head, struct extent_buffer, rcu_head);
4725 
4726 	__free_extent_buffer(eb);
4727 }
4728 
release_extent_buffer(struct extent_buffer * eb)4729 static int release_extent_buffer(struct extent_buffer *eb)
4730 	__releases(&eb->refs_lock)
4731 {
4732 	lockdep_assert_held(&eb->refs_lock);
4733 
4734 	WARN_ON(atomic_read(&eb->refs) == 0);
4735 	if (atomic_dec_and_test(&eb->refs)) {
4736 		if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
4737 			struct btrfs_fs_info *fs_info = eb->fs_info;
4738 
4739 			spin_unlock(&eb->refs_lock);
4740 
4741 			spin_lock(&fs_info->buffer_lock);
4742 			radix_tree_delete(&fs_info->buffer_radix,
4743 					  eb->start >> fs_info->sectorsize_bits);
4744 			spin_unlock(&fs_info->buffer_lock);
4745 		} else {
4746 			spin_unlock(&eb->refs_lock);
4747 		}
4748 
4749 		btrfs_leak_debug_del_eb(eb);
4750 		/* Should be safe to release our pages at this point */
4751 		btrfs_release_extent_buffer_pages(eb);
4752 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4753 		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
4754 			__free_extent_buffer(eb);
4755 			return 1;
4756 		}
4757 #endif
4758 		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4759 		return 1;
4760 	}
4761 	spin_unlock(&eb->refs_lock);
4762 
4763 	return 0;
4764 }
4765 
free_extent_buffer(struct extent_buffer * eb)4766 void free_extent_buffer(struct extent_buffer *eb)
4767 {
4768 	int refs;
4769 	if (!eb)
4770 		return;
4771 
4772 	refs = atomic_read(&eb->refs);
4773 	while (1) {
4774 		if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
4775 		    || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
4776 			refs == 1))
4777 			break;
4778 		if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
4779 			return;
4780 	}
4781 
4782 	spin_lock(&eb->refs_lock);
4783 	if (atomic_read(&eb->refs) == 2 &&
4784 	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4785 	    !extent_buffer_under_io(eb) &&
4786 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4787 		atomic_dec(&eb->refs);
4788 
4789 	/*
4790 	 * I know this is terrible, but it's temporary until we stop tracking
4791 	 * the uptodate bits and such for the extent buffers.
4792 	 */
4793 	release_extent_buffer(eb);
4794 }
4795 
free_extent_buffer_stale(struct extent_buffer * eb)4796 void free_extent_buffer_stale(struct extent_buffer *eb)
4797 {
4798 	if (!eb)
4799 		return;
4800 
4801 	spin_lock(&eb->refs_lock);
4802 	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4803 
4804 	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4805 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4806 		atomic_dec(&eb->refs);
4807 	release_extent_buffer(eb);
4808 }
4809 
btree_clear_page_dirty(struct page * page)4810 static void btree_clear_page_dirty(struct page *page)
4811 {
4812 	ASSERT(PageDirty(page));
4813 	ASSERT(PageLocked(page));
4814 	clear_page_dirty_for_io(page);
4815 	xa_lock_irq(&page->mapping->i_pages);
4816 	if (!PageDirty(page))
4817 		__xa_clear_mark(&page->mapping->i_pages,
4818 				page_index(page), PAGECACHE_TAG_DIRTY);
4819 	xa_unlock_irq(&page->mapping->i_pages);
4820 }
4821 
clear_subpage_extent_buffer_dirty(const struct extent_buffer * eb)4822 static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
4823 {
4824 	struct btrfs_fs_info *fs_info = eb->fs_info;
4825 	struct page *page = eb->pages[0];
4826 	bool last;
4827 
4828 	/* btree_clear_page_dirty() needs page locked */
4829 	lock_page(page);
4830 	last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start,
4831 						  eb->len);
4832 	if (last)
4833 		btree_clear_page_dirty(page);
4834 	unlock_page(page);
4835 	WARN_ON(atomic_read(&eb->refs) == 0);
4836 }
4837 
clear_extent_buffer_dirty(const struct extent_buffer * eb)4838 void clear_extent_buffer_dirty(const struct extent_buffer *eb)
4839 {
4840 	int i;
4841 	int num_pages;
4842 	struct page *page;
4843 
4844 	if (eb->fs_info->nodesize < PAGE_SIZE)
4845 		return clear_subpage_extent_buffer_dirty(eb);
4846 
4847 	num_pages = num_extent_pages(eb);
4848 
4849 	for (i = 0; i < num_pages; i++) {
4850 		page = eb->pages[i];
4851 		if (!PageDirty(page))
4852 			continue;
4853 		lock_page(page);
4854 		btree_clear_page_dirty(page);
4855 		ClearPageError(page);
4856 		unlock_page(page);
4857 	}
4858 	WARN_ON(atomic_read(&eb->refs) == 0);
4859 }
4860 
set_extent_buffer_dirty(struct extent_buffer * eb)4861 bool set_extent_buffer_dirty(struct extent_buffer *eb)
4862 {
4863 	int i;
4864 	int num_pages;
4865 	bool was_dirty;
4866 
4867 	check_buffer_tree_ref(eb);
4868 
4869 	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
4870 
4871 	num_pages = num_extent_pages(eb);
4872 	WARN_ON(atomic_read(&eb->refs) == 0);
4873 	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4874 
4875 	if (!was_dirty) {
4876 		bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
4877 
4878 		/*
4879 		 * For subpage case, we can have other extent buffers in the
4880 		 * same page, and in clear_subpage_extent_buffer_dirty() we
4881 		 * have to clear page dirty without subpage lock held.
4882 		 * This can cause race where our page gets dirty cleared after
4883 		 * we just set it.
4884 		 *
4885 		 * Thankfully, clear_subpage_extent_buffer_dirty() has locked
4886 		 * its page for other reasons, we can use page lock to prevent
4887 		 * the above race.
4888 		 */
4889 		if (subpage)
4890 			lock_page(eb->pages[0]);
4891 		for (i = 0; i < num_pages; i++)
4892 			btrfs_page_set_dirty(eb->fs_info, eb->pages[i],
4893 					     eb->start, eb->len);
4894 		if (subpage)
4895 			unlock_page(eb->pages[0]);
4896 	}
4897 #ifdef CONFIG_BTRFS_DEBUG
4898 	for (i = 0; i < num_pages; i++)
4899 		ASSERT(PageDirty(eb->pages[i]));
4900 #endif
4901 
4902 	return was_dirty;
4903 }
4904 
clear_extent_buffer_uptodate(struct extent_buffer * eb)4905 void clear_extent_buffer_uptodate(struct extent_buffer *eb)
4906 {
4907 	struct btrfs_fs_info *fs_info = eb->fs_info;
4908 	struct page *page;
4909 	int num_pages;
4910 	int i;
4911 
4912 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4913 	num_pages = num_extent_pages(eb);
4914 	for (i = 0; i < num_pages; i++) {
4915 		page = eb->pages[i];
4916 		if (!page)
4917 			continue;
4918 
4919 		/*
4920 		 * This is special handling for metadata subpage, as regular
4921 		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
4922 		 */
4923 		if (fs_info->nodesize >= PAGE_SIZE)
4924 			ClearPageUptodate(page);
4925 		else
4926 			btrfs_subpage_clear_uptodate(fs_info, page, eb->start,
4927 						     eb->len);
4928 	}
4929 }
4930 
set_extent_buffer_uptodate(struct extent_buffer * eb)4931 void set_extent_buffer_uptodate(struct extent_buffer *eb)
4932 {
4933 	struct btrfs_fs_info *fs_info = eb->fs_info;
4934 	struct page *page;
4935 	int num_pages;
4936 	int i;
4937 
4938 	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4939 	num_pages = num_extent_pages(eb);
4940 	for (i = 0; i < num_pages; i++) {
4941 		page = eb->pages[i];
4942 
4943 		/*
4944 		 * This is special handling for metadata subpage, as regular
4945 		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
4946 		 */
4947 		if (fs_info->nodesize >= PAGE_SIZE)
4948 			SetPageUptodate(page);
4949 		else
4950 			btrfs_subpage_set_uptodate(fs_info, page, eb->start,
4951 						   eb->len);
4952 	}
4953 }
4954 
read_extent_buffer_subpage(struct extent_buffer * eb,int wait,int mirror_num)4955 static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait,
4956 				      int mirror_num)
4957 {
4958 	struct btrfs_fs_info *fs_info = eb->fs_info;
4959 	struct extent_io_tree *io_tree;
4960 	struct page *page = eb->pages[0];
4961 	struct btrfs_bio_ctrl bio_ctrl = {
4962 		.mirror_num = mirror_num,
4963 	};
4964 	int ret = 0;
4965 
4966 	ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags));
4967 	ASSERT(PagePrivate(page));
4968 	io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
4969 
4970 	if (wait == WAIT_NONE) {
4971 		if (!try_lock_extent(io_tree, eb->start, eb->start + eb->len - 1))
4972 			return -EAGAIN;
4973 	} else {
4974 		ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1, NULL);
4975 		if (ret < 0)
4976 			return ret;
4977 	}
4978 
4979 	ret = 0;
4980 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags) ||
4981 	    PageUptodate(page) ||
4982 	    btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) {
4983 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4984 		unlock_extent(io_tree, eb->start, eb->start + eb->len - 1, NULL);
4985 		return ret;
4986 	}
4987 
4988 	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
4989 	eb->read_mirror = 0;
4990 	atomic_set(&eb->io_pages, 1);
4991 	check_buffer_tree_ref(eb);
4992 	bio_ctrl.end_io_func = end_bio_extent_readpage;
4993 
4994 	btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len);
4995 
4996 	btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len);
4997 	ret = submit_extent_page(REQ_OP_READ, NULL, &bio_ctrl,
4998 				 eb->start, page, eb->len,
4999 				 eb->start - page_offset(page), 0, true);
5000 	if (ret) {
5001 		/*
5002 		 * In the endio function, if we hit something wrong we will
5003 		 * increase the io_pages, so here we need to decrease it for
5004 		 * error path.
5005 		 */
5006 		atomic_dec(&eb->io_pages);
5007 	}
5008 	submit_one_bio(&bio_ctrl);
5009 	if (ret || wait != WAIT_COMPLETE)
5010 		return ret;
5011 
5012 	wait_extent_bit(io_tree, eb->start, eb->start + eb->len - 1, EXTENT_LOCKED);
5013 	if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
5014 		ret = -EIO;
5015 	return ret;
5016 }
5017 
read_extent_buffer_pages(struct extent_buffer * eb,int wait,int mirror_num)5018 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num)
5019 {
5020 	int i;
5021 	struct page *page;
5022 	int err;
5023 	int ret = 0;
5024 	int locked_pages = 0;
5025 	int all_uptodate = 1;
5026 	int num_pages;
5027 	unsigned long num_reads = 0;
5028 	struct btrfs_bio_ctrl bio_ctrl = {
5029 		.mirror_num = mirror_num,
5030 	};
5031 
5032 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
5033 		return 0;
5034 
5035 	/*
5036 	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
5037 	 * operation, which could potentially still be in flight.  In this case
5038 	 * we simply want to return an error.
5039 	 */
5040 	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
5041 		return -EIO;
5042 
5043 	if (eb->fs_info->nodesize < PAGE_SIZE)
5044 		return read_extent_buffer_subpage(eb, wait, mirror_num);
5045 
5046 	num_pages = num_extent_pages(eb);
5047 	for (i = 0; i < num_pages; i++) {
5048 		page = eb->pages[i];
5049 		if (wait == WAIT_NONE) {
5050 			/*
5051 			 * WAIT_NONE is only utilized by readahead. If we can't
5052 			 * acquire the lock atomically it means either the eb
5053 			 * is being read out or under modification.
5054 			 * Either way the eb will be or has been cached,
5055 			 * readahead can exit safely.
5056 			 */
5057 			if (!trylock_page(page))
5058 				goto unlock_exit;
5059 		} else {
5060 			lock_page(page);
5061 		}
5062 		locked_pages++;
5063 	}
5064 	/*
5065 	 * We need to firstly lock all pages to make sure that
5066 	 * the uptodate bit of our pages won't be affected by
5067 	 * clear_extent_buffer_uptodate().
5068 	 */
5069 	for (i = 0; i < num_pages; i++) {
5070 		page = eb->pages[i];
5071 		if (!PageUptodate(page)) {
5072 			num_reads++;
5073 			all_uptodate = 0;
5074 		}
5075 	}
5076 
5077 	if (all_uptodate) {
5078 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5079 		goto unlock_exit;
5080 	}
5081 
5082 	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
5083 	eb->read_mirror = 0;
5084 	atomic_set(&eb->io_pages, num_reads);
5085 	/*
5086 	 * It is possible for release_folio to clear the TREE_REF bit before we
5087 	 * set io_pages. See check_buffer_tree_ref for a more detailed comment.
5088 	 */
5089 	check_buffer_tree_ref(eb);
5090 	bio_ctrl.end_io_func = end_bio_extent_readpage;
5091 	for (i = 0; i < num_pages; i++) {
5092 		page = eb->pages[i];
5093 
5094 		if (!PageUptodate(page)) {
5095 			if (ret) {
5096 				atomic_dec(&eb->io_pages);
5097 				unlock_page(page);
5098 				continue;
5099 			}
5100 
5101 			ClearPageError(page);
5102 			err = submit_extent_page(REQ_OP_READ, NULL,
5103 					 &bio_ctrl, page_offset(page), page,
5104 					 PAGE_SIZE, 0, 0, false);
5105 			if (err) {
5106 				/*
5107 				 * We failed to submit the bio so it's the
5108 				 * caller's responsibility to perform cleanup
5109 				 * i.e unlock page/set error bit.
5110 				 */
5111 				ret = err;
5112 				SetPageError(page);
5113 				unlock_page(page);
5114 				atomic_dec(&eb->io_pages);
5115 			}
5116 		} else {
5117 			unlock_page(page);
5118 		}
5119 	}
5120 
5121 	submit_one_bio(&bio_ctrl);
5122 
5123 	if (ret || wait != WAIT_COMPLETE)
5124 		return ret;
5125 
5126 	for (i = 0; i < num_pages; i++) {
5127 		page = eb->pages[i];
5128 		wait_on_page_locked(page);
5129 		if (!PageUptodate(page))
5130 			ret = -EIO;
5131 	}
5132 
5133 	return ret;
5134 
5135 unlock_exit:
5136 	while (locked_pages > 0) {
5137 		locked_pages--;
5138 		page = eb->pages[locked_pages];
5139 		unlock_page(page);
5140 	}
5141 	return ret;
5142 }
5143 
report_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)5144 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
5145 			    unsigned long len)
5146 {
5147 	btrfs_warn(eb->fs_info,
5148 		"access to eb bytenr %llu len %lu out of range start %lu len %lu",
5149 		eb->start, eb->len, start, len);
5150 	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
5151 
5152 	return true;
5153 }
5154 
5155 /*
5156  * Check if the [start, start + len) range is valid before reading/writing
5157  * the eb.
5158  * NOTE: @start and @len are offset inside the eb, not logical address.
5159  *
5160  * Caller should not touch the dst/src memory if this function returns error.
5161  */
check_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)5162 static inline int check_eb_range(const struct extent_buffer *eb,
5163 				 unsigned long start, unsigned long len)
5164 {
5165 	unsigned long offset;
5166 
5167 	/* start, start + len should not go beyond eb->len nor overflow */
5168 	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
5169 		return report_eb_range(eb, start, len);
5170 
5171 	return false;
5172 }
5173 
read_extent_buffer(const struct extent_buffer * eb,void * dstv,unsigned long start,unsigned long len)5174 void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
5175 			unsigned long start, unsigned long len)
5176 {
5177 	size_t cur;
5178 	size_t offset;
5179 	struct page *page;
5180 	char *kaddr;
5181 	char *dst = (char *)dstv;
5182 	unsigned long i = get_eb_page_index(start);
5183 
5184 	if (check_eb_range(eb, start, len))
5185 		return;
5186 
5187 	offset = get_eb_offset_in_page(eb, start);
5188 
5189 	while (len > 0) {
5190 		page = eb->pages[i];
5191 
5192 		cur = min(len, (PAGE_SIZE - offset));
5193 		kaddr = page_address(page);
5194 		memcpy(dst, kaddr + offset, cur);
5195 
5196 		dst += cur;
5197 		len -= cur;
5198 		offset = 0;
5199 		i++;
5200 	}
5201 }
5202 
read_extent_buffer_to_user_nofault(const struct extent_buffer * eb,void __user * dstv,unsigned long start,unsigned long len)5203 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
5204 				       void __user *dstv,
5205 				       unsigned long start, unsigned long len)
5206 {
5207 	size_t cur;
5208 	size_t offset;
5209 	struct page *page;
5210 	char *kaddr;
5211 	char __user *dst = (char __user *)dstv;
5212 	unsigned long i = get_eb_page_index(start);
5213 	int ret = 0;
5214 
5215 	WARN_ON(start > eb->len);
5216 	WARN_ON(start + len > eb->start + eb->len);
5217 
5218 	offset = get_eb_offset_in_page(eb, start);
5219 
5220 	while (len > 0) {
5221 		page = eb->pages[i];
5222 
5223 		cur = min(len, (PAGE_SIZE - offset));
5224 		kaddr = page_address(page);
5225 		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
5226 			ret = -EFAULT;
5227 			break;
5228 		}
5229 
5230 		dst += cur;
5231 		len -= cur;
5232 		offset = 0;
5233 		i++;
5234 	}
5235 
5236 	return ret;
5237 }
5238 
memcmp_extent_buffer(const struct extent_buffer * eb,const void * ptrv,unsigned long start,unsigned long len)5239 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
5240 			 unsigned long start, unsigned long len)
5241 {
5242 	size_t cur;
5243 	size_t offset;
5244 	struct page *page;
5245 	char *kaddr;
5246 	char *ptr = (char *)ptrv;
5247 	unsigned long i = get_eb_page_index(start);
5248 	int ret = 0;
5249 
5250 	if (check_eb_range(eb, start, len))
5251 		return -EINVAL;
5252 
5253 	offset = get_eb_offset_in_page(eb, start);
5254 
5255 	while (len > 0) {
5256 		page = eb->pages[i];
5257 
5258 		cur = min(len, (PAGE_SIZE - offset));
5259 
5260 		kaddr = page_address(page);
5261 		ret = memcmp(ptr, kaddr + offset, cur);
5262 		if (ret)
5263 			break;
5264 
5265 		ptr += cur;
5266 		len -= cur;
5267 		offset = 0;
5268 		i++;
5269 	}
5270 	return ret;
5271 }
5272 
5273 /*
5274  * Check that the extent buffer is uptodate.
5275  *
5276  * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
5277  * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
5278  */
assert_eb_page_uptodate(const struct extent_buffer * eb,struct page * page)5279 static void assert_eb_page_uptodate(const struct extent_buffer *eb,
5280 				    struct page *page)
5281 {
5282 	struct btrfs_fs_info *fs_info = eb->fs_info;
5283 
5284 	/*
5285 	 * If we are using the commit root we could potentially clear a page
5286 	 * Uptodate while we're using the extent buffer that we've previously
5287 	 * looked up.  We don't want to complain in this case, as the page was
5288 	 * valid before, we just didn't write it out.  Instead we want to catch
5289 	 * the case where we didn't actually read the block properly, which
5290 	 * would have !PageUptodate && !PageError, as we clear PageError before
5291 	 * reading.
5292 	 */
5293 	if (fs_info->nodesize < PAGE_SIZE) {
5294 		bool uptodate, error;
5295 
5296 		uptodate = btrfs_subpage_test_uptodate(fs_info, page,
5297 						       eb->start, eb->len);
5298 		error = btrfs_subpage_test_error(fs_info, page, eb->start, eb->len);
5299 		WARN_ON(!uptodate && !error);
5300 	} else {
5301 		WARN_ON(!PageUptodate(page) && !PageError(page));
5302 	}
5303 }
5304 
write_extent_buffer_chunk_tree_uuid(const struct extent_buffer * eb,const void * srcv)5305 void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
5306 		const void *srcv)
5307 {
5308 	char *kaddr;
5309 
5310 	assert_eb_page_uptodate(eb, eb->pages[0]);
5311 	kaddr = page_address(eb->pages[0]) +
5312 		get_eb_offset_in_page(eb, offsetof(struct btrfs_header,
5313 						   chunk_tree_uuid));
5314 	memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
5315 }
5316 
write_extent_buffer_fsid(const struct extent_buffer * eb,const void * srcv)5317 void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv)
5318 {
5319 	char *kaddr;
5320 
5321 	assert_eb_page_uptodate(eb, eb->pages[0]);
5322 	kaddr = page_address(eb->pages[0]) +
5323 		get_eb_offset_in_page(eb, offsetof(struct btrfs_header, fsid));
5324 	memcpy(kaddr, srcv, BTRFS_FSID_SIZE);
5325 }
5326 
write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len)5327 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
5328 			 unsigned long start, unsigned long len)
5329 {
5330 	size_t cur;
5331 	size_t offset;
5332 	struct page *page;
5333 	char *kaddr;
5334 	char *src = (char *)srcv;
5335 	unsigned long i = get_eb_page_index(start);
5336 
5337 	WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
5338 
5339 	if (check_eb_range(eb, start, len))
5340 		return;
5341 
5342 	offset = get_eb_offset_in_page(eb, start);
5343 
5344 	while (len > 0) {
5345 		page = eb->pages[i];
5346 		assert_eb_page_uptodate(eb, page);
5347 
5348 		cur = min(len, PAGE_SIZE - offset);
5349 		kaddr = page_address(page);
5350 		memcpy(kaddr + offset, src, cur);
5351 
5352 		src += cur;
5353 		len -= cur;
5354 		offset = 0;
5355 		i++;
5356 	}
5357 }
5358 
memzero_extent_buffer(const struct extent_buffer * eb,unsigned long start,unsigned long len)5359 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
5360 		unsigned long len)
5361 {
5362 	size_t cur;
5363 	size_t offset;
5364 	struct page *page;
5365 	char *kaddr;
5366 	unsigned long i = get_eb_page_index(start);
5367 
5368 	if (check_eb_range(eb, start, len))
5369 		return;
5370 
5371 	offset = get_eb_offset_in_page(eb, start);
5372 
5373 	while (len > 0) {
5374 		page = eb->pages[i];
5375 		assert_eb_page_uptodate(eb, page);
5376 
5377 		cur = min(len, PAGE_SIZE - offset);
5378 		kaddr = page_address(page);
5379 		memset(kaddr + offset, 0, cur);
5380 
5381 		len -= cur;
5382 		offset = 0;
5383 		i++;
5384 	}
5385 }
5386 
copy_extent_buffer_full(const struct extent_buffer * dst,const struct extent_buffer * src)5387 void copy_extent_buffer_full(const struct extent_buffer *dst,
5388 			     const struct extent_buffer *src)
5389 {
5390 	int i;
5391 	int num_pages;
5392 
5393 	ASSERT(dst->len == src->len);
5394 
5395 	if (dst->fs_info->nodesize >= PAGE_SIZE) {
5396 		num_pages = num_extent_pages(dst);
5397 		for (i = 0; i < num_pages; i++)
5398 			copy_page(page_address(dst->pages[i]),
5399 				  page_address(src->pages[i]));
5400 	} else {
5401 		size_t src_offset = get_eb_offset_in_page(src, 0);
5402 		size_t dst_offset = get_eb_offset_in_page(dst, 0);
5403 
5404 		ASSERT(src->fs_info->nodesize < PAGE_SIZE);
5405 		memcpy(page_address(dst->pages[0]) + dst_offset,
5406 		       page_address(src->pages[0]) + src_offset,
5407 		       src->len);
5408 	}
5409 }
5410 
copy_extent_buffer(const struct extent_buffer * dst,const struct extent_buffer * src,unsigned long dst_offset,unsigned long src_offset,unsigned long len)5411 void copy_extent_buffer(const struct extent_buffer *dst,
5412 			const struct extent_buffer *src,
5413 			unsigned long dst_offset, unsigned long src_offset,
5414 			unsigned long len)
5415 {
5416 	u64 dst_len = dst->len;
5417 	size_t cur;
5418 	size_t offset;
5419 	struct page *page;
5420 	char *kaddr;
5421 	unsigned long i = get_eb_page_index(dst_offset);
5422 
5423 	if (check_eb_range(dst, dst_offset, len) ||
5424 	    check_eb_range(src, src_offset, len))
5425 		return;
5426 
5427 	WARN_ON(src->len != dst_len);
5428 
5429 	offset = get_eb_offset_in_page(dst, dst_offset);
5430 
5431 	while (len > 0) {
5432 		page = dst->pages[i];
5433 		assert_eb_page_uptodate(dst, page);
5434 
5435 		cur = min(len, (unsigned long)(PAGE_SIZE - offset));
5436 
5437 		kaddr = page_address(page);
5438 		read_extent_buffer(src, kaddr + offset, src_offset, cur);
5439 
5440 		src_offset += cur;
5441 		len -= cur;
5442 		offset = 0;
5443 		i++;
5444 	}
5445 }
5446 
5447 /*
5448  * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5449  * given bit number
5450  * @eb: the extent buffer
5451  * @start: offset of the bitmap item in the extent buffer
5452  * @nr: bit number
5453  * @page_index: return index of the page in the extent buffer that contains the
5454  * given bit number
5455  * @page_offset: return offset into the page given by page_index
5456  *
5457  * This helper hides the ugliness of finding the byte in an extent buffer which
5458  * contains a given bit.
5459  */
eb_bitmap_offset(const struct extent_buffer * eb,unsigned long start,unsigned long nr,unsigned long * page_index,size_t * page_offset)5460 static inline void eb_bitmap_offset(const struct extent_buffer *eb,
5461 				    unsigned long start, unsigned long nr,
5462 				    unsigned long *page_index,
5463 				    size_t *page_offset)
5464 {
5465 	size_t byte_offset = BIT_BYTE(nr);
5466 	size_t offset;
5467 
5468 	/*
5469 	 * The byte we want is the offset of the extent buffer + the offset of
5470 	 * the bitmap item in the extent buffer + the offset of the byte in the
5471 	 * bitmap item.
5472 	 */
5473 	offset = start + offset_in_page(eb->start) + byte_offset;
5474 
5475 	*page_index = offset >> PAGE_SHIFT;
5476 	*page_offset = offset_in_page(offset);
5477 }
5478 
5479 /**
5480  * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5481  * @eb: the extent buffer
5482  * @start: offset of the bitmap item in the extent buffer
5483  * @nr: bit number to test
5484  */
extent_buffer_test_bit(const struct extent_buffer * eb,unsigned long start,unsigned long nr)5485 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
5486 			   unsigned long nr)
5487 {
5488 	u8 *kaddr;
5489 	struct page *page;
5490 	unsigned long i;
5491 	size_t offset;
5492 
5493 	eb_bitmap_offset(eb, start, nr, &i, &offset);
5494 	page = eb->pages[i];
5495 	assert_eb_page_uptodate(eb, page);
5496 	kaddr = page_address(page);
5497 	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5498 }
5499 
5500 /**
5501  * extent_buffer_bitmap_set - set an area of a bitmap
5502  * @eb: the extent buffer
5503  * @start: offset of the bitmap item in the extent buffer
5504  * @pos: bit number of the first bit
5505  * @len: number of bits to set
5506  */
extent_buffer_bitmap_set(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)5507 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
5508 			      unsigned long pos, unsigned long len)
5509 {
5510 	u8 *kaddr;
5511 	struct page *page;
5512 	unsigned long i;
5513 	size_t offset;
5514 	const unsigned int size = pos + len;
5515 	int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
5516 	u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
5517 
5518 	eb_bitmap_offset(eb, start, pos, &i, &offset);
5519 	page = eb->pages[i];
5520 	assert_eb_page_uptodate(eb, page);
5521 	kaddr = page_address(page);
5522 
5523 	while (len >= bits_to_set) {
5524 		kaddr[offset] |= mask_to_set;
5525 		len -= bits_to_set;
5526 		bits_to_set = BITS_PER_BYTE;
5527 		mask_to_set = ~0;
5528 		if (++offset >= PAGE_SIZE && len > 0) {
5529 			offset = 0;
5530 			page = eb->pages[++i];
5531 			assert_eb_page_uptodate(eb, page);
5532 			kaddr = page_address(page);
5533 		}
5534 	}
5535 	if (len) {
5536 		mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5537 		kaddr[offset] |= mask_to_set;
5538 	}
5539 }
5540 
5541 
5542 /**
5543  * extent_buffer_bitmap_clear - clear an area of a bitmap
5544  * @eb: the extent buffer
5545  * @start: offset of the bitmap item in the extent buffer
5546  * @pos: bit number of the first bit
5547  * @len: number of bits to clear
5548  */
extent_buffer_bitmap_clear(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)5549 void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
5550 				unsigned long start, unsigned long pos,
5551 				unsigned long len)
5552 {
5553 	u8 *kaddr;
5554 	struct page *page;
5555 	unsigned long i;
5556 	size_t offset;
5557 	const unsigned int size = pos + len;
5558 	int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
5559 	u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
5560 
5561 	eb_bitmap_offset(eb, start, pos, &i, &offset);
5562 	page = eb->pages[i];
5563 	assert_eb_page_uptodate(eb, page);
5564 	kaddr = page_address(page);
5565 
5566 	while (len >= bits_to_clear) {
5567 		kaddr[offset] &= ~mask_to_clear;
5568 		len -= bits_to_clear;
5569 		bits_to_clear = BITS_PER_BYTE;
5570 		mask_to_clear = ~0;
5571 		if (++offset >= PAGE_SIZE && len > 0) {
5572 			offset = 0;
5573 			page = eb->pages[++i];
5574 			assert_eb_page_uptodate(eb, page);
5575 			kaddr = page_address(page);
5576 		}
5577 	}
5578 	if (len) {
5579 		mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5580 		kaddr[offset] &= ~mask_to_clear;
5581 	}
5582 }
5583 
areas_overlap(unsigned long src,unsigned long dst,unsigned long len)5584 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5585 {
5586 	unsigned long distance = (src > dst) ? src - dst : dst - src;
5587 	return distance < len;
5588 }
5589 
copy_pages(struct page * dst_page,struct page * src_page,unsigned long dst_off,unsigned long src_off,unsigned long len)5590 static void copy_pages(struct page *dst_page, struct page *src_page,
5591 		       unsigned long dst_off, unsigned long src_off,
5592 		       unsigned long len)
5593 {
5594 	char *dst_kaddr = page_address(dst_page);
5595 	char *src_kaddr;
5596 	int must_memmove = 0;
5597 
5598 	if (dst_page != src_page) {
5599 		src_kaddr = page_address(src_page);
5600 	} else {
5601 		src_kaddr = dst_kaddr;
5602 		if (areas_overlap(src_off, dst_off, len))
5603 			must_memmove = 1;
5604 	}
5605 
5606 	if (must_memmove)
5607 		memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5608 	else
5609 		memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
5610 }
5611 
memcpy_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)5612 void memcpy_extent_buffer(const struct extent_buffer *dst,
5613 			  unsigned long dst_offset, unsigned long src_offset,
5614 			  unsigned long len)
5615 {
5616 	size_t cur;
5617 	size_t dst_off_in_page;
5618 	size_t src_off_in_page;
5619 	unsigned long dst_i;
5620 	unsigned long src_i;
5621 
5622 	if (check_eb_range(dst, dst_offset, len) ||
5623 	    check_eb_range(dst, src_offset, len))
5624 		return;
5625 
5626 	while (len > 0) {
5627 		dst_off_in_page = get_eb_offset_in_page(dst, dst_offset);
5628 		src_off_in_page = get_eb_offset_in_page(dst, src_offset);
5629 
5630 		dst_i = get_eb_page_index(dst_offset);
5631 		src_i = get_eb_page_index(src_offset);
5632 
5633 		cur = min(len, (unsigned long)(PAGE_SIZE -
5634 					       src_off_in_page));
5635 		cur = min_t(unsigned long, cur,
5636 			(unsigned long)(PAGE_SIZE - dst_off_in_page));
5637 
5638 		copy_pages(dst->pages[dst_i], dst->pages[src_i],
5639 			   dst_off_in_page, src_off_in_page, cur);
5640 
5641 		src_offset += cur;
5642 		dst_offset += cur;
5643 		len -= cur;
5644 	}
5645 }
5646 
memmove_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)5647 void memmove_extent_buffer(const struct extent_buffer *dst,
5648 			   unsigned long dst_offset, unsigned long src_offset,
5649 			   unsigned long len)
5650 {
5651 	size_t cur;
5652 	size_t dst_off_in_page;
5653 	size_t src_off_in_page;
5654 	unsigned long dst_end = dst_offset + len - 1;
5655 	unsigned long src_end = src_offset + len - 1;
5656 	unsigned long dst_i;
5657 	unsigned long src_i;
5658 
5659 	if (check_eb_range(dst, dst_offset, len) ||
5660 	    check_eb_range(dst, src_offset, len))
5661 		return;
5662 	if (dst_offset < src_offset) {
5663 		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5664 		return;
5665 	}
5666 	while (len > 0) {
5667 		dst_i = get_eb_page_index(dst_end);
5668 		src_i = get_eb_page_index(src_end);
5669 
5670 		dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
5671 		src_off_in_page = get_eb_offset_in_page(dst, src_end);
5672 
5673 		cur = min_t(unsigned long, len, src_off_in_page + 1);
5674 		cur = min(cur, dst_off_in_page + 1);
5675 		copy_pages(dst->pages[dst_i], dst->pages[src_i],
5676 			   dst_off_in_page - cur + 1,
5677 			   src_off_in_page - cur + 1, cur);
5678 
5679 		dst_end -= cur;
5680 		src_end -= cur;
5681 		len -= cur;
5682 	}
5683 }
5684 
5685 #define GANG_LOOKUP_SIZE	16
get_next_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page,u64 bytenr)5686 static struct extent_buffer *get_next_extent_buffer(
5687 		struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
5688 {
5689 	struct extent_buffer *gang[GANG_LOOKUP_SIZE];
5690 	struct extent_buffer *found = NULL;
5691 	u64 page_start = page_offset(page);
5692 	u64 cur = page_start;
5693 
5694 	ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
5695 	lockdep_assert_held(&fs_info->buffer_lock);
5696 
5697 	while (cur < page_start + PAGE_SIZE) {
5698 		int ret;
5699 		int i;
5700 
5701 		ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
5702 				(void **)gang, cur >> fs_info->sectorsize_bits,
5703 				min_t(unsigned int, GANG_LOOKUP_SIZE,
5704 				      PAGE_SIZE / fs_info->nodesize));
5705 		if (ret == 0)
5706 			goto out;
5707 		for (i = 0; i < ret; i++) {
5708 			/* Already beyond page end */
5709 			if (gang[i]->start >= page_start + PAGE_SIZE)
5710 				goto out;
5711 			/* Found one */
5712 			if (gang[i]->start >= bytenr) {
5713 				found = gang[i];
5714 				goto out;
5715 			}
5716 		}
5717 		cur = gang[ret - 1]->start + gang[ret - 1]->len;
5718 	}
5719 out:
5720 	return found;
5721 }
5722 
try_release_subpage_extent_buffer(struct page * page)5723 static int try_release_subpage_extent_buffer(struct page *page)
5724 {
5725 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
5726 	u64 cur = page_offset(page);
5727 	const u64 end = page_offset(page) + PAGE_SIZE;
5728 	int ret;
5729 
5730 	while (cur < end) {
5731 		struct extent_buffer *eb = NULL;
5732 
5733 		/*
5734 		 * Unlike try_release_extent_buffer() which uses page->private
5735 		 * to grab buffer, for subpage case we rely on radix tree, thus
5736 		 * we need to ensure radix tree consistency.
5737 		 *
5738 		 * We also want an atomic snapshot of the radix tree, thus go
5739 		 * with spinlock rather than RCU.
5740 		 */
5741 		spin_lock(&fs_info->buffer_lock);
5742 		eb = get_next_extent_buffer(fs_info, page, cur);
5743 		if (!eb) {
5744 			/* No more eb in the page range after or at cur */
5745 			spin_unlock(&fs_info->buffer_lock);
5746 			break;
5747 		}
5748 		cur = eb->start + eb->len;
5749 
5750 		/*
5751 		 * The same as try_release_extent_buffer(), to ensure the eb
5752 		 * won't disappear out from under us.
5753 		 */
5754 		spin_lock(&eb->refs_lock);
5755 		if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5756 			spin_unlock(&eb->refs_lock);
5757 			spin_unlock(&fs_info->buffer_lock);
5758 			break;
5759 		}
5760 		spin_unlock(&fs_info->buffer_lock);
5761 
5762 		/*
5763 		 * If tree ref isn't set then we know the ref on this eb is a
5764 		 * real ref, so just return, this eb will likely be freed soon
5765 		 * anyway.
5766 		 */
5767 		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5768 			spin_unlock(&eb->refs_lock);
5769 			break;
5770 		}
5771 
5772 		/*
5773 		 * Here we don't care about the return value, we will always
5774 		 * check the page private at the end.  And
5775 		 * release_extent_buffer() will release the refs_lock.
5776 		 */
5777 		release_extent_buffer(eb);
5778 	}
5779 	/*
5780 	 * Finally to check if we have cleared page private, as if we have
5781 	 * released all ebs in the page, the page private should be cleared now.
5782 	 */
5783 	spin_lock(&page->mapping->private_lock);
5784 	if (!PagePrivate(page))
5785 		ret = 1;
5786 	else
5787 		ret = 0;
5788 	spin_unlock(&page->mapping->private_lock);
5789 	return ret;
5790 
5791 }
5792 
try_release_extent_buffer(struct page * page)5793 int try_release_extent_buffer(struct page *page)
5794 {
5795 	struct extent_buffer *eb;
5796 
5797 	if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
5798 		return try_release_subpage_extent_buffer(page);
5799 
5800 	/*
5801 	 * We need to make sure nobody is changing page->private, as we rely on
5802 	 * page->private as the pointer to extent buffer.
5803 	 */
5804 	spin_lock(&page->mapping->private_lock);
5805 	if (!PagePrivate(page)) {
5806 		spin_unlock(&page->mapping->private_lock);
5807 		return 1;
5808 	}
5809 
5810 	eb = (struct extent_buffer *)page->private;
5811 	BUG_ON(!eb);
5812 
5813 	/*
5814 	 * This is a little awful but should be ok, we need to make sure that
5815 	 * the eb doesn't disappear out from under us while we're looking at
5816 	 * this page.
5817 	 */
5818 	spin_lock(&eb->refs_lock);
5819 	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5820 		spin_unlock(&eb->refs_lock);
5821 		spin_unlock(&page->mapping->private_lock);
5822 		return 0;
5823 	}
5824 	spin_unlock(&page->mapping->private_lock);
5825 
5826 	/*
5827 	 * If tree ref isn't set then we know the ref on this eb is a real ref,
5828 	 * so just return, this page will likely be freed soon anyway.
5829 	 */
5830 	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5831 		spin_unlock(&eb->refs_lock);
5832 		return 0;
5833 	}
5834 
5835 	return release_extent_buffer(eb);
5836 }
5837 
5838 /*
5839  * btrfs_readahead_tree_block - attempt to readahead a child block
5840  * @fs_info:	the fs_info
5841  * @bytenr:	bytenr to read
5842  * @owner_root: objectid of the root that owns this eb
5843  * @gen:	generation for the uptodate check, can be 0
5844  * @level:	level for the eb
5845  *
5846  * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
5847  * normal uptodate check of the eb, without checking the generation.  If we have
5848  * to read the block we will not block on anything.
5849  */
btrfs_readahead_tree_block(struct btrfs_fs_info * fs_info,u64 bytenr,u64 owner_root,u64 gen,int level)5850 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
5851 				u64 bytenr, u64 owner_root, u64 gen, int level)
5852 {
5853 	struct extent_buffer *eb;
5854 	int ret;
5855 
5856 	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
5857 	if (IS_ERR(eb))
5858 		return;
5859 
5860 	if (btrfs_buffer_uptodate(eb, gen, 1)) {
5861 		free_extent_buffer(eb);
5862 		return;
5863 	}
5864 
5865 	ret = read_extent_buffer_pages(eb, WAIT_NONE, 0);
5866 	if (ret < 0)
5867 		free_extent_buffer_stale(eb);
5868 	else
5869 		free_extent_buffer(eb);
5870 }
5871 
5872 /*
5873  * btrfs_readahead_node_child - readahead a node's child block
5874  * @node:	parent node we're reading from
5875  * @slot:	slot in the parent node for the child we want to read
5876  *
5877  * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
5878  * the slot in the node provided.
5879  */
btrfs_readahead_node_child(struct extent_buffer * node,int slot)5880 void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
5881 {
5882 	btrfs_readahead_tree_block(node->fs_info,
5883 				   btrfs_node_blockptr(node, slot),
5884 				   btrfs_header_owner(node),
5885 				   btrfs_node_ptr_generation(node, slot),
5886 				   btrfs_header_level(node) - 1);
5887 }
5888