1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/sched/mm.h>
10 #include <linux/spinlock.h>
11 #include <linux/blkdev.h>
12 #include <linux/swap.h>
13 #include <linux/writeback.h>
14 #include <linux/pagevec.h>
15 #include <linux/prefetch.h>
16 #include <linux/fsverity.h>
17 #include "misc.h"
18 #include "extent_io.h"
19 #include "extent-io-tree.h"
20 #include "extent_map.h"
21 #include "ctree.h"
22 #include "btrfs_inode.h"
23 #include "bio.h"
24 #include "check-integrity.h"
25 #include "locking.h"
26 #include "rcu-string.h"
27 #include "backref.h"
28 #include "disk-io.h"
29 #include "subpage.h"
30 #include "zoned.h"
31 #include "block-group.h"
32 #include "compression.h"
33 #include "fs.h"
34 #include "accessors.h"
35 #include "file-item.h"
36 #include "file.h"
37 #include "dev-replace.h"
38 #include "super.h"
39 #include "transaction.h"
40 
41 static struct kmem_cache *extent_buffer_cache;
42 
43 #ifdef CONFIG_BTRFS_DEBUG
btrfs_leak_debug_add_eb(struct extent_buffer * eb)44 static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
45 {
46 	struct btrfs_fs_info *fs_info = eb->fs_info;
47 	unsigned long flags;
48 
49 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
50 	list_add(&eb->leak_list, &fs_info->allocated_ebs);
51 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
52 }
53 
btrfs_leak_debug_del_eb(struct extent_buffer * eb)54 static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
55 {
56 	struct btrfs_fs_info *fs_info = eb->fs_info;
57 	unsigned long flags;
58 
59 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
60 	list_del(&eb->leak_list);
61 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
62 }
63 
btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info * fs_info)64 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
65 {
66 	struct extent_buffer *eb;
67 	unsigned long flags;
68 
69 	/*
70 	 * If we didn't get into open_ctree our allocated_ebs will not be
71 	 * initialized, so just skip this.
72 	 */
73 	if (!fs_info->allocated_ebs.next)
74 		return;
75 
76 	WARN_ON(!list_empty(&fs_info->allocated_ebs));
77 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
78 	while (!list_empty(&fs_info->allocated_ebs)) {
79 		eb = list_first_entry(&fs_info->allocated_ebs,
80 				      struct extent_buffer, leak_list);
81 		pr_err(
82 	"BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
83 		       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
84 		       btrfs_header_owner(eb));
85 		list_del(&eb->leak_list);
86 		kmem_cache_free(extent_buffer_cache, eb);
87 	}
88 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
89 }
90 #else
91 #define btrfs_leak_debug_add_eb(eb)			do {} while (0)
92 #define btrfs_leak_debug_del_eb(eb)			do {} while (0)
93 #endif
94 
95 /*
96  * Structure to record info about the bio being assembled, and other info like
97  * how many bytes are there before stripe/ordered extent boundary.
98  */
99 struct btrfs_bio_ctrl {
100 	struct btrfs_bio *bbio;
101 	enum btrfs_compression_type compress_type;
102 	u32 len_to_oe_boundary;
103 	blk_opf_t opf;
104 	btrfs_bio_end_io_t end_io_func;
105 	struct writeback_control *wbc;
106 };
107 
submit_one_bio(struct btrfs_bio_ctrl * bio_ctrl)108 static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
109 {
110 	struct btrfs_bio *bbio = bio_ctrl->bbio;
111 
112 	if (!bbio)
113 		return;
114 
115 	/* Caller should ensure the bio has at least some range added */
116 	ASSERT(bbio->bio.bi_iter.bi_size);
117 
118 	if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
119 	    bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
120 		btrfs_submit_compressed_read(bbio);
121 	else
122 		btrfs_submit_bio(bbio, 0);
123 
124 	/* The bbio is owned by the end_io handler now */
125 	bio_ctrl->bbio = NULL;
126 }
127 
128 /*
129  * Submit or fail the current bio in the bio_ctrl structure.
130  */
submit_write_bio(struct btrfs_bio_ctrl * bio_ctrl,int ret)131 static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
132 {
133 	struct btrfs_bio *bbio = bio_ctrl->bbio;
134 
135 	if (!bbio)
136 		return;
137 
138 	if (ret) {
139 		ASSERT(ret < 0);
140 		btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
141 		/* The bio is owned by the end_io handler now */
142 		bio_ctrl->bbio = NULL;
143 	} else {
144 		submit_one_bio(bio_ctrl);
145 	}
146 }
147 
extent_buffer_init_cachep(void)148 int __init extent_buffer_init_cachep(void)
149 {
150 	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
151 			sizeof(struct extent_buffer), 0,
152 			SLAB_MEM_SPREAD, NULL);
153 	if (!extent_buffer_cache)
154 		return -ENOMEM;
155 
156 	return 0;
157 }
158 
extent_buffer_free_cachep(void)159 void __cold extent_buffer_free_cachep(void)
160 {
161 	/*
162 	 * Make sure all delayed rcu free are flushed before we
163 	 * destroy caches.
164 	 */
165 	rcu_barrier();
166 	kmem_cache_destroy(extent_buffer_cache);
167 }
168 
extent_range_clear_dirty_for_io(struct inode * inode,u64 start,u64 end)169 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
170 {
171 	unsigned long index = start >> PAGE_SHIFT;
172 	unsigned long end_index = end >> PAGE_SHIFT;
173 	struct page *page;
174 
175 	while (index <= end_index) {
176 		page = find_get_page(inode->i_mapping, index);
177 		BUG_ON(!page); /* Pages should be in the extent_io_tree */
178 		clear_page_dirty_for_io(page);
179 		put_page(page);
180 		index++;
181 	}
182 }
183 
process_one_page(struct btrfs_fs_info * fs_info,struct page * page,struct page * locked_page,unsigned long page_ops,u64 start,u64 end)184 static void process_one_page(struct btrfs_fs_info *fs_info,
185 			     struct page *page, struct page *locked_page,
186 			     unsigned long page_ops, u64 start, u64 end)
187 {
188 	u32 len;
189 
190 	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
191 	len = end + 1 - start;
192 
193 	if (page_ops & PAGE_SET_ORDERED)
194 		btrfs_page_clamp_set_ordered(fs_info, page, start, len);
195 	if (page_ops & PAGE_START_WRITEBACK) {
196 		btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
197 		btrfs_page_clamp_set_writeback(fs_info, page, start, len);
198 	}
199 	if (page_ops & PAGE_END_WRITEBACK)
200 		btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
201 
202 	if (page != locked_page && (page_ops & PAGE_UNLOCK))
203 		btrfs_page_end_writer_lock(fs_info, page, start, len);
204 }
205 
__process_pages_contig(struct address_space * mapping,struct page * locked_page,u64 start,u64 end,unsigned long page_ops)206 static void __process_pages_contig(struct address_space *mapping,
207 				   struct page *locked_page, u64 start, u64 end,
208 				   unsigned long page_ops)
209 {
210 	struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
211 	pgoff_t start_index = start >> PAGE_SHIFT;
212 	pgoff_t end_index = end >> PAGE_SHIFT;
213 	pgoff_t index = start_index;
214 	struct folio_batch fbatch;
215 	int i;
216 
217 	folio_batch_init(&fbatch);
218 	while (index <= end_index) {
219 		int found_folios;
220 
221 		found_folios = filemap_get_folios_contig(mapping, &index,
222 				end_index, &fbatch);
223 		for (i = 0; i < found_folios; i++) {
224 			struct folio *folio = fbatch.folios[i];
225 
226 			process_one_page(fs_info, &folio->page, locked_page,
227 					 page_ops, start, end);
228 		}
229 		folio_batch_release(&fbatch);
230 		cond_resched();
231 	}
232 }
233 
__unlock_for_delalloc(struct inode * inode,struct page * locked_page,u64 start,u64 end)234 static noinline void __unlock_for_delalloc(struct inode *inode,
235 					   struct page *locked_page,
236 					   u64 start, u64 end)
237 {
238 	unsigned long index = start >> PAGE_SHIFT;
239 	unsigned long end_index = end >> PAGE_SHIFT;
240 
241 	ASSERT(locked_page);
242 	if (index == locked_page->index && end_index == index)
243 		return;
244 
245 	__process_pages_contig(inode->i_mapping, locked_page, start, end,
246 			       PAGE_UNLOCK);
247 }
248 
lock_delalloc_pages(struct inode * inode,struct page * locked_page,u64 start,u64 end)249 static noinline int lock_delalloc_pages(struct inode *inode,
250 					struct page *locked_page,
251 					u64 start,
252 					u64 end)
253 {
254 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
255 	struct address_space *mapping = inode->i_mapping;
256 	pgoff_t start_index = start >> PAGE_SHIFT;
257 	pgoff_t end_index = end >> PAGE_SHIFT;
258 	pgoff_t index = start_index;
259 	u64 processed_end = start;
260 	struct folio_batch fbatch;
261 
262 	if (index == locked_page->index && index == end_index)
263 		return 0;
264 
265 	folio_batch_init(&fbatch);
266 	while (index <= end_index) {
267 		unsigned int found_folios, i;
268 
269 		found_folios = filemap_get_folios_contig(mapping, &index,
270 				end_index, &fbatch);
271 		if (found_folios == 0)
272 			goto out;
273 
274 		for (i = 0; i < found_folios; i++) {
275 			struct page *page = &fbatch.folios[i]->page;
276 			u32 len = end + 1 - start;
277 
278 			if (page == locked_page)
279 				continue;
280 
281 			if (btrfs_page_start_writer_lock(fs_info, page, start,
282 							 len))
283 				goto out;
284 
285 			if (!PageDirty(page) || page->mapping != mapping) {
286 				btrfs_page_end_writer_lock(fs_info, page, start,
287 							   len);
288 				goto out;
289 			}
290 
291 			processed_end = page_offset(page) + PAGE_SIZE - 1;
292 		}
293 		folio_batch_release(&fbatch);
294 		cond_resched();
295 	}
296 
297 	return 0;
298 out:
299 	folio_batch_release(&fbatch);
300 	if (processed_end > start)
301 		__unlock_for_delalloc(inode, locked_page, start, processed_end);
302 	return -EAGAIN;
303 }
304 
305 /*
306  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
307  * more than @max_bytes.
308  *
309  * @start:	The original start bytenr to search.
310  *		Will store the extent range start bytenr.
311  * @end:	The original end bytenr of the search range
312  *		Will store the extent range end bytenr.
313  *
314  * Return true if we find a delalloc range which starts inside the original
315  * range, and @start/@end will store the delalloc range start/end.
316  *
317  * Return false if we can't find any delalloc range which starts inside the
318  * original range, and @start/@end will be the non-delalloc range start/end.
319  */
320 EXPORT_FOR_TESTS
find_lock_delalloc_range(struct inode * inode,struct page * locked_page,u64 * start,u64 * end)321 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
322 				    struct page *locked_page, u64 *start,
323 				    u64 *end)
324 {
325 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
326 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
327 	const u64 orig_start = *start;
328 	const u64 orig_end = *end;
329 	/* The sanity tests may not set a valid fs_info. */
330 	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
331 	u64 delalloc_start;
332 	u64 delalloc_end;
333 	bool found;
334 	struct extent_state *cached_state = NULL;
335 	int ret;
336 	int loops = 0;
337 
338 	/* Caller should pass a valid @end to indicate the search range end */
339 	ASSERT(orig_end > orig_start);
340 
341 	/* The range should at least cover part of the page */
342 	ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
343 		 orig_end <= page_offset(locked_page)));
344 again:
345 	/* step one, find a bunch of delalloc bytes starting at start */
346 	delalloc_start = *start;
347 	delalloc_end = 0;
348 	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
349 					  max_bytes, &cached_state);
350 	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
351 		*start = delalloc_start;
352 
353 		/* @delalloc_end can be -1, never go beyond @orig_end */
354 		*end = min(delalloc_end, orig_end);
355 		free_extent_state(cached_state);
356 		return false;
357 	}
358 
359 	/*
360 	 * start comes from the offset of locked_page.  We have to lock
361 	 * pages in order, so we can't process delalloc bytes before
362 	 * locked_page
363 	 */
364 	if (delalloc_start < *start)
365 		delalloc_start = *start;
366 
367 	/*
368 	 * make sure to limit the number of pages we try to lock down
369 	 */
370 	if (delalloc_end + 1 - delalloc_start > max_bytes)
371 		delalloc_end = delalloc_start + max_bytes - 1;
372 
373 	/* step two, lock all the pages after the page that has start */
374 	ret = lock_delalloc_pages(inode, locked_page,
375 				  delalloc_start, delalloc_end);
376 	ASSERT(!ret || ret == -EAGAIN);
377 	if (ret == -EAGAIN) {
378 		/* some of the pages are gone, lets avoid looping by
379 		 * shortening the size of the delalloc range we're searching
380 		 */
381 		free_extent_state(cached_state);
382 		cached_state = NULL;
383 		if (!loops) {
384 			max_bytes = PAGE_SIZE;
385 			loops = 1;
386 			goto again;
387 		} else {
388 			found = false;
389 			goto out_failed;
390 		}
391 	}
392 
393 	/* step three, lock the state bits for the whole range */
394 	lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
395 
396 	/* then test to make sure it is all still delalloc */
397 	ret = test_range_bit(tree, delalloc_start, delalloc_end,
398 			     EXTENT_DELALLOC, 1, cached_state);
399 	if (!ret) {
400 		unlock_extent(tree, delalloc_start, delalloc_end,
401 			      &cached_state);
402 		__unlock_for_delalloc(inode, locked_page,
403 			      delalloc_start, delalloc_end);
404 		cond_resched();
405 		goto again;
406 	}
407 	free_extent_state(cached_state);
408 	*start = delalloc_start;
409 	*end = delalloc_end;
410 out_failed:
411 	return found;
412 }
413 
extent_clear_unlock_delalloc(struct btrfs_inode * inode,u64 start,u64 end,struct page * locked_page,u32 clear_bits,unsigned long page_ops)414 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
415 				  struct page *locked_page,
416 				  u32 clear_bits, unsigned long page_ops)
417 {
418 	clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL);
419 
420 	__process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
421 			       start, end, page_ops);
422 }
423 
btrfs_verify_page(struct page * page,u64 start)424 static bool btrfs_verify_page(struct page *page, u64 start)
425 {
426 	if (!fsverity_active(page->mapping->host) ||
427 	    PageUptodate(page) ||
428 	    start >= i_size_read(page->mapping->host))
429 		return true;
430 	return fsverity_verify_page(page);
431 }
432 
end_page_read(struct page * page,bool uptodate,u64 start,u32 len)433 static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
434 {
435 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
436 
437 	ASSERT(page_offset(page) <= start &&
438 	       start + len <= page_offset(page) + PAGE_SIZE);
439 
440 	if (uptodate && btrfs_verify_page(page, start))
441 		btrfs_page_set_uptodate(fs_info, page, start, len);
442 	else
443 		btrfs_page_clear_uptodate(fs_info, page, start, len);
444 
445 	if (!btrfs_is_subpage(fs_info, page))
446 		unlock_page(page);
447 	else
448 		btrfs_subpage_end_reader(fs_info, page, start, len);
449 }
450 
451 /*
452  * after a writepage IO is done, we need to:
453  * clear the uptodate bits on error
454  * clear the writeback bits in the extent tree for this IO
455  * end_page_writeback if the page has no more pending IO
456  *
457  * Scheduling is not allowed, so the extent state tree is expected
458  * to have one and only one object corresponding to this IO.
459  */
end_bio_extent_writepage(struct btrfs_bio * bbio)460 static void end_bio_extent_writepage(struct btrfs_bio *bbio)
461 {
462 	struct bio *bio = &bbio->bio;
463 	int error = blk_status_to_errno(bio->bi_status);
464 	struct bio_vec *bvec;
465 	struct bvec_iter_all iter_all;
466 
467 	ASSERT(!bio_flagged(bio, BIO_CLONED));
468 	bio_for_each_segment_all(bvec, bio, iter_all) {
469 		struct page *page = bvec->bv_page;
470 		struct inode *inode = page->mapping->host;
471 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
472 		const u32 sectorsize = fs_info->sectorsize;
473 		u64 start = page_offset(page) + bvec->bv_offset;
474 		u32 len = bvec->bv_len;
475 
476 		/* Our read/write should always be sector aligned. */
477 		if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
478 			btrfs_err(fs_info,
479 		"partial page write in btrfs with offset %u and length %u",
480 				  bvec->bv_offset, bvec->bv_len);
481 		else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
482 			btrfs_info(fs_info,
483 		"incomplete page write with offset %u and length %u",
484 				   bvec->bv_offset, bvec->bv_len);
485 
486 		btrfs_finish_ordered_extent(bbio->ordered, page, start, len, !error);
487 		if (error)
488 			mapping_set_error(page->mapping, error);
489 		btrfs_page_clear_writeback(fs_info, page, start, len);
490 	}
491 
492 	bio_put(bio);
493 }
494 
495 /*
496  * Record previously processed extent range
497  *
498  * For endio_readpage_release_extent() to handle a full extent range, reducing
499  * the extent io operations.
500  */
501 struct processed_extent {
502 	struct btrfs_inode *inode;
503 	/* Start of the range in @inode */
504 	u64 start;
505 	/* End of the range in @inode */
506 	u64 end;
507 	bool uptodate;
508 };
509 
510 /*
511  * Try to release processed extent range
512  *
513  * May not release the extent range right now if the current range is
514  * contiguous to processed extent.
515  *
516  * Will release processed extent when any of @inode, @uptodate, the range is
517  * no longer contiguous to the processed range.
518  *
519  * Passing @inode == NULL will force processed extent to be released.
520  */
endio_readpage_release_extent(struct processed_extent * processed,struct btrfs_inode * inode,u64 start,u64 end,bool uptodate)521 static void endio_readpage_release_extent(struct processed_extent *processed,
522 			      struct btrfs_inode *inode, u64 start, u64 end,
523 			      bool uptodate)
524 {
525 	struct extent_state *cached = NULL;
526 	struct extent_io_tree *tree;
527 
528 	/* The first extent, initialize @processed */
529 	if (!processed->inode)
530 		goto update;
531 
532 	/*
533 	 * Contiguous to processed extent, just uptodate the end.
534 	 *
535 	 * Several things to notice:
536 	 *
537 	 * - bio can be merged as long as on-disk bytenr is contiguous
538 	 *   This means we can have page belonging to other inodes, thus need to
539 	 *   check if the inode still matches.
540 	 * - bvec can contain range beyond current page for multi-page bvec
541 	 *   Thus we need to do processed->end + 1 >= start check
542 	 */
543 	if (processed->inode == inode && processed->uptodate == uptodate &&
544 	    processed->end + 1 >= start && end >= processed->end) {
545 		processed->end = end;
546 		return;
547 	}
548 
549 	tree = &processed->inode->io_tree;
550 	/*
551 	 * Now we don't have range contiguous to the processed range, release
552 	 * the processed range now.
553 	 */
554 	unlock_extent(tree, processed->start, processed->end, &cached);
555 
556 update:
557 	/* Update processed to current range */
558 	processed->inode = inode;
559 	processed->start = start;
560 	processed->end = end;
561 	processed->uptodate = uptodate;
562 }
563 
begin_page_read(struct btrfs_fs_info * fs_info,struct page * page)564 static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
565 {
566 	ASSERT(PageLocked(page));
567 	if (!btrfs_is_subpage(fs_info, page))
568 		return;
569 
570 	ASSERT(PagePrivate(page));
571 	btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
572 }
573 
574 /*
575  * after a readpage IO is done, we need to:
576  * clear the uptodate bits on error
577  * set the uptodate bits if things worked
578  * set the page up to date if all extents in the tree are uptodate
579  * clear the lock bit in the extent tree
580  * unlock the page if there are no other extents locked for it
581  *
582  * Scheduling is not allowed, so the extent state tree is expected
583  * to have one and only one object corresponding to this IO.
584  */
end_bio_extent_readpage(struct btrfs_bio * bbio)585 static void end_bio_extent_readpage(struct btrfs_bio *bbio)
586 {
587 	struct bio *bio = &bbio->bio;
588 	struct bio_vec *bvec;
589 	struct processed_extent processed = { 0 };
590 	/*
591 	 * The offset to the beginning of a bio, since one bio can never be
592 	 * larger than UINT_MAX, u32 here is enough.
593 	 */
594 	u32 bio_offset = 0;
595 	struct bvec_iter_all iter_all;
596 
597 	ASSERT(!bio_flagged(bio, BIO_CLONED));
598 	bio_for_each_segment_all(bvec, bio, iter_all) {
599 		bool uptodate = !bio->bi_status;
600 		struct page *page = bvec->bv_page;
601 		struct inode *inode = page->mapping->host;
602 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
603 		const u32 sectorsize = fs_info->sectorsize;
604 		u64 start;
605 		u64 end;
606 		u32 len;
607 
608 		btrfs_debug(fs_info,
609 			"end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
610 			bio->bi_iter.bi_sector, bio->bi_status,
611 			bbio->mirror_num);
612 
613 		/*
614 		 * We always issue full-sector reads, but if some block in a
615 		 * page fails to read, blk_update_request() will advance
616 		 * bv_offset and adjust bv_len to compensate.  Print a warning
617 		 * for unaligned offsets, and an error if they don't add up to
618 		 * a full sector.
619 		 */
620 		if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
621 			btrfs_err(fs_info,
622 		"partial page read in btrfs with offset %u and length %u",
623 				  bvec->bv_offset, bvec->bv_len);
624 		else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
625 				     sectorsize))
626 			btrfs_info(fs_info,
627 		"incomplete page read with offset %u and length %u",
628 				   bvec->bv_offset, bvec->bv_len);
629 
630 		start = page_offset(page) + bvec->bv_offset;
631 		end = start + bvec->bv_len - 1;
632 		len = bvec->bv_len;
633 
634 		if (likely(uptodate)) {
635 			loff_t i_size = i_size_read(inode);
636 			pgoff_t end_index = i_size >> PAGE_SHIFT;
637 
638 			/*
639 			 * Zero out the remaining part if this range straddles
640 			 * i_size.
641 			 *
642 			 * Here we should only zero the range inside the bvec,
643 			 * not touch anything else.
644 			 *
645 			 * NOTE: i_size is exclusive while end is inclusive.
646 			 */
647 			if (page->index == end_index && i_size <= end) {
648 				u32 zero_start = max(offset_in_page(i_size),
649 						     offset_in_page(start));
650 
651 				zero_user_segment(page, zero_start,
652 						  offset_in_page(end) + 1);
653 			}
654 		}
655 
656 		/* Update page status and unlock. */
657 		end_page_read(page, uptodate, start, len);
658 		endio_readpage_release_extent(&processed, BTRFS_I(inode),
659 					      start, end, uptodate);
660 
661 		ASSERT(bio_offset + len > bio_offset);
662 		bio_offset += len;
663 
664 	}
665 	/* Release the last extent */
666 	endio_readpage_release_extent(&processed, NULL, 0, 0, false);
667 	bio_put(bio);
668 }
669 
670 /*
671  * Populate every free slot in a provided array with pages.
672  *
673  * @nr_pages:   number of pages to allocate
674  * @page_array: the array to fill with pages; any existing non-null entries in
675  * 		the array will be skipped
676  *
677  * Return: 0        if all pages were able to be allocated;
678  *         -ENOMEM  otherwise, the partially allocated pages would be freed and
679  *                  the array slots zeroed
680  */
btrfs_alloc_page_array(unsigned int nr_pages,struct page ** page_array)681 int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
682 {
683 	unsigned int allocated;
684 
685 	for (allocated = 0; allocated < nr_pages;) {
686 		unsigned int last = allocated;
687 
688 		allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
689 
690 		if (allocated == nr_pages)
691 			return 0;
692 
693 		/*
694 		 * During this iteration, no page could be allocated, even
695 		 * though alloc_pages_bulk_array() falls back to alloc_page()
696 		 * if  it could not bulk-allocate. So we must be out of memory.
697 		 */
698 		if (allocated == last) {
699 			for (int i = 0; i < allocated; i++) {
700 				__free_page(page_array[i]);
701 				page_array[i] = NULL;
702 			}
703 			return -ENOMEM;
704 		}
705 
706 		memalloc_retry_wait(GFP_NOFS);
707 	}
708 	return 0;
709 }
710 
btrfs_bio_is_contig(struct btrfs_bio_ctrl * bio_ctrl,struct page * page,u64 disk_bytenr,unsigned int pg_offset)711 static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
712 				struct page *page, u64 disk_bytenr,
713 				unsigned int pg_offset)
714 {
715 	struct bio *bio = &bio_ctrl->bbio->bio;
716 	struct bio_vec *bvec = bio_last_bvec_all(bio);
717 	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
718 
719 	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
720 		/*
721 		 * For compression, all IO should have its logical bytenr set
722 		 * to the starting bytenr of the compressed extent.
723 		 */
724 		return bio->bi_iter.bi_sector == sector;
725 	}
726 
727 	/*
728 	 * The contig check requires the following conditions to be met:
729 	 *
730 	 * 1) The pages are belonging to the same inode
731 	 *    This is implied by the call chain.
732 	 *
733 	 * 2) The range has adjacent logical bytenr
734 	 *
735 	 * 3) The range has adjacent file offset
736 	 *    This is required for the usage of btrfs_bio->file_offset.
737 	 */
738 	return bio_end_sector(bio) == sector &&
739 		page_offset(bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
740 		page_offset(page) + pg_offset;
741 }
742 
alloc_new_bio(struct btrfs_inode * inode,struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,u64 file_offset)743 static void alloc_new_bio(struct btrfs_inode *inode,
744 			  struct btrfs_bio_ctrl *bio_ctrl,
745 			  u64 disk_bytenr, u64 file_offset)
746 {
747 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
748 	struct btrfs_bio *bbio;
749 
750 	bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
751 			       bio_ctrl->end_io_func, NULL);
752 	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
753 	bbio->inode = inode;
754 	bbio->file_offset = file_offset;
755 	bio_ctrl->bbio = bbio;
756 	bio_ctrl->len_to_oe_boundary = U32_MAX;
757 
758 	/* Limit data write bios to the ordered boundary. */
759 	if (bio_ctrl->wbc) {
760 		struct btrfs_ordered_extent *ordered;
761 
762 		ordered = btrfs_lookup_ordered_extent(inode, file_offset);
763 		if (ordered) {
764 			bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
765 					ordered->file_offset +
766 					ordered->disk_num_bytes - file_offset);
767 			bbio->ordered = ordered;
768 		}
769 
770 		/*
771 		 * Pick the last added device to support cgroup writeback.  For
772 		 * multi-device file systems this means blk-cgroup policies have
773 		 * to always be set on the last added/replaced device.
774 		 * This is a bit odd but has been like that for a long time.
775 		 */
776 		bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
777 		wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
778 	}
779 }
780 
781 /*
782  * @disk_bytenr: logical bytenr where the write will be
783  * @page:	page to add to the bio
784  * @size:	portion of page that we want to write to
785  * @pg_offset:	offset of the new bio or to check whether we are adding
786  *              a contiguous page to the previous one
787  *
788  * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
789  * new one in @bio_ctrl->bbio.
790  * The mirror number for this IO should already be initizlied in
791  * @bio_ctrl->mirror_num.
792  */
submit_extent_page(struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,struct page * page,size_t size,unsigned long pg_offset)793 static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
794 			       u64 disk_bytenr, struct page *page,
795 			       size_t size, unsigned long pg_offset)
796 {
797 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
798 
799 	ASSERT(pg_offset + size <= PAGE_SIZE);
800 	ASSERT(bio_ctrl->end_io_func);
801 
802 	if (bio_ctrl->bbio &&
803 	    !btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
804 		submit_one_bio(bio_ctrl);
805 
806 	do {
807 		u32 len = size;
808 
809 		/* Allocate new bio if needed */
810 		if (!bio_ctrl->bbio) {
811 			alloc_new_bio(inode, bio_ctrl, disk_bytenr,
812 				      page_offset(page) + pg_offset);
813 		}
814 
815 		/* Cap to the current ordered extent boundary if there is one. */
816 		if (len > bio_ctrl->len_to_oe_boundary) {
817 			ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
818 			ASSERT(is_data_inode(&inode->vfs_inode));
819 			len = bio_ctrl->len_to_oe_boundary;
820 		}
821 
822 		if (bio_add_page(&bio_ctrl->bbio->bio, page, len, pg_offset) != len) {
823 			/* bio full: move on to a new one */
824 			submit_one_bio(bio_ctrl);
825 			continue;
826 		}
827 
828 		if (bio_ctrl->wbc)
829 			wbc_account_cgroup_owner(bio_ctrl->wbc, page, len);
830 
831 		size -= len;
832 		pg_offset += len;
833 		disk_bytenr += len;
834 
835 		/*
836 		 * len_to_oe_boundary defaults to U32_MAX, which isn't page or
837 		 * sector aligned.  alloc_new_bio() then sets it to the end of
838 		 * our ordered extent for writes into zoned devices.
839 		 *
840 		 * When len_to_oe_boundary is tracking an ordered extent, we
841 		 * trust the ordered extent code to align things properly, and
842 		 * the check above to cap our write to the ordered extent
843 		 * boundary is correct.
844 		 *
845 		 * When len_to_oe_boundary is U32_MAX, the cap above would
846 		 * result in a 4095 byte IO for the last page right before
847 		 * we hit the bio limit of UINT_MAX.  bio_add_page() has all
848 		 * the checks required to make sure we don't overflow the bio,
849 		 * and we should just ignore len_to_oe_boundary completely
850 		 * unless we're using it to track an ordered extent.
851 		 *
852 		 * It's pretty hard to make a bio sized U32_MAX, but it can
853 		 * happen when the page cache is able to feed us contiguous
854 		 * pages for large extents.
855 		 */
856 		if (bio_ctrl->len_to_oe_boundary != U32_MAX)
857 			bio_ctrl->len_to_oe_boundary -= len;
858 
859 		/* Ordered extent boundary: move on to a new bio. */
860 		if (bio_ctrl->len_to_oe_boundary == 0)
861 			submit_one_bio(bio_ctrl);
862 	} while (size);
863 }
864 
attach_extent_buffer_page(struct extent_buffer * eb,struct page * page,struct btrfs_subpage * prealloc)865 static int attach_extent_buffer_page(struct extent_buffer *eb,
866 				     struct page *page,
867 				     struct btrfs_subpage *prealloc)
868 {
869 	struct btrfs_fs_info *fs_info = eb->fs_info;
870 	int ret = 0;
871 
872 	/*
873 	 * If the page is mapped to btree inode, we should hold the private
874 	 * lock to prevent race.
875 	 * For cloned or dummy extent buffers, their pages are not mapped and
876 	 * will not race with any other ebs.
877 	 */
878 	if (page->mapping)
879 		lockdep_assert_held(&page->mapping->private_lock);
880 
881 	if (fs_info->nodesize >= PAGE_SIZE) {
882 		if (!PagePrivate(page))
883 			attach_page_private(page, eb);
884 		else
885 			WARN_ON(page->private != (unsigned long)eb);
886 		return 0;
887 	}
888 
889 	/* Already mapped, just free prealloc */
890 	if (PagePrivate(page)) {
891 		btrfs_free_subpage(prealloc);
892 		return 0;
893 	}
894 
895 	if (prealloc)
896 		/* Has preallocated memory for subpage */
897 		attach_page_private(page, prealloc);
898 	else
899 		/* Do new allocation to attach subpage */
900 		ret = btrfs_attach_subpage(fs_info, page,
901 					   BTRFS_SUBPAGE_METADATA);
902 	return ret;
903 }
904 
set_page_extent_mapped(struct page * page)905 int set_page_extent_mapped(struct page *page)
906 {
907 	struct btrfs_fs_info *fs_info;
908 
909 	ASSERT(page->mapping);
910 
911 	if (PagePrivate(page))
912 		return 0;
913 
914 	fs_info = btrfs_sb(page->mapping->host->i_sb);
915 
916 	if (btrfs_is_subpage(fs_info, page))
917 		return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
918 
919 	attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
920 	return 0;
921 }
922 
clear_page_extent_mapped(struct page * page)923 void clear_page_extent_mapped(struct page *page)
924 {
925 	struct btrfs_fs_info *fs_info;
926 
927 	ASSERT(page->mapping);
928 
929 	if (!PagePrivate(page))
930 		return;
931 
932 	fs_info = btrfs_sb(page->mapping->host->i_sb);
933 	if (btrfs_is_subpage(fs_info, page))
934 		return btrfs_detach_subpage(fs_info, page);
935 
936 	detach_page_private(page);
937 }
938 
939 static struct extent_map *
__get_extent_map(struct inode * inode,struct page * page,size_t pg_offset,u64 start,u64 len,struct extent_map ** em_cached)940 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
941 		 u64 start, u64 len, struct extent_map **em_cached)
942 {
943 	struct extent_map *em;
944 
945 	if (em_cached && *em_cached) {
946 		em = *em_cached;
947 		if (extent_map_in_tree(em) && start >= em->start &&
948 		    start < extent_map_end(em)) {
949 			refcount_inc(&em->refs);
950 			return em;
951 		}
952 
953 		free_extent_map(em);
954 		*em_cached = NULL;
955 	}
956 
957 	em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
958 	if (em_cached && !IS_ERR(em)) {
959 		BUG_ON(*em_cached);
960 		refcount_inc(&em->refs);
961 		*em_cached = em;
962 	}
963 	return em;
964 }
965 /*
966  * basic readpage implementation.  Locked extent state structs are inserted
967  * into the tree that are removed when the IO is done (by the end_io
968  * handlers)
969  * XXX JDM: This needs looking at to ensure proper page locking
970  * return 0 on success, otherwise return error
971  */
btrfs_do_readpage(struct page * page,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)972 static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
973 		      struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
974 {
975 	struct inode *inode = page->mapping->host;
976 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
977 	u64 start = page_offset(page);
978 	const u64 end = start + PAGE_SIZE - 1;
979 	u64 cur = start;
980 	u64 extent_offset;
981 	u64 last_byte = i_size_read(inode);
982 	u64 block_start;
983 	struct extent_map *em;
984 	int ret = 0;
985 	size_t pg_offset = 0;
986 	size_t iosize;
987 	size_t blocksize = inode->i_sb->s_blocksize;
988 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
989 
990 	ret = set_page_extent_mapped(page);
991 	if (ret < 0) {
992 		unlock_extent(tree, start, end, NULL);
993 		unlock_page(page);
994 		return ret;
995 	}
996 
997 	if (page->index == last_byte >> PAGE_SHIFT) {
998 		size_t zero_offset = offset_in_page(last_byte);
999 
1000 		if (zero_offset) {
1001 			iosize = PAGE_SIZE - zero_offset;
1002 			memzero_page(page, zero_offset, iosize);
1003 		}
1004 	}
1005 	bio_ctrl->end_io_func = end_bio_extent_readpage;
1006 	begin_page_read(fs_info, page);
1007 	while (cur <= end) {
1008 		enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1009 		bool force_bio_submit = false;
1010 		u64 disk_bytenr;
1011 
1012 		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1013 		if (cur >= last_byte) {
1014 			iosize = PAGE_SIZE - pg_offset;
1015 			memzero_page(page, pg_offset, iosize);
1016 			unlock_extent(tree, cur, cur + iosize - 1, NULL);
1017 			end_page_read(page, true, cur, iosize);
1018 			break;
1019 		}
1020 		em = __get_extent_map(inode, page, pg_offset, cur,
1021 				      end - cur + 1, em_cached);
1022 		if (IS_ERR(em)) {
1023 			unlock_extent(tree, cur, end, NULL);
1024 			end_page_read(page, false, cur, end + 1 - cur);
1025 			return PTR_ERR(em);
1026 		}
1027 		extent_offset = cur - em->start;
1028 		BUG_ON(extent_map_end(em) <= cur);
1029 		BUG_ON(end < cur);
1030 
1031 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1032 			compress_type = em->compress_type;
1033 
1034 		iosize = min(extent_map_end(em) - cur, end - cur + 1);
1035 		iosize = ALIGN(iosize, blocksize);
1036 		if (compress_type != BTRFS_COMPRESS_NONE)
1037 			disk_bytenr = em->block_start;
1038 		else
1039 			disk_bytenr = em->block_start + extent_offset;
1040 		block_start = em->block_start;
1041 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1042 			block_start = EXTENT_MAP_HOLE;
1043 
1044 		/*
1045 		 * If we have a file range that points to a compressed extent
1046 		 * and it's followed by a consecutive file range that points
1047 		 * to the same compressed extent (possibly with a different
1048 		 * offset and/or length, so it either points to the whole extent
1049 		 * or only part of it), we must make sure we do not submit a
1050 		 * single bio to populate the pages for the 2 ranges because
1051 		 * this makes the compressed extent read zero out the pages
1052 		 * belonging to the 2nd range. Imagine the following scenario:
1053 		 *
1054 		 *  File layout
1055 		 *  [0 - 8K]                     [8K - 24K]
1056 		 *    |                               |
1057 		 *    |                               |
1058 		 * points to extent X,         points to extent X,
1059 		 * offset 4K, length of 8K     offset 0, length 16K
1060 		 *
1061 		 * [extent X, compressed length = 4K uncompressed length = 16K]
1062 		 *
1063 		 * If the bio to read the compressed extent covers both ranges,
1064 		 * it will decompress extent X into the pages belonging to the
1065 		 * first range and then it will stop, zeroing out the remaining
1066 		 * pages that belong to the other range that points to extent X.
1067 		 * So here we make sure we submit 2 bios, one for the first
1068 		 * range and another one for the third range. Both will target
1069 		 * the same physical extent from disk, but we can't currently
1070 		 * make the compressed bio endio callback populate the pages
1071 		 * for both ranges because each compressed bio is tightly
1072 		 * coupled with a single extent map, and each range can have
1073 		 * an extent map with a different offset value relative to the
1074 		 * uncompressed data of our extent and different lengths. This
1075 		 * is a corner case so we prioritize correctness over
1076 		 * non-optimal behavior (submitting 2 bios for the same extent).
1077 		 */
1078 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1079 		    prev_em_start && *prev_em_start != (u64)-1 &&
1080 		    *prev_em_start != em->start)
1081 			force_bio_submit = true;
1082 
1083 		if (prev_em_start)
1084 			*prev_em_start = em->start;
1085 
1086 		free_extent_map(em);
1087 		em = NULL;
1088 
1089 		/* we've found a hole, just zero and go on */
1090 		if (block_start == EXTENT_MAP_HOLE) {
1091 			memzero_page(page, pg_offset, iosize);
1092 
1093 			unlock_extent(tree, cur, cur + iosize - 1, NULL);
1094 			end_page_read(page, true, cur, iosize);
1095 			cur = cur + iosize;
1096 			pg_offset += iosize;
1097 			continue;
1098 		}
1099 		/* the get_extent function already copied into the page */
1100 		if (block_start == EXTENT_MAP_INLINE) {
1101 			unlock_extent(tree, cur, cur + iosize - 1, NULL);
1102 			end_page_read(page, true, cur, iosize);
1103 			cur = cur + iosize;
1104 			pg_offset += iosize;
1105 			continue;
1106 		}
1107 
1108 		if (bio_ctrl->compress_type != compress_type) {
1109 			submit_one_bio(bio_ctrl);
1110 			bio_ctrl->compress_type = compress_type;
1111 		}
1112 
1113 		if (force_bio_submit)
1114 			submit_one_bio(bio_ctrl);
1115 		submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1116 				   pg_offset);
1117 		cur = cur + iosize;
1118 		pg_offset += iosize;
1119 	}
1120 
1121 	return 0;
1122 }
1123 
btrfs_read_folio(struct file * file,struct folio * folio)1124 int btrfs_read_folio(struct file *file, struct folio *folio)
1125 {
1126 	struct page *page = &folio->page;
1127 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
1128 	u64 start = page_offset(page);
1129 	u64 end = start + PAGE_SIZE - 1;
1130 	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1131 	int ret;
1132 
1133 	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1134 
1135 	ret = btrfs_do_readpage(page, NULL, &bio_ctrl, NULL);
1136 	/*
1137 	 * If btrfs_do_readpage() failed we will want to submit the assembled
1138 	 * bio to do the cleanup.
1139 	 */
1140 	submit_one_bio(&bio_ctrl);
1141 	return ret;
1142 }
1143 
contiguous_readpages(struct page * pages[],int nr_pages,u64 start,u64 end,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)1144 static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1145 					u64 start, u64 end,
1146 					struct extent_map **em_cached,
1147 					struct btrfs_bio_ctrl *bio_ctrl,
1148 					u64 *prev_em_start)
1149 {
1150 	struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
1151 	int index;
1152 
1153 	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1154 
1155 	for (index = 0; index < nr_pages; index++) {
1156 		btrfs_do_readpage(pages[index], em_cached, bio_ctrl,
1157 				  prev_em_start);
1158 		put_page(pages[index]);
1159 	}
1160 }
1161 
1162 /*
1163  * helper for __extent_writepage, doing all of the delayed allocation setup.
1164  *
1165  * This returns 1 if btrfs_run_delalloc_range function did all the work required
1166  * to write the page (copy into inline extent).  In this case the IO has
1167  * been started and the page is already unlocked.
1168  *
1169  * This returns 0 if all went well (page still locked)
1170  * This returns < 0 if there were errors (page still locked)
1171  */
writepage_delalloc(struct btrfs_inode * inode,struct page * page,struct writeback_control * wbc)1172 static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1173 		struct page *page, struct writeback_control *wbc)
1174 {
1175 	const u64 page_start = page_offset(page);
1176 	const u64 page_end = page_start + PAGE_SIZE - 1;
1177 	u64 delalloc_start = page_start;
1178 	u64 delalloc_end = page_end;
1179 	u64 delalloc_to_write = 0;
1180 	int ret = 0;
1181 
1182 	while (delalloc_start < page_end) {
1183 		delalloc_end = page_end;
1184 		if (!find_lock_delalloc_range(&inode->vfs_inode, page,
1185 					      &delalloc_start, &delalloc_end)) {
1186 			delalloc_start = delalloc_end + 1;
1187 			continue;
1188 		}
1189 
1190 		ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
1191 					       delalloc_end, wbc);
1192 		if (ret < 0)
1193 			return ret;
1194 
1195 		delalloc_start = delalloc_end + 1;
1196 	}
1197 
1198 	/*
1199 	 * delalloc_end is already one less than the total length, so
1200 	 * we don't subtract one from PAGE_SIZE
1201 	 */
1202 	delalloc_to_write +=
1203 		DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1204 
1205 	/*
1206 	 * If btrfs_run_dealloc_range() already started I/O and unlocked
1207 	 * the pages, we just need to account for them here.
1208 	 */
1209 	if (ret == 1) {
1210 		wbc->nr_to_write -= delalloc_to_write;
1211 		return 1;
1212 	}
1213 
1214 	if (wbc->nr_to_write < delalloc_to_write) {
1215 		int thresh = 8192;
1216 
1217 		if (delalloc_to_write < thresh * 2)
1218 			thresh = delalloc_to_write;
1219 		wbc->nr_to_write = min_t(u64, delalloc_to_write,
1220 					 thresh);
1221 	}
1222 
1223 	return 0;
1224 }
1225 
1226 /*
1227  * Find the first byte we need to write.
1228  *
1229  * For subpage, one page can contain several sectors, and
1230  * __extent_writepage_io() will just grab all extent maps in the page
1231  * range and try to submit all non-inline/non-compressed extents.
1232  *
1233  * This is a big problem for subpage, we shouldn't re-submit already written
1234  * data at all.
1235  * This function will lookup subpage dirty bit to find which range we really
1236  * need to submit.
1237  *
1238  * Return the next dirty range in [@start, @end).
1239  * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1240  */
find_next_dirty_byte(struct btrfs_fs_info * fs_info,struct page * page,u64 * start,u64 * end)1241 static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1242 				 struct page *page, u64 *start, u64 *end)
1243 {
1244 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1245 	struct btrfs_subpage_info *spi = fs_info->subpage_info;
1246 	u64 orig_start = *start;
1247 	/* Declare as unsigned long so we can use bitmap ops */
1248 	unsigned long flags;
1249 	int range_start_bit;
1250 	int range_end_bit;
1251 
1252 	/*
1253 	 * For regular sector size == page size case, since one page only
1254 	 * contains one sector, we return the page offset directly.
1255 	 */
1256 	if (!btrfs_is_subpage(fs_info, page)) {
1257 		*start = page_offset(page);
1258 		*end = page_offset(page) + PAGE_SIZE;
1259 		return;
1260 	}
1261 
1262 	range_start_bit = spi->dirty_offset +
1263 			  (offset_in_page(orig_start) >> fs_info->sectorsize_bits);
1264 
1265 	/* We should have the page locked, but just in case */
1266 	spin_lock_irqsave(&subpage->lock, flags);
1267 	bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
1268 			       spi->dirty_offset + spi->bitmap_nr_bits);
1269 	spin_unlock_irqrestore(&subpage->lock, flags);
1270 
1271 	range_start_bit -= spi->dirty_offset;
1272 	range_end_bit -= spi->dirty_offset;
1273 
1274 	*start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1275 	*end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1276 }
1277 
1278 /*
1279  * helper for __extent_writepage.  This calls the writepage start hooks,
1280  * and does the loop to map the page into extents and bios.
1281  *
1282  * We return 1 if the IO is started and the page is unlocked,
1283  * 0 if all went well (page still locked)
1284  * < 0 if there were errors (page still locked)
1285  */
__extent_writepage_io(struct btrfs_inode * inode,struct page * page,struct btrfs_bio_ctrl * bio_ctrl,loff_t i_size,int * nr_ret)1286 static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
1287 				 struct page *page,
1288 				 struct btrfs_bio_ctrl *bio_ctrl,
1289 				 loff_t i_size,
1290 				 int *nr_ret)
1291 {
1292 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1293 	u64 cur = page_offset(page);
1294 	u64 end = cur + PAGE_SIZE - 1;
1295 	u64 extent_offset;
1296 	u64 block_start;
1297 	struct extent_map *em;
1298 	int ret = 0;
1299 	int nr = 0;
1300 
1301 	ret = btrfs_writepage_cow_fixup(page);
1302 	if (ret) {
1303 		/* Fixup worker will requeue */
1304 		redirty_page_for_writepage(bio_ctrl->wbc, page);
1305 		unlock_page(page);
1306 		return 1;
1307 	}
1308 
1309 	bio_ctrl->end_io_func = end_bio_extent_writepage;
1310 	while (cur <= end) {
1311 		u32 len = end - cur + 1;
1312 		u64 disk_bytenr;
1313 		u64 em_end;
1314 		u64 dirty_range_start = cur;
1315 		u64 dirty_range_end;
1316 		u32 iosize;
1317 
1318 		if (cur >= i_size) {
1319 			btrfs_mark_ordered_io_finished(inode, page, cur, len,
1320 						       true);
1321 			/*
1322 			 * This range is beyond i_size, thus we don't need to
1323 			 * bother writing back.
1324 			 * But we still need to clear the dirty subpage bit, or
1325 			 * the next time the page gets dirtied, we will try to
1326 			 * writeback the sectors with subpage dirty bits,
1327 			 * causing writeback without ordered extent.
1328 			 */
1329 			btrfs_page_clear_dirty(fs_info, page, cur, len);
1330 			break;
1331 		}
1332 
1333 		find_next_dirty_byte(fs_info, page, &dirty_range_start,
1334 				     &dirty_range_end);
1335 		if (cur < dirty_range_start) {
1336 			cur = dirty_range_start;
1337 			continue;
1338 		}
1339 
1340 		em = btrfs_get_extent(inode, NULL, 0, cur, len);
1341 		if (IS_ERR(em)) {
1342 			ret = PTR_ERR_OR_ZERO(em);
1343 			goto out_error;
1344 		}
1345 
1346 		extent_offset = cur - em->start;
1347 		em_end = extent_map_end(em);
1348 		ASSERT(cur <= em_end);
1349 		ASSERT(cur < end);
1350 		ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
1351 		ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1352 
1353 		block_start = em->block_start;
1354 		disk_bytenr = em->block_start + extent_offset;
1355 
1356 		ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags));
1357 		ASSERT(block_start != EXTENT_MAP_HOLE);
1358 		ASSERT(block_start != EXTENT_MAP_INLINE);
1359 
1360 		/*
1361 		 * Note that em_end from extent_map_end() and dirty_range_end from
1362 		 * find_next_dirty_byte() are all exclusive
1363 		 */
1364 		iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1365 		free_extent_map(em);
1366 		em = NULL;
1367 
1368 		btrfs_set_range_writeback(inode, cur, cur + iosize - 1);
1369 		if (!PageWriteback(page)) {
1370 			btrfs_err(inode->root->fs_info,
1371 				   "page %lu not writeback, cur %llu end %llu",
1372 			       page->index, cur, end);
1373 		}
1374 
1375 		/*
1376 		 * Although the PageDirty bit is cleared before entering this
1377 		 * function, subpage dirty bit is not cleared.
1378 		 * So clear subpage dirty bit here so next time we won't submit
1379 		 * page for range already written to disk.
1380 		 */
1381 		btrfs_page_clear_dirty(fs_info, page, cur, iosize);
1382 
1383 		submit_extent_page(bio_ctrl, disk_bytenr, page, iosize,
1384 				   cur - page_offset(page));
1385 		cur += iosize;
1386 		nr++;
1387 	}
1388 
1389 	btrfs_page_assert_not_dirty(fs_info, page);
1390 	*nr_ret = nr;
1391 	return 0;
1392 
1393 out_error:
1394 	/*
1395 	 * If we finish without problem, we should not only clear page dirty,
1396 	 * but also empty subpage dirty bits
1397 	 */
1398 	*nr_ret = nr;
1399 	return ret;
1400 }
1401 
1402 /*
1403  * the writepage semantics are similar to regular writepage.  extent
1404  * records are inserted to lock ranges in the tree, and as dirty areas
1405  * are found, they are marked writeback.  Then the lock bits are removed
1406  * and the end_io handler clears the writeback ranges
1407  *
1408  * Return 0 if everything goes well.
1409  * Return <0 for error.
1410  */
__extent_writepage(struct page * page,struct btrfs_bio_ctrl * bio_ctrl)1411 static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
1412 {
1413 	struct folio *folio = page_folio(page);
1414 	struct inode *inode = page->mapping->host;
1415 	const u64 page_start = page_offset(page);
1416 	int ret;
1417 	int nr = 0;
1418 	size_t pg_offset;
1419 	loff_t i_size = i_size_read(inode);
1420 	unsigned long end_index = i_size >> PAGE_SHIFT;
1421 
1422 	trace___extent_writepage(page, inode, bio_ctrl->wbc);
1423 
1424 	WARN_ON(!PageLocked(page));
1425 
1426 	pg_offset = offset_in_page(i_size);
1427 	if (page->index > end_index ||
1428 	   (page->index == end_index && !pg_offset)) {
1429 		folio_invalidate(folio, 0, folio_size(folio));
1430 		folio_unlock(folio);
1431 		return 0;
1432 	}
1433 
1434 	if (page->index == end_index)
1435 		memzero_page(page, pg_offset, PAGE_SIZE - pg_offset);
1436 
1437 	ret = set_page_extent_mapped(page);
1438 	if (ret < 0)
1439 		goto done;
1440 
1441 	ret = writepage_delalloc(BTRFS_I(inode), page, bio_ctrl->wbc);
1442 	if (ret == 1)
1443 		return 0;
1444 	if (ret)
1445 		goto done;
1446 
1447 	ret = __extent_writepage_io(BTRFS_I(inode), page, bio_ctrl, i_size, &nr);
1448 	if (ret == 1)
1449 		return 0;
1450 
1451 	bio_ctrl->wbc->nr_to_write--;
1452 
1453 done:
1454 	if (nr == 0) {
1455 		/* make sure the mapping tag for page dirty gets cleared */
1456 		set_page_writeback(page);
1457 		end_page_writeback(page);
1458 	}
1459 	if (ret) {
1460 		btrfs_mark_ordered_io_finished(BTRFS_I(inode), page, page_start,
1461 					       PAGE_SIZE, !ret);
1462 		mapping_set_error(page->mapping, ret);
1463 	}
1464 	unlock_page(page);
1465 	ASSERT(ret <= 0);
1466 	return ret;
1467 }
1468 
wait_on_extent_buffer_writeback(struct extent_buffer * eb)1469 void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1470 {
1471 	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
1472 		       TASK_UNINTERRUPTIBLE);
1473 }
1474 
1475 /*
1476  * Lock extent buffer status and pages for writeback.
1477  *
1478  * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1479  * extent buffer is not dirty)
1480  * Return %true is the extent buffer is submitted to bio.
1481  */
lock_extent_buffer_for_io(struct extent_buffer * eb,struct writeback_control * wbc)1482 static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1483 			  struct writeback_control *wbc)
1484 {
1485 	struct btrfs_fs_info *fs_info = eb->fs_info;
1486 	bool ret = false;
1487 
1488 	btrfs_tree_lock(eb);
1489 	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1490 		btrfs_tree_unlock(eb);
1491 		if (wbc->sync_mode != WB_SYNC_ALL)
1492 			return false;
1493 		wait_on_extent_buffer_writeback(eb);
1494 		btrfs_tree_lock(eb);
1495 	}
1496 
1497 	/*
1498 	 * We need to do this to prevent races in people who check if the eb is
1499 	 * under IO since we can end up having no IO bits set for a short period
1500 	 * of time.
1501 	 */
1502 	spin_lock(&eb->refs_lock);
1503 	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1504 		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1505 		spin_unlock(&eb->refs_lock);
1506 		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1507 		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1508 					 -eb->len,
1509 					 fs_info->dirty_metadata_batch);
1510 		ret = true;
1511 	} else {
1512 		spin_unlock(&eb->refs_lock);
1513 	}
1514 	btrfs_tree_unlock(eb);
1515 	return ret;
1516 }
1517 
set_btree_ioerr(struct extent_buffer * eb)1518 static void set_btree_ioerr(struct extent_buffer *eb)
1519 {
1520 	struct btrfs_fs_info *fs_info = eb->fs_info;
1521 
1522 	set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1523 
1524 	/*
1525 	 * A read may stumble upon this buffer later, make sure that it gets an
1526 	 * error and knows there was an error.
1527 	 */
1528 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1529 
1530 	/*
1531 	 * We need to set the mapping with the io error as well because a write
1532 	 * error will flip the file system readonly, and then syncfs() will
1533 	 * return a 0 because we are readonly if we don't modify the err seq for
1534 	 * the superblock.
1535 	 */
1536 	mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1537 
1538 	/*
1539 	 * If writeback for a btree extent that doesn't belong to a log tree
1540 	 * failed, increment the counter transaction->eb_write_errors.
1541 	 * We do this because while the transaction is running and before it's
1542 	 * committing (when we call filemap_fdata[write|wait]_range against
1543 	 * the btree inode), we might have
1544 	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1545 	 * returns an error or an error happens during writeback, when we're
1546 	 * committing the transaction we wouldn't know about it, since the pages
1547 	 * can be no longer dirty nor marked anymore for writeback (if a
1548 	 * subsequent modification to the extent buffer didn't happen before the
1549 	 * transaction commit), which makes filemap_fdata[write|wait]_range not
1550 	 * able to find the pages tagged with SetPageError at transaction
1551 	 * commit time. So if this happens we must abort the transaction,
1552 	 * otherwise we commit a super block with btree roots that point to
1553 	 * btree nodes/leafs whose content on disk is invalid - either garbage
1554 	 * or the content of some node/leaf from a past generation that got
1555 	 * cowed or deleted and is no longer valid.
1556 	 *
1557 	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1558 	 * not be enough - we need to distinguish between log tree extents vs
1559 	 * non-log tree extents, and the next filemap_fdatawait_range() call
1560 	 * will catch and clear such errors in the mapping - and that call might
1561 	 * be from a log sync and not from a transaction commit. Also, checking
1562 	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1563 	 * not done and would not be reliable - the eb might have been released
1564 	 * from memory and reading it back again means that flag would not be
1565 	 * set (since it's a runtime flag, not persisted on disk).
1566 	 *
1567 	 * Using the flags below in the btree inode also makes us achieve the
1568 	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1569 	 * writeback for all dirty pages and before filemap_fdatawait_range()
1570 	 * is called, the writeback for all dirty pages had already finished
1571 	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
1572 	 * filemap_fdatawait_range() would return success, as it could not know
1573 	 * that writeback errors happened (the pages were no longer tagged for
1574 	 * writeback).
1575 	 */
1576 	switch (eb->log_index) {
1577 	case -1:
1578 		set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1579 		break;
1580 	case 0:
1581 		set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1582 		break;
1583 	case 1:
1584 		set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1585 		break;
1586 	default:
1587 		BUG(); /* unexpected, logic error */
1588 	}
1589 }
1590 
1591 /*
1592  * The endio specific version which won't touch any unsafe spinlock in endio
1593  * context.
1594  */
find_extent_buffer_nolock(struct btrfs_fs_info * fs_info,u64 start)1595 static struct extent_buffer *find_extent_buffer_nolock(
1596 		struct btrfs_fs_info *fs_info, u64 start)
1597 {
1598 	struct extent_buffer *eb;
1599 
1600 	rcu_read_lock();
1601 	eb = radix_tree_lookup(&fs_info->buffer_radix,
1602 			       start >> fs_info->sectorsize_bits);
1603 	if (eb && atomic_inc_not_zero(&eb->refs)) {
1604 		rcu_read_unlock();
1605 		return eb;
1606 	}
1607 	rcu_read_unlock();
1608 	return NULL;
1609 }
1610 
extent_buffer_write_end_io(struct btrfs_bio * bbio)1611 static void extent_buffer_write_end_io(struct btrfs_bio *bbio)
1612 {
1613 	struct extent_buffer *eb = bbio->private;
1614 	struct btrfs_fs_info *fs_info = eb->fs_info;
1615 	bool uptodate = !bbio->bio.bi_status;
1616 	struct bvec_iter_all iter_all;
1617 	struct bio_vec *bvec;
1618 	u32 bio_offset = 0;
1619 
1620 	if (!uptodate)
1621 		set_btree_ioerr(eb);
1622 
1623 	bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
1624 		u64 start = eb->start + bio_offset;
1625 		struct page *page = bvec->bv_page;
1626 		u32 len = bvec->bv_len;
1627 
1628 		btrfs_page_clear_writeback(fs_info, page, start, len);
1629 		bio_offset += len;
1630 	}
1631 
1632 	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1633 	smp_mb__after_atomic();
1634 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
1635 
1636 	bio_put(&bbio->bio);
1637 }
1638 
prepare_eb_write(struct extent_buffer * eb)1639 static void prepare_eb_write(struct extent_buffer *eb)
1640 {
1641 	u32 nritems;
1642 	unsigned long start;
1643 	unsigned long end;
1644 
1645 	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1646 
1647 	/* Set btree blocks beyond nritems with 0 to avoid stale content */
1648 	nritems = btrfs_header_nritems(eb);
1649 	if (btrfs_header_level(eb) > 0) {
1650 		end = btrfs_node_key_ptr_offset(eb, nritems);
1651 		memzero_extent_buffer(eb, end, eb->len - end);
1652 	} else {
1653 		/*
1654 		 * Leaf:
1655 		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1656 		 */
1657 		start = btrfs_item_nr_offset(eb, nritems);
1658 		end = btrfs_item_nr_offset(eb, 0);
1659 		if (nritems == 0)
1660 			end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
1661 		else
1662 			end += btrfs_item_offset(eb, nritems - 1);
1663 		memzero_extent_buffer(eb, start, end - start);
1664 	}
1665 }
1666 
write_one_eb(struct extent_buffer * eb,struct writeback_control * wbc)1667 static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1668 					    struct writeback_control *wbc)
1669 {
1670 	struct btrfs_fs_info *fs_info = eb->fs_info;
1671 	struct btrfs_bio *bbio;
1672 
1673 	prepare_eb_write(eb);
1674 
1675 	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1676 			       REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1677 			       eb->fs_info, extent_buffer_write_end_io, eb);
1678 	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1679 	bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
1680 	wbc_init_bio(wbc, &bbio->bio);
1681 	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
1682 	bbio->file_offset = eb->start;
1683 	if (fs_info->nodesize < PAGE_SIZE) {
1684 		struct page *p = eb->pages[0];
1685 
1686 		lock_page(p);
1687 		btrfs_subpage_set_writeback(fs_info, p, eb->start, eb->len);
1688 		if (btrfs_subpage_clear_and_test_dirty(fs_info, p, eb->start,
1689 						       eb->len)) {
1690 			clear_page_dirty_for_io(p);
1691 			wbc->nr_to_write--;
1692 		}
1693 		__bio_add_page(&bbio->bio, p, eb->len, eb->start - page_offset(p));
1694 		wbc_account_cgroup_owner(wbc, p, eb->len);
1695 		unlock_page(p);
1696 	} else {
1697 		for (int i = 0; i < num_extent_pages(eb); i++) {
1698 			struct page *p = eb->pages[i];
1699 
1700 			lock_page(p);
1701 			clear_page_dirty_for_io(p);
1702 			set_page_writeback(p);
1703 			__bio_add_page(&bbio->bio, p, PAGE_SIZE, 0);
1704 			wbc_account_cgroup_owner(wbc, p, PAGE_SIZE);
1705 			wbc->nr_to_write--;
1706 			unlock_page(p);
1707 		}
1708 	}
1709 	btrfs_submit_bio(bbio, 0);
1710 }
1711 
1712 /*
1713  * Submit one subpage btree page.
1714  *
1715  * The main difference to submit_eb_page() is:
1716  * - Page locking
1717  *   For subpage, we don't rely on page locking at all.
1718  *
1719  * - Flush write bio
1720  *   We only flush bio if we may be unable to fit current extent buffers into
1721  *   current bio.
1722  *
1723  * Return >=0 for the number of submitted extent buffers.
1724  * Return <0 for fatal error.
1725  */
submit_eb_subpage(struct page * page,struct writeback_control * wbc)1726 static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1727 {
1728 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
1729 	int submitted = 0;
1730 	u64 page_start = page_offset(page);
1731 	int bit_start = 0;
1732 	int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1733 
1734 	/* Lock and write each dirty extent buffers in the range */
1735 	while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1736 		struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1737 		struct extent_buffer *eb;
1738 		unsigned long flags;
1739 		u64 start;
1740 
1741 		/*
1742 		 * Take private lock to ensure the subpage won't be detached
1743 		 * in the meantime.
1744 		 */
1745 		spin_lock(&page->mapping->private_lock);
1746 		if (!PagePrivate(page)) {
1747 			spin_unlock(&page->mapping->private_lock);
1748 			break;
1749 		}
1750 		spin_lock_irqsave(&subpage->lock, flags);
1751 		if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
1752 			      subpage->bitmaps)) {
1753 			spin_unlock_irqrestore(&subpage->lock, flags);
1754 			spin_unlock(&page->mapping->private_lock);
1755 			bit_start++;
1756 			continue;
1757 		}
1758 
1759 		start = page_start + bit_start * fs_info->sectorsize;
1760 		bit_start += sectors_per_node;
1761 
1762 		/*
1763 		 * Here we just want to grab the eb without touching extra
1764 		 * spin locks, so call find_extent_buffer_nolock().
1765 		 */
1766 		eb = find_extent_buffer_nolock(fs_info, start);
1767 		spin_unlock_irqrestore(&subpage->lock, flags);
1768 		spin_unlock(&page->mapping->private_lock);
1769 
1770 		/*
1771 		 * The eb has already reached 0 refs thus find_extent_buffer()
1772 		 * doesn't return it. We don't need to write back such eb
1773 		 * anyway.
1774 		 */
1775 		if (!eb)
1776 			continue;
1777 
1778 		if (lock_extent_buffer_for_io(eb, wbc)) {
1779 			write_one_eb(eb, wbc);
1780 			submitted++;
1781 		}
1782 		free_extent_buffer(eb);
1783 	}
1784 	return submitted;
1785 }
1786 
1787 /*
1788  * Submit all page(s) of one extent buffer.
1789  *
1790  * @page:	the page of one extent buffer
1791  * @eb_context:	to determine if we need to submit this page, if current page
1792  *		belongs to this eb, we don't need to submit
1793  *
1794  * The caller should pass each page in their bytenr order, and here we use
1795  * @eb_context to determine if we have submitted pages of one extent buffer.
1796  *
1797  * If we have, we just skip until we hit a new page that doesn't belong to
1798  * current @eb_context.
1799  *
1800  * If not, we submit all the page(s) of the extent buffer.
1801  *
1802  * Return >0 if we have submitted the extent buffer successfully.
1803  * Return 0 if we don't need to submit the page, as it's already submitted by
1804  * previous call.
1805  * Return <0 for fatal error.
1806  */
submit_eb_page(struct page * page,struct btrfs_eb_write_context * ctx)1807 static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
1808 {
1809 	struct writeback_control *wbc = ctx->wbc;
1810 	struct address_space *mapping = page->mapping;
1811 	struct extent_buffer *eb;
1812 	int ret;
1813 
1814 	if (!PagePrivate(page))
1815 		return 0;
1816 
1817 	if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
1818 		return submit_eb_subpage(page, wbc);
1819 
1820 	spin_lock(&mapping->private_lock);
1821 	if (!PagePrivate(page)) {
1822 		spin_unlock(&mapping->private_lock);
1823 		return 0;
1824 	}
1825 
1826 	eb = (struct extent_buffer *)page->private;
1827 
1828 	/*
1829 	 * Shouldn't happen and normally this would be a BUG_ON but no point
1830 	 * crashing the machine for something we can survive anyway.
1831 	 */
1832 	if (WARN_ON(!eb)) {
1833 		spin_unlock(&mapping->private_lock);
1834 		return 0;
1835 	}
1836 
1837 	if (eb == ctx->eb) {
1838 		spin_unlock(&mapping->private_lock);
1839 		return 0;
1840 	}
1841 	ret = atomic_inc_not_zero(&eb->refs);
1842 	spin_unlock(&mapping->private_lock);
1843 	if (!ret)
1844 		return 0;
1845 
1846 	ctx->eb = eb;
1847 
1848 	ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx);
1849 	if (ret) {
1850 		if (ret == -EBUSY)
1851 			ret = 0;
1852 		free_extent_buffer(eb);
1853 		return ret;
1854 	}
1855 
1856 	if (!lock_extent_buffer_for_io(eb, wbc)) {
1857 		free_extent_buffer(eb);
1858 		return 0;
1859 	}
1860 	/* Implies write in zoned mode. */
1861 	if (ctx->zoned_bg) {
1862 		/* Mark the last eb in the block group. */
1863 		btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb);
1864 		ctx->zoned_bg->meta_write_pointer += eb->len;
1865 	}
1866 	write_one_eb(eb, wbc);
1867 	free_extent_buffer(eb);
1868 	return 1;
1869 }
1870 
btree_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc)1871 int btree_write_cache_pages(struct address_space *mapping,
1872 				   struct writeback_control *wbc)
1873 {
1874 	struct btrfs_eb_write_context ctx = { .wbc = wbc };
1875 	struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
1876 	int ret = 0;
1877 	int done = 0;
1878 	int nr_to_write_done = 0;
1879 	struct folio_batch fbatch;
1880 	unsigned int nr_folios;
1881 	pgoff_t index;
1882 	pgoff_t end;		/* Inclusive */
1883 	int scanned = 0;
1884 	xa_mark_t tag;
1885 
1886 	folio_batch_init(&fbatch);
1887 	if (wbc->range_cyclic) {
1888 		index = mapping->writeback_index; /* Start from prev offset */
1889 		end = -1;
1890 		/*
1891 		 * Start from the beginning does not need to cycle over the
1892 		 * range, mark it as scanned.
1893 		 */
1894 		scanned = (index == 0);
1895 	} else {
1896 		index = wbc->range_start >> PAGE_SHIFT;
1897 		end = wbc->range_end >> PAGE_SHIFT;
1898 		scanned = 1;
1899 	}
1900 	if (wbc->sync_mode == WB_SYNC_ALL)
1901 		tag = PAGECACHE_TAG_TOWRITE;
1902 	else
1903 		tag = PAGECACHE_TAG_DIRTY;
1904 	btrfs_zoned_meta_io_lock(fs_info);
1905 retry:
1906 	if (wbc->sync_mode == WB_SYNC_ALL)
1907 		tag_pages_for_writeback(mapping, index, end);
1908 	while (!done && !nr_to_write_done && (index <= end) &&
1909 	       (nr_folios = filemap_get_folios_tag(mapping, &index, end,
1910 					    tag, &fbatch))) {
1911 		unsigned i;
1912 
1913 		for (i = 0; i < nr_folios; i++) {
1914 			struct folio *folio = fbatch.folios[i];
1915 
1916 			ret = submit_eb_page(&folio->page, &ctx);
1917 			if (ret == 0)
1918 				continue;
1919 			if (ret < 0) {
1920 				done = 1;
1921 				break;
1922 			}
1923 
1924 			/*
1925 			 * the filesystem may choose to bump up nr_to_write.
1926 			 * We have to make sure to honor the new nr_to_write
1927 			 * at any time
1928 			 */
1929 			nr_to_write_done = wbc->nr_to_write <= 0;
1930 		}
1931 		folio_batch_release(&fbatch);
1932 		cond_resched();
1933 	}
1934 	if (!scanned && !done) {
1935 		/*
1936 		 * We hit the last page and there is more work to be done: wrap
1937 		 * back to the start of the file
1938 		 */
1939 		scanned = 1;
1940 		index = 0;
1941 		goto retry;
1942 	}
1943 	/*
1944 	 * If something went wrong, don't allow any metadata write bio to be
1945 	 * submitted.
1946 	 *
1947 	 * This would prevent use-after-free if we had dirty pages not
1948 	 * cleaned up, which can still happen by fuzzed images.
1949 	 *
1950 	 * - Bad extent tree
1951 	 *   Allowing existing tree block to be allocated for other trees.
1952 	 *
1953 	 * - Log tree operations
1954 	 *   Exiting tree blocks get allocated to log tree, bumps its
1955 	 *   generation, then get cleaned in tree re-balance.
1956 	 *   Such tree block will not be written back, since it's clean,
1957 	 *   thus no WRITTEN flag set.
1958 	 *   And after log writes back, this tree block is not traced by
1959 	 *   any dirty extent_io_tree.
1960 	 *
1961 	 * - Offending tree block gets re-dirtied from its original owner
1962 	 *   Since it has bumped generation, no WRITTEN flag, it can be
1963 	 *   reused without COWing. This tree block will not be traced
1964 	 *   by btrfs_transaction::dirty_pages.
1965 	 *
1966 	 *   Now such dirty tree block will not be cleaned by any dirty
1967 	 *   extent io tree. Thus we don't want to submit such wild eb
1968 	 *   if the fs already has error.
1969 	 *
1970 	 * We can get ret > 0 from submit_extent_page() indicating how many ebs
1971 	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
1972 	 */
1973 	if (ret > 0)
1974 		ret = 0;
1975 	if (!ret && BTRFS_FS_ERROR(fs_info))
1976 		ret = -EROFS;
1977 
1978 	if (ctx.zoned_bg)
1979 		btrfs_put_block_group(ctx.zoned_bg);
1980 	btrfs_zoned_meta_io_unlock(fs_info);
1981 	return ret;
1982 }
1983 
1984 /*
1985  * Walk the list of dirty pages of the given address space and write all of them.
1986  *
1987  * @mapping:   address space structure to write
1988  * @wbc:       subtract the number of written pages from *@wbc->nr_to_write
1989  * @bio_ctrl:  holds context for the write, namely the bio
1990  *
1991  * If a page is already under I/O, write_cache_pages() skips it, even
1992  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
1993  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
1994  * and msync() need to guarantee that all the data which was dirty at the time
1995  * the call was made get new I/O started against them.  If wbc->sync_mode is
1996  * WB_SYNC_ALL then we were called for data integrity and we must wait for
1997  * existing IO to complete.
1998  */
extent_write_cache_pages(struct address_space * mapping,struct btrfs_bio_ctrl * bio_ctrl)1999 static int extent_write_cache_pages(struct address_space *mapping,
2000 			     struct btrfs_bio_ctrl *bio_ctrl)
2001 {
2002 	struct writeback_control *wbc = bio_ctrl->wbc;
2003 	struct inode *inode = mapping->host;
2004 	int ret = 0;
2005 	int done = 0;
2006 	int nr_to_write_done = 0;
2007 	struct folio_batch fbatch;
2008 	unsigned int nr_folios;
2009 	pgoff_t index;
2010 	pgoff_t end;		/* Inclusive */
2011 	pgoff_t done_index;
2012 	int range_whole = 0;
2013 	int scanned = 0;
2014 	xa_mark_t tag;
2015 
2016 	/*
2017 	 * We have to hold onto the inode so that ordered extents can do their
2018 	 * work when the IO finishes.  The alternative to this is failing to add
2019 	 * an ordered extent if the igrab() fails there and that is a huge pain
2020 	 * to deal with, so instead just hold onto the inode throughout the
2021 	 * writepages operation.  If it fails here we are freeing up the inode
2022 	 * anyway and we'd rather not waste our time writing out stuff that is
2023 	 * going to be truncated anyway.
2024 	 */
2025 	if (!igrab(inode))
2026 		return 0;
2027 
2028 	folio_batch_init(&fbatch);
2029 	if (wbc->range_cyclic) {
2030 		index = mapping->writeback_index; /* Start from prev offset */
2031 		end = -1;
2032 		/*
2033 		 * Start from the beginning does not need to cycle over the
2034 		 * range, mark it as scanned.
2035 		 */
2036 		scanned = (index == 0);
2037 	} else {
2038 		index = wbc->range_start >> PAGE_SHIFT;
2039 		end = wbc->range_end >> PAGE_SHIFT;
2040 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2041 			range_whole = 1;
2042 		scanned = 1;
2043 	}
2044 
2045 	/*
2046 	 * We do the tagged writepage as long as the snapshot flush bit is set
2047 	 * and we are the first one who do the filemap_flush() on this inode.
2048 	 *
2049 	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2050 	 * not race in and drop the bit.
2051 	 */
2052 	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2053 	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2054 			       &BTRFS_I(inode)->runtime_flags))
2055 		wbc->tagged_writepages = 1;
2056 
2057 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2058 		tag = PAGECACHE_TAG_TOWRITE;
2059 	else
2060 		tag = PAGECACHE_TAG_DIRTY;
2061 retry:
2062 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2063 		tag_pages_for_writeback(mapping, index, end);
2064 	done_index = index;
2065 	while (!done && !nr_to_write_done && (index <= end) &&
2066 			(nr_folios = filemap_get_folios_tag(mapping, &index,
2067 							end, tag, &fbatch))) {
2068 		unsigned i;
2069 
2070 		for (i = 0; i < nr_folios; i++) {
2071 			struct folio *folio = fbatch.folios[i];
2072 
2073 			done_index = folio_next_index(folio);
2074 			/*
2075 			 * At this point we hold neither the i_pages lock nor
2076 			 * the page lock: the page may be truncated or
2077 			 * invalidated (changing page->mapping to NULL),
2078 			 * or even swizzled back from swapper_space to
2079 			 * tmpfs file mapping
2080 			 */
2081 			if (!folio_trylock(folio)) {
2082 				submit_write_bio(bio_ctrl, 0);
2083 				folio_lock(folio);
2084 			}
2085 
2086 			if (unlikely(folio->mapping != mapping)) {
2087 				folio_unlock(folio);
2088 				continue;
2089 			}
2090 
2091 			if (!folio_test_dirty(folio)) {
2092 				/* Someone wrote it for us. */
2093 				folio_unlock(folio);
2094 				continue;
2095 			}
2096 
2097 			if (wbc->sync_mode != WB_SYNC_NONE) {
2098 				if (folio_test_writeback(folio))
2099 					submit_write_bio(bio_ctrl, 0);
2100 				folio_wait_writeback(folio);
2101 			}
2102 
2103 			if (folio_test_writeback(folio) ||
2104 			    !folio_clear_dirty_for_io(folio)) {
2105 				folio_unlock(folio);
2106 				continue;
2107 			}
2108 
2109 			ret = __extent_writepage(&folio->page, bio_ctrl);
2110 			if (ret < 0) {
2111 				done = 1;
2112 				break;
2113 			}
2114 
2115 			/*
2116 			 * The filesystem may choose to bump up nr_to_write.
2117 			 * We have to make sure to honor the new nr_to_write
2118 			 * at any time.
2119 			 */
2120 			nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2121 					    wbc->nr_to_write <= 0);
2122 		}
2123 		folio_batch_release(&fbatch);
2124 		cond_resched();
2125 	}
2126 	if (!scanned && !done) {
2127 		/*
2128 		 * We hit the last page and there is more work to be done: wrap
2129 		 * back to the start of the file
2130 		 */
2131 		scanned = 1;
2132 		index = 0;
2133 
2134 		/*
2135 		 * If we're looping we could run into a page that is locked by a
2136 		 * writer and that writer could be waiting on writeback for a
2137 		 * page in our current bio, and thus deadlock, so flush the
2138 		 * write bio here.
2139 		 */
2140 		submit_write_bio(bio_ctrl, 0);
2141 		goto retry;
2142 	}
2143 
2144 	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2145 		mapping->writeback_index = done_index;
2146 
2147 	btrfs_add_delayed_iput(BTRFS_I(inode));
2148 	return ret;
2149 }
2150 
2151 /*
2152  * Submit the pages in the range to bio for call sites which delalloc range has
2153  * already been ran (aka, ordered extent inserted) and all pages are still
2154  * locked.
2155  */
extent_write_locked_range(struct inode * inode,struct page * locked_page,u64 start,u64 end,struct writeback_control * wbc,bool pages_dirty)2156 void extent_write_locked_range(struct inode *inode, struct page *locked_page,
2157 			       u64 start, u64 end, struct writeback_control *wbc,
2158 			       bool pages_dirty)
2159 {
2160 	bool found_error = false;
2161 	int ret = 0;
2162 	struct address_space *mapping = inode->i_mapping;
2163 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2164 	const u32 sectorsize = fs_info->sectorsize;
2165 	loff_t i_size = i_size_read(inode);
2166 	u64 cur = start;
2167 	struct btrfs_bio_ctrl bio_ctrl = {
2168 		.wbc = wbc,
2169 		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2170 	};
2171 
2172 	if (wbc->no_cgroup_owner)
2173 		bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2174 
2175 	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2176 
2177 	while (cur <= end) {
2178 		u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2179 		u32 cur_len = cur_end + 1 - cur;
2180 		struct page *page;
2181 		int nr = 0;
2182 
2183 		page = find_get_page(mapping, cur >> PAGE_SHIFT);
2184 		ASSERT(PageLocked(page));
2185 		if (pages_dirty && page != locked_page) {
2186 			ASSERT(PageDirty(page));
2187 			clear_page_dirty_for_io(page);
2188 		}
2189 
2190 		ret = __extent_writepage_io(BTRFS_I(inode), page, &bio_ctrl,
2191 					    i_size, &nr);
2192 		if (ret == 1)
2193 			goto next_page;
2194 
2195 		/* Make sure the mapping tag for page dirty gets cleared. */
2196 		if (nr == 0) {
2197 			set_page_writeback(page);
2198 			end_page_writeback(page);
2199 		}
2200 		if (ret) {
2201 			btrfs_mark_ordered_io_finished(BTRFS_I(inode), page,
2202 						       cur, cur_len, !ret);
2203 			mapping_set_error(page->mapping, ret);
2204 		}
2205 		btrfs_page_unlock_writer(fs_info, page, cur, cur_len);
2206 		if (ret < 0)
2207 			found_error = true;
2208 next_page:
2209 		put_page(page);
2210 		cur = cur_end + 1;
2211 	}
2212 
2213 	submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2214 }
2215 
extent_writepages(struct address_space * mapping,struct writeback_control * wbc)2216 int extent_writepages(struct address_space *mapping,
2217 		      struct writeback_control *wbc)
2218 {
2219 	struct inode *inode = mapping->host;
2220 	int ret = 0;
2221 	struct btrfs_bio_ctrl bio_ctrl = {
2222 		.wbc = wbc,
2223 		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2224 	};
2225 
2226 	/*
2227 	 * Allow only a single thread to do the reloc work in zoned mode to
2228 	 * protect the write pointer updates.
2229 	 */
2230 	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2231 	ret = extent_write_cache_pages(mapping, &bio_ctrl);
2232 	submit_write_bio(&bio_ctrl, ret);
2233 	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2234 	return ret;
2235 }
2236 
extent_readahead(struct readahead_control * rac)2237 void extent_readahead(struct readahead_control *rac)
2238 {
2239 	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2240 	struct page *pagepool[16];
2241 	struct extent_map *em_cached = NULL;
2242 	u64 prev_em_start = (u64)-1;
2243 	int nr;
2244 
2245 	while ((nr = readahead_page_batch(rac, pagepool))) {
2246 		u64 contig_start = readahead_pos(rac);
2247 		u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2248 
2249 		contiguous_readpages(pagepool, nr, contig_start, contig_end,
2250 				&em_cached, &bio_ctrl, &prev_em_start);
2251 	}
2252 
2253 	if (em_cached)
2254 		free_extent_map(em_cached);
2255 	submit_one_bio(&bio_ctrl);
2256 }
2257 
2258 /*
2259  * basic invalidate_folio code, this waits on any locked or writeback
2260  * ranges corresponding to the folio, and then deletes any extent state
2261  * records from the tree
2262  */
extent_invalidate_folio(struct extent_io_tree * tree,struct folio * folio,size_t offset)2263 int extent_invalidate_folio(struct extent_io_tree *tree,
2264 			  struct folio *folio, size_t offset)
2265 {
2266 	struct extent_state *cached_state = NULL;
2267 	u64 start = folio_pos(folio);
2268 	u64 end = start + folio_size(folio) - 1;
2269 	size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
2270 
2271 	/* This function is only called for the btree inode */
2272 	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2273 
2274 	start += ALIGN(offset, blocksize);
2275 	if (start > end)
2276 		return 0;
2277 
2278 	lock_extent(tree, start, end, &cached_state);
2279 	folio_wait_writeback(folio);
2280 
2281 	/*
2282 	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2283 	 * so here we only need to unlock the extent range to free any
2284 	 * existing extent state.
2285 	 */
2286 	unlock_extent(tree, start, end, &cached_state);
2287 	return 0;
2288 }
2289 
2290 /*
2291  * a helper for release_folio, this tests for areas of the page that
2292  * are locked or under IO and drops the related state bits if it is safe
2293  * to drop the page.
2294  */
try_release_extent_state(struct extent_io_tree * tree,struct page * page,gfp_t mask)2295 static int try_release_extent_state(struct extent_io_tree *tree,
2296 				    struct page *page, gfp_t mask)
2297 {
2298 	u64 start = page_offset(page);
2299 	u64 end = start + PAGE_SIZE - 1;
2300 	int ret = 1;
2301 
2302 	if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
2303 		ret = 0;
2304 	} else {
2305 		u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2306 				   EXTENT_DELALLOC_NEW | EXTENT_CTLBITS |
2307 				   EXTENT_QGROUP_RESERVED);
2308 
2309 		/*
2310 		 * At this point we can safely clear everything except the
2311 		 * locked bit, the nodatasum bit and the delalloc new bit.
2312 		 * The delalloc new bit will be cleared by ordered extent
2313 		 * completion.
2314 		 */
2315 		ret = __clear_extent_bit(tree, start, end, clear_bits, NULL, NULL);
2316 
2317 		/* if clear_extent_bit failed for enomem reasons,
2318 		 * we can't allow the release to continue.
2319 		 */
2320 		if (ret < 0)
2321 			ret = 0;
2322 		else
2323 			ret = 1;
2324 	}
2325 	return ret;
2326 }
2327 
2328 /*
2329  * a helper for release_folio.  As long as there are no locked extents
2330  * in the range corresponding to the page, both state records and extent
2331  * map records are removed
2332  */
try_release_extent_mapping(struct page * page,gfp_t mask)2333 int try_release_extent_mapping(struct page *page, gfp_t mask)
2334 {
2335 	struct extent_map *em;
2336 	u64 start = page_offset(page);
2337 	u64 end = start + PAGE_SIZE - 1;
2338 	struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
2339 	struct extent_io_tree *tree = &btrfs_inode->io_tree;
2340 	struct extent_map_tree *map = &btrfs_inode->extent_tree;
2341 
2342 	if (gfpflags_allow_blocking(mask) &&
2343 	    page->mapping->host->i_size > SZ_16M) {
2344 		u64 len;
2345 		while (start <= end) {
2346 			struct btrfs_fs_info *fs_info;
2347 			u64 cur_gen;
2348 
2349 			len = end - start + 1;
2350 			write_lock(&map->lock);
2351 			em = lookup_extent_mapping(map, start, len);
2352 			if (!em) {
2353 				write_unlock(&map->lock);
2354 				break;
2355 			}
2356 			if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2357 			    em->start != start) {
2358 				write_unlock(&map->lock);
2359 				free_extent_map(em);
2360 				break;
2361 			}
2362 			if (test_range_bit(tree, em->start,
2363 					   extent_map_end(em) - 1,
2364 					   EXTENT_LOCKED, 0, NULL))
2365 				goto next;
2366 			/*
2367 			 * If it's not in the list of modified extents, used
2368 			 * by a fast fsync, we can remove it. If it's being
2369 			 * logged we can safely remove it since fsync took an
2370 			 * extra reference on the em.
2371 			 */
2372 			if (list_empty(&em->list) ||
2373 			    test_bit(EXTENT_FLAG_LOGGING, &em->flags))
2374 				goto remove_em;
2375 			/*
2376 			 * If it's in the list of modified extents, remove it
2377 			 * only if its generation is older then the current one,
2378 			 * in which case we don't need it for a fast fsync.
2379 			 * Otherwise don't remove it, we could be racing with an
2380 			 * ongoing fast fsync that could miss the new extent.
2381 			 */
2382 			fs_info = btrfs_inode->root->fs_info;
2383 			spin_lock(&fs_info->trans_lock);
2384 			cur_gen = fs_info->generation;
2385 			spin_unlock(&fs_info->trans_lock);
2386 			if (em->generation >= cur_gen)
2387 				goto next;
2388 remove_em:
2389 			/*
2390 			 * We only remove extent maps that are not in the list of
2391 			 * modified extents or that are in the list but with a
2392 			 * generation lower then the current generation, so there
2393 			 * is no need to set the full fsync flag on the inode (it
2394 			 * hurts the fsync performance for workloads with a data
2395 			 * size that exceeds or is close to the system's memory).
2396 			 */
2397 			remove_extent_mapping(map, em);
2398 			/* once for the rb tree */
2399 			free_extent_map(em);
2400 next:
2401 			start = extent_map_end(em);
2402 			write_unlock(&map->lock);
2403 
2404 			/* once for us */
2405 			free_extent_map(em);
2406 
2407 			cond_resched(); /* Allow large-extent preemption. */
2408 		}
2409 	}
2410 	return try_release_extent_state(tree, page, mask);
2411 }
2412 
2413 /*
2414  * To cache previous fiemap extent
2415  *
2416  * Will be used for merging fiemap extent
2417  */
2418 struct fiemap_cache {
2419 	u64 offset;
2420 	u64 phys;
2421 	u64 len;
2422 	u32 flags;
2423 	bool cached;
2424 };
2425 
2426 /*
2427  * Helper to submit fiemap extent.
2428  *
2429  * Will try to merge current fiemap extent specified by @offset, @phys,
2430  * @len and @flags with cached one.
2431  * And only when we fails to merge, cached one will be submitted as
2432  * fiemap extent.
2433  *
2434  * Return value is the same as fiemap_fill_next_extent().
2435  */
emit_fiemap_extent(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,u64 offset,u64 phys,u64 len,u32 flags)2436 static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
2437 				struct fiemap_cache *cache,
2438 				u64 offset, u64 phys, u64 len, u32 flags)
2439 {
2440 	u64 cache_end;
2441 	int ret = 0;
2442 
2443 	/* Set at the end of extent_fiemap(). */
2444 	ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2445 
2446 	if (!cache->cached)
2447 		goto assign;
2448 
2449 	/*
2450 	 * When iterating the extents of the inode, at extent_fiemap(), we may
2451 	 * find an extent that starts at an offset behind the end offset of the
2452 	 * previous extent we processed. This happens if fiemap is called
2453 	 * without FIEMAP_FLAG_SYNC and there are ordered extents completing
2454 	 * while we call btrfs_next_leaf() (through fiemap_next_leaf_item()).
2455 	 *
2456 	 * For example we are in leaf X processing its last item, which is the
2457 	 * file extent item for file range [512K, 1M[, and after
2458 	 * btrfs_next_leaf() releases the path, there's an ordered extent that
2459 	 * completes for the file range [768K, 2M[, and that results in trimming
2460 	 * the file extent item so that it now corresponds to the file range
2461 	 * [512K, 768K[ and a new file extent item is inserted for the file
2462 	 * range [768K, 2M[, which may end up as the last item of leaf X or as
2463 	 * the first item of the next leaf - in either case btrfs_next_leaf()
2464 	 * will leave us with a path pointing to the new extent item, for the
2465 	 * file range [768K, 2M[, since that's the first key that follows the
2466 	 * last one we processed. So in order not to report overlapping extents
2467 	 * to user space, we trim the length of the previously cached extent and
2468 	 * emit it.
2469 	 *
2470 	 * Upon calling btrfs_next_leaf() we may also find an extent with an
2471 	 * offset smaller than or equals to cache->offset, and this happens
2472 	 * when we had a hole or prealloc extent with several delalloc ranges in
2473 	 * it, but after btrfs_next_leaf() released the path, delalloc was
2474 	 * flushed and the resulting ordered extents were completed, so we can
2475 	 * now have found a file extent item for an offset that is smaller than
2476 	 * or equals to what we have in cache->offset. We deal with this as
2477 	 * described below.
2478 	 */
2479 	cache_end = cache->offset + cache->len;
2480 	if (cache_end > offset) {
2481 		if (offset == cache->offset) {
2482 			/*
2483 			 * We cached a dealloc range (found in the io tree) for
2484 			 * a hole or prealloc extent and we have now found a
2485 			 * file extent item for the same offset. What we have
2486 			 * now is more recent and up to date, so discard what
2487 			 * we had in the cache and use what we have just found.
2488 			 */
2489 			goto assign;
2490 		} else if (offset > cache->offset) {
2491 			/*
2492 			 * The extent range we previously found ends after the
2493 			 * offset of the file extent item we found and that
2494 			 * offset falls somewhere in the middle of that previous
2495 			 * extent range. So adjust the range we previously found
2496 			 * to end at the offset of the file extent item we have
2497 			 * just found, since this extent is more up to date.
2498 			 * Emit that adjusted range and cache the file extent
2499 			 * item we have just found. This corresponds to the case
2500 			 * where a previously found file extent item was split
2501 			 * due to an ordered extent completing.
2502 			 */
2503 			cache->len = offset - cache->offset;
2504 			goto emit;
2505 		} else {
2506 			const u64 range_end = offset + len;
2507 
2508 			/*
2509 			 * The offset of the file extent item we have just found
2510 			 * is behind the cached offset. This means we were
2511 			 * processing a hole or prealloc extent for which we
2512 			 * have found delalloc ranges (in the io tree), so what
2513 			 * we have in the cache is the last delalloc range we
2514 			 * found while the file extent item we found can be
2515 			 * either for a whole delalloc range we previously
2516 			 * emmitted or only a part of that range.
2517 			 *
2518 			 * We have two cases here:
2519 			 *
2520 			 * 1) The file extent item's range ends at or behind the
2521 			 *    cached extent's end. In this case just ignore the
2522 			 *    current file extent item because we don't want to
2523 			 *    overlap with previous ranges that may have been
2524 			 *    emmitted already;
2525 			 *
2526 			 * 2) The file extent item starts behind the currently
2527 			 *    cached extent but its end offset goes beyond the
2528 			 *    end offset of the cached extent. We don't want to
2529 			 *    overlap with a previous range that may have been
2530 			 *    emmitted already, so we emit the currently cached
2531 			 *    extent and then partially store the current file
2532 			 *    extent item's range in the cache, for the subrange
2533 			 *    going the cached extent's end to the end of the
2534 			 *    file extent item.
2535 			 */
2536 			if (range_end <= cache_end)
2537 				return 0;
2538 
2539 			if (!(flags & (FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_DELALLOC)))
2540 				phys += cache_end - offset;
2541 
2542 			offset = cache_end;
2543 			len = range_end - cache_end;
2544 			goto emit;
2545 		}
2546 	}
2547 
2548 	/*
2549 	 * Only merges fiemap extents if
2550 	 * 1) Their logical addresses are continuous
2551 	 *
2552 	 * 2) Their physical addresses are continuous
2553 	 *    So truly compressed (physical size smaller than logical size)
2554 	 *    extents won't get merged with each other
2555 	 *
2556 	 * 3) Share same flags
2557 	 */
2558 	if (cache->offset + cache->len  == offset &&
2559 	    cache->phys + cache->len == phys  &&
2560 	    cache->flags == flags) {
2561 		cache->len += len;
2562 		return 0;
2563 	}
2564 
2565 emit:
2566 	/* Not mergeable, need to submit cached one */
2567 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2568 				      cache->len, cache->flags);
2569 	cache->cached = false;
2570 	if (ret)
2571 		return ret;
2572 assign:
2573 	cache->cached = true;
2574 	cache->offset = offset;
2575 	cache->phys = phys;
2576 	cache->len = len;
2577 	cache->flags = flags;
2578 
2579 	return 0;
2580 }
2581 
2582 /*
2583  * Emit last fiemap cache
2584  *
2585  * The last fiemap cache may still be cached in the following case:
2586  * 0		      4k		    8k
2587  * |<- Fiemap range ->|
2588  * |<------------  First extent ----------->|
2589  *
2590  * In this case, the first extent range will be cached but not emitted.
2591  * So we must emit it before ending extent_fiemap().
2592  */
emit_last_fiemap_cache(struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache)2593 static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
2594 				  struct fiemap_cache *cache)
2595 {
2596 	int ret;
2597 
2598 	if (!cache->cached)
2599 		return 0;
2600 
2601 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
2602 				      cache->len, cache->flags);
2603 	cache->cached = false;
2604 	if (ret > 0)
2605 		ret = 0;
2606 	return ret;
2607 }
2608 
fiemap_next_leaf_item(struct btrfs_inode * inode,struct btrfs_path * path)2609 static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2610 {
2611 	struct extent_buffer *clone;
2612 	struct btrfs_key key;
2613 	int slot;
2614 	int ret;
2615 
2616 	path->slots[0]++;
2617 	if (path->slots[0] < btrfs_header_nritems(path->nodes[0]))
2618 		return 0;
2619 
2620 	ret = btrfs_next_leaf(inode->root, path);
2621 	if (ret != 0)
2622 		return ret;
2623 
2624 	/*
2625 	 * Don't bother with cloning if there are no more file extent items for
2626 	 * our inode.
2627 	 */
2628 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2629 	if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
2630 		return 1;
2631 
2632 	/* See the comment at fiemap_search_slot() about why we clone. */
2633 	clone = btrfs_clone_extent_buffer(path->nodes[0]);
2634 	if (!clone)
2635 		return -ENOMEM;
2636 
2637 	slot = path->slots[0];
2638 	btrfs_release_path(path);
2639 	path->nodes[0] = clone;
2640 	path->slots[0] = slot;
2641 
2642 	return 0;
2643 }
2644 
2645 /*
2646  * Search for the first file extent item that starts at a given file offset or
2647  * the one that starts immediately before that offset.
2648  * Returns: 0 on success, < 0 on error, 1 if not found.
2649  */
fiemap_search_slot(struct btrfs_inode * inode,struct btrfs_path * path,u64 file_offset)2650 static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2651 			      u64 file_offset)
2652 {
2653 	const u64 ino = btrfs_ino(inode);
2654 	struct btrfs_root *root = inode->root;
2655 	struct extent_buffer *clone;
2656 	struct btrfs_key key;
2657 	int slot;
2658 	int ret;
2659 
2660 	key.objectid = ino;
2661 	key.type = BTRFS_EXTENT_DATA_KEY;
2662 	key.offset = file_offset;
2663 
2664 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2665 	if (ret < 0)
2666 		return ret;
2667 
2668 	if (ret > 0 && path->slots[0] > 0) {
2669 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
2670 		if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2671 			path->slots[0]--;
2672 	}
2673 
2674 	if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2675 		ret = btrfs_next_leaf(root, path);
2676 		if (ret != 0)
2677 			return ret;
2678 
2679 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2680 		if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2681 			return 1;
2682 	}
2683 
2684 	/*
2685 	 * We clone the leaf and use it during fiemap. This is because while
2686 	 * using the leaf we do expensive things like checking if an extent is
2687 	 * shared, which can take a long time. In order to prevent blocking
2688 	 * other tasks for too long, we use a clone of the leaf. We have locked
2689 	 * the file range in the inode's io tree, so we know none of our file
2690 	 * extent items can change. This way we avoid blocking other tasks that
2691 	 * want to insert items for other inodes in the same leaf or b+tree
2692 	 * rebalance operations (triggered for example when someone is trying
2693 	 * to push items into this leaf when trying to insert an item in a
2694 	 * neighbour leaf).
2695 	 * We also need the private clone because holding a read lock on an
2696 	 * extent buffer of the subvolume's b+tree will make lockdep unhappy
2697 	 * when we call fiemap_fill_next_extent(), because that may cause a page
2698 	 * fault when filling the user space buffer with fiemap data.
2699 	 */
2700 	clone = btrfs_clone_extent_buffer(path->nodes[0]);
2701 	if (!clone)
2702 		return -ENOMEM;
2703 
2704 	slot = path->slots[0];
2705 	btrfs_release_path(path);
2706 	path->nodes[0] = clone;
2707 	path->slots[0] = slot;
2708 
2709 	return 0;
2710 }
2711 
2712 /*
2713  * Process a range which is a hole or a prealloc extent in the inode's subvolume
2714  * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2715  * extent. The end offset (@end) is inclusive.
2716  */
fiemap_process_hole(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,struct fiemap_cache * cache,struct extent_state ** delalloc_cached_state,struct btrfs_backref_share_check_ctx * backref_ctx,u64 disk_bytenr,u64 extent_offset,u64 extent_gen,u64 start,u64 end)2717 static int fiemap_process_hole(struct btrfs_inode *inode,
2718 			       struct fiemap_extent_info *fieinfo,
2719 			       struct fiemap_cache *cache,
2720 			       struct extent_state **delalloc_cached_state,
2721 			       struct btrfs_backref_share_check_ctx *backref_ctx,
2722 			       u64 disk_bytenr, u64 extent_offset,
2723 			       u64 extent_gen,
2724 			       u64 start, u64 end)
2725 {
2726 	const u64 i_size = i_size_read(&inode->vfs_inode);
2727 	u64 cur_offset = start;
2728 	u64 last_delalloc_end = 0;
2729 	u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2730 	bool checked_extent_shared = false;
2731 	int ret;
2732 
2733 	/*
2734 	 * There can be no delalloc past i_size, so don't waste time looking for
2735 	 * it beyond i_size.
2736 	 */
2737 	while (cur_offset < end && cur_offset < i_size) {
2738 		u64 delalloc_start;
2739 		u64 delalloc_end;
2740 		u64 prealloc_start;
2741 		u64 prealloc_len = 0;
2742 		bool delalloc;
2743 
2744 		delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end,
2745 							delalloc_cached_state,
2746 							&delalloc_start,
2747 							&delalloc_end);
2748 		if (!delalloc)
2749 			break;
2750 
2751 		/*
2752 		 * If this is a prealloc extent we have to report every section
2753 		 * of it that has no delalloc.
2754 		 */
2755 		if (disk_bytenr != 0) {
2756 			if (last_delalloc_end == 0) {
2757 				prealloc_start = start;
2758 				prealloc_len = delalloc_start - start;
2759 			} else {
2760 				prealloc_start = last_delalloc_end + 1;
2761 				prealloc_len = delalloc_start - prealloc_start;
2762 			}
2763 		}
2764 
2765 		if (prealloc_len > 0) {
2766 			if (!checked_extent_shared && fieinfo->fi_extents_max) {
2767 				ret = btrfs_is_data_extent_shared(inode,
2768 								  disk_bytenr,
2769 								  extent_gen,
2770 								  backref_ctx);
2771 				if (ret < 0)
2772 					return ret;
2773 				else if (ret > 0)
2774 					prealloc_flags |= FIEMAP_EXTENT_SHARED;
2775 
2776 				checked_extent_shared = true;
2777 			}
2778 			ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2779 						 disk_bytenr + extent_offset,
2780 						 prealloc_len, prealloc_flags);
2781 			if (ret)
2782 				return ret;
2783 			extent_offset += prealloc_len;
2784 		}
2785 
2786 		ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0,
2787 					 delalloc_end + 1 - delalloc_start,
2788 					 FIEMAP_EXTENT_DELALLOC |
2789 					 FIEMAP_EXTENT_UNKNOWN);
2790 		if (ret)
2791 			return ret;
2792 
2793 		last_delalloc_end = delalloc_end;
2794 		cur_offset = delalloc_end + 1;
2795 		extent_offset += cur_offset - delalloc_start;
2796 		cond_resched();
2797 	}
2798 
2799 	/*
2800 	 * Either we found no delalloc for the whole prealloc extent or we have
2801 	 * a prealloc extent that spans i_size or starts at or after i_size.
2802 	 */
2803 	if (disk_bytenr != 0 && last_delalloc_end < end) {
2804 		u64 prealloc_start;
2805 		u64 prealloc_len;
2806 
2807 		if (last_delalloc_end == 0) {
2808 			prealloc_start = start;
2809 			prealloc_len = end + 1 - start;
2810 		} else {
2811 			prealloc_start = last_delalloc_end + 1;
2812 			prealloc_len = end + 1 - prealloc_start;
2813 		}
2814 
2815 		if (!checked_extent_shared && fieinfo->fi_extents_max) {
2816 			ret = btrfs_is_data_extent_shared(inode,
2817 							  disk_bytenr,
2818 							  extent_gen,
2819 							  backref_ctx);
2820 			if (ret < 0)
2821 				return ret;
2822 			else if (ret > 0)
2823 				prealloc_flags |= FIEMAP_EXTENT_SHARED;
2824 		}
2825 		ret = emit_fiemap_extent(fieinfo, cache, prealloc_start,
2826 					 disk_bytenr + extent_offset,
2827 					 prealloc_len, prealloc_flags);
2828 		if (ret)
2829 			return ret;
2830 	}
2831 
2832 	return 0;
2833 }
2834 
fiemap_find_last_extent_offset(struct btrfs_inode * inode,struct btrfs_path * path,u64 * last_extent_end_ret)2835 static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2836 					  struct btrfs_path *path,
2837 					  u64 *last_extent_end_ret)
2838 {
2839 	const u64 ino = btrfs_ino(inode);
2840 	struct btrfs_root *root = inode->root;
2841 	struct extent_buffer *leaf;
2842 	struct btrfs_file_extent_item *ei;
2843 	struct btrfs_key key;
2844 	u64 disk_bytenr;
2845 	int ret;
2846 
2847 	/*
2848 	 * Lookup the last file extent. We're not using i_size here because
2849 	 * there might be preallocation past i_size.
2850 	 */
2851 	ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0);
2852 	/* There can't be a file extent item at offset (u64)-1 */
2853 	ASSERT(ret != 0);
2854 	if (ret < 0)
2855 		return ret;
2856 
2857 	/*
2858 	 * For a non-existing key, btrfs_search_slot() always leaves us at a
2859 	 * slot > 0, except if the btree is empty, which is impossible because
2860 	 * at least it has the inode item for this inode and all the items for
2861 	 * the root inode 256.
2862 	 */
2863 	ASSERT(path->slots[0] > 0);
2864 	path->slots[0]--;
2865 	leaf = path->nodes[0];
2866 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2867 	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
2868 		/* No file extent items in the subvolume tree. */
2869 		*last_extent_end_ret = 0;
2870 		return 0;
2871 	}
2872 
2873 	/*
2874 	 * For an inline extent, the disk_bytenr is where inline data starts at,
2875 	 * so first check if we have an inline extent item before checking if we
2876 	 * have an implicit hole (disk_bytenr == 0).
2877 	 */
2878 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
2879 	if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
2880 		*last_extent_end_ret = btrfs_file_extent_end(path);
2881 		return 0;
2882 	}
2883 
2884 	/*
2885 	 * Find the last file extent item that is not a hole (when NO_HOLES is
2886 	 * not enabled). This should take at most 2 iterations in the worst
2887 	 * case: we have one hole file extent item at slot 0 of a leaf and
2888 	 * another hole file extent item as the last item in the previous leaf.
2889 	 * This is because we merge file extent items that represent holes.
2890 	 */
2891 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2892 	while (disk_bytenr == 0) {
2893 		ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY);
2894 		if (ret < 0) {
2895 			return ret;
2896 		} else if (ret > 0) {
2897 			/* No file extent items that are not holes. */
2898 			*last_extent_end_ret = 0;
2899 			return 0;
2900 		}
2901 		leaf = path->nodes[0];
2902 		ei = btrfs_item_ptr(leaf, path->slots[0],
2903 				    struct btrfs_file_extent_item);
2904 		disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
2905 	}
2906 
2907 	*last_extent_end_ret = btrfs_file_extent_end(path);
2908 	return 0;
2909 }
2910 
extent_fiemap(struct btrfs_inode * inode,struct fiemap_extent_info * fieinfo,u64 start,u64 len)2911 int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
2912 		  u64 start, u64 len)
2913 {
2914 	const u64 ino = btrfs_ino(inode);
2915 	struct extent_state *cached_state = NULL;
2916 	struct extent_state *delalloc_cached_state = NULL;
2917 	struct btrfs_path *path;
2918 	struct fiemap_cache cache = { 0 };
2919 	struct btrfs_backref_share_check_ctx *backref_ctx;
2920 	u64 last_extent_end;
2921 	u64 prev_extent_end;
2922 	u64 lockstart;
2923 	u64 lockend;
2924 	bool stopped = false;
2925 	int ret;
2926 
2927 	backref_ctx = btrfs_alloc_backref_share_check_ctx();
2928 	path = btrfs_alloc_path();
2929 	if (!backref_ctx || !path) {
2930 		ret = -ENOMEM;
2931 		goto out;
2932 	}
2933 
2934 	lockstart = round_down(start, inode->root->fs_info->sectorsize);
2935 	lockend = round_up(start + len, inode->root->fs_info->sectorsize);
2936 	prev_extent_end = lockstart;
2937 
2938 	btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
2939 	lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
2940 
2941 	ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end);
2942 	if (ret < 0)
2943 		goto out_unlock;
2944 	btrfs_release_path(path);
2945 
2946 	path->reada = READA_FORWARD;
2947 	ret = fiemap_search_slot(inode, path, lockstart);
2948 	if (ret < 0) {
2949 		goto out_unlock;
2950 	} else if (ret > 0) {
2951 		/*
2952 		 * No file extent item found, but we may have delalloc between
2953 		 * the current offset and i_size. So check for that.
2954 		 */
2955 		ret = 0;
2956 		goto check_eof_delalloc;
2957 	}
2958 
2959 	while (prev_extent_end < lockend) {
2960 		struct extent_buffer *leaf = path->nodes[0];
2961 		struct btrfs_file_extent_item *ei;
2962 		struct btrfs_key key;
2963 		u64 extent_end;
2964 		u64 extent_len;
2965 		u64 extent_offset = 0;
2966 		u64 extent_gen;
2967 		u64 disk_bytenr = 0;
2968 		u64 flags = 0;
2969 		int extent_type;
2970 		u8 compression;
2971 
2972 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2973 		if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2974 			break;
2975 
2976 		extent_end = btrfs_file_extent_end(path);
2977 
2978 		/*
2979 		 * The first iteration can leave us at an extent item that ends
2980 		 * before our range's start. Move to the next item.
2981 		 */
2982 		if (extent_end <= lockstart)
2983 			goto next_item;
2984 
2985 		backref_ctx->curr_leaf_bytenr = leaf->start;
2986 
2987 		/* We have in implicit hole (NO_HOLES feature enabled). */
2988 		if (prev_extent_end < key.offset) {
2989 			const u64 range_end = min(key.offset, lockend) - 1;
2990 
2991 			ret = fiemap_process_hole(inode, fieinfo, &cache,
2992 						  &delalloc_cached_state,
2993 						  backref_ctx, 0, 0, 0,
2994 						  prev_extent_end, range_end);
2995 			if (ret < 0) {
2996 				goto out_unlock;
2997 			} else if (ret > 0) {
2998 				/* fiemap_fill_next_extent() told us to stop. */
2999 				stopped = true;
3000 				break;
3001 			}
3002 
3003 			/* We've reached the end of the fiemap range, stop. */
3004 			if (key.offset >= lockend) {
3005 				stopped = true;
3006 				break;
3007 			}
3008 		}
3009 
3010 		extent_len = extent_end - key.offset;
3011 		ei = btrfs_item_ptr(leaf, path->slots[0],
3012 				    struct btrfs_file_extent_item);
3013 		compression = btrfs_file_extent_compression(leaf, ei);
3014 		extent_type = btrfs_file_extent_type(leaf, ei);
3015 		extent_gen = btrfs_file_extent_generation(leaf, ei);
3016 
3017 		if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
3018 			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
3019 			if (compression == BTRFS_COMPRESS_NONE)
3020 				extent_offset = btrfs_file_extent_offset(leaf, ei);
3021 		}
3022 
3023 		if (compression != BTRFS_COMPRESS_NONE)
3024 			flags |= FIEMAP_EXTENT_ENCODED;
3025 
3026 		if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
3027 			flags |= FIEMAP_EXTENT_DATA_INLINE;
3028 			flags |= FIEMAP_EXTENT_NOT_ALIGNED;
3029 			ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0,
3030 						 extent_len, flags);
3031 		} else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
3032 			ret = fiemap_process_hole(inode, fieinfo, &cache,
3033 						  &delalloc_cached_state,
3034 						  backref_ctx,
3035 						  disk_bytenr, extent_offset,
3036 						  extent_gen, key.offset,
3037 						  extent_end - 1);
3038 		} else if (disk_bytenr == 0) {
3039 			/* We have an explicit hole. */
3040 			ret = fiemap_process_hole(inode, fieinfo, &cache,
3041 						  &delalloc_cached_state,
3042 						  backref_ctx, 0, 0, 0,
3043 						  key.offset, extent_end - 1);
3044 		} else {
3045 			/* We have a regular extent. */
3046 			if (fieinfo->fi_extents_max) {
3047 				ret = btrfs_is_data_extent_shared(inode,
3048 								  disk_bytenr,
3049 								  extent_gen,
3050 								  backref_ctx);
3051 				if (ret < 0)
3052 					goto out_unlock;
3053 				else if (ret > 0)
3054 					flags |= FIEMAP_EXTENT_SHARED;
3055 			}
3056 
3057 			ret = emit_fiemap_extent(fieinfo, &cache, key.offset,
3058 						 disk_bytenr + extent_offset,
3059 						 extent_len, flags);
3060 		}
3061 
3062 		if (ret < 0) {
3063 			goto out_unlock;
3064 		} else if (ret > 0) {
3065 			/* fiemap_fill_next_extent() told us to stop. */
3066 			stopped = true;
3067 			break;
3068 		}
3069 
3070 		prev_extent_end = extent_end;
3071 next_item:
3072 		if (fatal_signal_pending(current)) {
3073 			ret = -EINTR;
3074 			goto out_unlock;
3075 		}
3076 
3077 		ret = fiemap_next_leaf_item(inode, path);
3078 		if (ret < 0) {
3079 			goto out_unlock;
3080 		} else if (ret > 0) {
3081 			/* No more file extent items for this inode. */
3082 			break;
3083 		}
3084 		cond_resched();
3085 	}
3086 
3087 check_eof_delalloc:
3088 	/*
3089 	 * Release (and free) the path before emitting any final entries to
3090 	 * fiemap_fill_next_extent() to keep lockdep happy. This is because
3091 	 * once we find no more file extent items exist, we may have a
3092 	 * non-cloned leaf, and fiemap_fill_next_extent() can trigger page
3093 	 * faults when copying data to the user space buffer.
3094 	 */
3095 	btrfs_free_path(path);
3096 	path = NULL;
3097 
3098 	if (!stopped && prev_extent_end < lockend) {
3099 		ret = fiemap_process_hole(inode, fieinfo, &cache,
3100 					  &delalloc_cached_state, backref_ctx,
3101 					  0, 0, 0, prev_extent_end, lockend - 1);
3102 		if (ret < 0)
3103 			goto out_unlock;
3104 		prev_extent_end = lockend;
3105 	}
3106 
3107 	if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3108 		const u64 i_size = i_size_read(&inode->vfs_inode);
3109 
3110 		if (prev_extent_end < i_size) {
3111 			u64 delalloc_start;
3112 			u64 delalloc_end;
3113 			bool delalloc;
3114 
3115 			delalloc = btrfs_find_delalloc_in_range(inode,
3116 								prev_extent_end,
3117 								i_size - 1,
3118 								&delalloc_cached_state,
3119 								&delalloc_start,
3120 								&delalloc_end);
3121 			if (!delalloc)
3122 				cache.flags |= FIEMAP_EXTENT_LAST;
3123 		} else {
3124 			cache.flags |= FIEMAP_EXTENT_LAST;
3125 		}
3126 	}
3127 
3128 	ret = emit_last_fiemap_cache(fieinfo, &cache);
3129 
3130 out_unlock:
3131 	unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3132 	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3133 out:
3134 	free_extent_state(delalloc_cached_state);
3135 	btrfs_free_backref_share_ctx(backref_ctx);
3136 	btrfs_free_path(path);
3137 	return ret;
3138 }
3139 
__free_extent_buffer(struct extent_buffer * eb)3140 static void __free_extent_buffer(struct extent_buffer *eb)
3141 {
3142 	kmem_cache_free(extent_buffer_cache, eb);
3143 }
3144 
extent_buffer_under_io(const struct extent_buffer * eb)3145 static int extent_buffer_under_io(const struct extent_buffer *eb)
3146 {
3147 	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3148 		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3149 }
3150 
page_range_has_eb(struct btrfs_fs_info * fs_info,struct page * page)3151 static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
3152 {
3153 	struct btrfs_subpage *subpage;
3154 
3155 	lockdep_assert_held(&page->mapping->private_lock);
3156 
3157 	if (PagePrivate(page)) {
3158 		subpage = (struct btrfs_subpage *)page->private;
3159 		if (atomic_read(&subpage->eb_refs))
3160 			return true;
3161 		/*
3162 		 * Even there is no eb refs here, we may still have
3163 		 * end_page_read() call relying on page::private.
3164 		 */
3165 		if (atomic_read(&subpage->readers))
3166 			return true;
3167 	}
3168 	return false;
3169 }
3170 
detach_extent_buffer_page(struct extent_buffer * eb,struct page * page)3171 static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
3172 {
3173 	struct btrfs_fs_info *fs_info = eb->fs_info;
3174 	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3175 
3176 	/*
3177 	 * For mapped eb, we're going to change the page private, which should
3178 	 * be done under the private_lock.
3179 	 */
3180 	if (mapped)
3181 		spin_lock(&page->mapping->private_lock);
3182 
3183 	if (!PagePrivate(page)) {
3184 		if (mapped)
3185 			spin_unlock(&page->mapping->private_lock);
3186 		return;
3187 	}
3188 
3189 	if (fs_info->nodesize >= PAGE_SIZE) {
3190 		/*
3191 		 * We do this since we'll remove the pages after we've
3192 		 * removed the eb from the radix tree, so we could race
3193 		 * and have this page now attached to the new eb.  So
3194 		 * only clear page_private if it's still connected to
3195 		 * this eb.
3196 		 */
3197 		if (PagePrivate(page) &&
3198 		    page->private == (unsigned long)eb) {
3199 			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3200 			BUG_ON(PageDirty(page));
3201 			BUG_ON(PageWriteback(page));
3202 			/*
3203 			 * We need to make sure we haven't be attached
3204 			 * to a new eb.
3205 			 */
3206 			detach_page_private(page);
3207 		}
3208 		if (mapped)
3209 			spin_unlock(&page->mapping->private_lock);
3210 		return;
3211 	}
3212 
3213 	/*
3214 	 * For subpage, we can have dummy eb with page private.  In this case,
3215 	 * we can directly detach the private as such page is only attached to
3216 	 * one dummy eb, no sharing.
3217 	 */
3218 	if (!mapped) {
3219 		btrfs_detach_subpage(fs_info, page);
3220 		return;
3221 	}
3222 
3223 	btrfs_page_dec_eb_refs(fs_info, page);
3224 
3225 	/*
3226 	 * We can only detach the page private if there are no other ebs in the
3227 	 * page range and no unfinished IO.
3228 	 */
3229 	if (!page_range_has_eb(fs_info, page))
3230 		btrfs_detach_subpage(fs_info, page);
3231 
3232 	spin_unlock(&page->mapping->private_lock);
3233 }
3234 
3235 /* Release all pages attached to the extent buffer */
btrfs_release_extent_buffer_pages(struct extent_buffer * eb)3236 static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
3237 {
3238 	int i;
3239 	int num_pages;
3240 
3241 	ASSERT(!extent_buffer_under_io(eb));
3242 
3243 	num_pages = num_extent_pages(eb);
3244 	for (i = 0; i < num_pages; i++) {
3245 		struct page *page = eb->pages[i];
3246 
3247 		if (!page)
3248 			continue;
3249 
3250 		detach_extent_buffer_page(eb, page);
3251 
3252 		/* One for when we allocated the page */
3253 		put_page(page);
3254 	}
3255 }
3256 
3257 /*
3258  * Helper for releasing the extent buffer.
3259  */
btrfs_release_extent_buffer(struct extent_buffer * eb)3260 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3261 {
3262 	btrfs_release_extent_buffer_pages(eb);
3263 	btrfs_leak_debug_del_eb(eb);
3264 	__free_extent_buffer(eb);
3265 }
3266 
3267 static struct extent_buffer *
__alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)3268 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
3269 		      unsigned long len)
3270 {
3271 	struct extent_buffer *eb = NULL;
3272 
3273 	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3274 	eb->start = start;
3275 	eb->len = len;
3276 	eb->fs_info = fs_info;
3277 	init_rwsem(&eb->lock);
3278 
3279 	btrfs_leak_debug_add_eb(eb);
3280 
3281 	spin_lock_init(&eb->refs_lock);
3282 	atomic_set(&eb->refs, 1);
3283 
3284 	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3285 
3286 	return eb;
3287 }
3288 
btrfs_clone_extent_buffer(const struct extent_buffer * src)3289 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3290 {
3291 	int i;
3292 	struct extent_buffer *new;
3293 	int num_pages = num_extent_pages(src);
3294 	int ret;
3295 
3296 	new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
3297 	if (new == NULL)
3298 		return NULL;
3299 
3300 	/*
3301 	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3302 	 * btrfs_release_extent_buffer() have different behavior for
3303 	 * UNMAPPED subpage extent buffer.
3304 	 */
3305 	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
3306 
3307 	ret = btrfs_alloc_page_array(num_pages, new->pages);
3308 	if (ret) {
3309 		btrfs_release_extent_buffer(new);
3310 		return NULL;
3311 	}
3312 
3313 	for (i = 0; i < num_pages; i++) {
3314 		int ret;
3315 		struct page *p = new->pages[i];
3316 
3317 		ret = attach_extent_buffer_page(new, p, NULL);
3318 		if (ret < 0) {
3319 			btrfs_release_extent_buffer(new);
3320 			return NULL;
3321 		}
3322 		WARN_ON(PageDirty(p));
3323 	}
3324 	copy_extent_buffer_full(new, src);
3325 	set_extent_buffer_uptodate(new);
3326 
3327 	return new;
3328 }
3329 
__alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,unsigned long len)3330 struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3331 						  u64 start, unsigned long len)
3332 {
3333 	struct extent_buffer *eb;
3334 	int num_pages;
3335 	int i;
3336 	int ret;
3337 
3338 	eb = __alloc_extent_buffer(fs_info, start, len);
3339 	if (!eb)
3340 		return NULL;
3341 
3342 	num_pages = num_extent_pages(eb);
3343 	ret = btrfs_alloc_page_array(num_pages, eb->pages);
3344 	if (ret)
3345 		goto err;
3346 
3347 	for (i = 0; i < num_pages; i++) {
3348 		struct page *p = eb->pages[i];
3349 
3350 		ret = attach_extent_buffer_page(eb, p, NULL);
3351 		if (ret < 0)
3352 			goto err;
3353 	}
3354 
3355 	set_extent_buffer_uptodate(eb);
3356 	btrfs_set_header_nritems(eb, 0);
3357 	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3358 
3359 	return eb;
3360 err:
3361 	for (i = 0; i < num_pages; i++) {
3362 		if (eb->pages[i]) {
3363 			detach_extent_buffer_page(eb, eb->pages[i]);
3364 			__free_page(eb->pages[i]);
3365 		}
3366 	}
3367 	__free_extent_buffer(eb);
3368 	return NULL;
3369 }
3370 
alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3371 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3372 						u64 start)
3373 {
3374 	return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
3375 }
3376 
check_buffer_tree_ref(struct extent_buffer * eb)3377 static void check_buffer_tree_ref(struct extent_buffer *eb)
3378 {
3379 	int refs;
3380 	/*
3381 	 * The TREE_REF bit is first set when the extent_buffer is added
3382 	 * to the radix tree. It is also reset, if unset, when a new reference
3383 	 * is created by find_extent_buffer.
3384 	 *
3385 	 * It is only cleared in two cases: freeing the last non-tree
3386 	 * reference to the extent_buffer when its STALE bit is set or
3387 	 * calling release_folio when the tree reference is the only reference.
3388 	 *
3389 	 * In both cases, care is taken to ensure that the extent_buffer's
3390 	 * pages are not under io. However, release_folio can be concurrently
3391 	 * called with creating new references, which is prone to race
3392 	 * conditions between the calls to check_buffer_tree_ref in those
3393 	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
3394 	 *
3395 	 * The actual lifetime of the extent_buffer in the radix tree is
3396 	 * adequately protected by the refcount, but the TREE_REF bit and
3397 	 * its corresponding reference are not. To protect against this
3398 	 * class of races, we call check_buffer_tree_ref from the codepaths
3399 	 * which trigger io. Note that once io is initiated, TREE_REF can no
3400 	 * longer be cleared, so that is the moment at which any such race is
3401 	 * best fixed.
3402 	 */
3403 	refs = atomic_read(&eb->refs);
3404 	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3405 		return;
3406 
3407 	spin_lock(&eb->refs_lock);
3408 	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3409 		atomic_inc(&eb->refs);
3410 	spin_unlock(&eb->refs_lock);
3411 }
3412 
mark_extent_buffer_accessed(struct extent_buffer * eb,struct page * accessed)3413 static void mark_extent_buffer_accessed(struct extent_buffer *eb,
3414 		struct page *accessed)
3415 {
3416 	int num_pages, i;
3417 
3418 	check_buffer_tree_ref(eb);
3419 
3420 	num_pages = num_extent_pages(eb);
3421 	for (i = 0; i < num_pages; i++) {
3422 		struct page *p = eb->pages[i];
3423 
3424 		if (p != accessed)
3425 			mark_page_accessed(p);
3426 	}
3427 }
3428 
find_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3429 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3430 					 u64 start)
3431 {
3432 	struct extent_buffer *eb;
3433 
3434 	eb = find_extent_buffer_nolock(fs_info, start);
3435 	if (!eb)
3436 		return NULL;
3437 	/*
3438 	 * Lock our eb's refs_lock to avoid races with free_extent_buffer().
3439 	 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3440 	 * another task running free_extent_buffer() might have seen that flag
3441 	 * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3442 	 * writeback flags not set) and it's still in the tree (flag
3443 	 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3444 	 * decrementing the extent buffer's reference count twice.  So here we
3445 	 * could race and increment the eb's reference count, clear its stale
3446 	 * flag, mark it as dirty and drop our reference before the other task
3447 	 * finishes executing free_extent_buffer, which would later result in
3448 	 * an attempt to free an extent buffer that is dirty.
3449 	 */
3450 	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3451 		spin_lock(&eb->refs_lock);
3452 		spin_unlock(&eb->refs_lock);
3453 	}
3454 	mark_extent_buffer_accessed(eb, NULL);
3455 	return eb;
3456 }
3457 
3458 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
alloc_test_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3459 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3460 					u64 start)
3461 {
3462 	struct extent_buffer *eb, *exists = NULL;
3463 	int ret;
3464 
3465 	eb = find_extent_buffer(fs_info, start);
3466 	if (eb)
3467 		return eb;
3468 	eb = alloc_dummy_extent_buffer(fs_info, start);
3469 	if (!eb)
3470 		return ERR_PTR(-ENOMEM);
3471 	eb->fs_info = fs_info;
3472 again:
3473 	ret = radix_tree_preload(GFP_NOFS);
3474 	if (ret) {
3475 		exists = ERR_PTR(ret);
3476 		goto free_eb;
3477 	}
3478 	spin_lock(&fs_info->buffer_lock);
3479 	ret = radix_tree_insert(&fs_info->buffer_radix,
3480 				start >> fs_info->sectorsize_bits, eb);
3481 	spin_unlock(&fs_info->buffer_lock);
3482 	radix_tree_preload_end();
3483 	if (ret == -EEXIST) {
3484 		exists = find_extent_buffer(fs_info, start);
3485 		if (exists)
3486 			goto free_eb;
3487 		else
3488 			goto again;
3489 	}
3490 	check_buffer_tree_ref(eb);
3491 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3492 
3493 	return eb;
3494 free_eb:
3495 	btrfs_release_extent_buffer(eb);
3496 	return exists;
3497 }
3498 #endif
3499 
grab_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page)3500 static struct extent_buffer *grab_extent_buffer(
3501 		struct btrfs_fs_info *fs_info, struct page *page)
3502 {
3503 	struct extent_buffer *exists;
3504 
3505 	/*
3506 	 * For subpage case, we completely rely on radix tree to ensure we
3507 	 * don't try to insert two ebs for the same bytenr.  So here we always
3508 	 * return NULL and just continue.
3509 	 */
3510 	if (fs_info->nodesize < PAGE_SIZE)
3511 		return NULL;
3512 
3513 	/* Page not yet attached to an extent buffer */
3514 	if (!PagePrivate(page))
3515 		return NULL;
3516 
3517 	/*
3518 	 * We could have already allocated an eb for this page and attached one
3519 	 * so lets see if we can get a ref on the existing eb, and if we can we
3520 	 * know it's good and we can just return that one, else we know we can
3521 	 * just overwrite page->private.
3522 	 */
3523 	exists = (struct extent_buffer *)page->private;
3524 	if (atomic_inc_not_zero(&exists->refs))
3525 		return exists;
3526 
3527 	WARN_ON(PageDirty(page));
3528 	detach_page_private(page);
3529 	return NULL;
3530 }
3531 
check_eb_alignment(struct btrfs_fs_info * fs_info,u64 start)3532 static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3533 {
3534 	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3535 		btrfs_err(fs_info, "bad tree block start %llu", start);
3536 		return -EINVAL;
3537 	}
3538 
3539 	if (fs_info->nodesize < PAGE_SIZE &&
3540 	    offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3541 		btrfs_err(fs_info,
3542 		"tree block crosses page boundary, start %llu nodesize %u",
3543 			  start, fs_info->nodesize);
3544 		return -EINVAL;
3545 	}
3546 	if (fs_info->nodesize >= PAGE_SIZE &&
3547 	    !PAGE_ALIGNED(start)) {
3548 		btrfs_err(fs_info,
3549 		"tree block is not page aligned, start %llu nodesize %u",
3550 			  start, fs_info->nodesize);
3551 		return -EINVAL;
3552 	}
3553 	return 0;
3554 }
3555 
alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,u64 owner_root,int level)3556 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3557 					  u64 start, u64 owner_root, int level)
3558 {
3559 	unsigned long len = fs_info->nodesize;
3560 	int num_pages;
3561 	int i;
3562 	unsigned long index = start >> PAGE_SHIFT;
3563 	struct extent_buffer *eb;
3564 	struct extent_buffer *exists = NULL;
3565 	struct page *p;
3566 	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3567 	struct btrfs_subpage *prealloc = NULL;
3568 	u64 lockdep_owner = owner_root;
3569 	int uptodate = 1;
3570 	int ret;
3571 
3572 	if (check_eb_alignment(fs_info, start))
3573 		return ERR_PTR(-EINVAL);
3574 
3575 #if BITS_PER_LONG == 32
3576 	if (start >= MAX_LFS_FILESIZE) {
3577 		btrfs_err_rl(fs_info,
3578 		"extent buffer %llu is beyond 32bit page cache limit", start);
3579 		btrfs_err_32bit_limit(fs_info);
3580 		return ERR_PTR(-EOVERFLOW);
3581 	}
3582 	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3583 		btrfs_warn_32bit_limit(fs_info);
3584 #endif
3585 
3586 	eb = find_extent_buffer(fs_info, start);
3587 	if (eb)
3588 		return eb;
3589 
3590 	eb = __alloc_extent_buffer(fs_info, start, len);
3591 	if (!eb)
3592 		return ERR_PTR(-ENOMEM);
3593 
3594 	/*
3595 	 * The reloc trees are just snapshots, so we need them to appear to be
3596 	 * just like any other fs tree WRT lockdep.
3597 	 */
3598 	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3599 		lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3600 
3601 	btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3602 
3603 	num_pages = num_extent_pages(eb);
3604 
3605 	/*
3606 	 * Preallocate page->private for subpage case, so that we won't
3607 	 * allocate memory with private_lock nor page lock hold.
3608 	 *
3609 	 * The memory will be freed by attach_extent_buffer_page() or freed
3610 	 * manually if we exit earlier.
3611 	 */
3612 	if (fs_info->nodesize < PAGE_SIZE) {
3613 		prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
3614 		if (IS_ERR(prealloc)) {
3615 			exists = ERR_CAST(prealloc);
3616 			goto free_eb;
3617 		}
3618 	}
3619 
3620 	for (i = 0; i < num_pages; i++, index++) {
3621 		p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
3622 		if (!p) {
3623 			exists = ERR_PTR(-ENOMEM);
3624 			btrfs_free_subpage(prealloc);
3625 			goto free_eb;
3626 		}
3627 
3628 		spin_lock(&mapping->private_lock);
3629 		exists = grab_extent_buffer(fs_info, p);
3630 		if (exists) {
3631 			spin_unlock(&mapping->private_lock);
3632 			unlock_page(p);
3633 			put_page(p);
3634 			mark_extent_buffer_accessed(exists, p);
3635 			btrfs_free_subpage(prealloc);
3636 			goto free_eb;
3637 		}
3638 		/* Should not fail, as we have preallocated the memory */
3639 		ret = attach_extent_buffer_page(eb, p, prealloc);
3640 		ASSERT(!ret);
3641 		/*
3642 		 * To inform we have extra eb under allocation, so that
3643 		 * detach_extent_buffer_page() won't release the page private
3644 		 * when the eb hasn't yet been inserted into radix tree.
3645 		 *
3646 		 * The ref will be decreased when the eb released the page, in
3647 		 * detach_extent_buffer_page().
3648 		 * Thus needs no special handling in error path.
3649 		 */
3650 		btrfs_page_inc_eb_refs(fs_info, p);
3651 		spin_unlock(&mapping->private_lock);
3652 
3653 		WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
3654 		eb->pages[i] = p;
3655 		if (!btrfs_page_test_uptodate(fs_info, p, eb->start, eb->len))
3656 			uptodate = 0;
3657 
3658 		/*
3659 		 * We can't unlock the pages just yet since the extent buffer
3660 		 * hasn't been properly inserted in the radix tree, this
3661 		 * opens a race with btree_release_folio which can free a page
3662 		 * while we are still filling in all pages for the buffer and
3663 		 * we could crash.
3664 		 */
3665 	}
3666 	if (uptodate)
3667 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3668 again:
3669 	ret = radix_tree_preload(GFP_NOFS);
3670 	if (ret) {
3671 		exists = ERR_PTR(ret);
3672 		goto free_eb;
3673 	}
3674 
3675 	spin_lock(&fs_info->buffer_lock);
3676 	ret = radix_tree_insert(&fs_info->buffer_radix,
3677 				start >> fs_info->sectorsize_bits, eb);
3678 	spin_unlock(&fs_info->buffer_lock);
3679 	radix_tree_preload_end();
3680 	if (ret == -EEXIST) {
3681 		exists = find_extent_buffer(fs_info, start);
3682 		if (exists)
3683 			goto free_eb;
3684 		else
3685 			goto again;
3686 	}
3687 	/* add one reference for the tree */
3688 	check_buffer_tree_ref(eb);
3689 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3690 
3691 	/*
3692 	 * Now it's safe to unlock the pages because any calls to
3693 	 * btree_release_folio will correctly detect that a page belongs to a
3694 	 * live buffer and won't free them prematurely.
3695 	 */
3696 	for (i = 0; i < num_pages; i++)
3697 		unlock_page(eb->pages[i]);
3698 	return eb;
3699 
3700 free_eb:
3701 	WARN_ON(!atomic_dec_and_test(&eb->refs));
3702 	for (i = 0; i < num_pages; i++) {
3703 		if (eb->pages[i])
3704 			unlock_page(eb->pages[i]);
3705 	}
3706 
3707 	btrfs_release_extent_buffer(eb);
3708 	return exists;
3709 }
3710 
btrfs_release_extent_buffer_rcu(struct rcu_head * head)3711 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3712 {
3713 	struct extent_buffer *eb =
3714 			container_of(head, struct extent_buffer, rcu_head);
3715 
3716 	__free_extent_buffer(eb);
3717 }
3718 
release_extent_buffer(struct extent_buffer * eb)3719 static int release_extent_buffer(struct extent_buffer *eb)
3720 	__releases(&eb->refs_lock)
3721 {
3722 	lockdep_assert_held(&eb->refs_lock);
3723 
3724 	WARN_ON(atomic_read(&eb->refs) == 0);
3725 	if (atomic_dec_and_test(&eb->refs)) {
3726 		if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
3727 			struct btrfs_fs_info *fs_info = eb->fs_info;
3728 
3729 			spin_unlock(&eb->refs_lock);
3730 
3731 			spin_lock(&fs_info->buffer_lock);
3732 			radix_tree_delete(&fs_info->buffer_radix,
3733 					  eb->start >> fs_info->sectorsize_bits);
3734 			spin_unlock(&fs_info->buffer_lock);
3735 		} else {
3736 			spin_unlock(&eb->refs_lock);
3737 		}
3738 
3739 		btrfs_leak_debug_del_eb(eb);
3740 		/* Should be safe to release our pages at this point */
3741 		btrfs_release_extent_buffer_pages(eb);
3742 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3743 		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3744 			__free_extent_buffer(eb);
3745 			return 1;
3746 		}
3747 #endif
3748 		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3749 		return 1;
3750 	}
3751 	spin_unlock(&eb->refs_lock);
3752 
3753 	return 0;
3754 }
3755 
free_extent_buffer(struct extent_buffer * eb)3756 void free_extent_buffer(struct extent_buffer *eb)
3757 {
3758 	int refs;
3759 	if (!eb)
3760 		return;
3761 
3762 	refs = atomic_read(&eb->refs);
3763 	while (1) {
3764 		if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3765 		    || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3766 			refs == 1))
3767 			break;
3768 		if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
3769 			return;
3770 	}
3771 
3772 	spin_lock(&eb->refs_lock);
3773 	if (atomic_read(&eb->refs) == 2 &&
3774 	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3775 	    !extent_buffer_under_io(eb) &&
3776 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3777 		atomic_dec(&eb->refs);
3778 
3779 	/*
3780 	 * I know this is terrible, but it's temporary until we stop tracking
3781 	 * the uptodate bits and such for the extent buffers.
3782 	 */
3783 	release_extent_buffer(eb);
3784 }
3785 
free_extent_buffer_stale(struct extent_buffer * eb)3786 void free_extent_buffer_stale(struct extent_buffer *eb)
3787 {
3788 	if (!eb)
3789 		return;
3790 
3791 	spin_lock(&eb->refs_lock);
3792 	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3793 
3794 	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3795 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3796 		atomic_dec(&eb->refs);
3797 	release_extent_buffer(eb);
3798 }
3799 
btree_clear_page_dirty(struct page * page)3800 static void btree_clear_page_dirty(struct page *page)
3801 {
3802 	ASSERT(PageDirty(page));
3803 	ASSERT(PageLocked(page));
3804 	clear_page_dirty_for_io(page);
3805 	xa_lock_irq(&page->mapping->i_pages);
3806 	if (!PageDirty(page))
3807 		__xa_clear_mark(&page->mapping->i_pages,
3808 				page_index(page), PAGECACHE_TAG_DIRTY);
3809 	xa_unlock_irq(&page->mapping->i_pages);
3810 }
3811 
clear_subpage_extent_buffer_dirty(const struct extent_buffer * eb)3812 static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
3813 {
3814 	struct btrfs_fs_info *fs_info = eb->fs_info;
3815 	struct page *page = eb->pages[0];
3816 	bool last;
3817 
3818 	/* btree_clear_page_dirty() needs page locked */
3819 	lock_page(page);
3820 	last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start,
3821 						  eb->len);
3822 	if (last)
3823 		btree_clear_page_dirty(page);
3824 	unlock_page(page);
3825 	WARN_ON(atomic_read(&eb->refs) == 0);
3826 }
3827 
btrfs_clear_buffer_dirty(struct btrfs_trans_handle * trans,struct extent_buffer * eb)3828 void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3829 			      struct extent_buffer *eb)
3830 {
3831 	struct btrfs_fs_info *fs_info = eb->fs_info;
3832 	int i;
3833 	int num_pages;
3834 	struct page *page;
3835 
3836 	btrfs_assert_tree_write_locked(eb);
3837 
3838 	if (trans && btrfs_header_generation(eb) != trans->transid)
3839 		return;
3840 
3841 	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3842 		return;
3843 
3844 	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3845 				 fs_info->dirty_metadata_batch);
3846 
3847 	if (eb->fs_info->nodesize < PAGE_SIZE)
3848 		return clear_subpage_extent_buffer_dirty(eb);
3849 
3850 	num_pages = num_extent_pages(eb);
3851 
3852 	for (i = 0; i < num_pages; i++) {
3853 		page = eb->pages[i];
3854 		if (!PageDirty(page))
3855 			continue;
3856 		lock_page(page);
3857 		btree_clear_page_dirty(page);
3858 		unlock_page(page);
3859 	}
3860 	WARN_ON(atomic_read(&eb->refs) == 0);
3861 }
3862 
set_extent_buffer_dirty(struct extent_buffer * eb)3863 void set_extent_buffer_dirty(struct extent_buffer *eb)
3864 {
3865 	int i;
3866 	int num_pages;
3867 	bool was_dirty;
3868 
3869 	check_buffer_tree_ref(eb);
3870 
3871 	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3872 
3873 	num_pages = num_extent_pages(eb);
3874 	WARN_ON(atomic_read(&eb->refs) == 0);
3875 	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3876 
3877 	if (!was_dirty) {
3878 		bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
3879 
3880 		/*
3881 		 * For subpage case, we can have other extent buffers in the
3882 		 * same page, and in clear_subpage_extent_buffer_dirty() we
3883 		 * have to clear page dirty without subpage lock held.
3884 		 * This can cause race where our page gets dirty cleared after
3885 		 * we just set it.
3886 		 *
3887 		 * Thankfully, clear_subpage_extent_buffer_dirty() has locked
3888 		 * its page for other reasons, we can use page lock to prevent
3889 		 * the above race.
3890 		 */
3891 		if (subpage)
3892 			lock_page(eb->pages[0]);
3893 		for (i = 0; i < num_pages; i++)
3894 			btrfs_page_set_dirty(eb->fs_info, eb->pages[i],
3895 					     eb->start, eb->len);
3896 		if (subpage)
3897 			unlock_page(eb->pages[0]);
3898 		percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3899 					 eb->len,
3900 					 eb->fs_info->dirty_metadata_batch);
3901 	}
3902 #ifdef CONFIG_BTRFS_DEBUG
3903 	for (i = 0; i < num_pages; i++)
3904 		ASSERT(PageDirty(eb->pages[i]));
3905 #endif
3906 }
3907 
clear_extent_buffer_uptodate(struct extent_buffer * eb)3908 void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3909 {
3910 	struct btrfs_fs_info *fs_info = eb->fs_info;
3911 	struct page *page;
3912 	int num_pages;
3913 	int i;
3914 
3915 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3916 	num_pages = num_extent_pages(eb);
3917 	for (i = 0; i < num_pages; i++) {
3918 		page = eb->pages[i];
3919 		if (!page)
3920 			continue;
3921 
3922 		/*
3923 		 * This is special handling for metadata subpage, as regular
3924 		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
3925 		 */
3926 		if (fs_info->nodesize >= PAGE_SIZE)
3927 			ClearPageUptodate(page);
3928 		else
3929 			btrfs_subpage_clear_uptodate(fs_info, page, eb->start,
3930 						     eb->len);
3931 	}
3932 }
3933 
set_extent_buffer_uptodate(struct extent_buffer * eb)3934 void set_extent_buffer_uptodate(struct extent_buffer *eb)
3935 {
3936 	struct btrfs_fs_info *fs_info = eb->fs_info;
3937 	struct page *page;
3938 	int num_pages;
3939 	int i;
3940 
3941 	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3942 	num_pages = num_extent_pages(eb);
3943 	for (i = 0; i < num_pages; i++) {
3944 		page = eb->pages[i];
3945 
3946 		/*
3947 		 * This is special handling for metadata subpage, as regular
3948 		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
3949 		 */
3950 		if (fs_info->nodesize >= PAGE_SIZE)
3951 			SetPageUptodate(page);
3952 		else
3953 			btrfs_subpage_set_uptodate(fs_info, page, eb->start,
3954 						   eb->len);
3955 	}
3956 }
3957 
extent_buffer_read_end_io(struct btrfs_bio * bbio)3958 static void extent_buffer_read_end_io(struct btrfs_bio *bbio)
3959 {
3960 	struct extent_buffer *eb = bbio->private;
3961 	struct btrfs_fs_info *fs_info = eb->fs_info;
3962 	bool uptodate = !bbio->bio.bi_status;
3963 	struct bvec_iter_all iter_all;
3964 	struct bio_vec *bvec;
3965 	u32 bio_offset = 0;
3966 
3967 	eb->read_mirror = bbio->mirror_num;
3968 
3969 	if (uptodate &&
3970 	    btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3971 		uptodate = false;
3972 
3973 	if (uptodate) {
3974 		set_extent_buffer_uptodate(eb);
3975 	} else {
3976 		clear_extent_buffer_uptodate(eb);
3977 		set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3978 	}
3979 
3980 	bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
3981 		u64 start = eb->start + bio_offset;
3982 		struct page *page = bvec->bv_page;
3983 		u32 len = bvec->bv_len;
3984 
3985 		if (uptodate)
3986 			btrfs_page_set_uptodate(fs_info, page, start, len);
3987 		else
3988 			btrfs_page_clear_uptodate(fs_info, page, start, len);
3989 
3990 		bio_offset += len;
3991 	}
3992 
3993 	clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
3994 	smp_mb__after_atomic();
3995 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
3996 	free_extent_buffer(eb);
3997 
3998 	bio_put(&bbio->bio);
3999 }
4000 
read_extent_buffer_pages(struct extent_buffer * eb,int wait,int mirror_num,struct btrfs_tree_parent_check * check)4001 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
4002 			     struct btrfs_tree_parent_check *check)
4003 {
4004 	int num_pages = num_extent_pages(eb), i;
4005 	struct btrfs_bio *bbio;
4006 
4007 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4008 		return 0;
4009 
4010 	/*
4011 	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
4012 	 * operation, which could potentially still be in flight.  In this case
4013 	 * we simply want to return an error.
4014 	 */
4015 	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
4016 		return -EIO;
4017 
4018 	/* Someone else is already reading the buffer, just wait for it. */
4019 	if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
4020 		goto done;
4021 
4022 	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
4023 	eb->read_mirror = 0;
4024 	check_buffer_tree_ref(eb);
4025 	atomic_inc(&eb->refs);
4026 
4027 	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
4028 			       REQ_OP_READ | REQ_META, eb->fs_info,
4029 			       extent_buffer_read_end_io, eb);
4030 	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
4031 	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
4032 	bbio->file_offset = eb->start;
4033 	memcpy(&bbio->parent_check, check, sizeof(*check));
4034 	if (eb->fs_info->nodesize < PAGE_SIZE) {
4035 		__bio_add_page(&bbio->bio, eb->pages[0], eb->len,
4036 			       eb->start - page_offset(eb->pages[0]));
4037 	} else {
4038 		for (i = 0; i < num_pages; i++)
4039 			__bio_add_page(&bbio->bio, eb->pages[i], PAGE_SIZE, 0);
4040 	}
4041 	btrfs_submit_bio(bbio, mirror_num);
4042 
4043 done:
4044 	if (wait == WAIT_COMPLETE) {
4045 		wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
4046 		if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4047 			return -EIO;
4048 	}
4049 
4050 	return 0;
4051 }
4052 
report_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)4053 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
4054 			    unsigned long len)
4055 {
4056 	btrfs_warn(eb->fs_info,
4057 		"access to eb bytenr %llu len %lu out of range start %lu len %lu",
4058 		eb->start, eb->len, start, len);
4059 	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4060 
4061 	return true;
4062 }
4063 
4064 /*
4065  * Check if the [start, start + len) range is valid before reading/writing
4066  * the eb.
4067  * NOTE: @start and @len are offset inside the eb, not logical address.
4068  *
4069  * Caller should not touch the dst/src memory if this function returns error.
4070  */
check_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)4071 static inline int check_eb_range(const struct extent_buffer *eb,
4072 				 unsigned long start, unsigned long len)
4073 {
4074 	unsigned long offset;
4075 
4076 	/* start, start + len should not go beyond eb->len nor overflow */
4077 	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
4078 		return report_eb_range(eb, start, len);
4079 
4080 	return false;
4081 }
4082 
read_extent_buffer(const struct extent_buffer * eb,void * dstv,unsigned long start,unsigned long len)4083 void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
4084 			unsigned long start, unsigned long len)
4085 {
4086 	size_t cur;
4087 	size_t offset;
4088 	struct page *page;
4089 	char *kaddr;
4090 	char *dst = (char *)dstv;
4091 	unsigned long i = get_eb_page_index(start);
4092 
4093 	if (check_eb_range(eb, start, len)) {
4094 		/*
4095 		 * Invalid range hit, reset the memory, so callers won't get
4096 		 * some random garbage for their uninitialzed memory.
4097 		 */
4098 		memset(dstv, 0, len);
4099 		return;
4100 	}
4101 
4102 	offset = get_eb_offset_in_page(eb, start);
4103 
4104 	while (len > 0) {
4105 		page = eb->pages[i];
4106 
4107 		cur = min(len, (PAGE_SIZE - offset));
4108 		kaddr = page_address(page);
4109 		memcpy(dst, kaddr + offset, cur);
4110 
4111 		dst += cur;
4112 		len -= cur;
4113 		offset = 0;
4114 		i++;
4115 	}
4116 }
4117 
read_extent_buffer_to_user_nofault(const struct extent_buffer * eb,void __user * dstv,unsigned long start,unsigned long len)4118 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
4119 				       void __user *dstv,
4120 				       unsigned long start, unsigned long len)
4121 {
4122 	size_t cur;
4123 	size_t offset;
4124 	struct page *page;
4125 	char *kaddr;
4126 	char __user *dst = (char __user *)dstv;
4127 	unsigned long i = get_eb_page_index(start);
4128 	int ret = 0;
4129 
4130 	WARN_ON(start > eb->len);
4131 	WARN_ON(start + len > eb->start + eb->len);
4132 
4133 	offset = get_eb_offset_in_page(eb, start);
4134 
4135 	while (len > 0) {
4136 		page = eb->pages[i];
4137 
4138 		cur = min(len, (PAGE_SIZE - offset));
4139 		kaddr = page_address(page);
4140 		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
4141 			ret = -EFAULT;
4142 			break;
4143 		}
4144 
4145 		dst += cur;
4146 		len -= cur;
4147 		offset = 0;
4148 		i++;
4149 	}
4150 
4151 	return ret;
4152 }
4153 
memcmp_extent_buffer(const struct extent_buffer * eb,const void * ptrv,unsigned long start,unsigned long len)4154 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4155 			 unsigned long start, unsigned long len)
4156 {
4157 	size_t cur;
4158 	size_t offset;
4159 	struct page *page;
4160 	char *kaddr;
4161 	char *ptr = (char *)ptrv;
4162 	unsigned long i = get_eb_page_index(start);
4163 	int ret = 0;
4164 
4165 	if (check_eb_range(eb, start, len))
4166 		return -EINVAL;
4167 
4168 	offset = get_eb_offset_in_page(eb, start);
4169 
4170 	while (len > 0) {
4171 		page = eb->pages[i];
4172 
4173 		cur = min(len, (PAGE_SIZE - offset));
4174 
4175 		kaddr = page_address(page);
4176 		ret = memcmp(ptr, kaddr + offset, cur);
4177 		if (ret)
4178 			break;
4179 
4180 		ptr += cur;
4181 		len -= cur;
4182 		offset = 0;
4183 		i++;
4184 	}
4185 	return ret;
4186 }
4187 
4188 /*
4189  * Check that the extent buffer is uptodate.
4190  *
4191  * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4192  * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4193  */
assert_eb_page_uptodate(const struct extent_buffer * eb,struct page * page)4194 static void assert_eb_page_uptodate(const struct extent_buffer *eb,
4195 				    struct page *page)
4196 {
4197 	struct btrfs_fs_info *fs_info = eb->fs_info;
4198 
4199 	/*
4200 	 * If we are using the commit root we could potentially clear a page
4201 	 * Uptodate while we're using the extent buffer that we've previously
4202 	 * looked up.  We don't want to complain in this case, as the page was
4203 	 * valid before, we just didn't write it out.  Instead we want to catch
4204 	 * the case where we didn't actually read the block properly, which
4205 	 * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4206 	 */
4207 	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4208 		return;
4209 
4210 	if (fs_info->nodesize < PAGE_SIZE) {
4211 		if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, page,
4212 							 eb->start, eb->len)))
4213 			btrfs_subpage_dump_bitmap(fs_info, page, eb->start, eb->len);
4214 	} else {
4215 		WARN_ON(!PageUptodate(page));
4216 	}
4217 }
4218 
__write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len,bool use_memmove)4219 static void __write_extent_buffer(const struct extent_buffer *eb,
4220 				  const void *srcv, unsigned long start,
4221 				  unsigned long len, bool use_memmove)
4222 {
4223 	size_t cur;
4224 	size_t offset;
4225 	struct page *page;
4226 	char *kaddr;
4227 	char *src = (char *)srcv;
4228 	unsigned long i = get_eb_page_index(start);
4229 	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
4230 	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4231 
4232 	WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
4233 
4234 	if (check_eb_range(eb, start, len))
4235 		return;
4236 
4237 	offset = get_eb_offset_in_page(eb, start);
4238 
4239 	while (len > 0) {
4240 		page = eb->pages[i];
4241 		if (check_uptodate)
4242 			assert_eb_page_uptodate(eb, page);
4243 
4244 		cur = min(len, PAGE_SIZE - offset);
4245 		kaddr = page_address(page);
4246 		if (use_memmove)
4247 			memmove(kaddr + offset, src, cur);
4248 		else
4249 			memcpy(kaddr + offset, src, cur);
4250 
4251 		src += cur;
4252 		len -= cur;
4253 		offset = 0;
4254 		i++;
4255 	}
4256 }
4257 
write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len)4258 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4259 			 unsigned long start, unsigned long len)
4260 {
4261 	return __write_extent_buffer(eb, srcv, start, len, false);
4262 }
4263 
memset_extent_buffer(const struct extent_buffer * eb,int c,unsigned long start,unsigned long len)4264 static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4265 				 unsigned long start, unsigned long len)
4266 {
4267 	unsigned long cur = start;
4268 
4269 	while (cur < start + len) {
4270 		unsigned long index = get_eb_page_index(cur);
4271 		unsigned int offset = get_eb_offset_in_page(eb, cur);
4272 		unsigned int cur_len = min(start + len - cur, PAGE_SIZE - offset);
4273 		struct page *page = eb->pages[index];
4274 
4275 		assert_eb_page_uptodate(eb, page);
4276 		memset(page_address(page) + offset, c, cur_len);
4277 
4278 		cur += cur_len;
4279 	}
4280 }
4281 
memzero_extent_buffer(const struct extent_buffer * eb,unsigned long start,unsigned long len)4282 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4283 			   unsigned long len)
4284 {
4285 	if (check_eb_range(eb, start, len))
4286 		return;
4287 	return memset_extent_buffer(eb, 0, start, len);
4288 }
4289 
copy_extent_buffer_full(const struct extent_buffer * dst,const struct extent_buffer * src)4290 void copy_extent_buffer_full(const struct extent_buffer *dst,
4291 			     const struct extent_buffer *src)
4292 {
4293 	unsigned long cur = 0;
4294 
4295 	ASSERT(dst->len == src->len);
4296 
4297 	while (cur < src->len) {
4298 		unsigned long index = get_eb_page_index(cur);
4299 		unsigned long offset = get_eb_offset_in_page(src, cur);
4300 		unsigned long cur_len = min(src->len, PAGE_SIZE - offset);
4301 		void *addr = page_address(src->pages[index]) + offset;
4302 
4303 		write_extent_buffer(dst, addr, cur, cur_len);
4304 
4305 		cur += cur_len;
4306 	}
4307 }
4308 
copy_extent_buffer(const struct extent_buffer * dst,const struct extent_buffer * src,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4309 void copy_extent_buffer(const struct extent_buffer *dst,
4310 			const struct extent_buffer *src,
4311 			unsigned long dst_offset, unsigned long src_offset,
4312 			unsigned long len)
4313 {
4314 	u64 dst_len = dst->len;
4315 	size_t cur;
4316 	size_t offset;
4317 	struct page *page;
4318 	char *kaddr;
4319 	unsigned long i = get_eb_page_index(dst_offset);
4320 
4321 	if (check_eb_range(dst, dst_offset, len) ||
4322 	    check_eb_range(src, src_offset, len))
4323 		return;
4324 
4325 	WARN_ON(src->len != dst_len);
4326 
4327 	offset = get_eb_offset_in_page(dst, dst_offset);
4328 
4329 	while (len > 0) {
4330 		page = dst->pages[i];
4331 		assert_eb_page_uptodate(dst, page);
4332 
4333 		cur = min(len, (unsigned long)(PAGE_SIZE - offset));
4334 
4335 		kaddr = page_address(page);
4336 		read_extent_buffer(src, kaddr + offset, src_offset, cur);
4337 
4338 		src_offset += cur;
4339 		len -= cur;
4340 		offset = 0;
4341 		i++;
4342 	}
4343 }
4344 
4345 /*
4346  * eb_bitmap_offset() - calculate the page and offset of the byte containing the
4347  * given bit number
4348  * @eb: the extent buffer
4349  * @start: offset of the bitmap item in the extent buffer
4350  * @nr: bit number
4351  * @page_index: return index of the page in the extent buffer that contains the
4352  * given bit number
4353  * @page_offset: return offset into the page given by page_index
4354  *
4355  * This helper hides the ugliness of finding the byte in an extent buffer which
4356  * contains a given bit.
4357  */
eb_bitmap_offset(const struct extent_buffer * eb,unsigned long start,unsigned long nr,unsigned long * page_index,size_t * page_offset)4358 static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4359 				    unsigned long start, unsigned long nr,
4360 				    unsigned long *page_index,
4361 				    size_t *page_offset)
4362 {
4363 	size_t byte_offset = BIT_BYTE(nr);
4364 	size_t offset;
4365 
4366 	/*
4367 	 * The byte we want is the offset of the extent buffer + the offset of
4368 	 * the bitmap item in the extent buffer + the offset of the byte in the
4369 	 * bitmap item.
4370 	 */
4371 	offset = start + offset_in_page(eb->start) + byte_offset;
4372 
4373 	*page_index = offset >> PAGE_SHIFT;
4374 	*page_offset = offset_in_page(offset);
4375 }
4376 
4377 /*
4378  * Determine whether a bit in a bitmap item is set.
4379  *
4380  * @eb:     the extent buffer
4381  * @start:  offset of the bitmap item in the extent buffer
4382  * @nr:     bit number to test
4383  */
extent_buffer_test_bit(const struct extent_buffer * eb,unsigned long start,unsigned long nr)4384 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4385 			   unsigned long nr)
4386 {
4387 	u8 *kaddr;
4388 	struct page *page;
4389 	unsigned long i;
4390 	size_t offset;
4391 
4392 	eb_bitmap_offset(eb, start, nr, &i, &offset);
4393 	page = eb->pages[i];
4394 	assert_eb_page_uptodate(eb, page);
4395 	kaddr = page_address(page);
4396 	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4397 }
4398 
extent_buffer_get_byte(const struct extent_buffer * eb,unsigned long bytenr)4399 static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4400 {
4401 	unsigned long index = get_eb_page_index(bytenr);
4402 
4403 	if (check_eb_range(eb, bytenr, 1))
4404 		return NULL;
4405 	return page_address(eb->pages[index]) + get_eb_offset_in_page(eb, bytenr);
4406 }
4407 
4408 /*
4409  * Set an area of a bitmap to 1.
4410  *
4411  * @eb:     the extent buffer
4412  * @start:  offset of the bitmap item in the extent buffer
4413  * @pos:    bit number of the first bit
4414  * @len:    number of bits to set
4415  */
extent_buffer_bitmap_set(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)4416 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4417 			      unsigned long pos, unsigned long len)
4418 {
4419 	unsigned int first_byte = start + BIT_BYTE(pos);
4420 	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4421 	const bool same_byte = (first_byte == last_byte);
4422 	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4423 	u8 *kaddr;
4424 
4425 	if (same_byte)
4426 		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4427 
4428 	/* Handle the first byte. */
4429 	kaddr = extent_buffer_get_byte(eb, first_byte);
4430 	*kaddr |= mask;
4431 	if (same_byte)
4432 		return;
4433 
4434 	/* Handle the byte aligned part. */
4435 	ASSERT(first_byte + 1 <= last_byte);
4436 	memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1);
4437 
4438 	/* Handle the last byte. */
4439 	kaddr = extent_buffer_get_byte(eb, last_byte);
4440 	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4441 }
4442 
4443 
4444 /*
4445  * Clear an area of a bitmap.
4446  *
4447  * @eb:     the extent buffer
4448  * @start:  offset of the bitmap item in the extent buffer
4449  * @pos:    bit number of the first bit
4450  * @len:    number of bits to clear
4451  */
extent_buffer_bitmap_clear(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)4452 void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4453 				unsigned long start, unsigned long pos,
4454 				unsigned long len)
4455 {
4456 	unsigned int first_byte = start + BIT_BYTE(pos);
4457 	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4458 	const bool same_byte = (first_byte == last_byte);
4459 	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4460 	u8 *kaddr;
4461 
4462 	if (same_byte)
4463 		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4464 
4465 	/* Handle the first byte. */
4466 	kaddr = extent_buffer_get_byte(eb, first_byte);
4467 	*kaddr &= ~mask;
4468 	if (same_byte)
4469 		return;
4470 
4471 	/* Handle the byte aligned part. */
4472 	ASSERT(first_byte + 1 <= last_byte);
4473 	memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1);
4474 
4475 	/* Handle the last byte. */
4476 	kaddr = extent_buffer_get_byte(eb, last_byte);
4477 	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4478 }
4479 
areas_overlap(unsigned long src,unsigned long dst,unsigned long len)4480 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4481 {
4482 	unsigned long distance = (src > dst) ? src - dst : dst - src;
4483 	return distance < len;
4484 }
4485 
memcpy_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4486 void memcpy_extent_buffer(const struct extent_buffer *dst,
4487 			  unsigned long dst_offset, unsigned long src_offset,
4488 			  unsigned long len)
4489 {
4490 	unsigned long cur_off = 0;
4491 
4492 	if (check_eb_range(dst, dst_offset, len) ||
4493 	    check_eb_range(dst, src_offset, len))
4494 		return;
4495 
4496 	while (cur_off < len) {
4497 		unsigned long cur_src = cur_off + src_offset;
4498 		unsigned long pg_index = get_eb_page_index(cur_src);
4499 		unsigned long pg_off = get_eb_offset_in_page(dst, cur_src);
4500 		unsigned long cur_len = min(src_offset + len - cur_src,
4501 					    PAGE_SIZE - pg_off);
4502 		void *src_addr = page_address(dst->pages[pg_index]) + pg_off;
4503 		const bool use_memmove = areas_overlap(src_offset + cur_off,
4504 						       dst_offset + cur_off, cur_len);
4505 
4506 		__write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len,
4507 				      use_memmove);
4508 		cur_off += cur_len;
4509 	}
4510 }
4511 
memmove_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4512 void memmove_extent_buffer(const struct extent_buffer *dst,
4513 			   unsigned long dst_offset, unsigned long src_offset,
4514 			   unsigned long len)
4515 {
4516 	unsigned long dst_end = dst_offset + len - 1;
4517 	unsigned long src_end = src_offset + len - 1;
4518 
4519 	if (check_eb_range(dst, dst_offset, len) ||
4520 	    check_eb_range(dst, src_offset, len))
4521 		return;
4522 
4523 	if (dst_offset < src_offset) {
4524 		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4525 		return;
4526 	}
4527 
4528 	while (len > 0) {
4529 		unsigned long src_i;
4530 		size_t cur;
4531 		size_t dst_off_in_page;
4532 		size_t src_off_in_page;
4533 		void *src_addr;
4534 		bool use_memmove;
4535 
4536 		src_i = get_eb_page_index(src_end);
4537 
4538 		dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
4539 		src_off_in_page = get_eb_offset_in_page(dst, src_end);
4540 
4541 		cur = min_t(unsigned long, len, src_off_in_page + 1);
4542 		cur = min(cur, dst_off_in_page + 1);
4543 
4544 		src_addr = page_address(dst->pages[src_i]) + src_off_in_page -
4545 					cur + 1;
4546 		use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1,
4547 					    cur);
4548 
4549 		__write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur,
4550 				      use_memmove);
4551 
4552 		dst_end -= cur;
4553 		src_end -= cur;
4554 		len -= cur;
4555 	}
4556 }
4557 
4558 #define GANG_LOOKUP_SIZE	16
get_next_extent_buffer(struct btrfs_fs_info * fs_info,struct page * page,u64 bytenr)4559 static struct extent_buffer *get_next_extent_buffer(
4560 		struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4561 {
4562 	struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4563 	struct extent_buffer *found = NULL;
4564 	u64 page_start = page_offset(page);
4565 	u64 cur = page_start;
4566 
4567 	ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
4568 	lockdep_assert_held(&fs_info->buffer_lock);
4569 
4570 	while (cur < page_start + PAGE_SIZE) {
4571 		int ret;
4572 		int i;
4573 
4574 		ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4575 				(void **)gang, cur >> fs_info->sectorsize_bits,
4576 				min_t(unsigned int, GANG_LOOKUP_SIZE,
4577 				      PAGE_SIZE / fs_info->nodesize));
4578 		if (ret == 0)
4579 			goto out;
4580 		for (i = 0; i < ret; i++) {
4581 			/* Already beyond page end */
4582 			if (gang[i]->start >= page_start + PAGE_SIZE)
4583 				goto out;
4584 			/* Found one */
4585 			if (gang[i]->start >= bytenr) {
4586 				found = gang[i];
4587 				goto out;
4588 			}
4589 		}
4590 		cur = gang[ret - 1]->start + gang[ret - 1]->len;
4591 	}
4592 out:
4593 	return found;
4594 }
4595 
try_release_subpage_extent_buffer(struct page * page)4596 static int try_release_subpage_extent_buffer(struct page *page)
4597 {
4598 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
4599 	u64 cur = page_offset(page);
4600 	const u64 end = page_offset(page) + PAGE_SIZE;
4601 	int ret;
4602 
4603 	while (cur < end) {
4604 		struct extent_buffer *eb = NULL;
4605 
4606 		/*
4607 		 * Unlike try_release_extent_buffer() which uses page->private
4608 		 * to grab buffer, for subpage case we rely on radix tree, thus
4609 		 * we need to ensure radix tree consistency.
4610 		 *
4611 		 * We also want an atomic snapshot of the radix tree, thus go
4612 		 * with spinlock rather than RCU.
4613 		 */
4614 		spin_lock(&fs_info->buffer_lock);
4615 		eb = get_next_extent_buffer(fs_info, page, cur);
4616 		if (!eb) {
4617 			/* No more eb in the page range after or at cur */
4618 			spin_unlock(&fs_info->buffer_lock);
4619 			break;
4620 		}
4621 		cur = eb->start + eb->len;
4622 
4623 		/*
4624 		 * The same as try_release_extent_buffer(), to ensure the eb
4625 		 * won't disappear out from under us.
4626 		 */
4627 		spin_lock(&eb->refs_lock);
4628 		if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4629 			spin_unlock(&eb->refs_lock);
4630 			spin_unlock(&fs_info->buffer_lock);
4631 			break;
4632 		}
4633 		spin_unlock(&fs_info->buffer_lock);
4634 
4635 		/*
4636 		 * If tree ref isn't set then we know the ref on this eb is a
4637 		 * real ref, so just return, this eb will likely be freed soon
4638 		 * anyway.
4639 		 */
4640 		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4641 			spin_unlock(&eb->refs_lock);
4642 			break;
4643 		}
4644 
4645 		/*
4646 		 * Here we don't care about the return value, we will always
4647 		 * check the page private at the end.  And
4648 		 * release_extent_buffer() will release the refs_lock.
4649 		 */
4650 		release_extent_buffer(eb);
4651 	}
4652 	/*
4653 	 * Finally to check if we have cleared page private, as if we have
4654 	 * released all ebs in the page, the page private should be cleared now.
4655 	 */
4656 	spin_lock(&page->mapping->private_lock);
4657 	if (!PagePrivate(page))
4658 		ret = 1;
4659 	else
4660 		ret = 0;
4661 	spin_unlock(&page->mapping->private_lock);
4662 	return ret;
4663 
4664 }
4665 
try_release_extent_buffer(struct page * page)4666 int try_release_extent_buffer(struct page *page)
4667 {
4668 	struct extent_buffer *eb;
4669 
4670 	if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
4671 		return try_release_subpage_extent_buffer(page);
4672 
4673 	/*
4674 	 * We need to make sure nobody is changing page->private, as we rely on
4675 	 * page->private as the pointer to extent buffer.
4676 	 */
4677 	spin_lock(&page->mapping->private_lock);
4678 	if (!PagePrivate(page)) {
4679 		spin_unlock(&page->mapping->private_lock);
4680 		return 1;
4681 	}
4682 
4683 	eb = (struct extent_buffer *)page->private;
4684 	BUG_ON(!eb);
4685 
4686 	/*
4687 	 * This is a little awful but should be ok, we need to make sure that
4688 	 * the eb doesn't disappear out from under us while we're looking at
4689 	 * this page.
4690 	 */
4691 	spin_lock(&eb->refs_lock);
4692 	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4693 		spin_unlock(&eb->refs_lock);
4694 		spin_unlock(&page->mapping->private_lock);
4695 		return 0;
4696 	}
4697 	spin_unlock(&page->mapping->private_lock);
4698 
4699 	/*
4700 	 * If tree ref isn't set then we know the ref on this eb is a real ref,
4701 	 * so just return, this page will likely be freed soon anyway.
4702 	 */
4703 	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4704 		spin_unlock(&eb->refs_lock);
4705 		return 0;
4706 	}
4707 
4708 	return release_extent_buffer(eb);
4709 }
4710 
4711 /*
4712  * btrfs_readahead_tree_block - attempt to readahead a child block
4713  * @fs_info:	the fs_info
4714  * @bytenr:	bytenr to read
4715  * @owner_root: objectid of the root that owns this eb
4716  * @gen:	generation for the uptodate check, can be 0
4717  * @level:	level for the eb
4718  *
4719  * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
4720  * normal uptodate check of the eb, without checking the generation.  If we have
4721  * to read the block we will not block on anything.
4722  */
btrfs_readahead_tree_block(struct btrfs_fs_info * fs_info,u64 bytenr,u64 owner_root,u64 gen,int level)4723 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4724 				u64 bytenr, u64 owner_root, u64 gen, int level)
4725 {
4726 	struct btrfs_tree_parent_check check = {
4727 		.has_first_key = 0,
4728 		.level = level,
4729 		.transid = gen
4730 	};
4731 	struct extent_buffer *eb;
4732 	int ret;
4733 
4734 	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4735 	if (IS_ERR(eb))
4736 		return;
4737 
4738 	if (btrfs_buffer_uptodate(eb, gen, 1)) {
4739 		free_extent_buffer(eb);
4740 		return;
4741 	}
4742 
4743 	ret = read_extent_buffer_pages(eb, WAIT_NONE, 0, &check);
4744 	if (ret < 0)
4745 		free_extent_buffer_stale(eb);
4746 	else
4747 		free_extent_buffer(eb);
4748 }
4749 
4750 /*
4751  * btrfs_readahead_node_child - readahead a node's child block
4752  * @node:	parent node we're reading from
4753  * @slot:	slot in the parent node for the child we want to read
4754  *
4755  * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4756  * the slot in the node provided.
4757  */
btrfs_readahead_node_child(struct extent_buffer * node,int slot)4758 void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4759 {
4760 	btrfs_readahead_tree_block(node->fs_info,
4761 				   btrfs_node_blockptr(node, slot),
4762 				   btrfs_header_owner(node),
4763 				   btrfs_node_ptr_generation(node, slot),
4764 				   btrfs_header_level(node) - 1);
4765 }
4766