1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *	linux/mm/filemap.c
4  *
5  * Copyright (C) 1994-1999  Linus Torvalds
6  */
7 
8 /*
9  * This file handles the generic file mmap semantics used by
10  * most "normal" filesystems (but you don't /have/ to use this:
11  * the NFS filesystem used to do this differently, for example)
12  */
13 #include <linux/export.h>
14 #include <linux/compiler.h>
15 #include <linux/dax.h>
16 #include <linux/fs.h>
17 #include <linux/sched/signal.h>
18 #include <linux/uaccess.h>
19 #include <linux/capability.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/gfp.h>
22 #include <linux/mm.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/mman.h>
26 #include <linux/pagemap.h>
27 #include <linux/file.h>
28 #include <linux/uio.h>
29 #include <linux/error-injection.h>
30 #include <linux/hash.h>
31 #include <linux/writeback.h>
32 #include <linux/backing-dev.h>
33 #include <linux/pagevec.h>
34 #include <linux/security.h>
35 #include <linux/cpuset.h>
36 #include <linux/hugetlb.h>
37 #include <linux/memcontrol.h>
38 #include <linux/shmem_fs.h>
39 #include <linux/rmap.h>
40 #include <linux/delayacct.h>
41 #include <linux/psi.h>
42 #include <linux/ramfs.h>
43 #include <linux/page_idle.h>
44 #include <linux/migrate.h>
45 #include <asm/pgalloc.h>
46 #include <asm/tlbflush.h>
47 #include "internal.h"
48 
49 #define CREATE_TRACE_POINTS
50 #include <trace/events/filemap.h>
51 
52 /*
53  * FIXME: remove all knowledge of the buffer layer from the core VM
54  */
55 #include <linux/buffer_head.h> /* for try_to_free_buffers */
56 
57 #include <asm/mman.h>
58 
59 /*
60  * Shared mappings implemented 30.11.1994. It's not fully working yet,
61  * though.
62  *
63  * Shared mappings now work. 15.8.1995  Bruno.
64  *
65  * finished 'unifying' the page and buffer cache and SMP-threaded the
66  * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
67  *
68  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
69  */
70 
71 /*
72  * Lock ordering:
73  *
74  *  ->i_mmap_rwsem		(truncate_pagecache)
75  *    ->private_lock		(__free_pte->block_dirty_folio)
76  *      ->swap_lock		(exclusive_swap_page, others)
77  *        ->i_pages lock
78  *
79  *  ->i_rwsem
80  *    ->invalidate_lock		(acquired by fs in truncate path)
81  *      ->i_mmap_rwsem		(truncate->unmap_mapping_range)
82  *
83  *  ->mmap_lock
84  *    ->i_mmap_rwsem
85  *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
86  *        ->i_pages lock	(arch-dependent flush_dcache_mmap_lock)
87  *
88  *  ->mmap_lock
89  *    ->invalidate_lock		(filemap_fault)
90  *      ->lock_page		(filemap_fault, access_process_vm)
91  *
92  *  ->i_rwsem			(generic_perform_write)
93  *    ->mmap_lock		(fault_in_readable->do_page_fault)
94  *
95  *  bdi->wb.list_lock
96  *    sb_lock			(fs/fs-writeback.c)
97  *    ->i_pages lock		(__sync_single_inode)
98  *
99  *  ->i_mmap_rwsem
100  *    ->anon_vma.lock		(vma_adjust)
101  *
102  *  ->anon_vma.lock
103  *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
104  *
105  *  ->page_table_lock or pte_lock
106  *    ->swap_lock		(try_to_unmap_one)
107  *    ->private_lock		(try_to_unmap_one)
108  *    ->i_pages lock		(try_to_unmap_one)
109  *    ->lruvec->lru_lock	(follow_page->mark_page_accessed)
110  *    ->lruvec->lru_lock	(check_pte_range->isolate_lru_page)
111  *    ->private_lock		(page_remove_rmap->set_page_dirty)
112  *    ->i_pages lock		(page_remove_rmap->set_page_dirty)
113  *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
114  *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
115  *    ->memcg->move_lock	(page_remove_rmap->lock_page_memcg)
116  *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
117  *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
118  *    ->private_lock		(zap_pte_range->block_dirty_folio)
119  *
120  * ->i_mmap_rwsem
121  *   ->tasklist_lock            (memory_failure, collect_procs_ao)
122  */
123 
page_cache_delete(struct address_space * mapping,struct folio * folio,void * shadow)124 static void page_cache_delete(struct address_space *mapping,
125 				   struct folio *folio, void *shadow)
126 {
127 	XA_STATE(xas, &mapping->i_pages, folio->index);
128 	long nr = 1;
129 
130 	mapping_set_update(&xas, mapping);
131 
132 	/* hugetlb pages are represented by a single entry in the xarray */
133 	if (!folio_test_hugetlb(folio)) {
134 		xas_set_order(&xas, folio->index, folio_order(folio));
135 		nr = folio_nr_pages(folio);
136 	}
137 
138 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
139 
140 	xas_store(&xas, shadow);
141 	xas_init_marks(&xas);
142 
143 	folio->mapping = NULL;
144 	/* Leave page->index set: truncation lookup relies upon it */
145 	mapping->nrpages -= nr;
146 }
147 
filemap_unaccount_folio(struct address_space * mapping,struct folio * folio)148 static void filemap_unaccount_folio(struct address_space *mapping,
149 		struct folio *folio)
150 {
151 	long nr;
152 
153 	VM_BUG_ON_FOLIO(folio_mapped(folio), folio);
154 	if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) {
155 		pr_alert("BUG: Bad page cache in process %s  pfn:%05lx\n",
156 			 current->comm, folio_pfn(folio));
157 		dump_page(&folio->page, "still mapped when deleted");
158 		dump_stack();
159 		add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
160 
161 		if (mapping_exiting(mapping) && !folio_test_large(folio)) {
162 			int mapcount = page_mapcount(&folio->page);
163 
164 			if (folio_ref_count(folio) >= mapcount + 2) {
165 				/*
166 				 * All vmas have already been torn down, so it's
167 				 * a good bet that actually the page is unmapped
168 				 * and we'd rather not leak it: if we're wrong,
169 				 * another bad page check should catch it later.
170 				 */
171 				page_mapcount_reset(&folio->page);
172 				folio_ref_sub(folio, mapcount);
173 			}
174 		}
175 	}
176 
177 	/* hugetlb folios do not participate in page cache accounting. */
178 	if (folio_test_hugetlb(folio))
179 		return;
180 
181 	nr = folio_nr_pages(folio);
182 
183 	__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
184 	if (folio_test_swapbacked(folio)) {
185 		__lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
186 		if (folio_test_pmd_mappable(folio))
187 			__lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr);
188 	} else if (folio_test_pmd_mappable(folio)) {
189 		__lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr);
190 		filemap_nr_thps_dec(mapping);
191 	}
192 
193 	/*
194 	 * At this point folio must be either written or cleaned by
195 	 * truncate.  Dirty folio here signals a bug and loss of
196 	 * unwritten data - on ordinary filesystems.
197 	 *
198 	 * But it's harmless on in-memory filesystems like tmpfs; and can
199 	 * occur when a driver which did get_user_pages() sets page dirty
200 	 * before putting it, while the inode is being finally evicted.
201 	 *
202 	 * Below fixes dirty accounting after removing the folio entirely
203 	 * but leaves the dirty flag set: it has no effect for truncated
204 	 * folio and anyway will be cleared before returning folio to
205 	 * buddy allocator.
206 	 */
207 	if (WARN_ON_ONCE(folio_test_dirty(folio) &&
208 			 mapping_can_writeback(mapping)))
209 		folio_account_cleaned(folio, inode_to_wb(mapping->host));
210 }
211 
212 /*
213  * Delete a page from the page cache and free it. Caller has to make
214  * sure the page is locked and that nobody else uses it - or that usage
215  * is safe.  The caller must hold the i_pages lock.
216  */
__filemap_remove_folio(struct folio * folio,void * shadow)217 void __filemap_remove_folio(struct folio *folio, void *shadow)
218 {
219 	struct address_space *mapping = folio->mapping;
220 
221 	trace_mm_filemap_delete_from_page_cache(folio);
222 	filemap_unaccount_folio(mapping, folio);
223 	page_cache_delete(mapping, folio, shadow);
224 }
225 
filemap_free_folio(struct address_space * mapping,struct folio * folio)226 void filemap_free_folio(struct address_space *mapping, struct folio *folio)
227 {
228 	void (*free_folio)(struct folio *);
229 	int refs = 1;
230 
231 	free_folio = mapping->a_ops->free_folio;
232 	if (free_folio)
233 		free_folio(folio);
234 
235 	if (folio_test_large(folio) && !folio_test_hugetlb(folio))
236 		refs = folio_nr_pages(folio);
237 	folio_put_refs(folio, refs);
238 }
239 
240 /**
241  * filemap_remove_folio - Remove folio from page cache.
242  * @folio: The folio.
243  *
244  * This must be called only on folios that are locked and have been
245  * verified to be in the page cache.  It will never put the folio into
246  * the free list because the caller has a reference on the page.
247  */
filemap_remove_folio(struct folio * folio)248 void filemap_remove_folio(struct folio *folio)
249 {
250 	struct address_space *mapping = folio->mapping;
251 
252 	BUG_ON(!folio_test_locked(folio));
253 	spin_lock(&mapping->host->i_lock);
254 	xa_lock_irq(&mapping->i_pages);
255 	__filemap_remove_folio(folio, NULL);
256 	xa_unlock_irq(&mapping->i_pages);
257 	if (mapping_shrinkable(mapping))
258 		inode_add_lru(mapping->host);
259 	spin_unlock(&mapping->host->i_lock);
260 
261 	filemap_free_folio(mapping, folio);
262 }
263 
264 /*
265  * page_cache_delete_batch - delete several folios from page cache
266  * @mapping: the mapping to which folios belong
267  * @fbatch: batch of folios to delete
268  *
269  * The function walks over mapping->i_pages and removes folios passed in
270  * @fbatch from the mapping. The function expects @fbatch to be sorted
271  * by page index and is optimised for it to be dense.
272  * It tolerates holes in @fbatch (mapping entries at those indices are not
273  * modified).
274  *
275  * The function expects the i_pages lock to be held.
276  */
page_cache_delete_batch(struct address_space * mapping,struct folio_batch * fbatch)277 static void page_cache_delete_batch(struct address_space *mapping,
278 			     struct folio_batch *fbatch)
279 {
280 	XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);
281 	long total_pages = 0;
282 	int i = 0;
283 	struct folio *folio;
284 
285 	mapping_set_update(&xas, mapping);
286 	xas_for_each(&xas, folio, ULONG_MAX) {
287 		if (i >= folio_batch_count(fbatch))
288 			break;
289 
290 		/* A swap/dax/shadow entry got inserted? Skip it. */
291 		if (xa_is_value(folio))
292 			continue;
293 		/*
294 		 * A page got inserted in our range? Skip it. We have our
295 		 * pages locked so they are protected from being removed.
296 		 * If we see a page whose index is higher than ours, it
297 		 * means our page has been removed, which shouldn't be
298 		 * possible because we're holding the PageLock.
299 		 */
300 		if (folio != fbatch->folios[i]) {
301 			VM_BUG_ON_FOLIO(folio->index >
302 					fbatch->folios[i]->index, folio);
303 			continue;
304 		}
305 
306 		WARN_ON_ONCE(!folio_test_locked(folio));
307 
308 		folio->mapping = NULL;
309 		/* Leave folio->index set: truncation lookup relies on it */
310 
311 		i++;
312 		xas_store(&xas, NULL);
313 		total_pages += folio_nr_pages(folio);
314 	}
315 	mapping->nrpages -= total_pages;
316 }
317 
delete_from_page_cache_batch(struct address_space * mapping,struct folio_batch * fbatch)318 void delete_from_page_cache_batch(struct address_space *mapping,
319 				  struct folio_batch *fbatch)
320 {
321 	int i;
322 
323 	if (!folio_batch_count(fbatch))
324 		return;
325 
326 	spin_lock(&mapping->host->i_lock);
327 	xa_lock_irq(&mapping->i_pages);
328 	for (i = 0; i < folio_batch_count(fbatch); i++) {
329 		struct folio *folio = fbatch->folios[i];
330 
331 		trace_mm_filemap_delete_from_page_cache(folio);
332 		filemap_unaccount_folio(mapping, folio);
333 	}
334 	page_cache_delete_batch(mapping, fbatch);
335 	xa_unlock_irq(&mapping->i_pages);
336 	if (mapping_shrinkable(mapping))
337 		inode_add_lru(mapping->host);
338 	spin_unlock(&mapping->host->i_lock);
339 
340 	for (i = 0; i < folio_batch_count(fbatch); i++)
341 		filemap_free_folio(mapping, fbatch->folios[i]);
342 }
343 
filemap_check_errors(struct address_space * mapping)344 int filemap_check_errors(struct address_space *mapping)
345 {
346 	int ret = 0;
347 	/* Check for outstanding write errors */
348 	if (test_bit(AS_ENOSPC, &mapping->flags) &&
349 	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))
350 		ret = -ENOSPC;
351 	if (test_bit(AS_EIO, &mapping->flags) &&
352 	    test_and_clear_bit(AS_EIO, &mapping->flags))
353 		ret = -EIO;
354 	return ret;
355 }
356 EXPORT_SYMBOL(filemap_check_errors);
357 
filemap_check_and_keep_errors(struct address_space * mapping)358 static int filemap_check_and_keep_errors(struct address_space *mapping)
359 {
360 	/* Check for outstanding write errors */
361 	if (test_bit(AS_EIO, &mapping->flags))
362 		return -EIO;
363 	if (test_bit(AS_ENOSPC, &mapping->flags))
364 		return -ENOSPC;
365 	return 0;
366 }
367 
368 /**
369  * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
370  * @mapping:	address space structure to write
371  * @wbc:	the writeback_control controlling the writeout
372  *
373  * Call writepages on the mapping using the provided wbc to control the
374  * writeout.
375  *
376  * Return: %0 on success, negative error code otherwise.
377  */
filemap_fdatawrite_wbc(struct address_space * mapping,struct writeback_control * wbc)378 int filemap_fdatawrite_wbc(struct address_space *mapping,
379 			   struct writeback_control *wbc)
380 {
381 	int ret;
382 
383 	if (!mapping_can_writeback(mapping) ||
384 	    !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
385 		return 0;
386 
387 	wbc_attach_fdatawrite_inode(wbc, mapping->host);
388 	ret = do_writepages(mapping, wbc);
389 	wbc_detach_inode(wbc);
390 	return ret;
391 }
392 EXPORT_SYMBOL(filemap_fdatawrite_wbc);
393 
394 /**
395  * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
396  * @mapping:	address space structure to write
397  * @start:	offset in bytes where the range starts
398  * @end:	offset in bytes where the range ends (inclusive)
399  * @sync_mode:	enable synchronous operation
400  *
401  * Start writeback against all of a mapping's dirty pages that lie
402  * within the byte offsets <start, end> inclusive.
403  *
404  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
405  * opposed to a regular memory cleansing writeback.  The difference between
406  * these two operations is that if a dirty page/buffer is encountered, it must
407  * be waited upon, and not just skipped over.
408  *
409  * Return: %0 on success, negative error code otherwise.
410  */
__filemap_fdatawrite_range(struct address_space * mapping,loff_t start,loff_t end,int sync_mode)411 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
412 				loff_t end, int sync_mode)
413 {
414 	struct writeback_control wbc = {
415 		.sync_mode = sync_mode,
416 		.nr_to_write = LONG_MAX,
417 		.range_start = start,
418 		.range_end = end,
419 	};
420 
421 	return filemap_fdatawrite_wbc(mapping, &wbc);
422 }
423 
__filemap_fdatawrite(struct address_space * mapping,int sync_mode)424 static inline int __filemap_fdatawrite(struct address_space *mapping,
425 	int sync_mode)
426 {
427 	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
428 }
429 
filemap_fdatawrite(struct address_space * mapping)430 int filemap_fdatawrite(struct address_space *mapping)
431 {
432 	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
433 }
434 EXPORT_SYMBOL(filemap_fdatawrite);
435 
filemap_fdatawrite_range(struct address_space * mapping,loff_t start,loff_t end)436 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
437 				loff_t end)
438 {
439 	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
440 }
441 EXPORT_SYMBOL(filemap_fdatawrite_range);
442 
443 /**
444  * filemap_flush - mostly a non-blocking flush
445  * @mapping:	target address_space
446  *
447  * This is a mostly non-blocking flush.  Not suitable for data-integrity
448  * purposes - I/O may not be started against all dirty pages.
449  *
450  * Return: %0 on success, negative error code otherwise.
451  */
filemap_flush(struct address_space * mapping)452 int filemap_flush(struct address_space *mapping)
453 {
454 	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
455 }
456 EXPORT_SYMBOL(filemap_flush);
457 
458 /**
459  * filemap_range_has_page - check if a page exists in range.
460  * @mapping:           address space within which to check
461  * @start_byte:        offset in bytes where the range starts
462  * @end_byte:          offset in bytes where the range ends (inclusive)
463  *
464  * Find at least one page in the range supplied, usually used to check if
465  * direct writing in this range will trigger a writeback.
466  *
467  * Return: %true if at least one page exists in the specified range,
468  * %false otherwise.
469  */
filemap_range_has_page(struct address_space * mapping,loff_t start_byte,loff_t end_byte)470 bool filemap_range_has_page(struct address_space *mapping,
471 			   loff_t start_byte, loff_t end_byte)
472 {
473 	struct page *page;
474 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
475 	pgoff_t max = end_byte >> PAGE_SHIFT;
476 
477 	if (end_byte < start_byte)
478 		return false;
479 
480 	rcu_read_lock();
481 	for (;;) {
482 		page = xas_find(&xas, max);
483 		if (xas_retry(&xas, page))
484 			continue;
485 		/* Shadow entries don't count */
486 		if (xa_is_value(page))
487 			continue;
488 		/*
489 		 * We don't need to try to pin this page; we're about to
490 		 * release the RCU lock anyway.  It is enough to know that
491 		 * there was a page here recently.
492 		 */
493 		break;
494 	}
495 	rcu_read_unlock();
496 
497 	return page != NULL;
498 }
499 EXPORT_SYMBOL(filemap_range_has_page);
500 
__filemap_fdatawait_range(struct address_space * mapping,loff_t start_byte,loff_t end_byte)501 static void __filemap_fdatawait_range(struct address_space *mapping,
502 				     loff_t start_byte, loff_t end_byte)
503 {
504 	pgoff_t index = start_byte >> PAGE_SHIFT;
505 	pgoff_t end = end_byte >> PAGE_SHIFT;
506 	struct pagevec pvec;
507 	int nr_pages;
508 
509 	if (end_byte < start_byte)
510 		return;
511 
512 	pagevec_init(&pvec);
513 	while (index <= end) {
514 		unsigned i;
515 
516 		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
517 				end, PAGECACHE_TAG_WRITEBACK);
518 		if (!nr_pages)
519 			break;
520 
521 		for (i = 0; i < nr_pages; i++) {
522 			struct page *page = pvec.pages[i];
523 
524 			wait_on_page_writeback(page);
525 			ClearPageError(page);
526 		}
527 		pagevec_release(&pvec);
528 		cond_resched();
529 	}
530 }
531 
532 /**
533  * filemap_fdatawait_range - wait for writeback to complete
534  * @mapping:		address space structure to wait for
535  * @start_byte:		offset in bytes where the range starts
536  * @end_byte:		offset in bytes where the range ends (inclusive)
537  *
538  * Walk the list of under-writeback pages of the given address space
539  * in the given range and wait for all of them.  Check error status of
540  * the address space and return it.
541  *
542  * Since the error status of the address space is cleared by this function,
543  * callers are responsible for checking the return value and handling and/or
544  * reporting the error.
545  *
546  * Return: error status of the address space.
547  */
filemap_fdatawait_range(struct address_space * mapping,loff_t start_byte,loff_t end_byte)548 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
549 			    loff_t end_byte)
550 {
551 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
552 	return filemap_check_errors(mapping);
553 }
554 EXPORT_SYMBOL(filemap_fdatawait_range);
555 
556 /**
557  * filemap_fdatawait_range_keep_errors - wait for writeback to complete
558  * @mapping:		address space structure to wait for
559  * @start_byte:		offset in bytes where the range starts
560  * @end_byte:		offset in bytes where the range ends (inclusive)
561  *
562  * Walk the list of under-writeback pages of the given address space in the
563  * given range and wait for all of them.  Unlike filemap_fdatawait_range(),
564  * this function does not clear error status of the address space.
565  *
566  * Use this function if callers don't handle errors themselves.  Expected
567  * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
568  * fsfreeze(8)
569  */
filemap_fdatawait_range_keep_errors(struct address_space * mapping,loff_t start_byte,loff_t end_byte)570 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
571 		loff_t start_byte, loff_t end_byte)
572 {
573 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
574 	return filemap_check_and_keep_errors(mapping);
575 }
576 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors);
577 
578 /**
579  * file_fdatawait_range - wait for writeback to complete
580  * @file:		file pointing to address space structure to wait for
581  * @start_byte:		offset in bytes where the range starts
582  * @end_byte:		offset in bytes where the range ends (inclusive)
583  *
584  * Walk the list of under-writeback pages of the address space that file
585  * refers to, in the given range and wait for all of them.  Check error
586  * status of the address space vs. the file->f_wb_err cursor and return it.
587  *
588  * Since the error status of the file is advanced by this function,
589  * callers are responsible for checking the return value and handling and/or
590  * reporting the error.
591  *
592  * Return: error status of the address space vs. the file->f_wb_err cursor.
593  */
file_fdatawait_range(struct file * file,loff_t start_byte,loff_t end_byte)594 int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
595 {
596 	struct address_space *mapping = file->f_mapping;
597 
598 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
599 	return file_check_and_advance_wb_err(file);
600 }
601 EXPORT_SYMBOL(file_fdatawait_range);
602 
603 /**
604  * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
605  * @mapping: address space structure to wait for
606  *
607  * Walk the list of under-writeback pages of the given address space
608  * and wait for all of them.  Unlike filemap_fdatawait(), this function
609  * does not clear error status of the address space.
610  *
611  * Use this function if callers don't handle errors themselves.  Expected
612  * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
613  * fsfreeze(8)
614  *
615  * Return: error status of the address space.
616  */
filemap_fdatawait_keep_errors(struct address_space * mapping)617 int filemap_fdatawait_keep_errors(struct address_space *mapping)
618 {
619 	__filemap_fdatawait_range(mapping, 0, LLONG_MAX);
620 	return filemap_check_and_keep_errors(mapping);
621 }
622 EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
623 
624 /* Returns true if writeback might be needed or already in progress. */
mapping_needs_writeback(struct address_space * mapping)625 static bool mapping_needs_writeback(struct address_space *mapping)
626 {
627 	return mapping->nrpages;
628 }
629 
filemap_range_has_writeback(struct address_space * mapping,loff_t start_byte,loff_t end_byte)630 bool filemap_range_has_writeback(struct address_space *mapping,
631 				 loff_t start_byte, loff_t end_byte)
632 {
633 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
634 	pgoff_t max = end_byte >> PAGE_SHIFT;
635 	struct folio *folio;
636 
637 	if (end_byte < start_byte)
638 		return false;
639 
640 	rcu_read_lock();
641 	xas_for_each(&xas, folio, max) {
642 		if (xas_retry(&xas, folio))
643 			continue;
644 		if (xa_is_value(folio))
645 			continue;
646 		if (folio_test_dirty(folio) || folio_test_locked(folio) ||
647 				folio_test_writeback(folio))
648 			break;
649 	}
650 	rcu_read_unlock();
651 	return folio != NULL;
652 }
653 EXPORT_SYMBOL_GPL(filemap_range_has_writeback);
654 
655 /**
656  * filemap_write_and_wait_range - write out & wait on a file range
657  * @mapping:	the address_space for the pages
658  * @lstart:	offset in bytes where the range starts
659  * @lend:	offset in bytes where the range ends (inclusive)
660  *
661  * Write out and wait upon file offsets lstart->lend, inclusive.
662  *
663  * Note that @lend is inclusive (describes the last byte to be written) so
664  * that this function can be used to write to the very end-of-file (end = -1).
665  *
666  * Return: error status of the address space.
667  */
filemap_write_and_wait_range(struct address_space * mapping,loff_t lstart,loff_t lend)668 int filemap_write_and_wait_range(struct address_space *mapping,
669 				 loff_t lstart, loff_t lend)
670 {
671 	int err = 0, err2;
672 
673 	if (mapping_needs_writeback(mapping)) {
674 		err = __filemap_fdatawrite_range(mapping, lstart, lend,
675 						 WB_SYNC_ALL);
676 		/*
677 		 * Even if the above returned error, the pages may be
678 		 * written partially (e.g. -ENOSPC), so we wait for it.
679 		 * But the -EIO is special case, it may indicate the worst
680 		 * thing (e.g. bug) happened, so we avoid waiting for it.
681 		 */
682 		if (err != -EIO)
683 			__filemap_fdatawait_range(mapping, lstart, lend);
684 	}
685 	err2 = filemap_check_errors(mapping);
686 	if (!err)
687 		err = err2;
688 	return err;
689 }
690 EXPORT_SYMBOL(filemap_write_and_wait_range);
691 
__filemap_set_wb_err(struct address_space * mapping,int err)692 void __filemap_set_wb_err(struct address_space *mapping, int err)
693 {
694 	errseq_t eseq = errseq_set(&mapping->wb_err, err);
695 
696 	trace_filemap_set_wb_err(mapping, eseq);
697 }
698 EXPORT_SYMBOL(__filemap_set_wb_err);
699 
700 /**
701  * file_check_and_advance_wb_err - report wb error (if any) that was previously
702  * 				   and advance wb_err to current one
703  * @file: struct file on which the error is being reported
704  *
705  * When userland calls fsync (or something like nfsd does the equivalent), we
706  * want to report any writeback errors that occurred since the last fsync (or
707  * since the file was opened if there haven't been any).
708  *
709  * Grab the wb_err from the mapping. If it matches what we have in the file,
710  * then just quickly return 0. The file is all caught up.
711  *
712  * If it doesn't match, then take the mapping value, set the "seen" flag in
713  * it and try to swap it into place. If it works, or another task beat us
714  * to it with the new value, then update the f_wb_err and return the error
715  * portion. The error at this point must be reported via proper channels
716  * (a'la fsync, or NFS COMMIT operation, etc.).
717  *
718  * While we handle mapping->wb_err with atomic operations, the f_wb_err
719  * value is protected by the f_lock since we must ensure that it reflects
720  * the latest value swapped in for this file descriptor.
721  *
722  * Return: %0 on success, negative error code otherwise.
723  */
file_check_and_advance_wb_err(struct file * file)724 int file_check_and_advance_wb_err(struct file *file)
725 {
726 	int err = 0;
727 	errseq_t old = READ_ONCE(file->f_wb_err);
728 	struct address_space *mapping = file->f_mapping;
729 
730 	/* Locklessly handle the common case where nothing has changed */
731 	if (errseq_check(&mapping->wb_err, old)) {
732 		/* Something changed, must use slow path */
733 		spin_lock(&file->f_lock);
734 		old = file->f_wb_err;
735 		err = errseq_check_and_advance(&mapping->wb_err,
736 						&file->f_wb_err);
737 		trace_file_check_and_advance_wb_err(file, old);
738 		spin_unlock(&file->f_lock);
739 	}
740 
741 	/*
742 	 * We're mostly using this function as a drop in replacement for
743 	 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
744 	 * that the legacy code would have had on these flags.
745 	 */
746 	clear_bit(AS_EIO, &mapping->flags);
747 	clear_bit(AS_ENOSPC, &mapping->flags);
748 	return err;
749 }
750 EXPORT_SYMBOL(file_check_and_advance_wb_err);
751 
752 /**
753  * file_write_and_wait_range - write out & wait on a file range
754  * @file:	file pointing to address_space with pages
755  * @lstart:	offset in bytes where the range starts
756  * @lend:	offset in bytes where the range ends (inclusive)
757  *
758  * Write out and wait upon file offsets lstart->lend, inclusive.
759  *
760  * Note that @lend is inclusive (describes the last byte to be written) so
761  * that this function can be used to write to the very end-of-file (end = -1).
762  *
763  * After writing out and waiting on the data, we check and advance the
764  * f_wb_err cursor to the latest value, and return any errors detected there.
765  *
766  * Return: %0 on success, negative error code otherwise.
767  */
file_write_and_wait_range(struct file * file,loff_t lstart,loff_t lend)768 int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
769 {
770 	int err = 0, err2;
771 	struct address_space *mapping = file->f_mapping;
772 
773 	if (mapping_needs_writeback(mapping)) {
774 		err = __filemap_fdatawrite_range(mapping, lstart, lend,
775 						 WB_SYNC_ALL);
776 		/* See comment of filemap_write_and_wait() */
777 		if (err != -EIO)
778 			__filemap_fdatawait_range(mapping, lstart, lend);
779 	}
780 	err2 = file_check_and_advance_wb_err(file);
781 	if (!err)
782 		err = err2;
783 	return err;
784 }
785 EXPORT_SYMBOL(file_write_and_wait_range);
786 
787 /**
788  * replace_page_cache_page - replace a pagecache page with a new one
789  * @old:	page to be replaced
790  * @new:	page to replace with
791  *
792  * This function replaces a page in the pagecache with a new one.  On
793  * success it acquires the pagecache reference for the new page and
794  * drops it for the old page.  Both the old and new pages must be
795  * locked.  This function does not add the new page to the LRU, the
796  * caller must do that.
797  *
798  * The remove + add is atomic.  This function cannot fail.
799  */
replace_page_cache_page(struct page * old,struct page * new)800 void replace_page_cache_page(struct page *old, struct page *new)
801 {
802 	struct folio *fold = page_folio(old);
803 	struct folio *fnew = page_folio(new);
804 	struct address_space *mapping = old->mapping;
805 	void (*free_folio)(struct folio *) = mapping->a_ops->free_folio;
806 	pgoff_t offset = old->index;
807 	XA_STATE(xas, &mapping->i_pages, offset);
808 
809 	VM_BUG_ON_PAGE(!PageLocked(old), old);
810 	VM_BUG_ON_PAGE(!PageLocked(new), new);
811 	VM_BUG_ON_PAGE(new->mapping, new);
812 
813 	get_page(new);
814 	new->mapping = mapping;
815 	new->index = offset;
816 
817 	mem_cgroup_migrate(fold, fnew);
818 
819 	xas_lock_irq(&xas);
820 	xas_store(&xas, new);
821 
822 	old->mapping = NULL;
823 	/* hugetlb pages do not participate in page cache accounting. */
824 	if (!PageHuge(old))
825 		__dec_lruvec_page_state(old, NR_FILE_PAGES);
826 	if (!PageHuge(new))
827 		__inc_lruvec_page_state(new, NR_FILE_PAGES);
828 	if (PageSwapBacked(old))
829 		__dec_lruvec_page_state(old, NR_SHMEM);
830 	if (PageSwapBacked(new))
831 		__inc_lruvec_page_state(new, NR_SHMEM);
832 	xas_unlock_irq(&xas);
833 	if (free_folio)
834 		free_folio(fold);
835 	folio_put(fold);
836 }
837 EXPORT_SYMBOL_GPL(replace_page_cache_page);
838 
__filemap_add_folio(struct address_space * mapping,struct folio * folio,pgoff_t index,gfp_t gfp,void ** shadowp)839 noinline int __filemap_add_folio(struct address_space *mapping,
840 		struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp)
841 {
842 	XA_STATE(xas, &mapping->i_pages, index);
843 	int huge = folio_test_hugetlb(folio);
844 	bool charged = false;
845 	long nr = 1;
846 
847 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
848 	VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);
849 	mapping_set_update(&xas, mapping);
850 
851 	if (!huge) {
852 		int error = mem_cgroup_charge(folio, NULL, gfp);
853 		VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
854 		if (error)
855 			return error;
856 		charged = true;
857 		xas_set_order(&xas, index, folio_order(folio));
858 		nr = folio_nr_pages(folio);
859 	}
860 
861 	gfp &= GFP_RECLAIM_MASK;
862 	folio_ref_add(folio, nr);
863 	folio->mapping = mapping;
864 	folio->index = xas.xa_index;
865 
866 	do {
867 		unsigned int order = xa_get_order(xas.xa, xas.xa_index);
868 		void *entry, *old = NULL;
869 
870 		if (order > folio_order(folio))
871 			xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),
872 					order, gfp);
873 		xas_lock_irq(&xas);
874 		xas_for_each_conflict(&xas, entry) {
875 			old = entry;
876 			if (!xa_is_value(entry)) {
877 				xas_set_err(&xas, -EEXIST);
878 				goto unlock;
879 			}
880 		}
881 
882 		if (old) {
883 			if (shadowp)
884 				*shadowp = old;
885 			/* entry may have been split before we acquired lock */
886 			order = xa_get_order(xas.xa, xas.xa_index);
887 			if (order > folio_order(folio)) {
888 				/* How to handle large swap entries? */
889 				BUG_ON(shmem_mapping(mapping));
890 				xas_split(&xas, old, order);
891 				xas_reset(&xas);
892 			}
893 		}
894 
895 		xas_store(&xas, folio);
896 		if (xas_error(&xas))
897 			goto unlock;
898 
899 		mapping->nrpages += nr;
900 
901 		/* hugetlb pages do not participate in page cache accounting */
902 		if (!huge) {
903 			__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
904 			if (folio_test_pmd_mappable(folio))
905 				__lruvec_stat_mod_folio(folio,
906 						NR_FILE_THPS, nr);
907 		}
908 unlock:
909 		xas_unlock_irq(&xas);
910 	} while (xas_nomem(&xas, gfp));
911 
912 	if (xas_error(&xas))
913 		goto error;
914 
915 	trace_mm_filemap_add_to_page_cache(folio);
916 	return 0;
917 error:
918 	if (charged)
919 		mem_cgroup_uncharge(folio);
920 	folio->mapping = NULL;
921 	/* Leave page->index set: truncation relies upon it */
922 	folio_put_refs(folio, nr);
923 	return xas_error(&xas);
924 }
925 ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO);
926 
filemap_add_folio(struct address_space * mapping,struct folio * folio,pgoff_t index,gfp_t gfp)927 int filemap_add_folio(struct address_space *mapping, struct folio *folio,
928 				pgoff_t index, gfp_t gfp)
929 {
930 	void *shadow = NULL;
931 	int ret;
932 
933 	__folio_set_locked(folio);
934 	ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow);
935 	if (unlikely(ret))
936 		__folio_clear_locked(folio);
937 	else {
938 		/*
939 		 * The folio might have been evicted from cache only
940 		 * recently, in which case it should be activated like
941 		 * any other repeatedly accessed folio.
942 		 * The exception is folios getting rewritten; evicting other
943 		 * data from the working set, only to cache data that will
944 		 * get overwritten with something else, is a waste of memory.
945 		 */
946 		WARN_ON_ONCE(folio_test_active(folio));
947 		if (!(gfp & __GFP_WRITE) && shadow)
948 			workingset_refault(folio, shadow);
949 		folio_add_lru(folio);
950 	}
951 	return ret;
952 }
953 EXPORT_SYMBOL_GPL(filemap_add_folio);
954 
955 #ifdef CONFIG_NUMA
filemap_alloc_folio(gfp_t gfp,unsigned int order)956 struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)
957 {
958 	int n;
959 	struct folio *folio;
960 
961 	if (cpuset_do_page_mem_spread()) {
962 		unsigned int cpuset_mems_cookie;
963 		do {
964 			cpuset_mems_cookie = read_mems_allowed_begin();
965 			n = cpuset_mem_spread_node();
966 			folio = __folio_alloc_node(gfp, order, n);
967 		} while (!folio && read_mems_allowed_retry(cpuset_mems_cookie));
968 
969 		return folio;
970 	}
971 	return folio_alloc(gfp, order);
972 }
973 EXPORT_SYMBOL(filemap_alloc_folio);
974 #endif
975 
976 /*
977  * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
978  *
979  * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
980  *
981  * @mapping1: the first mapping to lock
982  * @mapping2: the second mapping to lock
983  */
filemap_invalidate_lock_two(struct address_space * mapping1,struct address_space * mapping2)984 void filemap_invalidate_lock_two(struct address_space *mapping1,
985 				 struct address_space *mapping2)
986 {
987 	if (mapping1 > mapping2)
988 		swap(mapping1, mapping2);
989 	if (mapping1)
990 		down_write(&mapping1->invalidate_lock);
991 	if (mapping2 && mapping1 != mapping2)
992 		down_write_nested(&mapping2->invalidate_lock, 1);
993 }
994 EXPORT_SYMBOL(filemap_invalidate_lock_two);
995 
996 /*
997  * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
998  *
999  * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1000  *
1001  * @mapping1: the first mapping to unlock
1002  * @mapping2: the second mapping to unlock
1003  */
filemap_invalidate_unlock_two(struct address_space * mapping1,struct address_space * mapping2)1004 void filemap_invalidate_unlock_two(struct address_space *mapping1,
1005 				   struct address_space *mapping2)
1006 {
1007 	if (mapping1)
1008 		up_write(&mapping1->invalidate_lock);
1009 	if (mapping2 && mapping1 != mapping2)
1010 		up_write(&mapping2->invalidate_lock);
1011 }
1012 EXPORT_SYMBOL(filemap_invalidate_unlock_two);
1013 
1014 /*
1015  * In order to wait for pages to become available there must be
1016  * waitqueues associated with pages. By using a hash table of
1017  * waitqueues where the bucket discipline is to maintain all
1018  * waiters on the same queue and wake all when any of the pages
1019  * become available, and for the woken contexts to check to be
1020  * sure the appropriate page became available, this saves space
1021  * at a cost of "thundering herd" phenomena during rare hash
1022  * collisions.
1023  */
1024 #define PAGE_WAIT_TABLE_BITS 8
1025 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
1026 static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
1027 
folio_waitqueue(struct folio * folio)1028 static wait_queue_head_t *folio_waitqueue(struct folio *folio)
1029 {
1030 	return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)];
1031 }
1032 
pagecache_init(void)1033 void __init pagecache_init(void)
1034 {
1035 	int i;
1036 
1037 	for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
1038 		init_waitqueue_head(&folio_wait_table[i]);
1039 
1040 	page_writeback_init();
1041 }
1042 
1043 /*
1044  * The page wait code treats the "wait->flags" somewhat unusually, because
1045  * we have multiple different kinds of waits, not just the usual "exclusive"
1046  * one.
1047  *
1048  * We have:
1049  *
1050  *  (a) no special bits set:
1051  *
1052  *	We're just waiting for the bit to be released, and when a waker
1053  *	calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up,
1054  *	and remove it from the wait queue.
1055  *
1056  *	Simple and straightforward.
1057  *
1058  *  (b) WQ_FLAG_EXCLUSIVE:
1059  *
1060  *	The waiter is waiting to get the lock, and only one waiter should
1061  *	be woken up to avoid any thundering herd behavior. We'll set the
1062  *	WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue.
1063  *
1064  *	This is the traditional exclusive wait.
1065  *
1066  *  (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM:
1067  *
1068  *	The waiter is waiting to get the bit, and additionally wants the
1069  *	lock to be transferred to it for fair lock behavior. If the lock
1070  *	cannot be taken, we stop walking the wait queue without waking
1071  *	the waiter.
1072  *
1073  *	This is the "fair lock handoff" case, and in addition to setting
1074  *	WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see
1075  *	that it now has the lock.
1076  */
wake_page_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * arg)1077 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
1078 {
1079 	unsigned int flags;
1080 	struct wait_page_key *key = arg;
1081 	struct wait_page_queue *wait_page
1082 		= container_of(wait, struct wait_page_queue, wait);
1083 
1084 	if (!wake_page_match(wait_page, key))
1085 		return 0;
1086 
1087 	/*
1088 	 * If it's a lock handoff wait, we get the bit for it, and
1089 	 * stop walking (and do not wake it up) if we can't.
1090 	 */
1091 	flags = wait->flags;
1092 	if (flags & WQ_FLAG_EXCLUSIVE) {
1093 		if (test_bit(key->bit_nr, &key->folio->flags))
1094 			return -1;
1095 		if (flags & WQ_FLAG_CUSTOM) {
1096 			if (test_and_set_bit(key->bit_nr, &key->folio->flags))
1097 				return -1;
1098 			flags |= WQ_FLAG_DONE;
1099 		}
1100 	}
1101 
1102 	/*
1103 	 * We are holding the wait-queue lock, but the waiter that
1104 	 * is waiting for this will be checking the flags without
1105 	 * any locking.
1106 	 *
1107 	 * So update the flags atomically, and wake up the waiter
1108 	 * afterwards to avoid any races. This store-release pairs
1109 	 * with the load-acquire in folio_wait_bit_common().
1110 	 */
1111 	smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
1112 	wake_up_state(wait->private, mode);
1113 
1114 	/*
1115 	 * Ok, we have successfully done what we're waiting for,
1116 	 * and we can unconditionally remove the wait entry.
1117 	 *
1118 	 * Note that this pairs with the "finish_wait()" in the
1119 	 * waiter, and has to be the absolute last thing we do.
1120 	 * After this list_del_init(&wait->entry) the wait entry
1121 	 * might be de-allocated and the process might even have
1122 	 * exited.
1123 	 */
1124 	list_del_init_careful(&wait->entry);
1125 	return (flags & WQ_FLAG_EXCLUSIVE) != 0;
1126 }
1127 
folio_wake_bit(struct folio * folio,int bit_nr)1128 static void folio_wake_bit(struct folio *folio, int bit_nr)
1129 {
1130 	wait_queue_head_t *q = folio_waitqueue(folio);
1131 	struct wait_page_key key;
1132 	unsigned long flags;
1133 	wait_queue_entry_t bookmark;
1134 
1135 	key.folio = folio;
1136 	key.bit_nr = bit_nr;
1137 	key.page_match = 0;
1138 
1139 	bookmark.flags = 0;
1140 	bookmark.private = NULL;
1141 	bookmark.func = NULL;
1142 	INIT_LIST_HEAD(&bookmark.entry);
1143 
1144 	spin_lock_irqsave(&q->lock, flags);
1145 	__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1146 
1147 	while (bookmark.flags & WQ_FLAG_BOOKMARK) {
1148 		/*
1149 		 * Take a breather from holding the lock,
1150 		 * allow pages that finish wake up asynchronously
1151 		 * to acquire the lock and remove themselves
1152 		 * from wait queue
1153 		 */
1154 		spin_unlock_irqrestore(&q->lock, flags);
1155 		cpu_relax();
1156 		spin_lock_irqsave(&q->lock, flags);
1157 		__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1158 	}
1159 
1160 	/*
1161 	 * It's possible to miss clearing waiters here, when we woke our page
1162 	 * waiters, but the hashed waitqueue has waiters for other pages on it.
1163 	 * That's okay, it's a rare case. The next waker will clear it.
1164 	 *
1165 	 * Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE,
1166 	 * other), the flag may be cleared in the course of freeing the page;
1167 	 * but that is not required for correctness.
1168 	 */
1169 	if (!waitqueue_active(q) || !key.page_match)
1170 		folio_clear_waiters(folio);
1171 
1172 	spin_unlock_irqrestore(&q->lock, flags);
1173 }
1174 
folio_wake(struct folio * folio,int bit)1175 static void folio_wake(struct folio *folio, int bit)
1176 {
1177 	if (!folio_test_waiters(folio))
1178 		return;
1179 	folio_wake_bit(folio, bit);
1180 }
1181 
1182 /*
1183  * A choice of three behaviors for folio_wait_bit_common():
1184  */
1185 enum behavior {
1186 	EXCLUSIVE,	/* Hold ref to page and take the bit when woken, like
1187 			 * __folio_lock() waiting on then setting PG_locked.
1188 			 */
1189 	SHARED,		/* Hold ref to page and check the bit when woken, like
1190 			 * folio_wait_writeback() waiting on PG_writeback.
1191 			 */
1192 	DROP,		/* Drop ref to page before wait, no check when woken,
1193 			 * like folio_put_wait_locked() on PG_locked.
1194 			 */
1195 };
1196 
1197 /*
1198  * Attempt to check (or get) the folio flag, and mark us done
1199  * if successful.
1200  */
folio_trylock_flag(struct folio * folio,int bit_nr,struct wait_queue_entry * wait)1201 static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
1202 					struct wait_queue_entry *wait)
1203 {
1204 	if (wait->flags & WQ_FLAG_EXCLUSIVE) {
1205 		if (test_and_set_bit(bit_nr, &folio->flags))
1206 			return false;
1207 	} else if (test_bit(bit_nr, &folio->flags))
1208 		return false;
1209 
1210 	wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
1211 	return true;
1212 }
1213 
1214 /* How many times do we accept lock stealing from under a waiter? */
1215 int sysctl_page_lock_unfairness = 5;
1216 
folio_wait_bit_common(struct folio * folio,int bit_nr,int state,enum behavior behavior)1217 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
1218 		int state, enum behavior behavior)
1219 {
1220 	wait_queue_head_t *q = folio_waitqueue(folio);
1221 	int unfairness = sysctl_page_lock_unfairness;
1222 	struct wait_page_queue wait_page;
1223 	wait_queue_entry_t *wait = &wait_page.wait;
1224 	bool thrashing = false;
1225 	unsigned long pflags;
1226 	bool in_thrashing;
1227 
1228 	if (bit_nr == PG_locked &&
1229 	    !folio_test_uptodate(folio) && folio_test_workingset(folio)) {
1230 		delayacct_thrashing_start(&in_thrashing);
1231 		psi_memstall_enter(&pflags);
1232 		thrashing = true;
1233 	}
1234 
1235 	init_wait(wait);
1236 	wait->func = wake_page_function;
1237 	wait_page.folio = folio;
1238 	wait_page.bit_nr = bit_nr;
1239 
1240 repeat:
1241 	wait->flags = 0;
1242 	if (behavior == EXCLUSIVE) {
1243 		wait->flags = WQ_FLAG_EXCLUSIVE;
1244 		if (--unfairness < 0)
1245 			wait->flags |= WQ_FLAG_CUSTOM;
1246 	}
1247 
1248 	/*
1249 	 * Do one last check whether we can get the
1250 	 * page bit synchronously.
1251 	 *
1252 	 * Do the folio_set_waiters() marking before that
1253 	 * to let any waker we _just_ missed know they
1254 	 * need to wake us up (otherwise they'll never
1255 	 * even go to the slow case that looks at the
1256 	 * page queue), and add ourselves to the wait
1257 	 * queue if we need to sleep.
1258 	 *
1259 	 * This part needs to be done under the queue
1260 	 * lock to avoid races.
1261 	 */
1262 	spin_lock_irq(&q->lock);
1263 	folio_set_waiters(folio);
1264 	if (!folio_trylock_flag(folio, bit_nr, wait))
1265 		__add_wait_queue_entry_tail(q, wait);
1266 	spin_unlock_irq(&q->lock);
1267 
1268 	/*
1269 	 * From now on, all the logic will be based on
1270 	 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to
1271 	 * see whether the page bit testing has already
1272 	 * been done by the wake function.
1273 	 *
1274 	 * We can drop our reference to the folio.
1275 	 */
1276 	if (behavior == DROP)
1277 		folio_put(folio);
1278 
1279 	/*
1280 	 * Note that until the "finish_wait()", or until
1281 	 * we see the WQ_FLAG_WOKEN flag, we need to
1282 	 * be very careful with the 'wait->flags', because
1283 	 * we may race with a waker that sets them.
1284 	 */
1285 	for (;;) {
1286 		unsigned int flags;
1287 
1288 		set_current_state(state);
1289 
1290 		/* Loop until we've been woken or interrupted */
1291 		flags = smp_load_acquire(&wait->flags);
1292 		if (!(flags & WQ_FLAG_WOKEN)) {
1293 			if (signal_pending_state(state, current))
1294 				break;
1295 
1296 			io_schedule();
1297 			continue;
1298 		}
1299 
1300 		/* If we were non-exclusive, we're done */
1301 		if (behavior != EXCLUSIVE)
1302 			break;
1303 
1304 		/* If the waker got the lock for us, we're done */
1305 		if (flags & WQ_FLAG_DONE)
1306 			break;
1307 
1308 		/*
1309 		 * Otherwise, if we're getting the lock, we need to
1310 		 * try to get it ourselves.
1311 		 *
1312 		 * And if that fails, we'll have to retry this all.
1313 		 */
1314 		if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0))))
1315 			goto repeat;
1316 
1317 		wait->flags |= WQ_FLAG_DONE;
1318 		break;
1319 	}
1320 
1321 	/*
1322 	 * If a signal happened, this 'finish_wait()' may remove the last
1323 	 * waiter from the wait-queues, but the folio waiters bit will remain
1324 	 * set. That's ok. The next wakeup will take care of it, and trying
1325 	 * to do it here would be difficult and prone to races.
1326 	 */
1327 	finish_wait(q, wait);
1328 
1329 	if (thrashing) {
1330 		delayacct_thrashing_end(&in_thrashing);
1331 		psi_memstall_leave(&pflags);
1332 	}
1333 
1334 	/*
1335 	 * NOTE! The wait->flags weren't stable until we've done the
1336 	 * 'finish_wait()', and we could have exited the loop above due
1337 	 * to a signal, and had a wakeup event happen after the signal
1338 	 * test but before the 'finish_wait()'.
1339 	 *
1340 	 * So only after the finish_wait() can we reliably determine
1341 	 * if we got woken up or not, so we can now figure out the final
1342 	 * return value based on that state without races.
1343 	 *
1344 	 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
1345 	 * waiter, but an exclusive one requires WQ_FLAG_DONE.
1346 	 */
1347 	if (behavior == EXCLUSIVE)
1348 		return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
1349 
1350 	return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
1351 }
1352 
1353 #ifdef CONFIG_MIGRATION
1354 /**
1355  * migration_entry_wait_on_locked - Wait for a migration entry to be removed
1356  * @entry: migration swap entry.
1357  * @ptep: mapped pte pointer. Will return with the ptep unmapped. Only required
1358  *        for pte entries, pass NULL for pmd entries.
1359  * @ptl: already locked ptl. This function will drop the lock.
1360  *
1361  * Wait for a migration entry referencing the given page to be removed. This is
1362  * equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except
1363  * this can be called without taking a reference on the page. Instead this
1364  * should be called while holding the ptl for the migration entry referencing
1365  * the page.
1366  *
1367  * Returns after unmapping and unlocking the pte/ptl with pte_unmap_unlock().
1368  *
1369  * This follows the same logic as folio_wait_bit_common() so see the comments
1370  * there.
1371  */
migration_entry_wait_on_locked(swp_entry_t entry,pte_t * ptep,spinlock_t * ptl)1372 void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep,
1373 				spinlock_t *ptl)
1374 {
1375 	struct wait_page_queue wait_page;
1376 	wait_queue_entry_t *wait = &wait_page.wait;
1377 	bool thrashing = false;
1378 	unsigned long pflags;
1379 	bool in_thrashing;
1380 	wait_queue_head_t *q;
1381 	struct folio *folio = page_folio(pfn_swap_entry_to_page(entry));
1382 
1383 	q = folio_waitqueue(folio);
1384 	if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
1385 		delayacct_thrashing_start(&in_thrashing);
1386 		psi_memstall_enter(&pflags);
1387 		thrashing = true;
1388 	}
1389 
1390 	init_wait(wait);
1391 	wait->func = wake_page_function;
1392 	wait_page.folio = folio;
1393 	wait_page.bit_nr = PG_locked;
1394 	wait->flags = 0;
1395 
1396 	spin_lock_irq(&q->lock);
1397 	folio_set_waiters(folio);
1398 	if (!folio_trylock_flag(folio, PG_locked, wait))
1399 		__add_wait_queue_entry_tail(q, wait);
1400 	spin_unlock_irq(&q->lock);
1401 
1402 	/*
1403 	 * If a migration entry exists for the page the migration path must hold
1404 	 * a valid reference to the page, and it must take the ptl to remove the
1405 	 * migration entry. So the page is valid until the ptl is dropped.
1406 	 */
1407 	if (ptep)
1408 		pte_unmap_unlock(ptep, ptl);
1409 	else
1410 		spin_unlock(ptl);
1411 
1412 	for (;;) {
1413 		unsigned int flags;
1414 
1415 		set_current_state(TASK_UNINTERRUPTIBLE);
1416 
1417 		/* Loop until we've been woken or interrupted */
1418 		flags = smp_load_acquire(&wait->flags);
1419 		if (!(flags & WQ_FLAG_WOKEN)) {
1420 			if (signal_pending_state(TASK_UNINTERRUPTIBLE, current))
1421 				break;
1422 
1423 			io_schedule();
1424 			continue;
1425 		}
1426 		break;
1427 	}
1428 
1429 	finish_wait(q, wait);
1430 
1431 	if (thrashing) {
1432 		delayacct_thrashing_end(&in_thrashing);
1433 		psi_memstall_leave(&pflags);
1434 	}
1435 }
1436 #endif
1437 
folio_wait_bit(struct folio * folio,int bit_nr)1438 void folio_wait_bit(struct folio *folio, int bit_nr)
1439 {
1440 	folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
1441 }
1442 EXPORT_SYMBOL(folio_wait_bit);
1443 
folio_wait_bit_killable(struct folio * folio,int bit_nr)1444 int folio_wait_bit_killable(struct folio *folio, int bit_nr)
1445 {
1446 	return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED);
1447 }
1448 EXPORT_SYMBOL(folio_wait_bit_killable);
1449 
1450 /**
1451  * folio_put_wait_locked - Drop a reference and wait for it to be unlocked
1452  * @folio: The folio to wait for.
1453  * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc).
1454  *
1455  * The caller should hold a reference on @folio.  They expect the page to
1456  * become unlocked relatively soon, but do not wish to hold up migration
1457  * (for example) by holding the reference while waiting for the folio to
1458  * come unlocked.  After this function returns, the caller should not
1459  * dereference @folio.
1460  *
1461  * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal.
1462  */
folio_put_wait_locked(struct folio * folio,int state)1463 static int folio_put_wait_locked(struct folio *folio, int state)
1464 {
1465 	return folio_wait_bit_common(folio, PG_locked, state, DROP);
1466 }
1467 
1468 /**
1469  * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue
1470  * @folio: Folio defining the wait queue of interest
1471  * @waiter: Waiter to add to the queue
1472  *
1473  * Add an arbitrary @waiter to the wait queue for the nominated @folio.
1474  */
folio_add_wait_queue(struct folio * folio,wait_queue_entry_t * waiter)1475 void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter)
1476 {
1477 	wait_queue_head_t *q = folio_waitqueue(folio);
1478 	unsigned long flags;
1479 
1480 	spin_lock_irqsave(&q->lock, flags);
1481 	__add_wait_queue_entry_tail(q, waiter);
1482 	folio_set_waiters(folio);
1483 	spin_unlock_irqrestore(&q->lock, flags);
1484 }
1485 EXPORT_SYMBOL_GPL(folio_add_wait_queue);
1486 
1487 #ifndef clear_bit_unlock_is_negative_byte
1488 
1489 /*
1490  * PG_waiters is the high bit in the same byte as PG_lock.
1491  *
1492  * On x86 (and on many other architectures), we can clear PG_lock and
1493  * test the sign bit at the same time. But if the architecture does
1494  * not support that special operation, we just do this all by hand
1495  * instead.
1496  *
1497  * The read of PG_waiters has to be after (or concurrently with) PG_locked
1498  * being cleared, but a memory barrier should be unnecessary since it is
1499  * in the same byte as PG_locked.
1500  */
clear_bit_unlock_is_negative_byte(long nr,volatile void * mem)1501 static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
1502 {
1503 	clear_bit_unlock(nr, mem);
1504 	/* smp_mb__after_atomic(); */
1505 	return test_bit(PG_waiters, mem);
1506 }
1507 
1508 #endif
1509 
1510 /**
1511  * folio_unlock - Unlock a locked folio.
1512  * @folio: The folio.
1513  *
1514  * Unlocks the folio and wakes up any thread sleeping on the page lock.
1515  *
1516  * Context: May be called from interrupt or process context.  May not be
1517  * called from NMI context.
1518  */
folio_unlock(struct folio * folio)1519 void folio_unlock(struct folio *folio)
1520 {
1521 	/* Bit 7 allows x86 to check the byte's sign bit */
1522 	BUILD_BUG_ON(PG_waiters != 7);
1523 	BUILD_BUG_ON(PG_locked > 7);
1524 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1525 	if (clear_bit_unlock_is_negative_byte(PG_locked, folio_flags(folio, 0)))
1526 		folio_wake_bit(folio, PG_locked);
1527 }
1528 EXPORT_SYMBOL(folio_unlock);
1529 
1530 /**
1531  * folio_end_private_2 - Clear PG_private_2 and wake any waiters.
1532  * @folio: The folio.
1533  *
1534  * Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for
1535  * it.  The folio reference held for PG_private_2 being set is released.
1536  *
1537  * This is, for example, used when a netfs folio is being written to a local
1538  * disk cache, thereby allowing writes to the cache for the same folio to be
1539  * serialised.
1540  */
folio_end_private_2(struct folio * folio)1541 void folio_end_private_2(struct folio *folio)
1542 {
1543 	VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio);
1544 	clear_bit_unlock(PG_private_2, folio_flags(folio, 0));
1545 	folio_wake_bit(folio, PG_private_2);
1546 	folio_put(folio);
1547 }
1548 EXPORT_SYMBOL(folio_end_private_2);
1549 
1550 /**
1551  * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio.
1552  * @folio: The folio to wait on.
1553  *
1554  * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio.
1555  */
folio_wait_private_2(struct folio * folio)1556 void folio_wait_private_2(struct folio *folio)
1557 {
1558 	while (folio_test_private_2(folio))
1559 		folio_wait_bit(folio, PG_private_2);
1560 }
1561 EXPORT_SYMBOL(folio_wait_private_2);
1562 
1563 /**
1564  * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio.
1565  * @folio: The folio to wait on.
1566  *
1567  * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio or until a
1568  * fatal signal is received by the calling task.
1569  *
1570  * Return:
1571  * - 0 if successful.
1572  * - -EINTR if a fatal signal was encountered.
1573  */
folio_wait_private_2_killable(struct folio * folio)1574 int folio_wait_private_2_killable(struct folio *folio)
1575 {
1576 	int ret = 0;
1577 
1578 	while (folio_test_private_2(folio)) {
1579 		ret = folio_wait_bit_killable(folio, PG_private_2);
1580 		if (ret < 0)
1581 			break;
1582 	}
1583 
1584 	return ret;
1585 }
1586 EXPORT_SYMBOL(folio_wait_private_2_killable);
1587 
1588 /**
1589  * folio_end_writeback - End writeback against a folio.
1590  * @folio: The folio.
1591  */
folio_end_writeback(struct folio * folio)1592 void folio_end_writeback(struct folio *folio)
1593 {
1594 	/*
1595 	 * folio_test_clear_reclaim() could be used here but it is an
1596 	 * atomic operation and overkill in this particular case. Failing
1597 	 * to shuffle a folio marked for immediate reclaim is too mild
1598 	 * a gain to justify taking an atomic operation penalty at the
1599 	 * end of every folio writeback.
1600 	 */
1601 	if (folio_test_reclaim(folio)) {
1602 		folio_clear_reclaim(folio);
1603 		folio_rotate_reclaimable(folio);
1604 	}
1605 
1606 	/*
1607 	 * Writeback does not hold a folio reference of its own, relying
1608 	 * on truncation to wait for the clearing of PG_writeback.
1609 	 * But here we must make sure that the folio is not freed and
1610 	 * reused before the folio_wake().
1611 	 */
1612 	folio_get(folio);
1613 	if (!__folio_end_writeback(folio))
1614 		BUG();
1615 
1616 	smp_mb__after_atomic();
1617 	folio_wake(folio, PG_writeback);
1618 	acct_reclaim_writeback(folio);
1619 	folio_put(folio);
1620 }
1621 EXPORT_SYMBOL(folio_end_writeback);
1622 
1623 /*
1624  * After completing I/O on a page, call this routine to update the page
1625  * flags appropriately
1626  */
page_endio(struct page * page,bool is_write,int err)1627 void page_endio(struct page *page, bool is_write, int err)
1628 {
1629 	struct folio *folio = page_folio(page);
1630 
1631 	if (!is_write) {
1632 		if (!err) {
1633 			folio_mark_uptodate(folio);
1634 		} else {
1635 			folio_clear_uptodate(folio);
1636 			folio_set_error(folio);
1637 		}
1638 		folio_unlock(folio);
1639 	} else {
1640 		if (err) {
1641 			struct address_space *mapping;
1642 
1643 			folio_set_error(folio);
1644 			mapping = folio_mapping(folio);
1645 			if (mapping)
1646 				mapping_set_error(mapping, err);
1647 		}
1648 		folio_end_writeback(folio);
1649 	}
1650 }
1651 EXPORT_SYMBOL_GPL(page_endio);
1652 
1653 /**
1654  * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it.
1655  * @folio: The folio to lock
1656  */
__folio_lock(struct folio * folio)1657 void __folio_lock(struct folio *folio)
1658 {
1659 	folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE,
1660 				EXCLUSIVE);
1661 }
1662 EXPORT_SYMBOL(__folio_lock);
1663 
__folio_lock_killable(struct folio * folio)1664 int __folio_lock_killable(struct folio *folio)
1665 {
1666 	return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE,
1667 					EXCLUSIVE);
1668 }
1669 EXPORT_SYMBOL_GPL(__folio_lock_killable);
1670 
__folio_lock_async(struct folio * folio,struct wait_page_queue * wait)1671 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)
1672 {
1673 	struct wait_queue_head *q = folio_waitqueue(folio);
1674 	int ret = 0;
1675 
1676 	wait->folio = folio;
1677 	wait->bit_nr = PG_locked;
1678 
1679 	spin_lock_irq(&q->lock);
1680 	__add_wait_queue_entry_tail(q, &wait->wait);
1681 	folio_set_waiters(folio);
1682 	ret = !folio_trylock(folio);
1683 	/*
1684 	 * If we were successful now, we know we're still on the
1685 	 * waitqueue as we're still under the lock. This means it's
1686 	 * safe to remove and return success, we know the callback
1687 	 * isn't going to trigger.
1688 	 */
1689 	if (!ret)
1690 		__remove_wait_queue(q, &wait->wait);
1691 	else
1692 		ret = -EIOCBQUEUED;
1693 	spin_unlock_irq(&q->lock);
1694 	return ret;
1695 }
1696 
1697 /*
1698  * Return values:
1699  * true - folio is locked; mmap_lock is still held.
1700  * false - folio is not locked.
1701  *     mmap_lock has been released (mmap_read_unlock(), unless flags had both
1702  *     FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
1703  *     which case mmap_lock is still held.
1704  *
1705  * If neither ALLOW_RETRY nor KILLABLE are set, will always return true
1706  * with the folio locked and the mmap_lock unperturbed.
1707  */
__folio_lock_or_retry(struct folio * folio,struct mm_struct * mm,unsigned int flags)1708 bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm,
1709 			 unsigned int flags)
1710 {
1711 	if (fault_flag_allow_retry_first(flags)) {
1712 		/*
1713 		 * CAUTION! In this case, mmap_lock is not released
1714 		 * even though return 0.
1715 		 */
1716 		if (flags & FAULT_FLAG_RETRY_NOWAIT)
1717 			return false;
1718 
1719 		mmap_read_unlock(mm);
1720 		if (flags & FAULT_FLAG_KILLABLE)
1721 			folio_wait_locked_killable(folio);
1722 		else
1723 			folio_wait_locked(folio);
1724 		return false;
1725 	}
1726 	if (flags & FAULT_FLAG_KILLABLE) {
1727 		bool ret;
1728 
1729 		ret = __folio_lock_killable(folio);
1730 		if (ret) {
1731 			mmap_read_unlock(mm);
1732 			return false;
1733 		}
1734 	} else {
1735 		__folio_lock(folio);
1736 	}
1737 
1738 	return true;
1739 }
1740 
1741 /**
1742  * page_cache_next_miss() - Find the next gap in the page cache.
1743  * @mapping: Mapping.
1744  * @index: Index.
1745  * @max_scan: Maximum range to search.
1746  *
1747  * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1748  * gap with the lowest index.
1749  *
1750  * This function may be called under the rcu_read_lock.  However, this will
1751  * not atomically search a snapshot of the cache at a single point in time.
1752  * For example, if a gap is created at index 5, then subsequently a gap is
1753  * created at index 10, page_cache_next_miss covering both indices may
1754  * return 10 if called under the rcu_read_lock.
1755  *
1756  * Return: The index of the gap if found, otherwise an index outside the
1757  * range specified (in which case 'return - index >= max_scan' will be true).
1758  * In the rare case of index wrap-around, 0 will be returned.
1759  */
page_cache_next_miss(struct address_space * mapping,pgoff_t index,unsigned long max_scan)1760 pgoff_t page_cache_next_miss(struct address_space *mapping,
1761 			     pgoff_t index, unsigned long max_scan)
1762 {
1763 	XA_STATE(xas, &mapping->i_pages, index);
1764 
1765 	while (max_scan--) {
1766 		void *entry = xas_next(&xas);
1767 		if (!entry || xa_is_value(entry))
1768 			break;
1769 		if (xas.xa_index == 0)
1770 			break;
1771 	}
1772 
1773 	return xas.xa_index;
1774 }
1775 EXPORT_SYMBOL(page_cache_next_miss);
1776 
1777 /**
1778  * page_cache_prev_miss() - Find the previous gap in the page cache.
1779  * @mapping: Mapping.
1780  * @index: Index.
1781  * @max_scan: Maximum range to search.
1782  *
1783  * Search the range [max(index - max_scan + 1, 0), index] for the
1784  * gap with the highest index.
1785  *
1786  * This function may be called under the rcu_read_lock.  However, this will
1787  * not atomically search a snapshot of the cache at a single point in time.
1788  * For example, if a gap is created at index 10, then subsequently a gap is
1789  * created at index 5, page_cache_prev_miss() covering both indices may
1790  * return 5 if called under the rcu_read_lock.
1791  *
1792  * Return: The index of the gap if found, otherwise an index outside the
1793  * range specified (in which case 'index - return >= max_scan' will be true).
1794  * In the rare case of wrap-around, ULONG_MAX will be returned.
1795  */
page_cache_prev_miss(struct address_space * mapping,pgoff_t index,unsigned long max_scan)1796 pgoff_t page_cache_prev_miss(struct address_space *mapping,
1797 			     pgoff_t index, unsigned long max_scan)
1798 {
1799 	XA_STATE(xas, &mapping->i_pages, index);
1800 
1801 	while (max_scan--) {
1802 		void *entry = xas_prev(&xas);
1803 		if (!entry || xa_is_value(entry))
1804 			break;
1805 		if (xas.xa_index == ULONG_MAX)
1806 			break;
1807 	}
1808 
1809 	return xas.xa_index;
1810 }
1811 EXPORT_SYMBOL(page_cache_prev_miss);
1812 
1813 /*
1814  * Lockless page cache protocol:
1815  * On the lookup side:
1816  * 1. Load the folio from i_pages
1817  * 2. Increment the refcount if it's not zero
1818  * 3. If the folio is not found by xas_reload(), put the refcount and retry
1819  *
1820  * On the removal side:
1821  * A. Freeze the page (by zeroing the refcount if nobody else has a reference)
1822  * B. Remove the page from i_pages
1823  * C. Return the page to the page allocator
1824  *
1825  * This means that any page may have its reference count temporarily
1826  * increased by a speculative page cache (or fast GUP) lookup as it can
1827  * be allocated by another user before the RCU grace period expires.
1828  * Because the refcount temporarily acquired here may end up being the
1829  * last refcount on the page, any page allocation must be freeable by
1830  * folio_put().
1831  */
1832 
1833 /*
1834  * mapping_get_entry - Get a page cache entry.
1835  * @mapping: the address_space to search
1836  * @index: The page cache index.
1837  *
1838  * Looks up the page cache entry at @mapping & @index.  If it is a folio,
1839  * it is returned with an increased refcount.  If it is a shadow entry
1840  * of a previously evicted folio, or a swap entry from shmem/tmpfs,
1841  * it is returned without further action.
1842  *
1843  * Return: The folio, swap or shadow entry, %NULL if nothing is found.
1844  */
mapping_get_entry(struct address_space * mapping,pgoff_t index)1845 static void *mapping_get_entry(struct address_space *mapping, pgoff_t index)
1846 {
1847 	XA_STATE(xas, &mapping->i_pages, index);
1848 	struct folio *folio;
1849 
1850 	rcu_read_lock();
1851 repeat:
1852 	xas_reset(&xas);
1853 	folio = xas_load(&xas);
1854 	if (xas_retry(&xas, folio))
1855 		goto repeat;
1856 	/*
1857 	 * A shadow entry of a recently evicted page, or a swap entry from
1858 	 * shmem/tmpfs.  Return it without attempting to raise page count.
1859 	 */
1860 	if (!folio || xa_is_value(folio))
1861 		goto out;
1862 
1863 	if (!folio_try_get_rcu(folio))
1864 		goto repeat;
1865 
1866 	if (unlikely(folio != xas_reload(&xas))) {
1867 		folio_put(folio);
1868 		goto repeat;
1869 	}
1870 out:
1871 	rcu_read_unlock();
1872 
1873 	return folio;
1874 }
1875 
1876 /**
1877  * __filemap_get_folio - Find and get a reference to a folio.
1878  * @mapping: The address_space to search.
1879  * @index: The page index.
1880  * @fgp_flags: %FGP flags modify how the folio is returned.
1881  * @gfp: Memory allocation flags to use if %FGP_CREAT is specified.
1882  *
1883  * Looks up the page cache entry at @mapping & @index.
1884  *
1885  * @fgp_flags can be zero or more of these flags:
1886  *
1887  * * %FGP_ACCESSED - The folio will be marked accessed.
1888  * * %FGP_LOCK - The folio is returned locked.
1889  * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it
1890  *   instead of allocating a new folio to replace it.
1891  * * %FGP_CREAT - If no page is present then a new page is allocated using
1892  *   @gfp and added to the page cache and the VM's LRU list.
1893  *   The page is returned locked and with an increased refcount.
1894  * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1895  *   page is already in cache.  If the page was allocated, unlock it before
1896  *   returning so the caller can do the same dance.
1897  * * %FGP_WRITE - The page will be written to by the caller.
1898  * * %FGP_NOFS - __GFP_FS will get cleared in gfp.
1899  * * %FGP_NOWAIT - Don't get blocked by page lock.
1900  * * %FGP_STABLE - Wait for the folio to be stable (finished writeback)
1901  *
1902  * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even
1903  * if the %GFP flags specified for %FGP_CREAT are atomic.
1904  *
1905  * If there is a page cache page, it is returned with an increased refcount.
1906  *
1907  * Return: The found folio or %NULL otherwise.
1908  */
__filemap_get_folio(struct address_space * mapping,pgoff_t index,int fgp_flags,gfp_t gfp)1909 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
1910 		int fgp_flags, gfp_t gfp)
1911 {
1912 	struct folio *folio;
1913 
1914 repeat:
1915 	folio = mapping_get_entry(mapping, index);
1916 	if (xa_is_value(folio)) {
1917 		if (fgp_flags & FGP_ENTRY)
1918 			return folio;
1919 		folio = NULL;
1920 	}
1921 	if (!folio)
1922 		goto no_page;
1923 
1924 	if (fgp_flags & FGP_LOCK) {
1925 		if (fgp_flags & FGP_NOWAIT) {
1926 			if (!folio_trylock(folio)) {
1927 				folio_put(folio);
1928 				return NULL;
1929 			}
1930 		} else {
1931 			folio_lock(folio);
1932 		}
1933 
1934 		/* Has the page been truncated? */
1935 		if (unlikely(folio->mapping != mapping)) {
1936 			folio_unlock(folio);
1937 			folio_put(folio);
1938 			goto repeat;
1939 		}
1940 		VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
1941 	}
1942 
1943 	if (fgp_flags & FGP_ACCESSED)
1944 		folio_mark_accessed(folio);
1945 	else if (fgp_flags & FGP_WRITE) {
1946 		/* Clear idle flag for buffer write */
1947 		if (folio_test_idle(folio))
1948 			folio_clear_idle(folio);
1949 	}
1950 
1951 	if (fgp_flags & FGP_STABLE)
1952 		folio_wait_stable(folio);
1953 no_page:
1954 	if (!folio && (fgp_flags & FGP_CREAT)) {
1955 		int err;
1956 		if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
1957 			gfp |= __GFP_WRITE;
1958 		if (fgp_flags & FGP_NOFS)
1959 			gfp &= ~__GFP_FS;
1960 		if (fgp_flags & FGP_NOWAIT) {
1961 			gfp &= ~GFP_KERNEL;
1962 			gfp |= GFP_NOWAIT | __GFP_NOWARN;
1963 		}
1964 
1965 		folio = filemap_alloc_folio(gfp, 0);
1966 		if (!folio)
1967 			return NULL;
1968 
1969 		if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
1970 			fgp_flags |= FGP_LOCK;
1971 
1972 		/* Init accessed so avoid atomic mark_page_accessed later */
1973 		if (fgp_flags & FGP_ACCESSED)
1974 			__folio_set_referenced(folio);
1975 
1976 		err = filemap_add_folio(mapping, folio, index, gfp);
1977 		if (unlikely(err)) {
1978 			folio_put(folio);
1979 			folio = NULL;
1980 			if (err == -EEXIST)
1981 				goto repeat;
1982 		}
1983 
1984 		/*
1985 		 * filemap_add_folio locks the page, and for mmap
1986 		 * we expect an unlocked page.
1987 		 */
1988 		if (folio && (fgp_flags & FGP_FOR_MMAP))
1989 			folio_unlock(folio);
1990 	}
1991 
1992 	return folio;
1993 }
1994 EXPORT_SYMBOL(__filemap_get_folio);
1995 
find_get_entry(struct xa_state * xas,pgoff_t max,xa_mark_t mark)1996 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
1997 		xa_mark_t mark)
1998 {
1999 	struct folio *folio;
2000 
2001 retry:
2002 	if (mark == XA_PRESENT)
2003 		folio = xas_find(xas, max);
2004 	else
2005 		folio = xas_find_marked(xas, max, mark);
2006 
2007 	if (xas_retry(xas, folio))
2008 		goto retry;
2009 	/*
2010 	 * A shadow entry of a recently evicted page, a swap
2011 	 * entry from shmem/tmpfs or a DAX entry.  Return it
2012 	 * without attempting to raise page count.
2013 	 */
2014 	if (!folio || xa_is_value(folio))
2015 		return folio;
2016 
2017 	if (!folio_try_get_rcu(folio))
2018 		goto reset;
2019 
2020 	if (unlikely(folio != xas_reload(xas))) {
2021 		folio_put(folio);
2022 		goto reset;
2023 	}
2024 
2025 	return folio;
2026 reset:
2027 	xas_reset(xas);
2028 	goto retry;
2029 }
2030 
2031 /**
2032  * find_get_entries - gang pagecache lookup
2033  * @mapping:	The address_space to search
2034  * @start:	The starting page cache index
2035  * @end:	The final page index (inclusive).
2036  * @fbatch:	Where the resulting entries are placed.
2037  * @indices:	The cache indices corresponding to the entries in @entries
2038  *
2039  * find_get_entries() will search for and return a batch of entries in
2040  * the mapping.  The entries are placed in @fbatch.  find_get_entries()
2041  * takes a reference on any actual folios it returns.
2042  *
2043  * The entries have ascending indexes.  The indices may not be consecutive
2044  * due to not-present entries or large folios.
2045  *
2046  * Any shadow entries of evicted folios, or swap entries from
2047  * shmem/tmpfs, are included in the returned array.
2048  *
2049  * Return: The number of entries which were found.
2050  */
find_get_entries(struct address_space * mapping,pgoff_t start,pgoff_t end,struct folio_batch * fbatch,pgoff_t * indices)2051 unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
2052 		pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
2053 {
2054 	XA_STATE(xas, &mapping->i_pages, start);
2055 	struct folio *folio;
2056 
2057 	rcu_read_lock();
2058 	while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) {
2059 		indices[fbatch->nr] = xas.xa_index;
2060 		if (!folio_batch_add(fbatch, folio))
2061 			break;
2062 	}
2063 	rcu_read_unlock();
2064 
2065 	return folio_batch_count(fbatch);
2066 }
2067 
2068 /**
2069  * find_lock_entries - Find a batch of pagecache entries.
2070  * @mapping:	The address_space to search.
2071  * @start:	The starting page cache index.
2072  * @end:	The final page index (inclusive).
2073  * @fbatch:	Where the resulting entries are placed.
2074  * @indices:	The cache indices of the entries in @fbatch.
2075  *
2076  * find_lock_entries() will return a batch of entries from @mapping.
2077  * Swap, shadow and DAX entries are included.  Folios are returned
2078  * locked and with an incremented refcount.  Folios which are locked
2079  * by somebody else or under writeback are skipped.  Folios which are
2080  * partially outside the range are not returned.
2081  *
2082  * The entries have ascending indexes.  The indices may not be consecutive
2083  * due to not-present entries, large folios, folios which could not be
2084  * locked or folios under writeback.
2085  *
2086  * Return: The number of entries which were found.
2087  */
find_lock_entries(struct address_space * mapping,pgoff_t start,pgoff_t end,struct folio_batch * fbatch,pgoff_t * indices)2088 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
2089 		pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
2090 {
2091 	XA_STATE(xas, &mapping->i_pages, start);
2092 	struct folio *folio;
2093 
2094 	rcu_read_lock();
2095 	while ((folio = find_get_entry(&xas, end, XA_PRESENT))) {
2096 		if (!xa_is_value(folio)) {
2097 			if (folio->index < start)
2098 				goto put;
2099 			if (folio->index + folio_nr_pages(folio) - 1 > end)
2100 				goto put;
2101 			if (!folio_trylock(folio))
2102 				goto put;
2103 			if (folio->mapping != mapping ||
2104 			    folio_test_writeback(folio))
2105 				goto unlock;
2106 			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
2107 					folio);
2108 		}
2109 		indices[fbatch->nr] = xas.xa_index;
2110 		if (!folio_batch_add(fbatch, folio))
2111 			break;
2112 		continue;
2113 unlock:
2114 		folio_unlock(folio);
2115 put:
2116 		folio_put(folio);
2117 	}
2118 	rcu_read_unlock();
2119 
2120 	return folio_batch_count(fbatch);
2121 }
2122 
2123 /**
2124  * filemap_get_folios - Get a batch of folios
2125  * @mapping:	The address_space to search
2126  * @start:	The starting page index
2127  * @end:	The final page index (inclusive)
2128  * @fbatch:	The batch to fill.
2129  *
2130  * Search for and return a batch of folios in the mapping starting at
2131  * index @start and up to index @end (inclusive).  The folios are returned
2132  * in @fbatch with an elevated reference count.
2133  *
2134  * The first folio may start before @start; if it does, it will contain
2135  * @start.  The final folio may extend beyond @end; if it does, it will
2136  * contain @end.  The folios have ascending indices.  There may be gaps
2137  * between the folios if there are indices which have no folio in the
2138  * page cache.  If folios are added to or removed from the page cache
2139  * while this is running, they may or may not be found by this call.
2140  *
2141  * Return: The number of folios which were found.
2142  * We also update @start to index the next folio for the traversal.
2143  */
filemap_get_folios(struct address_space * mapping,pgoff_t * start,pgoff_t end,struct folio_batch * fbatch)2144 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
2145 		pgoff_t end, struct folio_batch *fbatch)
2146 {
2147 	XA_STATE(xas, &mapping->i_pages, *start);
2148 	struct folio *folio;
2149 
2150 	rcu_read_lock();
2151 	while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) {
2152 		/* Skip over shadow, swap and DAX entries */
2153 		if (xa_is_value(folio))
2154 			continue;
2155 		if (!folio_batch_add(fbatch, folio)) {
2156 			unsigned long nr = folio_nr_pages(folio);
2157 
2158 			if (folio_test_hugetlb(folio))
2159 				nr = 1;
2160 			*start = folio->index + nr;
2161 			goto out;
2162 		}
2163 	}
2164 
2165 	/*
2166 	 * We come here when there is no page beyond @end. We take care to not
2167 	 * overflow the index @start as it confuses some of the callers. This
2168 	 * breaks the iteration when there is a page at index -1 but that is
2169 	 * already broken anyway.
2170 	 */
2171 	if (end == (pgoff_t)-1)
2172 		*start = (pgoff_t)-1;
2173 	else
2174 		*start = end + 1;
2175 out:
2176 	rcu_read_unlock();
2177 
2178 	return folio_batch_count(fbatch);
2179 }
2180 EXPORT_SYMBOL(filemap_get_folios);
2181 
2182 static inline
folio_more_pages(struct folio * folio,pgoff_t index,pgoff_t max)2183 bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max)
2184 {
2185 	if (!folio_test_large(folio) || folio_test_hugetlb(folio))
2186 		return false;
2187 	if (index >= max)
2188 		return false;
2189 	return index < folio->index + folio_nr_pages(folio) - 1;
2190 }
2191 
2192 /**
2193  * filemap_get_folios_contig - Get a batch of contiguous folios
2194  * @mapping:	The address_space to search
2195  * @start:	The starting page index
2196  * @end:	The final page index (inclusive)
2197  * @fbatch:	The batch to fill
2198  *
2199  * filemap_get_folios_contig() works exactly like filemap_get_folios(),
2200  * except the returned folios are guaranteed to be contiguous. This may
2201  * not return all contiguous folios if the batch gets filled up.
2202  *
2203  * Return: The number of folios found.
2204  * Also update @start to be positioned for traversal of the next folio.
2205  */
2206 
filemap_get_folios_contig(struct address_space * mapping,pgoff_t * start,pgoff_t end,struct folio_batch * fbatch)2207 unsigned filemap_get_folios_contig(struct address_space *mapping,
2208 		pgoff_t *start, pgoff_t end, struct folio_batch *fbatch)
2209 {
2210 	XA_STATE(xas, &mapping->i_pages, *start);
2211 	unsigned long nr;
2212 	struct folio *folio;
2213 
2214 	rcu_read_lock();
2215 
2216 	for (folio = xas_load(&xas); folio && xas.xa_index <= end;
2217 			folio = xas_next(&xas)) {
2218 		if (xas_retry(&xas, folio))
2219 			continue;
2220 		/*
2221 		 * If the entry has been swapped out, we can stop looking.
2222 		 * No current caller is looking for DAX entries.
2223 		 */
2224 		if (xa_is_value(folio))
2225 			goto update_start;
2226 
2227 		if (!folio_try_get_rcu(folio))
2228 			goto retry;
2229 
2230 		if (unlikely(folio != xas_reload(&xas)))
2231 			goto put_folio;
2232 
2233 		if (!folio_batch_add(fbatch, folio)) {
2234 			nr = folio_nr_pages(folio);
2235 
2236 			if (folio_test_hugetlb(folio))
2237 				nr = 1;
2238 			*start = folio->index + nr;
2239 			goto out;
2240 		}
2241 		continue;
2242 put_folio:
2243 		folio_put(folio);
2244 
2245 retry:
2246 		xas_reset(&xas);
2247 	}
2248 
2249 update_start:
2250 	nr = folio_batch_count(fbatch);
2251 
2252 	if (nr) {
2253 		folio = fbatch->folios[nr - 1];
2254 		if (folio_test_hugetlb(folio))
2255 			*start = folio->index + 1;
2256 		else
2257 			*start = folio->index + folio_nr_pages(folio);
2258 	}
2259 out:
2260 	rcu_read_unlock();
2261 	return folio_batch_count(fbatch);
2262 }
2263 EXPORT_SYMBOL(filemap_get_folios_contig);
2264 
2265 /**
2266  * find_get_pages_range_tag - Find and return head pages matching @tag.
2267  * @mapping:	the address_space to search
2268  * @index:	the starting page index
2269  * @end:	The final page index (inclusive)
2270  * @tag:	the tag index
2271  * @nr_pages:	the maximum number of pages
2272  * @pages:	where the resulting pages are placed
2273  *
2274  * Like find_get_pages_range(), except we only return head pages which are
2275  * tagged with @tag.  @index is updated to the index immediately after the
2276  * last page we return, ready for the next iteration.
2277  *
2278  * Return: the number of pages which were found.
2279  */
find_get_pages_range_tag(struct address_space * mapping,pgoff_t * index,pgoff_t end,xa_mark_t tag,unsigned int nr_pages,struct page ** pages)2280 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
2281 			pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
2282 			struct page **pages)
2283 {
2284 	XA_STATE(xas, &mapping->i_pages, *index);
2285 	struct folio *folio;
2286 	unsigned ret = 0;
2287 
2288 	if (unlikely(!nr_pages))
2289 		return 0;
2290 
2291 	rcu_read_lock();
2292 	while ((folio = find_get_entry(&xas, end, tag))) {
2293 		/*
2294 		 * Shadow entries should never be tagged, but this iteration
2295 		 * is lockless so there is a window for page reclaim to evict
2296 		 * a page we saw tagged.  Skip over it.
2297 		 */
2298 		if (xa_is_value(folio))
2299 			continue;
2300 
2301 		pages[ret] = &folio->page;
2302 		if (++ret == nr_pages) {
2303 			*index = folio->index + folio_nr_pages(folio);
2304 			goto out;
2305 		}
2306 	}
2307 
2308 	/*
2309 	 * We come here when we got to @end. We take care to not overflow the
2310 	 * index @index as it confuses some of the callers. This breaks the
2311 	 * iteration when there is a page at index -1 but that is already
2312 	 * broken anyway.
2313 	 */
2314 	if (end == (pgoff_t)-1)
2315 		*index = (pgoff_t)-1;
2316 	else
2317 		*index = end + 1;
2318 out:
2319 	rcu_read_unlock();
2320 
2321 	return ret;
2322 }
2323 EXPORT_SYMBOL(find_get_pages_range_tag);
2324 
2325 /*
2326  * CD/DVDs are error prone. When a medium error occurs, the driver may fail
2327  * a _large_ part of the i/o request. Imagine the worst scenario:
2328  *
2329  *      ---R__________________________________________B__________
2330  *         ^ reading here                             ^ bad block(assume 4k)
2331  *
2332  * read(R) => miss => readahead(R...B) => media error => frustrating retries
2333  * => failing the whole request => read(R) => read(R+1) =>
2334  * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
2335  * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
2336  * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
2337  *
2338  * It is going insane. Fix it by quickly scaling down the readahead size.
2339  */
shrink_readahead_size_eio(struct file_ra_state * ra)2340 static void shrink_readahead_size_eio(struct file_ra_state *ra)
2341 {
2342 	ra->ra_pages /= 4;
2343 }
2344 
2345 /*
2346  * filemap_get_read_batch - Get a batch of folios for read
2347  *
2348  * Get a batch of folios which represent a contiguous range of bytes in
2349  * the file.  No exceptional entries will be returned.  If @index is in
2350  * the middle of a folio, the entire folio will be returned.  The last
2351  * folio in the batch may have the readahead flag set or the uptodate flag
2352  * clear so that the caller can take the appropriate action.
2353  */
filemap_get_read_batch(struct address_space * mapping,pgoff_t index,pgoff_t max,struct folio_batch * fbatch)2354 static void filemap_get_read_batch(struct address_space *mapping,
2355 		pgoff_t index, pgoff_t max, struct folio_batch *fbatch)
2356 {
2357 	XA_STATE(xas, &mapping->i_pages, index);
2358 	struct folio *folio;
2359 
2360 	rcu_read_lock();
2361 	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
2362 		if (xas_retry(&xas, folio))
2363 			continue;
2364 		if (xas.xa_index > max || xa_is_value(folio))
2365 			break;
2366 		if (xa_is_sibling(folio))
2367 			break;
2368 		if (!folio_try_get_rcu(folio))
2369 			goto retry;
2370 
2371 		if (unlikely(folio != xas_reload(&xas)))
2372 			goto put_folio;
2373 
2374 		if (!folio_batch_add(fbatch, folio))
2375 			break;
2376 		if (!folio_test_uptodate(folio))
2377 			break;
2378 		if (folio_test_readahead(folio))
2379 			break;
2380 		xas_advance(&xas, folio->index + folio_nr_pages(folio) - 1);
2381 		continue;
2382 put_folio:
2383 		folio_put(folio);
2384 retry:
2385 		xas_reset(&xas);
2386 	}
2387 	rcu_read_unlock();
2388 }
2389 
filemap_read_folio(struct file * file,filler_t filler,struct folio * folio)2390 static int filemap_read_folio(struct file *file, filler_t filler,
2391 		struct folio *folio)
2392 {
2393 	bool workingset = folio_test_workingset(folio);
2394 	unsigned long pflags;
2395 	int error;
2396 
2397 	/*
2398 	 * A previous I/O error may have been due to temporary failures,
2399 	 * eg. multipath errors.  PG_error will be set again if read_folio
2400 	 * fails.
2401 	 */
2402 	folio_clear_error(folio);
2403 
2404 	/* Start the actual read. The read will unlock the page. */
2405 	if (unlikely(workingset))
2406 		psi_memstall_enter(&pflags);
2407 	error = filler(file, folio);
2408 	if (unlikely(workingset))
2409 		psi_memstall_leave(&pflags);
2410 	if (error)
2411 		return error;
2412 
2413 	error = folio_wait_locked_killable(folio);
2414 	if (error)
2415 		return error;
2416 	if (folio_test_uptodate(folio))
2417 		return 0;
2418 	if (file)
2419 		shrink_readahead_size_eio(&file->f_ra);
2420 	return -EIO;
2421 }
2422 
filemap_range_uptodate(struct address_space * mapping,loff_t pos,struct iov_iter * iter,struct folio * folio)2423 static bool filemap_range_uptodate(struct address_space *mapping,
2424 		loff_t pos, struct iov_iter *iter, struct folio *folio)
2425 {
2426 	int count;
2427 
2428 	if (folio_test_uptodate(folio))
2429 		return true;
2430 	/* pipes can't handle partially uptodate pages */
2431 	if (iov_iter_is_pipe(iter))
2432 		return false;
2433 	if (!mapping->a_ops->is_partially_uptodate)
2434 		return false;
2435 	if (mapping->host->i_blkbits >= folio_shift(folio))
2436 		return false;
2437 
2438 	count = iter->count;
2439 	if (folio_pos(folio) > pos) {
2440 		count -= folio_pos(folio) - pos;
2441 		pos = 0;
2442 	} else {
2443 		pos -= folio_pos(folio);
2444 	}
2445 
2446 	return mapping->a_ops->is_partially_uptodate(folio, pos, count);
2447 }
2448 
filemap_update_page(struct kiocb * iocb,struct address_space * mapping,struct iov_iter * iter,struct folio * folio)2449 static int filemap_update_page(struct kiocb *iocb,
2450 		struct address_space *mapping, struct iov_iter *iter,
2451 		struct folio *folio)
2452 {
2453 	int error;
2454 
2455 	if (iocb->ki_flags & IOCB_NOWAIT) {
2456 		if (!filemap_invalidate_trylock_shared(mapping))
2457 			return -EAGAIN;
2458 	} else {
2459 		filemap_invalidate_lock_shared(mapping);
2460 	}
2461 
2462 	if (!folio_trylock(folio)) {
2463 		error = -EAGAIN;
2464 		if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))
2465 			goto unlock_mapping;
2466 		if (!(iocb->ki_flags & IOCB_WAITQ)) {
2467 			filemap_invalidate_unlock_shared(mapping);
2468 			/*
2469 			 * This is where we usually end up waiting for a
2470 			 * previously submitted readahead to finish.
2471 			 */
2472 			folio_put_wait_locked(folio, TASK_KILLABLE);
2473 			return AOP_TRUNCATED_PAGE;
2474 		}
2475 		error = __folio_lock_async(folio, iocb->ki_waitq);
2476 		if (error)
2477 			goto unlock_mapping;
2478 	}
2479 
2480 	error = AOP_TRUNCATED_PAGE;
2481 	if (!folio->mapping)
2482 		goto unlock;
2483 
2484 	error = 0;
2485 	if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, folio))
2486 		goto unlock;
2487 
2488 	error = -EAGAIN;
2489 	if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ))
2490 		goto unlock;
2491 
2492 	error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,
2493 			folio);
2494 	goto unlock_mapping;
2495 unlock:
2496 	folio_unlock(folio);
2497 unlock_mapping:
2498 	filemap_invalidate_unlock_shared(mapping);
2499 	if (error == AOP_TRUNCATED_PAGE)
2500 		folio_put(folio);
2501 	return error;
2502 }
2503 
filemap_create_folio(struct file * file,struct address_space * mapping,pgoff_t index,struct folio_batch * fbatch)2504 static int filemap_create_folio(struct file *file,
2505 		struct address_space *mapping, pgoff_t index,
2506 		struct folio_batch *fbatch)
2507 {
2508 	struct folio *folio;
2509 	int error;
2510 
2511 	folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0);
2512 	if (!folio)
2513 		return -ENOMEM;
2514 
2515 	/*
2516 	 * Protect against truncate / hole punch. Grabbing invalidate_lock
2517 	 * here assures we cannot instantiate and bring uptodate new
2518 	 * pagecache folios after evicting page cache during truncate
2519 	 * and before actually freeing blocks.	Note that we could
2520 	 * release invalidate_lock after inserting the folio into
2521 	 * the page cache as the locked folio would then be enough to
2522 	 * synchronize with hole punching. But there are code paths
2523 	 * such as filemap_update_page() filling in partially uptodate
2524 	 * pages or ->readahead() that need to hold invalidate_lock
2525 	 * while mapping blocks for IO so let's hold the lock here as
2526 	 * well to keep locking rules simple.
2527 	 */
2528 	filemap_invalidate_lock_shared(mapping);
2529 	error = filemap_add_folio(mapping, folio, index,
2530 			mapping_gfp_constraint(mapping, GFP_KERNEL));
2531 	if (error == -EEXIST)
2532 		error = AOP_TRUNCATED_PAGE;
2533 	if (error)
2534 		goto error;
2535 
2536 	error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
2537 	if (error)
2538 		goto error;
2539 
2540 	filemap_invalidate_unlock_shared(mapping);
2541 	folio_batch_add(fbatch, folio);
2542 	return 0;
2543 error:
2544 	filemap_invalidate_unlock_shared(mapping);
2545 	folio_put(folio);
2546 	return error;
2547 }
2548 
filemap_readahead(struct kiocb * iocb,struct file * file,struct address_space * mapping,struct folio * folio,pgoff_t last_index)2549 static int filemap_readahead(struct kiocb *iocb, struct file *file,
2550 		struct address_space *mapping, struct folio *folio,
2551 		pgoff_t last_index)
2552 {
2553 	DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index);
2554 
2555 	if (iocb->ki_flags & IOCB_NOIO)
2556 		return -EAGAIN;
2557 	page_cache_async_ra(&ractl, folio, last_index - folio->index);
2558 	return 0;
2559 }
2560 
filemap_get_pages(struct kiocb * iocb,struct iov_iter * iter,struct folio_batch * fbatch)2561 static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter,
2562 		struct folio_batch *fbatch)
2563 {
2564 	struct file *filp = iocb->ki_filp;
2565 	struct address_space *mapping = filp->f_mapping;
2566 	struct file_ra_state *ra = &filp->f_ra;
2567 	pgoff_t index = iocb->ki_pos >> PAGE_SHIFT;
2568 	pgoff_t last_index;
2569 	struct folio *folio;
2570 	int err = 0;
2571 
2572 	last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE);
2573 retry:
2574 	if (fatal_signal_pending(current))
2575 		return -EINTR;
2576 
2577 	filemap_get_read_batch(mapping, index, last_index, fbatch);
2578 	if (!folio_batch_count(fbatch)) {
2579 		if (iocb->ki_flags & IOCB_NOIO)
2580 			return -EAGAIN;
2581 		page_cache_sync_readahead(mapping, ra, filp, index,
2582 				last_index - index);
2583 		filemap_get_read_batch(mapping, index, last_index, fbatch);
2584 	}
2585 	if (!folio_batch_count(fbatch)) {
2586 		if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
2587 			return -EAGAIN;
2588 		err = filemap_create_folio(filp, mapping,
2589 				iocb->ki_pos >> PAGE_SHIFT, fbatch);
2590 		if (err == AOP_TRUNCATED_PAGE)
2591 			goto retry;
2592 		return err;
2593 	}
2594 
2595 	folio = fbatch->folios[folio_batch_count(fbatch) - 1];
2596 	if (folio_test_readahead(folio)) {
2597 		err = filemap_readahead(iocb, filp, mapping, folio, last_index);
2598 		if (err)
2599 			goto err;
2600 	}
2601 	if (!folio_test_uptodate(folio)) {
2602 		if ((iocb->ki_flags & IOCB_WAITQ) &&
2603 		    folio_batch_count(fbatch) > 1)
2604 			iocb->ki_flags |= IOCB_NOWAIT;
2605 		err = filemap_update_page(iocb, mapping, iter, folio);
2606 		if (err)
2607 			goto err;
2608 	}
2609 
2610 	return 0;
2611 err:
2612 	if (err < 0)
2613 		folio_put(folio);
2614 	if (likely(--fbatch->nr))
2615 		return 0;
2616 	if (err == AOP_TRUNCATED_PAGE)
2617 		goto retry;
2618 	return err;
2619 }
2620 
pos_same_folio(loff_t pos1,loff_t pos2,struct folio * folio)2621 static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio)
2622 {
2623 	unsigned int shift = folio_shift(folio);
2624 
2625 	return (pos1 >> shift == pos2 >> shift);
2626 }
2627 
2628 /**
2629  * filemap_read - Read data from the page cache.
2630  * @iocb: The iocb to read.
2631  * @iter: Destination for the data.
2632  * @already_read: Number of bytes already read by the caller.
2633  *
2634  * Copies data from the page cache.  If the data is not currently present,
2635  * uses the readahead and read_folio address_space operations to fetch it.
2636  *
2637  * Return: Total number of bytes copied, including those already read by
2638  * the caller.  If an error happens before any bytes are copied, returns
2639  * a negative error number.
2640  */
filemap_read(struct kiocb * iocb,struct iov_iter * iter,ssize_t already_read)2641 ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
2642 		ssize_t already_read)
2643 {
2644 	struct file *filp = iocb->ki_filp;
2645 	struct file_ra_state *ra = &filp->f_ra;
2646 	struct address_space *mapping = filp->f_mapping;
2647 	struct inode *inode = mapping->host;
2648 	struct folio_batch fbatch;
2649 	int i, error = 0;
2650 	bool writably_mapped;
2651 	loff_t isize, end_offset;
2652 
2653 	if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes))
2654 		return 0;
2655 	if (unlikely(!iov_iter_count(iter)))
2656 		return 0;
2657 
2658 	iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
2659 	folio_batch_init(&fbatch);
2660 
2661 	do {
2662 		cond_resched();
2663 
2664 		/*
2665 		 * If we've already successfully copied some data, then we
2666 		 * can no longer safely return -EIOCBQUEUED. Hence mark
2667 		 * an async read NOWAIT at that point.
2668 		 */
2669 		if ((iocb->ki_flags & IOCB_WAITQ) && already_read)
2670 			iocb->ki_flags |= IOCB_NOWAIT;
2671 
2672 		if (unlikely(iocb->ki_pos >= i_size_read(inode)))
2673 			break;
2674 
2675 		error = filemap_get_pages(iocb, iter, &fbatch);
2676 		if (error < 0)
2677 			break;
2678 
2679 		/*
2680 		 * i_size must be checked after we know the pages are Uptodate.
2681 		 *
2682 		 * Checking i_size after the check allows us to calculate
2683 		 * the correct value for "nr", which means the zero-filled
2684 		 * part of the page is not copied back to userspace (unless
2685 		 * another truncate extends the file - this is desired though).
2686 		 */
2687 		isize = i_size_read(inode);
2688 		if (unlikely(iocb->ki_pos >= isize))
2689 			goto put_folios;
2690 		end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count);
2691 
2692 		/*
2693 		 * Once we start copying data, we don't want to be touching any
2694 		 * cachelines that might be contended:
2695 		 */
2696 		writably_mapped = mapping_writably_mapped(mapping);
2697 
2698 		/*
2699 		 * When a read accesses the same folio several times, only
2700 		 * mark it as accessed the first time.
2701 		 */
2702 		if (!pos_same_folio(iocb->ki_pos, ra->prev_pos - 1,
2703 							fbatch.folios[0]))
2704 			folio_mark_accessed(fbatch.folios[0]);
2705 
2706 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
2707 			struct folio *folio = fbatch.folios[i];
2708 			size_t fsize = folio_size(folio);
2709 			size_t offset = iocb->ki_pos & (fsize - 1);
2710 			size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos,
2711 					     fsize - offset);
2712 			size_t copied;
2713 
2714 			if (end_offset < folio_pos(folio))
2715 				break;
2716 			if (i > 0)
2717 				folio_mark_accessed(folio);
2718 			/*
2719 			 * If users can be writing to this folio using arbitrary
2720 			 * virtual addresses, take care of potential aliasing
2721 			 * before reading the folio on the kernel side.
2722 			 */
2723 			if (writably_mapped)
2724 				flush_dcache_folio(folio);
2725 
2726 			copied = copy_folio_to_iter(folio, offset, bytes, iter);
2727 
2728 			already_read += copied;
2729 			iocb->ki_pos += copied;
2730 			ra->prev_pos = iocb->ki_pos;
2731 
2732 			if (copied < bytes) {
2733 				error = -EFAULT;
2734 				break;
2735 			}
2736 		}
2737 put_folios:
2738 		for (i = 0; i < folio_batch_count(&fbatch); i++)
2739 			folio_put(fbatch.folios[i]);
2740 		folio_batch_init(&fbatch);
2741 	} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
2742 
2743 	file_accessed(filp);
2744 
2745 	return already_read ? already_read : error;
2746 }
2747 EXPORT_SYMBOL_GPL(filemap_read);
2748 
2749 /**
2750  * generic_file_read_iter - generic filesystem read routine
2751  * @iocb:	kernel I/O control block
2752  * @iter:	destination for the data read
2753  *
2754  * This is the "read_iter()" routine for all filesystems
2755  * that can use the page cache directly.
2756  *
2757  * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2758  * be returned when no data can be read without waiting for I/O requests
2759  * to complete; it doesn't prevent readahead.
2760  *
2761  * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2762  * requests shall be made for the read or for readahead.  When no data
2763  * can be read, -EAGAIN shall be returned.  When readahead would be
2764  * triggered, a partial, possibly empty read shall be returned.
2765  *
2766  * Return:
2767  * * number of bytes copied, even for partial reads
2768  * * negative error code (or 0 if IOCB_NOIO) if nothing was read
2769  */
2770 ssize_t
generic_file_read_iter(struct kiocb * iocb,struct iov_iter * iter)2771 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2772 {
2773 	size_t count = iov_iter_count(iter);
2774 	ssize_t retval = 0;
2775 
2776 	if (!count)
2777 		return 0; /* skip atime */
2778 
2779 	if (iocb->ki_flags & IOCB_DIRECT) {
2780 		struct file *file = iocb->ki_filp;
2781 		struct address_space *mapping = file->f_mapping;
2782 		struct inode *inode = mapping->host;
2783 
2784 		if (iocb->ki_flags & IOCB_NOWAIT) {
2785 			if (filemap_range_needs_writeback(mapping, iocb->ki_pos,
2786 						iocb->ki_pos + count - 1))
2787 				return -EAGAIN;
2788 		} else {
2789 			retval = filemap_write_and_wait_range(mapping,
2790 						iocb->ki_pos,
2791 					        iocb->ki_pos + count - 1);
2792 			if (retval < 0)
2793 				return retval;
2794 		}
2795 
2796 		file_accessed(file);
2797 
2798 		retval = mapping->a_ops->direct_IO(iocb, iter);
2799 		if (retval >= 0) {
2800 			iocb->ki_pos += retval;
2801 			count -= retval;
2802 		}
2803 		if (retval != -EIOCBQUEUED)
2804 			iov_iter_revert(iter, count - iov_iter_count(iter));
2805 
2806 		/*
2807 		 * Btrfs can have a short DIO read if we encounter
2808 		 * compressed extents, so if there was an error, or if
2809 		 * we've already read everything we wanted to, or if
2810 		 * there was a short read because we hit EOF, go ahead
2811 		 * and return.  Otherwise fallthrough to buffered io for
2812 		 * the rest of the read.  Buffered reads will not work for
2813 		 * DAX files, so don't bother trying.
2814 		 */
2815 		if (retval < 0 || !count || IS_DAX(inode))
2816 			return retval;
2817 		if (iocb->ki_pos >= i_size_read(inode))
2818 			return retval;
2819 	}
2820 
2821 	return filemap_read(iocb, iter, retval);
2822 }
2823 EXPORT_SYMBOL(generic_file_read_iter);
2824 
folio_seek_hole_data(struct xa_state * xas,struct address_space * mapping,struct folio * folio,loff_t start,loff_t end,bool seek_data)2825 static inline loff_t folio_seek_hole_data(struct xa_state *xas,
2826 		struct address_space *mapping, struct folio *folio,
2827 		loff_t start, loff_t end, bool seek_data)
2828 {
2829 	const struct address_space_operations *ops = mapping->a_ops;
2830 	size_t offset, bsz = i_blocksize(mapping->host);
2831 
2832 	if (xa_is_value(folio) || folio_test_uptodate(folio))
2833 		return seek_data ? start : end;
2834 	if (!ops->is_partially_uptodate)
2835 		return seek_data ? end : start;
2836 
2837 	xas_pause(xas);
2838 	rcu_read_unlock();
2839 	folio_lock(folio);
2840 	if (unlikely(folio->mapping != mapping))
2841 		goto unlock;
2842 
2843 	offset = offset_in_folio(folio, start) & ~(bsz - 1);
2844 
2845 	do {
2846 		if (ops->is_partially_uptodate(folio, offset, bsz) ==
2847 							seek_data)
2848 			break;
2849 		start = (start + bsz) & ~(bsz - 1);
2850 		offset += bsz;
2851 	} while (offset < folio_size(folio));
2852 unlock:
2853 	folio_unlock(folio);
2854 	rcu_read_lock();
2855 	return start;
2856 }
2857 
seek_folio_size(struct xa_state * xas,struct folio * folio)2858 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
2859 {
2860 	if (xa_is_value(folio))
2861 		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
2862 	return folio_size(folio);
2863 }
2864 
2865 /**
2866  * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
2867  * @mapping: Address space to search.
2868  * @start: First byte to consider.
2869  * @end: Limit of search (exclusive).
2870  * @whence: Either SEEK_HOLE or SEEK_DATA.
2871  *
2872  * If the page cache knows which blocks contain holes and which blocks
2873  * contain data, your filesystem can use this function to implement
2874  * SEEK_HOLE and SEEK_DATA.  This is useful for filesystems which are
2875  * entirely memory-based such as tmpfs, and filesystems which support
2876  * unwritten extents.
2877  *
2878  * Return: The requested offset on success, or -ENXIO if @whence specifies
2879  * SEEK_DATA and there is no data after @start.  There is an implicit hole
2880  * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
2881  * and @end contain data.
2882  */
mapping_seek_hole_data(struct address_space * mapping,loff_t start,loff_t end,int whence)2883 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
2884 		loff_t end, int whence)
2885 {
2886 	XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);
2887 	pgoff_t max = (end - 1) >> PAGE_SHIFT;
2888 	bool seek_data = (whence == SEEK_DATA);
2889 	struct folio *folio;
2890 
2891 	if (end <= start)
2892 		return -ENXIO;
2893 
2894 	rcu_read_lock();
2895 	while ((folio = find_get_entry(&xas, max, XA_PRESENT))) {
2896 		loff_t pos = (u64)xas.xa_index << PAGE_SHIFT;
2897 		size_t seek_size;
2898 
2899 		if (start < pos) {
2900 			if (!seek_data)
2901 				goto unlock;
2902 			start = pos;
2903 		}
2904 
2905 		seek_size = seek_folio_size(&xas, folio);
2906 		pos = round_up((u64)pos + 1, seek_size);
2907 		start = folio_seek_hole_data(&xas, mapping, folio, start, pos,
2908 				seek_data);
2909 		if (start < pos)
2910 			goto unlock;
2911 		if (start >= end)
2912 			break;
2913 		if (seek_size > PAGE_SIZE)
2914 			xas_set(&xas, pos >> PAGE_SHIFT);
2915 		if (!xa_is_value(folio))
2916 			folio_put(folio);
2917 	}
2918 	if (seek_data)
2919 		start = -ENXIO;
2920 unlock:
2921 	rcu_read_unlock();
2922 	if (folio && !xa_is_value(folio))
2923 		folio_put(folio);
2924 	if (start > end)
2925 		return end;
2926 	return start;
2927 }
2928 
2929 #ifdef CONFIG_MMU
2930 #define MMAP_LOTSAMISS  (100)
2931 /*
2932  * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2933  * @vmf - the vm_fault for this fault.
2934  * @folio - the folio to lock.
2935  * @fpin - the pointer to the file we may pin (or is already pinned).
2936  *
2937  * This works similar to lock_folio_or_retry in that it can drop the
2938  * mmap_lock.  It differs in that it actually returns the folio locked
2939  * if it returns 1 and 0 if it couldn't lock the folio.  If we did have
2940  * to drop the mmap_lock then fpin will point to the pinned file and
2941  * needs to be fput()'ed at a later point.
2942  */
lock_folio_maybe_drop_mmap(struct vm_fault * vmf,struct folio * folio,struct file ** fpin)2943 static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio,
2944 				     struct file **fpin)
2945 {
2946 	if (folio_trylock(folio))
2947 		return 1;
2948 
2949 	/*
2950 	 * NOTE! This will make us return with VM_FAULT_RETRY, but with
2951 	 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT
2952 	 * is supposed to work. We have way too many special cases..
2953 	 */
2954 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
2955 		return 0;
2956 
2957 	*fpin = maybe_unlock_mmap_for_io(vmf, *fpin);
2958 	if (vmf->flags & FAULT_FLAG_KILLABLE) {
2959 		if (__folio_lock_killable(folio)) {
2960 			/*
2961 			 * We didn't have the right flags to drop the mmap_lock,
2962 			 * but all fault_handlers only check for fatal signals
2963 			 * if we return VM_FAULT_RETRY, so we need to drop the
2964 			 * mmap_lock here and return 0 if we don't have a fpin.
2965 			 */
2966 			if (*fpin == NULL)
2967 				mmap_read_unlock(vmf->vma->vm_mm);
2968 			return 0;
2969 		}
2970 	} else
2971 		__folio_lock(folio);
2972 
2973 	return 1;
2974 }
2975 
2976 /*
2977  * Synchronous readahead happens when we don't even find a page in the page
2978  * cache at all.  We don't want to perform IO under the mmap sem, so if we have
2979  * to drop the mmap sem we return the file that was pinned in order for us to do
2980  * that.  If we didn't pin a file then we return NULL.  The file that is
2981  * returned needs to be fput()'ed when we're done with it.
2982  */
do_sync_mmap_readahead(struct vm_fault * vmf)2983 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
2984 {
2985 	struct file *file = vmf->vma->vm_file;
2986 	struct file_ra_state *ra = &file->f_ra;
2987 	struct address_space *mapping = file->f_mapping;
2988 	DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff);
2989 	struct file *fpin = NULL;
2990 	unsigned long vm_flags = vmf->vma->vm_flags;
2991 	unsigned int mmap_miss;
2992 
2993 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2994 	/* Use the readahead code, even if readahead is disabled */
2995 	if (vm_flags & VM_HUGEPAGE) {
2996 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
2997 		ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1);
2998 		ra->size = HPAGE_PMD_NR;
2999 		/*
3000 		 * Fetch two PMD folios, so we get the chance to actually
3001 		 * readahead, unless we've been told not to.
3002 		 */
3003 		if (!(vm_flags & VM_RAND_READ))
3004 			ra->size *= 2;
3005 		ra->async_size = HPAGE_PMD_NR;
3006 		page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER);
3007 		return fpin;
3008 	}
3009 #endif
3010 
3011 	/* If we don't want any read-ahead, don't bother */
3012 	if (vm_flags & VM_RAND_READ)
3013 		return fpin;
3014 	if (!ra->ra_pages)
3015 		return fpin;
3016 
3017 	if (vm_flags & VM_SEQ_READ) {
3018 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3019 		page_cache_sync_ra(&ractl, ra->ra_pages);
3020 		return fpin;
3021 	}
3022 
3023 	/* Avoid banging the cache line if not needed */
3024 	mmap_miss = READ_ONCE(ra->mmap_miss);
3025 	if (mmap_miss < MMAP_LOTSAMISS * 10)
3026 		WRITE_ONCE(ra->mmap_miss, ++mmap_miss);
3027 
3028 	/*
3029 	 * Do we miss much more than hit in this file? If so,
3030 	 * stop bothering with read-ahead. It will only hurt.
3031 	 */
3032 	if (mmap_miss > MMAP_LOTSAMISS)
3033 		return fpin;
3034 
3035 	/*
3036 	 * mmap read-around
3037 	 */
3038 	fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3039 	ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
3040 	ra->size = ra->ra_pages;
3041 	ra->async_size = ra->ra_pages / 4;
3042 	ractl._index = ra->start;
3043 	page_cache_ra_order(&ractl, ra, 0);
3044 	return fpin;
3045 }
3046 
3047 /*
3048  * Asynchronous readahead happens when we find the page and PG_readahead,
3049  * so we want to possibly extend the readahead further.  We return the file that
3050  * was pinned if we have to drop the mmap_lock in order to do IO.
3051  */
do_async_mmap_readahead(struct vm_fault * vmf,struct folio * folio)3052 static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
3053 					    struct folio *folio)
3054 {
3055 	struct file *file = vmf->vma->vm_file;
3056 	struct file_ra_state *ra = &file->f_ra;
3057 	DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff);
3058 	struct file *fpin = NULL;
3059 	unsigned int mmap_miss;
3060 
3061 	/* If we don't want any read-ahead, don't bother */
3062 	if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
3063 		return fpin;
3064 
3065 	mmap_miss = READ_ONCE(ra->mmap_miss);
3066 	if (mmap_miss)
3067 		WRITE_ONCE(ra->mmap_miss, --mmap_miss);
3068 
3069 	if (folio_test_readahead(folio)) {
3070 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3071 		page_cache_async_ra(&ractl, folio, ra->ra_pages);
3072 	}
3073 	return fpin;
3074 }
3075 
3076 /**
3077  * filemap_fault - read in file data for page fault handling
3078  * @vmf:	struct vm_fault containing details of the fault
3079  *
3080  * filemap_fault() is invoked via the vma operations vector for a
3081  * mapped memory region to read in file data during a page fault.
3082  *
3083  * The goto's are kind of ugly, but this streamlines the normal case of having
3084  * it in the page cache, and handles the special cases reasonably without
3085  * having a lot of duplicated code.
3086  *
3087  * vma->vm_mm->mmap_lock must be held on entry.
3088  *
3089  * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
3090  * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap().
3091  *
3092  * If our return value does not have VM_FAULT_RETRY set, the mmap_lock
3093  * has not been released.
3094  *
3095  * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
3096  *
3097  * Return: bitwise-OR of %VM_FAULT_ codes.
3098  */
filemap_fault(struct vm_fault * vmf)3099 vm_fault_t filemap_fault(struct vm_fault *vmf)
3100 {
3101 	int error;
3102 	struct file *file = vmf->vma->vm_file;
3103 	struct file *fpin = NULL;
3104 	struct address_space *mapping = file->f_mapping;
3105 	struct inode *inode = mapping->host;
3106 	pgoff_t max_idx, index = vmf->pgoff;
3107 	struct folio *folio;
3108 	vm_fault_t ret = 0;
3109 	bool mapping_locked = false;
3110 
3111 	max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3112 	if (unlikely(index >= max_idx))
3113 		return VM_FAULT_SIGBUS;
3114 
3115 	/*
3116 	 * Do we have something in the page cache already?
3117 	 */
3118 	folio = filemap_get_folio(mapping, index);
3119 	if (likely(folio)) {
3120 		/*
3121 		 * We found the page, so try async readahead before waiting for
3122 		 * the lock.
3123 		 */
3124 		if (!(vmf->flags & FAULT_FLAG_TRIED))
3125 			fpin = do_async_mmap_readahead(vmf, folio);
3126 		if (unlikely(!folio_test_uptodate(folio))) {
3127 			filemap_invalidate_lock_shared(mapping);
3128 			mapping_locked = true;
3129 		}
3130 	} else {
3131 		/* No page in the page cache at all */
3132 		count_vm_event(PGMAJFAULT);
3133 		count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
3134 		ret = VM_FAULT_MAJOR;
3135 		fpin = do_sync_mmap_readahead(vmf);
3136 retry_find:
3137 		/*
3138 		 * See comment in filemap_create_folio() why we need
3139 		 * invalidate_lock
3140 		 */
3141 		if (!mapping_locked) {
3142 			filemap_invalidate_lock_shared(mapping);
3143 			mapping_locked = true;
3144 		}
3145 		folio = __filemap_get_folio(mapping, index,
3146 					  FGP_CREAT|FGP_FOR_MMAP,
3147 					  vmf->gfp_mask);
3148 		if (!folio) {
3149 			if (fpin)
3150 				goto out_retry;
3151 			filemap_invalidate_unlock_shared(mapping);
3152 			return VM_FAULT_OOM;
3153 		}
3154 	}
3155 
3156 	if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin))
3157 		goto out_retry;
3158 
3159 	/* Did it get truncated? */
3160 	if (unlikely(folio->mapping != mapping)) {
3161 		folio_unlock(folio);
3162 		folio_put(folio);
3163 		goto retry_find;
3164 	}
3165 	VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
3166 
3167 	/*
3168 	 * We have a locked page in the page cache, now we need to check
3169 	 * that it's up-to-date. If not, it is going to be due to an error.
3170 	 */
3171 	if (unlikely(!folio_test_uptodate(folio))) {
3172 		/*
3173 		 * The page was in cache and uptodate and now it is not.
3174 		 * Strange but possible since we didn't hold the page lock all
3175 		 * the time. Let's drop everything get the invalidate lock and
3176 		 * try again.
3177 		 */
3178 		if (!mapping_locked) {
3179 			folio_unlock(folio);
3180 			folio_put(folio);
3181 			goto retry_find;
3182 		}
3183 		goto page_not_uptodate;
3184 	}
3185 
3186 	/*
3187 	 * We've made it this far and we had to drop our mmap_lock, now is the
3188 	 * time to return to the upper layer and have it re-find the vma and
3189 	 * redo the fault.
3190 	 */
3191 	if (fpin) {
3192 		folio_unlock(folio);
3193 		goto out_retry;
3194 	}
3195 	if (mapping_locked)
3196 		filemap_invalidate_unlock_shared(mapping);
3197 
3198 	/*
3199 	 * Found the page and have a reference on it.
3200 	 * We must recheck i_size under page lock.
3201 	 */
3202 	max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3203 	if (unlikely(index >= max_idx)) {
3204 		folio_unlock(folio);
3205 		folio_put(folio);
3206 		return VM_FAULT_SIGBUS;
3207 	}
3208 
3209 	vmf->page = folio_file_page(folio, index);
3210 	return ret | VM_FAULT_LOCKED;
3211 
3212 page_not_uptodate:
3213 	/*
3214 	 * Umm, take care of errors if the page isn't up-to-date.
3215 	 * Try to re-read it _once_. We do this synchronously,
3216 	 * because there really aren't any performance issues here
3217 	 * and we need to check for errors.
3218 	 */
3219 	fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3220 	error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
3221 	if (fpin)
3222 		goto out_retry;
3223 	folio_put(folio);
3224 
3225 	if (!error || error == AOP_TRUNCATED_PAGE)
3226 		goto retry_find;
3227 	filemap_invalidate_unlock_shared(mapping);
3228 
3229 	return VM_FAULT_SIGBUS;
3230 
3231 out_retry:
3232 	/*
3233 	 * We dropped the mmap_lock, we need to return to the fault handler to
3234 	 * re-find the vma and come back and find our hopefully still populated
3235 	 * page.
3236 	 */
3237 	if (folio)
3238 		folio_put(folio);
3239 	if (mapping_locked)
3240 		filemap_invalidate_unlock_shared(mapping);
3241 	if (fpin)
3242 		fput(fpin);
3243 	return ret | VM_FAULT_RETRY;
3244 }
3245 EXPORT_SYMBOL(filemap_fault);
3246 
filemap_map_pmd(struct vm_fault * vmf,struct page * page)3247 static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page)
3248 {
3249 	struct mm_struct *mm = vmf->vma->vm_mm;
3250 
3251 	/* Huge page is mapped? No need to proceed. */
3252 	if (pmd_trans_huge(*vmf->pmd)) {
3253 		unlock_page(page);
3254 		put_page(page);
3255 		return true;
3256 	}
3257 
3258 	if (pmd_none(*vmf->pmd) && PageTransHuge(page)) {
3259 		vm_fault_t ret = do_set_pmd(vmf, page);
3260 		if (!ret) {
3261 			/* The page is mapped successfully, reference consumed. */
3262 			unlock_page(page);
3263 			return true;
3264 		}
3265 	}
3266 
3267 	if (pmd_none(*vmf->pmd))
3268 		pmd_install(mm, vmf->pmd, &vmf->prealloc_pte);
3269 
3270 	/* See comment in handle_pte_fault() */
3271 	if (pmd_devmap_trans_unstable(vmf->pmd)) {
3272 		unlock_page(page);
3273 		put_page(page);
3274 		return true;
3275 	}
3276 
3277 	return false;
3278 }
3279 
next_uptodate_page(struct folio * folio,struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3280 static struct folio *next_uptodate_page(struct folio *folio,
3281 				       struct address_space *mapping,
3282 				       struct xa_state *xas, pgoff_t end_pgoff)
3283 {
3284 	unsigned long max_idx;
3285 
3286 	do {
3287 		if (!folio)
3288 			return NULL;
3289 		if (xas_retry(xas, folio))
3290 			continue;
3291 		if (xa_is_value(folio))
3292 			continue;
3293 		if (folio_test_locked(folio))
3294 			continue;
3295 		if (!folio_try_get_rcu(folio))
3296 			continue;
3297 		/* Has the page moved or been split? */
3298 		if (unlikely(folio != xas_reload(xas)))
3299 			goto skip;
3300 		if (!folio_test_uptodate(folio) || folio_test_readahead(folio))
3301 			goto skip;
3302 		if (!folio_trylock(folio))
3303 			goto skip;
3304 		if (folio->mapping != mapping)
3305 			goto unlock;
3306 		if (!folio_test_uptodate(folio))
3307 			goto unlock;
3308 		max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
3309 		if (xas->xa_index >= max_idx)
3310 			goto unlock;
3311 		return folio;
3312 unlock:
3313 		folio_unlock(folio);
3314 skip:
3315 		folio_put(folio);
3316 	} while ((folio = xas_next_entry(xas, end_pgoff)) != NULL);
3317 
3318 	return NULL;
3319 }
3320 
first_map_page(struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3321 static inline struct folio *first_map_page(struct address_space *mapping,
3322 					  struct xa_state *xas,
3323 					  pgoff_t end_pgoff)
3324 {
3325 	return next_uptodate_page(xas_find(xas, end_pgoff),
3326 				  mapping, xas, end_pgoff);
3327 }
3328 
next_map_page(struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3329 static inline struct folio *next_map_page(struct address_space *mapping,
3330 					 struct xa_state *xas,
3331 					 pgoff_t end_pgoff)
3332 {
3333 	return next_uptodate_page(xas_next_entry(xas, end_pgoff),
3334 				  mapping, xas, end_pgoff);
3335 }
3336 
filemap_map_pages(struct vm_fault * vmf,pgoff_t start_pgoff,pgoff_t end_pgoff)3337 vm_fault_t filemap_map_pages(struct vm_fault *vmf,
3338 			     pgoff_t start_pgoff, pgoff_t end_pgoff)
3339 {
3340 	struct vm_area_struct *vma = vmf->vma;
3341 	struct file *file = vma->vm_file;
3342 	struct address_space *mapping = file->f_mapping;
3343 	pgoff_t last_pgoff = start_pgoff;
3344 	unsigned long addr;
3345 	XA_STATE(xas, &mapping->i_pages, start_pgoff);
3346 	struct folio *folio;
3347 	struct page *page;
3348 	unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss);
3349 	vm_fault_t ret = 0;
3350 
3351 	rcu_read_lock();
3352 	folio = first_map_page(mapping, &xas, end_pgoff);
3353 	if (!folio)
3354 		goto out;
3355 
3356 	if (filemap_map_pmd(vmf, &folio->page)) {
3357 		ret = VM_FAULT_NOPAGE;
3358 		goto out;
3359 	}
3360 
3361 	addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
3362 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
3363 	do {
3364 again:
3365 		page = folio_file_page(folio, xas.xa_index);
3366 		if (PageHWPoison(page))
3367 			goto unlock;
3368 
3369 		if (mmap_miss > 0)
3370 			mmap_miss--;
3371 
3372 		addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
3373 		vmf->pte += xas.xa_index - last_pgoff;
3374 		last_pgoff = xas.xa_index;
3375 
3376 		/*
3377 		 * NOTE: If there're PTE markers, we'll leave them to be
3378 		 * handled in the specific fault path, and it'll prohibit the
3379 		 * fault-around logic.
3380 		 */
3381 		if (!pte_none(*vmf->pte))
3382 			goto unlock;
3383 
3384 		/* We're about to handle the fault */
3385 		if (vmf->address == addr)
3386 			ret = VM_FAULT_NOPAGE;
3387 
3388 		do_set_pte(vmf, page, addr);
3389 		/* no need to invalidate: a not-present page won't be cached */
3390 		update_mmu_cache(vma, addr, vmf->pte);
3391 		if (folio_more_pages(folio, xas.xa_index, end_pgoff)) {
3392 			xas.xa_index++;
3393 			folio_ref_inc(folio);
3394 			goto again;
3395 		}
3396 		folio_unlock(folio);
3397 		continue;
3398 unlock:
3399 		if (folio_more_pages(folio, xas.xa_index, end_pgoff)) {
3400 			xas.xa_index++;
3401 			goto again;
3402 		}
3403 		folio_unlock(folio);
3404 		folio_put(folio);
3405 	} while ((folio = next_map_page(mapping, &xas, end_pgoff)) != NULL);
3406 	pte_unmap_unlock(vmf->pte, vmf->ptl);
3407 out:
3408 	rcu_read_unlock();
3409 	WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss);
3410 	return ret;
3411 }
3412 EXPORT_SYMBOL(filemap_map_pages);
3413 
filemap_page_mkwrite(struct vm_fault * vmf)3414 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
3415 {
3416 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
3417 	struct folio *folio = page_folio(vmf->page);
3418 	vm_fault_t ret = VM_FAULT_LOCKED;
3419 
3420 	sb_start_pagefault(mapping->host->i_sb);
3421 	file_update_time(vmf->vma->vm_file);
3422 	folio_lock(folio);
3423 	if (folio->mapping != mapping) {
3424 		folio_unlock(folio);
3425 		ret = VM_FAULT_NOPAGE;
3426 		goto out;
3427 	}
3428 	/*
3429 	 * We mark the folio dirty already here so that when freeze is in
3430 	 * progress, we are guaranteed that writeback during freezing will
3431 	 * see the dirty folio and writeprotect it again.
3432 	 */
3433 	folio_mark_dirty(folio);
3434 	folio_wait_stable(folio);
3435 out:
3436 	sb_end_pagefault(mapping->host->i_sb);
3437 	return ret;
3438 }
3439 
3440 const struct vm_operations_struct generic_file_vm_ops = {
3441 	.fault		= filemap_fault,
3442 	.map_pages	= filemap_map_pages,
3443 	.page_mkwrite	= filemap_page_mkwrite,
3444 };
3445 
3446 /* This is used for a general mmap of a disk file */
3447 
generic_file_mmap(struct file * file,struct vm_area_struct * vma)3448 int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
3449 {
3450 	struct address_space *mapping = file->f_mapping;
3451 
3452 	if (!mapping->a_ops->read_folio)
3453 		return -ENOEXEC;
3454 	file_accessed(file);
3455 	vma->vm_ops = &generic_file_vm_ops;
3456 	return 0;
3457 }
3458 
3459 /*
3460  * This is for filesystems which do not implement ->writepage.
3461  */
generic_file_readonly_mmap(struct file * file,struct vm_area_struct * vma)3462 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
3463 {
3464 	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
3465 		return -EINVAL;
3466 	return generic_file_mmap(file, vma);
3467 }
3468 #else
filemap_page_mkwrite(struct vm_fault * vmf)3469 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
3470 {
3471 	return VM_FAULT_SIGBUS;
3472 }
generic_file_mmap(struct file * file,struct vm_area_struct * vma)3473 int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
3474 {
3475 	return -ENOSYS;
3476 }
generic_file_readonly_mmap(struct file * file,struct vm_area_struct * vma)3477 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
3478 {
3479 	return -ENOSYS;
3480 }
3481 #endif /* CONFIG_MMU */
3482 
3483 EXPORT_SYMBOL(filemap_page_mkwrite);
3484 EXPORT_SYMBOL(generic_file_mmap);
3485 EXPORT_SYMBOL(generic_file_readonly_mmap);
3486 
do_read_cache_folio(struct address_space * mapping,pgoff_t index,filler_t filler,struct file * file,gfp_t gfp)3487 static struct folio *do_read_cache_folio(struct address_space *mapping,
3488 		pgoff_t index, filler_t filler, struct file *file, gfp_t gfp)
3489 {
3490 	struct folio *folio;
3491 	int err;
3492 
3493 	if (!filler)
3494 		filler = mapping->a_ops->read_folio;
3495 repeat:
3496 	folio = filemap_get_folio(mapping, index);
3497 	if (!folio) {
3498 		folio = filemap_alloc_folio(gfp, 0);
3499 		if (!folio)
3500 			return ERR_PTR(-ENOMEM);
3501 		err = filemap_add_folio(mapping, folio, index, gfp);
3502 		if (unlikely(err)) {
3503 			folio_put(folio);
3504 			if (err == -EEXIST)
3505 				goto repeat;
3506 			/* Presumably ENOMEM for xarray node */
3507 			return ERR_PTR(err);
3508 		}
3509 
3510 		goto filler;
3511 	}
3512 	if (folio_test_uptodate(folio))
3513 		goto out;
3514 
3515 	if (!folio_trylock(folio)) {
3516 		folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE);
3517 		goto repeat;
3518 	}
3519 
3520 	/* Folio was truncated from mapping */
3521 	if (!folio->mapping) {
3522 		folio_unlock(folio);
3523 		folio_put(folio);
3524 		goto repeat;
3525 	}
3526 
3527 	/* Someone else locked and filled the page in a very small window */
3528 	if (folio_test_uptodate(folio)) {
3529 		folio_unlock(folio);
3530 		goto out;
3531 	}
3532 
3533 filler:
3534 	err = filemap_read_folio(file, filler, folio);
3535 	if (err) {
3536 		folio_put(folio);
3537 		if (err == AOP_TRUNCATED_PAGE)
3538 			goto repeat;
3539 		return ERR_PTR(err);
3540 	}
3541 
3542 out:
3543 	folio_mark_accessed(folio);
3544 	return folio;
3545 }
3546 
3547 /**
3548  * read_cache_folio - Read into page cache, fill it if needed.
3549  * @mapping: The address_space to read from.
3550  * @index: The index to read.
3551  * @filler: Function to perform the read, or NULL to use aops->read_folio().
3552  * @file: Passed to filler function, may be NULL if not required.
3553  *
3554  * Read one page into the page cache.  If it succeeds, the folio returned
3555  * will contain @index, but it may not be the first page of the folio.
3556  *
3557  * If the filler function returns an error, it will be returned to the
3558  * caller.
3559  *
3560  * Context: May sleep.  Expects mapping->invalidate_lock to be held.
3561  * Return: An uptodate folio on success, ERR_PTR() on failure.
3562  */
read_cache_folio(struct address_space * mapping,pgoff_t index,filler_t filler,struct file * file)3563 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index,
3564 		filler_t filler, struct file *file)
3565 {
3566 	return do_read_cache_folio(mapping, index, filler, file,
3567 			mapping_gfp_mask(mapping));
3568 }
3569 EXPORT_SYMBOL(read_cache_folio);
3570 
do_read_cache_page(struct address_space * mapping,pgoff_t index,filler_t * filler,struct file * file,gfp_t gfp)3571 static struct page *do_read_cache_page(struct address_space *mapping,
3572 		pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp)
3573 {
3574 	struct folio *folio;
3575 
3576 	folio = do_read_cache_folio(mapping, index, filler, file, gfp);
3577 	if (IS_ERR(folio))
3578 		return &folio->page;
3579 	return folio_file_page(folio, index);
3580 }
3581 
read_cache_page(struct address_space * mapping,pgoff_t index,filler_t * filler,struct file * file)3582 struct page *read_cache_page(struct address_space *mapping,
3583 			pgoff_t index, filler_t *filler, struct file *file)
3584 {
3585 	return do_read_cache_page(mapping, index, filler, file,
3586 			mapping_gfp_mask(mapping));
3587 }
3588 EXPORT_SYMBOL(read_cache_page);
3589 
3590 /**
3591  * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3592  * @mapping:	the page's address_space
3593  * @index:	the page index
3594  * @gfp:	the page allocator flags to use if allocating
3595  *
3596  * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3597  * any new page allocations done using the specified allocation flags.
3598  *
3599  * If the page does not get brought uptodate, return -EIO.
3600  *
3601  * The function expects mapping->invalidate_lock to be already held.
3602  *
3603  * Return: up to date page on success, ERR_PTR() on failure.
3604  */
read_cache_page_gfp(struct address_space * mapping,pgoff_t index,gfp_t gfp)3605 struct page *read_cache_page_gfp(struct address_space *mapping,
3606 				pgoff_t index,
3607 				gfp_t gfp)
3608 {
3609 	return do_read_cache_page(mapping, index, NULL, NULL, gfp);
3610 }
3611 EXPORT_SYMBOL(read_cache_page_gfp);
3612 
3613 /*
3614  * Warn about a page cache invalidation failure during a direct I/O write.
3615  */
dio_warn_stale_pagecache(struct file * filp)3616 void dio_warn_stale_pagecache(struct file *filp)
3617 {
3618 	static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
3619 	char pathname[128];
3620 	char *path;
3621 
3622 	errseq_set(&filp->f_mapping->wb_err, -EIO);
3623 	if (__ratelimit(&_rs)) {
3624 		path = file_path(filp, pathname, sizeof(pathname));
3625 		if (IS_ERR(path))
3626 			path = "(unknown)";
3627 		pr_crit("Page cache invalidation failure on direct I/O.  Possible data corruption due to collision with buffered I/O!\n");
3628 		pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
3629 			current->comm);
3630 	}
3631 }
3632 
3633 ssize_t
generic_file_direct_write(struct kiocb * iocb,struct iov_iter * from)3634 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
3635 {
3636 	struct file	*file = iocb->ki_filp;
3637 	struct address_space *mapping = file->f_mapping;
3638 	struct inode	*inode = mapping->host;
3639 	loff_t		pos = iocb->ki_pos;
3640 	ssize_t		written;
3641 	size_t		write_len;
3642 	pgoff_t		end;
3643 
3644 	write_len = iov_iter_count(from);
3645 	end = (pos + write_len - 1) >> PAGE_SHIFT;
3646 
3647 	if (iocb->ki_flags & IOCB_NOWAIT) {
3648 		/* If there are pages to writeback, return */
3649 		if (filemap_range_has_page(file->f_mapping, pos,
3650 					   pos + write_len - 1))
3651 			return -EAGAIN;
3652 	} else {
3653 		written = filemap_write_and_wait_range(mapping, pos,
3654 							pos + write_len - 1);
3655 		if (written)
3656 			goto out;
3657 	}
3658 
3659 	/*
3660 	 * After a write we want buffered reads to be sure to go to disk to get
3661 	 * the new data.  We invalidate clean cached page from the region we're
3662 	 * about to write.  We do this *before* the write so that we can return
3663 	 * without clobbering -EIOCBQUEUED from ->direct_IO().
3664 	 */
3665 	written = invalidate_inode_pages2_range(mapping,
3666 					pos >> PAGE_SHIFT, end);
3667 	/*
3668 	 * If a page can not be invalidated, return 0 to fall back
3669 	 * to buffered write.
3670 	 */
3671 	if (written) {
3672 		if (written == -EBUSY)
3673 			return 0;
3674 		goto out;
3675 	}
3676 
3677 	written = mapping->a_ops->direct_IO(iocb, from);
3678 
3679 	/*
3680 	 * Finally, try again to invalidate clean pages which might have been
3681 	 * cached by non-direct readahead, or faulted in by get_user_pages()
3682 	 * if the source of the write was an mmap'ed region of the file
3683 	 * we're writing.  Either one is a pretty crazy thing to do,
3684 	 * so we don't support it 100%.  If this invalidation
3685 	 * fails, tough, the write still worked...
3686 	 *
3687 	 * Most of the time we do not need this since dio_complete() will do
3688 	 * the invalidation for us. However there are some file systems that
3689 	 * do not end up with dio_complete() being called, so let's not break
3690 	 * them by removing it completely.
3691 	 *
3692 	 * Noticeable example is a blkdev_direct_IO().
3693 	 *
3694 	 * Skip invalidation for async writes or if mapping has no pages.
3695 	 */
3696 	if (written > 0 && mapping->nrpages &&
3697 	    invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))
3698 		dio_warn_stale_pagecache(file);
3699 
3700 	if (written > 0) {
3701 		pos += written;
3702 		write_len -= written;
3703 		if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
3704 			i_size_write(inode, pos);
3705 			mark_inode_dirty(inode);
3706 		}
3707 		iocb->ki_pos = pos;
3708 	}
3709 	if (written != -EIOCBQUEUED)
3710 		iov_iter_revert(from, write_len - iov_iter_count(from));
3711 out:
3712 	return written;
3713 }
3714 EXPORT_SYMBOL(generic_file_direct_write);
3715 
generic_perform_write(struct kiocb * iocb,struct iov_iter * i)3716 ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i)
3717 {
3718 	struct file *file = iocb->ki_filp;
3719 	loff_t pos = iocb->ki_pos;
3720 	struct address_space *mapping = file->f_mapping;
3721 	const struct address_space_operations *a_ops = mapping->a_ops;
3722 	long status = 0;
3723 	ssize_t written = 0;
3724 
3725 	do {
3726 		struct page *page;
3727 		unsigned long offset;	/* Offset into pagecache page */
3728 		unsigned long bytes;	/* Bytes to write to page */
3729 		size_t copied;		/* Bytes copied from user */
3730 		void *fsdata = NULL;
3731 
3732 		offset = (pos & (PAGE_SIZE - 1));
3733 		bytes = min_t(unsigned long, PAGE_SIZE - offset,
3734 						iov_iter_count(i));
3735 
3736 again:
3737 		/*
3738 		 * Bring in the user page that we will copy from _first_.
3739 		 * Otherwise there's a nasty deadlock on copying from the
3740 		 * same page as we're writing to, without it being marked
3741 		 * up-to-date.
3742 		 */
3743 		if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
3744 			status = -EFAULT;
3745 			break;
3746 		}
3747 
3748 		if (fatal_signal_pending(current)) {
3749 			status = -EINTR;
3750 			break;
3751 		}
3752 
3753 		status = a_ops->write_begin(file, mapping, pos, bytes,
3754 						&page, &fsdata);
3755 		if (unlikely(status < 0))
3756 			break;
3757 
3758 		if (mapping_writably_mapped(mapping))
3759 			flush_dcache_page(page);
3760 
3761 		copied = copy_page_from_iter_atomic(page, offset, bytes, i);
3762 		flush_dcache_page(page);
3763 
3764 		status = a_ops->write_end(file, mapping, pos, bytes, copied,
3765 						page, fsdata);
3766 		if (unlikely(status != copied)) {
3767 			iov_iter_revert(i, copied - max(status, 0L));
3768 			if (unlikely(status < 0))
3769 				break;
3770 		}
3771 		cond_resched();
3772 
3773 		if (unlikely(status == 0)) {
3774 			/*
3775 			 * A short copy made ->write_end() reject the
3776 			 * thing entirely.  Might be memory poisoning
3777 			 * halfway through, might be a race with munmap,
3778 			 * might be severe memory pressure.
3779 			 */
3780 			if (copied)
3781 				bytes = copied;
3782 			goto again;
3783 		}
3784 		pos += status;
3785 		written += status;
3786 
3787 		balance_dirty_pages_ratelimited(mapping);
3788 	} while (iov_iter_count(i));
3789 
3790 	return written ? written : status;
3791 }
3792 EXPORT_SYMBOL(generic_perform_write);
3793 
3794 /**
3795  * __generic_file_write_iter - write data to a file
3796  * @iocb:	IO state structure (file, offset, etc.)
3797  * @from:	iov_iter with data to write
3798  *
3799  * This function does all the work needed for actually writing data to a
3800  * file. It does all basic checks, removes SUID from the file, updates
3801  * modification times and calls proper subroutines depending on whether we
3802  * do direct IO or a standard buffered write.
3803  *
3804  * It expects i_rwsem to be grabbed unless we work on a block device or similar
3805  * object which does not need locking at all.
3806  *
3807  * This function does *not* take care of syncing data in case of O_SYNC write.
3808  * A caller has to handle it. This is mainly due to the fact that we want to
3809  * avoid syncing under i_rwsem.
3810  *
3811  * Return:
3812  * * number of bytes written, even for truncated writes
3813  * * negative error code if no data has been written at all
3814  */
__generic_file_write_iter(struct kiocb * iocb,struct iov_iter * from)3815 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3816 {
3817 	struct file *file = iocb->ki_filp;
3818 	struct address_space *mapping = file->f_mapping;
3819 	struct inode 	*inode = mapping->host;
3820 	ssize_t		written = 0;
3821 	ssize_t		err;
3822 	ssize_t		status;
3823 
3824 	/* We can write back this queue in page reclaim */
3825 	current->backing_dev_info = inode_to_bdi(inode);
3826 	err = file_remove_privs(file);
3827 	if (err)
3828 		goto out;
3829 
3830 	err = file_update_time(file);
3831 	if (err)
3832 		goto out;
3833 
3834 	if (iocb->ki_flags & IOCB_DIRECT) {
3835 		loff_t pos, endbyte;
3836 
3837 		written = generic_file_direct_write(iocb, from);
3838 		/*
3839 		 * If the write stopped short of completing, fall back to
3840 		 * buffered writes.  Some filesystems do this for writes to
3841 		 * holes, for example.  For DAX files, a buffered write will
3842 		 * not succeed (even if it did, DAX does not handle dirty
3843 		 * page-cache pages correctly).
3844 		 */
3845 		if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
3846 			goto out;
3847 
3848 		pos = iocb->ki_pos;
3849 		status = generic_perform_write(iocb, from);
3850 		/*
3851 		 * If generic_perform_write() returned a synchronous error
3852 		 * then we want to return the number of bytes which were
3853 		 * direct-written, or the error code if that was zero.  Note
3854 		 * that this differs from normal direct-io semantics, which
3855 		 * will return -EFOO even if some bytes were written.
3856 		 */
3857 		if (unlikely(status < 0)) {
3858 			err = status;
3859 			goto out;
3860 		}
3861 		/*
3862 		 * We need to ensure that the page cache pages are written to
3863 		 * disk and invalidated to preserve the expected O_DIRECT
3864 		 * semantics.
3865 		 */
3866 		endbyte = pos + status - 1;
3867 		err = filemap_write_and_wait_range(mapping, pos, endbyte);
3868 		if (err == 0) {
3869 			iocb->ki_pos = endbyte + 1;
3870 			written += status;
3871 			invalidate_mapping_pages(mapping,
3872 						 pos >> PAGE_SHIFT,
3873 						 endbyte >> PAGE_SHIFT);
3874 		} else {
3875 			/*
3876 			 * We don't know how much we wrote, so just return
3877 			 * the number of bytes which were direct-written
3878 			 */
3879 		}
3880 	} else {
3881 		written = generic_perform_write(iocb, from);
3882 		if (likely(written > 0))
3883 			iocb->ki_pos += written;
3884 	}
3885 out:
3886 	current->backing_dev_info = NULL;
3887 	return written ? written : err;
3888 }
3889 EXPORT_SYMBOL(__generic_file_write_iter);
3890 
3891 /**
3892  * generic_file_write_iter - write data to a file
3893  * @iocb:	IO state structure
3894  * @from:	iov_iter with data to write
3895  *
3896  * This is a wrapper around __generic_file_write_iter() to be used by most
3897  * filesystems. It takes care of syncing the file in case of O_SYNC file
3898  * and acquires i_rwsem as needed.
3899  * Return:
3900  * * negative error code if no data has been written at all of
3901  *   vfs_fsync_range() failed for a synchronous write
3902  * * number of bytes written, even for truncated writes
3903  */
generic_file_write_iter(struct kiocb * iocb,struct iov_iter * from)3904 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3905 {
3906 	struct file *file = iocb->ki_filp;
3907 	struct inode *inode = file->f_mapping->host;
3908 	ssize_t ret;
3909 
3910 	inode_lock(inode);
3911 	ret = generic_write_checks(iocb, from);
3912 	if (ret > 0)
3913 		ret = __generic_file_write_iter(iocb, from);
3914 	inode_unlock(inode);
3915 
3916 	if (ret > 0)
3917 		ret = generic_write_sync(iocb, ret);
3918 	return ret;
3919 }
3920 EXPORT_SYMBOL(generic_file_write_iter);
3921 
3922 /**
3923  * filemap_release_folio() - Release fs-specific metadata on a folio.
3924  * @folio: The folio which the kernel is trying to free.
3925  * @gfp: Memory allocation flags (and I/O mode).
3926  *
3927  * The address_space is trying to release any data attached to a folio
3928  * (presumably at folio->private).
3929  *
3930  * This will also be called if the private_2 flag is set on a page,
3931  * indicating that the folio has other metadata associated with it.
3932  *
3933  * The @gfp argument specifies whether I/O may be performed to release
3934  * this page (__GFP_IO), and whether the call may block
3935  * (__GFP_RECLAIM & __GFP_FS).
3936  *
3937  * Return: %true if the release was successful, otherwise %false.
3938  */
filemap_release_folio(struct folio * folio,gfp_t gfp)3939 bool filemap_release_folio(struct folio *folio, gfp_t gfp)
3940 {
3941 	struct address_space * const mapping = folio->mapping;
3942 
3943 	BUG_ON(!folio_test_locked(folio));
3944 	if (folio_test_writeback(folio))
3945 		return false;
3946 
3947 	if (mapping && mapping->a_ops->release_folio)
3948 		return mapping->a_ops->release_folio(folio, gfp);
3949 	return try_to_free_buffers(folio);
3950 }
3951 EXPORT_SYMBOL(filemap_release_folio);
3952