1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
4  *
5  *  Swap reorganised 29.12.95, Stephen Tweedie.
6  *  kswapd added: 7.1.96  sct
7  *  Removed kswapd_ctl limits, and swap out as many pages as needed
8  *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9  *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
10  *  Multiqueue VM started 5.8.00, Rik van Riel.
11  */
12 
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 
15 #include <linux/mm.h>
16 #include <linux/sched/mm.h>
17 #include <linux/module.h>
18 #include <linux/gfp.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/swap.h>
21 #include <linux/pagemap.h>
22 #include <linux/init.h>
23 #include <linux/highmem.h>
24 #include <linux/vmpressure.h>
25 #include <linux/vmstat.h>
26 #include <linux/file.h>
27 #include <linux/writeback.h>
28 #include <linux/blkdev.h>
29 #include <linux/buffer_head.h>	/* for try_to_release_page(),
30 					buffer_heads_over_limit */
31 #include <linux/mm_inline.h>
32 #include <linux/backing-dev.h>
33 #include <linux/rmap.h>
34 #include <linux/topology.h>
35 #include <linux/cpu.h>
36 #include <linux/cpuset.h>
37 #include <linux/compaction.h>
38 #include <linux/notifier.h>
39 #include <linux/rwsem.h>
40 #include <linux/delay.h>
41 #include <linux/kthread.h>
42 #include <linux/freezer.h>
43 #include <linux/memcontrol.h>
44 #include <linux/migrate.h>
45 #include <linux/delayacct.h>
46 #include <linux/sysctl.h>
47 #include <linux/oom.h>
48 #include <linux/pagevec.h>
49 #include <linux/prefetch.h>
50 #include <linux/printk.h>
51 #include <linux/dax.h>
52 #include <linux/psi.h>
53 
54 #include <asm/tlbflush.h>
55 #include <asm/div64.h>
56 
57 #include <linux/swapops.h>
58 #include <linux/balloon_compaction.h>
59 #include <linux/sched/sysctl.h>
60 
61 #include "internal.h"
62 #include "swap.h"
63 
64 #define CREATE_TRACE_POINTS
65 #include <trace/events/vmscan.h>
66 
67 struct scan_control {
68 	/* How many pages shrink_list() should reclaim */
69 	unsigned long nr_to_reclaim;
70 
71 	/*
72 	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
73 	 * are scanned.
74 	 */
75 	nodemask_t	*nodemask;
76 
77 	/*
78 	 * The memory cgroup that hit its limit and as a result is the
79 	 * primary target of this reclaim invocation.
80 	 */
81 	struct mem_cgroup *target_mem_cgroup;
82 
83 	/*
84 	 * Scan pressure balancing between anon and file LRUs
85 	 */
86 	unsigned long	anon_cost;
87 	unsigned long	file_cost;
88 
89 	/* Can active pages be deactivated as part of reclaim? */
90 #define DEACTIVATE_ANON 1
91 #define DEACTIVATE_FILE 2
92 	unsigned int may_deactivate:2;
93 	unsigned int force_deactivate:1;
94 	unsigned int skipped_deactivate:1;
95 
96 	/* Writepage batching in laptop mode; RECLAIM_WRITE */
97 	unsigned int may_writepage:1;
98 
99 	/* Can mapped pages be reclaimed? */
100 	unsigned int may_unmap:1;
101 
102 	/* Can pages be swapped as part of reclaim? */
103 	unsigned int may_swap:1;
104 
105 	/*
106 	 * Cgroup memory below memory.low is protected as long as we
107 	 * don't threaten to OOM. If any cgroup is reclaimed at
108 	 * reduced force or passed over entirely due to its memory.low
109 	 * setting (memcg_low_skipped), and nothing is reclaimed as a
110 	 * result, then go back for one more cycle that reclaims the protected
111 	 * memory (memcg_low_reclaim) to avert OOM.
112 	 */
113 	unsigned int memcg_low_reclaim:1;
114 	unsigned int memcg_low_skipped:1;
115 
116 	unsigned int hibernation_mode:1;
117 
118 	/* One of the zones is ready for compaction */
119 	unsigned int compaction_ready:1;
120 
121 	/* There is easily reclaimable cold cache in the current node */
122 	unsigned int cache_trim_mode:1;
123 
124 	/* The file pages on the current node are dangerously low */
125 	unsigned int file_is_tiny:1;
126 
127 	/* Always discard instead of demoting to lower tier memory */
128 	unsigned int no_demotion:1;
129 
130 	/* Allocation order */
131 	s8 order;
132 
133 	/* Scan (total_size >> priority) pages at once */
134 	s8 priority;
135 
136 	/* The highest zone to isolate pages for reclaim from */
137 	s8 reclaim_idx;
138 
139 	/* This context's GFP mask */
140 	gfp_t gfp_mask;
141 
142 	/* Incremented by the number of inactive pages that were scanned */
143 	unsigned long nr_scanned;
144 
145 	/* Number of pages freed so far during a call to shrink_zones() */
146 	unsigned long nr_reclaimed;
147 
148 	struct {
149 		unsigned int dirty;
150 		unsigned int unqueued_dirty;
151 		unsigned int congested;
152 		unsigned int writeback;
153 		unsigned int immediate;
154 		unsigned int file_taken;
155 		unsigned int taken;
156 	} nr;
157 
158 	/* for recording the reclaimed slab by now */
159 	struct reclaim_state reclaim_state;
160 };
161 
162 #ifdef ARCH_HAS_PREFETCHW
163 #define prefetchw_prev_lru_page(_page, _base, _field)			\
164 	do {								\
165 		if ((_page)->lru.prev != _base) {			\
166 			struct page *prev;				\
167 									\
168 			prev = lru_to_page(&(_page->lru));		\
169 			prefetchw(&prev->_field);			\
170 		}							\
171 	} while (0)
172 #else
173 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
174 #endif
175 
176 /*
177  * From 0 .. 200.  Higher means more swappy.
178  */
179 int vm_swappiness = 60;
180 
set_task_reclaim_state(struct task_struct * task,struct reclaim_state * rs)181 static void set_task_reclaim_state(struct task_struct *task,
182 				   struct reclaim_state *rs)
183 {
184 	/* Check for an overwrite */
185 	WARN_ON_ONCE(rs && task->reclaim_state);
186 
187 	/* Check for the nulling of an already-nulled member */
188 	WARN_ON_ONCE(!rs && !task->reclaim_state);
189 
190 	task->reclaim_state = rs;
191 }
192 
193 static LIST_HEAD(shrinker_list);
194 static DECLARE_RWSEM(shrinker_rwsem);
195 
196 #ifdef CONFIG_MEMCG
197 static int shrinker_nr_max;
198 
199 /* The shrinker_info is expanded in a batch of BITS_PER_LONG */
shrinker_map_size(int nr_items)200 static inline int shrinker_map_size(int nr_items)
201 {
202 	return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long));
203 }
204 
shrinker_defer_size(int nr_items)205 static inline int shrinker_defer_size(int nr_items)
206 {
207 	return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t));
208 }
209 
shrinker_info_protected(struct mem_cgroup * memcg,int nid)210 static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
211 						     int nid)
212 {
213 	return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
214 					 lockdep_is_held(&shrinker_rwsem));
215 }
216 
expand_one_shrinker_info(struct mem_cgroup * memcg,int map_size,int defer_size,int old_map_size,int old_defer_size)217 static int expand_one_shrinker_info(struct mem_cgroup *memcg,
218 				    int map_size, int defer_size,
219 				    int old_map_size, int old_defer_size)
220 {
221 	struct shrinker_info *new, *old;
222 	struct mem_cgroup_per_node *pn;
223 	int nid;
224 	int size = map_size + defer_size;
225 
226 	for_each_node(nid) {
227 		pn = memcg->nodeinfo[nid];
228 		old = shrinker_info_protected(memcg, nid);
229 		/* Not yet online memcg */
230 		if (!old)
231 			return 0;
232 
233 		new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
234 		if (!new)
235 			return -ENOMEM;
236 
237 		new->nr_deferred = (atomic_long_t *)(new + 1);
238 		new->map = (void *)new->nr_deferred + defer_size;
239 
240 		/* map: set all old bits, clear all new bits */
241 		memset(new->map, (int)0xff, old_map_size);
242 		memset((void *)new->map + old_map_size, 0, map_size - old_map_size);
243 		/* nr_deferred: copy old values, clear all new values */
244 		memcpy(new->nr_deferred, old->nr_deferred, old_defer_size);
245 		memset((void *)new->nr_deferred + old_defer_size, 0,
246 		       defer_size - old_defer_size);
247 
248 		rcu_assign_pointer(pn->shrinker_info, new);
249 		kvfree_rcu(old, rcu);
250 	}
251 
252 	return 0;
253 }
254 
free_shrinker_info(struct mem_cgroup * memcg)255 void free_shrinker_info(struct mem_cgroup *memcg)
256 {
257 	struct mem_cgroup_per_node *pn;
258 	struct shrinker_info *info;
259 	int nid;
260 
261 	for_each_node(nid) {
262 		pn = memcg->nodeinfo[nid];
263 		info = rcu_dereference_protected(pn->shrinker_info, true);
264 		kvfree(info);
265 		rcu_assign_pointer(pn->shrinker_info, NULL);
266 	}
267 }
268 
alloc_shrinker_info(struct mem_cgroup * memcg)269 int alloc_shrinker_info(struct mem_cgroup *memcg)
270 {
271 	struct shrinker_info *info;
272 	int nid, size, ret = 0;
273 	int map_size, defer_size = 0;
274 
275 	down_write(&shrinker_rwsem);
276 	map_size = shrinker_map_size(shrinker_nr_max);
277 	defer_size = shrinker_defer_size(shrinker_nr_max);
278 	size = map_size + defer_size;
279 	for_each_node(nid) {
280 		info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid);
281 		if (!info) {
282 			free_shrinker_info(memcg);
283 			ret = -ENOMEM;
284 			break;
285 		}
286 		info->nr_deferred = (atomic_long_t *)(info + 1);
287 		info->map = (void *)info->nr_deferred + defer_size;
288 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
289 	}
290 	up_write(&shrinker_rwsem);
291 
292 	return ret;
293 }
294 
need_expand(int nr_max)295 static inline bool need_expand(int nr_max)
296 {
297 	return round_up(nr_max, BITS_PER_LONG) >
298 	       round_up(shrinker_nr_max, BITS_PER_LONG);
299 }
300 
expand_shrinker_info(int new_id)301 static int expand_shrinker_info(int new_id)
302 {
303 	int ret = 0;
304 	int new_nr_max = new_id + 1;
305 	int map_size, defer_size = 0;
306 	int old_map_size, old_defer_size = 0;
307 	struct mem_cgroup *memcg;
308 
309 	if (!need_expand(new_nr_max))
310 		goto out;
311 
312 	if (!root_mem_cgroup)
313 		goto out;
314 
315 	lockdep_assert_held(&shrinker_rwsem);
316 
317 	map_size = shrinker_map_size(new_nr_max);
318 	defer_size = shrinker_defer_size(new_nr_max);
319 	old_map_size = shrinker_map_size(shrinker_nr_max);
320 	old_defer_size = shrinker_defer_size(shrinker_nr_max);
321 
322 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
323 	do {
324 		ret = expand_one_shrinker_info(memcg, map_size, defer_size,
325 					       old_map_size, old_defer_size);
326 		if (ret) {
327 			mem_cgroup_iter_break(NULL, memcg);
328 			goto out;
329 		}
330 	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
331 out:
332 	if (!ret)
333 		shrinker_nr_max = new_nr_max;
334 
335 	return ret;
336 }
337 
set_shrinker_bit(struct mem_cgroup * memcg,int nid,int shrinker_id)338 void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
339 {
340 	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
341 		struct shrinker_info *info;
342 
343 		rcu_read_lock();
344 		info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
345 		/* Pairs with smp mb in shrink_slab() */
346 		smp_mb__before_atomic();
347 		set_bit(shrinker_id, info->map);
348 		rcu_read_unlock();
349 	}
350 }
351 
352 static DEFINE_IDR(shrinker_idr);
353 
prealloc_memcg_shrinker(struct shrinker * shrinker)354 static int prealloc_memcg_shrinker(struct shrinker *shrinker)
355 {
356 	int id, ret = -ENOMEM;
357 
358 	if (mem_cgroup_disabled())
359 		return -ENOSYS;
360 
361 	down_write(&shrinker_rwsem);
362 	/* This may call shrinker, so it must use down_read_trylock() */
363 	id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
364 	if (id < 0)
365 		goto unlock;
366 
367 	if (id >= shrinker_nr_max) {
368 		if (expand_shrinker_info(id)) {
369 			idr_remove(&shrinker_idr, id);
370 			goto unlock;
371 		}
372 	}
373 	shrinker->id = id;
374 	ret = 0;
375 unlock:
376 	up_write(&shrinker_rwsem);
377 	return ret;
378 }
379 
unregister_memcg_shrinker(struct shrinker * shrinker)380 static void unregister_memcg_shrinker(struct shrinker *shrinker)
381 {
382 	int id = shrinker->id;
383 
384 	BUG_ON(id < 0);
385 
386 	lockdep_assert_held(&shrinker_rwsem);
387 
388 	idr_remove(&shrinker_idr, id);
389 }
390 
xchg_nr_deferred_memcg(int nid,struct shrinker * shrinker,struct mem_cgroup * memcg)391 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
392 				   struct mem_cgroup *memcg)
393 {
394 	struct shrinker_info *info;
395 
396 	info = shrinker_info_protected(memcg, nid);
397 	return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0);
398 }
399 
add_nr_deferred_memcg(long nr,int nid,struct shrinker * shrinker,struct mem_cgroup * memcg)400 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
401 				  struct mem_cgroup *memcg)
402 {
403 	struct shrinker_info *info;
404 
405 	info = shrinker_info_protected(memcg, nid);
406 	return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]);
407 }
408 
reparent_shrinker_deferred(struct mem_cgroup * memcg)409 void reparent_shrinker_deferred(struct mem_cgroup *memcg)
410 {
411 	int i, nid;
412 	long nr;
413 	struct mem_cgroup *parent;
414 	struct shrinker_info *child_info, *parent_info;
415 
416 	parent = parent_mem_cgroup(memcg);
417 	if (!parent)
418 		parent = root_mem_cgroup;
419 
420 	/* Prevent from concurrent shrinker_info expand */
421 	down_read(&shrinker_rwsem);
422 	for_each_node(nid) {
423 		child_info = shrinker_info_protected(memcg, nid);
424 		parent_info = shrinker_info_protected(parent, nid);
425 		for (i = 0; i < shrinker_nr_max; i++) {
426 			nr = atomic_long_read(&child_info->nr_deferred[i]);
427 			atomic_long_add(nr, &parent_info->nr_deferred[i]);
428 		}
429 	}
430 	up_read(&shrinker_rwsem);
431 }
432 
cgroup_reclaim(struct scan_control * sc)433 static bool cgroup_reclaim(struct scan_control *sc)
434 {
435 	return sc->target_mem_cgroup;
436 }
437 
438 /**
439  * writeback_throttling_sane - is the usual dirty throttling mechanism available?
440  * @sc: scan_control in question
441  *
442  * The normal page dirty throttling mechanism in balance_dirty_pages() is
443  * completely broken with the legacy memcg and direct stalling in
444  * shrink_page_list() is used for throttling instead, which lacks all the
445  * niceties such as fairness, adaptive pausing, bandwidth proportional
446  * allocation and configurability.
447  *
448  * This function tests whether the vmscan currently in progress can assume
449  * that the normal dirty throttling mechanism is operational.
450  */
writeback_throttling_sane(struct scan_control * sc)451 static bool writeback_throttling_sane(struct scan_control *sc)
452 {
453 	if (!cgroup_reclaim(sc))
454 		return true;
455 #ifdef CONFIG_CGROUP_WRITEBACK
456 	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
457 		return true;
458 #endif
459 	return false;
460 }
461 #else
prealloc_memcg_shrinker(struct shrinker * shrinker)462 static int prealloc_memcg_shrinker(struct shrinker *shrinker)
463 {
464 	return -ENOSYS;
465 }
466 
unregister_memcg_shrinker(struct shrinker * shrinker)467 static void unregister_memcg_shrinker(struct shrinker *shrinker)
468 {
469 }
470 
xchg_nr_deferred_memcg(int nid,struct shrinker * shrinker,struct mem_cgroup * memcg)471 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
472 				   struct mem_cgroup *memcg)
473 {
474 	return 0;
475 }
476 
add_nr_deferred_memcg(long nr,int nid,struct shrinker * shrinker,struct mem_cgroup * memcg)477 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
478 				  struct mem_cgroup *memcg)
479 {
480 	return 0;
481 }
482 
cgroup_reclaim(struct scan_control * sc)483 static bool cgroup_reclaim(struct scan_control *sc)
484 {
485 	return false;
486 }
487 
writeback_throttling_sane(struct scan_control * sc)488 static bool writeback_throttling_sane(struct scan_control *sc)
489 {
490 	return true;
491 }
492 #endif
493 
xchg_nr_deferred(struct shrinker * shrinker,struct shrink_control * sc)494 static long xchg_nr_deferred(struct shrinker *shrinker,
495 			     struct shrink_control *sc)
496 {
497 	int nid = sc->nid;
498 
499 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
500 		nid = 0;
501 
502 	if (sc->memcg &&
503 	    (shrinker->flags & SHRINKER_MEMCG_AWARE))
504 		return xchg_nr_deferred_memcg(nid, shrinker,
505 					      sc->memcg);
506 
507 	return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
508 }
509 
510 
add_nr_deferred(long nr,struct shrinker * shrinker,struct shrink_control * sc)511 static long add_nr_deferred(long nr, struct shrinker *shrinker,
512 			    struct shrink_control *sc)
513 {
514 	int nid = sc->nid;
515 
516 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
517 		nid = 0;
518 
519 	if (sc->memcg &&
520 	    (shrinker->flags & SHRINKER_MEMCG_AWARE))
521 		return add_nr_deferred_memcg(nr, nid, shrinker,
522 					     sc->memcg);
523 
524 	return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
525 }
526 
can_demote(int nid,struct scan_control * sc)527 static bool can_demote(int nid, struct scan_control *sc)
528 {
529 	if (!numa_demotion_enabled)
530 		return false;
531 	if (sc && sc->no_demotion)
532 		return false;
533 	if (next_demotion_node(nid) == NUMA_NO_NODE)
534 		return false;
535 
536 	return true;
537 }
538 
can_reclaim_anon_pages(struct mem_cgroup * memcg,int nid,struct scan_control * sc)539 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
540 					  int nid,
541 					  struct scan_control *sc)
542 {
543 	if (memcg == NULL) {
544 		/*
545 		 * For non-memcg reclaim, is there
546 		 * space in any swap device?
547 		 */
548 		if (get_nr_swap_pages() > 0)
549 			return true;
550 	} else {
551 		/* Is the memcg below its swap limit? */
552 		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
553 			return true;
554 	}
555 
556 	/*
557 	 * The page can not be swapped.
558 	 *
559 	 * Can it be reclaimed from this node via demotion?
560 	 */
561 	return can_demote(nid, sc);
562 }
563 
564 /*
565  * This misses isolated pages which are not accounted for to save counters.
566  * As the data only determines if reclaim or compaction continues, it is
567  * not expected that isolated pages will be a dominating factor.
568  */
zone_reclaimable_pages(struct zone * zone)569 unsigned long zone_reclaimable_pages(struct zone *zone)
570 {
571 	unsigned long nr;
572 
573 	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
574 		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
575 	if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL))
576 		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
577 			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
578 
579 	return nr;
580 }
581 
582 /**
583  * lruvec_lru_size -  Returns the number of pages on the given LRU list.
584  * @lruvec: lru vector
585  * @lru: lru to use
586  * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list)
587  */
lruvec_lru_size(struct lruvec * lruvec,enum lru_list lru,int zone_idx)588 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
589 				     int zone_idx)
590 {
591 	unsigned long size = 0;
592 	int zid;
593 
594 	for (zid = 0; zid <= zone_idx; zid++) {
595 		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
596 
597 		if (!managed_zone(zone))
598 			continue;
599 
600 		if (!mem_cgroup_disabled())
601 			size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
602 		else
603 			size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
604 	}
605 	return size;
606 }
607 
608 /*
609  * Add a shrinker callback to be called from the vm.
610  */
prealloc_shrinker(struct shrinker * shrinker)611 int prealloc_shrinker(struct shrinker *shrinker)
612 {
613 	unsigned int size;
614 	int err;
615 
616 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
617 		err = prealloc_memcg_shrinker(shrinker);
618 		if (err != -ENOSYS)
619 			return err;
620 
621 		shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
622 	}
623 
624 	size = sizeof(*shrinker->nr_deferred);
625 	if (shrinker->flags & SHRINKER_NUMA_AWARE)
626 		size *= nr_node_ids;
627 
628 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
629 	if (!shrinker->nr_deferred)
630 		return -ENOMEM;
631 
632 	return 0;
633 }
634 
free_prealloced_shrinker(struct shrinker * shrinker)635 void free_prealloced_shrinker(struct shrinker *shrinker)
636 {
637 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
638 		down_write(&shrinker_rwsem);
639 		unregister_memcg_shrinker(shrinker);
640 		up_write(&shrinker_rwsem);
641 		return;
642 	}
643 
644 	kfree(shrinker->nr_deferred);
645 	shrinker->nr_deferred = NULL;
646 }
647 
register_shrinker_prepared(struct shrinker * shrinker)648 void register_shrinker_prepared(struct shrinker *shrinker)
649 {
650 	down_write(&shrinker_rwsem);
651 	list_add_tail(&shrinker->list, &shrinker_list);
652 	shrinker->flags |= SHRINKER_REGISTERED;
653 	up_write(&shrinker_rwsem);
654 }
655 
register_shrinker(struct shrinker * shrinker)656 int register_shrinker(struct shrinker *shrinker)
657 {
658 	int err = prealloc_shrinker(shrinker);
659 
660 	if (err)
661 		return err;
662 	register_shrinker_prepared(shrinker);
663 	return 0;
664 }
665 EXPORT_SYMBOL(register_shrinker);
666 
667 /*
668  * Remove one
669  */
unregister_shrinker(struct shrinker * shrinker)670 void unregister_shrinker(struct shrinker *shrinker)
671 {
672 	if (!(shrinker->flags & SHRINKER_REGISTERED))
673 		return;
674 
675 	down_write(&shrinker_rwsem);
676 	list_del(&shrinker->list);
677 	shrinker->flags &= ~SHRINKER_REGISTERED;
678 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
679 		unregister_memcg_shrinker(shrinker);
680 	up_write(&shrinker_rwsem);
681 
682 	kfree(shrinker->nr_deferred);
683 	shrinker->nr_deferred = NULL;
684 }
685 EXPORT_SYMBOL(unregister_shrinker);
686 
687 /**
688  * synchronize_shrinkers - Wait for all running shrinkers to complete.
689  *
690  * This is equivalent to calling unregister_shrink() and register_shrinker(),
691  * but atomically and with less overhead. This is useful to guarantee that all
692  * shrinker invocations have seen an update, before freeing memory, similar to
693  * rcu.
694  */
synchronize_shrinkers(void)695 void synchronize_shrinkers(void)
696 {
697 	down_write(&shrinker_rwsem);
698 	up_write(&shrinker_rwsem);
699 }
700 EXPORT_SYMBOL(synchronize_shrinkers);
701 
702 #define SHRINK_BATCH 128
703 
do_shrink_slab(struct shrink_control * shrinkctl,struct shrinker * shrinker,int priority)704 static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
705 				    struct shrinker *shrinker, int priority)
706 {
707 	unsigned long freed = 0;
708 	unsigned long long delta;
709 	long total_scan;
710 	long freeable;
711 	long nr;
712 	long new_nr;
713 	long batch_size = shrinker->batch ? shrinker->batch
714 					  : SHRINK_BATCH;
715 	long scanned = 0, next_deferred;
716 
717 	freeable = shrinker->count_objects(shrinker, shrinkctl);
718 	if (freeable == 0 || freeable == SHRINK_EMPTY)
719 		return freeable;
720 
721 	/*
722 	 * copy the current shrinker scan count into a local variable
723 	 * and zero it so that other concurrent shrinker invocations
724 	 * don't also do this scanning work.
725 	 */
726 	nr = xchg_nr_deferred(shrinker, shrinkctl);
727 
728 	if (shrinker->seeks) {
729 		delta = freeable >> priority;
730 		delta *= 4;
731 		do_div(delta, shrinker->seeks);
732 	} else {
733 		/*
734 		 * These objects don't require any IO to create. Trim
735 		 * them aggressively under memory pressure to keep
736 		 * them from causing refetches in the IO caches.
737 		 */
738 		delta = freeable / 2;
739 	}
740 
741 	total_scan = nr >> priority;
742 	total_scan += delta;
743 	total_scan = min(total_scan, (2 * freeable));
744 
745 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
746 				   freeable, delta, total_scan, priority);
747 
748 	/*
749 	 * Normally, we should not scan less than batch_size objects in one
750 	 * pass to avoid too frequent shrinker calls, but if the slab has less
751 	 * than batch_size objects in total and we are really tight on memory,
752 	 * we will try to reclaim all available objects, otherwise we can end
753 	 * up failing allocations although there are plenty of reclaimable
754 	 * objects spread over several slabs with usage less than the
755 	 * batch_size.
756 	 *
757 	 * We detect the "tight on memory" situations by looking at the total
758 	 * number of objects we want to scan (total_scan). If it is greater
759 	 * than the total number of objects on slab (freeable), we must be
760 	 * scanning at high prio and therefore should try to reclaim as much as
761 	 * possible.
762 	 */
763 	while (total_scan >= batch_size ||
764 	       total_scan >= freeable) {
765 		unsigned long ret;
766 		unsigned long nr_to_scan = min(batch_size, total_scan);
767 
768 		shrinkctl->nr_to_scan = nr_to_scan;
769 		shrinkctl->nr_scanned = nr_to_scan;
770 		ret = shrinker->scan_objects(shrinker, shrinkctl);
771 		if (ret == SHRINK_STOP)
772 			break;
773 		freed += ret;
774 
775 		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
776 		total_scan -= shrinkctl->nr_scanned;
777 		scanned += shrinkctl->nr_scanned;
778 
779 		cond_resched();
780 	}
781 
782 	/*
783 	 * The deferred work is increased by any new work (delta) that wasn't
784 	 * done, decreased by old deferred work that was done now.
785 	 *
786 	 * And it is capped to two times of the freeable items.
787 	 */
788 	next_deferred = max_t(long, (nr + delta - scanned), 0);
789 	next_deferred = min(next_deferred, (2 * freeable));
790 
791 	/*
792 	 * move the unused scan count back into the shrinker in a
793 	 * manner that handles concurrent updates.
794 	 */
795 	new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
796 
797 	trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
798 	return freed;
799 }
800 
801 #ifdef CONFIG_MEMCG
shrink_slab_memcg(gfp_t gfp_mask,int nid,struct mem_cgroup * memcg,int priority)802 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
803 			struct mem_cgroup *memcg, int priority)
804 {
805 	struct shrinker_info *info;
806 	unsigned long ret, freed = 0;
807 	int i;
808 
809 	if (!mem_cgroup_online(memcg))
810 		return 0;
811 
812 	if (!down_read_trylock(&shrinker_rwsem))
813 		return 0;
814 
815 	info = shrinker_info_protected(memcg, nid);
816 	if (unlikely(!info))
817 		goto unlock;
818 
819 	for_each_set_bit(i, info->map, shrinker_nr_max) {
820 		struct shrink_control sc = {
821 			.gfp_mask = gfp_mask,
822 			.nid = nid,
823 			.memcg = memcg,
824 		};
825 		struct shrinker *shrinker;
826 
827 		shrinker = idr_find(&shrinker_idr, i);
828 		if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
829 			if (!shrinker)
830 				clear_bit(i, info->map);
831 			continue;
832 		}
833 
834 		/* Call non-slab shrinkers even though kmem is disabled */
835 		if (!memcg_kmem_enabled() &&
836 		    !(shrinker->flags & SHRINKER_NONSLAB))
837 			continue;
838 
839 		ret = do_shrink_slab(&sc, shrinker, priority);
840 		if (ret == SHRINK_EMPTY) {
841 			clear_bit(i, info->map);
842 			/*
843 			 * After the shrinker reported that it had no objects to
844 			 * free, but before we cleared the corresponding bit in
845 			 * the memcg shrinker map, a new object might have been
846 			 * added. To make sure, we have the bit set in this
847 			 * case, we invoke the shrinker one more time and reset
848 			 * the bit if it reports that it is not empty anymore.
849 			 * The memory barrier here pairs with the barrier in
850 			 * set_shrinker_bit():
851 			 *
852 			 * list_lru_add()     shrink_slab_memcg()
853 			 *   list_add_tail()    clear_bit()
854 			 *   <MB>               <MB>
855 			 *   set_bit()          do_shrink_slab()
856 			 */
857 			smp_mb__after_atomic();
858 			ret = do_shrink_slab(&sc, shrinker, priority);
859 			if (ret == SHRINK_EMPTY)
860 				ret = 0;
861 			else
862 				set_shrinker_bit(memcg, nid, i);
863 		}
864 		freed += ret;
865 
866 		if (rwsem_is_contended(&shrinker_rwsem)) {
867 			freed = freed ? : 1;
868 			break;
869 		}
870 	}
871 unlock:
872 	up_read(&shrinker_rwsem);
873 	return freed;
874 }
875 #else /* CONFIG_MEMCG */
shrink_slab_memcg(gfp_t gfp_mask,int nid,struct mem_cgroup * memcg,int priority)876 static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
877 			struct mem_cgroup *memcg, int priority)
878 {
879 	return 0;
880 }
881 #endif /* CONFIG_MEMCG */
882 
883 /**
884  * shrink_slab - shrink slab caches
885  * @gfp_mask: allocation context
886  * @nid: node whose slab caches to target
887  * @memcg: memory cgroup whose slab caches to target
888  * @priority: the reclaim priority
889  *
890  * Call the shrink functions to age shrinkable caches.
891  *
892  * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
893  * unaware shrinkers will receive a node id of 0 instead.
894  *
895  * @memcg specifies the memory cgroup to target. Unaware shrinkers
896  * are called only if it is the root cgroup.
897  *
898  * @priority is sc->priority, we take the number of objects and >> by priority
899  * in order to get the scan target.
900  *
901  * Returns the number of reclaimed slab objects.
902  */
shrink_slab(gfp_t gfp_mask,int nid,struct mem_cgroup * memcg,int priority)903 static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
904 				 struct mem_cgroup *memcg,
905 				 int priority)
906 {
907 	unsigned long ret, freed = 0;
908 	struct shrinker *shrinker;
909 
910 	/*
911 	 * The root memcg might be allocated even though memcg is disabled
912 	 * via "cgroup_disable=memory" boot parameter.  This could make
913 	 * mem_cgroup_is_root() return false, then just run memcg slab
914 	 * shrink, but skip global shrink.  This may result in premature
915 	 * oom.
916 	 */
917 	if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
918 		return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
919 
920 	if (!down_read_trylock(&shrinker_rwsem))
921 		goto out;
922 
923 	list_for_each_entry(shrinker, &shrinker_list, list) {
924 		struct shrink_control sc = {
925 			.gfp_mask = gfp_mask,
926 			.nid = nid,
927 			.memcg = memcg,
928 		};
929 
930 		ret = do_shrink_slab(&sc, shrinker, priority);
931 		if (ret == SHRINK_EMPTY)
932 			ret = 0;
933 		freed += ret;
934 		/*
935 		 * Bail out if someone want to register a new shrinker to
936 		 * prevent the registration from being stalled for long periods
937 		 * by parallel ongoing shrinking.
938 		 */
939 		if (rwsem_is_contended(&shrinker_rwsem)) {
940 			freed = freed ? : 1;
941 			break;
942 		}
943 	}
944 
945 	up_read(&shrinker_rwsem);
946 out:
947 	cond_resched();
948 	return freed;
949 }
950 
drop_slab_node(int nid)951 static void drop_slab_node(int nid)
952 {
953 	unsigned long freed;
954 	int shift = 0;
955 
956 	do {
957 		struct mem_cgroup *memcg = NULL;
958 
959 		if (fatal_signal_pending(current))
960 			return;
961 
962 		freed = 0;
963 		memcg = mem_cgroup_iter(NULL, NULL, NULL);
964 		do {
965 			freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
966 		} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
967 	} while ((freed >> shift++) > 1);
968 }
969 
drop_slab(void)970 void drop_slab(void)
971 {
972 	int nid;
973 
974 	for_each_online_node(nid)
975 		drop_slab_node(nid);
976 }
977 
is_page_cache_freeable(struct folio * folio)978 static inline int is_page_cache_freeable(struct folio *folio)
979 {
980 	/*
981 	 * A freeable page cache page is referenced only by the caller
982 	 * that isolated the page, the page cache and optional buffer
983 	 * heads at page->private.
984 	 */
985 	return folio_ref_count(folio) - folio_test_private(folio) ==
986 		1 + folio_nr_pages(folio);
987 }
988 
989 /*
990  * We detected a synchronous write error writing a folio out.  Probably
991  * -ENOSPC.  We need to propagate that into the address_space for a subsequent
992  * fsync(), msync() or close().
993  *
994  * The tricky part is that after writepage we cannot touch the mapping: nothing
995  * prevents it from being freed up.  But we have a ref on the folio and once
996  * that folio is locked, the mapping is pinned.
997  *
998  * We're allowed to run sleeping folio_lock() here because we know the caller has
999  * __GFP_FS.
1000  */
handle_write_error(struct address_space * mapping,struct folio * folio,int error)1001 static void handle_write_error(struct address_space *mapping,
1002 				struct folio *folio, int error)
1003 {
1004 	folio_lock(folio);
1005 	if (folio_mapping(folio) == mapping)
1006 		mapping_set_error(mapping, error);
1007 	folio_unlock(folio);
1008 }
1009 
skip_throttle_noprogress(pg_data_t * pgdat)1010 static bool skip_throttle_noprogress(pg_data_t *pgdat)
1011 {
1012 	int reclaimable = 0, write_pending = 0;
1013 	int i;
1014 
1015 	/*
1016 	 * If kswapd is disabled, reschedule if necessary but do not
1017 	 * throttle as the system is likely near OOM.
1018 	 */
1019 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
1020 		return true;
1021 
1022 	/*
1023 	 * If there are a lot of dirty/writeback pages then do not
1024 	 * throttle as throttling will occur when the pages cycle
1025 	 * towards the end of the LRU if still under writeback.
1026 	 */
1027 	for (i = 0; i < MAX_NR_ZONES; i++) {
1028 		struct zone *zone = pgdat->node_zones + i;
1029 
1030 		if (!managed_zone(zone))
1031 			continue;
1032 
1033 		reclaimable += zone_reclaimable_pages(zone);
1034 		write_pending += zone_page_state_snapshot(zone,
1035 						  NR_ZONE_WRITE_PENDING);
1036 	}
1037 	if (2 * write_pending <= reclaimable)
1038 		return true;
1039 
1040 	return false;
1041 }
1042 
reclaim_throttle(pg_data_t * pgdat,enum vmscan_throttle_state reason)1043 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
1044 {
1045 	wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];
1046 	long timeout, ret;
1047 	DEFINE_WAIT(wait);
1048 
1049 	/*
1050 	 * Do not throttle IO workers, kthreads other than kswapd or
1051 	 * workqueues. They may be required for reclaim to make
1052 	 * forward progress (e.g. journalling workqueues or kthreads).
1053 	 */
1054 	if (!current_is_kswapd() &&
1055 	    current->flags & (PF_IO_WORKER|PF_KTHREAD)) {
1056 		cond_resched();
1057 		return;
1058 	}
1059 
1060 	/*
1061 	 * These figures are pulled out of thin air.
1062 	 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
1063 	 * parallel reclaimers which is a short-lived event so the timeout is
1064 	 * short. Failing to make progress or waiting on writeback are
1065 	 * potentially long-lived events so use a longer timeout. This is shaky
1066 	 * logic as a failure to make progress could be due to anything from
1067 	 * writeback to a slow device to excessive references pages at the tail
1068 	 * of the inactive LRU.
1069 	 */
1070 	switch(reason) {
1071 	case VMSCAN_THROTTLE_WRITEBACK:
1072 		timeout = HZ/10;
1073 
1074 		if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {
1075 			WRITE_ONCE(pgdat->nr_reclaim_start,
1076 				node_page_state(pgdat, NR_THROTTLED_WRITTEN));
1077 		}
1078 
1079 		break;
1080 	case VMSCAN_THROTTLE_CONGESTED:
1081 		fallthrough;
1082 	case VMSCAN_THROTTLE_NOPROGRESS:
1083 		if (skip_throttle_noprogress(pgdat)) {
1084 			cond_resched();
1085 			return;
1086 		}
1087 
1088 		timeout = 1;
1089 
1090 		break;
1091 	case VMSCAN_THROTTLE_ISOLATED:
1092 		timeout = HZ/50;
1093 		break;
1094 	default:
1095 		WARN_ON_ONCE(1);
1096 		timeout = HZ;
1097 		break;
1098 	}
1099 
1100 	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1101 	ret = schedule_timeout(timeout);
1102 	finish_wait(wqh, &wait);
1103 
1104 	if (reason == VMSCAN_THROTTLE_WRITEBACK)
1105 		atomic_dec(&pgdat->nr_writeback_throttled);
1106 
1107 	trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),
1108 				jiffies_to_usecs(timeout - ret),
1109 				reason);
1110 }
1111 
1112 /*
1113  * Account for pages written if tasks are throttled waiting on dirty
1114  * pages to clean. If enough pages have been cleaned since throttling
1115  * started then wakeup the throttled tasks.
1116  */
__acct_reclaim_writeback(pg_data_t * pgdat,struct folio * folio,int nr_throttled)1117 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
1118 							int nr_throttled)
1119 {
1120 	unsigned long nr_written;
1121 
1122 	node_stat_add_folio(folio, NR_THROTTLED_WRITTEN);
1123 
1124 	/*
1125 	 * This is an inaccurate read as the per-cpu deltas may not
1126 	 * be synchronised. However, given that the system is
1127 	 * writeback throttled, it is not worth taking the penalty
1128 	 * of getting an accurate count. At worst, the throttle
1129 	 * timeout guarantees forward progress.
1130 	 */
1131 	nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -
1132 		READ_ONCE(pgdat->nr_reclaim_start);
1133 
1134 	if (nr_written > SWAP_CLUSTER_MAX * nr_throttled)
1135 		wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);
1136 }
1137 
1138 /* possible outcome of pageout() */
1139 typedef enum {
1140 	/* failed to write page out, page is locked */
1141 	PAGE_KEEP,
1142 	/* move page to the active list, page is locked */
1143 	PAGE_ACTIVATE,
1144 	/* page has been sent to the disk successfully, page is unlocked */
1145 	PAGE_SUCCESS,
1146 	/* page is clean and locked */
1147 	PAGE_CLEAN,
1148 } pageout_t;
1149 
1150 /*
1151  * pageout is called by shrink_page_list() for each dirty page.
1152  * Calls ->writepage().
1153  */
pageout(struct folio * folio,struct address_space * mapping,struct swap_iocb ** plug)1154 static pageout_t pageout(struct folio *folio, struct address_space *mapping,
1155 			 struct swap_iocb **plug)
1156 {
1157 	/*
1158 	 * If the folio is dirty, only perform writeback if that write
1159 	 * will be non-blocking.  To prevent this allocation from being
1160 	 * stalled by pagecache activity.  But note that there may be
1161 	 * stalls if we need to run get_block().  We could test
1162 	 * PagePrivate for that.
1163 	 *
1164 	 * If this process is currently in __generic_file_write_iter() against
1165 	 * this folio's queue, we can perform writeback even if that
1166 	 * will block.
1167 	 *
1168 	 * If the folio is swapcache, write it back even if that would
1169 	 * block, for some throttling. This happens by accident, because
1170 	 * swap_backing_dev_info is bust: it doesn't reflect the
1171 	 * congestion state of the swapdevs.  Easy to fix, if needed.
1172 	 */
1173 	if (!is_page_cache_freeable(folio))
1174 		return PAGE_KEEP;
1175 	if (!mapping) {
1176 		/*
1177 		 * Some data journaling orphaned folios can have
1178 		 * folio->mapping == NULL while being dirty with clean buffers.
1179 		 */
1180 		if (folio_test_private(folio)) {
1181 			if (try_to_free_buffers(folio)) {
1182 				folio_clear_dirty(folio);
1183 				pr_info("%s: orphaned folio\n", __func__);
1184 				return PAGE_CLEAN;
1185 			}
1186 		}
1187 		return PAGE_KEEP;
1188 	}
1189 	if (mapping->a_ops->writepage == NULL)
1190 		return PAGE_ACTIVATE;
1191 
1192 	if (folio_clear_dirty_for_io(folio)) {
1193 		int res;
1194 		struct writeback_control wbc = {
1195 			.sync_mode = WB_SYNC_NONE,
1196 			.nr_to_write = SWAP_CLUSTER_MAX,
1197 			.range_start = 0,
1198 			.range_end = LLONG_MAX,
1199 			.for_reclaim = 1,
1200 			.swap_plug = plug,
1201 		};
1202 
1203 		folio_set_reclaim(folio);
1204 		res = mapping->a_ops->writepage(&folio->page, &wbc);
1205 		if (res < 0)
1206 			handle_write_error(mapping, folio, res);
1207 		if (res == AOP_WRITEPAGE_ACTIVATE) {
1208 			folio_clear_reclaim(folio);
1209 			return PAGE_ACTIVATE;
1210 		}
1211 
1212 		if (!folio_test_writeback(folio)) {
1213 			/* synchronous write or broken a_ops? */
1214 			folio_clear_reclaim(folio);
1215 		}
1216 		trace_mm_vmscan_write_folio(folio);
1217 		node_stat_add_folio(folio, NR_VMSCAN_WRITE);
1218 		return PAGE_SUCCESS;
1219 	}
1220 
1221 	return PAGE_CLEAN;
1222 }
1223 
1224 /*
1225  * Same as remove_mapping, but if the page is removed from the mapping, it
1226  * gets returned with a refcount of 0.
1227  */
__remove_mapping(struct address_space * mapping,struct folio * folio,bool reclaimed,struct mem_cgroup * target_memcg)1228 static int __remove_mapping(struct address_space *mapping, struct folio *folio,
1229 			    bool reclaimed, struct mem_cgroup *target_memcg)
1230 {
1231 	int refcount;
1232 	void *shadow = NULL;
1233 
1234 	BUG_ON(!folio_test_locked(folio));
1235 	BUG_ON(mapping != folio_mapping(folio));
1236 
1237 	if (!folio_test_swapcache(folio))
1238 		spin_lock(&mapping->host->i_lock);
1239 	xa_lock_irq(&mapping->i_pages);
1240 	/*
1241 	 * The non racy check for a busy page.
1242 	 *
1243 	 * Must be careful with the order of the tests. When someone has
1244 	 * a ref to the page, it may be possible that they dirty it then
1245 	 * drop the reference. So if PageDirty is tested before page_count
1246 	 * here, then the following race may occur:
1247 	 *
1248 	 * get_user_pages(&page);
1249 	 * [user mapping goes away]
1250 	 * write_to(page);
1251 	 *				!PageDirty(page)    [good]
1252 	 * SetPageDirty(page);
1253 	 * put_page(page);
1254 	 *				!page_count(page)   [good, discard it]
1255 	 *
1256 	 * [oops, our write_to data is lost]
1257 	 *
1258 	 * Reversing the order of the tests ensures such a situation cannot
1259 	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
1260 	 * load is not satisfied before that of page->_refcount.
1261 	 *
1262 	 * Note that if SetPageDirty is always performed via set_page_dirty,
1263 	 * and thus under the i_pages lock, then this ordering is not required.
1264 	 */
1265 	refcount = 1 + folio_nr_pages(folio);
1266 	if (!folio_ref_freeze(folio, refcount))
1267 		goto cannot_free;
1268 	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
1269 	if (unlikely(folio_test_dirty(folio))) {
1270 		folio_ref_unfreeze(folio, refcount);
1271 		goto cannot_free;
1272 	}
1273 
1274 	if (folio_test_swapcache(folio)) {
1275 		swp_entry_t swap = folio_swap_entry(folio);
1276 		mem_cgroup_swapout(folio, swap);
1277 		if (reclaimed && !mapping_exiting(mapping))
1278 			shadow = workingset_eviction(folio, target_memcg);
1279 		__delete_from_swap_cache(&folio->page, swap, shadow);
1280 		xa_unlock_irq(&mapping->i_pages);
1281 		put_swap_page(&folio->page, swap);
1282 	} else {
1283 		void (*free_folio)(struct folio *);
1284 
1285 		free_folio = mapping->a_ops->free_folio;
1286 		/*
1287 		 * Remember a shadow entry for reclaimed file cache in
1288 		 * order to detect refaults, thus thrashing, later on.
1289 		 *
1290 		 * But don't store shadows in an address space that is
1291 		 * already exiting.  This is not just an optimization,
1292 		 * inode reclaim needs to empty out the radix tree or
1293 		 * the nodes are lost.  Don't plant shadows behind its
1294 		 * back.
1295 		 *
1296 		 * We also don't store shadows for DAX mappings because the
1297 		 * only page cache pages found in these are zero pages
1298 		 * covering holes, and because we don't want to mix DAX
1299 		 * exceptional entries and shadow exceptional entries in the
1300 		 * same address_space.
1301 		 */
1302 		if (reclaimed && folio_is_file_lru(folio) &&
1303 		    !mapping_exiting(mapping) && !dax_mapping(mapping))
1304 			shadow = workingset_eviction(folio, target_memcg);
1305 		__filemap_remove_folio(folio, shadow);
1306 		xa_unlock_irq(&mapping->i_pages);
1307 		if (mapping_shrinkable(mapping))
1308 			inode_add_lru(mapping->host);
1309 		spin_unlock(&mapping->host->i_lock);
1310 
1311 		if (free_folio)
1312 			free_folio(folio);
1313 	}
1314 
1315 	return 1;
1316 
1317 cannot_free:
1318 	xa_unlock_irq(&mapping->i_pages);
1319 	if (!folio_test_swapcache(folio))
1320 		spin_unlock(&mapping->host->i_lock);
1321 	return 0;
1322 }
1323 
1324 /**
1325  * remove_mapping() - Attempt to remove a folio from its mapping.
1326  * @mapping: The address space.
1327  * @folio: The folio to remove.
1328  *
1329  * If the folio is dirty, under writeback or if someone else has a ref
1330  * on it, removal will fail.
1331  * Return: The number of pages removed from the mapping.  0 if the folio
1332  * could not be removed.
1333  * Context: The caller should have a single refcount on the folio and
1334  * hold its lock.
1335  */
remove_mapping(struct address_space * mapping,struct folio * folio)1336 long remove_mapping(struct address_space *mapping, struct folio *folio)
1337 {
1338 	if (__remove_mapping(mapping, folio, false, NULL)) {
1339 		/*
1340 		 * Unfreezing the refcount with 1 effectively
1341 		 * drops the pagecache ref for us without requiring another
1342 		 * atomic operation.
1343 		 */
1344 		folio_ref_unfreeze(folio, 1);
1345 		return folio_nr_pages(folio);
1346 	}
1347 	return 0;
1348 }
1349 
1350 /**
1351  * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1352  * @folio: Folio to be returned to an LRU list.
1353  *
1354  * Add previously isolated @folio to appropriate LRU list.
1355  * The folio may still be unevictable for other reasons.
1356  *
1357  * Context: lru_lock must not be held, interrupts must be enabled.
1358  */
folio_putback_lru(struct folio * folio)1359 void folio_putback_lru(struct folio *folio)
1360 {
1361 	folio_add_lru(folio);
1362 	folio_put(folio);		/* drop ref from isolate */
1363 }
1364 
1365 enum page_references {
1366 	PAGEREF_RECLAIM,
1367 	PAGEREF_RECLAIM_CLEAN,
1368 	PAGEREF_KEEP,
1369 	PAGEREF_ACTIVATE,
1370 };
1371 
folio_check_references(struct folio * folio,struct scan_control * sc)1372 static enum page_references folio_check_references(struct folio *folio,
1373 						  struct scan_control *sc)
1374 {
1375 	int referenced_ptes, referenced_folio;
1376 	unsigned long vm_flags;
1377 
1378 	referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
1379 					   &vm_flags);
1380 	referenced_folio = folio_test_clear_referenced(folio);
1381 
1382 	/*
1383 	 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1384 	 * Let the folio, now marked Mlocked, be moved to the unevictable list.
1385 	 */
1386 	if (vm_flags & VM_LOCKED)
1387 		return PAGEREF_ACTIVATE;
1388 
1389 	/* rmap lock contention: rotate */
1390 	if (referenced_ptes == -1)
1391 		return PAGEREF_KEEP;
1392 
1393 	if (referenced_ptes) {
1394 		/*
1395 		 * All mapped folios start out with page table
1396 		 * references from the instantiating fault, so we need
1397 		 * to look twice if a mapped file/anon folio is used more
1398 		 * than once.
1399 		 *
1400 		 * Mark it and spare it for another trip around the
1401 		 * inactive list.  Another page table reference will
1402 		 * lead to its activation.
1403 		 *
1404 		 * Note: the mark is set for activated folios as well
1405 		 * so that recently deactivated but used folios are
1406 		 * quickly recovered.
1407 		 */
1408 		folio_set_referenced(folio);
1409 
1410 		if (referenced_folio || referenced_ptes > 1)
1411 			return PAGEREF_ACTIVATE;
1412 
1413 		/*
1414 		 * Activate file-backed executable folios after first usage.
1415 		 */
1416 		if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))
1417 			return PAGEREF_ACTIVATE;
1418 
1419 		return PAGEREF_KEEP;
1420 	}
1421 
1422 	/* Reclaim if clean, defer dirty folios to writeback */
1423 	if (referenced_folio && folio_is_file_lru(folio))
1424 		return PAGEREF_RECLAIM_CLEAN;
1425 
1426 	return PAGEREF_RECLAIM;
1427 }
1428 
1429 /* Check if a page is dirty or under writeback */
folio_check_dirty_writeback(struct folio * folio,bool * dirty,bool * writeback)1430 static void folio_check_dirty_writeback(struct folio *folio,
1431 				       bool *dirty, bool *writeback)
1432 {
1433 	struct address_space *mapping;
1434 
1435 	/*
1436 	 * Anonymous pages are not handled by flushers and must be written
1437 	 * from reclaim context. Do not stall reclaim based on them.
1438 	 * MADV_FREE anonymous pages are put into inactive file list too.
1439 	 * They could be mistakenly treated as file lru. So further anon
1440 	 * test is needed.
1441 	 */
1442 	if (!folio_is_file_lru(folio) ||
1443 	    (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
1444 		*dirty = false;
1445 		*writeback = false;
1446 		return;
1447 	}
1448 
1449 	/* By default assume that the folio flags are accurate */
1450 	*dirty = folio_test_dirty(folio);
1451 	*writeback = folio_test_writeback(folio);
1452 
1453 	/* Verify dirty/writeback state if the filesystem supports it */
1454 	if (!folio_test_private(folio))
1455 		return;
1456 
1457 	mapping = folio_mapping(folio);
1458 	if (mapping && mapping->a_ops->is_dirty_writeback)
1459 		mapping->a_ops->is_dirty_writeback(folio, dirty, writeback);
1460 }
1461 
alloc_demote_page(struct page * page,unsigned long node)1462 static struct page *alloc_demote_page(struct page *page, unsigned long node)
1463 {
1464 	struct migration_target_control mtc = {
1465 		/*
1466 		 * Allocate from 'node', or fail quickly and quietly.
1467 		 * When this happens, 'page' will likely just be discarded
1468 		 * instead of migrated.
1469 		 */
1470 		.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
1471 			    __GFP_THISNODE  | __GFP_NOWARN |
1472 			    __GFP_NOMEMALLOC | GFP_NOWAIT,
1473 		.nid = node
1474 	};
1475 
1476 	return alloc_migration_target(page, (unsigned long)&mtc);
1477 }
1478 
1479 /*
1480  * Take pages on @demote_list and attempt to demote them to
1481  * another node.  Pages which are not demoted are left on
1482  * @demote_pages.
1483  */
demote_page_list(struct list_head * demote_pages,struct pglist_data * pgdat)1484 static unsigned int demote_page_list(struct list_head *demote_pages,
1485 				     struct pglist_data *pgdat)
1486 {
1487 	int target_nid = next_demotion_node(pgdat->node_id);
1488 	unsigned int nr_succeeded;
1489 
1490 	if (list_empty(demote_pages))
1491 		return 0;
1492 
1493 	if (target_nid == NUMA_NO_NODE)
1494 		return 0;
1495 
1496 	/* Demotion ignores all cpuset and mempolicy settings */
1497 	migrate_pages(demote_pages, alloc_demote_page, NULL,
1498 			    target_nid, MIGRATE_ASYNC, MR_DEMOTION,
1499 			    &nr_succeeded);
1500 
1501 	if (current_is_kswapd())
1502 		__count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded);
1503 	else
1504 		__count_vm_events(PGDEMOTE_DIRECT, nr_succeeded);
1505 
1506 	return nr_succeeded;
1507 }
1508 
may_enter_fs(struct folio * folio,gfp_t gfp_mask)1509 static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
1510 {
1511 	if (gfp_mask & __GFP_FS)
1512 		return true;
1513 	if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO))
1514 		return false;
1515 	/*
1516 	 * We can "enter_fs" for swap-cache with only __GFP_IO
1517 	 * providing this isn't SWP_FS_OPS.
1518 	 * ->flags can be updated non-atomicially (scan_swap_map_slots),
1519 	 * but that will never affect SWP_FS_OPS, so the data_race
1520 	 * is safe.
1521 	 */
1522 	return !data_race(page_swap_flags(&folio->page) & SWP_FS_OPS);
1523 }
1524 
1525 /*
1526  * shrink_page_list() returns the number of reclaimed pages
1527  */
shrink_page_list(struct list_head * page_list,struct pglist_data * pgdat,struct scan_control * sc,struct reclaim_stat * stat,bool ignore_references)1528 static unsigned int shrink_page_list(struct list_head *page_list,
1529 				     struct pglist_data *pgdat,
1530 				     struct scan_control *sc,
1531 				     struct reclaim_stat *stat,
1532 				     bool ignore_references)
1533 {
1534 	LIST_HEAD(ret_pages);
1535 	LIST_HEAD(free_pages);
1536 	LIST_HEAD(demote_pages);
1537 	unsigned int nr_reclaimed = 0;
1538 	unsigned int pgactivate = 0;
1539 	bool do_demote_pass;
1540 	struct swap_iocb *plug = NULL;
1541 
1542 	memset(stat, 0, sizeof(*stat));
1543 	cond_resched();
1544 	do_demote_pass = can_demote(pgdat->node_id, sc);
1545 
1546 retry:
1547 	while (!list_empty(page_list)) {
1548 		struct address_space *mapping;
1549 		struct folio *folio;
1550 		enum page_references references = PAGEREF_RECLAIM;
1551 		bool dirty, writeback;
1552 		unsigned int nr_pages;
1553 
1554 		cond_resched();
1555 
1556 		folio = lru_to_folio(page_list);
1557 		list_del(&folio->lru);
1558 
1559 		if (!folio_trylock(folio))
1560 			goto keep;
1561 
1562 		VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1563 
1564 		nr_pages = folio_nr_pages(folio);
1565 
1566 		/* Account the number of base pages */
1567 		sc->nr_scanned += nr_pages;
1568 
1569 		if (unlikely(!folio_evictable(folio)))
1570 			goto activate_locked;
1571 
1572 		if (!sc->may_unmap && folio_mapped(folio))
1573 			goto keep_locked;
1574 
1575 		/*
1576 		 * The number of dirty pages determines if a node is marked
1577 		 * reclaim_congested. kswapd will stall and start writing
1578 		 * folios if the tail of the LRU is all dirty unqueued folios.
1579 		 */
1580 		folio_check_dirty_writeback(folio, &dirty, &writeback);
1581 		if (dirty || writeback)
1582 			stat->nr_dirty += nr_pages;
1583 
1584 		if (dirty && !writeback)
1585 			stat->nr_unqueued_dirty += nr_pages;
1586 
1587 		/*
1588 		 * Treat this folio as congested if folios are cycling
1589 		 * through the LRU so quickly that the folios marked
1590 		 * for immediate reclaim are making it to the end of
1591 		 * the LRU a second time.
1592 		 */
1593 		if (writeback && folio_test_reclaim(folio))
1594 			stat->nr_congested += nr_pages;
1595 
1596 		/*
1597 		 * If a folio at the tail of the LRU is under writeback, there
1598 		 * are three cases to consider.
1599 		 *
1600 		 * 1) If reclaim is encountering an excessive number
1601 		 *    of folios under writeback and this folio has both
1602 		 *    the writeback and reclaim flags set, then it
1603 		 *    indicates that folios are being queued for I/O but
1604 		 *    are being recycled through the LRU before the I/O
1605 		 *    can complete. Waiting on the folio itself risks an
1606 		 *    indefinite stall if it is impossible to writeback
1607 		 *    the folio due to I/O error or disconnected storage
1608 		 *    so instead note that the LRU is being scanned too
1609 		 *    quickly and the caller can stall after the folio
1610 		 *    list has been processed.
1611 		 *
1612 		 * 2) Global or new memcg reclaim encounters a folio that is
1613 		 *    not marked for immediate reclaim, or the caller does not
1614 		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
1615 		 *    not to fs). In this case mark the folio for immediate
1616 		 *    reclaim and continue scanning.
1617 		 *
1618 		 *    Require may_enter_fs() because we would wait on fs, which
1619 		 *    may not have submitted I/O yet. And the loop driver might
1620 		 *    enter reclaim, and deadlock if it waits on a folio for
1621 		 *    which it is needed to do the write (loop masks off
1622 		 *    __GFP_IO|__GFP_FS for this reason); but more thought
1623 		 *    would probably show more reasons.
1624 		 *
1625 		 * 3) Legacy memcg encounters a folio that already has the
1626 		 *    reclaim flag set. memcg does not have any dirty folio
1627 		 *    throttling so we could easily OOM just because too many
1628 		 *    folios are in writeback and there is nothing else to
1629 		 *    reclaim. Wait for the writeback to complete.
1630 		 *
1631 		 * In cases 1) and 2) we activate the folios to get them out of
1632 		 * the way while we continue scanning for clean folios on the
1633 		 * inactive list and refilling from the active list. The
1634 		 * observation here is that waiting for disk writes is more
1635 		 * expensive than potentially causing reloads down the line.
1636 		 * Since they're marked for immediate reclaim, they won't put
1637 		 * memory pressure on the cache working set any longer than it
1638 		 * takes to write them to disk.
1639 		 */
1640 		if (folio_test_writeback(folio)) {
1641 			/* Case 1 above */
1642 			if (current_is_kswapd() &&
1643 			    folio_test_reclaim(folio) &&
1644 			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1645 				stat->nr_immediate += nr_pages;
1646 				goto activate_locked;
1647 
1648 			/* Case 2 above */
1649 			} else if (writeback_throttling_sane(sc) ||
1650 			    !folio_test_reclaim(folio) ||
1651 			    !may_enter_fs(folio, sc->gfp_mask)) {
1652 				/*
1653 				 * This is slightly racy -
1654 				 * folio_end_writeback() might have
1655 				 * just cleared the reclaim flag, then
1656 				 * setting the reclaim flag here ends up
1657 				 * interpreted as the readahead flag - but
1658 				 * that does not matter enough to care.
1659 				 * What we do want is for this folio to
1660 				 * have the reclaim flag set next time
1661 				 * memcg reclaim reaches the tests above,
1662 				 * so it will then wait for writeback to
1663 				 * avoid OOM; and it's also appropriate
1664 				 * in global reclaim.
1665 				 */
1666 				folio_set_reclaim(folio);
1667 				stat->nr_writeback += nr_pages;
1668 				goto activate_locked;
1669 
1670 			/* Case 3 above */
1671 			} else {
1672 				folio_unlock(folio);
1673 				folio_wait_writeback(folio);
1674 				/* then go back and try same folio again */
1675 				list_add_tail(&folio->lru, page_list);
1676 				continue;
1677 			}
1678 		}
1679 
1680 		if (!ignore_references)
1681 			references = folio_check_references(folio, sc);
1682 
1683 		switch (references) {
1684 		case PAGEREF_ACTIVATE:
1685 			goto activate_locked;
1686 		case PAGEREF_KEEP:
1687 			stat->nr_ref_keep += nr_pages;
1688 			goto keep_locked;
1689 		case PAGEREF_RECLAIM:
1690 		case PAGEREF_RECLAIM_CLEAN:
1691 			; /* try to reclaim the folio below */
1692 		}
1693 
1694 		/*
1695 		 * Before reclaiming the folio, try to relocate
1696 		 * its contents to another node.
1697 		 */
1698 		if (do_demote_pass &&
1699 		    (thp_migration_supported() || !folio_test_large(folio))) {
1700 			list_add(&folio->lru, &demote_pages);
1701 			folio_unlock(folio);
1702 			continue;
1703 		}
1704 
1705 		/*
1706 		 * Anonymous process memory has backing store?
1707 		 * Try to allocate it some swap space here.
1708 		 * Lazyfree folio could be freed directly
1709 		 */
1710 		if (folio_test_anon(folio) && folio_test_swapbacked(folio)) {
1711 			if (!folio_test_swapcache(folio)) {
1712 				if (!(sc->gfp_mask & __GFP_IO))
1713 					goto keep_locked;
1714 				if (folio_maybe_dma_pinned(folio))
1715 					goto keep_locked;
1716 				if (folio_test_large(folio)) {
1717 					/* cannot split folio, skip it */
1718 					if (!can_split_folio(folio, NULL))
1719 						goto activate_locked;
1720 					/*
1721 					 * Split folios without a PMD map right
1722 					 * away. Chances are some or all of the
1723 					 * tail pages can be freed without IO.
1724 					 */
1725 					if (!folio_entire_mapcount(folio) &&
1726 					    split_folio_to_list(folio,
1727 								page_list))
1728 						goto activate_locked;
1729 				}
1730 				if (!add_to_swap(folio)) {
1731 					if (!folio_test_large(folio))
1732 						goto activate_locked_split;
1733 					/* Fallback to swap normal pages */
1734 					if (split_folio_to_list(folio,
1735 								page_list))
1736 						goto activate_locked;
1737 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1738 					count_vm_event(THP_SWPOUT_FALLBACK);
1739 #endif
1740 					if (!add_to_swap(folio))
1741 						goto activate_locked_split;
1742 				}
1743 			}
1744 		} else if (folio_test_swapbacked(folio) &&
1745 			   folio_test_large(folio)) {
1746 			/* Split shmem folio */
1747 			if (split_folio_to_list(folio, page_list))
1748 				goto keep_locked;
1749 		}
1750 
1751 		/*
1752 		 * If the folio was split above, the tail pages will make
1753 		 * their own pass through this function and be accounted
1754 		 * then.
1755 		 */
1756 		if ((nr_pages > 1) && !folio_test_large(folio)) {
1757 			sc->nr_scanned -= (nr_pages - 1);
1758 			nr_pages = 1;
1759 		}
1760 
1761 		/*
1762 		 * The folio is mapped into the page tables of one or more
1763 		 * processes. Try to unmap it here.
1764 		 */
1765 		if (folio_mapped(folio)) {
1766 			enum ttu_flags flags = TTU_BATCH_FLUSH;
1767 			bool was_swapbacked = folio_test_swapbacked(folio);
1768 
1769 			if (folio_test_pmd_mappable(folio))
1770 				flags |= TTU_SPLIT_HUGE_PMD;
1771 
1772 			try_to_unmap(folio, flags);
1773 			if (folio_mapped(folio)) {
1774 				stat->nr_unmap_fail += nr_pages;
1775 				if (!was_swapbacked &&
1776 				    folio_test_swapbacked(folio))
1777 					stat->nr_lazyfree_fail += nr_pages;
1778 				goto activate_locked;
1779 			}
1780 		}
1781 
1782 		mapping = folio_mapping(folio);
1783 		if (folio_test_dirty(folio)) {
1784 			/*
1785 			 * Only kswapd can writeback filesystem folios
1786 			 * to avoid risk of stack overflow. But avoid
1787 			 * injecting inefficient single-folio I/O into
1788 			 * flusher writeback as much as possible: only
1789 			 * write folios when we've encountered many
1790 			 * dirty folios, and when we've already scanned
1791 			 * the rest of the LRU for clean folios and see
1792 			 * the same dirty folios again (with the reclaim
1793 			 * flag set).
1794 			 */
1795 			if (folio_is_file_lru(folio) &&
1796 			    (!current_is_kswapd() ||
1797 			     !folio_test_reclaim(folio) ||
1798 			     !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1799 				/*
1800 				 * Immediately reclaim when written back.
1801 				 * Similar in principle to deactivate_page()
1802 				 * except we already have the folio isolated
1803 				 * and know it's dirty
1804 				 */
1805 				node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE,
1806 						nr_pages);
1807 				folio_set_reclaim(folio);
1808 
1809 				goto activate_locked;
1810 			}
1811 
1812 			if (references == PAGEREF_RECLAIM_CLEAN)
1813 				goto keep_locked;
1814 			if (!may_enter_fs(folio, sc->gfp_mask))
1815 				goto keep_locked;
1816 			if (!sc->may_writepage)
1817 				goto keep_locked;
1818 
1819 			/*
1820 			 * Folio is dirty. Flush the TLB if a writable entry
1821 			 * potentially exists to avoid CPU writes after I/O
1822 			 * starts and then write it out here.
1823 			 */
1824 			try_to_unmap_flush_dirty();
1825 			switch (pageout(folio, mapping, &plug)) {
1826 			case PAGE_KEEP:
1827 				goto keep_locked;
1828 			case PAGE_ACTIVATE:
1829 				goto activate_locked;
1830 			case PAGE_SUCCESS:
1831 				stat->nr_pageout += nr_pages;
1832 
1833 				if (folio_test_writeback(folio))
1834 					goto keep;
1835 				if (folio_test_dirty(folio))
1836 					goto keep;
1837 
1838 				/*
1839 				 * A synchronous write - probably a ramdisk.  Go
1840 				 * ahead and try to reclaim the folio.
1841 				 */
1842 				if (!folio_trylock(folio))
1843 					goto keep;
1844 				if (folio_test_dirty(folio) ||
1845 				    folio_test_writeback(folio))
1846 					goto keep_locked;
1847 				mapping = folio_mapping(folio);
1848 				fallthrough;
1849 			case PAGE_CLEAN:
1850 				; /* try to free the folio below */
1851 			}
1852 		}
1853 
1854 		/*
1855 		 * If the folio has buffers, try to free the buffer
1856 		 * mappings associated with this folio. If we succeed
1857 		 * we try to free the folio as well.
1858 		 *
1859 		 * We do this even if the folio is dirty.
1860 		 * filemap_release_folio() does not perform I/O, but it
1861 		 * is possible for a folio to have the dirty flag set,
1862 		 * but it is actually clean (all its buffers are clean).
1863 		 * This happens if the buffers were written out directly,
1864 		 * with submit_bh(). ext3 will do this, as well as
1865 		 * the blockdev mapping.  filemap_release_folio() will
1866 		 * discover that cleanness and will drop the buffers
1867 		 * and mark the folio clean - it can be freed.
1868 		 *
1869 		 * Rarely, folios can have buffers and no ->mapping.
1870 		 * These are the folios which were not successfully
1871 		 * invalidated in truncate_cleanup_folio().  We try to
1872 		 * drop those buffers here and if that worked, and the
1873 		 * folio is no longer mapped into process address space
1874 		 * (refcount == 1) it can be freed.  Otherwise, leave
1875 		 * the folio on the LRU so it is swappable.
1876 		 */
1877 		if (folio_has_private(folio)) {
1878 			if (!filemap_release_folio(folio, sc->gfp_mask))
1879 				goto activate_locked;
1880 			if (!mapping && folio_ref_count(folio) == 1) {
1881 				folio_unlock(folio);
1882 				if (folio_put_testzero(folio))
1883 					goto free_it;
1884 				else {
1885 					/*
1886 					 * rare race with speculative reference.
1887 					 * the speculative reference will free
1888 					 * this folio shortly, so we may
1889 					 * increment nr_reclaimed here (and
1890 					 * leave it off the LRU).
1891 					 */
1892 					nr_reclaimed += nr_pages;
1893 					continue;
1894 				}
1895 			}
1896 		}
1897 
1898 		if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) {
1899 			/* follow __remove_mapping for reference */
1900 			if (!folio_ref_freeze(folio, 1))
1901 				goto keep_locked;
1902 			/*
1903 			 * The folio has only one reference left, which is
1904 			 * from the isolation. After the caller puts the
1905 			 * folio back on the lru and drops the reference, the
1906 			 * folio will be freed anyway. It doesn't matter
1907 			 * which lru it goes on. So we don't bother checking
1908 			 * the dirty flag here.
1909 			 */
1910 			count_vm_events(PGLAZYFREED, nr_pages);
1911 			count_memcg_folio_events(folio, PGLAZYFREED, nr_pages);
1912 		} else if (!mapping || !__remove_mapping(mapping, folio, true,
1913 							 sc->target_mem_cgroup))
1914 			goto keep_locked;
1915 
1916 		folio_unlock(folio);
1917 free_it:
1918 		/*
1919 		 * Folio may get swapped out as a whole, need to account
1920 		 * all pages in it.
1921 		 */
1922 		nr_reclaimed += nr_pages;
1923 
1924 		/*
1925 		 * Is there need to periodically free_page_list? It would
1926 		 * appear not as the counts should be low
1927 		 */
1928 		if (unlikely(folio_test_large(folio)))
1929 			destroy_compound_page(&folio->page);
1930 		else
1931 			list_add(&folio->lru, &free_pages);
1932 		continue;
1933 
1934 activate_locked_split:
1935 		/*
1936 		 * The tail pages that are failed to add into swap cache
1937 		 * reach here.  Fixup nr_scanned and nr_pages.
1938 		 */
1939 		if (nr_pages > 1) {
1940 			sc->nr_scanned -= (nr_pages - 1);
1941 			nr_pages = 1;
1942 		}
1943 activate_locked:
1944 		/* Not a candidate for swapping, so reclaim swap space. */
1945 		if (folio_test_swapcache(folio) &&
1946 		    (mem_cgroup_swap_full(&folio->page) ||
1947 		     folio_test_mlocked(folio)))
1948 			try_to_free_swap(&folio->page);
1949 		VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1950 		if (!folio_test_mlocked(folio)) {
1951 			int type = folio_is_file_lru(folio);
1952 			folio_set_active(folio);
1953 			stat->nr_activate[type] += nr_pages;
1954 			count_memcg_folio_events(folio, PGACTIVATE, nr_pages);
1955 		}
1956 keep_locked:
1957 		folio_unlock(folio);
1958 keep:
1959 		list_add(&folio->lru, &ret_pages);
1960 		VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
1961 				folio_test_unevictable(folio), folio);
1962 	}
1963 	/* 'page_list' is always empty here */
1964 
1965 	/* Migrate folios selected for demotion */
1966 	nr_reclaimed += demote_page_list(&demote_pages, pgdat);
1967 	/* Folios that could not be demoted are still in @demote_pages */
1968 	if (!list_empty(&demote_pages)) {
1969 		/* Folios which weren't demoted go back on @page_list for retry: */
1970 		list_splice_init(&demote_pages, page_list);
1971 		do_demote_pass = false;
1972 		goto retry;
1973 	}
1974 
1975 	pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
1976 
1977 	mem_cgroup_uncharge_list(&free_pages);
1978 	try_to_unmap_flush();
1979 	free_unref_page_list(&free_pages);
1980 
1981 	list_splice(&ret_pages, page_list);
1982 	count_vm_events(PGACTIVATE, pgactivate);
1983 
1984 	if (plug)
1985 		swap_write_unplug(plug);
1986 	return nr_reclaimed;
1987 }
1988 
reclaim_clean_pages_from_list(struct zone * zone,struct list_head * page_list)1989 unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1990 					    struct list_head *page_list)
1991 {
1992 	struct scan_control sc = {
1993 		.gfp_mask = GFP_KERNEL,
1994 		.may_unmap = 1,
1995 	};
1996 	struct reclaim_stat stat;
1997 	unsigned int nr_reclaimed;
1998 	struct page *page, *next;
1999 	LIST_HEAD(clean_pages);
2000 	unsigned int noreclaim_flag;
2001 
2002 	list_for_each_entry_safe(page, next, page_list, lru) {
2003 		if (!PageHuge(page) && page_is_file_lru(page) &&
2004 		    !PageDirty(page) && !__PageMovable(page) &&
2005 		    !PageUnevictable(page)) {
2006 			ClearPageActive(page);
2007 			list_move(&page->lru, &clean_pages);
2008 		}
2009 	}
2010 
2011 	/*
2012 	 * We should be safe here since we are only dealing with file pages and
2013 	 * we are not kswapd and therefore cannot write dirty file pages. But
2014 	 * call memalloc_noreclaim_save() anyway, just in case these conditions
2015 	 * change in the future.
2016 	 */
2017 	noreclaim_flag = memalloc_noreclaim_save();
2018 	nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
2019 					&stat, true);
2020 	memalloc_noreclaim_restore(noreclaim_flag);
2021 
2022 	list_splice(&clean_pages, page_list);
2023 	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2024 			    -(long)nr_reclaimed);
2025 	/*
2026 	 * Since lazyfree pages are isolated from file LRU from the beginning,
2027 	 * they will rotate back to anonymous LRU in the end if it failed to
2028 	 * discard so isolated count will be mismatched.
2029 	 * Compensate the isolated count for both LRU lists.
2030 	 */
2031 	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
2032 			    stat.nr_lazyfree_fail);
2033 	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2034 			    -(long)stat.nr_lazyfree_fail);
2035 	return nr_reclaimed;
2036 }
2037 
2038 /*
2039  * Update LRU sizes after isolating pages. The LRU size updates must
2040  * be complete before mem_cgroup_update_lru_size due to a sanity check.
2041  */
update_lru_sizes(struct lruvec * lruvec,enum lru_list lru,unsigned long * nr_zone_taken)2042 static __always_inline void update_lru_sizes(struct lruvec *lruvec,
2043 			enum lru_list lru, unsigned long *nr_zone_taken)
2044 {
2045 	int zid;
2046 
2047 	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2048 		if (!nr_zone_taken[zid])
2049 			continue;
2050 
2051 		update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
2052 	}
2053 
2054 }
2055 
2056 /*
2057  * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
2058  *
2059  * lruvec->lru_lock is heavily contended.  Some of the functions that
2060  * shrink the lists perform better by taking out a batch of pages
2061  * and working on them outside the LRU lock.
2062  *
2063  * For pagecache intensive workloads, this function is the hottest
2064  * spot in the kernel (apart from copy_*_user functions).
2065  *
2066  * Lru_lock must be held before calling this function.
2067  *
2068  * @nr_to_scan:	The number of eligible pages to look through on the list.
2069  * @lruvec:	The LRU vector to pull pages from.
2070  * @dst:	The temp list to put pages on to.
2071  * @nr_scanned:	The number of pages that were scanned.
2072  * @sc:		The scan_control struct for this reclaim session
2073  * @lru:	LRU list id for isolating
2074  *
2075  * returns how many pages were moved onto *@dst.
2076  */
isolate_lru_pages(unsigned long nr_to_scan,struct lruvec * lruvec,struct list_head * dst,unsigned long * nr_scanned,struct scan_control * sc,enum lru_list lru)2077 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
2078 		struct lruvec *lruvec, struct list_head *dst,
2079 		unsigned long *nr_scanned, struct scan_control *sc,
2080 		enum lru_list lru)
2081 {
2082 	struct list_head *src = &lruvec->lists[lru];
2083 	unsigned long nr_taken = 0;
2084 	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
2085 	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
2086 	unsigned long skipped = 0;
2087 	unsigned long scan, total_scan, nr_pages;
2088 	LIST_HEAD(pages_skipped);
2089 
2090 	total_scan = 0;
2091 	scan = 0;
2092 	while (scan < nr_to_scan && !list_empty(src)) {
2093 		struct list_head *move_to = src;
2094 		struct page *page;
2095 
2096 		page = lru_to_page(src);
2097 		prefetchw_prev_lru_page(page, src, flags);
2098 
2099 		nr_pages = compound_nr(page);
2100 		total_scan += nr_pages;
2101 
2102 		if (page_zonenum(page) > sc->reclaim_idx) {
2103 			nr_skipped[page_zonenum(page)] += nr_pages;
2104 			move_to = &pages_skipped;
2105 			goto move;
2106 		}
2107 
2108 		/*
2109 		 * Do not count skipped pages because that makes the function
2110 		 * return with no isolated pages if the LRU mostly contains
2111 		 * ineligible pages.  This causes the VM to not reclaim any
2112 		 * pages, triggering a premature OOM.
2113 		 * Account all tail pages of THP.
2114 		 */
2115 		scan += nr_pages;
2116 
2117 		if (!PageLRU(page))
2118 			goto move;
2119 		if (!sc->may_unmap && page_mapped(page))
2120 			goto move;
2121 
2122 		/*
2123 		 * Be careful not to clear PageLRU until after we're
2124 		 * sure the page is not being freed elsewhere -- the
2125 		 * page release code relies on it.
2126 		 */
2127 		if (unlikely(!get_page_unless_zero(page)))
2128 			goto move;
2129 
2130 		if (!TestClearPageLRU(page)) {
2131 			/* Another thread is already isolating this page */
2132 			put_page(page);
2133 			goto move;
2134 		}
2135 
2136 		nr_taken += nr_pages;
2137 		nr_zone_taken[page_zonenum(page)] += nr_pages;
2138 		move_to = dst;
2139 move:
2140 		list_move(&page->lru, move_to);
2141 	}
2142 
2143 	/*
2144 	 * Splice any skipped pages to the start of the LRU list. Note that
2145 	 * this disrupts the LRU order when reclaiming for lower zones but
2146 	 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
2147 	 * scanning would soon rescan the same pages to skip and waste lots
2148 	 * of cpu cycles.
2149 	 */
2150 	if (!list_empty(&pages_skipped)) {
2151 		int zid;
2152 
2153 		list_splice(&pages_skipped, src);
2154 		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2155 			if (!nr_skipped[zid])
2156 				continue;
2157 
2158 			__count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
2159 			skipped += nr_skipped[zid];
2160 		}
2161 	}
2162 	*nr_scanned = total_scan;
2163 	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
2164 				    total_scan, skipped, nr_taken,
2165 				    sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru);
2166 	update_lru_sizes(lruvec, lru, nr_zone_taken);
2167 	return nr_taken;
2168 }
2169 
2170 /**
2171  * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2172  * @folio: Folio to isolate from its LRU list.
2173  *
2174  * Isolate a @folio from an LRU list and adjust the vmstat statistic
2175  * corresponding to whatever LRU list the folio was on.
2176  *
2177  * The folio will have its LRU flag cleared.  If it was found on the
2178  * active list, it will have the Active flag set.  If it was found on the
2179  * unevictable list, it will have the Unevictable flag set.  These flags
2180  * may need to be cleared by the caller before letting the page go.
2181  *
2182  * Context:
2183  *
2184  * (1) Must be called with an elevated refcount on the page. This is a
2185  *     fundamental difference from isolate_lru_pages() (which is called
2186  *     without a stable reference).
2187  * (2) The lru_lock must not be held.
2188  * (3) Interrupts must be enabled.
2189  *
2190  * Return: 0 if the folio was removed from an LRU list.
2191  * -EBUSY if the folio was not on an LRU list.
2192  */
folio_isolate_lru(struct folio * folio)2193 int folio_isolate_lru(struct folio *folio)
2194 {
2195 	int ret = -EBUSY;
2196 
2197 	VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
2198 
2199 	if (folio_test_clear_lru(folio)) {
2200 		struct lruvec *lruvec;
2201 
2202 		folio_get(folio);
2203 		lruvec = folio_lruvec_lock_irq(folio);
2204 		lruvec_del_folio(lruvec, folio);
2205 		unlock_page_lruvec_irq(lruvec);
2206 		ret = 0;
2207 	}
2208 
2209 	return ret;
2210 }
2211 
2212 /*
2213  * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
2214  * then get rescheduled. When there are massive number of tasks doing page
2215  * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
2216  * the LRU list will go small and be scanned faster than necessary, leading to
2217  * unnecessary swapping, thrashing and OOM.
2218  */
too_many_isolated(struct pglist_data * pgdat,int file,struct scan_control * sc)2219 static int too_many_isolated(struct pglist_data *pgdat, int file,
2220 		struct scan_control *sc)
2221 {
2222 	unsigned long inactive, isolated;
2223 	bool too_many;
2224 
2225 	if (current_is_kswapd())
2226 		return 0;
2227 
2228 	if (!writeback_throttling_sane(sc))
2229 		return 0;
2230 
2231 	if (file) {
2232 		inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
2233 		isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
2234 	} else {
2235 		inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
2236 		isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
2237 	}
2238 
2239 	/*
2240 	 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2241 	 * won't get blocked by normal direct-reclaimers, forming a circular
2242 	 * deadlock.
2243 	 */
2244 	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
2245 		inactive >>= 3;
2246 
2247 	too_many = isolated > inactive;
2248 
2249 	/* Wake up tasks throttled due to too_many_isolated. */
2250 	if (!too_many)
2251 		wake_throttle_isolated(pgdat);
2252 
2253 	return too_many;
2254 }
2255 
2256 /*
2257  * move_pages_to_lru() moves pages from private @list to appropriate LRU list.
2258  * On return, @list is reused as a list of pages to be freed by the caller.
2259  *
2260  * Returns the number of pages moved to the given lruvec.
2261  */
move_pages_to_lru(struct lruvec * lruvec,struct list_head * list)2262 static unsigned int move_pages_to_lru(struct lruvec *lruvec,
2263 				      struct list_head *list)
2264 {
2265 	int nr_pages, nr_moved = 0;
2266 	LIST_HEAD(pages_to_free);
2267 	struct page *page;
2268 
2269 	while (!list_empty(list)) {
2270 		page = lru_to_page(list);
2271 		VM_BUG_ON_PAGE(PageLRU(page), page);
2272 		list_del(&page->lru);
2273 		if (unlikely(!page_evictable(page))) {
2274 			spin_unlock_irq(&lruvec->lru_lock);
2275 			putback_lru_page(page);
2276 			spin_lock_irq(&lruvec->lru_lock);
2277 			continue;
2278 		}
2279 
2280 		/*
2281 		 * The SetPageLRU needs to be kept here for list integrity.
2282 		 * Otherwise:
2283 		 *   #0 move_pages_to_lru             #1 release_pages
2284 		 *   if !put_page_testzero
2285 		 *				      if (put_page_testzero())
2286 		 *				        !PageLRU //skip lru_lock
2287 		 *     SetPageLRU()
2288 		 *     list_add(&page->lru,)
2289 		 *                                        list_add(&page->lru,)
2290 		 */
2291 		SetPageLRU(page);
2292 
2293 		if (unlikely(put_page_testzero(page))) {
2294 			__clear_page_lru_flags(page);
2295 
2296 			if (unlikely(PageCompound(page))) {
2297 				spin_unlock_irq(&lruvec->lru_lock);
2298 				destroy_compound_page(page);
2299 				spin_lock_irq(&lruvec->lru_lock);
2300 			} else
2301 				list_add(&page->lru, &pages_to_free);
2302 
2303 			continue;
2304 		}
2305 
2306 		/*
2307 		 * All pages were isolated from the same lruvec (and isolation
2308 		 * inhibits memcg migration).
2309 		 */
2310 		VM_BUG_ON_PAGE(!folio_matches_lruvec(page_folio(page), lruvec), page);
2311 		add_page_to_lru_list(page, lruvec);
2312 		nr_pages = thp_nr_pages(page);
2313 		nr_moved += nr_pages;
2314 		if (PageActive(page))
2315 			workingset_age_nonresident(lruvec, nr_pages);
2316 	}
2317 
2318 	/*
2319 	 * To save our caller's stack, now use input list for pages to free.
2320 	 */
2321 	list_splice(&pages_to_free, list);
2322 
2323 	return nr_moved;
2324 }
2325 
2326 /*
2327  * If a kernel thread (such as nfsd for loop-back mounts) services a backing
2328  * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case
2329  * we should not throttle.  Otherwise it is safe to do so.
2330  */
current_may_throttle(void)2331 static int current_may_throttle(void)
2332 {
2333 	return !(current->flags & PF_LOCAL_THROTTLE);
2334 }
2335 
2336 /*
2337  * shrink_inactive_list() is a helper for shrink_node().  It returns the number
2338  * of reclaimed pages
2339  */
2340 static unsigned long
shrink_inactive_list(unsigned long nr_to_scan,struct lruvec * lruvec,struct scan_control * sc,enum lru_list lru)2341 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
2342 		     struct scan_control *sc, enum lru_list lru)
2343 {
2344 	LIST_HEAD(page_list);
2345 	unsigned long nr_scanned;
2346 	unsigned int nr_reclaimed = 0;
2347 	unsigned long nr_taken;
2348 	struct reclaim_stat stat;
2349 	bool file = is_file_lru(lru);
2350 	enum vm_event_item item;
2351 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2352 	bool stalled = false;
2353 
2354 	while (unlikely(too_many_isolated(pgdat, file, sc))) {
2355 		if (stalled)
2356 			return 0;
2357 
2358 		/* wait a bit for the reclaimer. */
2359 		stalled = true;
2360 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
2361 
2362 		/* We are about to die and free our memory. Return now. */
2363 		if (fatal_signal_pending(current))
2364 			return SWAP_CLUSTER_MAX;
2365 	}
2366 
2367 	lru_add_drain();
2368 
2369 	spin_lock_irq(&lruvec->lru_lock);
2370 
2371 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
2372 				     &nr_scanned, sc, lru);
2373 
2374 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2375 	item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
2376 	if (!cgroup_reclaim(sc))
2377 		__count_vm_events(item, nr_scanned);
2378 	__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
2379 	__count_vm_events(PGSCAN_ANON + file, nr_scanned);
2380 
2381 	spin_unlock_irq(&lruvec->lru_lock);
2382 
2383 	if (nr_taken == 0)
2384 		return 0;
2385 
2386 	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
2387 
2388 	spin_lock_irq(&lruvec->lru_lock);
2389 	move_pages_to_lru(lruvec, &page_list);
2390 
2391 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2392 	item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
2393 	if (!cgroup_reclaim(sc))
2394 		__count_vm_events(item, nr_reclaimed);
2395 	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2396 	__count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
2397 	spin_unlock_irq(&lruvec->lru_lock);
2398 
2399 	lru_note_cost(lruvec, file, stat.nr_pageout);
2400 	mem_cgroup_uncharge_list(&page_list);
2401 	free_unref_page_list(&page_list);
2402 
2403 	/*
2404 	 * If dirty pages are scanned that are not queued for IO, it
2405 	 * implies that flushers are not doing their job. This can
2406 	 * happen when memory pressure pushes dirty pages to the end of
2407 	 * the LRU before the dirty limits are breached and the dirty
2408 	 * data has expired. It can also happen when the proportion of
2409 	 * dirty pages grows not through writes but through memory
2410 	 * pressure reclaiming all the clean cache. And in some cases,
2411 	 * the flushers simply cannot keep up with the allocation
2412 	 * rate. Nudge the flusher threads in case they are asleep.
2413 	 */
2414 	if (stat.nr_unqueued_dirty == nr_taken)
2415 		wakeup_flusher_threads(WB_REASON_VMSCAN);
2416 
2417 	sc->nr.dirty += stat.nr_dirty;
2418 	sc->nr.congested += stat.nr_congested;
2419 	sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2420 	sc->nr.writeback += stat.nr_writeback;
2421 	sc->nr.immediate += stat.nr_immediate;
2422 	sc->nr.taken += nr_taken;
2423 	if (file)
2424 		sc->nr.file_taken += nr_taken;
2425 
2426 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2427 			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2428 	return nr_reclaimed;
2429 }
2430 
2431 /*
2432  * shrink_active_list() moves pages from the active LRU to the inactive LRU.
2433  *
2434  * We move them the other way if the page is referenced by one or more
2435  * processes.
2436  *
2437  * If the pages are mostly unmapped, the processing is fast and it is
2438  * appropriate to hold lru_lock across the whole operation.  But if
2439  * the pages are mapped, the processing is slow (folio_referenced()), so
2440  * we should drop lru_lock around each page.  It's impossible to balance
2441  * this, so instead we remove the pages from the LRU while processing them.
2442  * It is safe to rely on PG_active against the non-LRU pages in here because
2443  * nobody will play with that bit on a non-LRU page.
2444  *
2445  * The downside is that we have to touch page->_refcount against each page.
2446  * But we had to alter page->flags anyway.
2447  */
shrink_active_list(unsigned long nr_to_scan,struct lruvec * lruvec,struct scan_control * sc,enum lru_list lru)2448 static void shrink_active_list(unsigned long nr_to_scan,
2449 			       struct lruvec *lruvec,
2450 			       struct scan_control *sc,
2451 			       enum lru_list lru)
2452 {
2453 	unsigned long nr_taken;
2454 	unsigned long nr_scanned;
2455 	unsigned long vm_flags;
2456 	LIST_HEAD(l_hold);	/* The pages which were snipped off */
2457 	LIST_HEAD(l_active);
2458 	LIST_HEAD(l_inactive);
2459 	unsigned nr_deactivate, nr_activate;
2460 	unsigned nr_rotated = 0;
2461 	int file = is_file_lru(lru);
2462 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2463 
2464 	lru_add_drain();
2465 
2466 	spin_lock_irq(&lruvec->lru_lock);
2467 
2468 	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2469 				     &nr_scanned, sc, lru);
2470 
2471 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2472 
2473 	if (!cgroup_reclaim(sc))
2474 		__count_vm_events(PGREFILL, nr_scanned);
2475 	__count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2476 
2477 	spin_unlock_irq(&lruvec->lru_lock);
2478 
2479 	while (!list_empty(&l_hold)) {
2480 		struct folio *folio;
2481 		struct page *page;
2482 
2483 		cond_resched();
2484 		folio = lru_to_folio(&l_hold);
2485 		list_del(&folio->lru);
2486 		page = &folio->page;
2487 
2488 		if (unlikely(!page_evictable(page))) {
2489 			putback_lru_page(page);
2490 			continue;
2491 		}
2492 
2493 		if (unlikely(buffer_heads_over_limit)) {
2494 			if (page_has_private(page) && trylock_page(page)) {
2495 				if (page_has_private(page))
2496 					try_to_release_page(page, 0);
2497 				unlock_page(page);
2498 			}
2499 		}
2500 
2501 		/* Referenced or rmap lock contention: rotate */
2502 		if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2503 				     &vm_flags) != 0) {
2504 			/*
2505 			 * Identify referenced, file-backed active pages and
2506 			 * give them one more trip around the active list. So
2507 			 * that executable code get better chances to stay in
2508 			 * memory under moderate memory pressure.  Anon pages
2509 			 * are not likely to be evicted by use-once streaming
2510 			 * IO, plus JVM can create lots of anon VM_EXEC pages,
2511 			 * so we ignore them here.
2512 			 */
2513 			if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) {
2514 				nr_rotated += thp_nr_pages(page);
2515 				list_add(&page->lru, &l_active);
2516 				continue;
2517 			}
2518 		}
2519 
2520 		ClearPageActive(page);	/* we are de-activating */
2521 		SetPageWorkingset(page);
2522 		list_add(&page->lru, &l_inactive);
2523 	}
2524 
2525 	/*
2526 	 * Move pages back to the lru list.
2527 	 */
2528 	spin_lock_irq(&lruvec->lru_lock);
2529 
2530 	nr_activate = move_pages_to_lru(lruvec, &l_active);
2531 	nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
2532 	/* Keep all free pages in l_active list */
2533 	list_splice(&l_inactive, &l_active);
2534 
2535 	__count_vm_events(PGDEACTIVATE, nr_deactivate);
2536 	__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2537 
2538 	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2539 	spin_unlock_irq(&lruvec->lru_lock);
2540 
2541 	mem_cgroup_uncharge_list(&l_active);
2542 	free_unref_page_list(&l_active);
2543 	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2544 			nr_deactivate, nr_rotated, sc->priority, file);
2545 }
2546 
reclaim_page_list(struct list_head * page_list,struct pglist_data * pgdat)2547 static unsigned int reclaim_page_list(struct list_head *page_list,
2548 				      struct pglist_data *pgdat)
2549 {
2550 	struct reclaim_stat dummy_stat;
2551 	unsigned int nr_reclaimed;
2552 	struct folio *folio;
2553 	struct scan_control sc = {
2554 		.gfp_mask = GFP_KERNEL,
2555 		.may_writepage = 1,
2556 		.may_unmap = 1,
2557 		.may_swap = 1,
2558 		.no_demotion = 1,
2559 	};
2560 
2561 	nr_reclaimed = shrink_page_list(page_list, pgdat, &sc, &dummy_stat, false);
2562 	while (!list_empty(page_list)) {
2563 		folio = lru_to_folio(page_list);
2564 		list_del(&folio->lru);
2565 		folio_putback_lru(folio);
2566 	}
2567 
2568 	return nr_reclaimed;
2569 }
2570 
reclaim_pages(struct list_head * page_list)2571 unsigned long reclaim_pages(struct list_head *page_list)
2572 {
2573 	int nid;
2574 	unsigned int nr_reclaimed = 0;
2575 	LIST_HEAD(node_page_list);
2576 	struct page *page;
2577 	unsigned int noreclaim_flag;
2578 
2579 	if (list_empty(page_list))
2580 		return nr_reclaimed;
2581 
2582 	noreclaim_flag = memalloc_noreclaim_save();
2583 
2584 	nid = page_to_nid(lru_to_page(page_list));
2585 	do {
2586 		page = lru_to_page(page_list);
2587 
2588 		if (nid == page_to_nid(page)) {
2589 			ClearPageActive(page);
2590 			list_move(&page->lru, &node_page_list);
2591 			continue;
2592 		}
2593 
2594 		nr_reclaimed += reclaim_page_list(&node_page_list, NODE_DATA(nid));
2595 		nid = page_to_nid(lru_to_page(page_list));
2596 	} while (!list_empty(page_list));
2597 
2598 	nr_reclaimed += reclaim_page_list(&node_page_list, NODE_DATA(nid));
2599 
2600 	memalloc_noreclaim_restore(noreclaim_flag);
2601 
2602 	return nr_reclaimed;
2603 }
2604 
shrink_list(enum lru_list lru,unsigned long nr_to_scan,struct lruvec * lruvec,struct scan_control * sc)2605 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2606 				 struct lruvec *lruvec, struct scan_control *sc)
2607 {
2608 	if (is_active_lru(lru)) {
2609 		if (sc->may_deactivate & (1 << is_file_lru(lru)))
2610 			shrink_active_list(nr_to_scan, lruvec, sc, lru);
2611 		else
2612 			sc->skipped_deactivate = 1;
2613 		return 0;
2614 	}
2615 
2616 	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2617 }
2618 
2619 /*
2620  * The inactive anon list should be small enough that the VM never has
2621  * to do too much work.
2622  *
2623  * The inactive file list should be small enough to leave most memory
2624  * to the established workingset on the scan-resistant active list,
2625  * but large enough to avoid thrashing the aggregate readahead window.
2626  *
2627  * Both inactive lists should also be large enough that each inactive
2628  * page has a chance to be referenced again before it is reclaimed.
2629  *
2630  * If that fails and refaulting is observed, the inactive list grows.
2631  *
2632  * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2633  * on this LRU, maintained by the pageout code. An inactive_ratio
2634  * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2635  *
2636  * total     target    max
2637  * memory    ratio     inactive
2638  * -------------------------------------
2639  *   10MB       1         5MB
2640  *  100MB       1        50MB
2641  *    1GB       3       250MB
2642  *   10GB      10       0.9GB
2643  *  100GB      31         3GB
2644  *    1TB     101        10GB
2645  *   10TB     320        32GB
2646  */
inactive_is_low(struct lruvec * lruvec,enum lru_list inactive_lru)2647 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2648 {
2649 	enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2650 	unsigned long inactive, active;
2651 	unsigned long inactive_ratio;
2652 	unsigned long gb;
2653 
2654 	inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2655 	active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2656 
2657 	gb = (inactive + active) >> (30 - PAGE_SHIFT);
2658 	if (gb)
2659 		inactive_ratio = int_sqrt(10 * gb);
2660 	else
2661 		inactive_ratio = 1;
2662 
2663 	return inactive * inactive_ratio < active;
2664 }
2665 
2666 enum scan_balance {
2667 	SCAN_EQUAL,
2668 	SCAN_FRACT,
2669 	SCAN_ANON,
2670 	SCAN_FILE,
2671 };
2672 
2673 /*
2674  * Determine how aggressively the anon and file LRU lists should be
2675  * scanned.
2676  *
2677  * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2678  * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2679  */
get_scan_count(struct lruvec * lruvec,struct scan_control * sc,unsigned long * nr)2680 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
2681 			   unsigned long *nr)
2682 {
2683 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2684 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2685 	unsigned long anon_cost, file_cost, total_cost;
2686 	int swappiness = mem_cgroup_swappiness(memcg);
2687 	u64 fraction[ANON_AND_FILE];
2688 	u64 denominator = 0;	/* gcc */
2689 	enum scan_balance scan_balance;
2690 	unsigned long ap, fp;
2691 	enum lru_list lru;
2692 
2693 	/* If we have no swap space, do not bother scanning anon pages. */
2694 	if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
2695 		scan_balance = SCAN_FILE;
2696 		goto out;
2697 	}
2698 
2699 	/*
2700 	 * Global reclaim will swap to prevent OOM even with no
2701 	 * swappiness, but memcg users want to use this knob to
2702 	 * disable swapping for individual groups completely when
2703 	 * using the memory controller's swap limit feature would be
2704 	 * too expensive.
2705 	 */
2706 	if (cgroup_reclaim(sc) && !swappiness) {
2707 		scan_balance = SCAN_FILE;
2708 		goto out;
2709 	}
2710 
2711 	/*
2712 	 * Do not apply any pressure balancing cleverness when the
2713 	 * system is close to OOM, scan both anon and file equally
2714 	 * (unless the swappiness setting disagrees with swapping).
2715 	 */
2716 	if (!sc->priority && swappiness) {
2717 		scan_balance = SCAN_EQUAL;
2718 		goto out;
2719 	}
2720 
2721 	/*
2722 	 * If the system is almost out of file pages, force-scan anon.
2723 	 */
2724 	if (sc->file_is_tiny) {
2725 		scan_balance = SCAN_ANON;
2726 		goto out;
2727 	}
2728 
2729 	/*
2730 	 * If there is enough inactive page cache, we do not reclaim
2731 	 * anything from the anonymous working right now.
2732 	 */
2733 	if (sc->cache_trim_mode) {
2734 		scan_balance = SCAN_FILE;
2735 		goto out;
2736 	}
2737 
2738 	scan_balance = SCAN_FRACT;
2739 	/*
2740 	 * Calculate the pressure balance between anon and file pages.
2741 	 *
2742 	 * The amount of pressure we put on each LRU is inversely
2743 	 * proportional to the cost of reclaiming each list, as
2744 	 * determined by the share of pages that are refaulting, times
2745 	 * the relative IO cost of bringing back a swapped out
2746 	 * anonymous page vs reloading a filesystem page (swappiness).
2747 	 *
2748 	 * Although we limit that influence to ensure no list gets
2749 	 * left behind completely: at least a third of the pressure is
2750 	 * applied, before swappiness.
2751 	 *
2752 	 * With swappiness at 100, anon and file have equal IO cost.
2753 	 */
2754 	total_cost = sc->anon_cost + sc->file_cost;
2755 	anon_cost = total_cost + sc->anon_cost;
2756 	file_cost = total_cost + sc->file_cost;
2757 	total_cost = anon_cost + file_cost;
2758 
2759 	ap = swappiness * (total_cost + 1);
2760 	ap /= anon_cost + 1;
2761 
2762 	fp = (200 - swappiness) * (total_cost + 1);
2763 	fp /= file_cost + 1;
2764 
2765 	fraction[0] = ap;
2766 	fraction[1] = fp;
2767 	denominator = ap + fp;
2768 out:
2769 	for_each_evictable_lru(lru) {
2770 		int file = is_file_lru(lru);
2771 		unsigned long lruvec_size;
2772 		unsigned long low, min;
2773 		unsigned long scan;
2774 
2775 		lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2776 		mem_cgroup_protection(sc->target_mem_cgroup, memcg,
2777 				      &min, &low);
2778 
2779 		if (min || low) {
2780 			/*
2781 			 * Scale a cgroup's reclaim pressure by proportioning
2782 			 * its current usage to its memory.low or memory.min
2783 			 * setting.
2784 			 *
2785 			 * This is important, as otherwise scanning aggression
2786 			 * becomes extremely binary -- from nothing as we
2787 			 * approach the memory protection threshold, to totally
2788 			 * nominal as we exceed it.  This results in requiring
2789 			 * setting extremely liberal protection thresholds. It
2790 			 * also means we simply get no protection at all if we
2791 			 * set it too low, which is not ideal.
2792 			 *
2793 			 * If there is any protection in place, we reduce scan
2794 			 * pressure by how much of the total memory used is
2795 			 * within protection thresholds.
2796 			 *
2797 			 * There is one special case: in the first reclaim pass,
2798 			 * we skip over all groups that are within their low
2799 			 * protection. If that fails to reclaim enough pages to
2800 			 * satisfy the reclaim goal, we come back and override
2801 			 * the best-effort low protection. However, we still
2802 			 * ideally want to honor how well-behaved groups are in
2803 			 * that case instead of simply punishing them all
2804 			 * equally. As such, we reclaim them based on how much
2805 			 * memory they are using, reducing the scan pressure
2806 			 * again by how much of the total memory used is under
2807 			 * hard protection.
2808 			 */
2809 			unsigned long cgroup_size = mem_cgroup_size(memcg);
2810 			unsigned long protection;
2811 
2812 			/* memory.low scaling, make sure we retry before OOM */
2813 			if (!sc->memcg_low_reclaim && low > min) {
2814 				protection = low;
2815 				sc->memcg_low_skipped = 1;
2816 			} else {
2817 				protection = min;
2818 			}
2819 
2820 			/* Avoid TOCTOU with earlier protection check */
2821 			cgroup_size = max(cgroup_size, protection);
2822 
2823 			scan = lruvec_size - lruvec_size * protection /
2824 				(cgroup_size + 1);
2825 
2826 			/*
2827 			 * Minimally target SWAP_CLUSTER_MAX pages to keep
2828 			 * reclaim moving forwards, avoiding decrementing
2829 			 * sc->priority further than desirable.
2830 			 */
2831 			scan = max(scan, SWAP_CLUSTER_MAX);
2832 		} else {
2833 			scan = lruvec_size;
2834 		}
2835 
2836 		scan >>= sc->priority;
2837 
2838 		/*
2839 		 * If the cgroup's already been deleted, make sure to
2840 		 * scrape out the remaining cache.
2841 		 */
2842 		if (!scan && !mem_cgroup_online(memcg))
2843 			scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2844 
2845 		switch (scan_balance) {
2846 		case SCAN_EQUAL:
2847 			/* Scan lists relative to size */
2848 			break;
2849 		case SCAN_FRACT:
2850 			/*
2851 			 * Scan types proportional to swappiness and
2852 			 * their relative recent reclaim efficiency.
2853 			 * Make sure we don't miss the last page on
2854 			 * the offlined memory cgroups because of a
2855 			 * round-off error.
2856 			 */
2857 			scan = mem_cgroup_online(memcg) ?
2858 			       div64_u64(scan * fraction[file], denominator) :
2859 			       DIV64_U64_ROUND_UP(scan * fraction[file],
2860 						  denominator);
2861 			break;
2862 		case SCAN_FILE:
2863 		case SCAN_ANON:
2864 			/* Scan one type exclusively */
2865 			if ((scan_balance == SCAN_FILE) != file)
2866 				scan = 0;
2867 			break;
2868 		default:
2869 			/* Look ma, no brain */
2870 			BUG();
2871 		}
2872 
2873 		nr[lru] = scan;
2874 	}
2875 }
2876 
2877 /*
2878  * Anonymous LRU management is a waste if there is
2879  * ultimately no way to reclaim the memory.
2880  */
can_age_anon_pages(struct pglist_data * pgdat,struct scan_control * sc)2881 static bool can_age_anon_pages(struct pglist_data *pgdat,
2882 			       struct scan_control *sc)
2883 {
2884 	/* Aging the anon LRU is valuable if swap is present: */
2885 	if (total_swap_pages > 0)
2886 		return true;
2887 
2888 	/* Also valuable if anon pages can be demoted: */
2889 	return can_demote(pgdat->node_id, sc);
2890 }
2891 
shrink_lruvec(struct lruvec * lruvec,struct scan_control * sc)2892 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
2893 {
2894 	unsigned long nr[NR_LRU_LISTS];
2895 	unsigned long targets[NR_LRU_LISTS];
2896 	unsigned long nr_to_scan;
2897 	enum lru_list lru;
2898 	unsigned long nr_reclaimed = 0;
2899 	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2900 	struct blk_plug plug;
2901 	bool scan_adjusted;
2902 
2903 	get_scan_count(lruvec, sc, nr);
2904 
2905 	/* Record the original scan target for proportional adjustments later */
2906 	memcpy(targets, nr, sizeof(nr));
2907 
2908 	/*
2909 	 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2910 	 * event that can occur when there is little memory pressure e.g.
2911 	 * multiple streaming readers/writers. Hence, we do not abort scanning
2912 	 * when the requested number of pages are reclaimed when scanning at
2913 	 * DEF_PRIORITY on the assumption that the fact we are direct
2914 	 * reclaiming implies that kswapd is not keeping up and it is best to
2915 	 * do a batch of work at once. For memcg reclaim one check is made to
2916 	 * abort proportional reclaim if either the file or anon lru has already
2917 	 * dropped to zero at the first pass.
2918 	 */
2919 	scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
2920 			 sc->priority == DEF_PRIORITY);
2921 
2922 	blk_start_plug(&plug);
2923 	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2924 					nr[LRU_INACTIVE_FILE]) {
2925 		unsigned long nr_anon, nr_file, percentage;
2926 		unsigned long nr_scanned;
2927 
2928 		for_each_evictable_lru(lru) {
2929 			if (nr[lru]) {
2930 				nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2931 				nr[lru] -= nr_to_scan;
2932 
2933 				nr_reclaimed += shrink_list(lru, nr_to_scan,
2934 							    lruvec, sc);
2935 			}
2936 		}
2937 
2938 		cond_resched();
2939 
2940 		if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2941 			continue;
2942 
2943 		/*
2944 		 * For kswapd and memcg, reclaim at least the number of pages
2945 		 * requested. Ensure that the anon and file LRUs are scanned
2946 		 * proportionally what was requested by get_scan_count(). We
2947 		 * stop reclaiming one LRU and reduce the amount scanning
2948 		 * proportional to the original scan target.
2949 		 */
2950 		nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2951 		nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2952 
2953 		/*
2954 		 * It's just vindictive to attack the larger once the smaller
2955 		 * has gone to zero.  And given the way we stop scanning the
2956 		 * smaller below, this makes sure that we only make one nudge
2957 		 * towards proportionality once we've got nr_to_reclaim.
2958 		 */
2959 		if (!nr_file || !nr_anon)
2960 			break;
2961 
2962 		if (nr_file > nr_anon) {
2963 			unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2964 						targets[LRU_ACTIVE_ANON] + 1;
2965 			lru = LRU_BASE;
2966 			percentage = nr_anon * 100 / scan_target;
2967 		} else {
2968 			unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2969 						targets[LRU_ACTIVE_FILE] + 1;
2970 			lru = LRU_FILE;
2971 			percentage = nr_file * 100 / scan_target;
2972 		}
2973 
2974 		/* Stop scanning the smaller of the LRU */
2975 		nr[lru] = 0;
2976 		nr[lru + LRU_ACTIVE] = 0;
2977 
2978 		/*
2979 		 * Recalculate the other LRU scan count based on its original
2980 		 * scan target and the percentage scanning already complete
2981 		 */
2982 		lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2983 		nr_scanned = targets[lru] - nr[lru];
2984 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2985 		nr[lru] -= min(nr[lru], nr_scanned);
2986 
2987 		lru += LRU_ACTIVE;
2988 		nr_scanned = targets[lru] - nr[lru];
2989 		nr[lru] = targets[lru] * (100 - percentage) / 100;
2990 		nr[lru] -= min(nr[lru], nr_scanned);
2991 
2992 		scan_adjusted = true;
2993 	}
2994 	blk_finish_plug(&plug);
2995 	sc->nr_reclaimed += nr_reclaimed;
2996 
2997 	/*
2998 	 * Even if we did not try to evict anon pages at all, we want to
2999 	 * rebalance the anon lru active/inactive ratio.
3000 	 */
3001 	if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
3002 	    inactive_is_low(lruvec, LRU_INACTIVE_ANON))
3003 		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3004 				   sc, LRU_ACTIVE_ANON);
3005 }
3006 
3007 /* Use reclaim/compaction for costly allocs or under memory pressure */
in_reclaim_compaction(struct scan_control * sc)3008 static bool in_reclaim_compaction(struct scan_control *sc)
3009 {
3010 	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
3011 			(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
3012 			 sc->priority < DEF_PRIORITY - 2))
3013 		return true;
3014 
3015 	return false;
3016 }
3017 
3018 /*
3019  * Reclaim/compaction is used for high-order allocation requests. It reclaims
3020  * order-0 pages before compacting the zone. should_continue_reclaim() returns
3021  * true if more pages should be reclaimed such that when the page allocator
3022  * calls try_to_compact_pages() that it will have enough free pages to succeed.
3023  * It will give up earlier than that if there is difficulty reclaiming pages.
3024  */
should_continue_reclaim(struct pglist_data * pgdat,unsigned long nr_reclaimed,struct scan_control * sc)3025 static inline bool should_continue_reclaim(struct pglist_data *pgdat,
3026 					unsigned long nr_reclaimed,
3027 					struct scan_control *sc)
3028 {
3029 	unsigned long pages_for_compaction;
3030 	unsigned long inactive_lru_pages;
3031 	int z;
3032 
3033 	/* If not in reclaim/compaction mode, stop */
3034 	if (!in_reclaim_compaction(sc))
3035 		return false;
3036 
3037 	/*
3038 	 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
3039 	 * number of pages that were scanned. This will return to the caller
3040 	 * with the risk reclaim/compaction and the resulting allocation attempt
3041 	 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
3042 	 * allocations through requiring that the full LRU list has been scanned
3043 	 * first, by assuming that zero delta of sc->nr_scanned means full LRU
3044 	 * scan, but that approximation was wrong, and there were corner cases
3045 	 * where always a non-zero amount of pages were scanned.
3046 	 */
3047 	if (!nr_reclaimed)
3048 		return false;
3049 
3050 	/* If compaction would go ahead or the allocation would succeed, stop */
3051 	for (z = 0; z <= sc->reclaim_idx; z++) {
3052 		struct zone *zone = &pgdat->node_zones[z];
3053 		if (!managed_zone(zone))
3054 			continue;
3055 
3056 		switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
3057 		case COMPACT_SUCCESS:
3058 		case COMPACT_CONTINUE:
3059 			return false;
3060 		default:
3061 			/* check next zone */
3062 			;
3063 		}
3064 	}
3065 
3066 	/*
3067 	 * If we have not reclaimed enough pages for compaction and the
3068 	 * inactive lists are large enough, continue reclaiming
3069 	 */
3070 	pages_for_compaction = compact_gap(sc->order);
3071 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
3072 	if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
3073 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
3074 
3075 	return inactive_lru_pages > pages_for_compaction;
3076 }
3077 
shrink_node_memcgs(pg_data_t * pgdat,struct scan_control * sc)3078 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
3079 {
3080 	struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
3081 	struct mem_cgroup *memcg;
3082 
3083 	memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
3084 	do {
3085 		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3086 		unsigned long reclaimed;
3087 		unsigned long scanned;
3088 
3089 		/*
3090 		 * This loop can become CPU-bound when target memcgs
3091 		 * aren't eligible for reclaim - either because they
3092 		 * don't have any reclaimable pages, or because their
3093 		 * memory is explicitly protected. Avoid soft lockups.
3094 		 */
3095 		cond_resched();
3096 
3097 		mem_cgroup_calculate_protection(target_memcg, memcg);
3098 
3099 		if (mem_cgroup_below_min(memcg)) {
3100 			/*
3101 			 * Hard protection.
3102 			 * If there is no reclaimable memory, OOM.
3103 			 */
3104 			continue;
3105 		} else if (mem_cgroup_below_low(memcg)) {
3106 			/*
3107 			 * Soft protection.
3108 			 * Respect the protection only as long as
3109 			 * there is an unprotected supply
3110 			 * of reclaimable memory from other cgroups.
3111 			 */
3112 			if (!sc->memcg_low_reclaim) {
3113 				sc->memcg_low_skipped = 1;
3114 				continue;
3115 			}
3116 			memcg_memory_event(memcg, MEMCG_LOW);
3117 		}
3118 
3119 		reclaimed = sc->nr_reclaimed;
3120 		scanned = sc->nr_scanned;
3121 
3122 		shrink_lruvec(lruvec, sc);
3123 
3124 		shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
3125 			    sc->priority);
3126 
3127 		/* Record the group's reclaim efficiency */
3128 		vmpressure(sc->gfp_mask, memcg, false,
3129 			   sc->nr_scanned - scanned,
3130 			   sc->nr_reclaimed - reclaimed);
3131 
3132 	} while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
3133 }
3134 
shrink_node(pg_data_t * pgdat,struct scan_control * sc)3135 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
3136 {
3137 	struct reclaim_state *reclaim_state = current->reclaim_state;
3138 	unsigned long nr_reclaimed, nr_scanned;
3139 	struct lruvec *target_lruvec;
3140 	bool reclaimable = false;
3141 	unsigned long file;
3142 
3143 	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
3144 
3145 again:
3146 	/*
3147 	 * Flush the memory cgroup stats, so that we read accurate per-memcg
3148 	 * lruvec stats for heuristics.
3149 	 */
3150 	mem_cgroup_flush_stats();
3151 
3152 	memset(&sc->nr, 0, sizeof(sc->nr));
3153 
3154 	nr_reclaimed = sc->nr_reclaimed;
3155 	nr_scanned = sc->nr_scanned;
3156 
3157 	/*
3158 	 * Determine the scan balance between anon and file LRUs.
3159 	 */
3160 	spin_lock_irq(&target_lruvec->lru_lock);
3161 	sc->anon_cost = target_lruvec->anon_cost;
3162 	sc->file_cost = target_lruvec->file_cost;
3163 	spin_unlock_irq(&target_lruvec->lru_lock);
3164 
3165 	/*
3166 	 * Target desirable inactive:active list ratios for the anon
3167 	 * and file LRU lists.
3168 	 */
3169 	if (!sc->force_deactivate) {
3170 		unsigned long refaults;
3171 
3172 		refaults = lruvec_page_state(target_lruvec,
3173 				WORKINGSET_ACTIVATE_ANON);
3174 		if (refaults != target_lruvec->refaults[0] ||
3175 			inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
3176 			sc->may_deactivate |= DEACTIVATE_ANON;
3177 		else
3178 			sc->may_deactivate &= ~DEACTIVATE_ANON;
3179 
3180 		/*
3181 		 * When refaults are being observed, it means a new
3182 		 * workingset is being established. Deactivate to get
3183 		 * rid of any stale active pages quickly.
3184 		 */
3185 		refaults = lruvec_page_state(target_lruvec,
3186 				WORKINGSET_ACTIVATE_FILE);
3187 		if (refaults != target_lruvec->refaults[1] ||
3188 		    inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
3189 			sc->may_deactivate |= DEACTIVATE_FILE;
3190 		else
3191 			sc->may_deactivate &= ~DEACTIVATE_FILE;
3192 	} else
3193 		sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
3194 
3195 	/*
3196 	 * If we have plenty of inactive file pages that aren't
3197 	 * thrashing, try to reclaim those first before touching
3198 	 * anonymous pages.
3199 	 */
3200 	file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
3201 	if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
3202 		sc->cache_trim_mode = 1;
3203 	else
3204 		sc->cache_trim_mode = 0;
3205 
3206 	/*
3207 	 * Prevent the reclaimer from falling into the cache trap: as
3208 	 * cache pages start out inactive, every cache fault will tip
3209 	 * the scan balance towards the file LRU.  And as the file LRU
3210 	 * shrinks, so does the window for rotation from references.
3211 	 * This means we have a runaway feedback loop where a tiny
3212 	 * thrashing file LRU becomes infinitely more attractive than
3213 	 * anon pages.  Try to detect this based on file LRU size.
3214 	 */
3215 	if (!cgroup_reclaim(sc)) {
3216 		unsigned long total_high_wmark = 0;
3217 		unsigned long free, anon;
3218 		int z;
3219 
3220 		free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
3221 		file = node_page_state(pgdat, NR_ACTIVE_FILE) +
3222 			   node_page_state(pgdat, NR_INACTIVE_FILE);
3223 
3224 		for (z = 0; z < MAX_NR_ZONES; z++) {
3225 			struct zone *zone = &pgdat->node_zones[z];
3226 			if (!managed_zone(zone))
3227 				continue;
3228 
3229 			total_high_wmark += high_wmark_pages(zone);
3230 		}
3231 
3232 		/*
3233 		 * Consider anon: if that's low too, this isn't a
3234 		 * runaway file reclaim problem, but rather just
3235 		 * extreme pressure. Reclaim as per usual then.
3236 		 */
3237 		anon = node_page_state(pgdat, NR_INACTIVE_ANON);
3238 
3239 		sc->file_is_tiny =
3240 			file + free <= total_high_wmark &&
3241 			!(sc->may_deactivate & DEACTIVATE_ANON) &&
3242 			anon >> sc->priority;
3243 	}
3244 
3245 	shrink_node_memcgs(pgdat, sc);
3246 
3247 	if (reclaim_state) {
3248 		sc->nr_reclaimed += reclaim_state->reclaimed_slab;
3249 		reclaim_state->reclaimed_slab = 0;
3250 	}
3251 
3252 	/* Record the subtree's reclaim efficiency */
3253 	vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
3254 		   sc->nr_scanned - nr_scanned,
3255 		   sc->nr_reclaimed - nr_reclaimed);
3256 
3257 	if (sc->nr_reclaimed - nr_reclaimed)
3258 		reclaimable = true;
3259 
3260 	if (current_is_kswapd()) {
3261 		/*
3262 		 * If reclaim is isolating dirty pages under writeback,
3263 		 * it implies that the long-lived page allocation rate
3264 		 * is exceeding the page laundering rate. Either the
3265 		 * global limits are not being effective at throttling
3266 		 * processes due to the page distribution throughout
3267 		 * zones or there is heavy usage of a slow backing
3268 		 * device. The only option is to throttle from reclaim
3269 		 * context which is not ideal as there is no guarantee
3270 		 * the dirtying process is throttled in the same way
3271 		 * balance_dirty_pages() manages.
3272 		 *
3273 		 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
3274 		 * count the number of pages under pages flagged for
3275 		 * immediate reclaim and stall if any are encountered
3276 		 * in the nr_immediate check below.
3277 		 */
3278 		if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
3279 			set_bit(PGDAT_WRITEBACK, &pgdat->flags);
3280 
3281 		/* Allow kswapd to start writing pages during reclaim.*/
3282 		if (sc->nr.unqueued_dirty == sc->nr.file_taken)
3283 			set_bit(PGDAT_DIRTY, &pgdat->flags);
3284 
3285 		/*
3286 		 * If kswapd scans pages marked for immediate
3287 		 * reclaim and under writeback (nr_immediate), it
3288 		 * implies that pages are cycling through the LRU
3289 		 * faster than they are written so forcibly stall
3290 		 * until some pages complete writeback.
3291 		 */
3292 		if (sc->nr.immediate)
3293 			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
3294 	}
3295 
3296 	/*
3297 	 * Tag a node/memcg as congested if all the dirty pages were marked
3298 	 * for writeback and immediate reclaim (counted in nr.congested).
3299 	 *
3300 	 * Legacy memcg will stall in page writeback so avoid forcibly
3301 	 * stalling in reclaim_throttle().
3302 	 */
3303 	if ((current_is_kswapd() ||
3304 	     (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
3305 	    sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
3306 		set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
3307 
3308 	/*
3309 	 * Stall direct reclaim for IO completions if the lruvec is
3310 	 * node is congested. Allow kswapd to continue until it
3311 	 * starts encountering unqueued dirty pages or cycling through
3312 	 * the LRU too quickly.
3313 	 */
3314 	if (!current_is_kswapd() && current_may_throttle() &&
3315 	    !sc->hibernation_mode &&
3316 	    test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
3317 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);
3318 
3319 	if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
3320 				    sc))
3321 		goto again;
3322 
3323 	/*
3324 	 * Kswapd gives up on balancing particular nodes after too
3325 	 * many failures to reclaim anything from them and goes to
3326 	 * sleep. On reclaim progress, reset the failure counter. A
3327 	 * successful direct reclaim run will revive a dormant kswapd.
3328 	 */
3329 	if (reclaimable)
3330 		pgdat->kswapd_failures = 0;
3331 }
3332 
3333 /*
3334  * Returns true if compaction should go ahead for a costly-order request, or
3335  * the allocation would already succeed without compaction. Return false if we
3336  * should reclaim first.
3337  */
compaction_ready(struct zone * zone,struct scan_control * sc)3338 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
3339 {
3340 	unsigned long watermark;
3341 	enum compact_result suitable;
3342 
3343 	suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
3344 	if (suitable == COMPACT_SUCCESS)
3345 		/* Allocation should succeed already. Don't reclaim. */
3346 		return true;
3347 	if (suitable == COMPACT_SKIPPED)
3348 		/* Compaction cannot yet proceed. Do reclaim. */
3349 		return false;
3350 
3351 	/*
3352 	 * Compaction is already possible, but it takes time to run and there
3353 	 * are potentially other callers using the pages just freed. So proceed
3354 	 * with reclaim to make a buffer of free pages available to give
3355 	 * compaction a reasonable chance of completing and allocating the page.
3356 	 * Note that we won't actually reclaim the whole buffer in one attempt
3357 	 * as the target watermark in should_continue_reclaim() is lower. But if
3358 	 * we are already above the high+gap watermark, don't reclaim at all.
3359 	 */
3360 	watermark = high_wmark_pages(zone) + compact_gap(sc->order);
3361 
3362 	return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
3363 }
3364 
consider_reclaim_throttle(pg_data_t * pgdat,struct scan_control * sc)3365 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
3366 {
3367 	/*
3368 	 * If reclaim is making progress greater than 12% efficiency then
3369 	 * wake all the NOPROGRESS throttled tasks.
3370 	 */
3371 	if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) {
3372 		wait_queue_head_t *wqh;
3373 
3374 		wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];
3375 		if (waitqueue_active(wqh))
3376 			wake_up(wqh);
3377 
3378 		return;
3379 	}
3380 
3381 	/*
3382 	 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
3383 	 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
3384 	 * under writeback and marked for immediate reclaim at the tail of the
3385 	 * LRU.
3386 	 */
3387 	if (current_is_kswapd() || cgroup_reclaim(sc))
3388 		return;
3389 
3390 	/* Throttle if making no progress at high prioities. */
3391 	if (sc->priority == 1 && !sc->nr_reclaimed)
3392 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
3393 }
3394 
3395 /*
3396  * This is the direct reclaim path, for page-allocating processes.  We only
3397  * try to reclaim pages from zones which will satisfy the caller's allocation
3398  * request.
3399  *
3400  * If a zone is deemed to be full of pinned pages then just give it a light
3401  * scan then give up on it.
3402  */
shrink_zones(struct zonelist * zonelist,struct scan_control * sc)3403 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
3404 {
3405 	struct zoneref *z;
3406 	struct zone *zone;
3407 	unsigned long nr_soft_reclaimed;
3408 	unsigned long nr_soft_scanned;
3409 	gfp_t orig_mask;
3410 	pg_data_t *last_pgdat = NULL;
3411 	pg_data_t *first_pgdat = NULL;
3412 
3413 	/*
3414 	 * If the number of buffer_heads in the machine exceeds the maximum
3415 	 * allowed level, force direct reclaim to scan the highmem zone as
3416 	 * highmem pages could be pinning lowmem pages storing buffer_heads
3417 	 */
3418 	orig_mask = sc->gfp_mask;
3419 	if (buffer_heads_over_limit) {
3420 		sc->gfp_mask |= __GFP_HIGHMEM;
3421 		sc->reclaim_idx = gfp_zone(sc->gfp_mask);
3422 	}
3423 
3424 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
3425 					sc->reclaim_idx, sc->nodemask) {
3426 		/*
3427 		 * Take care memory controller reclaiming has small influence
3428 		 * to global LRU.
3429 		 */
3430 		if (!cgroup_reclaim(sc)) {
3431 			if (!cpuset_zone_allowed(zone,
3432 						 GFP_KERNEL | __GFP_HARDWALL))
3433 				continue;
3434 
3435 			/*
3436 			 * If we already have plenty of memory free for
3437 			 * compaction in this zone, don't free any more.
3438 			 * Even though compaction is invoked for any
3439 			 * non-zero order, only frequent costly order
3440 			 * reclamation is disruptive enough to become a
3441 			 * noticeable problem, like transparent huge
3442 			 * page allocations.
3443 			 */
3444 			if (IS_ENABLED(CONFIG_COMPACTION) &&
3445 			    sc->order > PAGE_ALLOC_COSTLY_ORDER &&
3446 			    compaction_ready(zone, sc)) {
3447 				sc->compaction_ready = true;
3448 				continue;
3449 			}
3450 
3451 			/*
3452 			 * Shrink each node in the zonelist once. If the
3453 			 * zonelist is ordered by zone (not the default) then a
3454 			 * node may be shrunk multiple times but in that case
3455 			 * the user prefers lower zones being preserved.
3456 			 */
3457 			if (zone->zone_pgdat == last_pgdat)
3458 				continue;
3459 
3460 			/*
3461 			 * This steals pages from memory cgroups over softlimit
3462 			 * and returns the number of reclaimed pages and
3463 			 * scanned pages. This works for global memory pressure
3464 			 * and balancing, not for a memcg's limit.
3465 			 */
3466 			nr_soft_scanned = 0;
3467 			nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
3468 						sc->order, sc->gfp_mask,
3469 						&nr_soft_scanned);
3470 			sc->nr_reclaimed += nr_soft_reclaimed;
3471 			sc->nr_scanned += nr_soft_scanned;
3472 			/* need some check for avoid more shrink_zone() */
3473 		}
3474 
3475 		if (!first_pgdat)
3476 			first_pgdat = zone->zone_pgdat;
3477 
3478 		/* See comment about same check for global reclaim above */
3479 		if (zone->zone_pgdat == last_pgdat)
3480 			continue;
3481 		last_pgdat = zone->zone_pgdat;
3482 		shrink_node(zone->zone_pgdat, sc);
3483 	}
3484 
3485 	if (first_pgdat)
3486 		consider_reclaim_throttle(first_pgdat, sc);
3487 
3488 	/*
3489 	 * Restore to original mask to avoid the impact on the caller if we
3490 	 * promoted it to __GFP_HIGHMEM.
3491 	 */
3492 	sc->gfp_mask = orig_mask;
3493 }
3494 
snapshot_refaults(struct mem_cgroup * target_memcg,pg_data_t * pgdat)3495 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
3496 {
3497 	struct lruvec *target_lruvec;
3498 	unsigned long refaults;
3499 
3500 	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
3501 	refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
3502 	target_lruvec->refaults[0] = refaults;
3503 	refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
3504 	target_lruvec->refaults[1] = refaults;
3505 }
3506 
3507 /*
3508  * This is the main entry point to direct page reclaim.
3509  *
3510  * If a full scan of the inactive list fails to free enough memory then we
3511  * are "out of memory" and something needs to be killed.
3512  *
3513  * If the caller is !__GFP_FS then the probability of a failure is reasonably
3514  * high - the zone may be full of dirty or under-writeback pages, which this
3515  * caller can't do much about.  We kick the writeback threads and take explicit
3516  * naps in the hope that some of these pages can be written.  But if the
3517  * allocating task holds filesystem locks which prevent writeout this might not
3518  * work, and the allocation attempt will fail.
3519  *
3520  * returns:	0, if no pages reclaimed
3521  * 		else, the number of pages reclaimed
3522  */
do_try_to_free_pages(struct zonelist * zonelist,struct scan_control * sc)3523 static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
3524 					  struct scan_control *sc)
3525 {
3526 	int initial_priority = sc->priority;
3527 	pg_data_t *last_pgdat;
3528 	struct zoneref *z;
3529 	struct zone *zone;
3530 retry:
3531 	delayacct_freepages_start();
3532 
3533 	if (!cgroup_reclaim(sc))
3534 		__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3535 
3536 	do {
3537 		vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
3538 				sc->priority);
3539 		sc->nr_scanned = 0;
3540 		shrink_zones(zonelist, sc);
3541 
3542 		if (sc->nr_reclaimed >= sc->nr_to_reclaim)
3543 			break;
3544 
3545 		if (sc->compaction_ready)
3546 			break;
3547 
3548 		/*
3549 		 * If we're getting trouble reclaiming, start doing
3550 		 * writepage even in laptop mode.
3551 		 */
3552 		if (sc->priority < DEF_PRIORITY - 2)
3553 			sc->may_writepage = 1;
3554 	} while (--sc->priority >= 0);
3555 
3556 	last_pgdat = NULL;
3557 	for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
3558 					sc->nodemask) {
3559 		if (zone->zone_pgdat == last_pgdat)
3560 			continue;
3561 		last_pgdat = zone->zone_pgdat;
3562 
3563 		snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3564 
3565 		if (cgroup_reclaim(sc)) {
3566 			struct lruvec *lruvec;
3567 
3568 			lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
3569 						   zone->zone_pgdat);
3570 			clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3571 		}
3572 	}
3573 
3574 	delayacct_freepages_end();
3575 
3576 	if (sc->nr_reclaimed)
3577 		return sc->nr_reclaimed;
3578 
3579 	/* Aborted reclaim to try compaction? don't OOM, then */
3580 	if (sc->compaction_ready)
3581 		return 1;
3582 
3583 	/*
3584 	 * We make inactive:active ratio decisions based on the node's
3585 	 * composition of memory, but a restrictive reclaim_idx or a
3586 	 * memory.low cgroup setting can exempt large amounts of
3587 	 * memory from reclaim. Neither of which are very common, so
3588 	 * instead of doing costly eligibility calculations of the
3589 	 * entire cgroup subtree up front, we assume the estimates are
3590 	 * good, and retry with forcible deactivation if that fails.
3591 	 */
3592 	if (sc->skipped_deactivate) {
3593 		sc->priority = initial_priority;
3594 		sc->force_deactivate = 1;
3595 		sc->skipped_deactivate = 0;
3596 		goto retry;
3597 	}
3598 
3599 	/* Untapped cgroup reserves?  Don't OOM, retry. */
3600 	if (sc->memcg_low_skipped) {
3601 		sc->priority = initial_priority;
3602 		sc->force_deactivate = 0;
3603 		sc->memcg_low_reclaim = 1;
3604 		sc->memcg_low_skipped = 0;
3605 		goto retry;
3606 	}
3607 
3608 	return 0;
3609 }
3610 
allow_direct_reclaim(pg_data_t * pgdat)3611 static bool allow_direct_reclaim(pg_data_t *pgdat)
3612 {
3613 	struct zone *zone;
3614 	unsigned long pfmemalloc_reserve = 0;
3615 	unsigned long free_pages = 0;
3616 	int i;
3617 	bool wmark_ok;
3618 
3619 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3620 		return true;
3621 
3622 	for (i = 0; i <= ZONE_NORMAL; i++) {
3623 		zone = &pgdat->node_zones[i];
3624 		if (!managed_zone(zone))
3625 			continue;
3626 
3627 		if (!zone_reclaimable_pages(zone))
3628 			continue;
3629 
3630 		pfmemalloc_reserve += min_wmark_pages(zone);
3631 		free_pages += zone_page_state(zone, NR_FREE_PAGES);
3632 	}
3633 
3634 	/* If there are no reserves (unexpected config) then do not throttle */
3635 	if (!pfmemalloc_reserve)
3636 		return true;
3637 
3638 	wmark_ok = free_pages > pfmemalloc_reserve / 2;
3639 
3640 	/* kswapd must be awake if processes are being throttled */
3641 	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3642 		if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
3643 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
3644 
3645 		wake_up_interruptible(&pgdat->kswapd_wait);
3646 	}
3647 
3648 	return wmark_ok;
3649 }
3650 
3651 /*
3652  * Throttle direct reclaimers if backing storage is backed by the network
3653  * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3654  * depleted. kswapd will continue to make progress and wake the processes
3655  * when the low watermark is reached.
3656  *
3657  * Returns true if a fatal signal was delivered during throttling. If this
3658  * happens, the page allocator should not consider triggering the OOM killer.
3659  */
throttle_direct_reclaim(gfp_t gfp_mask,struct zonelist * zonelist,nodemask_t * nodemask)3660 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
3661 					nodemask_t *nodemask)
3662 {
3663 	struct zoneref *z;
3664 	struct zone *zone;
3665 	pg_data_t *pgdat = NULL;
3666 
3667 	/*
3668 	 * Kernel threads should not be throttled as they may be indirectly
3669 	 * responsible for cleaning pages necessary for reclaim to make forward
3670 	 * progress. kjournald for example may enter direct reclaim while
3671 	 * committing a transaction where throttling it could forcing other
3672 	 * processes to block on log_wait_commit().
3673 	 */
3674 	if (current->flags & PF_KTHREAD)
3675 		goto out;
3676 
3677 	/*
3678 	 * If a fatal signal is pending, this process should not throttle.
3679 	 * It should return quickly so it can exit and free its memory
3680 	 */
3681 	if (fatal_signal_pending(current))
3682 		goto out;
3683 
3684 	/*
3685 	 * Check if the pfmemalloc reserves are ok by finding the first node
3686 	 * with a usable ZONE_NORMAL or lower zone. The expectation is that
3687 	 * GFP_KERNEL will be required for allocating network buffers when
3688 	 * swapping over the network so ZONE_HIGHMEM is unusable.
3689 	 *
3690 	 * Throttling is based on the first usable node and throttled processes
3691 	 * wait on a queue until kswapd makes progress and wakes them. There
3692 	 * is an affinity then between processes waking up and where reclaim
3693 	 * progress has been made assuming the process wakes on the same node.
3694 	 * More importantly, processes running on remote nodes will not compete
3695 	 * for remote pfmemalloc reserves and processes on different nodes
3696 	 * should make reasonable progress.
3697 	 */
3698 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
3699 					gfp_zone(gfp_mask), nodemask) {
3700 		if (zone_idx(zone) > ZONE_NORMAL)
3701 			continue;
3702 
3703 		/* Throttle based on the first usable node */
3704 		pgdat = zone->zone_pgdat;
3705 		if (allow_direct_reclaim(pgdat))
3706 			goto out;
3707 		break;
3708 	}
3709 
3710 	/* If no zone was usable by the allocation flags then do not throttle */
3711 	if (!pgdat)
3712 		goto out;
3713 
3714 	/* Account for the throttling */
3715 	count_vm_event(PGSCAN_DIRECT_THROTTLE);
3716 
3717 	/*
3718 	 * If the caller cannot enter the filesystem, it's possible that it
3719 	 * is due to the caller holding an FS lock or performing a journal
3720 	 * transaction in the case of a filesystem like ext[3|4]. In this case,
3721 	 * it is not safe to block on pfmemalloc_wait as kswapd could be
3722 	 * blocked waiting on the same lock. Instead, throttle for up to a
3723 	 * second before continuing.
3724 	 */
3725 	if (!(gfp_mask & __GFP_FS))
3726 		wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3727 			allow_direct_reclaim(pgdat), HZ);
3728 	else
3729 		/* Throttle until kswapd wakes the process */
3730 		wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3731 			allow_direct_reclaim(pgdat));
3732 
3733 	if (fatal_signal_pending(current))
3734 		return true;
3735 
3736 out:
3737 	return false;
3738 }
3739 
try_to_free_pages(struct zonelist * zonelist,int order,gfp_t gfp_mask,nodemask_t * nodemask)3740 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3741 				gfp_t gfp_mask, nodemask_t *nodemask)
3742 {
3743 	unsigned long nr_reclaimed;
3744 	struct scan_control sc = {
3745 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
3746 		.gfp_mask = current_gfp_context(gfp_mask),
3747 		.reclaim_idx = gfp_zone(gfp_mask),
3748 		.order = order,
3749 		.nodemask = nodemask,
3750 		.priority = DEF_PRIORITY,
3751 		.may_writepage = !laptop_mode,
3752 		.may_unmap = 1,
3753 		.may_swap = 1,
3754 	};
3755 
3756 	/*
3757 	 * scan_control uses s8 fields for order, priority, and reclaim_idx.
3758 	 * Confirm they are large enough for max values.
3759 	 */
3760 	BUILD_BUG_ON(MAX_ORDER > S8_MAX);
3761 	BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
3762 	BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
3763 
3764 	/*
3765 	 * Do not enter reclaim if fatal signal was delivered while throttled.
3766 	 * 1 is returned so that the page allocator does not OOM kill at this
3767 	 * point.
3768 	 */
3769 	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3770 		return 1;
3771 
3772 	set_task_reclaim_state(current, &sc.reclaim_state);
3773 	trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3774 
3775 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3776 
3777 	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3778 	set_task_reclaim_state(current, NULL);
3779 
3780 	return nr_reclaimed;
3781 }
3782 
3783 #ifdef CONFIG_MEMCG
3784 
3785 /* Only used by soft limit reclaim. Do not reuse for anything else. */
mem_cgroup_shrink_node(struct mem_cgroup * memcg,gfp_t gfp_mask,bool noswap,pg_data_t * pgdat,unsigned long * nr_scanned)3786 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3787 						gfp_t gfp_mask, bool noswap,
3788 						pg_data_t *pgdat,
3789 						unsigned long *nr_scanned)
3790 {
3791 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3792 	struct scan_control sc = {
3793 		.nr_to_reclaim = SWAP_CLUSTER_MAX,
3794 		.target_mem_cgroup = memcg,
3795 		.may_writepage = !laptop_mode,
3796 		.may_unmap = 1,
3797 		.reclaim_idx = MAX_NR_ZONES - 1,
3798 		.may_swap = !noswap,
3799 	};
3800 
3801 	WARN_ON_ONCE(!current->reclaim_state);
3802 
3803 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3804 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3805 
3806 	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3807 						      sc.gfp_mask);
3808 
3809 	/*
3810 	 * NOTE: Although we can get the priority field, using it
3811 	 * here is not a good idea, since it limits the pages we can scan.
3812 	 * if we don't reclaim here, the shrink_node from balance_pgdat
3813 	 * will pick up pages from other mem cgroup's as well. We hack
3814 	 * the priority and make it zero.
3815 	 */
3816 	shrink_lruvec(lruvec, &sc);
3817 
3818 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3819 
3820 	*nr_scanned = sc.nr_scanned;
3821 
3822 	return sc.nr_reclaimed;
3823 }
3824 
try_to_free_mem_cgroup_pages(struct mem_cgroup * memcg,unsigned long nr_pages,gfp_t gfp_mask,bool may_swap)3825 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3826 					   unsigned long nr_pages,
3827 					   gfp_t gfp_mask,
3828 					   bool may_swap)
3829 {
3830 	unsigned long nr_reclaimed;
3831 	unsigned int noreclaim_flag;
3832 	struct scan_control sc = {
3833 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3834 		.gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
3835 				(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3836 		.reclaim_idx = MAX_NR_ZONES - 1,
3837 		.target_mem_cgroup = memcg,
3838 		.priority = DEF_PRIORITY,
3839 		.may_writepage = !laptop_mode,
3840 		.may_unmap = 1,
3841 		.may_swap = may_swap,
3842 	};
3843 	/*
3844 	 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3845 	 * equal pressure on all the nodes. This is based on the assumption that
3846 	 * the reclaim does not bail out early.
3847 	 */
3848 	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3849 
3850 	set_task_reclaim_state(current, &sc.reclaim_state);
3851 	trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3852 	noreclaim_flag = memalloc_noreclaim_save();
3853 
3854 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3855 
3856 	memalloc_noreclaim_restore(noreclaim_flag);
3857 	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3858 	set_task_reclaim_state(current, NULL);
3859 
3860 	return nr_reclaimed;
3861 }
3862 #endif
3863 
age_active_anon(struct pglist_data * pgdat,struct scan_control * sc)3864 static void age_active_anon(struct pglist_data *pgdat,
3865 				struct scan_control *sc)
3866 {
3867 	struct mem_cgroup *memcg;
3868 	struct lruvec *lruvec;
3869 
3870 	if (!can_age_anon_pages(pgdat, sc))
3871 		return;
3872 
3873 	lruvec = mem_cgroup_lruvec(NULL, pgdat);
3874 	if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
3875 		return;
3876 
3877 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3878 	do {
3879 		lruvec = mem_cgroup_lruvec(memcg, pgdat);
3880 		shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3881 				   sc, LRU_ACTIVE_ANON);
3882 		memcg = mem_cgroup_iter(NULL, memcg, NULL);
3883 	} while (memcg);
3884 }
3885 
pgdat_watermark_boosted(pg_data_t * pgdat,int highest_zoneidx)3886 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
3887 {
3888 	int i;
3889 	struct zone *zone;
3890 
3891 	/*
3892 	 * Check for watermark boosts top-down as the higher zones
3893 	 * are more likely to be boosted. Both watermarks and boosts
3894 	 * should not be checked at the same time as reclaim would
3895 	 * start prematurely when there is no boosting and a lower
3896 	 * zone is balanced.
3897 	 */
3898 	for (i = highest_zoneidx; i >= 0; i--) {
3899 		zone = pgdat->node_zones + i;
3900 		if (!managed_zone(zone))
3901 			continue;
3902 
3903 		if (zone->watermark_boost)
3904 			return true;
3905 	}
3906 
3907 	return false;
3908 }
3909 
3910 /*
3911  * Returns true if there is an eligible zone balanced for the request order
3912  * and highest_zoneidx
3913  */
pgdat_balanced(pg_data_t * pgdat,int order,int highest_zoneidx)3914 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
3915 {
3916 	int i;
3917 	unsigned long mark = -1;
3918 	struct zone *zone;
3919 
3920 	/*
3921 	 * Check watermarks bottom-up as lower zones are more likely to
3922 	 * meet watermarks.
3923 	 */
3924 	for (i = 0; i <= highest_zoneidx; i++) {
3925 		zone = pgdat->node_zones + i;
3926 
3927 		if (!managed_zone(zone))
3928 			continue;
3929 
3930 		if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
3931 			mark = wmark_pages(zone, WMARK_PROMO);
3932 		else
3933 			mark = high_wmark_pages(zone);
3934 		if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
3935 			return true;
3936 	}
3937 
3938 	/*
3939 	 * If a node has no managed zone within highest_zoneidx, it does not
3940 	 * need balancing by definition. This can happen if a zone-restricted
3941 	 * allocation tries to wake a remote kswapd.
3942 	 */
3943 	if (mark == -1)
3944 		return true;
3945 
3946 	return false;
3947 }
3948 
3949 /* Clear pgdat state for congested, dirty or under writeback. */
clear_pgdat_congested(pg_data_t * pgdat)3950 static void clear_pgdat_congested(pg_data_t *pgdat)
3951 {
3952 	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
3953 
3954 	clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3955 	clear_bit(PGDAT_DIRTY, &pgdat->flags);
3956 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3957 }
3958 
3959 /*
3960  * Prepare kswapd for sleeping. This verifies that there are no processes
3961  * waiting in throttle_direct_reclaim() and that watermarks have been met.
3962  *
3963  * Returns true if kswapd is ready to sleep
3964  */
prepare_kswapd_sleep(pg_data_t * pgdat,int order,int highest_zoneidx)3965 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
3966 				int highest_zoneidx)
3967 {
3968 	/*
3969 	 * The throttled processes are normally woken up in balance_pgdat() as
3970 	 * soon as allow_direct_reclaim() is true. But there is a potential
3971 	 * race between when kswapd checks the watermarks and a process gets
3972 	 * throttled. There is also a potential race if processes get
3973 	 * throttled, kswapd wakes, a large process exits thereby balancing the
3974 	 * zones, which causes kswapd to exit balance_pgdat() before reaching
3975 	 * the wake up checks. If kswapd is going to sleep, no process should
3976 	 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3977 	 * the wake up is premature, processes will wake kswapd and get
3978 	 * throttled again. The difference from wake ups in balance_pgdat() is
3979 	 * that here we are under prepare_to_wait().
3980 	 */
3981 	if (waitqueue_active(&pgdat->pfmemalloc_wait))
3982 		wake_up_all(&pgdat->pfmemalloc_wait);
3983 
3984 	/* Hopeless node, leave it to direct reclaim */
3985 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3986 		return true;
3987 
3988 	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
3989 		clear_pgdat_congested(pgdat);
3990 		return true;
3991 	}
3992 
3993 	return false;
3994 }
3995 
3996 /*
3997  * kswapd shrinks a node of pages that are at or below the highest usable
3998  * zone that is currently unbalanced.
3999  *
4000  * Returns true if kswapd scanned at least the requested number of pages to
4001  * reclaim or if the lack of progress was due to pages under writeback.
4002  * This is used to determine if the scanning priority needs to be raised.
4003  */
kswapd_shrink_node(pg_data_t * pgdat,struct scan_control * sc)4004 static bool kswapd_shrink_node(pg_data_t *pgdat,
4005 			       struct scan_control *sc)
4006 {
4007 	struct zone *zone;
4008 	int z;
4009 
4010 	/* Reclaim a number of pages proportional to the number of zones */
4011 	sc->nr_to_reclaim = 0;
4012 	for (z = 0; z <= sc->reclaim_idx; z++) {
4013 		zone = pgdat->node_zones + z;
4014 		if (!managed_zone(zone))
4015 			continue;
4016 
4017 		sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
4018 	}
4019 
4020 	/*
4021 	 * Historically care was taken to put equal pressure on all zones but
4022 	 * now pressure is applied based on node LRU order.
4023 	 */
4024 	shrink_node(pgdat, sc);
4025 
4026 	/*
4027 	 * Fragmentation may mean that the system cannot be rebalanced for
4028 	 * high-order allocations. If twice the allocation size has been
4029 	 * reclaimed then recheck watermarks only at order-0 to prevent
4030 	 * excessive reclaim. Assume that a process requested a high-order
4031 	 * can direct reclaim/compact.
4032 	 */
4033 	if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
4034 		sc->order = 0;
4035 
4036 	return sc->nr_scanned >= sc->nr_to_reclaim;
4037 }
4038 
4039 /* Page allocator PCP high watermark is lowered if reclaim is active. */
4040 static inline void
update_reclaim_active(pg_data_t * pgdat,int highest_zoneidx,bool active)4041 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
4042 {
4043 	int i;
4044 	struct zone *zone;
4045 
4046 	for (i = 0; i <= highest_zoneidx; i++) {
4047 		zone = pgdat->node_zones + i;
4048 
4049 		if (!managed_zone(zone))
4050 			continue;
4051 
4052 		if (active)
4053 			set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
4054 		else
4055 			clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
4056 	}
4057 }
4058 
4059 static inline void
set_reclaim_active(pg_data_t * pgdat,int highest_zoneidx)4060 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
4061 {
4062 	update_reclaim_active(pgdat, highest_zoneidx, true);
4063 }
4064 
4065 static inline void
clear_reclaim_active(pg_data_t * pgdat,int highest_zoneidx)4066 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
4067 {
4068 	update_reclaim_active(pgdat, highest_zoneidx, false);
4069 }
4070 
4071 /*
4072  * For kswapd, balance_pgdat() will reclaim pages across a node from zones
4073  * that are eligible for use by the caller until at least one zone is
4074  * balanced.
4075  *
4076  * Returns the order kswapd finished reclaiming at.
4077  *
4078  * kswapd scans the zones in the highmem->normal->dma direction.  It skips
4079  * zones which have free_pages > high_wmark_pages(zone), but once a zone is
4080  * found to have free_pages <= high_wmark_pages(zone), any page in that zone
4081  * or lower is eligible for reclaim until at least one usable zone is
4082  * balanced.
4083  */
balance_pgdat(pg_data_t * pgdat,int order,int highest_zoneidx)4084 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
4085 {
4086 	int i;
4087 	unsigned long nr_soft_reclaimed;
4088 	unsigned long nr_soft_scanned;
4089 	unsigned long pflags;
4090 	unsigned long nr_boost_reclaim;
4091 	unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
4092 	bool boosted;
4093 	struct zone *zone;
4094 	struct scan_control sc = {
4095 		.gfp_mask = GFP_KERNEL,
4096 		.order = order,
4097 		.may_unmap = 1,
4098 	};
4099 
4100 	set_task_reclaim_state(current, &sc.reclaim_state);
4101 	psi_memstall_enter(&pflags);
4102 	__fs_reclaim_acquire(_THIS_IP_);
4103 
4104 	count_vm_event(PAGEOUTRUN);
4105 
4106 	/*
4107 	 * Account for the reclaim boost. Note that the zone boost is left in
4108 	 * place so that parallel allocations that are near the watermark will
4109 	 * stall or direct reclaim until kswapd is finished.
4110 	 */
4111 	nr_boost_reclaim = 0;
4112 	for (i = 0; i <= highest_zoneidx; i++) {
4113 		zone = pgdat->node_zones + i;
4114 		if (!managed_zone(zone))
4115 			continue;
4116 
4117 		nr_boost_reclaim += zone->watermark_boost;
4118 		zone_boosts[i] = zone->watermark_boost;
4119 	}
4120 	boosted = nr_boost_reclaim;
4121 
4122 restart:
4123 	set_reclaim_active(pgdat, highest_zoneidx);
4124 	sc.priority = DEF_PRIORITY;
4125 	do {
4126 		unsigned long nr_reclaimed = sc.nr_reclaimed;
4127 		bool raise_priority = true;
4128 		bool balanced;
4129 		bool ret;
4130 
4131 		sc.reclaim_idx = highest_zoneidx;
4132 
4133 		/*
4134 		 * If the number of buffer_heads exceeds the maximum allowed
4135 		 * then consider reclaiming from all zones. This has a dual
4136 		 * purpose -- on 64-bit systems it is expected that
4137 		 * buffer_heads are stripped during active rotation. On 32-bit
4138 		 * systems, highmem pages can pin lowmem memory and shrinking
4139 		 * buffers can relieve lowmem pressure. Reclaim may still not
4140 		 * go ahead if all eligible zones for the original allocation
4141 		 * request are balanced to avoid excessive reclaim from kswapd.
4142 		 */
4143 		if (buffer_heads_over_limit) {
4144 			for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
4145 				zone = pgdat->node_zones + i;
4146 				if (!managed_zone(zone))
4147 					continue;
4148 
4149 				sc.reclaim_idx = i;
4150 				break;
4151 			}
4152 		}
4153 
4154 		/*
4155 		 * If the pgdat is imbalanced then ignore boosting and preserve
4156 		 * the watermarks for a later time and restart. Note that the
4157 		 * zone watermarks will be still reset at the end of balancing
4158 		 * on the grounds that the normal reclaim should be enough to
4159 		 * re-evaluate if boosting is required when kswapd next wakes.
4160 		 */
4161 		balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
4162 		if (!balanced && nr_boost_reclaim) {
4163 			nr_boost_reclaim = 0;
4164 			goto restart;
4165 		}
4166 
4167 		/*
4168 		 * If boosting is not active then only reclaim if there are no
4169 		 * eligible zones. Note that sc.reclaim_idx is not used as
4170 		 * buffer_heads_over_limit may have adjusted it.
4171 		 */
4172 		if (!nr_boost_reclaim && balanced)
4173 			goto out;
4174 
4175 		/* Limit the priority of boosting to avoid reclaim writeback */
4176 		if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
4177 			raise_priority = false;
4178 
4179 		/*
4180 		 * Do not writeback or swap pages for boosted reclaim. The
4181 		 * intent is to relieve pressure not issue sub-optimal IO
4182 		 * from reclaim context. If no pages are reclaimed, the
4183 		 * reclaim will be aborted.
4184 		 */
4185 		sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
4186 		sc.may_swap = !nr_boost_reclaim;
4187 
4188 		/*
4189 		 * Do some background aging of the anon list, to give
4190 		 * pages a chance to be referenced before reclaiming. All
4191 		 * pages are rotated regardless of classzone as this is
4192 		 * about consistent aging.
4193 		 */
4194 		age_active_anon(pgdat, &sc);
4195 
4196 		/*
4197 		 * If we're getting trouble reclaiming, start doing writepage
4198 		 * even in laptop mode.
4199 		 */
4200 		if (sc.priority < DEF_PRIORITY - 2)
4201 			sc.may_writepage = 1;
4202 
4203 		/* Call soft limit reclaim before calling shrink_node. */
4204 		sc.nr_scanned = 0;
4205 		nr_soft_scanned = 0;
4206 		nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
4207 						sc.gfp_mask, &nr_soft_scanned);
4208 		sc.nr_reclaimed += nr_soft_reclaimed;
4209 
4210 		/*
4211 		 * There should be no need to raise the scanning priority if
4212 		 * enough pages are already being scanned that that high
4213 		 * watermark would be met at 100% efficiency.
4214 		 */
4215 		if (kswapd_shrink_node(pgdat, &sc))
4216 			raise_priority = false;
4217 
4218 		/*
4219 		 * If the low watermark is met there is no need for processes
4220 		 * to be throttled on pfmemalloc_wait as they should not be
4221 		 * able to safely make forward progress. Wake them
4222 		 */
4223 		if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
4224 				allow_direct_reclaim(pgdat))
4225 			wake_up_all(&pgdat->pfmemalloc_wait);
4226 
4227 		/* Check if kswapd should be suspending */
4228 		__fs_reclaim_release(_THIS_IP_);
4229 		ret = try_to_freeze();
4230 		__fs_reclaim_acquire(_THIS_IP_);
4231 		if (ret || kthread_should_stop())
4232 			break;
4233 
4234 		/*
4235 		 * Raise priority if scanning rate is too low or there was no
4236 		 * progress in reclaiming pages
4237 		 */
4238 		nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
4239 		nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
4240 
4241 		/*
4242 		 * If reclaim made no progress for a boost, stop reclaim as
4243 		 * IO cannot be queued and it could be an infinite loop in
4244 		 * extreme circumstances.
4245 		 */
4246 		if (nr_boost_reclaim && !nr_reclaimed)
4247 			break;
4248 
4249 		if (raise_priority || !nr_reclaimed)
4250 			sc.priority--;
4251 	} while (sc.priority >= 1);
4252 
4253 	if (!sc.nr_reclaimed)
4254 		pgdat->kswapd_failures++;
4255 
4256 out:
4257 	clear_reclaim_active(pgdat, highest_zoneidx);
4258 
4259 	/* If reclaim was boosted, account for the reclaim done in this pass */
4260 	if (boosted) {
4261 		unsigned long flags;
4262 
4263 		for (i = 0; i <= highest_zoneidx; i++) {
4264 			if (!zone_boosts[i])
4265 				continue;
4266 
4267 			/* Increments are under the zone lock */
4268 			zone = pgdat->node_zones + i;
4269 			spin_lock_irqsave(&zone->lock, flags);
4270 			zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
4271 			spin_unlock_irqrestore(&zone->lock, flags);
4272 		}
4273 
4274 		/*
4275 		 * As there is now likely space, wakeup kcompact to defragment
4276 		 * pageblocks.
4277 		 */
4278 		wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
4279 	}
4280 
4281 	snapshot_refaults(NULL, pgdat);
4282 	__fs_reclaim_release(_THIS_IP_);
4283 	psi_memstall_leave(&pflags);
4284 	set_task_reclaim_state(current, NULL);
4285 
4286 	/*
4287 	 * Return the order kswapd stopped reclaiming at as
4288 	 * prepare_kswapd_sleep() takes it into account. If another caller
4289 	 * entered the allocator slow path while kswapd was awake, order will
4290 	 * remain at the higher level.
4291 	 */
4292 	return sc.order;
4293 }
4294 
4295 /*
4296  * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
4297  * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
4298  * not a valid index then either kswapd runs for first time or kswapd couldn't
4299  * sleep after previous reclaim attempt (node is still unbalanced). In that
4300  * case return the zone index of the previous kswapd reclaim cycle.
4301  */
kswapd_highest_zoneidx(pg_data_t * pgdat,enum zone_type prev_highest_zoneidx)4302 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
4303 					   enum zone_type prev_highest_zoneidx)
4304 {
4305 	enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4306 
4307 	return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
4308 }
4309 
kswapd_try_to_sleep(pg_data_t * pgdat,int alloc_order,int reclaim_order,unsigned int highest_zoneidx)4310 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
4311 				unsigned int highest_zoneidx)
4312 {
4313 	long remaining = 0;
4314 	DEFINE_WAIT(wait);
4315 
4316 	if (freezing(current) || kthread_should_stop())
4317 		return;
4318 
4319 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4320 
4321 	/*
4322 	 * Try to sleep for a short interval. Note that kcompactd will only be
4323 	 * woken if it is possible to sleep for a short interval. This is
4324 	 * deliberate on the assumption that if reclaim cannot keep an
4325 	 * eligible zone balanced that it's also unlikely that compaction will
4326 	 * succeed.
4327 	 */
4328 	if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4329 		/*
4330 		 * Compaction records what page blocks it recently failed to
4331 		 * isolate pages from and skips them in the future scanning.
4332 		 * When kswapd is going to sleep, it is reasonable to assume
4333 		 * that pages and compaction may succeed so reset the cache.
4334 		 */
4335 		reset_isolation_suitable(pgdat);
4336 
4337 		/*
4338 		 * We have freed the memory, now we should compact it to make
4339 		 * allocation of the requested order possible.
4340 		 */
4341 		wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
4342 
4343 		remaining = schedule_timeout(HZ/10);
4344 
4345 		/*
4346 		 * If woken prematurely then reset kswapd_highest_zoneidx and
4347 		 * order. The values will either be from a wakeup request or
4348 		 * the previous request that slept prematurely.
4349 		 */
4350 		if (remaining) {
4351 			WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
4352 					kswapd_highest_zoneidx(pgdat,
4353 							highest_zoneidx));
4354 
4355 			if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
4356 				WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
4357 		}
4358 
4359 		finish_wait(&pgdat->kswapd_wait, &wait);
4360 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4361 	}
4362 
4363 	/*
4364 	 * After a short sleep, check if it was a premature sleep. If not, then
4365 	 * go fully to sleep until explicitly woken up.
4366 	 */
4367 	if (!remaining &&
4368 	    prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4369 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
4370 
4371 		/*
4372 		 * vmstat counters are not perfectly accurate and the estimated
4373 		 * value for counters such as NR_FREE_PAGES can deviate from the
4374 		 * true value by nr_online_cpus * threshold. To avoid the zone
4375 		 * watermarks being breached while under pressure, we reduce the
4376 		 * per-cpu vmstat threshold while kswapd is awake and restore
4377 		 * them before going back to sleep.
4378 		 */
4379 		set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
4380 
4381 		if (!kthread_should_stop())
4382 			schedule();
4383 
4384 		set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
4385 	} else {
4386 		if (remaining)
4387 			count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
4388 		else
4389 			count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
4390 	}
4391 	finish_wait(&pgdat->kswapd_wait, &wait);
4392 }
4393 
4394 /*
4395  * The background pageout daemon, started as a kernel thread
4396  * from the init process.
4397  *
4398  * This basically trickles out pages so that we have _some_
4399  * free memory available even if there is no other activity
4400  * that frees anything up. This is needed for things like routing
4401  * etc, where we otherwise might have all activity going on in
4402  * asynchronous contexts that cannot page things out.
4403  *
4404  * If there are applications that are active memory-allocators
4405  * (most normal use), this basically shouldn't matter.
4406  */
kswapd(void * p)4407 static int kswapd(void *p)
4408 {
4409 	unsigned int alloc_order, reclaim_order;
4410 	unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
4411 	pg_data_t *pgdat = (pg_data_t *)p;
4412 	struct task_struct *tsk = current;
4413 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
4414 
4415 	if (!cpumask_empty(cpumask))
4416 		set_cpus_allowed_ptr(tsk, cpumask);
4417 
4418 	/*
4419 	 * Tell the memory management that we're a "memory allocator",
4420 	 * and that if we need more memory we should get access to it
4421 	 * regardless (see "__alloc_pages()"). "kswapd" should
4422 	 * never get caught in the normal page freeing logic.
4423 	 *
4424 	 * (Kswapd normally doesn't need memory anyway, but sometimes
4425 	 * you need a small amount of memory in order to be able to
4426 	 * page out something else, and this flag essentially protects
4427 	 * us from recursively trying to free more memory as we're
4428 	 * trying to free the first piece of memory in the first place).
4429 	 */
4430 	tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
4431 	set_freezable();
4432 
4433 	WRITE_ONCE(pgdat->kswapd_order, 0);
4434 	WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4435 	atomic_set(&pgdat->nr_writeback_throttled, 0);
4436 	for ( ; ; ) {
4437 		bool ret;
4438 
4439 		alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
4440 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4441 							highest_zoneidx);
4442 
4443 kswapd_try_sleep:
4444 		kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
4445 					highest_zoneidx);
4446 
4447 		/* Read the new order and highest_zoneidx */
4448 		alloc_order = READ_ONCE(pgdat->kswapd_order);
4449 		highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4450 							highest_zoneidx);
4451 		WRITE_ONCE(pgdat->kswapd_order, 0);
4452 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4453 
4454 		ret = try_to_freeze();
4455 		if (kthread_should_stop())
4456 			break;
4457 
4458 		/*
4459 		 * We can speed up thawing tasks if we don't call balance_pgdat
4460 		 * after returning from the refrigerator
4461 		 */
4462 		if (ret)
4463 			continue;
4464 
4465 		/*
4466 		 * Reclaim begins at the requested order but if a high-order
4467 		 * reclaim fails then kswapd falls back to reclaiming for
4468 		 * order-0. If that happens, kswapd will consider sleeping
4469 		 * for the order it finished reclaiming at (reclaim_order)
4470 		 * but kcompactd is woken to compact for the original
4471 		 * request (alloc_order).
4472 		 */
4473 		trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
4474 						alloc_order);
4475 		reclaim_order = balance_pgdat(pgdat, alloc_order,
4476 						highest_zoneidx);
4477 		if (reclaim_order < alloc_order)
4478 			goto kswapd_try_sleep;
4479 	}
4480 
4481 	tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
4482 
4483 	return 0;
4484 }
4485 
4486 /*
4487  * A zone is low on free memory or too fragmented for high-order memory.  If
4488  * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
4489  * pgdat.  It will wake up kcompactd after reclaiming memory.  If kswapd reclaim
4490  * has failed or is not needed, still wake up kcompactd if only compaction is
4491  * needed.
4492  */
wakeup_kswapd(struct zone * zone,gfp_t gfp_flags,int order,enum zone_type highest_zoneidx)4493 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
4494 		   enum zone_type highest_zoneidx)
4495 {
4496 	pg_data_t *pgdat;
4497 	enum zone_type curr_idx;
4498 
4499 	if (!managed_zone(zone))
4500 		return;
4501 
4502 	if (!cpuset_zone_allowed(zone, gfp_flags))
4503 		return;
4504 
4505 	pgdat = zone->zone_pgdat;
4506 	curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4507 
4508 	if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
4509 		WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
4510 
4511 	if (READ_ONCE(pgdat->kswapd_order) < order)
4512 		WRITE_ONCE(pgdat->kswapd_order, order);
4513 
4514 	if (!waitqueue_active(&pgdat->kswapd_wait))
4515 		return;
4516 
4517 	/* Hopeless node, leave it to direct reclaim if possible */
4518 	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
4519 	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&
4520 	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
4521 		/*
4522 		 * There may be plenty of free memory available, but it's too
4523 		 * fragmented for high-order allocations.  Wake up kcompactd
4524 		 * and rely on compaction_suitable() to determine if it's
4525 		 * needed.  If it fails, it will defer subsequent attempts to
4526 		 * ratelimit its work.
4527 		 */
4528 		if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
4529 			wakeup_kcompactd(pgdat, order, highest_zoneidx);
4530 		return;
4531 	}
4532 
4533 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
4534 				      gfp_flags);
4535 	wake_up_interruptible(&pgdat->kswapd_wait);
4536 }
4537 
4538 #ifdef CONFIG_HIBERNATION
4539 /*
4540  * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
4541  * freed pages.
4542  *
4543  * Rather than trying to age LRUs the aim is to preserve the overall
4544  * LRU order by reclaiming preferentially
4545  * inactive > active > active referenced > active mapped
4546  */
shrink_all_memory(unsigned long nr_to_reclaim)4547 unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
4548 {
4549 	struct scan_control sc = {
4550 		.nr_to_reclaim = nr_to_reclaim,
4551 		.gfp_mask = GFP_HIGHUSER_MOVABLE,
4552 		.reclaim_idx = MAX_NR_ZONES - 1,
4553 		.priority = DEF_PRIORITY,
4554 		.may_writepage = 1,
4555 		.may_unmap = 1,
4556 		.may_swap = 1,
4557 		.hibernation_mode = 1,
4558 	};
4559 	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
4560 	unsigned long nr_reclaimed;
4561 	unsigned int noreclaim_flag;
4562 
4563 	fs_reclaim_acquire(sc.gfp_mask);
4564 	noreclaim_flag = memalloc_noreclaim_save();
4565 	set_task_reclaim_state(current, &sc.reclaim_state);
4566 
4567 	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
4568 
4569 	set_task_reclaim_state(current, NULL);
4570 	memalloc_noreclaim_restore(noreclaim_flag);
4571 	fs_reclaim_release(sc.gfp_mask);
4572 
4573 	return nr_reclaimed;
4574 }
4575 #endif /* CONFIG_HIBERNATION */
4576 
4577 /*
4578  * This kswapd start function will be called by init and node-hot-add.
4579  */
kswapd_run(int nid)4580 void kswapd_run(int nid)
4581 {
4582 	pg_data_t *pgdat = NODE_DATA(nid);
4583 
4584 	if (pgdat->kswapd)
4585 		return;
4586 
4587 	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
4588 	if (IS_ERR(pgdat->kswapd)) {
4589 		/* failure at boot is fatal */
4590 		BUG_ON(system_state < SYSTEM_RUNNING);
4591 		pr_err("Failed to start kswapd on node %d\n", nid);
4592 		pgdat->kswapd = NULL;
4593 	}
4594 }
4595 
4596 /*
4597  * Called by memory hotplug when all memory in a node is offlined.  Caller must
4598  * hold mem_hotplug_begin/end().
4599  */
kswapd_stop(int nid)4600 void kswapd_stop(int nid)
4601 {
4602 	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
4603 
4604 	if (kswapd) {
4605 		kthread_stop(kswapd);
4606 		NODE_DATA(nid)->kswapd = NULL;
4607 	}
4608 }
4609 
kswapd_init(void)4610 static int __init kswapd_init(void)
4611 {
4612 	int nid;
4613 
4614 	swap_setup();
4615 	for_each_node_state(nid, N_MEMORY)
4616  		kswapd_run(nid);
4617 	return 0;
4618 }
4619 
4620 module_init(kswapd_init)
4621 
4622 #ifdef CONFIG_NUMA
4623 /*
4624  * Node reclaim mode
4625  *
4626  * If non-zero call node_reclaim when the number of free pages falls below
4627  * the watermarks.
4628  */
4629 int node_reclaim_mode __read_mostly;
4630 
4631 /*
4632  * Priority for NODE_RECLAIM. This determines the fraction of pages
4633  * of a node considered for each zone_reclaim. 4 scans 1/16th of
4634  * a zone.
4635  */
4636 #define NODE_RECLAIM_PRIORITY 4
4637 
4638 /*
4639  * Percentage of pages in a zone that must be unmapped for node_reclaim to
4640  * occur.
4641  */
4642 int sysctl_min_unmapped_ratio = 1;
4643 
4644 /*
4645  * If the number of slab pages in a zone grows beyond this percentage then
4646  * slab reclaim needs to occur.
4647  */
4648 int sysctl_min_slab_ratio = 5;
4649 
node_unmapped_file_pages(struct pglist_data * pgdat)4650 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
4651 {
4652 	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
4653 	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
4654 		node_page_state(pgdat, NR_ACTIVE_FILE);
4655 
4656 	/*
4657 	 * It's possible for there to be more file mapped pages than
4658 	 * accounted for by the pages on the file LRU lists because
4659 	 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4660 	 */
4661 	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
4662 }
4663 
4664 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
node_pagecache_reclaimable(struct pglist_data * pgdat)4665 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
4666 {
4667 	unsigned long nr_pagecache_reclaimable;
4668 	unsigned long delta = 0;
4669 
4670 	/*
4671 	 * If RECLAIM_UNMAP is set, then all file pages are considered
4672 	 * potentially reclaimable. Otherwise, we have to worry about
4673 	 * pages like swapcache and node_unmapped_file_pages() provides
4674 	 * a better estimate
4675 	 */
4676 	if (node_reclaim_mode & RECLAIM_UNMAP)
4677 		nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
4678 	else
4679 		nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
4680 
4681 	/* If we can't clean pages, remove dirty pages from consideration */
4682 	if (!(node_reclaim_mode & RECLAIM_WRITE))
4683 		delta += node_page_state(pgdat, NR_FILE_DIRTY);
4684 
4685 	/* Watch for any possible underflows due to delta */
4686 	if (unlikely(delta > nr_pagecache_reclaimable))
4687 		delta = nr_pagecache_reclaimable;
4688 
4689 	return nr_pagecache_reclaimable - delta;
4690 }
4691 
4692 /*
4693  * Try to free up some pages from this node through reclaim.
4694  */
__node_reclaim(struct pglist_data * pgdat,gfp_t gfp_mask,unsigned int order)4695 static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4696 {
4697 	/* Minimum pages needed in order to stay on node */
4698 	const unsigned long nr_pages = 1 << order;
4699 	struct task_struct *p = current;
4700 	unsigned int noreclaim_flag;
4701 	struct scan_control sc = {
4702 		.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
4703 		.gfp_mask = current_gfp_context(gfp_mask),
4704 		.order = order,
4705 		.priority = NODE_RECLAIM_PRIORITY,
4706 		.may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
4707 		.may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
4708 		.may_swap = 1,
4709 		.reclaim_idx = gfp_zone(gfp_mask),
4710 	};
4711 	unsigned long pflags;
4712 
4713 	trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
4714 					   sc.gfp_mask);
4715 
4716 	cond_resched();
4717 	psi_memstall_enter(&pflags);
4718 	fs_reclaim_acquire(sc.gfp_mask);
4719 	/*
4720 	 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4721 	 */
4722 	noreclaim_flag = memalloc_noreclaim_save();
4723 	set_task_reclaim_state(p, &sc.reclaim_state);
4724 
4725 	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||
4726 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {
4727 		/*
4728 		 * Free memory by calling shrink node with increasing
4729 		 * priorities until we have enough memory freed.
4730 		 */
4731 		do {
4732 			shrink_node(pgdat, &sc);
4733 		} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4734 	}
4735 
4736 	set_task_reclaim_state(p, NULL);
4737 	memalloc_noreclaim_restore(noreclaim_flag);
4738 	fs_reclaim_release(sc.gfp_mask);
4739 	psi_memstall_leave(&pflags);
4740 
4741 	trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
4742 
4743 	return sc.nr_reclaimed >= nr_pages;
4744 }
4745 
node_reclaim(struct pglist_data * pgdat,gfp_t gfp_mask,unsigned int order)4746 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4747 {
4748 	int ret;
4749 
4750 	/*
4751 	 * Node reclaim reclaims unmapped file backed pages and
4752 	 * slab pages if we are over the defined limits.
4753 	 *
4754 	 * A small portion of unmapped file backed pages is needed for
4755 	 * file I/O otherwise pages read by file I/O will be immediately
4756 	 * thrown out if the node is overallocated. So we do not reclaim
4757 	 * if less than a specified percentage of the node is used by
4758 	 * unmapped file backed pages.
4759 	 */
4760 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4761 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
4762 	    pgdat->min_slab_pages)
4763 		return NODE_RECLAIM_FULL;
4764 
4765 	/*
4766 	 * Do not scan if the allocation should not be delayed.
4767 	 */
4768 	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
4769 		return NODE_RECLAIM_NOSCAN;
4770 
4771 	/*
4772 	 * Only run node reclaim on the local node or on nodes that do not
4773 	 * have associated processors. This will favor the local processor
4774 	 * over remote processors and spread off node memory allocations
4775 	 * as wide as possible.
4776 	 */
4777 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
4778 		return NODE_RECLAIM_NOSCAN;
4779 
4780 	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
4781 		return NODE_RECLAIM_NOSCAN;
4782 
4783 	ret = __node_reclaim(pgdat, gfp_mask, order);
4784 	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
4785 
4786 	if (!ret)
4787 		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
4788 
4789 	return ret;
4790 }
4791 #endif
4792 
4793 /**
4794  * check_move_unevictable_pages - check pages for evictability and move to
4795  * appropriate zone lru list
4796  * @pvec: pagevec with lru pages to check
4797  *
4798  * Checks pages for evictability, if an evictable page is in the unevictable
4799  * lru list, moves it to the appropriate evictable lru list. This function
4800  * should be only used for lru pages.
4801  */
check_move_unevictable_pages(struct pagevec * pvec)4802 void check_move_unevictable_pages(struct pagevec *pvec)
4803 {
4804 	struct lruvec *lruvec = NULL;
4805 	int pgscanned = 0;
4806 	int pgrescued = 0;
4807 	int i;
4808 
4809 	for (i = 0; i < pvec->nr; i++) {
4810 		struct page *page = pvec->pages[i];
4811 		struct folio *folio = page_folio(page);
4812 		int nr_pages;
4813 
4814 		if (PageTransTail(page))
4815 			continue;
4816 
4817 		nr_pages = thp_nr_pages(page);
4818 		pgscanned += nr_pages;
4819 
4820 		/* block memcg migration during page moving between lru */
4821 		if (!TestClearPageLRU(page))
4822 			continue;
4823 
4824 		lruvec = folio_lruvec_relock_irq(folio, lruvec);
4825 		if (page_evictable(page) && PageUnevictable(page)) {
4826 			del_page_from_lru_list(page, lruvec);
4827 			ClearPageUnevictable(page);
4828 			add_page_to_lru_list(page, lruvec);
4829 			pgrescued += nr_pages;
4830 		}
4831 		SetPageLRU(page);
4832 	}
4833 
4834 	if (lruvec) {
4835 		__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4836 		__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4837 		unlock_page_lruvec_irq(lruvec);
4838 	} else if (pgscanned) {
4839 		count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4840 	}
4841 }
4842 EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
4843