1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/mm/vmstat.c
4 *
5 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 *
8 * zoned VM statistics
9 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter
12 */
13 #include <linux/fs.h>
14 #include <linux/mm.h>
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/debugfs.h>
24 #include <linux/sched.h>
25 #include <linux/math64.h>
26 #include <linux/writeback.h>
27 #include <linux/compaction.h>
28 #include <linux/mm_inline.h>
29 #include <linux/page_ext.h>
30 #include <linux/page_owner.h>
31 #include <linux/migrate.h>
32
33 #include "internal.h"
34
35 #ifdef CONFIG_NUMA
36 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
37
38 /* zero numa counters within a zone */
zero_zone_numa_counters(struct zone * zone)39 static void zero_zone_numa_counters(struct zone *zone)
40 {
41 int item, cpu;
42
43 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) {
44 atomic_long_set(&zone->vm_numa_event[item], 0);
45 for_each_online_cpu(cpu) {
46 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]
47 = 0;
48 }
49 }
50 }
51
52 /* zero numa counters of all the populated zones */
zero_zones_numa_counters(void)53 static void zero_zones_numa_counters(void)
54 {
55 struct zone *zone;
56
57 for_each_populated_zone(zone)
58 zero_zone_numa_counters(zone);
59 }
60
61 /* zero global numa counters */
zero_global_numa_counters(void)62 static void zero_global_numa_counters(void)
63 {
64 int item;
65
66 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
67 atomic_long_set(&vm_numa_event[item], 0);
68 }
69
invalid_numa_statistics(void)70 static void invalid_numa_statistics(void)
71 {
72 zero_zones_numa_counters();
73 zero_global_numa_counters();
74 }
75
76 static DEFINE_MUTEX(vm_numa_stat_lock);
77
sysctl_vm_numa_stat_handler(struct ctl_table * table,int write,void * buffer,size_t * length,loff_t * ppos)78 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
79 void *buffer, size_t *length, loff_t *ppos)
80 {
81 int ret, oldval;
82
83 mutex_lock(&vm_numa_stat_lock);
84 if (write)
85 oldval = sysctl_vm_numa_stat;
86 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
87 if (ret || !write)
88 goto out;
89
90 if (oldval == sysctl_vm_numa_stat)
91 goto out;
92 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
93 static_branch_enable(&vm_numa_stat_key);
94 pr_info("enable numa statistics\n");
95 } else {
96 static_branch_disable(&vm_numa_stat_key);
97 invalid_numa_statistics();
98 pr_info("disable numa statistics, and clear numa counters\n");
99 }
100
101 out:
102 mutex_unlock(&vm_numa_stat_lock);
103 return ret;
104 }
105 #endif
106
107 #ifdef CONFIG_VM_EVENT_COUNTERS
108 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
109 EXPORT_PER_CPU_SYMBOL(vm_event_states);
110
sum_vm_events(unsigned long * ret)111 static void sum_vm_events(unsigned long *ret)
112 {
113 int cpu;
114 int i;
115
116 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
117
118 for_each_online_cpu(cpu) {
119 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
120
121 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
122 ret[i] += this->event[i];
123 }
124 }
125
126 /*
127 * Accumulate the vm event counters across all CPUs.
128 * The result is unavoidably approximate - it can change
129 * during and after execution of this function.
130 */
all_vm_events(unsigned long * ret)131 void all_vm_events(unsigned long *ret)
132 {
133 cpus_read_lock();
134 sum_vm_events(ret);
135 cpus_read_unlock();
136 }
137 EXPORT_SYMBOL_GPL(all_vm_events);
138
139 /*
140 * Fold the foreign cpu events into our own.
141 *
142 * This is adding to the events on one processor
143 * but keeps the global counts constant.
144 */
vm_events_fold_cpu(int cpu)145 void vm_events_fold_cpu(int cpu)
146 {
147 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
148 int i;
149
150 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
151 count_vm_events(i, fold_state->event[i]);
152 fold_state->event[i] = 0;
153 }
154 }
155
156 #endif /* CONFIG_VM_EVENT_COUNTERS */
157
158 /*
159 * Manage combined zone based / global counters
160 *
161 * vm_stat contains the global counters
162 */
163 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
164 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
165 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp;
166 EXPORT_SYMBOL(vm_zone_stat);
167 EXPORT_SYMBOL(vm_node_stat);
168
169 #ifdef CONFIG_NUMA
fold_vm_zone_numa_events(struct zone * zone)170 static void fold_vm_zone_numa_events(struct zone *zone)
171 {
172 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, };
173 int cpu;
174 enum numa_stat_item item;
175
176 for_each_online_cpu(cpu) {
177 struct per_cpu_zonestat *pzstats;
178
179 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
180 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
181 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0);
182 }
183
184 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
185 zone_numa_event_add(zone_numa_events[item], zone, item);
186 }
187
fold_vm_numa_events(void)188 void fold_vm_numa_events(void)
189 {
190 struct zone *zone;
191
192 for_each_populated_zone(zone)
193 fold_vm_zone_numa_events(zone);
194 }
195 #endif
196
197 #ifdef CONFIG_SMP
198
calculate_pressure_threshold(struct zone * zone)199 int calculate_pressure_threshold(struct zone *zone)
200 {
201 int threshold;
202 int watermark_distance;
203
204 /*
205 * As vmstats are not up to date, there is drift between the estimated
206 * and real values. For high thresholds and a high number of CPUs, it
207 * is possible for the min watermark to be breached while the estimated
208 * value looks fine. The pressure threshold is a reduced value such
209 * that even the maximum amount of drift will not accidentally breach
210 * the min watermark
211 */
212 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
213 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
214
215 /*
216 * Maximum threshold is 125
217 */
218 threshold = min(125, threshold);
219
220 return threshold;
221 }
222
calculate_normal_threshold(struct zone * zone)223 int calculate_normal_threshold(struct zone *zone)
224 {
225 int threshold;
226 int mem; /* memory in 128 MB units */
227
228 /*
229 * The threshold scales with the number of processors and the amount
230 * of memory per zone. More memory means that we can defer updates for
231 * longer, more processors could lead to more contention.
232 * fls() is used to have a cheap way of logarithmic scaling.
233 *
234 * Some sample thresholds:
235 *
236 * Threshold Processors (fls) Zonesize fls(mem)+1
237 * ------------------------------------------------------------------
238 * 8 1 1 0.9-1 GB 4
239 * 16 2 2 0.9-1 GB 4
240 * 20 2 2 1-2 GB 5
241 * 24 2 2 2-4 GB 6
242 * 28 2 2 4-8 GB 7
243 * 32 2 2 8-16 GB 8
244 * 4 2 2 <128M 1
245 * 30 4 3 2-4 GB 5
246 * 48 4 3 8-16 GB 8
247 * 32 8 4 1-2 GB 4
248 * 32 8 4 0.9-1GB 4
249 * 10 16 5 <128M 1
250 * 40 16 5 900M 4
251 * 70 64 7 2-4 GB 5
252 * 84 64 7 4-8 GB 6
253 * 108 512 9 4-8 GB 6
254 * 125 1024 10 8-16 GB 8
255 * 125 1024 10 16-32 GB 9
256 */
257
258 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);
259
260 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
261
262 /*
263 * Maximum threshold is 125
264 */
265 threshold = min(125, threshold);
266
267 return threshold;
268 }
269
270 /*
271 * Refresh the thresholds for each zone.
272 */
refresh_zone_stat_thresholds(void)273 void refresh_zone_stat_thresholds(void)
274 {
275 struct pglist_data *pgdat;
276 struct zone *zone;
277 int cpu;
278 int threshold;
279
280 /* Zero current pgdat thresholds */
281 for_each_online_pgdat(pgdat) {
282 for_each_online_cpu(cpu) {
283 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
284 }
285 }
286
287 for_each_populated_zone(zone) {
288 struct pglist_data *pgdat = zone->zone_pgdat;
289 unsigned long max_drift, tolerate_drift;
290
291 threshold = calculate_normal_threshold(zone);
292
293 for_each_online_cpu(cpu) {
294 int pgdat_threshold;
295
296 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
297 = threshold;
298
299 /* Base nodestat threshold on the largest populated zone. */
300 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
301 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
302 = max(threshold, pgdat_threshold);
303 }
304
305 /*
306 * Only set percpu_drift_mark if there is a danger that
307 * NR_FREE_PAGES reports the low watermark is ok when in fact
308 * the min watermark could be breached by an allocation
309 */
310 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
311 max_drift = num_online_cpus() * threshold;
312 if (max_drift > tolerate_drift)
313 zone->percpu_drift_mark = high_wmark_pages(zone) +
314 max_drift;
315 }
316 }
317
set_pgdat_percpu_threshold(pg_data_t * pgdat,int (* calculate_pressure)(struct zone *))318 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
319 int (*calculate_pressure)(struct zone *))
320 {
321 struct zone *zone;
322 int cpu;
323 int threshold;
324 int i;
325
326 for (i = 0; i < pgdat->nr_zones; i++) {
327 zone = &pgdat->node_zones[i];
328 if (!zone->percpu_drift_mark)
329 continue;
330
331 threshold = (*calculate_pressure)(zone);
332 for_each_online_cpu(cpu)
333 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
334 = threshold;
335 }
336 }
337
338 /*
339 * For use when we know that interrupts are disabled,
340 * or when we know that preemption is disabled and that
341 * particular counter cannot be updated from interrupt context.
342 */
__mod_zone_page_state(struct zone * zone,enum zone_stat_item item,long delta)343 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
344 long delta)
345 {
346 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
347 s8 __percpu *p = pcp->vm_stat_diff + item;
348 long x;
349 long t;
350
351 /*
352 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels,
353 * atomicity is provided by IRQs being disabled -- either explicitly
354 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables
355 * CPU migrations and preemption potentially corrupts a counter so
356 * disable preemption.
357 */
358 if (IS_ENABLED(CONFIG_PREEMPT_RT))
359 preempt_disable();
360
361 x = delta + __this_cpu_read(*p);
362
363 t = __this_cpu_read(pcp->stat_threshold);
364
365 if (unlikely(abs(x) > t)) {
366 zone_page_state_add(x, zone, item);
367 x = 0;
368 }
369 __this_cpu_write(*p, x);
370
371 if (IS_ENABLED(CONFIG_PREEMPT_RT))
372 preempt_enable();
373 }
374 EXPORT_SYMBOL(__mod_zone_page_state);
375
__mod_node_page_state(struct pglist_data * pgdat,enum node_stat_item item,long delta)376 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
377 long delta)
378 {
379 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
380 s8 __percpu *p = pcp->vm_node_stat_diff + item;
381 long x;
382 long t;
383
384 if (vmstat_item_in_bytes(item)) {
385 /*
386 * Only cgroups use subpage accounting right now; at
387 * the global level, these items still change in
388 * multiples of whole pages. Store them as pages
389 * internally to keep the per-cpu counters compact.
390 */
391 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
392 delta >>= PAGE_SHIFT;
393 }
394
395 /* See __mod_node_page_state */
396 if (IS_ENABLED(CONFIG_PREEMPT_RT))
397 preempt_disable();
398
399 x = delta + __this_cpu_read(*p);
400
401 t = __this_cpu_read(pcp->stat_threshold);
402
403 if (unlikely(abs(x) > t)) {
404 node_page_state_add(x, pgdat, item);
405 x = 0;
406 }
407 __this_cpu_write(*p, x);
408
409 if (IS_ENABLED(CONFIG_PREEMPT_RT))
410 preempt_enable();
411 }
412 EXPORT_SYMBOL(__mod_node_page_state);
413
414 /*
415 * Optimized increment and decrement functions.
416 *
417 * These are only for a single page and therefore can take a struct page *
418 * argument instead of struct zone *. This allows the inclusion of the code
419 * generated for page_zone(page) into the optimized functions.
420 *
421 * No overflow check is necessary and therefore the differential can be
422 * incremented or decremented in place which may allow the compilers to
423 * generate better code.
424 * The increment or decrement is known and therefore one boundary check can
425 * be omitted.
426 *
427 * NOTE: These functions are very performance sensitive. Change only
428 * with care.
429 *
430 * Some processors have inc/dec instructions that are atomic vs an interrupt.
431 * However, the code must first determine the differential location in a zone
432 * based on the processor number and then inc/dec the counter. There is no
433 * guarantee without disabling preemption that the processor will not change
434 * in between and therefore the atomicity vs. interrupt cannot be exploited
435 * in a useful way here.
436 */
__inc_zone_state(struct zone * zone,enum zone_stat_item item)437 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
438 {
439 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
440 s8 __percpu *p = pcp->vm_stat_diff + item;
441 s8 v, t;
442
443 /* See __mod_node_page_state */
444 if (IS_ENABLED(CONFIG_PREEMPT_RT))
445 preempt_disable();
446
447 v = __this_cpu_inc_return(*p);
448 t = __this_cpu_read(pcp->stat_threshold);
449 if (unlikely(v > t)) {
450 s8 overstep = t >> 1;
451
452 zone_page_state_add(v + overstep, zone, item);
453 __this_cpu_write(*p, -overstep);
454 }
455
456 if (IS_ENABLED(CONFIG_PREEMPT_RT))
457 preempt_enable();
458 }
459
__inc_node_state(struct pglist_data * pgdat,enum node_stat_item item)460 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
461 {
462 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
463 s8 __percpu *p = pcp->vm_node_stat_diff + item;
464 s8 v, t;
465
466 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
467
468 /* See __mod_node_page_state */
469 if (IS_ENABLED(CONFIG_PREEMPT_RT))
470 preempt_disable();
471
472 v = __this_cpu_inc_return(*p);
473 t = __this_cpu_read(pcp->stat_threshold);
474 if (unlikely(v > t)) {
475 s8 overstep = t >> 1;
476
477 node_page_state_add(v + overstep, pgdat, item);
478 __this_cpu_write(*p, -overstep);
479 }
480
481 if (IS_ENABLED(CONFIG_PREEMPT_RT))
482 preempt_enable();
483 }
484
__inc_zone_page_state(struct page * page,enum zone_stat_item item)485 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
486 {
487 __inc_zone_state(page_zone(page), item);
488 }
489 EXPORT_SYMBOL(__inc_zone_page_state);
490
__inc_node_page_state(struct page * page,enum node_stat_item item)491 void __inc_node_page_state(struct page *page, enum node_stat_item item)
492 {
493 __inc_node_state(page_pgdat(page), item);
494 }
495 EXPORT_SYMBOL(__inc_node_page_state);
496
__dec_zone_state(struct zone * zone,enum zone_stat_item item)497 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
498 {
499 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
500 s8 __percpu *p = pcp->vm_stat_diff + item;
501 s8 v, t;
502
503 /* See __mod_node_page_state */
504 if (IS_ENABLED(CONFIG_PREEMPT_RT))
505 preempt_disable();
506
507 v = __this_cpu_dec_return(*p);
508 t = __this_cpu_read(pcp->stat_threshold);
509 if (unlikely(v < - t)) {
510 s8 overstep = t >> 1;
511
512 zone_page_state_add(v - overstep, zone, item);
513 __this_cpu_write(*p, overstep);
514 }
515
516 if (IS_ENABLED(CONFIG_PREEMPT_RT))
517 preempt_enable();
518 }
519
__dec_node_state(struct pglist_data * pgdat,enum node_stat_item item)520 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
521 {
522 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
523 s8 __percpu *p = pcp->vm_node_stat_diff + item;
524 s8 v, t;
525
526 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
527
528 /* See __mod_node_page_state */
529 if (IS_ENABLED(CONFIG_PREEMPT_RT))
530 preempt_disable();
531
532 v = __this_cpu_dec_return(*p);
533 t = __this_cpu_read(pcp->stat_threshold);
534 if (unlikely(v < - t)) {
535 s8 overstep = t >> 1;
536
537 node_page_state_add(v - overstep, pgdat, item);
538 __this_cpu_write(*p, overstep);
539 }
540
541 if (IS_ENABLED(CONFIG_PREEMPT_RT))
542 preempt_enable();
543 }
544
__dec_zone_page_state(struct page * page,enum zone_stat_item item)545 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
546 {
547 __dec_zone_state(page_zone(page), item);
548 }
549 EXPORT_SYMBOL(__dec_zone_page_state);
550
__dec_node_page_state(struct page * page,enum node_stat_item item)551 void __dec_node_page_state(struct page *page, enum node_stat_item item)
552 {
553 __dec_node_state(page_pgdat(page), item);
554 }
555 EXPORT_SYMBOL(__dec_node_page_state);
556
557 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
558 /*
559 * If we have cmpxchg_local support then we do not need to incur the overhead
560 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
561 *
562 * mod_state() modifies the zone counter state through atomic per cpu
563 * operations.
564 *
565 * Overstep mode specifies how overstep should handled:
566 * 0 No overstepping
567 * 1 Overstepping half of threshold
568 * -1 Overstepping minus half of threshold
569 */
mod_zone_state(struct zone * zone,enum zone_stat_item item,long delta,int overstep_mode)570 static inline void mod_zone_state(struct zone *zone,
571 enum zone_stat_item item, long delta, int overstep_mode)
572 {
573 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
574 s8 __percpu *p = pcp->vm_stat_diff + item;
575 long o, n, t, z;
576
577 do {
578 z = 0; /* overflow to zone counters */
579
580 /*
581 * The fetching of the stat_threshold is racy. We may apply
582 * a counter threshold to the wrong the cpu if we get
583 * rescheduled while executing here. However, the next
584 * counter update will apply the threshold again and
585 * therefore bring the counter under the threshold again.
586 *
587 * Most of the time the thresholds are the same anyways
588 * for all cpus in a zone.
589 */
590 t = this_cpu_read(pcp->stat_threshold);
591
592 o = this_cpu_read(*p);
593 n = delta + o;
594
595 if (abs(n) > t) {
596 int os = overstep_mode * (t >> 1) ;
597
598 /* Overflow must be added to zone counters */
599 z = n + os;
600 n = -os;
601 }
602 } while (this_cpu_cmpxchg(*p, o, n) != o);
603
604 if (z)
605 zone_page_state_add(z, zone, item);
606 }
607
mod_zone_page_state(struct zone * zone,enum zone_stat_item item,long delta)608 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
609 long delta)
610 {
611 mod_zone_state(zone, item, delta, 0);
612 }
613 EXPORT_SYMBOL(mod_zone_page_state);
614
inc_zone_page_state(struct page * page,enum zone_stat_item item)615 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
616 {
617 mod_zone_state(page_zone(page), item, 1, 1);
618 }
619 EXPORT_SYMBOL(inc_zone_page_state);
620
dec_zone_page_state(struct page * page,enum zone_stat_item item)621 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
622 {
623 mod_zone_state(page_zone(page), item, -1, -1);
624 }
625 EXPORT_SYMBOL(dec_zone_page_state);
626
mod_node_state(struct pglist_data * pgdat,enum node_stat_item item,int delta,int overstep_mode)627 static inline void mod_node_state(struct pglist_data *pgdat,
628 enum node_stat_item item, int delta, int overstep_mode)
629 {
630 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
631 s8 __percpu *p = pcp->vm_node_stat_diff + item;
632 long o, n, t, z;
633
634 if (vmstat_item_in_bytes(item)) {
635 /*
636 * Only cgroups use subpage accounting right now; at
637 * the global level, these items still change in
638 * multiples of whole pages. Store them as pages
639 * internally to keep the per-cpu counters compact.
640 */
641 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
642 delta >>= PAGE_SHIFT;
643 }
644
645 do {
646 z = 0; /* overflow to node counters */
647
648 /*
649 * The fetching of the stat_threshold is racy. We may apply
650 * a counter threshold to the wrong the cpu if we get
651 * rescheduled while executing here. However, the next
652 * counter update will apply the threshold again and
653 * therefore bring the counter under the threshold again.
654 *
655 * Most of the time the thresholds are the same anyways
656 * for all cpus in a node.
657 */
658 t = this_cpu_read(pcp->stat_threshold);
659
660 o = this_cpu_read(*p);
661 n = delta + o;
662
663 if (abs(n) > t) {
664 int os = overstep_mode * (t >> 1) ;
665
666 /* Overflow must be added to node counters */
667 z = n + os;
668 n = -os;
669 }
670 } while (this_cpu_cmpxchg(*p, o, n) != o);
671
672 if (z)
673 node_page_state_add(z, pgdat, item);
674 }
675
mod_node_page_state(struct pglist_data * pgdat,enum node_stat_item item,long delta)676 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
677 long delta)
678 {
679 mod_node_state(pgdat, item, delta, 0);
680 }
681 EXPORT_SYMBOL(mod_node_page_state);
682
inc_node_state(struct pglist_data * pgdat,enum node_stat_item item)683 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
684 {
685 mod_node_state(pgdat, item, 1, 1);
686 }
687
inc_node_page_state(struct page * page,enum node_stat_item item)688 void inc_node_page_state(struct page *page, enum node_stat_item item)
689 {
690 mod_node_state(page_pgdat(page), item, 1, 1);
691 }
692 EXPORT_SYMBOL(inc_node_page_state);
693
dec_node_page_state(struct page * page,enum node_stat_item item)694 void dec_node_page_state(struct page *page, enum node_stat_item item)
695 {
696 mod_node_state(page_pgdat(page), item, -1, -1);
697 }
698 EXPORT_SYMBOL(dec_node_page_state);
699 #else
700 /*
701 * Use interrupt disable to serialize counter updates
702 */
mod_zone_page_state(struct zone * zone,enum zone_stat_item item,long delta)703 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
704 long delta)
705 {
706 unsigned long flags;
707
708 local_irq_save(flags);
709 __mod_zone_page_state(zone, item, delta);
710 local_irq_restore(flags);
711 }
712 EXPORT_SYMBOL(mod_zone_page_state);
713
inc_zone_page_state(struct page * page,enum zone_stat_item item)714 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
715 {
716 unsigned long flags;
717 struct zone *zone;
718
719 zone = page_zone(page);
720 local_irq_save(flags);
721 __inc_zone_state(zone, item);
722 local_irq_restore(flags);
723 }
724 EXPORT_SYMBOL(inc_zone_page_state);
725
dec_zone_page_state(struct page * page,enum zone_stat_item item)726 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
727 {
728 unsigned long flags;
729
730 local_irq_save(flags);
731 __dec_zone_page_state(page, item);
732 local_irq_restore(flags);
733 }
734 EXPORT_SYMBOL(dec_zone_page_state);
735
inc_node_state(struct pglist_data * pgdat,enum node_stat_item item)736 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
737 {
738 unsigned long flags;
739
740 local_irq_save(flags);
741 __inc_node_state(pgdat, item);
742 local_irq_restore(flags);
743 }
744 EXPORT_SYMBOL(inc_node_state);
745
mod_node_page_state(struct pglist_data * pgdat,enum node_stat_item item,long delta)746 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
747 long delta)
748 {
749 unsigned long flags;
750
751 local_irq_save(flags);
752 __mod_node_page_state(pgdat, item, delta);
753 local_irq_restore(flags);
754 }
755 EXPORT_SYMBOL(mod_node_page_state);
756
inc_node_page_state(struct page * page,enum node_stat_item item)757 void inc_node_page_state(struct page *page, enum node_stat_item item)
758 {
759 unsigned long flags;
760 struct pglist_data *pgdat;
761
762 pgdat = page_pgdat(page);
763 local_irq_save(flags);
764 __inc_node_state(pgdat, item);
765 local_irq_restore(flags);
766 }
767 EXPORT_SYMBOL(inc_node_page_state);
768
dec_node_page_state(struct page * page,enum node_stat_item item)769 void dec_node_page_state(struct page *page, enum node_stat_item item)
770 {
771 unsigned long flags;
772
773 local_irq_save(flags);
774 __dec_node_page_state(page, item);
775 local_irq_restore(flags);
776 }
777 EXPORT_SYMBOL(dec_node_page_state);
778 #endif
779
780 /*
781 * Fold a differential into the global counters.
782 * Returns the number of counters updated.
783 */
fold_diff(int * zone_diff,int * node_diff)784 static int fold_diff(int *zone_diff, int *node_diff)
785 {
786 int i;
787 int changes = 0;
788
789 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
790 if (zone_diff[i]) {
791 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
792 changes++;
793 }
794
795 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
796 if (node_diff[i]) {
797 atomic_long_add(node_diff[i], &vm_node_stat[i]);
798 changes++;
799 }
800 return changes;
801 }
802
803 /*
804 * Update the zone counters for the current cpu.
805 *
806 * Note that refresh_cpu_vm_stats strives to only access
807 * node local memory. The per cpu pagesets on remote zones are placed
808 * in the memory local to the processor using that pageset. So the
809 * loop over all zones will access a series of cachelines local to
810 * the processor.
811 *
812 * The call to zone_page_state_add updates the cachelines with the
813 * statistics in the remote zone struct as well as the global cachelines
814 * with the global counters. These could cause remote node cache line
815 * bouncing and will have to be only done when necessary.
816 *
817 * The function returns the number of global counters updated.
818 */
refresh_cpu_vm_stats(bool do_pagesets)819 static int refresh_cpu_vm_stats(bool do_pagesets)
820 {
821 struct pglist_data *pgdat;
822 struct zone *zone;
823 int i;
824 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
825 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
826 int changes = 0;
827
828 for_each_populated_zone(zone) {
829 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
830 #ifdef CONFIG_NUMA
831 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;
832 #endif
833
834 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
835 int v;
836
837 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0);
838 if (v) {
839
840 atomic_long_add(v, &zone->vm_stat[i]);
841 global_zone_diff[i] += v;
842 #ifdef CONFIG_NUMA
843 /* 3 seconds idle till flush */
844 __this_cpu_write(pcp->expire, 3);
845 #endif
846 }
847 }
848 #ifdef CONFIG_NUMA
849
850 if (do_pagesets) {
851 cond_resched();
852 /*
853 * Deal with draining the remote pageset of this
854 * processor
855 *
856 * Check if there are pages remaining in this pageset
857 * if not then there is nothing to expire.
858 */
859 if (!__this_cpu_read(pcp->expire) ||
860 !__this_cpu_read(pcp->count))
861 continue;
862
863 /*
864 * We never drain zones local to this processor.
865 */
866 if (zone_to_nid(zone) == numa_node_id()) {
867 __this_cpu_write(pcp->expire, 0);
868 continue;
869 }
870
871 if (__this_cpu_dec_return(pcp->expire))
872 continue;
873
874 if (__this_cpu_read(pcp->count)) {
875 drain_zone_pages(zone, this_cpu_ptr(pcp));
876 changes++;
877 }
878 }
879 #endif
880 }
881
882 for_each_online_pgdat(pgdat) {
883 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
884
885 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
886 int v;
887
888 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
889 if (v) {
890 atomic_long_add(v, &pgdat->vm_stat[i]);
891 global_node_diff[i] += v;
892 }
893 }
894 }
895
896 changes += fold_diff(global_zone_diff, global_node_diff);
897 return changes;
898 }
899
900 /*
901 * Fold the data for an offline cpu into the global array.
902 * There cannot be any access by the offline cpu and therefore
903 * synchronization is simplified.
904 */
cpu_vm_stats_fold(int cpu)905 void cpu_vm_stats_fold(int cpu)
906 {
907 struct pglist_data *pgdat;
908 struct zone *zone;
909 int i;
910 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
911 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
912
913 for_each_populated_zone(zone) {
914 struct per_cpu_zonestat *pzstats;
915
916 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
917
918 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
919 if (pzstats->vm_stat_diff[i]) {
920 int v;
921
922 v = pzstats->vm_stat_diff[i];
923 pzstats->vm_stat_diff[i] = 0;
924 atomic_long_add(v, &zone->vm_stat[i]);
925 global_zone_diff[i] += v;
926 }
927 }
928 #ifdef CONFIG_NUMA
929 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) {
930 if (pzstats->vm_numa_event[i]) {
931 unsigned long v;
932
933 v = pzstats->vm_numa_event[i];
934 pzstats->vm_numa_event[i] = 0;
935 zone_numa_event_add(v, zone, i);
936 }
937 }
938 #endif
939 }
940
941 for_each_online_pgdat(pgdat) {
942 struct per_cpu_nodestat *p;
943
944 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
945
946 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
947 if (p->vm_node_stat_diff[i]) {
948 int v;
949
950 v = p->vm_node_stat_diff[i];
951 p->vm_node_stat_diff[i] = 0;
952 atomic_long_add(v, &pgdat->vm_stat[i]);
953 global_node_diff[i] += v;
954 }
955 }
956
957 fold_diff(global_zone_diff, global_node_diff);
958 }
959
960 /*
961 * this is only called if !populated_zone(zone), which implies no other users of
962 * pset->vm_stat_diff[] exist.
963 */
drain_zonestat(struct zone * zone,struct per_cpu_zonestat * pzstats)964 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)
965 {
966 unsigned long v;
967 int i;
968
969 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
970 if (pzstats->vm_stat_diff[i]) {
971 v = pzstats->vm_stat_diff[i];
972 pzstats->vm_stat_diff[i] = 0;
973 zone_page_state_add(v, zone, i);
974 }
975 }
976
977 #ifdef CONFIG_NUMA
978 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) {
979 if (pzstats->vm_numa_event[i]) {
980 v = pzstats->vm_numa_event[i];
981 pzstats->vm_numa_event[i] = 0;
982 zone_numa_event_add(v, zone, i);
983 }
984 }
985 #endif
986 }
987 #endif
988
989 #ifdef CONFIG_NUMA
990 /*
991 * Determine the per node value of a stat item. This function
992 * is called frequently in a NUMA machine, so try to be as
993 * frugal as possible.
994 */
sum_zone_node_page_state(int node,enum zone_stat_item item)995 unsigned long sum_zone_node_page_state(int node,
996 enum zone_stat_item item)
997 {
998 struct zone *zones = NODE_DATA(node)->node_zones;
999 int i;
1000 unsigned long count = 0;
1001
1002 for (i = 0; i < MAX_NR_ZONES; i++)
1003 count += zone_page_state(zones + i, item);
1004
1005 return count;
1006 }
1007
1008 /* Determine the per node value of a numa stat item. */
sum_zone_numa_event_state(int node,enum numa_stat_item item)1009 unsigned long sum_zone_numa_event_state(int node,
1010 enum numa_stat_item item)
1011 {
1012 struct zone *zones = NODE_DATA(node)->node_zones;
1013 unsigned long count = 0;
1014 int i;
1015
1016 for (i = 0; i < MAX_NR_ZONES; i++)
1017 count += zone_numa_event_state(zones + i, item);
1018
1019 return count;
1020 }
1021
1022 /*
1023 * Determine the per node value of a stat item.
1024 */
node_page_state_pages(struct pglist_data * pgdat,enum node_stat_item item)1025 unsigned long node_page_state_pages(struct pglist_data *pgdat,
1026 enum node_stat_item item)
1027 {
1028 long x = atomic_long_read(&pgdat->vm_stat[item]);
1029 #ifdef CONFIG_SMP
1030 if (x < 0)
1031 x = 0;
1032 #endif
1033 return x;
1034 }
1035
node_page_state(struct pglist_data * pgdat,enum node_stat_item item)1036 unsigned long node_page_state(struct pglist_data *pgdat,
1037 enum node_stat_item item)
1038 {
1039 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
1040
1041 return node_page_state_pages(pgdat, item);
1042 }
1043 #endif
1044
1045 #ifdef CONFIG_COMPACTION
1046
1047 struct contig_page_info {
1048 unsigned long free_pages;
1049 unsigned long free_blocks_total;
1050 unsigned long free_blocks_suitable;
1051 };
1052
1053 /*
1054 * Calculate the number of free pages in a zone, how many contiguous
1055 * pages are free and how many are large enough to satisfy an allocation of
1056 * the target size. Note that this function makes no attempt to estimate
1057 * how many suitable free blocks there *might* be if MOVABLE pages were
1058 * migrated. Calculating that is possible, but expensive and can be
1059 * figured out from userspace
1060 */
fill_contig_page_info(struct zone * zone,unsigned int suitable_order,struct contig_page_info * info)1061 static void fill_contig_page_info(struct zone *zone,
1062 unsigned int suitable_order,
1063 struct contig_page_info *info)
1064 {
1065 unsigned int order;
1066
1067 info->free_pages = 0;
1068 info->free_blocks_total = 0;
1069 info->free_blocks_suitable = 0;
1070
1071 for (order = 0; order < MAX_ORDER; order++) {
1072 unsigned long blocks;
1073
1074 /*
1075 * Count number of free blocks.
1076 *
1077 * Access to nr_free is lockless as nr_free is used only for
1078 * diagnostic purposes. Use data_race to avoid KCSAN warning.
1079 */
1080 blocks = data_race(zone->free_area[order].nr_free);
1081 info->free_blocks_total += blocks;
1082
1083 /* Count free base pages */
1084 info->free_pages += blocks << order;
1085
1086 /* Count the suitable free blocks */
1087 if (order >= suitable_order)
1088 info->free_blocks_suitable += blocks <<
1089 (order - suitable_order);
1090 }
1091 }
1092
1093 /*
1094 * A fragmentation index only makes sense if an allocation of a requested
1095 * size would fail. If that is true, the fragmentation index indicates
1096 * whether external fragmentation or a lack of memory was the problem.
1097 * The value can be used to determine if page reclaim or compaction
1098 * should be used
1099 */
__fragmentation_index(unsigned int order,struct contig_page_info * info)1100 static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1101 {
1102 unsigned long requested = 1UL << order;
1103
1104 if (WARN_ON_ONCE(order >= MAX_ORDER))
1105 return 0;
1106
1107 if (!info->free_blocks_total)
1108 return 0;
1109
1110 /* Fragmentation index only makes sense when a request would fail */
1111 if (info->free_blocks_suitable)
1112 return -1000;
1113
1114 /*
1115 * Index is between 0 and 1 so return within 3 decimal places
1116 *
1117 * 0 => allocation would fail due to lack of memory
1118 * 1 => allocation would fail due to fragmentation
1119 */
1120 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1121 }
1122
1123 /*
1124 * Calculates external fragmentation within a zone wrt the given order.
1125 * It is defined as the percentage of pages found in blocks of size
1126 * less than 1 << order. It returns values in range [0, 100].
1127 */
extfrag_for_order(struct zone * zone,unsigned int order)1128 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)
1129 {
1130 struct contig_page_info info;
1131
1132 fill_contig_page_info(zone, order, &info);
1133 if (info.free_pages == 0)
1134 return 0;
1135
1136 return div_u64((info.free_pages -
1137 (info.free_blocks_suitable << order)) * 100,
1138 info.free_pages);
1139 }
1140
1141 /* Same as __fragmentation index but allocs contig_page_info on stack */
fragmentation_index(struct zone * zone,unsigned int order)1142 int fragmentation_index(struct zone *zone, unsigned int order)
1143 {
1144 struct contig_page_info info;
1145
1146 fill_contig_page_info(zone, order, &info);
1147 return __fragmentation_index(order, &info);
1148 }
1149 #endif
1150
1151 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
1152 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
1153 #ifdef CONFIG_ZONE_DMA
1154 #define TEXT_FOR_DMA(xx) xx "_dma",
1155 #else
1156 #define TEXT_FOR_DMA(xx)
1157 #endif
1158
1159 #ifdef CONFIG_ZONE_DMA32
1160 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1161 #else
1162 #define TEXT_FOR_DMA32(xx)
1163 #endif
1164
1165 #ifdef CONFIG_HIGHMEM
1166 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1167 #else
1168 #define TEXT_FOR_HIGHMEM(xx)
1169 #endif
1170
1171 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1172 TEXT_FOR_HIGHMEM(xx) xx "_movable",
1173
1174 const char * const vmstat_text[] = {
1175 /* enum zone_stat_item counters */
1176 "nr_free_pages",
1177 "nr_zone_inactive_anon",
1178 "nr_zone_active_anon",
1179 "nr_zone_inactive_file",
1180 "nr_zone_active_file",
1181 "nr_zone_unevictable",
1182 "nr_zone_write_pending",
1183 "nr_mlock",
1184 "nr_bounce",
1185 #if IS_ENABLED(CONFIG_ZSMALLOC)
1186 "nr_zspages",
1187 #endif
1188 "nr_free_cma",
1189
1190 /* enum numa_stat_item counters */
1191 #ifdef CONFIG_NUMA
1192 "numa_hit",
1193 "numa_miss",
1194 "numa_foreign",
1195 "numa_interleave",
1196 "numa_local",
1197 "numa_other",
1198 #endif
1199
1200 /* enum node_stat_item counters */
1201 "nr_inactive_anon",
1202 "nr_active_anon",
1203 "nr_inactive_file",
1204 "nr_active_file",
1205 "nr_unevictable",
1206 "nr_slab_reclaimable",
1207 "nr_slab_unreclaimable",
1208 "nr_isolated_anon",
1209 "nr_isolated_file",
1210 "workingset_nodes",
1211 "workingset_refault_anon",
1212 "workingset_refault_file",
1213 "workingset_activate_anon",
1214 "workingset_activate_file",
1215 "workingset_restore_anon",
1216 "workingset_restore_file",
1217 "workingset_nodereclaim",
1218 "nr_anon_pages",
1219 "nr_mapped",
1220 "nr_file_pages",
1221 "nr_dirty",
1222 "nr_writeback",
1223 "nr_writeback_temp",
1224 "nr_shmem",
1225 "nr_shmem_hugepages",
1226 "nr_shmem_pmdmapped",
1227 "nr_file_hugepages",
1228 "nr_file_pmdmapped",
1229 "nr_anon_transparent_hugepages",
1230 "nr_vmscan_write",
1231 "nr_vmscan_immediate_reclaim",
1232 "nr_dirtied",
1233 "nr_written",
1234 "nr_throttled_written",
1235 "nr_kernel_misc_reclaimable",
1236 "nr_foll_pin_acquired",
1237 "nr_foll_pin_released",
1238 "nr_kernel_stack",
1239 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1240 "nr_shadow_call_stack",
1241 #endif
1242 "nr_page_table_pages",
1243 #ifdef CONFIG_SWAP
1244 "nr_swapcached",
1245 #endif
1246 #ifdef CONFIG_NUMA_BALANCING
1247 "pgpromote_success",
1248 #endif
1249
1250 /* enum writeback_stat_item counters */
1251 "nr_dirty_threshold",
1252 "nr_dirty_background_threshold",
1253
1254 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
1255 /* enum vm_event_item counters */
1256 "pgpgin",
1257 "pgpgout",
1258 "pswpin",
1259 "pswpout",
1260
1261 TEXTS_FOR_ZONES("pgalloc")
1262 TEXTS_FOR_ZONES("allocstall")
1263 TEXTS_FOR_ZONES("pgskip")
1264
1265 "pgfree",
1266 "pgactivate",
1267 "pgdeactivate",
1268 "pglazyfree",
1269
1270 "pgfault",
1271 "pgmajfault",
1272 "pglazyfreed",
1273
1274 "pgrefill",
1275 "pgreuse",
1276 "pgsteal_kswapd",
1277 "pgsteal_direct",
1278 "pgdemote_kswapd",
1279 "pgdemote_direct",
1280 "pgscan_kswapd",
1281 "pgscan_direct",
1282 "pgscan_direct_throttle",
1283 "pgscan_anon",
1284 "pgscan_file",
1285 "pgsteal_anon",
1286 "pgsteal_file",
1287
1288 #ifdef CONFIG_NUMA
1289 "zone_reclaim_failed",
1290 #endif
1291 "pginodesteal",
1292 "slabs_scanned",
1293 "kswapd_inodesteal",
1294 "kswapd_low_wmark_hit_quickly",
1295 "kswapd_high_wmark_hit_quickly",
1296 "pageoutrun",
1297
1298 "pgrotated",
1299
1300 "drop_pagecache",
1301 "drop_slab",
1302 "oom_kill",
1303
1304 #ifdef CONFIG_NUMA_BALANCING
1305 "numa_pte_updates",
1306 "numa_huge_pte_updates",
1307 "numa_hint_faults",
1308 "numa_hint_faults_local",
1309 "numa_pages_migrated",
1310 #endif
1311 #ifdef CONFIG_MIGRATION
1312 "pgmigrate_success",
1313 "pgmigrate_fail",
1314 "thp_migration_success",
1315 "thp_migration_fail",
1316 "thp_migration_split",
1317 #endif
1318 #ifdef CONFIG_COMPACTION
1319 "compact_migrate_scanned",
1320 "compact_free_scanned",
1321 "compact_isolated",
1322 "compact_stall",
1323 "compact_fail",
1324 "compact_success",
1325 "compact_daemon_wake",
1326 "compact_daemon_migrate_scanned",
1327 "compact_daemon_free_scanned",
1328 #endif
1329
1330 #ifdef CONFIG_HUGETLB_PAGE
1331 "htlb_buddy_alloc_success",
1332 "htlb_buddy_alloc_fail",
1333 #endif
1334 #ifdef CONFIG_CMA
1335 "cma_alloc_success",
1336 "cma_alloc_fail",
1337 #endif
1338 "unevictable_pgs_culled",
1339 "unevictable_pgs_scanned",
1340 "unevictable_pgs_rescued",
1341 "unevictable_pgs_mlocked",
1342 "unevictable_pgs_munlocked",
1343 "unevictable_pgs_cleared",
1344 "unevictable_pgs_stranded",
1345
1346 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1347 "thp_fault_alloc",
1348 "thp_fault_fallback",
1349 "thp_fault_fallback_charge",
1350 "thp_collapse_alloc",
1351 "thp_collapse_alloc_failed",
1352 "thp_file_alloc",
1353 "thp_file_fallback",
1354 "thp_file_fallback_charge",
1355 "thp_file_mapped",
1356 "thp_split_page",
1357 "thp_split_page_failed",
1358 "thp_deferred_split_page",
1359 "thp_split_pmd",
1360 "thp_scan_exceed_none_pte",
1361 "thp_scan_exceed_swap_pte",
1362 "thp_scan_exceed_share_pte",
1363 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1364 "thp_split_pud",
1365 #endif
1366 "thp_zero_page_alloc",
1367 "thp_zero_page_alloc_failed",
1368 "thp_swpout",
1369 "thp_swpout_fallback",
1370 #endif
1371 #ifdef CONFIG_MEMORY_BALLOON
1372 "balloon_inflate",
1373 "balloon_deflate",
1374 #ifdef CONFIG_BALLOON_COMPACTION
1375 "balloon_migrate",
1376 #endif
1377 #endif /* CONFIG_MEMORY_BALLOON */
1378 #ifdef CONFIG_DEBUG_TLBFLUSH
1379 "nr_tlb_remote_flush",
1380 "nr_tlb_remote_flush_received",
1381 "nr_tlb_local_flush_all",
1382 "nr_tlb_local_flush_one",
1383 #endif /* CONFIG_DEBUG_TLBFLUSH */
1384
1385 #ifdef CONFIG_DEBUG_VM_VMACACHE
1386 "vmacache_find_calls",
1387 "vmacache_find_hits",
1388 #endif
1389 #ifdef CONFIG_SWAP
1390 "swap_ra",
1391 "swap_ra_hit",
1392 #ifdef CONFIG_KSM
1393 "ksm_swpin_copy",
1394 #endif
1395 #endif
1396 #ifdef CONFIG_KSM
1397 "cow_ksm",
1398 #endif
1399 #ifdef CONFIG_ZSWAP
1400 "zswpin",
1401 "zswpout",
1402 #endif
1403 #ifdef CONFIG_X86
1404 "direct_map_level2_splits",
1405 "direct_map_level3_splits",
1406 #endif
1407 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
1408 };
1409 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
1410
1411 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1412 defined(CONFIG_PROC_FS)
frag_start(struct seq_file * m,loff_t * pos)1413 static void *frag_start(struct seq_file *m, loff_t *pos)
1414 {
1415 pg_data_t *pgdat;
1416 loff_t node = *pos;
1417
1418 for (pgdat = first_online_pgdat();
1419 pgdat && node;
1420 pgdat = next_online_pgdat(pgdat))
1421 --node;
1422
1423 return pgdat;
1424 }
1425
frag_next(struct seq_file * m,void * arg,loff_t * pos)1426 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1427 {
1428 pg_data_t *pgdat = (pg_data_t *)arg;
1429
1430 (*pos)++;
1431 return next_online_pgdat(pgdat);
1432 }
1433
frag_stop(struct seq_file * m,void * arg)1434 static void frag_stop(struct seq_file *m, void *arg)
1435 {
1436 }
1437
1438 /*
1439 * Walk zones in a node and print using a callback.
1440 * If @assert_populated is true, only use callback for zones that are populated.
1441 */
walk_zones_in_node(struct seq_file * m,pg_data_t * pgdat,bool assert_populated,bool nolock,void (* print)(struct seq_file * m,pg_data_t *,struct zone *))1442 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1443 bool assert_populated, bool nolock,
1444 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1445 {
1446 struct zone *zone;
1447 struct zone *node_zones = pgdat->node_zones;
1448 unsigned long flags;
1449
1450 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1451 if (assert_populated && !populated_zone(zone))
1452 continue;
1453
1454 if (!nolock)
1455 spin_lock_irqsave(&zone->lock, flags);
1456 print(m, pgdat, zone);
1457 if (!nolock)
1458 spin_unlock_irqrestore(&zone->lock, flags);
1459 }
1460 }
1461 #endif
1462
1463 #ifdef CONFIG_PROC_FS
frag_show_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)1464 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1465 struct zone *zone)
1466 {
1467 int order;
1468
1469 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1470 for (order = 0; order < MAX_ORDER; ++order)
1471 /*
1472 * Access to nr_free is lockless as nr_free is used only for
1473 * printing purposes. Use data_race to avoid KCSAN warning.
1474 */
1475 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));
1476 seq_putc(m, '\n');
1477 }
1478
1479 /*
1480 * This walks the free areas for each zone.
1481 */
frag_show(struct seq_file * m,void * arg)1482 static int frag_show(struct seq_file *m, void *arg)
1483 {
1484 pg_data_t *pgdat = (pg_data_t *)arg;
1485 walk_zones_in_node(m, pgdat, true, false, frag_show_print);
1486 return 0;
1487 }
1488
pagetypeinfo_showfree_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)1489 static void pagetypeinfo_showfree_print(struct seq_file *m,
1490 pg_data_t *pgdat, struct zone *zone)
1491 {
1492 int order, mtype;
1493
1494 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1495 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1496 pgdat->node_id,
1497 zone->name,
1498 migratetype_names[mtype]);
1499 for (order = 0; order < MAX_ORDER; ++order) {
1500 unsigned long freecount = 0;
1501 struct free_area *area;
1502 struct list_head *curr;
1503 bool overflow = false;
1504
1505 area = &(zone->free_area[order]);
1506
1507 list_for_each(curr, &area->free_list[mtype]) {
1508 /*
1509 * Cap the free_list iteration because it might
1510 * be really large and we are under a spinlock
1511 * so a long time spent here could trigger a
1512 * hard lockup detector. Anyway this is a
1513 * debugging tool so knowing there is a handful
1514 * of pages of this order should be more than
1515 * sufficient.
1516 */
1517 if (++freecount >= 100000) {
1518 overflow = true;
1519 break;
1520 }
1521 }
1522 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount);
1523 spin_unlock_irq(&zone->lock);
1524 cond_resched();
1525 spin_lock_irq(&zone->lock);
1526 }
1527 seq_putc(m, '\n');
1528 }
1529 }
1530
1531 /* Print out the free pages at each order for each migatetype */
pagetypeinfo_showfree(struct seq_file * m,void * arg)1532 static void pagetypeinfo_showfree(struct seq_file *m, void *arg)
1533 {
1534 int order;
1535 pg_data_t *pgdat = (pg_data_t *)arg;
1536
1537 /* Print header */
1538 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1539 for (order = 0; order < MAX_ORDER; ++order)
1540 seq_printf(m, "%6d ", order);
1541 seq_putc(m, '\n');
1542
1543 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
1544 }
1545
pagetypeinfo_showblockcount_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)1546 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1547 pg_data_t *pgdat, struct zone *zone)
1548 {
1549 int mtype;
1550 unsigned long pfn;
1551 unsigned long start_pfn = zone->zone_start_pfn;
1552 unsigned long end_pfn = zone_end_pfn(zone);
1553 unsigned long count[MIGRATE_TYPES] = { 0, };
1554
1555 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1556 struct page *page;
1557
1558 page = pfn_to_online_page(pfn);
1559 if (!page)
1560 continue;
1561
1562 if (page_zone(page) != zone)
1563 continue;
1564
1565 mtype = get_pageblock_migratetype(page);
1566
1567 if (mtype < MIGRATE_TYPES)
1568 count[mtype]++;
1569 }
1570
1571 /* Print counts */
1572 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1573 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1574 seq_printf(m, "%12lu ", count[mtype]);
1575 seq_putc(m, '\n');
1576 }
1577
1578 /* Print out the number of pageblocks for each migratetype */
pagetypeinfo_showblockcount(struct seq_file * m,void * arg)1579 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1580 {
1581 int mtype;
1582 pg_data_t *pgdat = (pg_data_t *)arg;
1583
1584 seq_printf(m, "\n%-23s", "Number of blocks type ");
1585 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1586 seq_printf(m, "%12s ", migratetype_names[mtype]);
1587 seq_putc(m, '\n');
1588 walk_zones_in_node(m, pgdat, true, false,
1589 pagetypeinfo_showblockcount_print);
1590 }
1591
1592 /*
1593 * Print out the number of pageblocks for each migratetype that contain pages
1594 * of other types. This gives an indication of how well fallbacks are being
1595 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1596 * to determine what is going on
1597 */
pagetypeinfo_showmixedcount(struct seq_file * m,pg_data_t * pgdat)1598 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1599 {
1600 #ifdef CONFIG_PAGE_OWNER
1601 int mtype;
1602
1603 if (!static_branch_unlikely(&page_owner_inited))
1604 return;
1605
1606 drain_all_pages(NULL);
1607
1608 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1609 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1610 seq_printf(m, "%12s ", migratetype_names[mtype]);
1611 seq_putc(m, '\n');
1612
1613 walk_zones_in_node(m, pgdat, true, true,
1614 pagetypeinfo_showmixedcount_print);
1615 #endif /* CONFIG_PAGE_OWNER */
1616 }
1617
1618 /*
1619 * This prints out statistics in relation to grouping pages by mobility.
1620 * It is expensive to collect so do not constantly read the file.
1621 */
pagetypeinfo_show(struct seq_file * m,void * arg)1622 static int pagetypeinfo_show(struct seq_file *m, void *arg)
1623 {
1624 pg_data_t *pgdat = (pg_data_t *)arg;
1625
1626 /* check memoryless node */
1627 if (!node_state(pgdat->node_id, N_MEMORY))
1628 return 0;
1629
1630 seq_printf(m, "Page block order: %d\n", pageblock_order);
1631 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1632 seq_putc(m, '\n');
1633 pagetypeinfo_showfree(m, pgdat);
1634 pagetypeinfo_showblockcount(m, pgdat);
1635 pagetypeinfo_showmixedcount(m, pgdat);
1636
1637 return 0;
1638 }
1639
1640 static const struct seq_operations fragmentation_op = {
1641 .start = frag_start,
1642 .next = frag_next,
1643 .stop = frag_stop,
1644 .show = frag_show,
1645 };
1646
1647 static const struct seq_operations pagetypeinfo_op = {
1648 .start = frag_start,
1649 .next = frag_next,
1650 .stop = frag_stop,
1651 .show = pagetypeinfo_show,
1652 };
1653
is_zone_first_populated(pg_data_t * pgdat,struct zone * zone)1654 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1655 {
1656 int zid;
1657
1658 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1659 struct zone *compare = &pgdat->node_zones[zid];
1660
1661 if (populated_zone(compare))
1662 return zone == compare;
1663 }
1664
1665 return false;
1666 }
1667
zoneinfo_show_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)1668 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1669 struct zone *zone)
1670 {
1671 int i;
1672 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1673 if (is_zone_first_populated(pgdat, zone)) {
1674 seq_printf(m, "\n per-node stats");
1675 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1676 unsigned long pages = node_page_state_pages(pgdat, i);
1677
1678 if (vmstat_item_print_in_thp(i))
1679 pages /= HPAGE_PMD_NR;
1680 seq_printf(m, "\n %-12s %lu", node_stat_name(i),
1681 pages);
1682 }
1683 }
1684 seq_printf(m,
1685 "\n pages free %lu"
1686 "\n boost %lu"
1687 "\n min %lu"
1688 "\n low %lu"
1689 "\n high %lu"
1690 "\n spanned %lu"
1691 "\n present %lu"
1692 "\n managed %lu"
1693 "\n cma %lu",
1694 zone_page_state(zone, NR_FREE_PAGES),
1695 zone->watermark_boost,
1696 min_wmark_pages(zone),
1697 low_wmark_pages(zone),
1698 high_wmark_pages(zone),
1699 zone->spanned_pages,
1700 zone->present_pages,
1701 zone_managed_pages(zone),
1702 zone_cma_pages(zone));
1703
1704 seq_printf(m,
1705 "\n protection: (%ld",
1706 zone->lowmem_reserve[0]);
1707 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1708 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1709 seq_putc(m, ')');
1710
1711 /* If unpopulated, no other information is useful */
1712 if (!populated_zone(zone)) {
1713 seq_putc(m, '\n');
1714 return;
1715 }
1716
1717 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1718 seq_printf(m, "\n %-12s %lu", zone_stat_name(i),
1719 zone_page_state(zone, i));
1720
1721 #ifdef CONFIG_NUMA
1722 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
1723 seq_printf(m, "\n %-12s %lu", numa_stat_name(i),
1724 zone_numa_event_state(zone, i));
1725 #endif
1726
1727 seq_printf(m, "\n pagesets");
1728 for_each_online_cpu(i) {
1729 struct per_cpu_pages *pcp;
1730 struct per_cpu_zonestat __maybe_unused *pzstats;
1731
1732 pcp = per_cpu_ptr(zone->per_cpu_pageset, i);
1733 seq_printf(m,
1734 "\n cpu: %i"
1735 "\n count: %i"
1736 "\n high: %i"
1737 "\n batch: %i",
1738 i,
1739 pcp->count,
1740 pcp->high,
1741 pcp->batch);
1742 #ifdef CONFIG_SMP
1743 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);
1744 seq_printf(m, "\n vm stats threshold: %d",
1745 pzstats->stat_threshold);
1746 #endif
1747 }
1748 seq_printf(m,
1749 "\n node_unreclaimable: %u"
1750 "\n start_pfn: %lu",
1751 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
1752 zone->zone_start_pfn);
1753 seq_putc(m, '\n');
1754 }
1755
1756 /*
1757 * Output information about zones in @pgdat. All zones are printed regardless
1758 * of whether they are populated or not: lowmem_reserve_ratio operates on the
1759 * set of all zones and userspace would not be aware of such zones if they are
1760 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
1761 */
zoneinfo_show(struct seq_file * m,void * arg)1762 static int zoneinfo_show(struct seq_file *m, void *arg)
1763 {
1764 pg_data_t *pgdat = (pg_data_t *)arg;
1765 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print);
1766 return 0;
1767 }
1768
1769 static const struct seq_operations zoneinfo_op = {
1770 .start = frag_start, /* iterate over all zones. The same as in
1771 * fragmentation. */
1772 .next = frag_next,
1773 .stop = frag_stop,
1774 .show = zoneinfo_show,
1775 };
1776
1777 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1778 NR_VM_NUMA_EVENT_ITEMS + \
1779 NR_VM_NODE_STAT_ITEMS + \
1780 NR_VM_WRITEBACK_STAT_ITEMS + \
1781 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1782 NR_VM_EVENT_ITEMS : 0))
1783
vmstat_start(struct seq_file * m,loff_t * pos)1784 static void *vmstat_start(struct seq_file *m, loff_t *pos)
1785 {
1786 unsigned long *v;
1787 int i;
1788
1789 if (*pos >= NR_VMSTAT_ITEMS)
1790 return NULL;
1791
1792 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS);
1793 fold_vm_numa_events();
1794 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL);
1795 m->private = v;
1796 if (!v)
1797 return ERR_PTR(-ENOMEM);
1798 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1799 v[i] = global_zone_page_state(i);
1800 v += NR_VM_ZONE_STAT_ITEMS;
1801
1802 #ifdef CONFIG_NUMA
1803 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
1804 v[i] = global_numa_event_state(i);
1805 v += NR_VM_NUMA_EVENT_ITEMS;
1806 #endif
1807
1808 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1809 v[i] = global_node_page_state_pages(i);
1810 if (vmstat_item_print_in_thp(i))
1811 v[i] /= HPAGE_PMD_NR;
1812 }
1813 v += NR_VM_NODE_STAT_ITEMS;
1814
1815 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1816 v + NR_DIRTY_THRESHOLD);
1817 v += NR_VM_WRITEBACK_STAT_ITEMS;
1818
1819 #ifdef CONFIG_VM_EVENT_COUNTERS
1820 all_vm_events(v);
1821 v[PGPGIN] /= 2; /* sectors -> kbytes */
1822 v[PGPGOUT] /= 2;
1823 #endif
1824 return (unsigned long *)m->private + *pos;
1825 }
1826
vmstat_next(struct seq_file * m,void * arg,loff_t * pos)1827 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1828 {
1829 (*pos)++;
1830 if (*pos >= NR_VMSTAT_ITEMS)
1831 return NULL;
1832 return (unsigned long *)m->private + *pos;
1833 }
1834
vmstat_show(struct seq_file * m,void * arg)1835 static int vmstat_show(struct seq_file *m, void *arg)
1836 {
1837 unsigned long *l = arg;
1838 unsigned long off = l - (unsigned long *)m->private;
1839
1840 seq_puts(m, vmstat_text[off]);
1841 seq_put_decimal_ull(m, " ", *l);
1842 seq_putc(m, '\n');
1843
1844 if (off == NR_VMSTAT_ITEMS - 1) {
1845 /*
1846 * We've come to the end - add any deprecated counters to avoid
1847 * breaking userspace which might depend on them being present.
1848 */
1849 seq_puts(m, "nr_unstable 0\n");
1850 }
1851 return 0;
1852 }
1853
vmstat_stop(struct seq_file * m,void * arg)1854 static void vmstat_stop(struct seq_file *m, void *arg)
1855 {
1856 kfree(m->private);
1857 m->private = NULL;
1858 }
1859
1860 static const struct seq_operations vmstat_op = {
1861 .start = vmstat_start,
1862 .next = vmstat_next,
1863 .stop = vmstat_stop,
1864 .show = vmstat_show,
1865 };
1866 #endif /* CONFIG_PROC_FS */
1867
1868 #ifdef CONFIG_SMP
1869 static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1870 int sysctl_stat_interval __read_mostly = HZ;
1871
1872 #ifdef CONFIG_PROC_FS
refresh_vm_stats(struct work_struct * work)1873 static void refresh_vm_stats(struct work_struct *work)
1874 {
1875 refresh_cpu_vm_stats(true);
1876 }
1877
vmstat_refresh(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)1878 int vmstat_refresh(struct ctl_table *table, int write,
1879 void *buffer, size_t *lenp, loff_t *ppos)
1880 {
1881 long val;
1882 int err;
1883 int i;
1884
1885 /*
1886 * The regular update, every sysctl_stat_interval, may come later
1887 * than expected: leaving a significant amount in per_cpu buckets.
1888 * This is particularly misleading when checking a quantity of HUGE
1889 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1890 * which can equally be echo'ed to or cat'ted from (by root),
1891 * can be used to update the stats just before reading them.
1892 *
1893 * Oh, and since global_zone_page_state() etc. are so careful to hide
1894 * transiently negative values, report an error here if any of
1895 * the stats is negative, so we know to go looking for imbalance.
1896 */
1897 err = schedule_on_each_cpu(refresh_vm_stats);
1898 if (err)
1899 return err;
1900 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
1901 /*
1902 * Skip checking stats known to go negative occasionally.
1903 */
1904 switch (i) {
1905 case NR_ZONE_WRITE_PENDING:
1906 case NR_FREE_CMA_PAGES:
1907 continue;
1908 }
1909 val = atomic_long_read(&vm_zone_stat[i]);
1910 if (val < 0) {
1911 pr_warn("%s: %s %ld\n",
1912 __func__, zone_stat_name(i), val);
1913 }
1914 }
1915 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1916 /*
1917 * Skip checking stats known to go negative occasionally.
1918 */
1919 switch (i) {
1920 case NR_WRITEBACK:
1921 continue;
1922 }
1923 val = atomic_long_read(&vm_node_stat[i]);
1924 if (val < 0) {
1925 pr_warn("%s: %s %ld\n",
1926 __func__, node_stat_name(i), val);
1927 }
1928 }
1929 if (write)
1930 *ppos += *lenp;
1931 else
1932 *lenp = 0;
1933 return 0;
1934 }
1935 #endif /* CONFIG_PROC_FS */
1936
vmstat_update(struct work_struct * w)1937 static void vmstat_update(struct work_struct *w)
1938 {
1939 if (refresh_cpu_vm_stats(true)) {
1940 /*
1941 * Counters were updated so we expect more updates
1942 * to occur in the future. Keep on running the
1943 * update worker thread.
1944 */
1945 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq,
1946 this_cpu_ptr(&vmstat_work),
1947 round_jiffies_relative(sysctl_stat_interval));
1948 }
1949 }
1950
1951 /*
1952 * Check if the diffs for a certain cpu indicate that
1953 * an update is needed.
1954 */
need_update(int cpu)1955 static bool need_update(int cpu)
1956 {
1957 pg_data_t *last_pgdat = NULL;
1958 struct zone *zone;
1959
1960 for_each_populated_zone(zone) {
1961 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
1962 struct per_cpu_nodestat *n;
1963
1964 /*
1965 * The fast way of checking if there are any vmstat diffs.
1966 */
1967 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff)))
1968 return true;
1969
1970 if (last_pgdat == zone->zone_pgdat)
1971 continue;
1972 last_pgdat = zone->zone_pgdat;
1973 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);
1974 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff)))
1975 return true;
1976 }
1977 return false;
1978 }
1979
1980 /*
1981 * Switch off vmstat processing and then fold all the remaining differentials
1982 * until the diffs stay at zero. The function is used by NOHZ and can only be
1983 * invoked when tick processing is not active.
1984 */
quiet_vmstat(void)1985 void quiet_vmstat(void)
1986 {
1987 if (system_state != SYSTEM_RUNNING)
1988 return;
1989
1990 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
1991 return;
1992
1993 if (!need_update(smp_processor_id()))
1994 return;
1995
1996 /*
1997 * Just refresh counters and do not care about the pending delayed
1998 * vmstat_update. It doesn't fire that often to matter and canceling
1999 * it would be too expensive from this path.
2000 * vmstat_shepherd will take care about that for us.
2001 */
2002 refresh_cpu_vm_stats(false);
2003 }
2004
2005 /*
2006 * Shepherd worker thread that checks the
2007 * differentials of processors that have their worker
2008 * threads for vm statistics updates disabled because of
2009 * inactivity.
2010 */
2011 static void vmstat_shepherd(struct work_struct *w);
2012
2013 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
2014
vmstat_shepherd(struct work_struct * w)2015 static void vmstat_shepherd(struct work_struct *w)
2016 {
2017 int cpu;
2018
2019 cpus_read_lock();
2020 /* Check processors whose vmstat worker threads have been disabled */
2021 for_each_online_cpu(cpu) {
2022 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
2023
2024 if (!delayed_work_pending(dw) && need_update(cpu))
2025 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
2026
2027 cond_resched();
2028 }
2029 cpus_read_unlock();
2030
2031 schedule_delayed_work(&shepherd,
2032 round_jiffies_relative(sysctl_stat_interval));
2033 }
2034
start_shepherd_timer(void)2035 static void __init start_shepherd_timer(void)
2036 {
2037 int cpu;
2038
2039 for_each_possible_cpu(cpu)
2040 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
2041 vmstat_update);
2042
2043 schedule_delayed_work(&shepherd,
2044 round_jiffies_relative(sysctl_stat_interval));
2045 }
2046
init_cpu_node_state(void)2047 static void __init init_cpu_node_state(void)
2048 {
2049 int node;
2050
2051 for_each_online_node(node) {
2052 if (!cpumask_empty(cpumask_of_node(node)))
2053 node_set_state(node, N_CPU);
2054 }
2055 }
2056
vmstat_cpu_online(unsigned int cpu)2057 static int vmstat_cpu_online(unsigned int cpu)
2058 {
2059 refresh_zone_stat_thresholds();
2060
2061 if (!node_state(cpu_to_node(cpu), N_CPU)) {
2062 node_set_state(cpu_to_node(cpu), N_CPU);
2063 set_migration_target_nodes();
2064 }
2065
2066 return 0;
2067 }
2068
vmstat_cpu_down_prep(unsigned int cpu)2069 static int vmstat_cpu_down_prep(unsigned int cpu)
2070 {
2071 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
2072 return 0;
2073 }
2074
vmstat_cpu_dead(unsigned int cpu)2075 static int vmstat_cpu_dead(unsigned int cpu)
2076 {
2077 const struct cpumask *node_cpus;
2078 int node;
2079
2080 node = cpu_to_node(cpu);
2081
2082 refresh_zone_stat_thresholds();
2083 node_cpus = cpumask_of_node(node);
2084 if (!cpumask_empty(node_cpus))
2085 return 0;
2086
2087 node_clear_state(node, N_CPU);
2088 set_migration_target_nodes();
2089
2090 return 0;
2091 }
2092
2093 #endif
2094
2095 struct workqueue_struct *mm_percpu_wq;
2096
init_mm_internals(void)2097 void __init init_mm_internals(void)
2098 {
2099 int ret __maybe_unused;
2100
2101 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0);
2102
2103 #ifdef CONFIG_SMP
2104 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
2105 NULL, vmstat_cpu_dead);
2106 if (ret < 0)
2107 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2108
2109 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
2110 vmstat_cpu_online,
2111 vmstat_cpu_down_prep);
2112 if (ret < 0)
2113 pr_err("vmstat: failed to register 'online' hotplug state\n");
2114
2115 cpus_read_lock();
2116 init_cpu_node_state();
2117 cpus_read_unlock();
2118
2119 start_shepherd_timer();
2120 #endif
2121 migrate_on_reclaim_init();
2122 #ifdef CONFIG_PROC_FS
2123 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
2124 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
2125 proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
2126 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
2127 #endif
2128 }
2129
2130 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2131
2132 /*
2133 * Return an index indicating how much of the available free memory is
2134 * unusable for an allocation of the requested size.
2135 */
unusable_free_index(unsigned int order,struct contig_page_info * info)2136 static int unusable_free_index(unsigned int order,
2137 struct contig_page_info *info)
2138 {
2139 /* No free memory is interpreted as all free memory is unusable */
2140 if (info->free_pages == 0)
2141 return 1000;
2142
2143 /*
2144 * Index should be a value between 0 and 1. Return a value to 3
2145 * decimal places.
2146 *
2147 * 0 => no fragmentation
2148 * 1 => high fragmentation
2149 */
2150 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2151
2152 }
2153
unusable_show_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)2154 static void unusable_show_print(struct seq_file *m,
2155 pg_data_t *pgdat, struct zone *zone)
2156 {
2157 unsigned int order;
2158 int index;
2159 struct contig_page_info info;
2160
2161 seq_printf(m, "Node %d, zone %8s ",
2162 pgdat->node_id,
2163 zone->name);
2164 for (order = 0; order < MAX_ORDER; ++order) {
2165 fill_contig_page_info(zone, order, &info);
2166 index = unusable_free_index(order, &info);
2167 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2168 }
2169
2170 seq_putc(m, '\n');
2171 }
2172
2173 /*
2174 * Display unusable free space index
2175 *
2176 * The unusable free space index measures how much of the available free
2177 * memory cannot be used to satisfy an allocation of a given size and is a
2178 * value between 0 and 1. The higher the value, the more of free memory is
2179 * unusable and by implication, the worse the external fragmentation is. This
2180 * can be expressed as a percentage by multiplying by 100.
2181 */
unusable_show(struct seq_file * m,void * arg)2182 static int unusable_show(struct seq_file *m, void *arg)
2183 {
2184 pg_data_t *pgdat = (pg_data_t *)arg;
2185
2186 /* check memoryless node */
2187 if (!node_state(pgdat->node_id, N_MEMORY))
2188 return 0;
2189
2190 walk_zones_in_node(m, pgdat, true, false, unusable_show_print);
2191
2192 return 0;
2193 }
2194
2195 static const struct seq_operations unusable_sops = {
2196 .start = frag_start,
2197 .next = frag_next,
2198 .stop = frag_stop,
2199 .show = unusable_show,
2200 };
2201
2202 DEFINE_SEQ_ATTRIBUTE(unusable);
2203
extfrag_show_print(struct seq_file * m,pg_data_t * pgdat,struct zone * zone)2204 static void extfrag_show_print(struct seq_file *m,
2205 pg_data_t *pgdat, struct zone *zone)
2206 {
2207 unsigned int order;
2208 int index;
2209
2210 /* Alloc on stack as interrupts are disabled for zone walk */
2211 struct contig_page_info info;
2212
2213 seq_printf(m, "Node %d, zone %8s ",
2214 pgdat->node_id,
2215 zone->name);
2216 for (order = 0; order < MAX_ORDER; ++order) {
2217 fill_contig_page_info(zone, order, &info);
2218 index = __fragmentation_index(order, &info);
2219 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000);
2220 }
2221
2222 seq_putc(m, '\n');
2223 }
2224
2225 /*
2226 * Display fragmentation index for orders that allocations would fail for
2227 */
extfrag_show(struct seq_file * m,void * arg)2228 static int extfrag_show(struct seq_file *m, void *arg)
2229 {
2230 pg_data_t *pgdat = (pg_data_t *)arg;
2231
2232 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print);
2233
2234 return 0;
2235 }
2236
2237 static const struct seq_operations extfrag_sops = {
2238 .start = frag_start,
2239 .next = frag_next,
2240 .stop = frag_stop,
2241 .show = extfrag_show,
2242 };
2243
2244 DEFINE_SEQ_ATTRIBUTE(extfrag);
2245
extfrag_debug_init(void)2246 static int __init extfrag_debug_init(void)
2247 {
2248 struct dentry *extfrag_debug_root;
2249
2250 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
2251
2252 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL,
2253 &unusable_fops);
2254
2255 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL,
2256 &extfrag_fops);
2257
2258 return 0;
2259 }
2260
2261 module_init(extfrag_debug_init);
2262 #endif
2263