1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_MMZONE_H
3 #define _LINUX_MMZONE_H
4 
5 #ifndef __ASSEMBLY__
6 #ifndef __GENERATING_BOUNDS_H
7 
8 #include <linux/spinlock.h>
9 #include <linux/list.h>
10 #include <linux/wait.h>
11 #include <linux/bitops.h>
12 #include <linux/cache.h>
13 #include <linux/threads.h>
14 #include <linux/numa.h>
15 #include <linux/init.h>
16 #include <linux/seqlock.h>
17 #include <linux/nodemask.h>
18 #include <linux/pageblock-flags.h>
19 #include <linux/page-flags-layout.h>
20 #include <linux/atomic.h>
21 #include <linux/mm_types.h>
22 #include <linux/page-flags.h>
23 #include <linux/local_lock.h>
24 #include <asm/page.h>
25 
26 /* Free memory management - zoned buddy allocator.  */
27 #ifndef CONFIG_FORCE_MAX_ZONEORDER
28 #define MAX_ORDER 11
29 #else
30 #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
31 #endif
32 #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
33 
34 /*
35  * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
36  * costly to service.  That is between allocation orders which should
37  * coalesce naturally under reasonable reclaim pressure and those which
38  * will not.
39  */
40 #define PAGE_ALLOC_COSTLY_ORDER 3
41 
42 enum migratetype {
43 	MIGRATE_UNMOVABLE,
44 	MIGRATE_MOVABLE,
45 	MIGRATE_RECLAIMABLE,
46 	MIGRATE_PCPTYPES,	/* the number of types on the pcp lists */
47 	MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
48 #ifdef CONFIG_CMA
49 	/*
50 	 * MIGRATE_CMA migration type is designed to mimic the way
51 	 * ZONE_MOVABLE works.  Only movable pages can be allocated
52 	 * from MIGRATE_CMA pageblocks and page allocator never
53 	 * implicitly change migration type of MIGRATE_CMA pageblock.
54 	 *
55 	 * The way to use it is to change migratetype of a range of
56 	 * pageblocks to MIGRATE_CMA which can be done by
57 	 * __free_pageblock_cma() function.
58 	 */
59 	MIGRATE_CMA,
60 #endif
61 #ifdef CONFIG_MEMORY_ISOLATION
62 	MIGRATE_ISOLATE,	/* can't allocate from here */
63 #endif
64 	MIGRATE_TYPES
65 };
66 
67 /* In mm/page_alloc.c; keep in sync also with show_migration_types() there */
68 extern const char * const migratetype_names[MIGRATE_TYPES];
69 
70 #ifdef CONFIG_CMA
71 #  define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
72 #  define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)
73 #else
74 #  define is_migrate_cma(migratetype) false
75 #  define is_migrate_cma_page(_page) false
76 #endif
77 
is_migrate_movable(int mt)78 static inline bool is_migrate_movable(int mt)
79 {
80 	return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
81 }
82 
83 /*
84  * Check whether a migratetype can be merged with another migratetype.
85  *
86  * It is only mergeable when it can fall back to other migratetypes for
87  * allocation. See fallbacks[MIGRATE_TYPES][3] in page_alloc.c.
88  */
migratetype_is_mergeable(int mt)89 static inline bool migratetype_is_mergeable(int mt)
90 {
91 	return mt < MIGRATE_PCPTYPES;
92 }
93 
94 #define for_each_migratetype_order(order, type) \
95 	for (order = 0; order < MAX_ORDER; order++) \
96 		for (type = 0; type < MIGRATE_TYPES; type++)
97 
98 extern int page_group_by_mobility_disabled;
99 
100 #define MIGRATETYPE_MASK ((1UL << PB_migratetype_bits) - 1)
101 
102 #define get_pageblock_migratetype(page)					\
103 	get_pfnblock_flags_mask(page, page_to_pfn(page), MIGRATETYPE_MASK)
104 
105 struct free_area {
106 	struct list_head	free_list[MIGRATE_TYPES];
107 	unsigned long		nr_free;
108 };
109 
get_page_from_free_area(struct free_area * area,int migratetype)110 static inline struct page *get_page_from_free_area(struct free_area *area,
111 					    int migratetype)
112 {
113 	return list_first_entry_or_null(&area->free_list[migratetype],
114 					struct page, lru);
115 }
116 
free_area_empty(struct free_area * area,int migratetype)117 static inline bool free_area_empty(struct free_area *area, int migratetype)
118 {
119 	return list_empty(&area->free_list[migratetype]);
120 }
121 
122 struct pglist_data;
123 
124 /*
125  * Add a wild amount of padding here to ensure data fall into separate
126  * cachelines.  There are very few zone structures in the machine, so space
127  * consumption is not a concern here.
128  */
129 #if defined(CONFIG_SMP)
130 struct zone_padding {
131 	char x[0];
132 } ____cacheline_internodealigned_in_smp;
133 #define ZONE_PADDING(name)	struct zone_padding name;
134 #else
135 #define ZONE_PADDING(name)
136 #endif
137 
138 #ifdef CONFIG_NUMA
139 enum numa_stat_item {
140 	NUMA_HIT,		/* allocated in intended node */
141 	NUMA_MISS,		/* allocated in non intended node */
142 	NUMA_FOREIGN,		/* was intended here, hit elsewhere */
143 	NUMA_INTERLEAVE_HIT,	/* interleaver preferred this zone */
144 	NUMA_LOCAL,		/* allocation from local node */
145 	NUMA_OTHER,		/* allocation from other node */
146 	NR_VM_NUMA_EVENT_ITEMS
147 };
148 #else
149 #define NR_VM_NUMA_EVENT_ITEMS 0
150 #endif
151 
152 enum zone_stat_item {
153 	/* First 128 byte cacheline (assuming 64 bit words) */
154 	NR_FREE_PAGES,
155 	NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */
156 	NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
157 	NR_ZONE_ACTIVE_ANON,
158 	NR_ZONE_INACTIVE_FILE,
159 	NR_ZONE_ACTIVE_FILE,
160 	NR_ZONE_UNEVICTABLE,
161 	NR_ZONE_WRITE_PENDING,	/* Count of dirty, writeback and unstable pages */
162 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
163 	/* Second 128 byte cacheline */
164 	NR_BOUNCE,
165 #if IS_ENABLED(CONFIG_ZSMALLOC)
166 	NR_ZSPAGES,		/* allocated in zsmalloc */
167 #endif
168 	NR_FREE_CMA_PAGES,
169 	NR_VM_ZONE_STAT_ITEMS };
170 
171 enum node_stat_item {
172 	NR_LRU_BASE,
173 	NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
174 	NR_ACTIVE_ANON,		/*  "     "     "   "       "         */
175 	NR_INACTIVE_FILE,	/*  "     "     "   "       "         */
176 	NR_ACTIVE_FILE,		/*  "     "     "   "       "         */
177 	NR_UNEVICTABLE,		/*  "     "     "   "       "         */
178 	NR_SLAB_RECLAIMABLE_B,
179 	NR_SLAB_UNRECLAIMABLE_B,
180 	NR_ISOLATED_ANON,	/* Temporary isolated pages from anon lru */
181 	NR_ISOLATED_FILE,	/* Temporary isolated pages from file lru */
182 	WORKINGSET_NODES,
183 	WORKINGSET_REFAULT_BASE,
184 	WORKINGSET_REFAULT_ANON = WORKINGSET_REFAULT_BASE,
185 	WORKINGSET_REFAULT_FILE,
186 	WORKINGSET_ACTIVATE_BASE,
187 	WORKINGSET_ACTIVATE_ANON = WORKINGSET_ACTIVATE_BASE,
188 	WORKINGSET_ACTIVATE_FILE,
189 	WORKINGSET_RESTORE_BASE,
190 	WORKINGSET_RESTORE_ANON = WORKINGSET_RESTORE_BASE,
191 	WORKINGSET_RESTORE_FILE,
192 	WORKINGSET_NODERECLAIM,
193 	NR_ANON_MAPPED,	/* Mapped anonymous pages */
194 	NR_FILE_MAPPED,	/* pagecache pages mapped into pagetables.
195 			   only modified from process context */
196 	NR_FILE_PAGES,
197 	NR_FILE_DIRTY,
198 	NR_WRITEBACK,
199 	NR_WRITEBACK_TEMP,	/* Writeback using temporary buffers */
200 	NR_SHMEM,		/* shmem pages (included tmpfs/GEM pages) */
201 	NR_SHMEM_THPS,
202 	NR_SHMEM_PMDMAPPED,
203 	NR_FILE_THPS,
204 	NR_FILE_PMDMAPPED,
205 	NR_ANON_THPS,
206 	NR_VMSCAN_WRITE,
207 	NR_VMSCAN_IMMEDIATE,	/* Prioritise for reclaim when writeback ends */
208 	NR_DIRTIED,		/* page dirtyings since bootup */
209 	NR_WRITTEN,		/* page writings since bootup */
210 	NR_THROTTLED_WRITTEN,	/* NR_WRITTEN while reclaim throttled */
211 	NR_KERNEL_MISC_RECLAIMABLE,	/* reclaimable non-slab kernel pages */
212 	NR_FOLL_PIN_ACQUIRED,	/* via: pin_user_page(), gup flag: FOLL_PIN */
213 	NR_FOLL_PIN_RELEASED,	/* pages returned via unpin_user_page() */
214 	NR_KERNEL_STACK_KB,	/* measured in KiB */
215 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
216 	NR_KERNEL_SCS_KB,	/* measured in KiB */
217 #endif
218 	NR_PAGETABLE,		/* used for pagetables */
219 #ifdef CONFIG_SWAP
220 	NR_SWAPCACHE,
221 #endif
222 #ifdef CONFIG_NUMA_BALANCING
223 	PGPROMOTE_SUCCESS,	/* promote successfully */
224 #endif
225 	NR_VM_NODE_STAT_ITEMS
226 };
227 
228 /*
229  * Returns true if the item should be printed in THPs (/proc/vmstat
230  * currently prints number of anon, file and shmem THPs. But the item
231  * is charged in pages).
232  */
vmstat_item_print_in_thp(enum node_stat_item item)233 static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item)
234 {
235 	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
236 		return false;
237 
238 	return item == NR_ANON_THPS ||
239 	       item == NR_FILE_THPS ||
240 	       item == NR_SHMEM_THPS ||
241 	       item == NR_SHMEM_PMDMAPPED ||
242 	       item == NR_FILE_PMDMAPPED;
243 }
244 
245 /*
246  * Returns true if the value is measured in bytes (most vmstat values are
247  * measured in pages). This defines the API part, the internal representation
248  * might be different.
249  */
vmstat_item_in_bytes(int idx)250 static __always_inline bool vmstat_item_in_bytes(int idx)
251 {
252 	/*
253 	 * Global and per-node slab counters track slab pages.
254 	 * It's expected that changes are multiples of PAGE_SIZE.
255 	 * Internally values are stored in pages.
256 	 *
257 	 * Per-memcg and per-lruvec counters track memory, consumed
258 	 * by individual slab objects. These counters are actually
259 	 * byte-precise.
260 	 */
261 	return (idx == NR_SLAB_RECLAIMABLE_B ||
262 		idx == NR_SLAB_UNRECLAIMABLE_B);
263 }
264 
265 /*
266  * We do arithmetic on the LRU lists in various places in the code,
267  * so it is important to keep the active lists LRU_ACTIVE higher in
268  * the array than the corresponding inactive lists, and to keep
269  * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
270  *
271  * This has to be kept in sync with the statistics in zone_stat_item
272  * above and the descriptions in vmstat_text in mm/vmstat.c
273  */
274 #define LRU_BASE 0
275 #define LRU_ACTIVE 1
276 #define LRU_FILE 2
277 
278 enum lru_list {
279 	LRU_INACTIVE_ANON = LRU_BASE,
280 	LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
281 	LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
282 	LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
283 	LRU_UNEVICTABLE,
284 	NR_LRU_LISTS
285 };
286 
287 enum vmscan_throttle_state {
288 	VMSCAN_THROTTLE_WRITEBACK,
289 	VMSCAN_THROTTLE_ISOLATED,
290 	VMSCAN_THROTTLE_NOPROGRESS,
291 	VMSCAN_THROTTLE_CONGESTED,
292 	NR_VMSCAN_THROTTLE,
293 };
294 
295 #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
296 
297 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
298 
is_file_lru(enum lru_list lru)299 static inline bool is_file_lru(enum lru_list lru)
300 {
301 	return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
302 }
303 
is_active_lru(enum lru_list lru)304 static inline bool is_active_lru(enum lru_list lru)
305 {
306 	return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
307 }
308 
309 #define ANON_AND_FILE 2
310 
311 enum lruvec_flags {
312 	LRUVEC_CONGESTED,		/* lruvec has many dirty pages
313 					 * backed by a congested BDI
314 					 */
315 };
316 
317 struct lruvec {
318 	struct list_head		lists[NR_LRU_LISTS];
319 	/* per lruvec lru_lock for memcg */
320 	spinlock_t			lru_lock;
321 	/*
322 	 * These track the cost of reclaiming one LRU - file or anon -
323 	 * over the other. As the observed cost of reclaiming one LRU
324 	 * increases, the reclaim scan balance tips toward the other.
325 	 */
326 	unsigned long			anon_cost;
327 	unsigned long			file_cost;
328 	/* Non-resident age, driven by LRU movement */
329 	atomic_long_t			nonresident_age;
330 	/* Refaults at the time of last reclaim cycle */
331 	unsigned long			refaults[ANON_AND_FILE];
332 	/* Various lruvec state flags (enum lruvec_flags) */
333 	unsigned long			flags;
334 #ifdef CONFIG_MEMCG
335 	struct pglist_data *pgdat;
336 #endif
337 };
338 
339 /* Isolate unmapped pages */
340 #define ISOLATE_UNMAPPED	((__force isolate_mode_t)0x2)
341 /* Isolate for asynchronous migration */
342 #define ISOLATE_ASYNC_MIGRATE	((__force isolate_mode_t)0x4)
343 /* Isolate unevictable pages */
344 #define ISOLATE_UNEVICTABLE	((__force isolate_mode_t)0x8)
345 
346 /* LRU Isolation modes. */
347 typedef unsigned __bitwise isolate_mode_t;
348 
349 enum zone_watermarks {
350 	WMARK_MIN,
351 	WMARK_LOW,
352 	WMARK_HIGH,
353 	WMARK_PROMO,
354 	NR_WMARK
355 };
356 
357 /*
358  * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER plus one additional
359  * for pageblock size for THP if configured.
360  */
361 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
362 #define NR_PCP_THP 1
363 #else
364 #define NR_PCP_THP 0
365 #endif
366 #define NR_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1 + NR_PCP_THP))
367 
368 /*
369  * Shift to encode migratetype and order in the same integer, with order
370  * in the least significant bits.
371  */
372 #define NR_PCP_ORDER_WIDTH 8
373 #define NR_PCP_ORDER_MASK ((1<<NR_PCP_ORDER_WIDTH) - 1)
374 
375 #define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
376 #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
377 #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
378 #define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)
379 
380 /* Fields and list protected by pagesets local_lock in page_alloc.c */
381 struct per_cpu_pages {
382 	int count;		/* number of pages in the list */
383 	int high;		/* high watermark, emptying needed */
384 	int batch;		/* chunk size for buddy add/remove */
385 	short free_factor;	/* batch scaling factor during free */
386 #ifdef CONFIG_NUMA
387 	short expire;		/* When 0, remote pagesets are drained */
388 #endif
389 
390 	/* Lists of pages, one per migrate type stored on the pcp-lists */
391 	struct list_head lists[NR_PCP_LISTS];
392 };
393 
394 struct per_cpu_zonestat {
395 #ifdef CONFIG_SMP
396 	s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
397 	s8 stat_threshold;
398 #endif
399 #ifdef CONFIG_NUMA
400 	/*
401 	 * Low priority inaccurate counters that are only folded
402 	 * on demand. Use a large type to avoid the overhead of
403 	 * folding during refresh_cpu_vm_stats.
404 	 */
405 	unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
406 #endif
407 };
408 
409 struct per_cpu_nodestat {
410 	s8 stat_threshold;
411 	s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];
412 };
413 
414 #endif /* !__GENERATING_BOUNDS.H */
415 
416 enum zone_type {
417 	/*
418 	 * ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able
419 	 * to DMA to all of the addressable memory (ZONE_NORMAL).
420 	 * On architectures where this area covers the whole 32 bit address
421 	 * space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller
422 	 * DMA addressing constraints. This distinction is important as a 32bit
423 	 * DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit
424 	 * platforms may need both zones as they support peripherals with
425 	 * different DMA addressing limitations.
426 	 */
427 #ifdef CONFIG_ZONE_DMA
428 	ZONE_DMA,
429 #endif
430 #ifdef CONFIG_ZONE_DMA32
431 	ZONE_DMA32,
432 #endif
433 	/*
434 	 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
435 	 * performed on pages in ZONE_NORMAL if the DMA devices support
436 	 * transfers to all addressable memory.
437 	 */
438 	ZONE_NORMAL,
439 #ifdef CONFIG_HIGHMEM
440 	/*
441 	 * A memory area that is only addressable by the kernel through
442 	 * mapping portions into its own address space. This is for example
443 	 * used by i386 to allow the kernel to address the memory beyond
444 	 * 900MB. The kernel will set up special mappings (page
445 	 * table entries on i386) for each page that the kernel needs to
446 	 * access.
447 	 */
448 	ZONE_HIGHMEM,
449 #endif
450 	/*
451 	 * ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains
452 	 * movable pages with few exceptional cases described below. Main use
453 	 * cases for ZONE_MOVABLE are to make memory offlining/unplug more
454 	 * likely to succeed, and to locally limit unmovable allocations - e.g.,
455 	 * to increase the number of THP/huge pages. Notable special cases are:
456 	 *
457 	 * 1. Pinned pages: (long-term) pinning of movable pages might
458 	 *    essentially turn such pages unmovable. Therefore, we do not allow
459 	 *    pinning long-term pages in ZONE_MOVABLE. When pages are pinned and
460 	 *    faulted, they come from the right zone right away. However, it is
461 	 *    still possible that address space already has pages in
462 	 *    ZONE_MOVABLE at the time when pages are pinned (i.e. user has
463 	 *    touches that memory before pinning). In such case we migrate them
464 	 *    to a different zone. When migration fails - pinning fails.
465 	 * 2. memblock allocations: kernelcore/movablecore setups might create
466 	 *    situations where ZONE_MOVABLE contains unmovable allocations
467 	 *    after boot. Memory offlining and allocations fail early.
468 	 * 3. Memory holes: kernelcore/movablecore setups might create very rare
469 	 *    situations where ZONE_MOVABLE contains memory holes after boot,
470 	 *    for example, if we have sections that are only partially
471 	 *    populated. Memory offlining and allocations fail early.
472 	 * 4. PG_hwpoison pages: while poisoned pages can be skipped during
473 	 *    memory offlining, such pages cannot be allocated.
474 	 * 5. Unmovable PG_offline pages: in paravirtualized environments,
475 	 *    hotplugged memory blocks might only partially be managed by the
476 	 *    buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The
477 	 *    parts not manged by the buddy are unmovable PG_offline pages. In
478 	 *    some cases (virtio-mem), such pages can be skipped during
479 	 *    memory offlining, however, cannot be moved/allocated. These
480 	 *    techniques might use alloc_contig_range() to hide previously
481 	 *    exposed pages from the buddy again (e.g., to implement some sort
482 	 *    of memory unplug in virtio-mem).
483 	 * 6. ZERO_PAGE(0), kernelcore/movablecore setups might create
484 	 *    situations where ZERO_PAGE(0) which is allocated differently
485 	 *    on different platforms may end up in a movable zone. ZERO_PAGE(0)
486 	 *    cannot be migrated.
487 	 * 7. Memory-hotplug: when using memmap_on_memory and onlining the
488 	 *    memory to the MOVABLE zone, the vmemmap pages are also placed in
489 	 *    such zone. Such pages cannot be really moved around as they are
490 	 *    self-stored in the range, but they are treated as movable when
491 	 *    the range they describe is about to be offlined.
492 	 *
493 	 * In general, no unmovable allocations that degrade memory offlining
494 	 * should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range())
495 	 * have to expect that migrating pages in ZONE_MOVABLE can fail (even
496 	 * if has_unmovable_pages() states that there are no unmovable pages,
497 	 * there can be false negatives).
498 	 */
499 	ZONE_MOVABLE,
500 #ifdef CONFIG_ZONE_DEVICE
501 	ZONE_DEVICE,
502 #endif
503 	__MAX_NR_ZONES
504 
505 };
506 
507 #ifndef __GENERATING_BOUNDS_H
508 
509 #define ASYNC_AND_SYNC 2
510 
511 struct zone {
512 	/* Read-mostly fields */
513 
514 	/* zone watermarks, access with *_wmark_pages(zone) macros */
515 	unsigned long _watermark[NR_WMARK];
516 	unsigned long watermark_boost;
517 
518 	unsigned long nr_reserved_highatomic;
519 
520 	/*
521 	 * We don't know if the memory that we're going to allocate will be
522 	 * freeable or/and it will be released eventually, so to avoid totally
523 	 * wasting several GB of ram we must reserve some of the lower zone
524 	 * memory (otherwise we risk to run OOM on the lower zones despite
525 	 * there being tons of freeable ram on the higher zones).  This array is
526 	 * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl
527 	 * changes.
528 	 */
529 	long lowmem_reserve[MAX_NR_ZONES];
530 
531 #ifdef CONFIG_NUMA
532 	int node;
533 #endif
534 	struct pglist_data	*zone_pgdat;
535 	struct per_cpu_pages	__percpu *per_cpu_pageset;
536 	struct per_cpu_zonestat	__percpu *per_cpu_zonestats;
537 	/*
538 	 * the high and batch values are copied to individual pagesets for
539 	 * faster access
540 	 */
541 	int pageset_high;
542 	int pageset_batch;
543 
544 #ifndef CONFIG_SPARSEMEM
545 	/*
546 	 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
547 	 * In SPARSEMEM, this map is stored in struct mem_section
548 	 */
549 	unsigned long		*pageblock_flags;
550 #endif /* CONFIG_SPARSEMEM */
551 
552 	/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
553 	unsigned long		zone_start_pfn;
554 
555 	/*
556 	 * spanned_pages is the total pages spanned by the zone, including
557 	 * holes, which is calculated as:
558 	 * 	spanned_pages = zone_end_pfn - zone_start_pfn;
559 	 *
560 	 * present_pages is physical pages existing within the zone, which
561 	 * is calculated as:
562 	 *	present_pages = spanned_pages - absent_pages(pages in holes);
563 	 *
564 	 * present_early_pages is present pages existing within the zone
565 	 * located on memory available since early boot, excluding hotplugged
566 	 * memory.
567 	 *
568 	 * managed_pages is present pages managed by the buddy system, which
569 	 * is calculated as (reserved_pages includes pages allocated by the
570 	 * bootmem allocator):
571 	 *	managed_pages = present_pages - reserved_pages;
572 	 *
573 	 * cma pages is present pages that are assigned for CMA use
574 	 * (MIGRATE_CMA).
575 	 *
576 	 * So present_pages may be used by memory hotplug or memory power
577 	 * management logic to figure out unmanaged pages by checking
578 	 * (present_pages - managed_pages). And managed_pages should be used
579 	 * by page allocator and vm scanner to calculate all kinds of watermarks
580 	 * and thresholds.
581 	 *
582 	 * Locking rules:
583 	 *
584 	 * zone_start_pfn and spanned_pages are protected by span_seqlock.
585 	 * It is a seqlock because it has to be read outside of zone->lock,
586 	 * and it is done in the main allocator path.  But, it is written
587 	 * quite infrequently.
588 	 *
589 	 * The span_seq lock is declared along with zone->lock because it is
590 	 * frequently read in proximity to zone->lock.  It's good to
591 	 * give them a chance of being in the same cacheline.
592 	 *
593 	 * Write access to present_pages at runtime should be protected by
594 	 * mem_hotplug_begin/end(). Any reader who can't tolerant drift of
595 	 * present_pages should get_online_mems() to get a stable value.
596 	 */
597 	atomic_long_t		managed_pages;
598 	unsigned long		spanned_pages;
599 	unsigned long		present_pages;
600 #if defined(CONFIG_MEMORY_HOTPLUG)
601 	unsigned long		present_early_pages;
602 #endif
603 #ifdef CONFIG_CMA
604 	unsigned long		cma_pages;
605 #endif
606 
607 	const char		*name;
608 
609 #ifdef CONFIG_MEMORY_ISOLATION
610 	/*
611 	 * Number of isolated pageblock. It is used to solve incorrect
612 	 * freepage counting problem due to racy retrieving migratetype
613 	 * of pageblock. Protected by zone->lock.
614 	 */
615 	unsigned long		nr_isolate_pageblock;
616 #endif
617 
618 #ifdef CONFIG_MEMORY_HOTPLUG
619 	/* see spanned/present_pages for more description */
620 	seqlock_t		span_seqlock;
621 #endif
622 
623 	int initialized;
624 
625 	/* Write-intensive fields used from the page allocator */
626 	ZONE_PADDING(_pad1_)
627 
628 	/* free areas of different sizes */
629 	struct free_area	free_area[MAX_ORDER];
630 
631 	/* zone flags, see below */
632 	unsigned long		flags;
633 
634 	/* Primarily protects free_area */
635 	spinlock_t		lock;
636 
637 	/* Write-intensive fields used by compaction and vmstats. */
638 	ZONE_PADDING(_pad2_)
639 
640 	/*
641 	 * When free pages are below this point, additional steps are taken
642 	 * when reading the number of free pages to avoid per-cpu counter
643 	 * drift allowing watermarks to be breached
644 	 */
645 	unsigned long percpu_drift_mark;
646 
647 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
648 	/* pfn where compaction free scanner should start */
649 	unsigned long		compact_cached_free_pfn;
650 	/* pfn where compaction migration scanner should start */
651 	unsigned long		compact_cached_migrate_pfn[ASYNC_AND_SYNC];
652 	unsigned long		compact_init_migrate_pfn;
653 	unsigned long		compact_init_free_pfn;
654 #endif
655 
656 #ifdef CONFIG_COMPACTION
657 	/*
658 	 * On compaction failure, 1<<compact_defer_shift compactions
659 	 * are skipped before trying again. The number attempted since
660 	 * last failure is tracked with compact_considered.
661 	 * compact_order_failed is the minimum compaction failed order.
662 	 */
663 	unsigned int		compact_considered;
664 	unsigned int		compact_defer_shift;
665 	int			compact_order_failed;
666 #endif
667 
668 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
669 	/* Set to true when the PG_migrate_skip bits should be cleared */
670 	bool			compact_blockskip_flush;
671 #endif
672 
673 	bool			contiguous;
674 
675 	ZONE_PADDING(_pad3_)
676 	/* Zone statistics */
677 	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
678 	atomic_long_t		vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
679 } ____cacheline_internodealigned_in_smp;
680 
681 enum pgdat_flags {
682 	PGDAT_DIRTY,			/* reclaim scanning has recently found
683 					 * many dirty file pages at the tail
684 					 * of the LRU.
685 					 */
686 	PGDAT_WRITEBACK,		/* reclaim scanning has recently found
687 					 * many pages under writeback
688 					 */
689 	PGDAT_RECLAIM_LOCKED,		/* prevents concurrent reclaim */
690 };
691 
692 enum zone_flags {
693 	ZONE_BOOSTED_WATERMARK,		/* zone recently boosted watermarks.
694 					 * Cleared when kswapd is woken.
695 					 */
696 	ZONE_RECLAIM_ACTIVE,		/* kswapd may be scanning the zone. */
697 };
698 
zone_managed_pages(struct zone * zone)699 static inline unsigned long zone_managed_pages(struct zone *zone)
700 {
701 	return (unsigned long)atomic_long_read(&zone->managed_pages);
702 }
703 
zone_cma_pages(struct zone * zone)704 static inline unsigned long zone_cma_pages(struct zone *zone)
705 {
706 #ifdef CONFIG_CMA
707 	return zone->cma_pages;
708 #else
709 	return 0;
710 #endif
711 }
712 
zone_end_pfn(const struct zone * zone)713 static inline unsigned long zone_end_pfn(const struct zone *zone)
714 {
715 	return zone->zone_start_pfn + zone->spanned_pages;
716 }
717 
zone_spans_pfn(const struct zone * zone,unsigned long pfn)718 static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
719 {
720 	return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
721 }
722 
zone_is_initialized(struct zone * zone)723 static inline bool zone_is_initialized(struct zone *zone)
724 {
725 	return zone->initialized;
726 }
727 
zone_is_empty(struct zone * zone)728 static inline bool zone_is_empty(struct zone *zone)
729 {
730 	return zone->spanned_pages == 0;
731 }
732 
733 /*
734  * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty
735  * intersection with the given zone
736  */
zone_intersects(struct zone * zone,unsigned long start_pfn,unsigned long nr_pages)737 static inline bool zone_intersects(struct zone *zone,
738 		unsigned long start_pfn, unsigned long nr_pages)
739 {
740 	if (zone_is_empty(zone))
741 		return false;
742 	if (start_pfn >= zone_end_pfn(zone) ||
743 	    start_pfn + nr_pages <= zone->zone_start_pfn)
744 		return false;
745 
746 	return true;
747 }
748 
749 /*
750  * The "priority" of VM scanning is how much of the queues we will scan in one
751  * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
752  * queues ("queue_length >> 12") during an aging round.
753  */
754 #define DEF_PRIORITY 12
755 
756 /* Maximum number of zones on a zonelist */
757 #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
758 
759 enum {
760 	ZONELIST_FALLBACK,	/* zonelist with fallback */
761 #ifdef CONFIG_NUMA
762 	/*
763 	 * The NUMA zonelists are doubled because we need zonelists that
764 	 * restrict the allocations to a single node for __GFP_THISNODE.
765 	 */
766 	ZONELIST_NOFALLBACK,	/* zonelist without fallback (__GFP_THISNODE) */
767 #endif
768 	MAX_ZONELISTS
769 };
770 
771 /*
772  * This struct contains information about a zone in a zonelist. It is stored
773  * here to avoid dereferences into large structures and lookups of tables
774  */
775 struct zoneref {
776 	struct zone *zone;	/* Pointer to actual zone */
777 	int zone_idx;		/* zone_idx(zoneref->zone) */
778 };
779 
780 /*
781  * One allocation request operates on a zonelist. A zonelist
782  * is a list of zones, the first one is the 'goal' of the
783  * allocation, the other zones are fallback zones, in decreasing
784  * priority.
785  *
786  * To speed the reading of the zonelist, the zonerefs contain the zone index
787  * of the entry being read. Helper functions to access information given
788  * a struct zoneref are
789  *
790  * zonelist_zone()	- Return the struct zone * for an entry in _zonerefs
791  * zonelist_zone_idx()	- Return the index of the zone for an entry
792  * zonelist_node_idx()	- Return the index of the node for an entry
793  */
794 struct zonelist {
795 	struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
796 };
797 
798 /*
799  * The array of struct pages for flatmem.
800  * It must be declared for SPARSEMEM as well because there are configurations
801  * that rely on that.
802  */
803 extern struct page *mem_map;
804 
805 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
806 struct deferred_split {
807 	spinlock_t split_queue_lock;
808 	struct list_head split_queue;
809 	unsigned long split_queue_len;
810 };
811 #endif
812 
813 /*
814  * On NUMA machines, each NUMA node would have a pg_data_t to describe
815  * it's memory layout. On UMA machines there is a single pglist_data which
816  * describes the whole memory.
817  *
818  * Memory statistics and page replacement data structures are maintained on a
819  * per-zone basis.
820  */
821 typedef struct pglist_data {
822 	/*
823 	 * node_zones contains just the zones for THIS node. Not all of the
824 	 * zones may be populated, but it is the full list. It is referenced by
825 	 * this node's node_zonelists as well as other node's node_zonelists.
826 	 */
827 	struct zone node_zones[MAX_NR_ZONES];
828 
829 	/*
830 	 * node_zonelists contains references to all zones in all nodes.
831 	 * Generally the first zones will be references to this node's
832 	 * node_zones.
833 	 */
834 	struct zonelist node_zonelists[MAX_ZONELISTS];
835 
836 	int nr_zones; /* number of populated zones in this node */
837 #ifdef CONFIG_FLATMEM	/* means !SPARSEMEM */
838 	struct page *node_mem_map;
839 #ifdef CONFIG_PAGE_EXTENSION
840 	struct page_ext *node_page_ext;
841 #endif
842 #endif
843 #if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
844 	/*
845 	 * Must be held any time you expect node_start_pfn,
846 	 * node_present_pages, node_spanned_pages or nr_zones to stay constant.
847 	 * Also synchronizes pgdat->first_deferred_pfn during deferred page
848 	 * init.
849 	 *
850 	 * pgdat_resize_lock() and pgdat_resize_unlock() are provided to
851 	 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG
852 	 * or CONFIG_DEFERRED_STRUCT_PAGE_INIT.
853 	 *
854 	 * Nests above zone->lock and zone->span_seqlock
855 	 */
856 	spinlock_t node_size_lock;
857 #endif
858 	unsigned long node_start_pfn;
859 	unsigned long node_present_pages; /* total number of physical pages */
860 	unsigned long node_spanned_pages; /* total size of physical page
861 					     range, including holes */
862 	int node_id;
863 	wait_queue_head_t kswapd_wait;
864 	wait_queue_head_t pfmemalloc_wait;
865 
866 	/* workqueues for throttling reclaim for different reasons. */
867 	wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE];
868 
869 	atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */
870 	unsigned long nr_reclaim_start;	/* nr pages written while throttled
871 					 * when throttling started. */
872 	struct task_struct *kswapd;	/* Protected by
873 					   mem_hotplug_begin/end() */
874 	int kswapd_order;
875 	enum zone_type kswapd_highest_zoneidx;
876 
877 	int kswapd_failures;		/* Number of 'reclaimed == 0' runs */
878 
879 #ifdef CONFIG_COMPACTION
880 	int kcompactd_max_order;
881 	enum zone_type kcompactd_highest_zoneidx;
882 	wait_queue_head_t kcompactd_wait;
883 	struct task_struct *kcompactd;
884 	bool proactive_compact_trigger;
885 #endif
886 	/*
887 	 * This is a per-node reserve of pages that are not available
888 	 * to userspace allocations.
889 	 */
890 	unsigned long		totalreserve_pages;
891 
892 #ifdef CONFIG_NUMA
893 	/*
894 	 * node reclaim becomes active if more unmapped pages exist.
895 	 */
896 	unsigned long		min_unmapped_pages;
897 	unsigned long		min_slab_pages;
898 #endif /* CONFIG_NUMA */
899 
900 	/* Write-intensive fields used by page reclaim */
901 	ZONE_PADDING(_pad1_)
902 
903 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
904 	/*
905 	 * If memory initialisation on large machines is deferred then this
906 	 * is the first PFN that needs to be initialised.
907 	 */
908 	unsigned long first_deferred_pfn;
909 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
910 
911 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
912 	struct deferred_split deferred_split_queue;
913 #endif
914 
915 	/* Fields commonly accessed by the page reclaim scanner */
916 
917 	/*
918 	 * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
919 	 *
920 	 * Use mem_cgroup_lruvec() to look up lruvecs.
921 	 */
922 	struct lruvec		__lruvec;
923 
924 	unsigned long		flags;
925 
926 	ZONE_PADDING(_pad2_)
927 
928 	/* Per-node vmstats */
929 	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
930 	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
931 } pg_data_t;
932 
933 #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
934 #define node_spanned_pages(nid)	(NODE_DATA(nid)->node_spanned_pages)
935 
936 #define node_start_pfn(nid)	(NODE_DATA(nid)->node_start_pfn)
937 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
938 
pgdat_end_pfn(pg_data_t * pgdat)939 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
940 {
941 	return pgdat->node_start_pfn + pgdat->node_spanned_pages;
942 }
943 
pgdat_is_empty(pg_data_t * pgdat)944 static inline bool pgdat_is_empty(pg_data_t *pgdat)
945 {
946 	return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
947 }
948 
949 #include <linux/memory_hotplug.h>
950 
951 void build_all_zonelists(pg_data_t *pgdat);
952 void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order,
953 		   enum zone_type highest_zoneidx);
954 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
955 			 int highest_zoneidx, unsigned int alloc_flags,
956 			 long free_pages);
957 bool zone_watermark_ok(struct zone *z, unsigned int order,
958 		unsigned long mark, int highest_zoneidx,
959 		unsigned int alloc_flags);
960 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
961 		unsigned long mark, int highest_zoneidx);
962 /*
963  * Memory initialization context, use to differentiate memory added by
964  * the platform statically or via memory hotplug interface.
965  */
966 enum meminit_context {
967 	MEMINIT_EARLY,
968 	MEMINIT_HOTPLUG,
969 };
970 
971 extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
972 				     unsigned long size);
973 
974 extern void lruvec_init(struct lruvec *lruvec);
975 
lruvec_pgdat(struct lruvec * lruvec)976 static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
977 {
978 #ifdef CONFIG_MEMCG
979 	return lruvec->pgdat;
980 #else
981 	return container_of(lruvec, struct pglist_data, __lruvec);
982 #endif
983 }
984 
985 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
986 int local_memory_node(int node_id);
987 #else
local_memory_node(int node_id)988 static inline int local_memory_node(int node_id) { return node_id; };
989 #endif
990 
991 /*
992  * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
993  */
994 #define zone_idx(zone)		((zone) - (zone)->zone_pgdat->node_zones)
995 
996 #ifdef CONFIG_ZONE_DEVICE
zone_is_zone_device(struct zone * zone)997 static inline bool zone_is_zone_device(struct zone *zone)
998 {
999 	return zone_idx(zone) == ZONE_DEVICE;
1000 }
1001 #else
zone_is_zone_device(struct zone * zone)1002 static inline bool zone_is_zone_device(struct zone *zone)
1003 {
1004 	return false;
1005 }
1006 #endif
1007 
1008 /*
1009  * Returns true if a zone has pages managed by the buddy allocator.
1010  * All the reclaim decisions have to use this function rather than
1011  * populated_zone(). If the whole zone is reserved then we can easily
1012  * end up with populated_zone() && !managed_zone().
1013  */
managed_zone(struct zone * zone)1014 static inline bool managed_zone(struct zone *zone)
1015 {
1016 	return zone_managed_pages(zone);
1017 }
1018 
1019 /* Returns true if a zone has memory */
populated_zone(struct zone * zone)1020 static inline bool populated_zone(struct zone *zone)
1021 {
1022 	return zone->present_pages;
1023 }
1024 
1025 #ifdef CONFIG_NUMA
zone_to_nid(struct zone * zone)1026 static inline int zone_to_nid(struct zone *zone)
1027 {
1028 	return zone->node;
1029 }
1030 
zone_set_nid(struct zone * zone,int nid)1031 static inline void zone_set_nid(struct zone *zone, int nid)
1032 {
1033 	zone->node = nid;
1034 }
1035 #else
zone_to_nid(struct zone * zone)1036 static inline int zone_to_nid(struct zone *zone)
1037 {
1038 	return 0;
1039 }
1040 
zone_set_nid(struct zone * zone,int nid)1041 static inline void zone_set_nid(struct zone *zone, int nid) {}
1042 #endif
1043 
1044 extern int movable_zone;
1045 
is_highmem_idx(enum zone_type idx)1046 static inline int is_highmem_idx(enum zone_type idx)
1047 {
1048 #ifdef CONFIG_HIGHMEM
1049 	return (idx == ZONE_HIGHMEM ||
1050 		(idx == ZONE_MOVABLE && movable_zone == ZONE_HIGHMEM));
1051 #else
1052 	return 0;
1053 #endif
1054 }
1055 
1056 #ifdef CONFIG_ZONE_DMA
1057 bool has_managed_dma(void);
1058 #else
has_managed_dma(void)1059 static inline bool has_managed_dma(void)
1060 {
1061 	return false;
1062 }
1063 #endif
1064 
1065 /**
1066  * is_highmem - helper function to quickly check if a struct zone is a
1067  *              highmem zone or not.  This is an attempt to keep references
1068  *              to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
1069  * @zone: pointer to struct zone variable
1070  * Return: 1 for a highmem zone, 0 otherwise
1071  */
is_highmem(struct zone * zone)1072 static inline int is_highmem(struct zone *zone)
1073 {
1074 #ifdef CONFIG_HIGHMEM
1075 	return is_highmem_idx(zone_idx(zone));
1076 #else
1077 	return 0;
1078 #endif
1079 }
1080 
1081 /* These two functions are used to setup the per zone pages min values */
1082 struct ctl_table;
1083 
1084 int min_free_kbytes_sysctl_handler(struct ctl_table *, int, void *, size_t *,
1085 		loff_t *);
1086 int watermark_scale_factor_sysctl_handler(struct ctl_table *, int, void *,
1087 		size_t *, loff_t *);
1088 extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES];
1089 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, void *,
1090 		size_t *, loff_t *);
1091 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *, int,
1092 		void *, size_t *, loff_t *);
1093 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
1094 		void *, size_t *, loff_t *);
1095 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
1096 		void *, size_t *, loff_t *);
1097 int numa_zonelist_order_handler(struct ctl_table *, int,
1098 		void *, size_t *, loff_t *);
1099 extern int percpu_pagelist_high_fraction;
1100 extern char numa_zonelist_order[];
1101 #define NUMA_ZONELIST_ORDER_LEN	16
1102 
1103 #ifndef CONFIG_NUMA
1104 
1105 extern struct pglist_data contig_page_data;
NODE_DATA(int nid)1106 static inline struct pglist_data *NODE_DATA(int nid)
1107 {
1108 	return &contig_page_data;
1109 }
1110 
1111 #else /* CONFIG_NUMA */
1112 
1113 #include <asm/mmzone.h>
1114 
1115 #endif /* !CONFIG_NUMA */
1116 
1117 extern struct pglist_data *first_online_pgdat(void);
1118 extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
1119 extern struct zone *next_zone(struct zone *zone);
1120 
1121 /**
1122  * for_each_online_pgdat - helper macro to iterate over all online nodes
1123  * @pgdat: pointer to a pg_data_t variable
1124  */
1125 #define for_each_online_pgdat(pgdat)			\
1126 	for (pgdat = first_online_pgdat();		\
1127 	     pgdat;					\
1128 	     pgdat = next_online_pgdat(pgdat))
1129 /**
1130  * for_each_zone - helper macro to iterate over all memory zones
1131  * @zone: pointer to struct zone variable
1132  *
1133  * The user only needs to declare the zone variable, for_each_zone
1134  * fills it in.
1135  */
1136 #define for_each_zone(zone)			        \
1137 	for (zone = (first_online_pgdat())->node_zones; \
1138 	     zone;					\
1139 	     zone = next_zone(zone))
1140 
1141 #define for_each_populated_zone(zone)		        \
1142 	for (zone = (first_online_pgdat())->node_zones; \
1143 	     zone;					\
1144 	     zone = next_zone(zone))			\
1145 		if (!populated_zone(zone))		\
1146 			; /* do nothing */		\
1147 		else
1148 
zonelist_zone(struct zoneref * zoneref)1149 static inline struct zone *zonelist_zone(struct zoneref *zoneref)
1150 {
1151 	return zoneref->zone;
1152 }
1153 
zonelist_zone_idx(struct zoneref * zoneref)1154 static inline int zonelist_zone_idx(struct zoneref *zoneref)
1155 {
1156 	return zoneref->zone_idx;
1157 }
1158 
zonelist_node_idx(struct zoneref * zoneref)1159 static inline int zonelist_node_idx(struct zoneref *zoneref)
1160 {
1161 	return zone_to_nid(zoneref->zone);
1162 }
1163 
1164 struct zoneref *__next_zones_zonelist(struct zoneref *z,
1165 					enum zone_type highest_zoneidx,
1166 					nodemask_t *nodes);
1167 
1168 /**
1169  * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
1170  * @z: The cursor used as a starting point for the search
1171  * @highest_zoneidx: The zone index of the highest zone to return
1172  * @nodes: An optional nodemask to filter the zonelist with
1173  *
1174  * This function returns the next zone at or below a given zone index that is
1175  * within the allowed nodemask using a cursor as the starting point for the
1176  * search. The zoneref returned is a cursor that represents the current zone
1177  * being examined. It should be advanced by one before calling
1178  * next_zones_zonelist again.
1179  *
1180  * Return: the next zone at or below highest_zoneidx within the allowed
1181  * nodemask using a cursor within a zonelist as a starting point
1182  */
next_zones_zonelist(struct zoneref * z,enum zone_type highest_zoneidx,nodemask_t * nodes)1183 static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
1184 					enum zone_type highest_zoneidx,
1185 					nodemask_t *nodes)
1186 {
1187 	if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
1188 		return z;
1189 	return __next_zones_zonelist(z, highest_zoneidx, nodes);
1190 }
1191 
1192 /**
1193  * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
1194  * @zonelist: The zonelist to search for a suitable zone
1195  * @highest_zoneidx: The zone index of the highest zone to return
1196  * @nodes: An optional nodemask to filter the zonelist with
1197  *
1198  * This function returns the first zone at or below a given zone index that is
1199  * within the allowed nodemask. The zoneref returned is a cursor that can be
1200  * used to iterate the zonelist with next_zones_zonelist by advancing it by
1201  * one before calling.
1202  *
1203  * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is
1204  * never NULL). This may happen either genuinely, or due to concurrent nodemask
1205  * update due to cpuset modification.
1206  *
1207  * Return: Zoneref pointer for the first suitable zone found
1208  */
first_zones_zonelist(struct zonelist * zonelist,enum zone_type highest_zoneidx,nodemask_t * nodes)1209 static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
1210 					enum zone_type highest_zoneidx,
1211 					nodemask_t *nodes)
1212 {
1213 	return next_zones_zonelist(zonelist->_zonerefs,
1214 							highest_zoneidx, nodes);
1215 }
1216 
1217 /**
1218  * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
1219  * @zone: The current zone in the iterator
1220  * @z: The current pointer within zonelist->_zonerefs being iterated
1221  * @zlist: The zonelist being iterated
1222  * @highidx: The zone index of the highest zone to return
1223  * @nodemask: Nodemask allowed by the allocator
1224  *
1225  * This iterator iterates though all zones at or below a given zone index and
1226  * within a given nodemask
1227  */
1228 #define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
1229 	for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z);	\
1230 		zone;							\
1231 		z = next_zones_zonelist(++z, highidx, nodemask),	\
1232 			zone = zonelist_zone(z))
1233 
1234 #define for_next_zone_zonelist_nodemask(zone, z, highidx, nodemask) \
1235 	for (zone = z->zone;	\
1236 		zone;							\
1237 		z = next_zones_zonelist(++z, highidx, nodemask),	\
1238 			zone = zonelist_zone(z))
1239 
1240 
1241 /**
1242  * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
1243  * @zone: The current zone in the iterator
1244  * @z: The current pointer within zonelist->zones being iterated
1245  * @zlist: The zonelist being iterated
1246  * @highidx: The zone index of the highest zone to return
1247  *
1248  * This iterator iterates though all zones at or below a given zone index.
1249  */
1250 #define for_each_zone_zonelist(zone, z, zlist, highidx) \
1251 	for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
1252 
1253 /* Whether the 'nodes' are all movable nodes */
movable_only_nodes(nodemask_t * nodes)1254 static inline bool movable_only_nodes(nodemask_t *nodes)
1255 {
1256 	struct zonelist *zonelist;
1257 	struct zoneref *z;
1258 	int nid;
1259 
1260 	if (nodes_empty(*nodes))
1261 		return false;
1262 
1263 	/*
1264 	 * We can chose arbitrary node from the nodemask to get a
1265 	 * zonelist as they are interlinked. We just need to find
1266 	 * at least one zone that can satisfy kernel allocations.
1267 	 */
1268 	nid = first_node(*nodes);
1269 	zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
1270 	z = first_zones_zonelist(zonelist, ZONE_NORMAL,	nodes);
1271 	return (!z->zone) ? true : false;
1272 }
1273 
1274 
1275 #ifdef CONFIG_SPARSEMEM
1276 #include <asm/sparsemem.h>
1277 #endif
1278 
1279 #ifdef CONFIG_FLATMEM
1280 #define pfn_to_nid(pfn)		(0)
1281 #endif
1282 
1283 #ifdef CONFIG_SPARSEMEM
1284 
1285 /*
1286  * PA_SECTION_SHIFT		physical address to/from section number
1287  * PFN_SECTION_SHIFT		pfn to/from section number
1288  */
1289 #define PA_SECTION_SHIFT	(SECTION_SIZE_BITS)
1290 #define PFN_SECTION_SHIFT	(SECTION_SIZE_BITS - PAGE_SHIFT)
1291 
1292 #define NR_MEM_SECTIONS		(1UL << SECTIONS_SHIFT)
1293 
1294 #define PAGES_PER_SECTION       (1UL << PFN_SECTION_SHIFT)
1295 #define PAGE_SECTION_MASK	(~(PAGES_PER_SECTION-1))
1296 
1297 #define SECTION_BLOCKFLAGS_BITS \
1298 	((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1299 
1300 #if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
1301 #error Allocator MAX_ORDER exceeds SECTION_SIZE
1302 #endif
1303 
pfn_to_section_nr(unsigned long pfn)1304 static inline unsigned long pfn_to_section_nr(unsigned long pfn)
1305 {
1306 	return pfn >> PFN_SECTION_SHIFT;
1307 }
section_nr_to_pfn(unsigned long sec)1308 static inline unsigned long section_nr_to_pfn(unsigned long sec)
1309 {
1310 	return sec << PFN_SECTION_SHIFT;
1311 }
1312 
1313 #define SECTION_ALIGN_UP(pfn)	(((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
1314 #define SECTION_ALIGN_DOWN(pfn)	((pfn) & PAGE_SECTION_MASK)
1315 
1316 #define SUBSECTION_SHIFT 21
1317 #define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT)
1318 
1319 #define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)
1320 #define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)
1321 #define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
1322 
1323 #if SUBSECTION_SHIFT > SECTION_SIZE_BITS
1324 #error Subsection size exceeds section size
1325 #else
1326 #define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))
1327 #endif
1328 
1329 #define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)
1330 #define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
1331 
1332 struct mem_section_usage {
1333 #ifdef CONFIG_SPARSEMEM_VMEMMAP
1334 	DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);
1335 #endif
1336 	/* See declaration of similar field in struct zone */
1337 	unsigned long pageblock_flags[0];
1338 };
1339 
1340 void subsection_map_init(unsigned long pfn, unsigned long nr_pages);
1341 
1342 struct page;
1343 struct page_ext;
1344 struct mem_section {
1345 	/*
1346 	 * This is, logically, a pointer to an array of struct
1347 	 * pages.  However, it is stored with some other magic.
1348 	 * (see sparse.c::sparse_init_one_section())
1349 	 *
1350 	 * Additionally during early boot we encode node id of
1351 	 * the location of the section here to guide allocation.
1352 	 * (see sparse.c::memory_present())
1353 	 *
1354 	 * Making it a UL at least makes someone do a cast
1355 	 * before using it wrong.
1356 	 */
1357 	unsigned long section_mem_map;
1358 
1359 	struct mem_section_usage *usage;
1360 #ifdef CONFIG_PAGE_EXTENSION
1361 	/*
1362 	 * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
1363 	 * section. (see page_ext.h about this.)
1364 	 */
1365 	struct page_ext *page_ext;
1366 	unsigned long pad;
1367 #endif
1368 	/*
1369 	 * WARNING: mem_section must be a power-of-2 in size for the
1370 	 * calculation and use of SECTION_ROOT_MASK to make sense.
1371 	 */
1372 };
1373 
1374 #ifdef CONFIG_SPARSEMEM_EXTREME
1375 #define SECTIONS_PER_ROOT       (PAGE_SIZE / sizeof (struct mem_section))
1376 #else
1377 #define SECTIONS_PER_ROOT	1
1378 #endif
1379 
1380 #define SECTION_NR_TO_ROOT(sec)	((sec) / SECTIONS_PER_ROOT)
1381 #define NR_SECTION_ROOTS	DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
1382 #define SECTION_ROOT_MASK	(SECTIONS_PER_ROOT - 1)
1383 
1384 #ifdef CONFIG_SPARSEMEM_EXTREME
1385 extern struct mem_section **mem_section;
1386 #else
1387 extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
1388 #endif
1389 
section_to_usemap(struct mem_section * ms)1390 static inline unsigned long *section_to_usemap(struct mem_section *ms)
1391 {
1392 	return ms->usage->pageblock_flags;
1393 }
1394 
__nr_to_section(unsigned long nr)1395 static inline struct mem_section *__nr_to_section(unsigned long nr)
1396 {
1397 	unsigned long root = SECTION_NR_TO_ROOT(nr);
1398 
1399 	if (unlikely(root >= NR_SECTION_ROOTS))
1400 		return NULL;
1401 
1402 #ifdef CONFIG_SPARSEMEM_EXTREME
1403 	if (!mem_section || !mem_section[root])
1404 		return NULL;
1405 #endif
1406 	return &mem_section[root][nr & SECTION_ROOT_MASK];
1407 }
1408 extern size_t mem_section_usage_size(void);
1409 
1410 /*
1411  * We use the lower bits of the mem_map pointer to store
1412  * a little bit of information.  The pointer is calculated
1413  * as mem_map - section_nr_to_pfn(pnum).  The result is
1414  * aligned to the minimum alignment of the two values:
1415  *   1. All mem_map arrays are page-aligned.
1416  *   2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT
1417  *      lowest bits.  PFN_SECTION_SHIFT is arch-specific
1418  *      (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the
1419  *      worst combination is powerpc with 256k pages,
1420  *      which results in PFN_SECTION_SHIFT equal 6.
1421  * To sum it up, at least 6 bits are available.
1422  */
1423 #define SECTION_MARKED_PRESENT		(1UL<<0)
1424 #define SECTION_HAS_MEM_MAP		(1UL<<1)
1425 #define SECTION_IS_ONLINE		(1UL<<2)
1426 #define SECTION_IS_EARLY		(1UL<<3)
1427 #define SECTION_TAINT_ZONE_DEVICE	(1UL<<4)
1428 #define SECTION_MAP_LAST_BIT		(1UL<<5)
1429 #define SECTION_MAP_MASK		(~(SECTION_MAP_LAST_BIT-1))
1430 #define SECTION_NID_SHIFT		6
1431 
__section_mem_map_addr(struct mem_section * section)1432 static inline struct page *__section_mem_map_addr(struct mem_section *section)
1433 {
1434 	unsigned long map = section->section_mem_map;
1435 	map &= SECTION_MAP_MASK;
1436 	return (struct page *)map;
1437 }
1438 
present_section(struct mem_section * section)1439 static inline int present_section(struct mem_section *section)
1440 {
1441 	return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
1442 }
1443 
present_section_nr(unsigned long nr)1444 static inline int present_section_nr(unsigned long nr)
1445 {
1446 	return present_section(__nr_to_section(nr));
1447 }
1448 
valid_section(struct mem_section * section)1449 static inline int valid_section(struct mem_section *section)
1450 {
1451 	return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
1452 }
1453 
early_section(struct mem_section * section)1454 static inline int early_section(struct mem_section *section)
1455 {
1456 	return (section && (section->section_mem_map & SECTION_IS_EARLY));
1457 }
1458 
valid_section_nr(unsigned long nr)1459 static inline int valid_section_nr(unsigned long nr)
1460 {
1461 	return valid_section(__nr_to_section(nr));
1462 }
1463 
online_section(struct mem_section * section)1464 static inline int online_section(struct mem_section *section)
1465 {
1466 	return (section && (section->section_mem_map & SECTION_IS_ONLINE));
1467 }
1468 
online_device_section(struct mem_section * section)1469 static inline int online_device_section(struct mem_section *section)
1470 {
1471 	unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE;
1472 
1473 	return section && ((section->section_mem_map & flags) == flags);
1474 }
1475 
online_section_nr(unsigned long nr)1476 static inline int online_section_nr(unsigned long nr)
1477 {
1478 	return online_section(__nr_to_section(nr));
1479 }
1480 
1481 #ifdef CONFIG_MEMORY_HOTPLUG
1482 void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1483 void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
1484 #endif
1485 
__pfn_to_section(unsigned long pfn)1486 static inline struct mem_section *__pfn_to_section(unsigned long pfn)
1487 {
1488 	return __nr_to_section(pfn_to_section_nr(pfn));
1489 }
1490 
1491 extern unsigned long __highest_present_section_nr;
1492 
subsection_map_index(unsigned long pfn)1493 static inline int subsection_map_index(unsigned long pfn)
1494 {
1495 	return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;
1496 }
1497 
1498 #ifdef CONFIG_SPARSEMEM_VMEMMAP
pfn_section_valid(struct mem_section * ms,unsigned long pfn)1499 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1500 {
1501 	int idx = subsection_map_index(pfn);
1502 
1503 	return test_bit(idx, ms->usage->subsection_map);
1504 }
1505 #else
pfn_section_valid(struct mem_section * ms,unsigned long pfn)1506 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
1507 {
1508 	return 1;
1509 }
1510 #endif
1511 
1512 #ifndef CONFIG_HAVE_ARCH_PFN_VALID
1513 /**
1514  * pfn_valid - check if there is a valid memory map entry for a PFN
1515  * @pfn: the page frame number to check
1516  *
1517  * Check if there is a valid memory map entry aka struct page for the @pfn.
1518  * Note, that availability of the memory map entry does not imply that
1519  * there is actual usable memory at that @pfn. The struct page may
1520  * represent a hole or an unusable page frame.
1521  *
1522  * Return: 1 for PFNs that have memory map entries and 0 otherwise
1523  */
pfn_valid(unsigned long pfn)1524 static inline int pfn_valid(unsigned long pfn)
1525 {
1526 	struct mem_section *ms;
1527 
1528 	/*
1529 	 * Ensure the upper PAGE_SHIFT bits are clear in the
1530 	 * pfn. Else it might lead to false positives when
1531 	 * some of the upper bits are set, but the lower bits
1532 	 * match a valid pfn.
1533 	 */
1534 	if (PHYS_PFN(PFN_PHYS(pfn)) != pfn)
1535 		return 0;
1536 
1537 	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1538 		return 0;
1539 	ms = __pfn_to_section(pfn);
1540 	if (!valid_section(ms))
1541 		return 0;
1542 	/*
1543 	 * Traditionally early sections always returned pfn_valid() for
1544 	 * the entire section-sized span.
1545 	 */
1546 	return early_section(ms) || pfn_section_valid(ms, pfn);
1547 }
1548 #endif
1549 
pfn_in_present_section(unsigned long pfn)1550 static inline int pfn_in_present_section(unsigned long pfn)
1551 {
1552 	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1553 		return 0;
1554 	return present_section(__pfn_to_section(pfn));
1555 }
1556 
next_present_section_nr(unsigned long section_nr)1557 static inline unsigned long next_present_section_nr(unsigned long section_nr)
1558 {
1559 	while (++section_nr <= __highest_present_section_nr) {
1560 		if (present_section_nr(section_nr))
1561 			return section_nr;
1562 	}
1563 
1564 	return -1;
1565 }
1566 
1567 /*
1568  * These are _only_ used during initialisation, therefore they
1569  * can use __initdata ...  They could have names to indicate
1570  * this restriction.
1571  */
1572 #ifdef CONFIG_NUMA
1573 #define pfn_to_nid(pfn)							\
1574 ({									\
1575 	unsigned long __pfn_to_nid_pfn = (pfn);				\
1576 	page_to_nid(pfn_to_page(__pfn_to_nid_pfn));			\
1577 })
1578 #else
1579 #define pfn_to_nid(pfn)		(0)
1580 #endif
1581 
1582 void sparse_init(void);
1583 #else
1584 #define sparse_init()	do {} while (0)
1585 #define sparse_index_init(_sec, _nid)  do {} while (0)
1586 #define pfn_in_present_section pfn_valid
1587 #define subsection_map_init(_pfn, _nr_pages) do {} while (0)
1588 #endif /* CONFIG_SPARSEMEM */
1589 
1590 #endif /* !__GENERATING_BOUNDS.H */
1591 #endif /* !__ASSEMBLY__ */
1592 #endif /* _LINUX_MMZONE_H */
1593