1 #include <linux/gfp.h>
2 #include <linux/initrd.h>
3 #include <linux/ioport.h>
4 #include <linux/swap.h>
5 #include <linux/memblock.h>
6 #include <linux/swapfile.h>
7 #include <linux/swapops.h>
8 #include <linux/kmemleak.h>
9 #include <linux/sched/task.h>
10 
11 #include <asm/set_memory.h>
12 #include <asm/cpu_device_id.h>
13 #include <asm/e820/api.h>
14 #include <asm/init.h>
15 #include <asm/page.h>
16 #include <asm/page_types.h>
17 #include <asm/sections.h>
18 #include <asm/setup.h>
19 #include <asm/tlbflush.h>
20 #include <asm/tlb.h>
21 #include <asm/proto.h>
22 #include <asm/dma.h>		/* for MAX_DMA_PFN */
23 #include <asm/kaslr.h>
24 #include <asm/hypervisor.h>
25 #include <asm/cpufeature.h>
26 #include <asm/pti.h>
27 #include <asm/text-patching.h>
28 #include <asm/memtype.h>
29 #include <asm/paravirt.h>
30 
31 /*
32  * We need to define the tracepoints somewhere, and tlb.c
33  * is only compiled when SMP=y.
34  */
35 #include <trace/events/tlb.h>
36 
37 #include "mm_internal.h"
38 
39 /*
40  * Tables translating between page_cache_type_t and pte encoding.
41  *
42  * The default values are defined statically as minimal supported mode;
43  * WC and WT fall back to UC-.  pat_init() updates these values to support
44  * more cache modes, WC and WT, when it is safe to do so.  See pat_init()
45  * for the details.  Note, __early_ioremap() used during early boot-time
46  * takes pgprot_t (pte encoding) and does not use these tables.
47  *
48  *   Index into __cachemode2pte_tbl[] is the cachemode.
49  *
50  *   Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte
51  *   (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
52  */
53 static uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
54 	[_PAGE_CACHE_MODE_WB      ]	= 0         | 0        ,
55 	[_PAGE_CACHE_MODE_WC      ]	= 0         | _PAGE_PCD,
56 	[_PAGE_CACHE_MODE_UC_MINUS]	= 0         | _PAGE_PCD,
57 	[_PAGE_CACHE_MODE_UC      ]	= _PAGE_PWT | _PAGE_PCD,
58 	[_PAGE_CACHE_MODE_WT      ]	= 0         | _PAGE_PCD,
59 	[_PAGE_CACHE_MODE_WP      ]	= 0         | _PAGE_PCD,
60 };
61 
cachemode2protval(enum page_cache_mode pcm)62 unsigned long cachemode2protval(enum page_cache_mode pcm)
63 {
64 	if (likely(pcm == 0))
65 		return 0;
66 	return __cachemode2pte_tbl[pcm];
67 }
68 EXPORT_SYMBOL(cachemode2protval);
69 
70 static uint8_t __pte2cachemode_tbl[8] = {
71 	[__pte2cm_idx( 0        | 0         | 0        )] = _PAGE_CACHE_MODE_WB,
72 	[__pte2cm_idx(_PAGE_PWT | 0         | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
73 	[__pte2cm_idx( 0        | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
74 	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC,
75 	[__pte2cm_idx( 0        | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
76 	[__pte2cm_idx(_PAGE_PWT | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
77 	[__pte2cm_idx(0         | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
78 	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
79 };
80 
81 /*
82  * Check that the write-protect PAT entry is set for write-protect.
83  * To do this without making assumptions how PAT has been set up (Xen has
84  * another layout than the kernel), translate the _PAGE_CACHE_MODE_WP cache
85  * mode via the __cachemode2pte_tbl[] into protection bits (those protection
86  * bits will select a cache mode of WP or better), and then translate the
87  * protection bits back into the cache mode using __pte2cm_idx() and the
88  * __pte2cachemode_tbl[] array. This will return the really used cache mode.
89  */
x86_has_pat_wp(void)90 bool x86_has_pat_wp(void)
91 {
92 	uint16_t prot = __cachemode2pte_tbl[_PAGE_CACHE_MODE_WP];
93 
94 	return __pte2cachemode_tbl[__pte2cm_idx(prot)] == _PAGE_CACHE_MODE_WP;
95 }
96 
pgprot2cachemode(pgprot_t pgprot)97 enum page_cache_mode pgprot2cachemode(pgprot_t pgprot)
98 {
99 	unsigned long masked;
100 
101 	masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK;
102 	if (likely(masked == 0))
103 		return 0;
104 	return __pte2cachemode_tbl[__pte2cm_idx(masked)];
105 }
106 
107 static unsigned long __initdata pgt_buf_start;
108 static unsigned long __initdata pgt_buf_end;
109 static unsigned long __initdata pgt_buf_top;
110 
111 static unsigned long min_pfn_mapped;
112 
113 static bool __initdata can_use_brk_pgt = true;
114 
115 /*
116  * Pages returned are already directly mapped.
117  *
118  * Changing that is likely to break Xen, see commit:
119  *
120  *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
121  *
122  * for detailed information.
123  */
alloc_low_pages(unsigned int num)124 __ref void *alloc_low_pages(unsigned int num)
125 {
126 	unsigned long pfn;
127 	int i;
128 
129 	if (after_bootmem) {
130 		unsigned int order;
131 
132 		order = get_order((unsigned long)num << PAGE_SHIFT);
133 		return (void *)__get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
134 	}
135 
136 	if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
137 		unsigned long ret = 0;
138 
139 		if (min_pfn_mapped < max_pfn_mapped) {
140 			ret = memblock_phys_alloc_range(
141 					PAGE_SIZE * num, PAGE_SIZE,
142 					min_pfn_mapped << PAGE_SHIFT,
143 					max_pfn_mapped << PAGE_SHIFT);
144 		}
145 		if (!ret && can_use_brk_pgt)
146 			ret = __pa(extend_brk(PAGE_SIZE * num, PAGE_SIZE));
147 
148 		if (!ret)
149 			panic("alloc_low_pages: can not alloc memory");
150 
151 		pfn = ret >> PAGE_SHIFT;
152 	} else {
153 		pfn = pgt_buf_end;
154 		pgt_buf_end += num;
155 	}
156 
157 	for (i = 0; i < num; i++) {
158 		void *adr;
159 
160 		adr = __va((pfn + i) << PAGE_SHIFT);
161 		clear_page(adr);
162 	}
163 
164 	return __va(pfn << PAGE_SHIFT);
165 }
166 
167 /*
168  * By default need to be able to allocate page tables below PGD firstly for
169  * the 0-ISA_END_ADDRESS range and secondly for the initial PMD_SIZE mapping.
170  * With KASLR memory randomization, depending on the machine e820 memory and the
171  * PUD alignment, twice that many pages may be needed when KASLR memory
172  * randomization is enabled.
173  */
174 
175 #ifndef CONFIG_X86_5LEVEL
176 #define INIT_PGD_PAGE_TABLES    3
177 #else
178 #define INIT_PGD_PAGE_TABLES    4
179 #endif
180 
181 #ifndef CONFIG_RANDOMIZE_MEMORY
182 #define INIT_PGD_PAGE_COUNT      (2 * INIT_PGD_PAGE_TABLES)
183 #else
184 #define INIT_PGD_PAGE_COUNT      (4 * INIT_PGD_PAGE_TABLES)
185 #endif
186 
187 #define INIT_PGT_BUF_SIZE	(INIT_PGD_PAGE_COUNT * PAGE_SIZE)
188 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
early_alloc_pgt_buf(void)189 void  __init early_alloc_pgt_buf(void)
190 {
191 	unsigned long tables = INIT_PGT_BUF_SIZE;
192 	phys_addr_t base;
193 
194 	base = __pa(extend_brk(tables, PAGE_SIZE));
195 
196 	pgt_buf_start = base >> PAGE_SHIFT;
197 	pgt_buf_end = pgt_buf_start;
198 	pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
199 }
200 
201 int after_bootmem;
202 
203 early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES);
204 
205 struct map_range {
206 	unsigned long start;
207 	unsigned long end;
208 	unsigned page_size_mask;
209 };
210 
211 static int page_size_mask;
212 
213 /*
214  * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
215  * enable and PPro Global page enable), so that any CPU's that boot
216  * up after us can get the correct flags. Invoked on the boot CPU.
217  */
cr4_set_bits_and_update_boot(unsigned long mask)218 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
219 {
220 	mmu_cr4_features |= mask;
221 	if (trampoline_cr4_features)
222 		*trampoline_cr4_features = mmu_cr4_features;
223 	cr4_set_bits(mask);
224 }
225 
probe_page_size_mask(void)226 static void __init probe_page_size_mask(void)
227 {
228 	/*
229 	 * For pagealloc debugging, identity mapping will use small pages.
230 	 * This will simplify cpa(), which otherwise needs to support splitting
231 	 * large pages into small in interrupt context, etc.
232 	 */
233 	if (boot_cpu_has(X86_FEATURE_PSE) && !debug_pagealloc_enabled())
234 		page_size_mask |= 1 << PG_LEVEL_2M;
235 	else
236 		direct_gbpages = 0;
237 
238 	/* Enable PSE if available */
239 	if (boot_cpu_has(X86_FEATURE_PSE))
240 		cr4_set_bits_and_update_boot(X86_CR4_PSE);
241 
242 	/* Enable PGE if available */
243 	__supported_pte_mask &= ~_PAGE_GLOBAL;
244 	if (boot_cpu_has(X86_FEATURE_PGE)) {
245 		cr4_set_bits_and_update_boot(X86_CR4_PGE);
246 		__supported_pte_mask |= _PAGE_GLOBAL;
247 	}
248 
249 	/* By the default is everything supported: */
250 	__default_kernel_pte_mask = __supported_pte_mask;
251 	/* Except when with PTI where the kernel is mostly non-Global: */
252 	if (cpu_feature_enabled(X86_FEATURE_PTI))
253 		__default_kernel_pte_mask &= ~_PAGE_GLOBAL;
254 
255 	/* Enable 1 GB linear kernel mappings if available: */
256 	if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) {
257 		printk(KERN_INFO "Using GB pages for direct mapping\n");
258 		page_size_mask |= 1 << PG_LEVEL_1G;
259 	} else {
260 		direct_gbpages = 0;
261 	}
262 }
263 
264 #define INTEL_MATCH(_model) { .vendor  = X86_VENDOR_INTEL,	\
265 			      .family  = 6,			\
266 			      .model = _model,			\
267 			    }
268 /*
269  * INVLPG may not properly flush Global entries
270  * on these CPUs when PCIDs are enabled.
271  */
272 static const struct x86_cpu_id invlpg_miss_ids[] = {
273 	INTEL_MATCH(INTEL_FAM6_ALDERLAKE   ),
274 	INTEL_MATCH(INTEL_FAM6_ALDERLAKE_L ),
275 	INTEL_MATCH(INTEL_FAM6_ATOM_GRACEMONT ),
276 	INTEL_MATCH(INTEL_FAM6_RAPTORLAKE  ),
277 	INTEL_MATCH(INTEL_FAM6_RAPTORLAKE_P),
278 	INTEL_MATCH(INTEL_FAM6_RAPTORLAKE_S),
279 	{}
280 };
281 
setup_pcid(void)282 static void setup_pcid(void)
283 {
284 	if (!IS_ENABLED(CONFIG_X86_64))
285 		return;
286 
287 	if (!boot_cpu_has(X86_FEATURE_PCID))
288 		return;
289 
290 	if (x86_match_cpu(invlpg_miss_ids)) {
291 		pr_info("Incomplete global flushes, disabling PCID");
292 		setup_clear_cpu_cap(X86_FEATURE_PCID);
293 		return;
294 	}
295 
296 	if (boot_cpu_has(X86_FEATURE_PGE)) {
297 		/*
298 		 * This can't be cr4_set_bits_and_update_boot() -- the
299 		 * trampoline code can't handle CR4.PCIDE and it wouldn't
300 		 * do any good anyway.  Despite the name,
301 		 * cr4_set_bits_and_update_boot() doesn't actually cause
302 		 * the bits in question to remain set all the way through
303 		 * the secondary boot asm.
304 		 *
305 		 * Instead, we brute-force it and set CR4.PCIDE manually in
306 		 * start_secondary().
307 		 */
308 		cr4_set_bits(X86_CR4_PCIDE);
309 	} else {
310 		/*
311 		 * flush_tlb_all(), as currently implemented, won't work if
312 		 * PCID is on but PGE is not.  Since that combination
313 		 * doesn't exist on real hardware, there's no reason to try
314 		 * to fully support it, but it's polite to avoid corrupting
315 		 * data if we're on an improperly configured VM.
316 		 */
317 		setup_clear_cpu_cap(X86_FEATURE_PCID);
318 	}
319 }
320 
321 #ifdef CONFIG_X86_32
322 #define NR_RANGE_MR 3
323 #else /* CONFIG_X86_64 */
324 #define NR_RANGE_MR 5
325 #endif
326 
save_mr(struct map_range * mr,int nr_range,unsigned long start_pfn,unsigned long end_pfn,unsigned long page_size_mask)327 static int __meminit save_mr(struct map_range *mr, int nr_range,
328 			     unsigned long start_pfn, unsigned long end_pfn,
329 			     unsigned long page_size_mask)
330 {
331 	if (start_pfn < end_pfn) {
332 		if (nr_range >= NR_RANGE_MR)
333 			panic("run out of range for init_memory_mapping\n");
334 		mr[nr_range].start = start_pfn<<PAGE_SHIFT;
335 		mr[nr_range].end   = end_pfn<<PAGE_SHIFT;
336 		mr[nr_range].page_size_mask = page_size_mask;
337 		nr_range++;
338 	}
339 
340 	return nr_range;
341 }
342 
343 /*
344  * adjust the page_size_mask for small range to go with
345  *	big page size instead small one if nearby are ram too.
346  */
adjust_range_page_size_mask(struct map_range * mr,int nr_range)347 static void __ref adjust_range_page_size_mask(struct map_range *mr,
348 							 int nr_range)
349 {
350 	int i;
351 
352 	for (i = 0; i < nr_range; i++) {
353 		if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
354 		    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
355 			unsigned long start = round_down(mr[i].start, PMD_SIZE);
356 			unsigned long end = round_up(mr[i].end, PMD_SIZE);
357 
358 #ifdef CONFIG_X86_32
359 			if ((end >> PAGE_SHIFT) > max_low_pfn)
360 				continue;
361 #endif
362 
363 			if (memblock_is_region_memory(start, end - start))
364 				mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
365 		}
366 		if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
367 		    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
368 			unsigned long start = round_down(mr[i].start, PUD_SIZE);
369 			unsigned long end = round_up(mr[i].end, PUD_SIZE);
370 
371 			if (memblock_is_region_memory(start, end - start))
372 				mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
373 		}
374 	}
375 }
376 
page_size_string(struct map_range * mr)377 static const char *page_size_string(struct map_range *mr)
378 {
379 	static const char str_1g[] = "1G";
380 	static const char str_2m[] = "2M";
381 	static const char str_4m[] = "4M";
382 	static const char str_4k[] = "4k";
383 
384 	if (mr->page_size_mask & (1<<PG_LEVEL_1G))
385 		return str_1g;
386 	/*
387 	 * 32-bit without PAE has a 4M large page size.
388 	 * PG_LEVEL_2M is misnamed, but we can at least
389 	 * print out the right size in the string.
390 	 */
391 	if (IS_ENABLED(CONFIG_X86_32) &&
392 	    !IS_ENABLED(CONFIG_X86_PAE) &&
393 	    mr->page_size_mask & (1<<PG_LEVEL_2M))
394 		return str_4m;
395 
396 	if (mr->page_size_mask & (1<<PG_LEVEL_2M))
397 		return str_2m;
398 
399 	return str_4k;
400 }
401 
split_mem_range(struct map_range * mr,int nr_range,unsigned long start,unsigned long end)402 static int __meminit split_mem_range(struct map_range *mr, int nr_range,
403 				     unsigned long start,
404 				     unsigned long end)
405 {
406 	unsigned long start_pfn, end_pfn, limit_pfn;
407 	unsigned long pfn;
408 	int i;
409 
410 	limit_pfn = PFN_DOWN(end);
411 
412 	/* head if not big page alignment ? */
413 	pfn = start_pfn = PFN_DOWN(start);
414 #ifdef CONFIG_X86_32
415 	/*
416 	 * Don't use a large page for the first 2/4MB of memory
417 	 * because there are often fixed size MTRRs in there
418 	 * and overlapping MTRRs into large pages can cause
419 	 * slowdowns.
420 	 */
421 	if (pfn == 0)
422 		end_pfn = PFN_DOWN(PMD_SIZE);
423 	else
424 		end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
425 #else /* CONFIG_X86_64 */
426 	end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
427 #endif
428 	if (end_pfn > limit_pfn)
429 		end_pfn = limit_pfn;
430 	if (start_pfn < end_pfn) {
431 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
432 		pfn = end_pfn;
433 	}
434 
435 	/* big page (2M) range */
436 	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
437 #ifdef CONFIG_X86_32
438 	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
439 #else /* CONFIG_X86_64 */
440 	end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
441 	if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
442 		end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
443 #endif
444 
445 	if (start_pfn < end_pfn) {
446 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
447 				page_size_mask & (1<<PG_LEVEL_2M));
448 		pfn = end_pfn;
449 	}
450 
451 #ifdef CONFIG_X86_64
452 	/* big page (1G) range */
453 	start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
454 	end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
455 	if (start_pfn < end_pfn) {
456 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
457 				page_size_mask &
458 				 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
459 		pfn = end_pfn;
460 	}
461 
462 	/* tail is not big page (1G) alignment */
463 	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
464 	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
465 	if (start_pfn < end_pfn) {
466 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
467 				page_size_mask & (1<<PG_LEVEL_2M));
468 		pfn = end_pfn;
469 	}
470 #endif
471 
472 	/* tail is not big page (2M) alignment */
473 	start_pfn = pfn;
474 	end_pfn = limit_pfn;
475 	nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
476 
477 	if (!after_bootmem)
478 		adjust_range_page_size_mask(mr, nr_range);
479 
480 	/* try to merge same page size and continuous */
481 	for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
482 		unsigned long old_start;
483 		if (mr[i].end != mr[i+1].start ||
484 		    mr[i].page_size_mask != mr[i+1].page_size_mask)
485 			continue;
486 		/* move it */
487 		old_start = mr[i].start;
488 		memmove(&mr[i], &mr[i+1],
489 			(nr_range - 1 - i) * sizeof(struct map_range));
490 		mr[i--].start = old_start;
491 		nr_range--;
492 	}
493 
494 	for (i = 0; i < nr_range; i++)
495 		pr_debug(" [mem %#010lx-%#010lx] page %s\n",
496 				mr[i].start, mr[i].end - 1,
497 				page_size_string(&mr[i]));
498 
499 	return nr_range;
500 }
501 
502 struct range pfn_mapped[E820_MAX_ENTRIES];
503 int nr_pfn_mapped;
504 
add_pfn_range_mapped(unsigned long start_pfn,unsigned long end_pfn)505 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
506 {
507 	nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_MAX_ENTRIES,
508 					     nr_pfn_mapped, start_pfn, end_pfn);
509 	nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_MAX_ENTRIES);
510 
511 	max_pfn_mapped = max(max_pfn_mapped, end_pfn);
512 
513 	if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
514 		max_low_pfn_mapped = max(max_low_pfn_mapped,
515 					 min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
516 }
517 
pfn_range_is_mapped(unsigned long start_pfn,unsigned long end_pfn)518 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
519 {
520 	int i;
521 
522 	for (i = 0; i < nr_pfn_mapped; i++)
523 		if ((start_pfn >= pfn_mapped[i].start) &&
524 		    (end_pfn <= pfn_mapped[i].end))
525 			return true;
526 
527 	return false;
528 }
529 
530 /*
531  * Setup the direct mapping of the physical memory at PAGE_OFFSET.
532  * This runs before bootmem is initialized and gets pages directly from
533  * the physical memory. To access them they are temporarily mapped.
534  */
init_memory_mapping(unsigned long start,unsigned long end,pgprot_t prot)535 unsigned long __ref init_memory_mapping(unsigned long start,
536 					unsigned long end, pgprot_t prot)
537 {
538 	struct map_range mr[NR_RANGE_MR];
539 	unsigned long ret = 0;
540 	int nr_range, i;
541 
542 	pr_debug("init_memory_mapping: [mem %#010lx-%#010lx]\n",
543 	       start, end - 1);
544 
545 	memset(mr, 0, sizeof(mr));
546 	nr_range = split_mem_range(mr, 0, start, end);
547 
548 	for (i = 0; i < nr_range; i++)
549 		ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
550 						   mr[i].page_size_mask,
551 						   prot);
552 
553 	add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
554 
555 	return ret >> PAGE_SHIFT;
556 }
557 
558 /*
559  * We need to iterate through the E820 memory map and create direct mappings
560  * for only E820_TYPE_RAM and E820_KERN_RESERVED regions. We cannot simply
561  * create direct mappings for all pfns from [0 to max_low_pfn) and
562  * [4GB to max_pfn) because of possible memory holes in high addresses
563  * that cannot be marked as UC by fixed/variable range MTRRs.
564  * Depending on the alignment of E820 ranges, this may possibly result
565  * in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
566  *
567  * init_mem_mapping() calls init_range_memory_mapping() with big range.
568  * That range would have hole in the middle or ends, and only ram parts
569  * will be mapped in init_range_memory_mapping().
570  */
init_range_memory_mapping(unsigned long r_start,unsigned long r_end)571 static unsigned long __init init_range_memory_mapping(
572 					   unsigned long r_start,
573 					   unsigned long r_end)
574 {
575 	unsigned long start_pfn, end_pfn;
576 	unsigned long mapped_ram_size = 0;
577 	int i;
578 
579 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
580 		u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
581 		u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
582 		if (start >= end)
583 			continue;
584 
585 		/*
586 		 * if it is overlapping with brk pgt, we need to
587 		 * alloc pgt buf from memblock instead.
588 		 */
589 		can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
590 				    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
591 		init_memory_mapping(start, end, PAGE_KERNEL);
592 		mapped_ram_size += end - start;
593 		can_use_brk_pgt = true;
594 	}
595 
596 	return mapped_ram_size;
597 }
598 
get_new_step_size(unsigned long step_size)599 static unsigned long __init get_new_step_size(unsigned long step_size)
600 {
601 	/*
602 	 * Initial mapped size is PMD_SIZE (2M).
603 	 * We can not set step_size to be PUD_SIZE (1G) yet.
604 	 * In worse case, when we cross the 1G boundary, and
605 	 * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
606 	 * to map 1G range with PTE. Hence we use one less than the
607 	 * difference of page table level shifts.
608 	 *
609 	 * Don't need to worry about overflow in the top-down case, on 32bit,
610 	 * when step_size is 0, round_down() returns 0 for start, and that
611 	 * turns it into 0x100000000ULL.
612 	 * In the bottom-up case, round_up(x, 0) returns 0 though too, which
613 	 * needs to be taken into consideration by the code below.
614 	 */
615 	return step_size << (PMD_SHIFT - PAGE_SHIFT - 1);
616 }
617 
618 /**
619  * memory_map_top_down - Map [map_start, map_end) top down
620  * @map_start: start address of the target memory range
621  * @map_end: end address of the target memory range
622  *
623  * This function will setup direct mapping for memory range
624  * [map_start, map_end) in top-down. That said, the page tables
625  * will be allocated at the end of the memory, and we map the
626  * memory in top-down.
627  */
memory_map_top_down(unsigned long map_start,unsigned long map_end)628 static void __init memory_map_top_down(unsigned long map_start,
629 				       unsigned long map_end)
630 {
631 	unsigned long real_end, last_start;
632 	unsigned long step_size;
633 	unsigned long addr;
634 	unsigned long mapped_ram_size = 0;
635 
636 	/*
637 	 * Systems that have many reserved areas near top of the memory,
638 	 * e.g. QEMU with less than 1G RAM and EFI enabled, or Xen, will
639 	 * require lots of 4K mappings which may exhaust pgt_buf.
640 	 * Start with top-most PMD_SIZE range aligned at PMD_SIZE to ensure
641 	 * there is enough mapped memory that can be allocated from
642 	 * memblock.
643 	 */
644 	addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start,
645 					 map_end);
646 	memblock_phys_free(addr, PMD_SIZE);
647 	real_end = addr + PMD_SIZE;
648 
649 	/* step_size need to be small so pgt_buf from BRK could cover it */
650 	step_size = PMD_SIZE;
651 	max_pfn_mapped = 0; /* will get exact value next */
652 	min_pfn_mapped = real_end >> PAGE_SHIFT;
653 	last_start = real_end;
654 
655 	/*
656 	 * We start from the top (end of memory) and go to the bottom.
657 	 * The memblock_find_in_range() gets us a block of RAM from the
658 	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
659 	 * for page table.
660 	 */
661 	while (last_start > map_start) {
662 		unsigned long start;
663 
664 		if (last_start > step_size) {
665 			start = round_down(last_start - 1, step_size);
666 			if (start < map_start)
667 				start = map_start;
668 		} else
669 			start = map_start;
670 		mapped_ram_size += init_range_memory_mapping(start,
671 							last_start);
672 		last_start = start;
673 		min_pfn_mapped = last_start >> PAGE_SHIFT;
674 		if (mapped_ram_size >= step_size)
675 			step_size = get_new_step_size(step_size);
676 	}
677 
678 	if (real_end < map_end)
679 		init_range_memory_mapping(real_end, map_end);
680 }
681 
682 /**
683  * memory_map_bottom_up - Map [map_start, map_end) bottom up
684  * @map_start: start address of the target memory range
685  * @map_end: end address of the target memory range
686  *
687  * This function will setup direct mapping for memory range
688  * [map_start, map_end) in bottom-up. Since we have limited the
689  * bottom-up allocation above the kernel, the page tables will
690  * be allocated just above the kernel and we map the memory
691  * in [map_start, map_end) in bottom-up.
692  */
memory_map_bottom_up(unsigned long map_start,unsigned long map_end)693 static void __init memory_map_bottom_up(unsigned long map_start,
694 					unsigned long map_end)
695 {
696 	unsigned long next, start;
697 	unsigned long mapped_ram_size = 0;
698 	/* step_size need to be small so pgt_buf from BRK could cover it */
699 	unsigned long step_size = PMD_SIZE;
700 
701 	start = map_start;
702 	min_pfn_mapped = start >> PAGE_SHIFT;
703 
704 	/*
705 	 * We start from the bottom (@map_start) and go to the top (@map_end).
706 	 * The memblock_find_in_range() gets us a block of RAM from the
707 	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
708 	 * for page table.
709 	 */
710 	while (start < map_end) {
711 		if (step_size && map_end - start > step_size) {
712 			next = round_up(start + 1, step_size);
713 			if (next > map_end)
714 				next = map_end;
715 		} else {
716 			next = map_end;
717 		}
718 
719 		mapped_ram_size += init_range_memory_mapping(start, next);
720 		start = next;
721 
722 		if (mapped_ram_size >= step_size)
723 			step_size = get_new_step_size(step_size);
724 	}
725 }
726 
727 /*
728  * The real mode trampoline, which is required for bootstrapping CPUs
729  * occupies only a small area under the low 1MB.  See reserve_real_mode()
730  * for details.
731  *
732  * If KASLR is disabled the first PGD entry of the direct mapping is copied
733  * to map the real mode trampoline.
734  *
735  * If KASLR is enabled, copy only the PUD which covers the low 1MB
736  * area. This limits the randomization granularity to 1GB for both 4-level
737  * and 5-level paging.
738  */
init_trampoline(void)739 static void __init init_trampoline(void)
740 {
741 #ifdef CONFIG_X86_64
742 	/*
743 	 * The code below will alias kernel page-tables in the user-range of the
744 	 * address space, including the Global bit. So global TLB entries will
745 	 * be created when using the trampoline page-table.
746 	 */
747 	if (!kaslr_memory_enabled())
748 		trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
749 	else
750 		init_trampoline_kaslr();
751 #endif
752 }
753 
init_mem_mapping(void)754 void __init init_mem_mapping(void)
755 {
756 	unsigned long end;
757 
758 	pti_check_boottime_disable();
759 	probe_page_size_mask();
760 	setup_pcid();
761 
762 #ifdef CONFIG_X86_64
763 	end = max_pfn << PAGE_SHIFT;
764 #else
765 	end = max_low_pfn << PAGE_SHIFT;
766 #endif
767 
768 	/* the ISA range is always mapped regardless of memory holes */
769 	init_memory_mapping(0, ISA_END_ADDRESS, PAGE_KERNEL);
770 
771 	/* Init the trampoline, possibly with KASLR memory offset */
772 	init_trampoline();
773 
774 	/*
775 	 * If the allocation is in bottom-up direction, we setup direct mapping
776 	 * in bottom-up, otherwise we setup direct mapping in top-down.
777 	 */
778 	if (memblock_bottom_up()) {
779 		unsigned long kernel_end = __pa_symbol(_end);
780 
781 		/*
782 		 * we need two separate calls here. This is because we want to
783 		 * allocate page tables above the kernel. So we first map
784 		 * [kernel_end, end) to make memory above the kernel be mapped
785 		 * as soon as possible. And then use page tables allocated above
786 		 * the kernel to map [ISA_END_ADDRESS, kernel_end).
787 		 */
788 		memory_map_bottom_up(kernel_end, end);
789 		memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
790 	} else {
791 		memory_map_top_down(ISA_END_ADDRESS, end);
792 	}
793 
794 #ifdef CONFIG_X86_64
795 	if (max_pfn > max_low_pfn) {
796 		/* can we preserve max_low_pfn ?*/
797 		max_low_pfn = max_pfn;
798 	}
799 #else
800 	early_ioremap_page_table_range_init();
801 #endif
802 
803 	load_cr3(swapper_pg_dir);
804 	__flush_tlb_all();
805 
806 	x86_init.hyper.init_mem_mapping();
807 
808 	early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
809 }
810 
811 /*
812  * Initialize an mm_struct to be used during poking and a pointer to be used
813  * during patching.
814  */
poking_init(void)815 void __init poking_init(void)
816 {
817 	spinlock_t *ptl;
818 	pte_t *ptep;
819 
820 	poking_mm = mm_alloc();
821 	BUG_ON(!poking_mm);
822 
823 	/* Xen PV guests need the PGD to be pinned. */
824 	paravirt_enter_mmap(poking_mm);
825 
826 	/*
827 	 * Randomize the poking address, but make sure that the following page
828 	 * will be mapped at the same PMD. We need 2 pages, so find space for 3,
829 	 * and adjust the address if the PMD ends after the first one.
830 	 */
831 	poking_addr = TASK_UNMAPPED_BASE;
832 	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
833 		poking_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) %
834 			(TASK_SIZE - TASK_UNMAPPED_BASE - 3 * PAGE_SIZE);
835 
836 	if (((poking_addr + PAGE_SIZE) & ~PMD_MASK) == 0)
837 		poking_addr += PAGE_SIZE;
838 
839 	/*
840 	 * We need to trigger the allocation of the page-tables that will be
841 	 * needed for poking now. Later, poking may be performed in an atomic
842 	 * section, which might cause allocation to fail.
843 	 */
844 	ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
845 	BUG_ON(!ptep);
846 	pte_unmap_unlock(ptep, ptl);
847 }
848 
849 /*
850  * devmem_is_allowed() checks to see if /dev/mem access to a certain address
851  * is valid. The argument is a physical page number.
852  *
853  * On x86, access has to be given to the first megabyte of RAM because that
854  * area traditionally contains BIOS code and data regions used by X, dosemu,
855  * and similar apps. Since they map the entire memory range, the whole range
856  * must be allowed (for mapping), but any areas that would otherwise be
857  * disallowed are flagged as being "zero filled" instead of rejected.
858  * Access has to be given to non-kernel-ram areas as well, these contain the
859  * PCI mmio resources as well as potential bios/acpi data regions.
860  */
devmem_is_allowed(unsigned long pagenr)861 int devmem_is_allowed(unsigned long pagenr)
862 {
863 	if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,
864 				IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)
865 			!= REGION_DISJOINT) {
866 		/*
867 		 * For disallowed memory regions in the low 1MB range,
868 		 * request that the page be shown as all zeros.
869 		 */
870 		if (pagenr < 256)
871 			return 2;
872 
873 		return 0;
874 	}
875 
876 	/*
877 	 * This must follow RAM test, since System RAM is considered a
878 	 * restricted resource under CONFIG_STRICT_DEVMEM.
879 	 */
880 	if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {
881 		/* Low 1MB bypasses iomem restrictions. */
882 		if (pagenr < 256)
883 			return 1;
884 
885 		return 0;
886 	}
887 
888 	return 1;
889 }
890 
free_init_pages(const char * what,unsigned long begin,unsigned long end)891 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
892 {
893 	unsigned long begin_aligned, end_aligned;
894 
895 	/* Make sure boundaries are page aligned */
896 	begin_aligned = PAGE_ALIGN(begin);
897 	end_aligned   = end & PAGE_MASK;
898 
899 	if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
900 		begin = begin_aligned;
901 		end   = end_aligned;
902 	}
903 
904 	if (begin >= end)
905 		return;
906 
907 	/*
908 	 * If debugging page accesses then do not free this memory but
909 	 * mark them not present - any buggy init-section access will
910 	 * create a kernel page fault:
911 	 */
912 	if (debug_pagealloc_enabled()) {
913 		pr_info("debug: unmapping init [mem %#010lx-%#010lx]\n",
914 			begin, end - 1);
915 		/*
916 		 * Inform kmemleak about the hole in the memory since the
917 		 * corresponding pages will be unmapped.
918 		 */
919 		kmemleak_free_part((void *)begin, end - begin);
920 		set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
921 	} else {
922 		/*
923 		 * We just marked the kernel text read only above, now that
924 		 * we are going to free part of that, we need to make that
925 		 * writeable and non-executable first.
926 		 */
927 		set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
928 		set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
929 
930 		free_reserved_area((void *)begin, (void *)end,
931 				   POISON_FREE_INITMEM, what);
932 	}
933 }
934 
935 /*
936  * begin/end can be in the direct map or the "high kernel mapping"
937  * used for the kernel image only.  free_init_pages() will do the
938  * right thing for either kind of address.
939  */
free_kernel_image_pages(const char * what,void * begin,void * end)940 void free_kernel_image_pages(const char *what, void *begin, void *end)
941 {
942 	unsigned long begin_ul = (unsigned long)begin;
943 	unsigned long end_ul = (unsigned long)end;
944 	unsigned long len_pages = (end_ul - begin_ul) >> PAGE_SHIFT;
945 
946 	free_init_pages(what, begin_ul, end_ul);
947 
948 	/*
949 	 * PTI maps some of the kernel into userspace.  For performance,
950 	 * this includes some kernel areas that do not contain secrets.
951 	 * Those areas might be adjacent to the parts of the kernel image
952 	 * being freed, which may contain secrets.  Remove the "high kernel
953 	 * image mapping" for these freed areas, ensuring they are not even
954 	 * potentially vulnerable to Meltdown regardless of the specific
955 	 * optimizations PTI is currently using.
956 	 *
957 	 * The "noalias" prevents unmapping the direct map alias which is
958 	 * needed to access the freed pages.
959 	 *
960 	 * This is only valid for 64bit kernels. 32bit has only one mapping
961 	 * which can't be treated in this way for obvious reasons.
962 	 */
963 	if (IS_ENABLED(CONFIG_X86_64) && cpu_feature_enabled(X86_FEATURE_PTI))
964 		set_memory_np_noalias(begin_ul, len_pages);
965 }
966 
free_initmem(void)967 void __ref free_initmem(void)
968 {
969 	e820__reallocate_tables();
970 
971 	mem_encrypt_free_decrypted_mem();
972 
973 	free_kernel_image_pages("unused kernel image (initmem)",
974 				&__init_begin, &__init_end);
975 }
976 
977 #ifdef CONFIG_BLK_DEV_INITRD
free_initrd_mem(unsigned long start,unsigned long end)978 void __init free_initrd_mem(unsigned long start, unsigned long end)
979 {
980 	/*
981 	 * end could be not aligned, and We can not align that,
982 	 * decompressor could be confused by aligned initrd_end
983 	 * We already reserve the end partial page before in
984 	 *   - i386_start_kernel()
985 	 *   - x86_64_start_kernel()
986 	 *   - relocate_initrd()
987 	 * So here We can do PAGE_ALIGN() safely to get partial page to be freed
988 	 */
989 	free_init_pages("initrd", start, PAGE_ALIGN(end));
990 }
991 #endif
992 
993 /*
994  * Calculate the precise size of the DMA zone (first 16 MB of RAM),
995  * and pass it to the MM layer - to help it set zone watermarks more
996  * accurately.
997  *
998  * Done on 64-bit systems only for the time being, although 32-bit systems
999  * might benefit from this as well.
1000  */
memblock_find_dma_reserve(void)1001 void __init memblock_find_dma_reserve(void)
1002 {
1003 #ifdef CONFIG_X86_64
1004 	u64 nr_pages = 0, nr_free_pages = 0;
1005 	unsigned long start_pfn, end_pfn;
1006 	phys_addr_t start_addr, end_addr;
1007 	int i;
1008 	u64 u;
1009 
1010 	/*
1011 	 * Iterate over all memory ranges (free and reserved ones alike),
1012 	 * to calculate the total number of pages in the first 16 MB of RAM:
1013 	 */
1014 	nr_pages = 0;
1015 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
1016 		start_pfn = min(start_pfn, MAX_DMA_PFN);
1017 		end_pfn   = min(end_pfn,   MAX_DMA_PFN);
1018 
1019 		nr_pages += end_pfn - start_pfn;
1020 	}
1021 
1022 	/*
1023 	 * Iterate over free memory ranges to calculate the number of free
1024 	 * pages in the DMA zone, while not counting potential partial
1025 	 * pages at the beginning or the end of the range:
1026 	 */
1027 	nr_free_pages = 0;
1028 	for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) {
1029 		start_pfn = min_t(unsigned long, PFN_UP(start_addr), MAX_DMA_PFN);
1030 		end_pfn   = min_t(unsigned long, PFN_DOWN(end_addr), MAX_DMA_PFN);
1031 
1032 		if (start_pfn < end_pfn)
1033 			nr_free_pages += end_pfn - start_pfn;
1034 	}
1035 
1036 	set_dma_reserve(nr_pages - nr_free_pages);
1037 #endif
1038 }
1039 
zone_sizes_init(void)1040 void __init zone_sizes_init(void)
1041 {
1042 	unsigned long max_zone_pfns[MAX_NR_ZONES];
1043 
1044 	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
1045 
1046 #ifdef CONFIG_ZONE_DMA
1047 	max_zone_pfns[ZONE_DMA]		= min(MAX_DMA_PFN, max_low_pfn);
1048 #endif
1049 #ifdef CONFIG_ZONE_DMA32
1050 	max_zone_pfns[ZONE_DMA32]	= min(MAX_DMA32_PFN, max_low_pfn);
1051 #endif
1052 	max_zone_pfns[ZONE_NORMAL]	= max_low_pfn;
1053 #ifdef CONFIG_HIGHMEM
1054 	max_zone_pfns[ZONE_HIGHMEM]	= max_pfn;
1055 #endif
1056 
1057 	free_area_init(max_zone_pfns);
1058 }
1059 
1060 __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = {
1061 	.loaded_mm = &init_mm,
1062 	.next_asid = 1,
1063 	.cr4 = ~0UL,	/* fail hard if we screw up cr4 shadow initialization */
1064 };
1065 
1066 #ifdef CONFIG_ADDRESS_MASKING
1067 DEFINE_PER_CPU(u64, tlbstate_untag_mask);
1068 EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask);
1069 #endif
1070 
update_cache_mode_entry(unsigned entry,enum page_cache_mode cache)1071 void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
1072 {
1073 	/* entry 0 MUST be WB (hardwired to speed up translations) */
1074 	BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB);
1075 
1076 	__cachemode2pte_tbl[cache] = __cm_idx2pte(entry);
1077 	__pte2cachemode_tbl[entry] = cache;
1078 }
1079 
1080 #ifdef CONFIG_SWAP
arch_max_swapfile_size(void)1081 unsigned long arch_max_swapfile_size(void)
1082 {
1083 	unsigned long pages;
1084 
1085 	pages = generic_max_swapfile_size();
1086 
1087 	if (boot_cpu_has_bug(X86_BUG_L1TF) && l1tf_mitigation != L1TF_MITIGATION_OFF) {
1088 		/* Limit the swap file size to MAX_PA/2 for L1TF workaround */
1089 		unsigned long long l1tf_limit = l1tf_pfn_limit();
1090 		/*
1091 		 * We encode swap offsets also with 3 bits below those for pfn
1092 		 * which makes the usable limit higher.
1093 		 */
1094 #if CONFIG_PGTABLE_LEVELS > 2
1095 		l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT;
1096 #endif
1097 		pages = min_t(unsigned long long, l1tf_limit, pages);
1098 	}
1099 	return pages;
1100 }
1101 #endif
1102