1 #include <linux/gfp.h>
2 #include <linux/highmem.h>
3 #include <linux/kernel.h>
4 #include <linux/kmsan-checks.h>
5 #include <linux/mmdebug.h>
6 #include <linux/mm_types.h>
7 #include <linux/mm_inline.h>
8 #include <linux/pagemap.h>
9 #include <linux/rcupdate.h>
10 #include <linux/smp.h>
11 #include <linux/swap.h>
12 
13 #include <asm/pgalloc.h>
14 #include <asm/tlb.h>
15 
16 #ifndef CONFIG_MMU_GATHER_NO_GATHER
17 
tlb_next_batch(struct mmu_gather * tlb)18 static bool tlb_next_batch(struct mmu_gather *tlb)
19 {
20 	struct mmu_gather_batch *batch;
21 
22 	batch = tlb->active;
23 	if (batch->next) {
24 		tlb->active = batch->next;
25 		return true;
26 	}
27 
28 	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
29 		return false;
30 
31 	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
32 	if (!batch)
33 		return false;
34 
35 	tlb->batch_count++;
36 	batch->next = NULL;
37 	batch->nr   = 0;
38 	batch->max  = MAX_GATHER_BATCH;
39 
40 	tlb->active->next = batch;
41 	tlb->active = batch;
42 
43 	return true;
44 }
45 
tlb_batch_pages_flush(struct mmu_gather * tlb)46 static void tlb_batch_pages_flush(struct mmu_gather *tlb)
47 {
48 	struct mmu_gather_batch *batch;
49 
50 	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
51 		struct page **pages = batch->pages;
52 
53 		do {
54 			/*
55 			 * limit free batch count when PAGE_SIZE > 4K
56 			 */
57 			unsigned int nr = min(512U, batch->nr);
58 
59 			free_pages_and_swap_cache(pages, nr);
60 			pages += nr;
61 			batch->nr -= nr;
62 
63 			cond_resched();
64 		} while (batch->nr);
65 	}
66 	tlb->active = &tlb->local;
67 }
68 
tlb_batch_list_free(struct mmu_gather * tlb)69 static void tlb_batch_list_free(struct mmu_gather *tlb)
70 {
71 	struct mmu_gather_batch *batch, *next;
72 
73 	for (batch = tlb->local.next; batch; batch = next) {
74 		next = batch->next;
75 		free_pages((unsigned long)batch, 0);
76 	}
77 	tlb->local.next = NULL;
78 }
79 
__tlb_remove_page_size(struct mmu_gather * tlb,struct page * page,int page_size)80 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
81 {
82 	struct mmu_gather_batch *batch;
83 
84 	VM_BUG_ON(!tlb->end);
85 
86 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
87 	VM_WARN_ON(tlb->page_size != page_size);
88 #endif
89 
90 	batch = tlb->active;
91 	/*
92 	 * Add the page and check if we are full. If so
93 	 * force a flush.
94 	 */
95 	batch->pages[batch->nr++] = page;
96 	if (batch->nr == batch->max) {
97 		if (!tlb_next_batch(tlb))
98 			return true;
99 		batch = tlb->active;
100 	}
101 	VM_BUG_ON_PAGE(batch->nr > batch->max, page);
102 
103 	return false;
104 }
105 
106 #endif /* MMU_GATHER_NO_GATHER */
107 
108 #ifdef CONFIG_MMU_GATHER_TABLE_FREE
109 
__tlb_remove_table_free(struct mmu_table_batch * batch)110 static void __tlb_remove_table_free(struct mmu_table_batch *batch)
111 {
112 	int i;
113 
114 	for (i = 0; i < batch->nr; i++)
115 		__tlb_remove_table(batch->tables[i]);
116 
117 	free_page((unsigned long)batch);
118 }
119 
120 #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
121 
122 /*
123  * Semi RCU freeing of the page directories.
124  *
125  * This is needed by some architectures to implement software pagetable walkers.
126  *
127  * gup_fast() and other software pagetable walkers do a lockless page-table
128  * walk and therefore needs some synchronization with the freeing of the page
129  * directories. The chosen means to accomplish that is by disabling IRQs over
130  * the walk.
131  *
132  * Architectures that use IPIs to flush TLBs will then automagically DTRT,
133  * since we unlink the page, flush TLBs, free the page. Since the disabling of
134  * IRQs delays the completion of the TLB flush we can never observe an already
135  * freed page.
136  *
137  * Architectures that do not have this (PPC) need to delay the freeing by some
138  * other means, this is that means.
139  *
140  * What we do is batch the freed directory pages (tables) and RCU free them.
141  * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
142  * holds off grace periods.
143  *
144  * However, in order to batch these pages we need to allocate storage, this
145  * allocation is deep inside the MM code and can thus easily fail on memory
146  * pressure. To guarantee progress we fall back to single table freeing, see
147  * the implementation of tlb_remove_table_one().
148  *
149  */
150 
tlb_remove_table_smp_sync(void * arg)151 static void tlb_remove_table_smp_sync(void *arg)
152 {
153 	/* Simply deliver the interrupt */
154 }
155 
tlb_remove_table_sync_one(void)156 void tlb_remove_table_sync_one(void)
157 {
158 	/*
159 	 * This isn't an RCU grace period and hence the page-tables cannot be
160 	 * assumed to be actually RCU-freed.
161 	 *
162 	 * It is however sufficient for software page-table walkers that rely on
163 	 * IRQ disabling.
164 	 */
165 	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
166 }
167 
tlb_remove_table_rcu(struct rcu_head * head)168 static void tlb_remove_table_rcu(struct rcu_head *head)
169 {
170 	__tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
171 }
172 
tlb_remove_table_free(struct mmu_table_batch * batch)173 static void tlb_remove_table_free(struct mmu_table_batch *batch)
174 {
175 	call_rcu(&batch->rcu, tlb_remove_table_rcu);
176 }
177 
178 #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
179 
tlb_remove_table_free(struct mmu_table_batch * batch)180 static void tlb_remove_table_free(struct mmu_table_batch *batch)
181 {
182 	__tlb_remove_table_free(batch);
183 }
184 
185 #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
186 
187 /*
188  * If we want tlb_remove_table() to imply TLB invalidates.
189  */
tlb_table_invalidate(struct mmu_gather * tlb)190 static inline void tlb_table_invalidate(struct mmu_gather *tlb)
191 {
192 	if (tlb_needs_table_invalidate()) {
193 		/*
194 		 * Invalidate page-table caches used by hardware walkers. Then
195 		 * we still need to RCU-sched wait while freeing the pages
196 		 * because software walkers can still be in-flight.
197 		 */
198 		tlb_flush_mmu_tlbonly(tlb);
199 	}
200 }
201 
tlb_remove_table_one(void * table)202 static void tlb_remove_table_one(void *table)
203 {
204 	tlb_remove_table_sync_one();
205 	__tlb_remove_table(table);
206 }
207 
tlb_table_flush(struct mmu_gather * tlb)208 static void tlb_table_flush(struct mmu_gather *tlb)
209 {
210 	struct mmu_table_batch **batch = &tlb->batch;
211 
212 	if (*batch) {
213 		tlb_table_invalidate(tlb);
214 		tlb_remove_table_free(*batch);
215 		*batch = NULL;
216 	}
217 }
218 
tlb_remove_table(struct mmu_gather * tlb,void * table)219 void tlb_remove_table(struct mmu_gather *tlb, void *table)
220 {
221 	struct mmu_table_batch **batch = &tlb->batch;
222 
223 	if (*batch == NULL) {
224 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
225 		if (*batch == NULL) {
226 			tlb_table_invalidate(tlb);
227 			tlb_remove_table_one(table);
228 			return;
229 		}
230 		(*batch)->nr = 0;
231 	}
232 
233 	(*batch)->tables[(*batch)->nr++] = table;
234 	if ((*batch)->nr == MAX_TABLE_BATCH)
235 		tlb_table_flush(tlb);
236 }
237 
tlb_table_init(struct mmu_gather * tlb)238 static inline void tlb_table_init(struct mmu_gather *tlb)
239 {
240 	tlb->batch = NULL;
241 }
242 
243 #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
244 
tlb_table_flush(struct mmu_gather * tlb)245 static inline void tlb_table_flush(struct mmu_gather *tlb) { }
tlb_table_init(struct mmu_gather * tlb)246 static inline void tlb_table_init(struct mmu_gather *tlb) { }
247 
248 #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
249 
tlb_flush_mmu_free(struct mmu_gather * tlb)250 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
251 {
252 	tlb_table_flush(tlb);
253 #ifndef CONFIG_MMU_GATHER_NO_GATHER
254 	tlb_batch_pages_flush(tlb);
255 #endif
256 }
257 
tlb_flush_mmu(struct mmu_gather * tlb)258 void tlb_flush_mmu(struct mmu_gather *tlb)
259 {
260 	tlb_flush_mmu_tlbonly(tlb);
261 	tlb_flush_mmu_free(tlb);
262 }
263 
__tlb_gather_mmu(struct mmu_gather * tlb,struct mm_struct * mm,bool fullmm)264 static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
265 			     bool fullmm)
266 {
267 	/*
268 	 * struct mmu_gather contains 7 1-bit fields packed into a 32-bit
269 	 * unsigned int value. The remaining 25 bits remain uninitialized
270 	 * and are never used, but KMSAN updates the origin for them in
271 	 * zap_pXX_range() in mm/memory.c, thus creating very long origin
272 	 * chains. This is technically correct, but consumes too much memory.
273 	 * Unpoisoning the whole structure will prevent creating such chains.
274 	 */
275 	kmsan_unpoison_memory(tlb, sizeof(*tlb));
276 	tlb->mm = mm;
277 	tlb->fullmm = fullmm;
278 
279 #ifndef CONFIG_MMU_GATHER_NO_GATHER
280 	tlb->need_flush_all = 0;
281 	tlb->local.next = NULL;
282 	tlb->local.nr   = 0;
283 	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
284 	tlb->active     = &tlb->local;
285 	tlb->batch_count = 0;
286 #endif
287 
288 	tlb_table_init(tlb);
289 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
290 	tlb->page_size = 0;
291 #endif
292 
293 	__tlb_reset_range(tlb);
294 	inc_tlb_flush_pending(tlb->mm);
295 }
296 
297 /**
298  * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
299  * @tlb: the mmu_gather structure to initialize
300  * @mm: the mm_struct of the target address space
301  *
302  * Called to initialize an (on-stack) mmu_gather structure for page-table
303  * tear-down from @mm.
304  */
tlb_gather_mmu(struct mmu_gather * tlb,struct mm_struct * mm)305 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm)
306 {
307 	__tlb_gather_mmu(tlb, mm, false);
308 }
309 
310 /**
311  * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down
312  * @tlb: the mmu_gather structure to initialize
313  * @mm: the mm_struct of the target address space
314  *
315  * In this case, @mm is without users and we're going to destroy the
316  * full address space (exit/execve).
317  *
318  * Called to initialize an (on-stack) mmu_gather structure for page-table
319  * tear-down from @mm.
320  */
tlb_gather_mmu_fullmm(struct mmu_gather * tlb,struct mm_struct * mm)321 void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
322 {
323 	__tlb_gather_mmu(tlb, mm, true);
324 }
325 
326 /**
327  * tlb_finish_mmu - finish an mmu_gather structure
328  * @tlb: the mmu_gather structure to finish
329  *
330  * Called at the end of the shootdown operation to free up any resources that
331  * were required.
332  */
tlb_finish_mmu(struct mmu_gather * tlb)333 void tlb_finish_mmu(struct mmu_gather *tlb)
334 {
335 	/*
336 	 * If there are parallel threads are doing PTE changes on same range
337 	 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
338 	 * flush by batching, one thread may end up seeing inconsistent PTEs
339 	 * and result in having stale TLB entries.  So flush TLB forcefully
340 	 * if we detect parallel PTE batching threads.
341 	 *
342 	 * However, some syscalls, e.g. munmap(), may free page tables, this
343 	 * needs force flush everything in the given range. Otherwise this
344 	 * may result in having stale TLB entries for some architectures,
345 	 * e.g. aarch64, that could specify flush what level TLB.
346 	 */
347 	if (mm_tlb_flush_nested(tlb->mm)) {
348 		/*
349 		 * The aarch64 yields better performance with fullmm by
350 		 * avoiding multiple CPUs spamming TLBI messages at the
351 		 * same time.
352 		 *
353 		 * On x86 non-fullmm doesn't yield significant difference
354 		 * against fullmm.
355 		 */
356 		tlb->fullmm = 1;
357 		__tlb_reset_range(tlb);
358 		tlb->freed_tables = 1;
359 	}
360 
361 	tlb_flush_mmu(tlb);
362 
363 #ifndef CONFIG_MMU_GATHER_NO_GATHER
364 	tlb_batch_list_free(tlb);
365 #endif
366 	dec_tlb_flush_pending(tlb->mm);
367 }
368