1 /*
2 * native hashtable management.
3 *
4 * SMP scalability work:
5 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 */
12
13 #undef DEBUG_LOW
14
15 #include <linux/spinlock.h>
16 #include <linux/bitops.h>
17 #include <linux/threads.h>
18 #include <linux/smp.h>
19
20 #include <asm/abs_addr.h>
21 #include <asm/machdep.h>
22 #include <asm/mmu.h>
23 #include <asm/mmu_context.h>
24 #include <asm/pgtable.h>
25 #include <asm/tlbflush.h>
26 #include <asm/tlb.h>
27 #include <asm/cputable.h>
28 #include <asm/udbg.h>
29 #include <asm/kexec.h>
30 #include <asm/ppc-opcode.h>
31
32 #ifdef DEBUG_LOW
33 #define DBG_LOW(fmt...) udbg_printf(fmt)
34 #else
35 #define DBG_LOW(fmt...)
36 #endif
37
38 #define HPTE_LOCK_BIT 3
39
40 static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
41
__tlbie(unsigned long va,int psize,int ssize)42 static inline void __tlbie(unsigned long va, int psize, int ssize)
43 {
44 unsigned int penc;
45
46 /* clear top 16 bits, non SLS segment */
47 va &= ~(0xffffULL << 48);
48
49 switch (psize) {
50 case MMU_PAGE_4K:
51 va &= ~0xffful;
52 va |= ssize << 8;
53 asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0),
54 %2)
55 : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206)
56 : "memory");
57 break;
58 default:
59 penc = mmu_psize_defs[psize].penc;
60 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
61 va |= penc << 12;
62 va |= ssize << 8;
63 va |= 1; /* L */
64 asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0),
65 %2)
66 : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206)
67 : "memory");
68 break;
69 }
70 }
71
__tlbiel(unsigned long va,int psize,int ssize)72 static inline void __tlbiel(unsigned long va, int psize, int ssize)
73 {
74 unsigned int penc;
75
76 /* clear top 16 bits, non SLS segment */
77 va &= ~(0xffffULL << 48);
78
79 switch (psize) {
80 case MMU_PAGE_4K:
81 va &= ~0xffful;
82 va |= ssize << 8;
83 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
84 : : "r"(va) : "memory");
85 break;
86 default:
87 penc = mmu_psize_defs[psize].penc;
88 va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
89 va |= penc << 12;
90 va |= ssize << 8;
91 va |= 1; /* L */
92 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
93 : : "r"(va) : "memory");
94 break;
95 }
96
97 }
98
tlbie(unsigned long va,int psize,int ssize,int local)99 static inline void tlbie(unsigned long va, int psize, int ssize, int local)
100 {
101 unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);
102 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
103
104 if (use_local)
105 use_local = mmu_psize_defs[psize].tlbiel;
106 if (lock_tlbie && !use_local)
107 raw_spin_lock(&native_tlbie_lock);
108 asm volatile("ptesync": : :"memory");
109 if (use_local) {
110 __tlbiel(va, psize, ssize);
111 asm volatile("ptesync": : :"memory");
112 } else {
113 __tlbie(va, psize, ssize);
114 asm volatile("eieio; tlbsync; ptesync": : :"memory");
115 }
116 if (lock_tlbie && !use_local)
117 raw_spin_unlock(&native_tlbie_lock);
118 }
119
native_lock_hpte(struct hash_pte * hptep)120 static inline void native_lock_hpte(struct hash_pte *hptep)
121 {
122 unsigned long *word = &hptep->v;
123
124 while (1) {
125 if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
126 break;
127 while(test_bit(HPTE_LOCK_BIT, word))
128 cpu_relax();
129 }
130 }
131
native_unlock_hpte(struct hash_pte * hptep)132 static inline void native_unlock_hpte(struct hash_pte *hptep)
133 {
134 unsigned long *word = &hptep->v;
135
136 clear_bit_unlock(HPTE_LOCK_BIT, word);
137 }
138
native_hpte_insert(unsigned long hpte_group,unsigned long va,unsigned long pa,unsigned long rflags,unsigned long vflags,int psize,int ssize)139 static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
140 unsigned long pa, unsigned long rflags,
141 unsigned long vflags, int psize, int ssize)
142 {
143 struct hash_pte *hptep = htab_address + hpte_group;
144 unsigned long hpte_v, hpte_r;
145 int i;
146
147 if (!(vflags & HPTE_V_BOLTED)) {
148 DBG_LOW(" insert(group=%lx, va=%016lx, pa=%016lx,"
149 " rflags=%lx, vflags=%lx, psize=%d)\n",
150 hpte_group, va, pa, rflags, vflags, psize);
151 }
152
153 for (i = 0; i < HPTES_PER_GROUP; i++) {
154 if (! (hptep->v & HPTE_V_VALID)) {
155 /* retry with lock held */
156 native_lock_hpte(hptep);
157 if (! (hptep->v & HPTE_V_VALID))
158 break;
159 native_unlock_hpte(hptep);
160 }
161
162 hptep++;
163 }
164
165 if (i == HPTES_PER_GROUP)
166 return -1;
167
168 hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID;
169 hpte_r = hpte_encode_r(pa, psize) | rflags;
170
171 if (!(vflags & HPTE_V_BOLTED)) {
172 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
173 i, hpte_v, hpte_r);
174 }
175
176 hptep->r = hpte_r;
177 /* Guarantee the second dword is visible before the valid bit */
178 eieio();
179 /*
180 * Now set the first dword including the valid bit
181 * NOTE: this also unlocks the hpte
182 */
183 hptep->v = hpte_v;
184
185 __asm__ __volatile__ ("ptesync" : : : "memory");
186
187 return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
188 }
189
native_hpte_remove(unsigned long hpte_group)190 static long native_hpte_remove(unsigned long hpte_group)
191 {
192 struct hash_pte *hptep;
193 int i;
194 int slot_offset;
195 unsigned long hpte_v;
196
197 DBG_LOW(" remove(group=%lx)\n", hpte_group);
198
199 /* pick a random entry to start at */
200 slot_offset = mftb() & 0x7;
201
202 for (i = 0; i < HPTES_PER_GROUP; i++) {
203 hptep = htab_address + hpte_group + slot_offset;
204 hpte_v = hptep->v;
205
206 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
207 /* retry with lock held */
208 native_lock_hpte(hptep);
209 hpte_v = hptep->v;
210 if ((hpte_v & HPTE_V_VALID)
211 && !(hpte_v & HPTE_V_BOLTED))
212 break;
213 native_unlock_hpte(hptep);
214 }
215
216 slot_offset++;
217 slot_offset &= 0x7;
218 }
219
220 if (i == HPTES_PER_GROUP)
221 return -1;
222
223 /* Invalidate the hpte. NOTE: this also unlocks it */
224 hptep->v = 0;
225
226 return i;
227 }
228
native_hpte_updatepp(unsigned long slot,unsigned long newpp,unsigned long va,int psize,int ssize,int local)229 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
230 unsigned long va, int psize, int ssize,
231 int local)
232 {
233 struct hash_pte *hptep = htab_address + slot;
234 unsigned long hpte_v, want_v;
235 int ret = 0;
236
237 want_v = hpte_encode_v(va, psize, ssize);
238
239 DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",
240 va, want_v & HPTE_V_AVPN, slot, newpp);
241
242 native_lock_hpte(hptep);
243
244 hpte_v = hptep->v;
245
246 /* Even if we miss, we need to invalidate the TLB */
247 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
248 DBG_LOW(" -> miss\n");
249 ret = -1;
250 } else {
251 DBG_LOW(" -> hit\n");
252 /* Update the HPTE */
253 hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
254 (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C));
255 }
256 native_unlock_hpte(hptep);
257
258 /* Ensure it is out of the tlb too. */
259 tlbie(va, psize, ssize, local);
260
261 return ret;
262 }
263
native_hpte_find(unsigned long va,int psize,int ssize)264 static long native_hpte_find(unsigned long va, int psize, int ssize)
265 {
266 struct hash_pte *hptep;
267 unsigned long hash;
268 unsigned long i;
269 long slot;
270 unsigned long want_v, hpte_v;
271
272 hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize);
273 want_v = hpte_encode_v(va, psize, ssize);
274
275 /* Bolted mappings are only ever in the primary group */
276 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
277 for (i = 0; i < HPTES_PER_GROUP; i++) {
278 hptep = htab_address + slot;
279 hpte_v = hptep->v;
280
281 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
282 /* HPTE matches */
283 return slot;
284 ++slot;
285 }
286
287 return -1;
288 }
289
290 /*
291 * Update the page protection bits. Intended to be used to create
292 * guard pages for kernel data structures on pages which are bolted
293 * in the HPT. Assumes pages being operated on will not be stolen.
294 *
295 * No need to lock here because we should be the only user.
296 */
native_hpte_updateboltedpp(unsigned long newpp,unsigned long ea,int psize,int ssize)297 static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
298 int psize, int ssize)
299 {
300 unsigned long vsid, va;
301 long slot;
302 struct hash_pte *hptep;
303
304 vsid = get_kernel_vsid(ea, ssize);
305 va = hpt_va(ea, vsid, ssize);
306
307 slot = native_hpte_find(va, psize, ssize);
308 if (slot == -1)
309 panic("could not find page to bolt\n");
310 hptep = htab_address + slot;
311
312 /* Update the HPTE */
313 hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
314 (newpp & (HPTE_R_PP | HPTE_R_N));
315
316 /* Ensure it is out of the tlb too. */
317 tlbie(va, psize, ssize, 0);
318 }
319
native_hpte_invalidate(unsigned long slot,unsigned long va,int psize,int ssize,int local)320 static void native_hpte_invalidate(unsigned long slot, unsigned long va,
321 int psize, int ssize, int local)
322 {
323 struct hash_pte *hptep = htab_address + slot;
324 unsigned long hpte_v;
325 unsigned long want_v;
326 unsigned long flags;
327
328 local_irq_save(flags);
329
330 DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot);
331
332 want_v = hpte_encode_v(va, psize, ssize);
333 native_lock_hpte(hptep);
334 hpte_v = hptep->v;
335
336 /* Even if we miss, we need to invalidate the TLB */
337 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
338 native_unlock_hpte(hptep);
339 else
340 /* Invalidate the hpte. NOTE: this also unlocks it */
341 hptep->v = 0;
342
343 /* Invalidate the TLB */
344 tlbie(va, psize, ssize, local);
345
346 local_irq_restore(flags);
347 }
348
349 #define LP_SHIFT 12
350 #define LP_BITS 8
351 #define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT)
352
hpte_decode(struct hash_pte * hpte,unsigned long slot,int * psize,int * ssize,unsigned long * va)353 static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
354 int *psize, int *ssize, unsigned long *va)
355 {
356 unsigned long hpte_r = hpte->r;
357 unsigned long hpte_v = hpte->v;
358 unsigned long avpn;
359 int i, size, shift, penc;
360
361 if (!(hpte_v & HPTE_V_LARGE))
362 size = MMU_PAGE_4K;
363 else {
364 for (i = 0; i < LP_BITS; i++) {
365 if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1))
366 break;
367 }
368 penc = LP_MASK(i+1) >> LP_SHIFT;
369 for (size = 0; size < MMU_PAGE_COUNT; size++) {
370
371 /* 4K pages are not represented by LP */
372 if (size == MMU_PAGE_4K)
373 continue;
374
375 /* valid entries have a shift value */
376 if (!mmu_psize_defs[size].shift)
377 continue;
378
379 if (penc == mmu_psize_defs[size].penc)
380 break;
381 }
382 }
383
384 /* This works for all page sizes, and for 256M and 1T segments */
385 shift = mmu_psize_defs[size].shift;
386 avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm) << 23;
387
388 if (shift < 23) {
389 unsigned long vpi, vsid, pteg;
390
391 pteg = slot / HPTES_PER_GROUP;
392 if (hpte_v & HPTE_V_SECONDARY)
393 pteg = ~pteg;
394 switch (hpte_v >> HPTE_V_SSIZE_SHIFT) {
395 case MMU_SEGSIZE_256M:
396 vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask;
397 break;
398 case MMU_SEGSIZE_1T:
399 vsid = avpn >> 40;
400 vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
401 break;
402 default:
403 avpn = vpi = size = 0;
404 }
405 avpn |= (vpi << mmu_psize_defs[size].shift);
406 }
407
408 *va = avpn;
409 *psize = size;
410 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
411 }
412
413 /*
414 * clear all mappings on kexec. All cpus are in real mode (or they will
415 * be when they isi), and we are the only one left. We rely on our kernel
416 * mapping being 0xC0's and the hardware ignoring those two real bits.
417 *
418 * TODO: add batching support when enabled. remember, no dynamic memory here,
419 * athough there is the control page available...
420 */
native_hpte_clear(void)421 static void native_hpte_clear(void)
422 {
423 unsigned long slot, slots, flags;
424 struct hash_pte *hptep = htab_address;
425 unsigned long hpte_v, va;
426 unsigned long pteg_count;
427 int psize, ssize;
428
429 pteg_count = htab_hash_mask + 1;
430
431 local_irq_save(flags);
432
433 /* we take the tlbie lock and hold it. Some hardware will
434 * deadlock if we try to tlbie from two processors at once.
435 */
436 raw_spin_lock(&native_tlbie_lock);
437
438 slots = pteg_count * HPTES_PER_GROUP;
439
440 for (slot = 0; slot < slots; slot++, hptep++) {
441 /*
442 * we could lock the pte here, but we are the only cpu
443 * running, right? and for crash dump, we probably
444 * don't want to wait for a maybe bad cpu.
445 */
446 hpte_v = hptep->v;
447
448 /*
449 * Call __tlbie() here rather than tlbie() since we
450 * already hold the native_tlbie_lock.
451 */
452 if (hpte_v & HPTE_V_VALID) {
453 hpte_decode(hptep, slot, &psize, &ssize, &va);
454 hptep->v = 0;
455 __tlbie(va, psize, ssize);
456 }
457 }
458
459 asm volatile("eieio; tlbsync; ptesync":::"memory");
460 raw_spin_unlock(&native_tlbie_lock);
461 local_irq_restore(flags);
462 }
463
464 /*
465 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
466 * the lock all the time
467 */
native_flush_hash_range(unsigned long number,int local)468 static void native_flush_hash_range(unsigned long number, int local)
469 {
470 unsigned long va, hash, index, hidx, shift, slot;
471 struct hash_pte *hptep;
472 unsigned long hpte_v;
473 unsigned long want_v;
474 unsigned long flags;
475 real_pte_t pte;
476 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
477 unsigned long psize = batch->psize;
478 int ssize = batch->ssize;
479 int i;
480
481 local_irq_save(flags);
482
483 for (i = 0; i < number; i++) {
484 va = batch->vaddr[i];
485 pte = batch->pte[i];
486
487 pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
488 hash = hpt_hash(va, shift, ssize);
489 hidx = __rpte_to_hidx(pte, index);
490 if (hidx & _PTEIDX_SECONDARY)
491 hash = ~hash;
492 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
493 slot += hidx & _PTEIDX_GROUP_IX;
494 hptep = htab_address + slot;
495 want_v = hpte_encode_v(va, psize, ssize);
496 native_lock_hpte(hptep);
497 hpte_v = hptep->v;
498 if (!HPTE_V_COMPARE(hpte_v, want_v) ||
499 !(hpte_v & HPTE_V_VALID))
500 native_unlock_hpte(hptep);
501 else
502 hptep->v = 0;
503 } pte_iterate_hashed_end();
504 }
505
506 if (cpu_has_feature(CPU_FTR_TLBIEL) &&
507 mmu_psize_defs[psize].tlbiel && local) {
508 asm volatile("ptesync":::"memory");
509 for (i = 0; i < number; i++) {
510 va = batch->vaddr[i];
511 pte = batch->pte[i];
512
513 pte_iterate_hashed_subpages(pte, psize, va, index,
514 shift) {
515 __tlbiel(va, psize, ssize);
516 } pte_iterate_hashed_end();
517 }
518 asm volatile("ptesync":::"memory");
519 } else {
520 int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
521
522 if (lock_tlbie)
523 raw_spin_lock(&native_tlbie_lock);
524
525 asm volatile("ptesync":::"memory");
526 for (i = 0; i < number; i++) {
527 va = batch->vaddr[i];
528 pte = batch->pte[i];
529
530 pte_iterate_hashed_subpages(pte, psize, va, index,
531 shift) {
532 __tlbie(va, psize, ssize);
533 } pte_iterate_hashed_end();
534 }
535 asm volatile("eieio; tlbsync; ptesync":::"memory");
536
537 if (lock_tlbie)
538 raw_spin_unlock(&native_tlbie_lock);
539 }
540
541 local_irq_restore(flags);
542 }
543
544 #ifdef CONFIG_PPC_PSERIES
545 /* Disable TLB batching on nighthawk */
tlb_batching_enabled(void)546 static inline int tlb_batching_enabled(void)
547 {
548 struct device_node *root = of_find_node_by_path("/");
549 int enabled = 1;
550
551 if (root) {
552 const char *model = of_get_property(root, "model", NULL);
553 if (model && !strcmp(model, "IBM,9076-N81"))
554 enabled = 0;
555 of_node_put(root);
556 }
557
558 return enabled;
559 }
560 #else
tlb_batching_enabled(void)561 static inline int tlb_batching_enabled(void)
562 {
563 return 1;
564 }
565 #endif
566
hpte_init_native(void)567 void __init hpte_init_native(void)
568 {
569 ppc_md.hpte_invalidate = native_hpte_invalidate;
570 ppc_md.hpte_updatepp = native_hpte_updatepp;
571 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
572 ppc_md.hpte_insert = native_hpte_insert;
573 ppc_md.hpte_remove = native_hpte_remove;
574 ppc_md.hpte_clear_all = native_hpte_clear;
575 if (tlb_batching_enabled())
576 ppc_md.flush_hash_range = native_flush_hash_range;
577 }
578