1 /*
2 * PowerPC64 port by Mike Corrigan and Dave Engebretsen
3 * {mikejc|engebret}@us.ibm.com
4 *
5 * Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
6 *
7 * SMP scalability work:
8 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
9 *
10 * Module name: htab.c
11 *
12 * Description:
13 * PowerPC Hashed Page Table functions
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
21 #include <linux/config.h>
22 #include <linux/spinlock.h>
23 #include <linux/errno.h>
24 #include <linux/sched.h>
25 #include <linux/proc_fs.h>
26 #include <linux/stat.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/cache.h>
30
31 #include <asm/ppcdebug.h>
32 #include <asm/processor.h>
33 #include <asm/pgtable.h>
34 #include <asm/mmu.h>
35 #include <asm/mmu_context.h>
36 #include <asm/page.h>
37 #include <asm/types.h>
38 #include <asm/uaccess.h>
39 #include <asm/naca.h>
40 #include <asm/pmc.h>
41 #include <asm/machdep.h>
42 #include <asm/lmb.h>
43 #include <asm/abs_addr.h>
44 #include <asm/io.h>
45 #include <asm/eeh.h>
46 #include <asm/hvcall.h>
47 #include <asm/iSeries/LparData.h>
48 #include <asm/iSeries/HvCallHpt.h>
49 #include <asm/cputable.h>
50
51 /*
52 * Note: pte --> Linux PTE
53 * HPTE --> PowerPC Hashed Page Table Entry
54 *
55 * Execution context:
56 * htab_initialize is called with the MMU off (of course), but
57 * the kernel has been copied down to zero so it can directly
58 * reference global data. At this point it is very difficult
59 * to print debug info.
60 *
61 */
62
63 HTAB htab_data = {NULL, 0, 0, 0, 0};
64
65 extern unsigned long _SDR1;
66 extern unsigned long klimit;
67
68 void make_pte(HPTE *htab, unsigned long va, unsigned long pa,
69 int mode, unsigned long hash_mask, int large);
70 long plpar_pte_enter(unsigned long flags,
71 unsigned long ptex,
72 unsigned long new_pteh, unsigned long new_ptel,
73 unsigned long *old_pteh_ret, unsigned long *old_ptel_ret);
74 static long hpte_remove(unsigned long hpte_group);
75 static long rpa_lpar_hpte_remove(unsigned long hpte_group);
76 static long iSeries_hpte_remove(unsigned long hpte_group);
77 inline unsigned long get_lock_slot(unsigned long vpn);
78
79 static spinlock_t pSeries_tlbie_lock = SPIN_LOCK_UNLOCKED;
80 static spinlock_t pSeries_lpar_tlbie_lock = SPIN_LOCK_UNLOCKED;
81
82 #define LOCK_SPLIT
83 #ifdef LOCK_SPLIT
84 hash_table_lock_t hash_table_lock[128] __cacheline_aligned_in_smp = { [0 ... 31] = {SPIN_LOCK_UNLOCKED}};
85 #else
86 hash_table_lock_t hash_table_lock[1] __cacheline_aligned_in_smp = { [0] = {SPIN_LOCK_UNLOCKED}};
87 #endif
88
89 #define KB (1024)
90 #define MB (1024*KB)
91
92 static inline void
loop_forever(void)93 loop_forever(void)
94 {
95 volatile unsigned long x = 1;
96 for(;x;x|=1)
97 ;
98 }
99
100 static inline void
create_pte_mapping(unsigned long start,unsigned long end,unsigned long mode,unsigned long mask,int large)101 create_pte_mapping(unsigned long start, unsigned long end,
102 unsigned long mode, unsigned long mask, int large)
103 {
104 unsigned long addr;
105 HPTE *htab = (HPTE *)__v2a(htab_data.htab);
106 unsigned int step;
107
108 if (large)
109 step = 16*MB;
110 else
111 step = 4*KB;
112
113 for (addr = start; addr < end; addr += step) {
114 unsigned long vsid = get_kernel_vsid(addr);
115 unsigned long va = (vsid << 28) | (addr & 0xfffffff);
116 make_pte(htab, va, (unsigned long)__v2a(addr),
117 mode, mask, large);
118 }
119 }
120
121 void
htab_initialize(void)122 htab_initialize(void)
123 {
124 unsigned long table, htab_size_bytes;
125 unsigned long pteg_count;
126 unsigned long mode_rw, mask, lock_shift;
127
128 #if 0
129 /* Can't really do the call below since it calls the normal RTAS
130 * entry point and we're still relocate off at the moment.
131 * Temporarily diabling until it can call through the relocate off
132 * RTAS entry point. -Peter
133 */
134 ppc64_boot_msg(0x05, "htab init");
135 #endif
136 /*
137 * Calculate the required size of the htab. We want the number of
138 * PTEGs to equal one half the number of real pages.
139 */
140 htab_size_bytes = 1UL << naca->pftSize;
141 pteg_count = htab_size_bytes >> 7;
142
143 /* For debug, make the HTAB 1/8 as big as it normally would be. */
144 ifppcdebug(PPCDBG_HTABSIZE) {
145 pteg_count >>= 3;
146 htab_size_bytes = pteg_count << 7;
147 }
148
149 htab_data.htab_num_ptegs = pteg_count;
150 htab_data.htab_hash_mask = pteg_count - 1;
151
152 /*
153 * Calculate the number of bits to shift the pteg selector such that we
154 * use the high order 8 bits to select a page table lock.
155 */
156 asm ("cntlzd %0,%1" : "=r" (lock_shift) :
157 "r" (htab_data.htab_hash_mask));
158 htab_data.htab_lock_shift = (64 - lock_shift) - 8;
159
160 if(systemcfg->platform == PLATFORM_PSERIES) {
161 /* Find storage for the HPT. Must be contiguous in
162 * the absolute address space.
163 */
164 table = lmb_alloc(htab_size_bytes, htab_size_bytes);
165 if ( !table ) {
166 ppc64_terminate_msg(0x20, "hpt space");
167 loop_forever();
168 }
169 htab_data.htab = (HPTE *)__a2v(table);
170
171 /* htab absolute addr + encoded htabsize */
172 _SDR1 = table + __ilog2(pteg_count) - 11;
173
174 /* Initialize the HPT with no entries */
175 memset((void *)table, 0, htab_size_bytes);
176 } else {
177 /* Using a hypervisor which owns the htab */
178 htab_data.htab = NULL;
179 _SDR1 = 0;
180 }
181
182 mode_rw = _PAGE_ACCESSED | _PAGE_COHERENT | PP_RWXX;
183 mask = pteg_count-1;
184
185 /* XXX we currently map kernel text rw, should fix this */
186 if ((systemcfg->platform & PLATFORM_PSERIES) &&
187 (cur_cpu_spec->cpu_features &
188 CPU_FTR_16M_PAGE) &&
189 (systemcfg->physicalMemorySize > 256*MB)) {
190 create_pte_mapping((unsigned long)KERNELBASE,
191 KERNELBASE + 256*MB, mode_rw, mask, 0);
192 create_pte_mapping((unsigned long)KERNELBASE + 256*MB,
193 KERNELBASE + (systemcfg->physicalMemorySize),
194 mode_rw, mask, 1);
195 } else {
196 create_pte_mapping((unsigned long)KERNELBASE,
197 KERNELBASE+(systemcfg->physicalMemorySize),
198 mode_rw, mask, 0);
199 }
200 #if 0
201 /* Can't really do the call below since it calls the normal RTAS
202 * entry point and we're still relocate off at the moment.
203 * Temporarily diabling until it can call through the relocate off
204 * RTAS entry point. -Peter
205 */
206 ppc64_boot_msg(0x06, "htab done");
207 #endif
208 }
209 #undef KB
210 #undef MB
211
212 /*
213 * Create a pte. Used during initialization only.
214 * We assume the PTE will fit in the primary PTEG.
215 */
make_pte(HPTE * htab,unsigned long va,unsigned long pa,int mode,unsigned long hash_mask,int large)216 void make_pte(HPTE *htab, unsigned long va, unsigned long pa,
217 int mode, unsigned long hash_mask, int large)
218 {
219 HPTE *hptep, local_hpte, rhpte;
220 unsigned long hash, vpn, flags, lpar_rc;
221 unsigned long i, dummy1, dummy2;
222 long slot;
223
224 if (large)
225 vpn = va >> LARGE_PAGE_SHIFT;
226 else
227 vpn = va >> PAGE_SHIFT;
228
229 hash = hpt_hash(vpn, large);
230
231 local_hpte.dw1.dword1 = pa | mode;
232 local_hpte.dw0.dword0 = 0;
233 local_hpte.dw0.dw0.avpn = va >> 23;
234 local_hpte.dw0.dw0.bolted = 1; /* bolted */
235 if (large) {
236 local_hpte.dw0.dw0.l = 1; /* large page */
237 local_hpte.dw0.dw0.avpn &= ~0x1UL;
238 }
239 local_hpte.dw0.dw0.v = 1;
240
241 if (systemcfg->platform == PLATFORM_PSERIES) {
242 hptep = htab + ((hash & hash_mask)*HPTES_PER_GROUP);
243
244 for (i = 0; i < 8; ++i, ++hptep) {
245 if (hptep->dw0.dw0.v == 0) { /* !valid */
246 *hptep = local_hpte;
247 return;
248 }
249 }
250 } else if (systemcfg->platform == PLATFORM_PSERIES_LPAR) {
251 slot = ((hash & hash_mask)*HPTES_PER_GROUP);
252
253 /* Set CEC cookie to 0 */
254 /* Zero page = 0 */
255 /* I-cache Invalidate = 0 */
256 /* I-cache synchronize = 0 */
257 /* Exact = 0 - modify any entry in group */
258 flags = 0;
259
260 lpar_rc = plpar_pte_enter(flags, slot, local_hpte.dw0.dword0,
261 local_hpte.dw1.dword1,
262 &dummy1, &dummy2);
263 return;
264 } else if (systemcfg->platform == PLATFORM_ISERIES_LPAR) {
265 slot = HvCallHpt_findValid(&rhpte, vpn);
266 if (slot < 0) {
267 /* Must find space in primary group */
268 panic("hash_page: hpte already exists\n");
269 }
270 HvCallHpt_addValidate(slot, 0, (HPTE *)&local_hpte );
271 return;
272 }
273
274 /* We should _never_ get here and too early to call xmon. */
275 ppc64_terminate_msg(0x22, "hpte platform");
276 loop_forever();
277 }
278
279 /*
280 * find_linux_pte returns the address of a linux pte for a given
281 * effective address and directory. If not found, it returns zero.
282 */
find_linux_pte(pgd_t * pgdir,unsigned long ea)283 pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea)
284 {
285 pgd_t *pg;
286 pmd_t *pm;
287 pte_t *pt = NULL;
288 pte_t pte;
289
290 pg = pgdir + pgd_index(ea);
291 if (!pgd_none(*pg)) {
292 pm = pmd_offset(pg, ea);
293 if (!pmd_none(*pm)) {
294 pt = pte_offset(pm, ea);
295 pte = *pt;
296 if (!pte_present(pte))
297 pt = NULL;
298 }
299 }
300
301 return pt;
302 }
303
computeHptePP(unsigned long pte)304 static inline unsigned long computeHptePP(unsigned long pte)
305 {
306 return (pte & _PAGE_USER) |
307 (((pte & _PAGE_USER) >> 1) &
308 ((~((pte >> 2) & /* _PAGE_RW */
309 (pte >> 7))) & /* _PAGE_DIRTY */
310 1));
311 }
312
313 /*
314 * Handle a fault by adding an HPTE. If the address can't be determined
315 * to be valid via Linux page tables, return 1. If handled return 0
316 */
__hash_page(unsigned long ea,unsigned long access,unsigned long vsid,pte_t * ptep)317 int __hash_page(unsigned long ea, unsigned long access,
318 unsigned long vsid, pte_t *ptep)
319 {
320 unsigned long va, vpn;
321 unsigned long newpp, prpn;
322 unsigned long hpteflags, lock_slot;
323 long slot;
324 pte_t old_pte, new_pte;
325
326 /* Search the Linux page table for a match with va */
327 va = (vsid << 28) | (ea & 0x0fffffff);
328 vpn = va >> PAGE_SHIFT;
329 lock_slot = get_lock_slot(vpn);
330
331 /* Acquire the hash table lock to guarantee that the linux
332 * pte we fetch will not change
333 */
334 spin_lock(&hash_table_lock[lock_slot].lock);
335
336 /*
337 * Check the user's access rights to the page. If access should be
338 * prevented then send the problem up to do_page_fault.
339 */
340 #ifdef CONFIG_SHARED_MEMORY_ADDRESSING
341 access |= _PAGE_PRESENT;
342 if (unlikely(access & ~(pte_val(*ptep)))) {
343 if(!(((ea >> SMALLOC_EA_SHIFT) ==
344 (SMALLOC_START >> SMALLOC_EA_SHIFT)) &&
345 ((current->thread.flags) & PPC_FLAG_SHARED))) {
346 spin_unlock(&hash_table_lock[lock_slot].lock);
347 return 1;
348 }
349 }
350 #else
351 access |= _PAGE_PRESENT;
352 if (unlikely(access & ~(pte_val(*ptep)))) {
353 spin_unlock(&hash_table_lock[lock_slot].lock);
354 return 1;
355 }
356 #endif
357
358 /*
359 * We have found a pte (which was present).
360 * The spinlocks prevent this status from changing
361 * The hash_table_lock prevents the _PAGE_HASHPTE status
362 * from changing (RPN, DIRTY and ACCESSED too)
363 * The page_table_lock prevents the pte from being
364 * invalidated or modified
365 */
366
367 /*
368 * At this point, we have a pte (old_pte) which can be used to build
369 * or update an HPTE. There are 2 cases:
370 *
371 * 1. There is a valid (present) pte with no associated HPTE (this is
372 * the most common case)
373 * 2. There is a valid (present) pte with an associated HPTE. The
374 * current values of the pp bits in the HPTE prevent access
375 * because we are doing software DIRTY bit management and the
376 * page is currently not DIRTY.
377 */
378
379 old_pte = *ptep;
380 new_pte = old_pte;
381
382 /* If the attempted access was a store */
383 if (access & _PAGE_RW)
384 pte_val(new_pte) |= _PAGE_ACCESSED | _PAGE_DIRTY;
385 else
386 pte_val(new_pte) |= _PAGE_ACCESSED;
387
388 newpp = computeHptePP(pte_val(new_pte));
389
390 /* Check if pte already has an hpte (case 2) */
391 if (unlikely(pte_val(old_pte) & _PAGE_HASHPTE)) {
392 /* There MIGHT be an HPTE for this pte */
393 unsigned long hash, slot, secondary;
394
395 /* XXX fix large pte flag */
396 hash = hpt_hash(vpn, 0);
397 secondary = (pte_val(old_pte) & _PAGE_SECONDARY) >> 15;
398 if (secondary)
399 hash = ~hash;
400 slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
401 slot += (pte_val(old_pte) & _PAGE_GROUP_IX) >> 12;
402
403 /* XXX fix large pte flag */
404 if (ppc_md.hpte_updatepp(slot, secondary,
405 newpp, va, 0) == -1) {
406 pte_val(old_pte) &= ~_PAGE_HPTEFLAGS;
407 } else {
408 if (!pte_same(old_pte, new_pte)) {
409 *ptep = new_pte;
410 }
411 }
412 }
413
414 if (likely(!(pte_val(old_pte) & _PAGE_HASHPTE))) {
415 /* Update the linux pte with the HPTE slot */
416 pte_val(new_pte) &= ~_PAGE_HPTEFLAGS;
417 pte_val(new_pte) |= _PAGE_HASHPTE;
418 prpn = pte_val(old_pte) >> PTE_SHIFT;
419
420 /* copy appropriate flags from linux pte */
421 hpteflags = (pte_val(new_pte) & 0x1f8) | newpp;
422
423 slot = ppc_md.hpte_insert(vpn, prpn, hpteflags, 0, 0);
424
425 pte_val(new_pte) |= ((slot<<12) &
426 (_PAGE_GROUP_IX | _PAGE_SECONDARY));
427
428 *ptep = new_pte;
429 }
430
431 spin_unlock(&hash_table_lock[lock_slot].lock);
432
433 return 0;
434 }
435
436 /*
437 * Handle a fault by adding an HPTE. If the address can't be determined
438 * to be valid via Linux page tables, return 1. If handled return 0
439 */
hash_page(unsigned long ea,unsigned long access)440 int hash_page(unsigned long ea, unsigned long access)
441 {
442 void *pgdir;
443 unsigned long vsid;
444 struct mm_struct *mm;
445 pte_t *ptep;
446 int ret;
447
448 /* Check for invalid addresses. */
449 if (!IS_VALID_EA(ea)) return 1;
450
451 switch (REGION_ID(ea)) {
452 case USER_REGION_ID:
453 mm = current->mm;
454 if (mm == NULL) return 1;
455 vsid = get_vsid(mm->context, ea);
456 break;
457 case IO_REGION_ID:
458 mm = &ioremap_mm;
459 vsid = get_kernel_vsid(ea);
460 break;
461 case VMALLOC_REGION_ID:
462 mm = &init_mm;
463 vsid = get_kernel_vsid(ea);
464 #ifdef CONFIG_SHARED_MEMORY_ADDRESSING
465 /*
466 * Check if this is a user task with shared access to kernel
467 * data & we got a protection fault. If it is, the kernel
468 * must have faulted in the segment and the protection flags
469 * on the segment are kernel access only. Just flush the
470 * segment table & fault in the segment with the right flags.
471 */
472 if(((current->thread.flags) & PPC_FLAG_SHARED) &&
473 (access & _PAGE_USER)) {
474 flush_stab();
475 }
476 #endif
477 break;
478 case EEH_REGION_ID:
479 /*
480 * Should only be hit if there is an access to MMIO space
481 * which is protected by EEH.
482 * Send the problem up to do_page_fault
483 */
484 case KERNEL_REGION_ID:
485 /*
486 * Should never get here - entire 0xC0... region is bolted.
487 * Send the problem up to do_page_fault
488 */
489 default:
490 /* Not a valid range
491 * Send the problem up to do_page_fault
492 */
493 return 1;
494 break;
495 }
496
497 pgdir = mm->pgd;
498 if (pgdir == NULL) return 1;
499
500 /*
501 * Lock the Linux page table to prevent mmap and kswapd
502 * from modifying entries while we search and update
503 */
504 spin_lock(&mm->page_table_lock);
505
506 ptep = find_linux_pte(pgdir, ea);
507 /*
508 * If no pte found or not present, send the problem up to
509 * do_page_fault
510 */
511 if (ptep && pte_present(*ptep)) {
512 ret = __hash_page(ea, access, vsid, ptep);
513 } else {
514 /* If no pte, send the problem up to do_page_fault */
515 ret = 1;
516 }
517
518 spin_unlock(&mm->page_table_lock);
519
520 return ret;
521 }
522
flush_hash_page(unsigned long context,unsigned long ea,pte_t * ptep)523 void flush_hash_page(unsigned long context, unsigned long ea, pte_t *ptep)
524 {
525 unsigned long vsid, vpn, va, hash, secondary, slot, flags, lock_slot;
526 unsigned long large = 0, local = 0;
527 pte_t pte;
528
529 if ((ea >= USER_START) && (ea <= USER_END))
530 vsid = get_vsid(context, ea);
531 else
532 vsid = get_kernel_vsid(ea);
533
534 va = (vsid << 28) | (ea & 0x0fffffff);
535 if (large)
536 vpn = va >> LARGE_PAGE_SHIFT;
537 else
538 vpn = va >> PAGE_SHIFT;
539
540 lock_slot = get_lock_slot(vpn);
541 hash = hpt_hash(vpn, large);
542
543 spin_lock_irqsave(&hash_table_lock[lock_slot].lock, flags);
544
545 pte = __pte(pte_update(ptep, _PAGE_HPTEFLAGS, 0));
546 secondary = (pte_val(pte) & _PAGE_SECONDARY) >> 15;
547 if (secondary) hash = ~hash;
548 slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
549 slot += (pte_val(pte) & _PAGE_GROUP_IX) >> 12;
550
551 if (pte_val(pte) & _PAGE_HASHPTE) {
552 ppc_md.hpte_invalidate(slot, secondary, va, large, local);
553 }
554
555 spin_unlock_irqrestore(&hash_table_lock[lock_slot].lock, flags);
556 }
557
plpar_pte_enter(unsigned long flags,unsigned long ptex,unsigned long new_pteh,unsigned long new_ptel,unsigned long * old_pteh_ret,unsigned long * old_ptel_ret)558 long plpar_pte_enter(unsigned long flags,
559 unsigned long ptex,
560 unsigned long new_pteh, unsigned long new_ptel,
561 unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
562 {
563 unsigned long dummy, ret;
564 ret = plpar_hcall(H_ENTER, flags, ptex, new_pteh, new_ptel,
565 old_pteh_ret, old_ptel_ret, &dummy);
566 return(ret);
567 }
568
plpar_pte_remove(unsigned long flags,unsigned long ptex,unsigned long avpn,unsigned long * old_pteh_ret,unsigned long * old_ptel_ret)569 long plpar_pte_remove(unsigned long flags,
570 unsigned long ptex,
571 unsigned long avpn,
572 unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
573 {
574 unsigned long dummy;
575 return plpar_hcall(H_REMOVE, flags, ptex, avpn, 0,
576 old_pteh_ret, old_ptel_ret, &dummy);
577 }
578
plpar_pte_read(unsigned long flags,unsigned long ptex,unsigned long * old_pteh_ret,unsigned long * old_ptel_ret)579 long plpar_pte_read(unsigned long flags,
580 unsigned long ptex,
581 unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
582 {
583 unsigned long dummy;
584 return plpar_hcall(H_READ, flags, ptex, 0, 0,
585 old_pteh_ret, old_ptel_ret, &dummy);
586 }
587
plpar_pte_protect(unsigned long flags,unsigned long ptex,unsigned long avpn)588 long plpar_pte_protect(unsigned long flags,
589 unsigned long ptex,
590 unsigned long avpn)
591 {
592 return plpar_hcall_norets(H_PROTECT, flags, ptex, avpn);
593 }
594
set_pp_bit(unsigned long pp,HPTE * addr)595 static __inline__ void set_pp_bit(unsigned long pp, HPTE *addr)
596 {
597 unsigned long old;
598 unsigned long *p = &addr->dw1.dword1;
599
600 __asm__ __volatile__(
601 "1: ldarx %0,0,%3\n\
602 rldimi %0,%2,0,62\n\
603 stdcx. %0,0,%3\n\
604 bne 1b"
605 : "=&r" (old), "=m" (*p)
606 : "r" (pp), "r" (p), "m" (*p)
607 : "cc");
608 }
609
610 /*
611 * Calculate which hash_table_lock to use, based on the pteg being used.
612 *
613 * Given a VPN, use the high order 8 bits to select one of 2^7 locks. The
614 * highest order bit is used to indicate primary vs. secondary group. If the
615 * secondary is selected, complement the lock select bits. This results in
616 * both the primary and secondary groups being covered under the same lock.
617 */
get_lock_slot(unsigned long vpn)618 inline unsigned long get_lock_slot(unsigned long vpn)
619 {
620 unsigned long lock_slot;
621 #ifdef LOCK_SPLIT
622 lock_slot = (hpt_hash(vpn,0) >> htab_data.htab_lock_shift) & 0xff;
623 if(lock_slot & 0x80) lock_slot = (~lock_slot) & 0x7f;
624 #else
625 lock_slot = 0;
626 #endif
627 return(lock_slot);
628 }
629
630 /*
631 * Functions used to retrieve word 0 of a given page table entry.
632 *
633 * Input : slot : PTE index within the page table of the entry to retrieve
634 * Output: Contents of word 0 of the specified entry
635 */
rpa_lpar_hpte_getword0(unsigned long slot)636 static unsigned long rpa_lpar_hpte_getword0(unsigned long slot)
637 {
638 unsigned long dword0;
639 unsigned long lpar_rc;
640 unsigned long dummy_word1;
641 unsigned long flags;
642
643 /* Read 1 pte at a time */
644 /* Do not need RPN to logical page translation */
645 /* No cross CEC PFT access */
646 flags = 0;
647
648 lpar_rc = plpar_pte_read(flags, slot, &dword0, &dummy_word1);
649
650 if (lpar_rc != H_Success)
651 panic("Error on pte read in get_hpte0 rc = %lx\n", lpar_rc);
652
653 return dword0;
654 }
655
iSeries_hpte_getword0(unsigned long slot)656 unsigned long iSeries_hpte_getword0(unsigned long slot)
657 {
658 unsigned long dword0;
659
660 HPTE hpte;
661 HvCallHpt_get(&hpte, slot);
662 dword0 = hpte.dw0.dword0;
663
664 return dword0;
665 }
666
667 /*
668 * Functions used to find the PTE for a particular virtual address.
669 * Only used during boot when bolting pages.
670 *
671 * Input : vpn : virtual page number
672 * Output: PTE index within the page table of the entry
673 * -1 on failure
674 */
hpte_find(unsigned long vpn)675 static long hpte_find(unsigned long vpn)
676 {
677 HPTE *hptep;
678 unsigned long hash;
679 unsigned long i, j;
680 long slot;
681 Hpte_dword0 dw0;
682
683 hash = hpt_hash(vpn, 0);
684
685 for (j = 0; j < 2; j++) {
686 slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
687 for (i = 0; i < HPTES_PER_GROUP; i++) {
688 hptep = htab_data.htab + slot;
689 dw0 = hptep->dw0.dw0;
690
691 if ((dw0.avpn == (vpn >> 11)) && dw0.v &&
692 (dw0.h == j)) {
693 /* HPTE matches */
694 if (j)
695 slot = -slot;
696 return slot;
697 }
698 ++slot;
699 }
700 hash = ~hash;
701 }
702
703 return -1;
704 }
705
rpa_lpar_hpte_find(unsigned long vpn)706 static long rpa_lpar_hpte_find(unsigned long vpn)
707 {
708 unsigned long hash;
709 unsigned long i, j;
710 long slot;
711 union {
712 unsigned long dword0;
713 Hpte_dword0 dw0;
714 } hpte_dw0;
715 Hpte_dword0 dw0;
716
717 hash = hpt_hash(vpn, 0);
718
719 for (j = 0; j < 2; j++) {
720 slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
721 for (i = 0; i < HPTES_PER_GROUP; i++) {
722 hpte_dw0.dword0 = rpa_lpar_hpte_getword0(slot);
723 dw0 = hpte_dw0.dw0;
724
725 if ((dw0.avpn == (vpn >> 11)) && dw0.v &&
726 (dw0.h == j)) {
727 /* HPTE matches */
728 if (j)
729 slot = -slot;
730 return slot;
731 }
732 ++slot;
733 }
734 hash = ~hash;
735 }
736
737 return -1;
738 }
739
iSeries_hpte_find(unsigned long vpn)740 static long iSeries_hpte_find(unsigned long vpn)
741 {
742 HPTE hpte;
743 long slot;
744
745 /*
746 * The HvCallHpt_findValid interface is as follows:
747 * 0xffffffffffffffff : No entry found.
748 * 0x00000000xxxxxxxx : Entry found in primary group, slot x
749 * 0x80000000xxxxxxxx : Entry found in secondary group, slot x
750 */
751 slot = HvCallHpt_findValid(&hpte, vpn);
752 if (hpte.dw0.dw0.v) {
753 if (slot < 0) {
754 slot &= 0x7fffffffffffffff;
755 slot = -slot;
756 }
757 } else {
758 slot = -1;
759 }
760
761 return slot;
762 }
763
764 /*
765 * Functions used to invalidate a page table entry from the page table
766 * and tlb.
767 *
768 * Input : slot : PTE index within the page table of the entry to invalidated
769 * va : Virtual address of the entry being invalidated
770 * large : 1 = large page (16M)
771 * local : 1 = Use tlbiel to only invalidate the local tlb
772 */
hpte_invalidate(unsigned long slot,unsigned long secondary,unsigned long va,int large,int local)773 static void hpte_invalidate(unsigned long slot,
774 unsigned long secondary,
775 unsigned long va,
776 int large, int local)
777 {
778 HPTE *hptep = htab_data.htab + slot;
779 Hpte_dword0 dw0;
780 unsigned long vpn, avpn;
781 unsigned long flags;
782
783 if (large)
784 vpn = va >> LARGE_PAGE_SHIFT;
785 else
786 vpn = va >> PAGE_SHIFT;
787
788 avpn = vpn >> 11;
789
790 dw0 = hptep->dw0.dw0;
791
792 /*
793 * Do not remove bolted entries. Alternatively, we could check
794 * the AVPN, hash group, and valid bits. By doing it this way,
795 * it is common with the pSeries LPAR optimal path.
796 */
797 if (dw0.bolted) return;
798
799 /* Invalidate the hpte. */
800 hptep->dw0.dword0 = 0;
801
802 /* Invalidate the tlb */
803 spin_lock_irqsave(&pSeries_tlbie_lock, flags);
804 _tlbie(va, large);
805 spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
806 }
807
rpa_lpar_hpte_invalidate(unsigned long slot,unsigned long secondary,unsigned long va,int large,int local)808 static void rpa_lpar_hpte_invalidate(unsigned long slot,
809 unsigned long secondary,
810 unsigned long va,
811 int large, int local)
812 {
813 unsigned long lpar_rc;
814 unsigned long dummy1, dummy2;
815
816 /*
817 * Don't remove a bolted entry. This case can occur when we bolt
818 * pages dynamically after initial boot.
819 */
820 lpar_rc = plpar_pte_remove(H_ANDCOND, slot, (0x1UL << 4),
821 &dummy1, &dummy2);
822
823 if (lpar_rc != H_Success)
824 panic("Bad return code from invalidate rc = %lx\n", lpar_rc);
825 }
826
iSeries_hpte_invalidate(unsigned long slot,unsigned long secondary,unsigned long va,int large,int local)827 static void iSeries_hpte_invalidate(unsigned long slot,
828 unsigned long secondary,
829 unsigned long va,
830 int large, int local)
831 {
832 HPTE lhpte;
833 unsigned long vpn, avpn;
834
835 if (large)
836 vpn = va >> LARGE_PAGE_SHIFT;
837 else
838 vpn = va >> PAGE_SHIFT;
839
840 avpn = vpn >> 11;
841
842 lhpte.dw0.dword0 = iSeries_hpte_getword0(slot);
843
844 if ((lhpte.dw0.dw0.avpn == avpn) &&
845 (lhpte.dw0.dw0.v) &&
846 (lhpte.dw0.dw0.h == secondary)) {
847 HvCallHpt_invalidateSetSwBitsGet(slot, 0, 0);
848 }
849 }
850
851 /*
852 * Functions used to update page protection bits.
853 *
854 * Input : slot : PTE index within the page table of the entry to update
855 * newpp : new page protection bits
856 * va : Virtual address of the entry being updated
857 * large : 1 = large page (16M)
858 * Output: 0 on success, -1 on failure
859 */
hpte_updatepp(unsigned long slot,unsigned long secondary,unsigned long newpp,unsigned long va,int large)860 static long hpte_updatepp(unsigned long slot,
861 unsigned long secondary,
862 unsigned long newpp,
863 unsigned long va, int large)
864 {
865 HPTE *hptep = htab_data.htab + slot;
866 Hpte_dword0 dw0;
867 Hpte_dword1 dw1;
868 unsigned long vpn, avpn;
869 unsigned long flags;
870
871 if (large)
872 vpn = va >> LARGE_PAGE_SHIFT;
873 else
874 vpn = va >> PAGE_SHIFT;
875
876 avpn = vpn >> 11;
877
878 dw0 = hptep->dw0.dw0;
879 if ((dw0.avpn == avpn) &&
880 (dw0.v) && (dw0.h == secondary)) {
881 /* Turn off valid bit in HPTE */
882 dw0.v = 0;
883 hptep->dw0.dw0 = dw0;
884
885 /* Ensure it is out of the tlb too */
886 spin_lock_irqsave(&pSeries_tlbie_lock, flags);
887 _tlbie(va, large);
888 spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
889
890 /* Insert the new pp bits into the HPTE */
891 dw1 = hptep->dw1.dw1;
892 dw1.pp = newpp;
893 hptep->dw1.dw1 = dw1;
894
895 /* Ensure it is visible before validating */
896 __asm__ __volatile__ ("eieio" : : : "memory");
897
898 /* Turn the valid bit back on in HPTE */
899 dw0.v = 1;
900 hptep->dw0.dw0 = dw0;
901
902 __asm__ __volatile__ ("ptesync" : : : "memory");
903
904 return 0;
905 }
906
907 return -1;
908 }
909
rpa_lpar_hpte_updatepp(unsigned long slot,unsigned long secondary,unsigned long newpp,unsigned long va,int large)910 static long rpa_lpar_hpte_updatepp(unsigned long slot,
911 unsigned long secondary,
912 unsigned long newpp,
913 unsigned long va, int large)
914 {
915 unsigned long lpar_rc;
916 unsigned long flags = (newpp & 7);
917 unsigned long avpn = va >> 23;
918 HPTE hpte;
919
920 lpar_rc = plpar_pte_read(0, slot, &hpte.dw0.dword0, &hpte.dw1.dword1);
921
922 if ((hpte.dw0.dw0.avpn == avpn) &&
923 (hpte.dw0.dw0.v) &&
924 (hpte.dw0.dw0.h == secondary)) {
925 lpar_rc = plpar_pte_protect(flags, slot, 0);
926 if (lpar_rc != H_Success)
927 panic("bad return code from pte protect rc = %lx\n",
928 lpar_rc);
929 return 0;
930 }
931
932 return -1;
933 }
934
iSeries_hpte_updatepp(unsigned long slot,unsigned long secondary,unsigned long newpp,unsigned long va,int large)935 static long iSeries_hpte_updatepp(unsigned long slot,
936 unsigned long secondary,
937 unsigned long newpp,
938 unsigned long va, int large)
939 {
940 unsigned long vpn, avpn;
941 HPTE hpte;
942
943 if (large)
944 vpn = va >> LARGE_PAGE_SHIFT;
945 else
946 vpn = va >> PAGE_SHIFT;
947
948 avpn = vpn >> 11;
949
950 HvCallHpt_get(&hpte, slot);
951 if ((hpte.dw0.dw0.avpn == avpn) &&
952 (hpte.dw0.dw0.v) &&
953 (hpte.dw0.dw0.h == secondary)) {
954 HvCallHpt_setPp(slot, newpp);
955 return 0;
956 }
957 return -1;
958 }
959
960 /*
961 * Functions used to update the page protection bits. Intended to be used
962 * to create guard pages for kernel data structures on pages which are bolted
963 * in the HPT. Assumes pages being operated on will not be stolen.
964 * Does not work on large pages. No need to lock here because we are the
965 * only user.
966 *
967 * Input : newpp : page protection flags
968 * ea : effective kernel address to bolt.
969 */
hpte_updateboltedpp(unsigned long newpp,unsigned long ea)970 static void hpte_updateboltedpp(unsigned long newpp, unsigned long ea)
971 {
972 unsigned long vsid, va, vpn, flags;
973 long slot;
974 HPTE *hptep;
975
976 vsid = get_kernel_vsid(ea);
977 va = (vsid << 28) | (ea & 0x0fffffff);
978 vpn = va >> PAGE_SHIFT;
979
980 slot = hpte_find(vpn);
981 if (slot == -1)
982 panic("could not find page to bolt\n");
983 hptep = htab_data.htab + slot;
984
985 set_pp_bit(newpp, hptep);
986
987 /* Ensure it is out of the tlb too */
988 spin_lock_irqsave(&pSeries_tlbie_lock, flags);
989 _tlbie(va, 0);
990 spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
991 }
992
rpa_lpar_hpte_updateboltedpp(unsigned long newpp,unsigned long ea)993 static void rpa_lpar_hpte_updateboltedpp(unsigned long newpp, unsigned long ea)
994 {
995 unsigned long lpar_rc;
996 unsigned long vsid, va, vpn, flags;
997 long slot;
998
999 vsid = get_kernel_vsid(ea);
1000 va = (vsid << 28) | (ea & 0x0fffffff);
1001 vpn = va >> PAGE_SHIFT;
1002
1003 slot = rpa_lpar_hpte_find(vpn);
1004 if (slot == -1)
1005 panic("updateboltedpp: Could not find page to bolt\n");
1006
1007 flags = newpp & 3;
1008 lpar_rc = plpar_pte_protect(flags, slot, 0);
1009
1010 if (lpar_rc != H_Success)
1011 panic("Bad return code from pte bolted protect rc = %lx\n",
1012 lpar_rc);
1013 }
1014
iSeries_hpte_updateboltedpp(unsigned long newpp,unsigned long ea)1015 void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea)
1016 {
1017 unsigned long vsid,va,vpn;
1018 long slot;
1019
1020 vsid = get_kernel_vsid( ea );
1021 va = ( vsid << 28 ) | ( ea & 0x0fffffff );
1022 vpn = va >> PAGE_SHIFT;
1023
1024 slot = iSeries_hpte_find(vpn);
1025 if (slot == -1)
1026 panic("updateboltedpp: Could not find page to bolt\n");
1027
1028 HvCallHpt_setPp(slot, newpp);
1029 }
1030
1031 /*
1032 * Functions used to insert new hardware page table entries.
1033 * Will castout non-bolted entries as necessary using a random
1034 * algorithm.
1035 *
1036 * Input : vpn : virtual page number
1037 * prpn : real page number in absolute space
1038 * hpteflags: page protection flags
1039 * bolted : 1 = bolt the page
1040 * large : 1 = large page (16M)
1041 * Output: hsss, where h = hash group, sss = slot within that group
1042 */
hpte_insert(unsigned long vpn,unsigned long prpn,unsigned long hpteflags,int bolted,int large)1043 static long hpte_insert(unsigned long vpn, unsigned long prpn,
1044 unsigned long hpteflags, int bolted, int large)
1045 {
1046 HPTE *hptep;
1047 Hpte_dword0 dw0;
1048 HPTE lhpte;
1049 int i, secondary;
1050 unsigned long hash = hpt_hash(vpn, 0);
1051 unsigned long avpn = vpn >> 11;
1052 unsigned long arpn = physRpn_to_absRpn(prpn);
1053 unsigned long hpte_group;
1054
1055 repeat:
1056 secondary = 0;
1057 hpte_group = ((hash & htab_data.htab_hash_mask) *
1058 HPTES_PER_GROUP) & ~0x7UL;
1059 hptep = htab_data.htab + hpte_group;
1060
1061 for (i = 0; i < HPTES_PER_GROUP; i++) {
1062 dw0 = hptep->dw0.dw0;
1063 if (!dw0.v) {
1064 /* retry with lock held */
1065 dw0 = hptep->dw0.dw0;
1066 if (!dw0.v)
1067 break;
1068 }
1069 hptep++;
1070 }
1071
1072 if (i == HPTES_PER_GROUP) {
1073 secondary = 1;
1074 hpte_group = ((~hash & htab_data.htab_hash_mask) *
1075 HPTES_PER_GROUP) & ~0x7UL;
1076 hptep = htab_data.htab + hpte_group;
1077
1078 for (i = 0; i < HPTES_PER_GROUP; i++) {
1079 dw0 = hptep->dw0.dw0;
1080 if (!dw0.v) {
1081 /* retry with lock held */
1082 dw0 = hptep->dw0.dw0;
1083 if (!dw0.v)
1084 break;
1085 }
1086 hptep++;
1087 }
1088 if (i == HPTES_PER_GROUP) {
1089 if (mftb() & 0x1)
1090 hpte_group=((hash & htab_data.htab_hash_mask)*
1091 HPTES_PER_GROUP) & ~0x7UL;
1092
1093 hpte_remove(hpte_group);
1094 goto repeat;
1095 }
1096 }
1097
1098 lhpte.dw1.dword1 = 0;
1099 lhpte.dw1.dw1.rpn = arpn;
1100 lhpte.dw1.flags.flags = hpteflags;
1101
1102 lhpte.dw0.dword0 = 0;
1103 lhpte.dw0.dw0.avpn = avpn;
1104 lhpte.dw0.dw0.h = secondary;
1105 lhpte.dw0.dw0.bolted = bolted;
1106 lhpte.dw0.dw0.v = 1;
1107
1108 if (large) lhpte.dw0.dw0.l = 1;
1109
1110 hptep->dw1.dword1 = lhpte.dw1.dword1;
1111
1112 /* Guarantee the second dword is visible before the valid bit */
1113 __asm__ __volatile__ ("eieio" : : : "memory");
1114
1115 /*
1116 * Now set the first dword including the valid bit
1117 * NOTE: this also unlocks the hpte
1118 */
1119 hptep->dw0.dword0 = lhpte.dw0.dword0;
1120
1121 __asm__ __volatile__ ("ptesync" : : : "memory");
1122
1123 return ((secondary << 3) | i);
1124 }
1125
rpa_lpar_hpte_insert(unsigned long vpn,unsigned long prpn,unsigned long hpteflags,int bolted,int large)1126 static long rpa_lpar_hpte_insert(unsigned long vpn, unsigned long prpn,
1127 unsigned long hpteflags,
1128 int bolted, int large)
1129 {
1130 /* XXX fix for large page */
1131 unsigned long lpar_rc;
1132 unsigned long flags;
1133 unsigned long slot;
1134 HPTE lhpte;
1135 int secondary;
1136 unsigned long hash = hpt_hash(vpn, 0);
1137 unsigned long avpn = vpn >> 11;
1138 unsigned long arpn = physRpn_to_absRpn(prpn);
1139 unsigned long hpte_group;
1140
1141 /* Fill in the local HPTE with absolute rpn, avpn and flags */
1142 lhpte.dw1.dword1 = 0;
1143 lhpte.dw1.dw1.rpn = arpn;
1144 lhpte.dw1.flags.flags = hpteflags;
1145
1146 lhpte.dw0.dword0 = 0;
1147 lhpte.dw0.dw0.avpn = avpn;
1148 lhpte.dw0.dw0.bolted = bolted;
1149 lhpte.dw0.dw0.v = 1;
1150
1151 if (large) lhpte.dw0.dw0.l = 1;
1152
1153 /* Now fill in the actual HPTE */
1154 /* Set CEC cookie to 0 */
1155 /* Large page = 0 */
1156 /* Zero page = 0 */
1157 /* I-cache Invalidate = 0 */
1158 /* I-cache synchronize = 0 */
1159 /* Exact = 0 */
1160 flags = 0;
1161
1162 /* XXX why is this here? - Anton */
1163 /* -- Because at one point we hit a case where non cachable
1164 * pages where marked coherent & this is rejected by the HV.
1165 * Perhaps it is no longer an issue ... DRENG.
1166 */
1167 if (hpteflags & (_PAGE_GUARDED|_PAGE_NO_CACHE))
1168 lhpte.dw1.flags.flags &= ~_PAGE_COHERENT;
1169
1170 repeat:
1171 secondary = 0;
1172 lhpte.dw0.dw0.h = secondary;
1173 hpte_group = ((hash & htab_data.htab_hash_mask) *
1174 HPTES_PER_GROUP) & ~0x7UL;
1175
1176 __asm__ __volatile__ (
1177 H_ENTER_r3
1178 "mr 4, %2\n"
1179 "mr 5, %3\n"
1180 "mr 6, %4\n"
1181 "mr 7, %5\n"
1182 HVSC
1183 "mr %0, 3\n"
1184 "mr %1, 4\n"
1185 : "=r" (lpar_rc), "=r" (slot)
1186 : "r" (flags), "r" (hpte_group), "r" (lhpte.dw0.dword0),
1187 "r" (lhpte.dw1.dword1)
1188 : "r0", "r3", "r4", "r5", "r6", "r7",
1189 "r8", "r9", "r10", "r11", "r12", "cc");
1190
1191 if (lpar_rc == H_PTEG_Full) {
1192 secondary = 1;
1193 lhpte.dw0.dw0.h = secondary;
1194 hpte_group = ((~hash & htab_data.htab_hash_mask) *
1195 HPTES_PER_GROUP) & ~0x7UL;
1196
1197 __asm__ __volatile__ (
1198 H_ENTER_r3
1199 "mr 4, %2\n"
1200 "mr 5, %3\n"
1201 "mr 6, %4\n"
1202 "mr 7, %5\n"
1203 HVSC
1204 "mr %0, 3\n"
1205 "mr %1, 4\n"
1206 : "=r" (lpar_rc), "=r" (slot)
1207 : "r" (flags), "r" (hpte_group), "r" (lhpte.dw0.dword0),
1208 "r" (lhpte.dw1.dword1)
1209 : "r0", "r3", "r4", "r5", "r6", "r7",
1210 "r8", "r9", "r10", "r11", "r12", "cc");
1211 if (lpar_rc == H_PTEG_Full) {
1212 if (mftb() & 0x1)
1213 hpte_group=((hash & htab_data.htab_hash_mask)*
1214 HPTES_PER_GROUP) & ~0x7UL;
1215
1216 rpa_lpar_hpte_remove(hpte_group);
1217 goto repeat;
1218 }
1219 }
1220
1221 if (lpar_rc != H_Success)
1222 panic("Bad return code from pte enter rc = %lx\n", lpar_rc);
1223
1224 return ((secondary << 3) | (slot & 0x7));
1225 }
1226
iSeries_hpte_insert(unsigned long vpn,unsigned long prpn,unsigned long hpteflags,int bolted,int large)1227 static long iSeries_hpte_insert(unsigned long vpn, unsigned long prpn,
1228 unsigned long hpteflags,
1229 int bolted, int large)
1230 {
1231 HPTE lhpte;
1232 unsigned long hash, hpte_group;
1233 unsigned long avpn = vpn >> 11;
1234 unsigned long arpn = physRpn_to_absRpn( prpn );
1235 int secondary = 0;
1236 long slot;
1237
1238 hash = hpt_hash(vpn, 0);
1239
1240 repeat:
1241 slot = HvCallHpt_findValid(&lhpte, vpn);
1242 if (lhpte.dw0.dw0.v) {
1243 panic("select_hpte_slot found entry already valid\n");
1244 }
1245
1246 if (slot == -1) { /* No available entry found in either group */
1247 if (mftb() & 0x1) {
1248 hpte_group=((hash & htab_data.htab_hash_mask)*
1249 HPTES_PER_GROUP) & ~0x7UL;
1250 } else {
1251 hpte_group=((~hash & htab_data.htab_hash_mask)*
1252 HPTES_PER_GROUP) & ~0x7UL;
1253 }
1254
1255 hash = hpt_hash(vpn, 0);
1256 iSeries_hpte_remove(hpte_group);
1257 goto repeat;
1258 } else if (slot < 0) {
1259 slot &= 0x7fffffffffffffff;
1260 secondary = 1;
1261 }
1262
1263 /* Create the HPTE */
1264 lhpte.dw1.dword1 = 0;
1265 lhpte.dw1.dw1.rpn = arpn;
1266 lhpte.dw1.flags.flags = hpteflags;
1267
1268 lhpte.dw0.dword0 = 0;
1269 lhpte.dw0.dw0.avpn = avpn;
1270 lhpte.dw0.dw0.h = secondary;
1271 lhpte.dw0.dw0.bolted = bolted;
1272 lhpte.dw0.dw0.v = 1;
1273
1274 /* Now fill in the actual HPTE */
1275 HvCallHpt_addValidate(slot, secondary, (HPTE *)&lhpte);
1276 return ((secondary << 3) | (slot & 0x7));
1277 }
1278
1279 /*
1280 * Functions used to remove hardware page table entries.
1281 *
1282 * Input : hpte_group: PTE index of the first entry in a group
1283 * Output: offset within the group of the entry removed or
1284 * -1 on failure
1285 */
hpte_remove(unsigned long hpte_group)1286 static long hpte_remove(unsigned long hpte_group)
1287 {
1288 HPTE *hptep;
1289 Hpte_dword0 dw0;
1290 int i;
1291 int slot_offset;
1292 unsigned long vsid, group, pi, pi_high;
1293 unsigned long slot;
1294 unsigned long flags;
1295 int large;
1296 unsigned long va;
1297
1298 /* pick a random slot to start at */
1299 slot_offset = mftb() & 0x7;
1300
1301 for (i = 0; i < HPTES_PER_GROUP; i++) {
1302 hptep = htab_data.htab + hpte_group + slot_offset;
1303 dw0 = hptep->dw0.dw0;
1304
1305 if (dw0.v && !dw0.bolted) {
1306 /* retry with lock held */
1307 dw0 = hptep->dw0.dw0;
1308 if (dw0.v && !dw0.bolted)
1309 break;
1310 }
1311
1312 slot_offset++;
1313 slot_offset &= 0x7;
1314 }
1315
1316 if (i == HPTES_PER_GROUP)
1317 return -1;
1318
1319 large = dw0.l;
1320
1321 /* Invalidate the hpte. NOTE: this also unlocks it */
1322 hptep->dw0.dword0 = 0;
1323
1324 /* Invalidate the tlb */
1325 vsid = dw0.avpn >> 5;
1326 slot = hptep - htab_data.htab;
1327 group = slot >> 3;
1328 if (dw0.h)
1329 group = ~group;
1330 pi = (vsid ^ group) & 0x7ff;
1331 pi_high = (dw0.avpn & 0x1f) << 11;
1332 pi |= pi_high;
1333
1334 if (large)
1335 va = pi << LARGE_PAGE_SHIFT;
1336 else
1337 va = pi << PAGE_SHIFT;
1338
1339 spin_lock_irqsave(&pSeries_tlbie_lock, flags);
1340 _tlbie(va, large);
1341 spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
1342
1343 return i;
1344 }
1345
rpa_lpar_hpte_remove(unsigned long hpte_group)1346 static long rpa_lpar_hpte_remove(unsigned long hpte_group)
1347 {
1348 unsigned long slot_offset;
1349 unsigned long lpar_rc;
1350 int i;
1351 unsigned long dummy1, dummy2;
1352
1353 /* pick a random slot to start at */
1354 slot_offset = mftb() & 0x7;
1355
1356 for (i = 0; i < HPTES_PER_GROUP; i++) {
1357
1358 /* Don't remove a bolted entry */
1359 lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset,
1360 (0x1UL << 4), &dummy1, &dummy2);
1361
1362 if (lpar_rc == H_Success)
1363 return i;
1364
1365 if (lpar_rc != H_Not_Found)
1366 panic("Bad return code from pte remove rc = %lx\n",
1367 lpar_rc);
1368
1369 slot_offset++;
1370 slot_offset &= 0x7;
1371 }
1372
1373 return -1;
1374 }
1375
iSeries_hpte_remove(unsigned long hpte_group)1376 static long iSeries_hpte_remove(unsigned long hpte_group)
1377 {
1378 unsigned long slot_offset;
1379 int i;
1380 HPTE lhpte;
1381
1382 /* Pick a random slot to start at */
1383 slot_offset = mftb() & 0x7;
1384
1385 for (i = 0; i < HPTES_PER_GROUP; i++) {
1386 lhpte.dw0.dword0 =
1387 iSeries_hpte_getword0(hpte_group + slot_offset);
1388
1389 if (!lhpte.dw0.dw0.bolted) {
1390 HvCallHpt_invalidateSetSwBitsGet(hpte_group +
1391 slot_offset, 0, 0);
1392 return i;
1393 }
1394
1395 slot_offset++;
1396 slot_offset &= 0x7;
1397 }
1398
1399 return -1;
1400 }
1401
hpte_init_pSeries(void)1402 void hpte_init_pSeries(void)
1403 {
1404 ppc_md.hpte_invalidate = hpte_invalidate;
1405 ppc_md.hpte_updatepp = hpte_updatepp;
1406 ppc_md.hpte_updateboltedpp = hpte_updateboltedpp;
1407 ppc_md.hpte_insert = hpte_insert;
1408 ppc_md.hpte_remove = hpte_remove;
1409 }
1410
pSeries_lpar_mm_init(void)1411 void pSeries_lpar_mm_init(void)
1412 {
1413 ppc_md.hpte_invalidate = rpa_lpar_hpte_invalidate;
1414 ppc_md.hpte_updatepp = rpa_lpar_hpte_updatepp;
1415 ppc_md.hpte_updateboltedpp = rpa_lpar_hpte_updateboltedpp;
1416 ppc_md.hpte_insert = rpa_lpar_hpte_insert;
1417 ppc_md.hpte_remove = rpa_lpar_hpte_remove;
1418 }
1419
hpte_init_iSeries(void)1420 void hpte_init_iSeries(void)
1421 {
1422 ppc_md.hpte_invalidate = iSeries_hpte_invalidate;
1423 ppc_md.hpte_updatepp = iSeries_hpte_updatepp;
1424 ppc_md.hpte_updateboltedpp = iSeries_hpte_updateboltedpp;
1425 ppc_md.hpte_insert = iSeries_hpte_insert;
1426 ppc_md.hpte_remove = iSeries_hpte_remove;
1427 }
1428
1429