1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _SPARC64_TSB_H
3 #define _SPARC64_TSB_H
4 
5 /* The sparc64 TSB is similar to the powerpc hashtables.  It's a
6  * power-of-2 sized table of TAG/PTE pairs.  The cpu precomputes
7  * pointers into this table for 8K and 64K page sizes, and also a
8  * comparison TAG based upon the virtual address and context which
9  * faults.
10  *
11  * TLB miss trap handler software does the actual lookup via something
12  * of the form:
13  *
14  * 	ldxa		[%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
15  * 	ldxa		[%g0] ASI_{D,I}MMU, %g6
16  *	sllx		%g6, 22, %g6
17  *	srlx		%g6, 22, %g6
18  * 	ldda		[%g1] ASI_NUCLEUS_QUAD_LDD, %g4
19  * 	cmp		%g4, %g6
20  * 	bne,pn	%xcc, tsb_miss_{d,i}tlb
21  * 	 mov		FAULT_CODE_{D,I}TLB, %g3
22  * 	stxa		%g5, [%g0] ASI_{D,I}TLB_DATA_IN
23  * 	retry
24  *
25  *
26  * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
27  * PTE.  The TAG is of the same layout as the TLB TAG TARGET mmu
28  * register which is:
29  *
30  * -------------------------------------------------
31  * |  -  |  CONTEXT |  -  |    VADDR bits 63:22    |
32  * -------------------------------------------------
33  *  63 61 60      48 47 42 41                     0
34  *
35  * But actually, since we use per-mm TSB's, we zero out the CONTEXT
36  * field.
37  *
38  * Like the powerpc hashtables we need to use locking in order to
39  * synchronize while we update the entries.  PTE updates need locking
40  * as well.
41  *
42  * We need to carefully choose a lock bits for the TSB entry.  We
43  * choose to use bit 47 in the tag.  Also, since we never map anything
44  * at page zero in context zero, we use zero as an invalid tag entry.
45  * When the lock bit is set, this forces a tag comparison failure.
46  */
47 
48 #define TSB_TAG_LOCK_BIT	47
49 #define TSB_TAG_LOCK_HIGH	(1 << (TSB_TAG_LOCK_BIT - 32))
50 
51 #define TSB_TAG_INVALID_BIT	46
52 #define TSB_TAG_INVALID_HIGH	(1 << (TSB_TAG_INVALID_BIT - 32))
53 
54 /* Some cpus support physical address quad loads.  We want to use
55  * those if possible so we don't need to hard-lock the TSB mapping
56  * into the TLB.  We encode some instruction patching in order to
57  * support this.
58  *
59  * The kernel TSB is locked into the TLB by virtue of being in the
60  * kernel image, so we don't play these games for swapper_tsb access.
61  */
62 #ifndef __ASSEMBLY__
63 struct tsb_ldquad_phys_patch_entry {
64 	unsigned int	addr;
65 	unsigned int	sun4u_insn;
66 	unsigned int	sun4v_insn;
67 };
68 extern struct tsb_ldquad_phys_patch_entry __tsb_ldquad_phys_patch,
69 	__tsb_ldquad_phys_patch_end;
70 
71 struct tsb_phys_patch_entry {
72 	unsigned int	addr;
73 	unsigned int	insn;
74 };
75 extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
76 #endif
77 #define TSB_LOAD_QUAD(TSB, REG)	\
78 661:	ldda		[TSB] ASI_NUCLEUS_QUAD_LDD, REG; \
79 	.section	.tsb_ldquad_phys_patch, "ax"; \
80 	.word		661b; \
81 	ldda		[TSB] ASI_QUAD_LDD_PHYS, REG; \
82 	ldda		[TSB] ASI_QUAD_LDD_PHYS_4V, REG; \
83 	.previous
84 
85 #define TSB_LOAD_TAG_HIGH(TSB, REG) \
86 661:	lduwa		[TSB] ASI_N, REG; \
87 	.section	.tsb_phys_patch, "ax"; \
88 	.word		661b; \
89 	lduwa		[TSB] ASI_PHYS_USE_EC, REG; \
90 	.previous
91 
92 #define TSB_LOAD_TAG(TSB, REG) \
93 661:	ldxa		[TSB] ASI_N, REG; \
94 	.section	.tsb_phys_patch, "ax"; \
95 	.word		661b; \
96 	ldxa		[TSB] ASI_PHYS_USE_EC, REG; \
97 	.previous
98 
99 #define TSB_CAS_TAG_HIGH(TSB, REG1, REG2) \
100 661:	casa		[TSB] ASI_N, REG1, REG2; \
101 	.section	.tsb_phys_patch, "ax"; \
102 	.word		661b; \
103 	casa		[TSB] ASI_PHYS_USE_EC, REG1, REG2; \
104 	.previous
105 
106 #define TSB_CAS_TAG(TSB, REG1, REG2) \
107 661:	casxa		[TSB] ASI_N, REG1, REG2; \
108 	.section	.tsb_phys_patch, "ax"; \
109 	.word		661b; \
110 	casxa		[TSB] ASI_PHYS_USE_EC, REG1, REG2; \
111 	.previous
112 
113 #define TSB_STORE(ADDR, VAL) \
114 661:	stxa		VAL, [ADDR] ASI_N; \
115 	.section	.tsb_phys_patch, "ax"; \
116 	.word		661b; \
117 	stxa		VAL, [ADDR] ASI_PHYS_USE_EC; \
118 	.previous
119 
120 #define TSB_LOCK_TAG(TSB, REG1, REG2)	\
121 99:	TSB_LOAD_TAG_HIGH(TSB, REG1);	\
122 	sethi	%hi(TSB_TAG_LOCK_HIGH), REG2;\
123 	andcc	REG1, REG2, %g0;	\
124 	bne,pn	%icc, 99b;		\
125 	 nop;				\
126 	TSB_CAS_TAG_HIGH(TSB, REG1, REG2);	\
127 	cmp	REG1, REG2;		\
128 	bne,pn	%icc, 99b;		\
129 	 nop;				\
130 
131 #define TSB_WRITE(TSB, TTE, TAG) \
132 	add	TSB, 0x8, TSB;   \
133 	TSB_STORE(TSB, TTE);     \
134 	sub	TSB, 0x8, TSB;   \
135 	TSB_STORE(TSB, TAG);
136 
137 	/* Do a kernel page table walk.  Leaves valid PTE value in
138 	 * REG1.  Jumps to FAIL_LABEL on early page table walk
139 	 * termination.  VADDR will not be clobbered, but REG2 will.
140 	 *
141 	 * There are two masks we must apply to propagate bits from
142 	 * the virtual address into the PTE physical address field
143 	 * when dealing with huge pages.  This is because the page
144 	 * table boundaries do not match the huge page size(s) the
145 	 * hardware supports.
146 	 *
147 	 * In these cases we propagate the bits that are below the
148 	 * page table level where we saw the huge page mapping, but
149 	 * are still within the relevant physical bits for the huge
150 	 * page size in question.  So for PMD mappings (which fall on
151 	 * bit 23, for 8MB per PMD) we must propagate bit 22 for a
152 	 * 4MB huge page.  For huge PUDs (which fall on bit 33, for
153 	 * 8GB per PUD), we have to accommodate 256MB and 2GB huge
154 	 * pages.  So for those we propagate bits 32 to 28.
155 	 */
156 #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL)	\
157 	sethi		%hi(swapper_pg_dir), REG1; \
158 	or		REG1, %lo(swapper_pg_dir), REG1; \
159 	sllx		VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
160 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
161 	andn		REG2, 0x7, REG2; \
162 	ldx		[REG1 + REG2], REG1; \
163 	brz,pn		REG1, FAIL_LABEL; \
164 	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
165 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
166 	andn		REG2, 0x7, REG2; \
167 	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
168 	brz,pn		REG1, FAIL_LABEL; \
169 	sethi		%uhi(_PAGE_PUD_HUGE), REG2; \
170 	brz,pn		REG1, FAIL_LABEL; \
171 	 sllx		REG2, 32, REG2; \
172 	andcc		REG1, REG2, %g0; \
173 	sethi		%hi(0xf8000000), REG2; \
174 	bne,pt		%xcc, 697f; \
175 	 sllx		REG2, 1, REG2; \
176 	sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
177 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
178 	andn		REG2, 0x7, REG2; \
179 	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
180 	sethi		%uhi(_PAGE_PMD_HUGE), REG2; \
181 	brz,pn		REG1, FAIL_LABEL; \
182 	 sllx		REG2, 32, REG2; \
183 	andcc		REG1, REG2, %g0; \
184 	be,pn		%xcc, 698f; \
185 	 sethi		%hi(0x400000), REG2; \
186 697:	brgez,pn	REG1, FAIL_LABEL; \
187 	 andn		REG1, REG2, REG1; \
188 	and		VADDR, REG2, REG2; \
189 	ba,pt		%xcc, 699f; \
190 	 or		REG1, REG2, REG1; \
191 698:	sllx		VADDR, 64 - PMD_SHIFT, REG2; \
192 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
193 	andn		REG2, 0x7, REG2; \
194 	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
195 	brgez,pn	REG1, FAIL_LABEL; \
196 	 nop; \
197 699:
198 
199 	/* PUD has been loaded into REG1, interpret the value, seeing
200 	 * if it is a HUGE PUD or a normal one.  If it is not valid
201 	 * then jump to FAIL_LABEL.  If it is a HUGE PUD, and it
202 	 * translates to a valid PTE, branch to PTE_LABEL.
203 	 *
204 	 * We have to propagate bits [32:22] from the virtual address
205 	 * to resolve at 4M granularity.
206 	 */
207 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
208 #define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
209 700:	ba 700f;					\
210 	 nop;						\
211 	.section	.pud_huge_patch, "ax";		\
212 	.word		700b;				\
213 	nop;						\
214 	.previous;					\
215 	brz,pn		REG1, FAIL_LABEL;		\
216 	 sethi		%uhi(_PAGE_PUD_HUGE), REG2;	\
217 	sllx		REG2, 32, REG2;			\
218 	andcc		REG1, REG2, %g0;		\
219 	be,pt		%xcc, 700f;			\
220 	 sethi		%hi(0xffe00000), REG2;		\
221 	sllx		REG2, 1, REG2;			\
222 	brgez,pn	REG1, FAIL_LABEL;		\
223 	 andn		REG1, REG2, REG1;		\
224 	and		VADDR, REG2, REG2;		\
225 	brlz,pt		REG1, PTE_LABEL;		\
226 	 or		REG1, REG2, REG1;		\
227 700:
228 #else
229 #define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
230 	brz,pn		REG1, FAIL_LABEL; \
231 	 nop;
232 #endif
233 
234 	/* PMD has been loaded into REG1, interpret the value, seeing
235 	 * if it is a HUGE PMD or a normal one.  If it is not valid
236 	 * then jump to FAIL_LABEL.  If it is a HUGE PMD, and it
237 	 * translates to a valid PTE, branch to PTE_LABEL.
238 	 *
239 	 * We have to propagate the 4MB bit of the virtual address
240 	 * because we are fabricating 8MB pages using 4MB hw pages.
241 	 */
242 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
243 #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
244 	brz,pn		REG1, FAIL_LABEL;		\
245 	 sethi		%uhi(_PAGE_PMD_HUGE), REG2;	\
246 	sllx		REG2, 32, REG2;			\
247 	andcc		REG1, REG2, %g0;		\
248 	be,pt		%xcc, 700f;			\
249 	 sethi		%hi(4 * 1024 * 1024), REG2;	\
250 	brgez,pn	REG1, FAIL_LABEL;		\
251 	 andn		REG1, REG2, REG1;		\
252 	and		VADDR, REG2, REG2;		\
253 	brlz,pt		REG1, PTE_LABEL;		\
254 	 or		REG1, REG2, REG1;		\
255 700:
256 #else
257 #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
258 	brz,pn		REG1, FAIL_LABEL; \
259 	 nop;
260 #endif
261 
262 	/* Do a user page table walk in MMU globals.  Leaves final,
263 	 * valid, PTE value in REG1.  Jumps to FAIL_LABEL on early
264 	 * page table walk termination or if the PTE is not valid.
265 	 *
266 	 * Physical base of page tables is in PHYS_PGD which will not
267 	 * be modified.
268 	 *
269 	 * VADDR will not be clobbered, but REG1 and REG2 will.
270 	 */
271 #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL)	\
272 	sllx		VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
273 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
274 	andn		REG2, 0x7, REG2; \
275 	ldxa		[PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
276 	brz,pn		REG1, FAIL_LABEL; \
277 	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
278 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
279 	andn		REG2, 0x7, REG2; \
280 	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
281 	USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
282 	brz,pn		REG1, FAIL_LABEL; \
283 	 sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
284 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
285 	andn		REG2, 0x7, REG2; \
286 	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
287 	USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
288 	sllx		VADDR, 64 - PMD_SHIFT, REG2; \
289 	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
290 	andn		REG2, 0x7, REG2; \
291 	add		REG1, REG2, REG1; \
292 	ldxa		[REG1] ASI_PHYS_USE_EC, REG1; \
293 	brgez,pn	REG1, FAIL_LABEL; \
294 	 nop; \
295 800:
296 
297 /* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
298  * If no entry is found, FAIL_LABEL will be branched to.  On success
299  * the resulting PTE value will be left in REG1.  VADDR is preserved
300  * by this routine.
301  */
302 #define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
303 	sethi		%hi(prom_trans), REG1; \
304 	or		REG1, %lo(prom_trans), REG1; \
305 97:	ldx		[REG1 + 0x00], REG2; \
306 	brz,pn		REG2, FAIL_LABEL; \
307 	 nop; \
308 	ldx		[REG1 + 0x08], REG3; \
309 	add		REG2, REG3, REG3; \
310 	cmp		REG2, VADDR; \
311 	bgu,pt		%xcc, 98f; \
312 	 cmp		VADDR, REG3; \
313 	bgeu,pt		%xcc, 98f; \
314 	 ldx		[REG1 + 0x10], REG3; \
315 	sub		VADDR, REG2, REG2; \
316 	ba,pt		%xcc, 99f; \
317 	 add		REG3, REG2, REG1; \
318 98:	ba,pt		%xcc, 97b; \
319 	 add		REG1, (3 * 8), REG1; \
320 99:
321 
322 	/* We use a 32K TSB for the whole kernel, this allows to
323 	 * handle about 16MB of modules and vmalloc mappings without
324 	 * incurring many hash conflicts.
325 	 */
326 #define KERNEL_TSB_SIZE_BYTES	(32 * 1024)
327 #define KERNEL_TSB_NENTRIES	\
328 	(KERNEL_TSB_SIZE_BYTES / 16)
329 #define KERNEL_TSB4M_NENTRIES	4096
330 
331 	/* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
332 	 * on TSB hit.  REG1, REG2, REG3, and REG4 are used as temporaries
333 	 * and the found TTE will be left in REG1.  REG3 and REG4 must
334 	 * be an even/odd pair of registers.
335 	 *
336 	 * VADDR and TAG will be preserved and not clobbered by this macro.
337 	 */
338 #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
339 661:	sethi		%uhi(swapper_tsb), REG1; \
340 	sethi		%hi(swapper_tsb), REG2; \
341 	or		REG1, %ulo(swapper_tsb), REG1; \
342 	or		REG2, %lo(swapper_tsb), REG2; \
343 	.section	.swapper_tsb_phys_patch, "ax"; \
344 	.word		661b; \
345 	.previous; \
346 	sllx		REG1, 32, REG1; \
347 	or		REG1, REG2, REG1; \
348 	srlx		VADDR, PAGE_SHIFT, REG2; \
349 	and		REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
350 	sllx		REG2, 4, REG2; \
351 	add		REG1, REG2, REG2; \
352 	TSB_LOAD_QUAD(REG2, REG3); \
353 	cmp		REG3, TAG; \
354 	be,a,pt		%xcc, OK_LABEL; \
355 	 mov		REG4, REG1;
356 
357 #ifndef CONFIG_DEBUG_PAGEALLOC
358 	/* This version uses a trick, the TAG is already (VADDR >> 22) so
359 	 * we can make use of that for the index computation.
360 	 */
361 #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
362 661:	sethi		%uhi(swapper_4m_tsb), REG1; \
363 	sethi		%hi(swapper_4m_tsb), REG2; \
364 	or		REG1, %ulo(swapper_4m_tsb), REG1; \
365 	or		REG2, %lo(swapper_4m_tsb), REG2; \
366 	.section	.swapper_4m_tsb_phys_patch, "ax"; \
367 	.word		661b; \
368 	.previous; \
369 	sllx		REG1, 32, REG1; \
370 	or		REG1, REG2, REG1; \
371 	and		TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
372 	sllx		REG2, 4, REG2; \
373 	add		REG1, REG2, REG2; \
374 	TSB_LOAD_QUAD(REG2, REG3); \
375 	cmp		REG3, TAG; \
376 	be,a,pt		%xcc, OK_LABEL; \
377 	 mov		REG4, REG1;
378 #endif
379 
380 #endif /* !(_SPARC64_TSB_H) */
381