1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
2 *
3 * Copyright (C) 2006 <davem@davemloft.net>
4 */
5
6	.text
7	.align	32
8
9	/* Load ITLB fault information into VADDR and CTX, using BASE.  */
10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
11	ldx	[BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
12	ldx	[BASE + HV_FAULT_I_CTX_OFFSET], CTX;
13
14	/* Load DTLB fault information into VADDR and CTX, using BASE.  */
15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
16	ldx	[BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
17	ldx	[BASE + HV_FAULT_D_CTX_OFFSET], CTX;
18
19	/* DEST = (VADDR >> 22)
20	 *
21	 * Branch to ZERO_CTX_LABEL if context is zero.
22	 */
23#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
24	srlx	VADDR, 22, DEST; \
25	brz,pn	CTX, ZERO_CTX_LABEL; \
26	 nop;
27
28	/* Create TSB pointer.  This is something like:
29	 *
30	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
31	 * tsb_base = tsb_reg & ~0x7UL;
32	 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
33	 * tsb_ptr = tsb_base + (tsb_index * 16);
34	 */
35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
36	and	TSB_PTR, 0x7, TMP1;			\
37	mov	512, TMP2;				\
38	andn	TSB_PTR, 0x7, TSB_PTR;			\
39	sllx	TMP2, TMP1, TMP2;			\
40	srlx	VADDR, HASH_SHIFT, TMP1;		\
41	sub	TMP2, 1, TMP2;				\
42	and	TMP1, TMP2, TMP1;			\
43	sllx	TMP1, 4, TMP1;				\
44	add	TSB_PTR, TMP1, TSB_PTR;
45
46sun4v_itlb_miss:
47	/* Load MMU Miss base into %g2.  */
48	ldxa	[%g0] ASI_SCRATCHPAD, %g2
49
50	/* Load UTSB reg into %g1.  */
51	mov	SCRATCHPAD_UTSBREG1, %g1
52	ldxa	[%g1] ASI_SCRATCHPAD, %g1
53
54	LOAD_ITLB_INFO(%g2, %g4, %g5)
55	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
56	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
57
58	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
59	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
60	cmp	%g2, %g6
61	bne,a,pn %xcc, tsb_miss_page_table_walk
62	 mov	FAULT_CODE_ITLB, %g3
63	andcc	%g3, _PAGE_EXEC_4V, %g0
64	be,a,pn	%xcc, tsb_do_fault
65	 mov	FAULT_CODE_ITLB, %g3
66
67	/* We have a valid entry, make hypervisor call to load
68	 * I-TLB and return from trap.
69	 *
70	 * %g3:	PTE
71	 * %g4:	vaddr
72	 */
73sun4v_itlb_load:
74	ldxa	[%g0] ASI_SCRATCHPAD, %g6
75	mov	%o0, %g1		! save %o0
76	mov	%o1, %g2		! save %o1
77	mov	%o2, %g5		! save %o2
78	mov	%o3, %g7		! save %o3
79	mov	%g4, %o0		! vaddr
80	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1	! ctx
81	mov	%g3, %o2		! PTE
82	mov	HV_MMU_IMMU, %o3	! flags
83	ta	HV_MMU_MAP_ADDR_TRAP
84	brnz,pn	%o0, sun4v_itlb_error
85	 mov	%g2, %o1		! restore %o1
86	mov	%g1, %o0		! restore %o0
87	mov	%g5, %o2		! restore %o2
88	mov	%g7, %o3		! restore %o3
89
90	retry
91
92sun4v_dtlb_miss:
93	/* Load MMU Miss base into %g2.  */
94	ldxa	[%g0] ASI_SCRATCHPAD, %g2
95
96	/* Load UTSB reg into %g1.  */
97	mov	SCRATCHPAD_UTSBREG1, %g1
98	ldxa	[%g1] ASI_SCRATCHPAD, %g1
99
100	LOAD_DTLB_INFO(%g2, %g4, %g5)
101	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
102	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
103
104	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
105	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
106	cmp	%g2, %g6
107	bne,a,pn %xcc, tsb_miss_page_table_walk
108	 mov	FAULT_CODE_DTLB, %g3
109
110	/* We have a valid entry, make hypervisor call to load
111	 * D-TLB and return from trap.
112	 *
113	 * %g3:	PTE
114	 * %g4:	vaddr
115	 */
116sun4v_dtlb_load:
117	ldxa	[%g0] ASI_SCRATCHPAD, %g6
118	mov	%o0, %g1		! save %o0
119	mov	%o1, %g2		! save %o1
120	mov	%o2, %g5		! save %o2
121	mov	%o3, %g7		! save %o3
122	mov	%g4, %o0		! vaddr
123	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1	! ctx
124	mov	%g3, %o2		! PTE
125	mov	HV_MMU_DMMU, %o3	! flags
126	ta	HV_MMU_MAP_ADDR_TRAP
127	brnz,pn	%o0, sun4v_dtlb_error
128	 mov	%g2, %o1		! restore %o1
129	mov	%g1, %o0		! restore %o0
130	mov	%g5, %o2		! restore %o2
131	mov	%g7, %o3		! restore %o3
132
133	retry
134
135sun4v_dtlb_prot:
136	SET_GL(1)
137
138	/* Load MMU Miss base into %g5.  */
139	ldxa	[%g0] ASI_SCRATCHPAD, %g5
140
141	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
142	rdpr	%tl, %g1
143	cmp	%g1, 1
144	bgu,pn	%xcc, winfix_trampoline
145	 mov	FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
146	ba,pt	%xcc, sparc64_realfault_common
147	 nop
148
149	/* Called from trap table:
150	 * %g4:	vaddr
151	 * %g5:	context
152	 * %g6: TAG TARGET
153	 */
154sun4v_itsb_miss:
155	mov	SCRATCHPAD_UTSBREG1, %g1
156	ldxa	[%g1] ASI_SCRATCHPAD, %g1
157	brz,pn	%g5, kvmap_itlb_4v
158	 mov	FAULT_CODE_ITLB, %g3
159	ba,a,pt	%xcc, sun4v_tsb_miss_common
160
161	/* Called from trap table:
162	 * %g4:	vaddr
163	 * %g5:	context
164	 * %g6: TAG TARGET
165	 */
166sun4v_dtsb_miss:
167	mov	SCRATCHPAD_UTSBREG1, %g1
168	ldxa	[%g1] ASI_SCRATCHPAD, %g1
169	brz,pn	%g5, kvmap_dtlb_4v
170	 mov	FAULT_CODE_DTLB, %g3
171
172	/* fallthrough */
173
174sun4v_tsb_miss_common:
175	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
176
177	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
178
179#ifdef CONFIG_HUGETLB_PAGE
180	mov	SCRATCHPAD_UTSBREG2, %g5
181	ldxa	[%g5] ASI_SCRATCHPAD, %g5
182	cmp	%g5, -1
183	be,pt	%xcc, 80f
184	 nop
185	COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7)
186
187	/* That clobbered %g2, reload it.  */
188	ldxa	[%g0] ASI_SCRATCHPAD, %g2
189	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
190
19180:	stx	%g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
192#endif
193
194	ba,pt	%xcc, tsb_miss_page_table_walk_sun4v_fastpath
195	 ldx	[%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
196
197sun4v_itlb_error:
198	sethi	%hi(sun4v_err_itlb_vaddr), %g1
199	stx	%g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
200	sethi	%hi(sun4v_err_itlb_ctx), %g1
201	ldxa	[%g0] ASI_SCRATCHPAD, %g6
202	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1
203	stx	%o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
204	sethi	%hi(sun4v_err_itlb_pte), %g1
205	stx	%g3, [%g1 + %lo(sun4v_err_itlb_pte)]
206	sethi	%hi(sun4v_err_itlb_error), %g1
207	stx	%o0, [%g1 + %lo(sun4v_err_itlb_error)]
208
209	rdpr	%tl, %g4
210	cmp	%g4, 1
211	ble,pt	%icc, 1f
212	 sethi	%hi(2f), %g7
213	ba,pt	%xcc, etraptl1
214	 or	%g7, %lo(2f), %g7
215
2161:	ba,pt	%xcc, etrap
2172:	 or	%g7, %lo(2b), %g7
218	mov	%l4, %o1
219	call	sun4v_itlb_error_report
220	 add	%sp, PTREGS_OFF, %o0
221
222	/* NOTREACHED */
223
224sun4v_dtlb_error:
225	sethi	%hi(sun4v_err_dtlb_vaddr), %g1
226	stx	%g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
227	sethi	%hi(sun4v_err_dtlb_ctx), %g1
228	ldxa	[%g0] ASI_SCRATCHPAD, %g6
229	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1
230	stx	%o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
231	sethi	%hi(sun4v_err_dtlb_pte), %g1
232	stx	%g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
233	sethi	%hi(sun4v_err_dtlb_error), %g1
234	stx	%o0, [%g1 + %lo(sun4v_err_dtlb_error)]
235
236	rdpr	%tl, %g4
237	cmp	%g4, 1
238	ble,pt	%icc, 1f
239	 sethi	%hi(2f), %g7
240	ba,pt	%xcc, etraptl1
241	 or	%g7, %lo(2f), %g7
242
2431:	ba,pt	%xcc, etrap
2442:	 or	%g7, %lo(2b), %g7
245	mov	%l4, %o1
246	call	sun4v_dtlb_error_report
247	 add	%sp, PTREGS_OFF, %o0
248
249	/* NOTREACHED */
250
251	/* Instruction Access Exception, tl0. */
252sun4v_iacc:
253	ldxa	[%g0] ASI_SCRATCHPAD, %g2
254	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
255	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
256	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
257	sllx	%g3, 16, %g3
258	or	%g5, %g3, %g5
259	ba,pt	%xcc, etrap
260	 rd	%pc, %g7
261	mov	%l4, %o1
262	mov	%l5, %o2
263	call	sun4v_insn_access_exception
264	 add	%sp, PTREGS_OFF, %o0
265	ba,a,pt	%xcc, rtrap
266
267	/* Instruction Access Exception, tl1. */
268sun4v_iacc_tl1:
269	ldxa	[%g0] ASI_SCRATCHPAD, %g2
270	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
271	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
272	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
273	sllx	%g3, 16, %g3
274	or	%g5, %g3, %g5
275	ba,pt	%xcc, etraptl1
276	 rd	%pc, %g7
277	mov	%l4, %o1
278	mov	%l5, %o2
279	call	sun4v_insn_access_exception_tl1
280	 add	%sp, PTREGS_OFF, %o0
281	ba,a,pt	%xcc, rtrap
282
283	/* Data Access Exception, tl0. */
284sun4v_dacc:
285	ldxa	[%g0] ASI_SCRATCHPAD, %g2
286	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
287	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
288	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
289	sllx	%g3, 16, %g3
290	or	%g5, %g3, %g5
291	ba,pt	%xcc, etrap
292	 rd	%pc, %g7
293	mov	%l4, %o1
294	mov	%l5, %o2
295	call	sun4v_data_access_exception
296	 add	%sp, PTREGS_OFF, %o0
297	ba,a,pt	%xcc, rtrap
298
299	/* Data Access Exception, tl1. */
300sun4v_dacc_tl1:
301	ldxa	[%g0] ASI_SCRATCHPAD, %g2
302	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
303	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
304	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
305	sllx	%g3, 16, %g3
306	or	%g5, %g3, %g5
307	ba,pt	%xcc, etraptl1
308	 rd	%pc, %g7
309	mov	%l4, %o1
310	mov	%l5, %o2
311	call	sun4v_data_access_exception_tl1
312	 add	%sp, PTREGS_OFF, %o0
313	ba,a,pt	%xcc, rtrap
314
315	/* Memory Address Unaligned.  */
316sun4v_mna:
317	/* Window fixup? */
318	rdpr	%tl, %g2
319	cmp	%g2, 1
320	ble,pt	%icc, 1f
321	 nop
322
323	SET_GL(1)
324	ldxa	[%g0] ASI_SCRATCHPAD, %g2
325	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
326	mov	HV_FAULT_TYPE_UNALIGNED, %g3
327	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g4
328	sllx	%g3, 16, %g3
329	or	%g4, %g3, %g4
330	ba,pt	%xcc, winfix_mna
331	 rdpr	%tpc, %g3
332	/* not reached */
333
3341:	ldxa	[%g0] ASI_SCRATCHPAD, %g2
335	mov	HV_FAULT_TYPE_UNALIGNED, %g3
336	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
337	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
338	sllx	%g3, 16, %g3
339	or	%g5, %g3, %g5
340
341	ba,pt	%xcc, etrap
342	 rd	%pc, %g7
343	mov	%l4, %o1
344	mov	%l5, %o2
345	call	sun4v_do_mna
346	 add	%sp, PTREGS_OFF, %o0
347	ba,a,pt	%xcc, rtrap
348
349	/* Privileged Action.  */
350sun4v_privact:
351	ba,pt	%xcc, etrap
352	 rd	%pc, %g7
353	call	do_privact
354	 add	%sp, PTREGS_OFF, %o0
355	ba,a,pt	%xcc, rtrap
356
357	/* Unaligned ldd float, tl0. */
358sun4v_lddfmna:
359	ldxa	[%g0] ASI_SCRATCHPAD, %g2
360	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
361	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
362	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
363	sllx	%g3, 16, %g3
364	or	%g5, %g3, %g5
365	ba,pt	%xcc, etrap
366	 rd	%pc, %g7
367	mov	%l4, %o1
368	mov	%l5, %o2
369	call	handle_lddfmna
370	 add	%sp, PTREGS_OFF, %o0
371	ba,a,pt	%xcc, rtrap
372
373	/* Unaligned std float, tl0. */
374sun4v_stdfmna:
375	ldxa	[%g0] ASI_SCRATCHPAD, %g2
376	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
377	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
378	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
379	sllx	%g3, 16, %g3
380	or	%g5, %g3, %g5
381	ba,pt	%xcc, etrap
382	 rd	%pc, %g7
383	mov	%l4, %o1
384	mov	%l5, %o2
385	call	handle_stdfmna
386	 add	%sp, PTREGS_OFF, %o0
387	ba,a,pt	%xcc, rtrap
388
389#define BRANCH_ALWAYS	0x10680000
390#define NOP		0x01000000
391#define SUN4V_DO_PATCH(OLD, NEW)	\
392	sethi	%hi(NEW), %g1; \
393	or	%g1, %lo(NEW), %g1; \
394	sethi	%hi(OLD), %g2; \
395	or	%g2, %lo(OLD), %g2; \
396	sub	%g1, %g2, %g1; \
397	sethi	%hi(BRANCH_ALWAYS), %g3; \
398	sll	%g1, 11, %g1; \
399	srl	%g1, 11 + 2, %g1; \
400	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
401	or	%g3, %g1, %g3; \
402	stw	%g3, [%g2]; \
403	sethi	%hi(NOP), %g3; \
404	or	%g3, %lo(NOP), %g3; \
405	stw	%g3, [%g2 + 0x4]; \
406	flush	%g2;
407
408	.globl	sun4v_patch_tlb_handlers
409	.type	sun4v_patch_tlb_handlers,#function
410sun4v_patch_tlb_handlers:
411	SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
412	SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
413	SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
414	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
415	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
416	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
417	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
418	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
419	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
420	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
421	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
422	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
423	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
424	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
425	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
426	retl
427	 nop
428	.size	sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
429