1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * hypersparc.S: High speed Hypersparc mmu/cache operations.
4 *
5 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
6 */
7
8#include <asm/ptrace.h>
9#include <asm/psr.h>
10#include <asm/asm-offsets.h>
11#include <asm/asi.h>
12#include <asm/page.h>
13#include <asm/pgtable.h>
14#include <asm/pgtsrmmu.h>
15#include <linux/init.h>
16
17	.text
18	.align	4
19
20	.globl	hypersparc_flush_cache_all, hypersparc_flush_cache_mm
21	.globl	hypersparc_flush_cache_range, hypersparc_flush_cache_page
22	.globl	hypersparc_flush_page_to_ram
23	.globl	hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
24	.globl	hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
25	.globl	hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
26
27hypersparc_flush_cache_all:
28	WINDOW_FLUSH(%g4, %g5)
29	sethi	%hi(vac_cache_size), %g4
30	ld	[%g4 + %lo(vac_cache_size)], %g5
31	sethi	%hi(vac_line_size), %g1
32	ld	[%g1 + %lo(vac_line_size)], %g2
331:
34	subcc	%g5, %g2, %g5			! hyper_flush_unconditional_combined
35	bne	1b
36	 sta	%g0, [%g5] ASI_M_FLUSH_CTX
37	retl
38	 sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE	! hyper_flush_whole_icache
39
40	/* We expand the window flush to get maximum performance. */
41hypersparc_flush_cache_mm:
42#ifndef CONFIG_SMP
43	ld	[%o0 + AOFF_mm_context], %g1
44	cmp	%g1, -1
45	be	hypersparc_flush_cache_mm_out
46#endif
47	WINDOW_FLUSH(%g4, %g5)
48
49	sethi	%hi(vac_line_size), %g1
50	ld	[%g1 + %lo(vac_line_size)], %o1
51	sethi	%hi(vac_cache_size), %g2
52	ld	[%g2 + %lo(vac_cache_size)], %o0
53	add	%o1, %o1, %g1
54	add	%o1, %g1, %g2
55	add	%o1, %g2, %g3
56	add	%o1, %g3, %g4
57	add	%o1, %g4, %g5
58	add	%o1, %g5, %o4
59	add	%o1, %o4, %o5
60
61	/* BLAMMO! */
621:
63	subcc	%o0, %o5, %o0				! hyper_flush_cache_user
64	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_USER
65	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_USER
66	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_USER
67	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_USER
68	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_USER
69	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_USER
70	sta	%g0, [%o0 + %g5] ASI_M_FLUSH_USER
71	bne	1b
72	 sta	%g0, [%o0 + %o4] ASI_M_FLUSH_USER
73hypersparc_flush_cache_mm_out:
74	retl
75	 nop
76
77	/* The things we do for performance... */
78hypersparc_flush_cache_range:
79	ld	[%o0 + VMA_VM_MM], %o0
80#ifndef CONFIG_SMP
81	ld	[%o0 + AOFF_mm_context], %g1
82	cmp	%g1, -1
83	be	hypersparc_flush_cache_range_out
84#endif
85	WINDOW_FLUSH(%g4, %g5)
86
87	sethi	%hi(vac_line_size), %g1
88	ld	[%g1 + %lo(vac_line_size)], %o4
89	sethi	%hi(vac_cache_size), %g2
90	ld	[%g2 + %lo(vac_cache_size)], %o3
91
92	/* Here comes the fun part... */
93	add	%o2, (PAGE_SIZE - 1), %o2
94	andn	%o1, (PAGE_SIZE - 1), %o1
95	add	%o4, %o4, %o5
96	andn	%o2, (PAGE_SIZE - 1), %o2
97	add	%o4, %o5, %g1
98	sub	%o2, %o1, %g4
99	add	%o4, %g1, %g2
100	sll	%o3, 2, %g5
101	add	%o4, %g2, %g3
102	cmp	%g4, %g5
103	add	%o4, %g3, %g4
104	blu	0f
105	 add	%o4, %g4, %g5
106	add	%o4, %g5, %g7
107
108	/* Flush entire user space, believe it or not this is quicker
109	 * than page at a time flushings for range > (cache_size<<2).
110	 */
1111:
112	subcc	%o3, %g7, %o3
113	sta	%g0, [%o3 + %g0] ASI_M_FLUSH_USER
114	sta	%g0, [%o3 + %o4] ASI_M_FLUSH_USER
115	sta	%g0, [%o3 + %o5] ASI_M_FLUSH_USER
116	sta	%g0, [%o3 + %g1] ASI_M_FLUSH_USER
117	sta	%g0, [%o3 + %g2] ASI_M_FLUSH_USER
118	sta	%g0, [%o3 + %g3] ASI_M_FLUSH_USER
119	sta	%g0, [%o3 + %g4] ASI_M_FLUSH_USER
120	bne	1b
121	 sta	%g0, [%o3 + %g5] ASI_M_FLUSH_USER
122	retl
123	 nop
124
125	/* Below our threshold, flush one page at a time. */
1260:
127	ld	[%o0 + AOFF_mm_context], %o0
128	mov	SRMMU_CTX_REG, %g7
129	lda	[%g7] ASI_M_MMUREGS, %o3
130	sta	%o0, [%g7] ASI_M_MMUREGS
131	add	%o2, -PAGE_SIZE, %o0
1321:
133	or	%o0, 0x400, %g7
134	lda	[%g7] ASI_M_FLUSH_PROBE, %g7
135	orcc	%g7, 0, %g0
136	be,a	3f
137	 mov	%o0, %o2
138	add	%o4, %g5, %g7
1392:
140	sub	%o2, %g7, %o2
141	sta	%g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
142	sta	%g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
143	sta	%g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
144	sta	%g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
145	sta	%g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
146	sta	%g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
147	andcc	%o2, 0xffc, %g0
148	sta	%g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
149	bne	2b
150	 sta	%g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
1513:
152	cmp	%o2, %o1
153	bne	1b
154	 add	%o2, -PAGE_SIZE, %o0
155	mov	SRMMU_FAULT_STATUS, %g5
156	lda	[%g5] ASI_M_MMUREGS, %g0
157	mov	SRMMU_CTX_REG, %g7
158	sta	%o3, [%g7] ASI_M_MMUREGS
159hypersparc_flush_cache_range_out:
160	retl
161	 nop
162
163	/* HyperSparc requires a valid mapping where we are about to flush
164	 * in order to check for a physical tag match during the flush.
165	 */
166	/* Verified, my ass... */
167hypersparc_flush_cache_page:
168	ld	[%o0 + VMA_VM_MM], %o0
169	ld	[%o0 + AOFF_mm_context], %g2
170#ifndef CONFIG_SMP
171	cmp	%g2, -1
172	be	hypersparc_flush_cache_page_out
173#endif
174	WINDOW_FLUSH(%g4, %g5)
175
176	sethi	%hi(vac_line_size), %g1
177	ld	[%g1 + %lo(vac_line_size)], %o4
178	mov	SRMMU_CTX_REG, %o3
179	andn	%o1, (PAGE_SIZE - 1), %o1
180	lda	[%o3] ASI_M_MMUREGS, %o2
181	sta	%g2, [%o3] ASI_M_MMUREGS
182	or	%o1, 0x400, %o5
183	lda	[%o5] ASI_M_FLUSH_PROBE, %g1
184	orcc	%g0, %g1, %g0
185	be	2f
186	 add	%o4, %o4, %o5
187	sub	%o1, -PAGE_SIZE, %o1
188	add	%o4, %o5, %g1
189	add	%o4, %g1, %g2
190	add	%o4, %g2, %g3
191	add	%o4, %g3, %g4
192	add	%o4, %g4, %g5
193	add	%o4, %g5, %g7
194
195	/* BLAMMO! */
1961:
197	sub	%o1, %g7, %o1
198	sta	%g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
199	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
200	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
201	sta	%g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
202	sta	%g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
203	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
204	andcc	%o1, 0xffc, %g0
205	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
206	bne	1b
207	 sta	%g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
2082:
209	mov	SRMMU_FAULT_STATUS, %g7
210	mov	SRMMU_CTX_REG, %g4
211	lda	[%g7] ASI_M_MMUREGS, %g0
212	sta	%o2, [%g4] ASI_M_MMUREGS
213hypersparc_flush_cache_page_out:
214	retl
215	 nop
216
217hypersparc_flush_sig_insns:
218	flush	%o1
219	retl
220	 flush	%o1 + 4
221
222	/* HyperSparc is copy-back. */
223hypersparc_flush_page_to_ram:
224	sethi	%hi(vac_line_size), %g1
225	ld	[%g1 + %lo(vac_line_size)], %o4
226	andn	%o0, (PAGE_SIZE - 1), %o0
227	add	%o4, %o4, %o5
228	or	%o0, 0x400, %g7
229	lda	[%g7] ASI_M_FLUSH_PROBE, %g5
230	add	%o4, %o5, %g1
231	orcc	%g5, 0, %g0
232	be	2f
233	 add	%o4, %g1, %g2
234	add	%o4, %g2, %g3
235	sub	%o0, -PAGE_SIZE, %o0
236	add	%o4, %g3, %g4
237	add	%o4, %g4, %g5
238	add	%o4, %g5, %g7
239
240	/* BLAMMO! */
2411:
242	sub	%o0, %g7, %o0
243	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
244	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
245	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
246	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
247	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
248	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
249	andcc	%o0, 0xffc, %g0
250	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
251	bne	1b
252	 sta	%g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
2532:
254	mov	SRMMU_FAULT_STATUS, %g1
255	retl
256	 lda	[%g1] ASI_M_MMUREGS, %g0
257
258	/* HyperSparc is IO cache coherent. */
259hypersparc_flush_page_for_dma:
260	retl
261	 nop
262
263	/* It was noted that at boot time a TLB flush all in a delay slot
264	 * can deliver an illegal instruction to the processor if the timing
265	 * is just right...
266	 */
267hypersparc_flush_tlb_all:
268	mov	0x400, %g1
269	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
270	retl
271	 nop
272
273hypersparc_flush_tlb_mm:
274	mov	SRMMU_CTX_REG, %g1
275	ld	[%o0 + AOFF_mm_context], %o1
276	lda	[%g1] ASI_M_MMUREGS, %g5
277#ifndef CONFIG_SMP
278	cmp	%o1, -1
279	be	hypersparc_flush_tlb_mm_out
280#endif
281	 mov	0x300, %g2
282	sta	%o1, [%g1] ASI_M_MMUREGS
283	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
284hypersparc_flush_tlb_mm_out:
285	retl
286	 sta	%g5, [%g1] ASI_M_MMUREGS
287
288hypersparc_flush_tlb_range:
289	ld	[%o0 + VMA_VM_MM], %o0
290	mov	SRMMU_CTX_REG, %g1
291	ld	[%o0 + AOFF_mm_context], %o3
292	lda	[%g1] ASI_M_MMUREGS, %g5
293#ifndef CONFIG_SMP
294	cmp	%o3, -1
295	be	hypersparc_flush_tlb_range_out
296#endif
297	 sethi	%hi(~((1 << PGDIR_SHIFT) - 1)), %o4
298	sta	%o3, [%g1] ASI_M_MMUREGS
299	and	%o1, %o4, %o1
300	add	%o1, 0x200, %o1
301	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
3021:
303	sub	%o1, %o4, %o1
304	cmp	%o1, %o2
305	blu,a	1b
306	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
307hypersparc_flush_tlb_range_out:
308	retl
309	 sta	%g5, [%g1] ASI_M_MMUREGS
310
311hypersparc_flush_tlb_page:
312	ld	[%o0 + VMA_VM_MM], %o0
313	mov	SRMMU_CTX_REG, %g1
314	ld	[%o0 + AOFF_mm_context], %o3
315	andn	%o1, (PAGE_SIZE - 1), %o1
316#ifndef CONFIG_SMP
317	cmp	%o3, -1
318	be	hypersparc_flush_tlb_page_out
319#endif
320	 lda	[%g1] ASI_M_MMUREGS, %g5
321	sta	%o3, [%g1] ASI_M_MMUREGS
322	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
323hypersparc_flush_tlb_page_out:
324	retl
325	 sta	%g5, [%g1] ASI_M_MMUREGS
326
327	__INIT
328
329	/* High speed page clear/copy. */
330hypersparc_bzero_1page:
331/* NOTE: This routine has to be shorter than 40insns --jj */
332	clr	%g1
333	mov	32, %g2
334	mov	64, %g3
335	mov	96, %g4
336	mov	128, %g5
337	mov	160, %g7
338	mov	192, %o2
339	mov	224, %o3
340	mov	16, %o1
3411:
342	stda	%g0, [%o0 + %g0] ASI_M_BFILL
343	stda	%g0, [%o0 + %g2] ASI_M_BFILL
344	stda	%g0, [%o0 + %g3] ASI_M_BFILL
345	stda	%g0, [%o0 + %g4] ASI_M_BFILL
346	stda	%g0, [%o0 + %g5] ASI_M_BFILL
347	stda	%g0, [%o0 + %g7] ASI_M_BFILL
348	stda	%g0, [%o0 + %o2] ASI_M_BFILL
349	stda	%g0, [%o0 + %o3] ASI_M_BFILL
350	subcc	%o1, 1, %o1
351	bne	1b
352	 add	%o0, 256, %o0
353
354	retl
355	 nop
356
357hypersparc_copy_1page:
358/* NOTE: This routine has to be shorter than 70insns --jj */
359	sub	%o1, %o0, %o2		! difference
360	mov	16, %g1
3611:
362	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
363	add	%o0, 32, %o0
364	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
365	add	%o0, 32, %o0
366	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
367	add	%o0, 32, %o0
368	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
369	add	%o0, 32, %o0
370	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
371	add	%o0, 32, %o0
372	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
373	add	%o0, 32, %o0
374	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
375	add	%o0, 32, %o0
376	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
377	subcc	%g1, 1, %g1
378	bne	1b
379	 add	%o0, 32, %o0
380
381	retl
382	 nop
383
384	.globl	hypersparc_setup_blockops
385hypersparc_setup_blockops:
386	sethi	%hi(bzero_1page), %o0
387	or	%o0, %lo(bzero_1page), %o0
388	sethi	%hi(hypersparc_bzero_1page), %o1
389	or	%o1, %lo(hypersparc_bzero_1page), %o1
390	sethi	%hi(hypersparc_copy_1page), %o2
391	or	%o2, %lo(hypersparc_copy_1page), %o2
392	ld	[%o1], %o4
3931:
394	add	%o1, 4, %o1
395	st	%o4, [%o0]
396	add	%o0, 4, %o0
397	cmp	%o1, %o2
398	bne	1b
399	 ld	[%o1], %o4
400	sethi	%hi(__copy_1page), %o0
401	or	%o0, %lo(__copy_1page), %o0
402	sethi	%hi(hypersparc_setup_blockops), %o2
403	or	%o2, %lo(hypersparc_setup_blockops), %o2
404	ld	[%o1], %o4
4051:
406	add	%o1, 4, %o1
407	st	%o4, [%o0]
408	add	%o0, 4, %o0
409	cmp	%o1, %o2
410	bne	1b
411	 ld	[%o1], %o4
412	sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE
413	retl
414	 nop
415