1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#include <linux/export.h>
7#include <asm/alternative-asm.h>
8#include <asm/asm.h>
9#include <asm/asmmacro.h>
10#include <asm/asm-extable.h>
11#include <asm/cpu.h>
12#include <asm/regdef.h>
13
14SYM_FUNC_START(__clear_user)
15	/*
16	 * Some CPUs support hardware unaligned access
17	 */
18	ALTERNATIVE	"b __clear_user_generic",	\
19			"b __clear_user_fast", CPU_FEATURE_UAL
20SYM_FUNC_END(__clear_user)
21
22EXPORT_SYMBOL(__clear_user)
23
24/*
25 * unsigned long __clear_user_generic(void *addr, size_t size)
26 *
27 * a0: addr
28 * a1: size
29 */
30SYM_FUNC_START(__clear_user_generic)
31	beqz	a1, 2f
32
331:	st.b	zero, a0, 0
34	addi.d	a0, a0, 1
35	addi.d	a1, a1, -1
36	bgtz	a1, 1b
37
382:	move	a0, a1
39	jr	ra
40
41	_asm_extable 1b, 2b
42SYM_FUNC_END(__clear_user_generic)
43
44/*
45 * unsigned long __clear_user_fast(void *addr, unsigned long size)
46 *
47 * a0: addr
48 * a1: size
49 */
50SYM_FUNC_START(__clear_user_fast)
51	sltui	t0, a1, 9
52	bnez	t0, .Lsmall
53
54	add.d	a2, a0, a1
550:	st.d	zero, a0, 0
56
57	/* align up address */
58	addi.d	a0, a0, 8
59	bstrins.d	a0, zero, 2, 0
60
61	addi.d	a3, a2, -64
62	bgeu	a0, a3, .Llt64
63
64	/* set 64 bytes at a time */
65.Lloop64:
661:	st.d	zero, a0, 0
672:	st.d	zero, a0, 8
683:	st.d	zero, a0, 16
694:	st.d	zero, a0, 24
705:	st.d	zero, a0, 32
716:	st.d	zero, a0, 40
727:	st.d	zero, a0, 48
738:	st.d	zero, a0, 56
74	addi.d	a0, a0, 64
75	bltu	a0, a3, .Lloop64
76
77	/* set the remaining bytes */
78.Llt64:
79	addi.d	a3, a2, -32
80	bgeu	a0, a3, .Llt32
819:	st.d	zero, a0, 0
8210:	st.d	zero, a0, 8
8311:	st.d	zero, a0, 16
8412:	st.d	zero, a0, 24
85	addi.d	a0, a0, 32
86
87.Llt32:
88	addi.d	a3, a2, -16
89	bgeu	a0, a3, .Llt16
9013:	st.d	zero, a0, 0
9114:	st.d	zero, a0, 8
92	addi.d	a0, a0, 16
93
94.Llt16:
95	addi.d	a3, a2, -8
96	bgeu	a0, a3, .Llt8
9715:	st.d	zero, a0, 0
98	addi.d	a0, a0, 8
99
100.Llt8:
10116:	st.d	zero, a2, -8
102
103	/* return */
104	move	a0, zero
105	jr	ra
106
107	.align	4
108.Lsmall:
109	pcaddi	t0, 4
110	slli.d	a2, a1, 4
111	add.d	t0, t0, a2
112	jr	t0
113
114	.align	4
115	move	a0, zero
116	jr	ra
117
118	.align	4
11917:	st.b	zero, a0, 0
120	move	a0, zero
121	jr	ra
122
123	.align	4
12418:	st.h	zero, a0, 0
125	move	a0, zero
126	jr	ra
127
128	.align	4
12919:	st.h	zero, a0, 0
13020:	st.b	zero, a0, 2
131	move	a0, zero
132	jr	ra
133
134	.align	4
13521:	st.w	zero, a0, 0
136	move	a0, zero
137	jr	ra
138
139	.align	4
14022:	st.w	zero, a0, 0
14123:	st.b	zero, a0, 4
142	move	a0, zero
143	jr	ra
144
145	.align	4
14624:	st.w	zero, a0, 0
14725:	st.h	zero, a0, 4
148	move	a0, zero
149	jr	ra
150
151	.align	4
15226:	st.w	zero, a0, 0
15327:	st.w	zero, a0, 3
154	move	a0, zero
155	jr	ra
156
157	.align	4
15828:	st.d	zero, a0, 0
159	move	a0, zero
160	jr	ra
161
162	/* fixup and ex_table */
163.Llarge_fixup:
164	sub.d	a1, a2, a0
165
166.Lsmall_fixup:
16729:	st.b	zero, a0, 0
168	addi.d	a0, a0, 1
169	addi.d	a1, a1, -1
170	bgt	a1, zero, 29b
171
172.Lexit:
173	move	a0, a1
174	jr	ra
175
176	_asm_extable 0b, .Lsmall_fixup
177	_asm_extable 1b, .Llarge_fixup
178	_asm_extable 2b, .Llarge_fixup
179	_asm_extable 3b, .Llarge_fixup
180	_asm_extable 4b, .Llarge_fixup
181	_asm_extable 5b, .Llarge_fixup
182	_asm_extable 6b, .Llarge_fixup
183	_asm_extable 7b, .Llarge_fixup
184	_asm_extable 8b, .Llarge_fixup
185	_asm_extable 9b, .Llarge_fixup
186	_asm_extable 10b, .Llarge_fixup
187	_asm_extable 11b, .Llarge_fixup
188	_asm_extable 12b, .Llarge_fixup
189	_asm_extable 13b, .Llarge_fixup
190	_asm_extable 14b, .Llarge_fixup
191	_asm_extable 15b, .Llarge_fixup
192	_asm_extable 16b, .Llarge_fixup
193	_asm_extable 17b, .Lexit
194	_asm_extable 18b, .Lsmall_fixup
195	_asm_extable 19b, .Lsmall_fixup
196	_asm_extable 20b, .Lsmall_fixup
197	_asm_extable 21b, .Lsmall_fixup
198	_asm_extable 22b, .Lsmall_fixup
199	_asm_extable 23b, .Lsmall_fixup
200	_asm_extable 24b, .Lsmall_fixup
201	_asm_extable 25b, .Lsmall_fixup
202	_asm_extable 26b, .Lsmall_fixup
203	_asm_extable 27b, .Lsmall_fixup
204	_asm_extable 28b, .Lsmall_fixup
205	_asm_extable 29b, .Lexit
206SYM_FUNC_END(__clear_user_fast)
207