1!
2! Fast SH memcpy
3!
4! by Toshiyasu Morita (tm@netcom.com)
5! hacked by J"orn Rernnecke (joern.rennecke@superh.com) ("o for o-umlaut)
6! SH5 code Copyright 2002 SuperH Ltd.
7!
8! Entry: ARG0: destination pointer
9!        ARG1: source pointer
10!        ARG2: byte count
11!
12! Exit:  RESULT: destination pointer
13!        any other registers in the range r0-r7: trashed
14!
15! Notes: Usually one wants to do small reads and write a longword, but
16!        unfortunately it is difficult in some cases to concatanate bytes
17!        into a longword on the SH, so this does a longword read and small
18!        writes.
19!
20! This implementation makes two assumptions about how it is called:
21!
22! 1.: If the byte count is nonzero, the address of the last byte to be
23!     copied is unsigned greater than the address of the first byte to
24!     be copied.  This could be easily swapped for a signed comparison,
25!     but the algorithm used needs some comparison.
26!
27! 2.: When there are two or three bytes in the last word of an 11-or-more
28!     bytes memory chunk to b copied, the rest of the word can be read
29!     without side effects.
30!     This could be easily changed by increasing the minimum size of
31!     a fast memcpy and the amount subtracted from r7 before L_2l_loop be 2,
32!     however, this would cost a few extra cyles on average.
33!     For SHmedia, the assumption is that any quadword can be read in its
34!     enirety if at least one byte is included in the copy.
35
36/* Imported into Linux kernel by Richard Curnow.  This is used to implement the
37   __copy_user function in the general case, so it has to be a distinct
38   function from intra-kernel memcpy to allow for exception fix-ups in the
39   event that the user pointer is bad somewhere in the copy (e.g. due to
40   running off the end of the vma).
41
42   Note, this algorithm will be slightly wasteful in the case where the source
43   and destination pointers are equally aligned, because the stlo/sthi pairs
44   could then be merged back into single stores.  If there are a lot of cache
45   misses, this is probably offset by the stall lengths on the preloads.
46
47*/
48
49/* NOTE : Prefetches removed and allocos guarded by synco to avoid TAKum03020
50 * erratum.  The first two prefetches are nop-ed out to avoid upsetting the
51 * instruction counts used in the jump address calculation.
52 * */
53
54	.section .text..SHmedia32,"ax"
55	.little
56	.balign 32
57	.global copy_user_memcpy
58	.global copy_user_memcpy_end
59copy_user_memcpy:
60
61#define LDUAQ(P,O,D0,D1) ldlo.q P,O,D0; ldhi.q P,O+7,D1
62#define STUAQ(P,O,D0,D1) stlo.q P,O,D0; sthi.q P,O+7,D1
63#define LDUAL(P,O,D0,D1) ldlo.l P,O,D0; ldhi.l P,O+3,D1
64#define STUAL(P,O,D0,D1) stlo.l P,O,D0; sthi.l P,O+3,D1
65
66	nop ! ld.b r3,0,r63 ! TAKum03020
67	pta/l Large,tr0
68	movi 25,r0
69	bgeu/u r4,r0,tr0
70	nsb r4,r0
71	shlli r0,5,r0
72	movi (L1-L0+63*32 + 1) & 0xffff,r1
73	sub r1, r0, r0
74L0:	ptrel r0,tr0
75	add r2,r4,r5
76	ptabs r18,tr1
77	add r3,r4,r6
78	blink tr0,r63
79
80/* Rearranged to make cut2 safe */
81	.balign 8
82L4_7:	/* 4..7 byte memcpy cntd. */
83	stlo.l r2, 0, r0
84	or r6, r7, r6
85	sthi.l r5, -1, r6
86	stlo.l r5, -4, r6
87	blink tr1,r63
88
89	.balign 8
90L1:	/* 0 byte memcpy */
91	nop
92	blink tr1,r63
93	nop
94	nop
95	nop
96	nop
97
98L2_3:	/* 2 or 3 byte memcpy cntd. */
99	st.b r5,-1,r6
100	blink tr1,r63
101
102	/* 1 byte memcpy */
103	ld.b r3,0,r0
104	st.b r2,0,r0
105	blink tr1,r63
106
107L8_15:	/* 8..15 byte memcpy cntd. */
108	stlo.q r2, 0, r0
109	or r6, r7, r6
110	sthi.q r5, -1, r6
111	stlo.q r5, -8, r6
112	blink tr1,r63
113
114	/* 2 or 3 byte memcpy */
115	ld.b r3,0,r0
116	nop ! ld.b r2,0,r63 ! TAKum03020
117	ld.b r3,1,r1
118	st.b r2,0,r0
119	pta/l L2_3,tr0
120	ld.b r6,-1,r6
121	st.b r2,1,r1
122	blink tr0, r63
123
124	/* 4 .. 7 byte memcpy */
125	LDUAL (r3, 0, r0, r1)
126	pta L4_7, tr0
127	ldlo.l r6, -4, r7
128	or r0, r1, r0
129	sthi.l r2, 3, r0
130	ldhi.l r6, -1, r6
131	blink tr0, r63
132
133	/* 8 .. 15 byte memcpy */
134	LDUAQ (r3, 0, r0, r1)
135	pta L8_15, tr0
136	ldlo.q r6, -8, r7
137	or r0, r1, r0
138	sthi.q r2, 7, r0
139	ldhi.q r6, -1, r6
140	blink tr0, r63
141
142	/* 16 .. 24 byte memcpy */
143	LDUAQ (r3, 0, r0, r1)
144	LDUAQ (r3, 8, r8, r9)
145	or r0, r1, r0
146	sthi.q r2, 7, r0
147	or r8, r9, r8
148	sthi.q r2, 15, r8
149	ldlo.q r6, -8, r7
150	ldhi.q r6, -1, r6
151	stlo.q r2, 8, r8
152	stlo.q r2, 0, r0
153	or r6, r7, r6
154	sthi.q r5, -1, r6
155	stlo.q r5, -8, r6
156	blink tr1,r63
157
158Large:
159	! ld.b r2, 0, r63 ! TAKum03020
160	pta/l  Loop_ua, tr1
161	ori r3, -8, r7
162	sub r2, r7, r22
163	sub r3, r2, r6
164	add r2, r4, r5
165	ldlo.q r3, 0, r0
166	addi r5, -16, r5
167	movi 64+8, r27 ! could subtract r7 from that.
168	stlo.q r2, 0, r0
169	sthi.q r2, 7, r0
170	ldx.q r22, r6, r0
171	bgtu/l r27, r4, tr1
172
173	addi r5, -48, r27
174	pta/l Loop_line, tr0
175	addi r6, 64, r36
176	addi r6, -24, r19
177	addi r6, -16, r20
178	addi r6, -8, r21
179
180Loop_line:
181	! ldx.q r22, r36, r63 ! TAKum03020
182	alloco r22, 32
183	synco
184	addi r22, 32, r22
185	ldx.q r22, r19, r23
186	sthi.q r22, -25, r0
187	ldx.q r22, r20, r24
188	ldx.q r22, r21, r25
189	stlo.q r22, -32, r0
190	ldx.q r22, r6,  r0
191	sthi.q r22, -17, r23
192	sthi.q r22,  -9, r24
193	sthi.q r22,  -1, r25
194	stlo.q r22, -24, r23
195	stlo.q r22, -16, r24
196	stlo.q r22,  -8, r25
197	bgeu r27, r22, tr0
198
199Loop_ua:
200	addi r22, 8, r22
201	sthi.q r22, -1, r0
202	stlo.q r22, -8, r0
203	ldx.q r22, r6, r0
204	bgtu/l r5, r22, tr1
205
206	add r3, r4, r7
207	ldlo.q r7, -8, r1
208	sthi.q r22, 7, r0
209	ldhi.q r7, -1, r7
210	ptabs r18,tr1
211	stlo.q r22, 0, r0
212	or r1, r7, r1
213	sthi.q r5, 15, r1
214	stlo.q r5, 8, r1
215	blink tr1, r63
216copy_user_memcpy_end:
217	nop
218