1/*
2 * Memory copy functions for 32-bit PowerPC.
3 *
4 * Copyright (C) 1996-2005 Paul Mackerras.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11#include <asm/processor.h>
12#include <asm/cache.h>
13#include <asm/errno.h>
14#include <asm/ppc_asm.h>
15
16#define COPY_16_BYTES		\
17	lwz	r7,4(r4);	\
18	lwz	r8,8(r4);	\
19	lwz	r9,12(r4);	\
20	lwzu	r10,16(r4);	\
21	stw	r7,4(r6);	\
22	stw	r8,8(r6);	\
23	stw	r9,12(r6);	\
24	stwu	r10,16(r6)
25
26#define COPY_16_BYTES_WITHEX(n)	\
278 ## n ## 0:			\
28	lwz	r7,4(r4);	\
298 ## n ## 1:			\
30	lwz	r8,8(r4);	\
318 ## n ## 2:			\
32	lwz	r9,12(r4);	\
338 ## n ## 3:			\
34	lwzu	r10,16(r4);	\
358 ## n ## 4:			\
36	stw	r7,4(r6);	\
378 ## n ## 5:			\
38	stw	r8,8(r6);	\
398 ## n ## 6:			\
40	stw	r9,12(r6);	\
418 ## n ## 7:			\
42	stwu	r10,16(r6)
43
44#define COPY_16_BYTES_EXCODE(n)			\
459 ## n ## 0:					\
46	addi	r5,r5,-(16 * n);		\
47	b	104f;				\
489 ## n ## 1:					\
49	addi	r5,r5,-(16 * n);		\
50	b	105f;				\
51.section __ex_table,"a";			\
52	.align	2;				\
53	.long	8 ## n ## 0b,9 ## n ## 0b;	\
54	.long	8 ## n ## 1b,9 ## n ## 0b;	\
55	.long	8 ## n ## 2b,9 ## n ## 0b;	\
56	.long	8 ## n ## 3b,9 ## n ## 0b;	\
57	.long	8 ## n ## 4b,9 ## n ## 1b;	\
58	.long	8 ## n ## 5b,9 ## n ## 1b;	\
59	.long	8 ## n ## 6b,9 ## n ## 1b;	\
60	.long	8 ## n ## 7b,9 ## n ## 1b;	\
61	.text
62
63	.text
64	.stabs	"arch/powerpc/lib/",N_SO,0,0,0f
65	.stabs	"copy_32.S",N_SO,0,0,0f
660:
67
68CACHELINE_BYTES = L1_CACHE_BYTES
69LG_CACHELINE_BYTES = L1_CACHE_SHIFT
70CACHELINE_MASK = (L1_CACHE_BYTES-1)
71
72/*
73 * Use dcbz on the complete cache lines in the destination
74 * to set them to zero.  This requires that the destination
75 * area is cacheable.  -- paulus
76 */
77_GLOBAL(cacheable_memzero)
78	mr	r5,r4
79	li	r4,0
80	addi	r6,r3,-4
81	cmplwi	0,r5,4
82	blt	7f
83	stwu	r4,4(r6)
84	beqlr
85	andi.	r0,r6,3
86	add	r5,r0,r5
87	subf	r6,r0,r6
88	clrlwi	r7,r6,32-LG_CACHELINE_BYTES
89	add	r8,r7,r5
90	srwi	r9,r8,LG_CACHELINE_BYTES
91	addic.	r9,r9,-1	/* total number of complete cachelines */
92	ble	2f
93	xori	r0,r7,CACHELINE_MASK & ~3
94	srwi.	r0,r0,2
95	beq	3f
96	mtctr	r0
974:	stwu	r4,4(r6)
98	bdnz	4b
993:	mtctr	r9
100	li	r7,4
10110:	dcbz	r7,r6
102	addi	r6,r6,CACHELINE_BYTES
103	bdnz	10b
104	clrlwi	r5,r8,32-LG_CACHELINE_BYTES
105	addi	r5,r5,4
1062:	srwi	r0,r5,2
107	mtctr	r0
108	bdz	6f
1091:	stwu	r4,4(r6)
110	bdnz	1b
1116:	andi.	r5,r5,3
1127:	cmpwi	0,r5,0
113	beqlr
114	mtctr	r5
115	addi	r6,r6,3
1168:	stbu	r4,1(r6)
117	bdnz	8b
118	blr
119
120_GLOBAL(memset)
121	rlwimi	r4,r4,8,16,23
122	rlwimi	r4,r4,16,0,15
123	addi	r6,r3,-4
124	cmplwi	0,r5,4
125	blt	7f
126	stwu	r4,4(r6)
127	beqlr
128	andi.	r0,r6,3
129	add	r5,r0,r5
130	subf	r6,r0,r6
131	srwi	r0,r5,2
132	mtctr	r0
133	bdz	6f
1341:	stwu	r4,4(r6)
135	bdnz	1b
1366:	andi.	r5,r5,3
1377:	cmpwi	0,r5,0
138	beqlr
139	mtctr	r5
140	addi	r6,r6,3
1418:	stbu	r4,1(r6)
142	bdnz	8b
143	blr
144
145/*
146 * This version uses dcbz on the complete cache lines in the
147 * destination area to reduce memory traffic.  This requires that
148 * the destination area is cacheable.
149 * We only use this version if the source and dest don't overlap.
150 * -- paulus.
151 */
152_GLOBAL(cacheable_memcpy)
153	add	r7,r3,r5		/* test if the src & dst overlap */
154	add	r8,r4,r5
155	cmplw	0,r4,r7
156	cmplw	1,r3,r8
157	crand	0,0,4			/* cr0.lt &= cr1.lt */
158	blt	memcpy			/* if regions overlap */
159
160	addi	r4,r4,-4
161	addi	r6,r3,-4
162	neg	r0,r3
163	andi.	r0,r0,CACHELINE_MASK	/* # bytes to start of cache line */
164	beq	58f
165
166	cmplw	0,r5,r0			/* is this more than total to do? */
167	blt	63f			/* if not much to do */
168	andi.	r8,r0,3			/* get it word-aligned first */
169	subf	r5,r0,r5
170	mtctr	r8
171	beq+	61f
17270:	lbz	r9,4(r4)		/* do some bytes */
173	stb	r9,4(r6)
174	addi	r4,r4,1
175	addi	r6,r6,1
176	bdnz	70b
17761:	srwi.	r0,r0,2
178	mtctr	r0
179	beq	58f
18072:	lwzu	r9,4(r4)		/* do some words */
181	stwu	r9,4(r6)
182	bdnz	72b
183
18458:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
185	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
186	li	r11,4
187	mtctr	r0
188	beq	63f
18953:
190	dcbz	r11,r6
191	COPY_16_BYTES
192#if L1_CACHE_BYTES >= 32
193	COPY_16_BYTES
194#if L1_CACHE_BYTES >= 64
195	COPY_16_BYTES
196	COPY_16_BYTES
197#if L1_CACHE_BYTES >= 128
198	COPY_16_BYTES
199	COPY_16_BYTES
200	COPY_16_BYTES
201	COPY_16_BYTES
202#endif
203#endif
204#endif
205	bdnz	53b
206
20763:	srwi.	r0,r5,2
208	mtctr	r0
209	beq	64f
21030:	lwzu	r0,4(r4)
211	stwu	r0,4(r6)
212	bdnz	30b
213
21464:	andi.	r0,r5,3
215	mtctr	r0
216	beq+	65f
21740:	lbz	r0,4(r4)
218	stb	r0,4(r6)
219	addi	r4,r4,1
220	addi	r6,r6,1
221	bdnz	40b
22265:	blr
223
224_GLOBAL(memmove)
225	cmplw	0,r3,r4
226	bgt	backwards_memcpy
227	/* fall through */
228
229_GLOBAL(memcpy)
230	srwi.	r7,r5,3
231	addi	r6,r3,-4
232	addi	r4,r4,-4
233	beq	2f			/* if less than 8 bytes to do */
234	andi.	r0,r6,3			/* get dest word aligned */
235	mtctr	r7
236	bne	5f
2371:	lwz	r7,4(r4)
238	lwzu	r8,8(r4)
239	stw	r7,4(r6)
240	stwu	r8,8(r6)
241	bdnz	1b
242	andi.	r5,r5,7
2432:	cmplwi	0,r5,4
244	blt	3f
245	lwzu	r0,4(r4)
246	addi	r5,r5,-4
247	stwu	r0,4(r6)
2483:	cmpwi	0,r5,0
249	beqlr
250	mtctr	r5
251	addi	r4,r4,3
252	addi	r6,r6,3
2534:	lbzu	r0,1(r4)
254	stbu	r0,1(r6)
255	bdnz	4b
256	blr
2575:	subfic	r0,r0,4
258	mtctr	r0
2596:	lbz	r7,4(r4)
260	addi	r4,r4,1
261	stb	r7,4(r6)
262	addi	r6,r6,1
263	bdnz	6b
264	subf	r5,r0,r5
265	rlwinm.	r7,r5,32-3,3,31
266	beq	2b
267	mtctr	r7
268	b	1b
269
270_GLOBAL(backwards_memcpy)
271	rlwinm.	r7,r5,32-3,3,31		/* r0 = r5 >> 3 */
272	add	r6,r3,r5
273	add	r4,r4,r5
274	beq	2f
275	andi.	r0,r6,3
276	mtctr	r7
277	bne	5f
2781:	lwz	r7,-4(r4)
279	lwzu	r8,-8(r4)
280	stw	r7,-4(r6)
281	stwu	r8,-8(r6)
282	bdnz	1b
283	andi.	r5,r5,7
2842:	cmplwi	0,r5,4
285	blt	3f
286	lwzu	r0,-4(r4)
287	subi	r5,r5,4
288	stwu	r0,-4(r6)
2893:	cmpwi	0,r5,0
290	beqlr
291	mtctr	r5
2924:	lbzu	r0,-1(r4)
293	stbu	r0,-1(r6)
294	bdnz	4b
295	blr
2965:	mtctr	r0
2976:	lbzu	r7,-1(r4)
298	stbu	r7,-1(r6)
299	bdnz	6b
300	subf	r5,r0,r5
301	rlwinm.	r7,r5,32-3,3,31
302	beq	2b
303	mtctr	r7
304	b	1b
305
306_GLOBAL(__copy_tofrom_user)
307	addi	r4,r4,-4
308	addi	r6,r3,-4
309	neg	r0,r3
310	andi.	r0,r0,CACHELINE_MASK	/* # bytes to start of cache line */
311	beq	58f
312
313	cmplw	0,r5,r0			/* is this more than total to do? */
314	blt	63f			/* if not much to do */
315	andi.	r8,r0,3			/* get it word-aligned first */
316	mtctr	r8
317	beq+	61f
31870:	lbz	r9,4(r4)		/* do some bytes */
31971:	stb	r9,4(r6)
320	addi	r4,r4,1
321	addi	r6,r6,1
322	bdnz	70b
32361:	subf	r5,r0,r5
324	srwi.	r0,r0,2
325	mtctr	r0
326	beq	58f
32772:	lwzu	r9,4(r4)		/* do some words */
32873:	stwu	r9,4(r6)
329	bdnz	72b
330
331	.section __ex_table,"a"
332	.align	2
333	.long	70b,100f
334	.long	71b,101f
335	.long	72b,102f
336	.long	73b,103f
337	.text
338
33958:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
340	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
341	li	r11,4
342	beq	63f
343
344	/* Here we decide how far ahead to prefetch the source */
345	li	r3,4
346	cmpwi	r0,1
347	li	r7,0
348	ble	114f
349	li	r7,1
350#if MAX_COPY_PREFETCH > 1
351	/* Heuristically, for large transfers we prefetch
352	   MAX_COPY_PREFETCH cachelines ahead.  For small transfers
353	   we prefetch 1 cacheline ahead. */
354	cmpwi	r0,MAX_COPY_PREFETCH
355	ble	112f
356	li	r7,MAX_COPY_PREFETCH
357112:	mtctr	r7
358111:	dcbt	r3,r4
359	addi	r3,r3,CACHELINE_BYTES
360	bdnz	111b
361#else
362	dcbt	r3,r4
363	addi	r3,r3,CACHELINE_BYTES
364#endif /* MAX_COPY_PREFETCH > 1 */
365
366114:	subf	r8,r7,r0
367	mr	r0,r7
368	mtctr	r8
369
37053:	dcbt	r3,r4
37154:	dcbz	r11,r6
372	.section __ex_table,"a"
373	.align	2
374	.long	54b,105f
375	.text
376/* the main body of the cacheline loop */
377	COPY_16_BYTES_WITHEX(0)
378#if L1_CACHE_BYTES >= 32
379	COPY_16_BYTES_WITHEX(1)
380#if L1_CACHE_BYTES >= 64
381	COPY_16_BYTES_WITHEX(2)
382	COPY_16_BYTES_WITHEX(3)
383#if L1_CACHE_BYTES >= 128
384	COPY_16_BYTES_WITHEX(4)
385	COPY_16_BYTES_WITHEX(5)
386	COPY_16_BYTES_WITHEX(6)
387	COPY_16_BYTES_WITHEX(7)
388#endif
389#endif
390#endif
391	bdnz	53b
392	cmpwi	r0,0
393	li	r3,4
394	li	r7,0
395	bne	114b
396
39763:	srwi.	r0,r5,2
398	mtctr	r0
399	beq	64f
40030:	lwzu	r0,4(r4)
40131:	stwu	r0,4(r6)
402	bdnz	30b
403
40464:	andi.	r0,r5,3
405	mtctr	r0
406	beq+	65f
40740:	lbz	r0,4(r4)
40841:	stb	r0,4(r6)
409	addi	r4,r4,1
410	addi	r6,r6,1
411	bdnz	40b
41265:	li	r3,0
413	blr
414
415/* read fault, initial single-byte copy */
416100:	li	r9,0
417	b	90f
418/* write fault, initial single-byte copy */
419101:	li	r9,1
42090:	subf	r5,r8,r5
421	li	r3,0
422	b	99f
423/* read fault, initial word copy */
424102:	li	r9,0
425	b	91f
426/* write fault, initial word copy */
427103:	li	r9,1
42891:	li	r3,2
429	b	99f
430
431/*
432 * this stuff handles faults in the cacheline loop and branches to either
433 * 104f (if in read part) or 105f (if in write part), after updating r5
434 */
435	COPY_16_BYTES_EXCODE(0)
436#if L1_CACHE_BYTES >= 32
437	COPY_16_BYTES_EXCODE(1)
438#if L1_CACHE_BYTES >= 64
439	COPY_16_BYTES_EXCODE(2)
440	COPY_16_BYTES_EXCODE(3)
441#if L1_CACHE_BYTES >= 128
442	COPY_16_BYTES_EXCODE(4)
443	COPY_16_BYTES_EXCODE(5)
444	COPY_16_BYTES_EXCODE(6)
445	COPY_16_BYTES_EXCODE(7)
446#endif
447#endif
448#endif
449
450/* read fault in cacheline loop */
451104:	li	r9,0
452	b	92f
453/* fault on dcbz (effectively a write fault) */
454/* or write fault in cacheline loop */
455105:	li	r9,1
45692:	li	r3,LG_CACHELINE_BYTES
457	mfctr	r8
458	add	r0,r0,r8
459	b	106f
460/* read fault in final word loop */
461108:	li	r9,0
462	b	93f
463/* write fault in final word loop */
464109:	li	r9,1
46593:	andi.	r5,r5,3
466	li	r3,2
467	b	99f
468/* read fault in final byte loop */
469110:	li	r9,0
470	b	94f
471/* write fault in final byte loop */
472111:	li	r9,1
47394:	li	r5,0
474	li	r3,0
475/*
476 * At this stage the number of bytes not copied is
477 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
478 */
47999:	mfctr	r0
480106:	slw	r3,r0,r3
481	add.	r3,r3,r5
482	beq	120f			/* shouldn't happen */
483	cmpwi	0,r9,0
484	bne	120f
485/* for a read fault, first try to continue the copy one byte at a time */
486	mtctr	r3
487130:	lbz	r0,4(r4)
488131:	stb	r0,4(r6)
489	addi	r4,r4,1
490	addi	r6,r6,1
491	bdnz	130b
492/* then clear out the destination: r3 bytes starting at 4(r6) */
493132:	mfctr	r3
494	srwi.	r0,r3,2
495	li	r9,0
496	mtctr	r0
497	beq	113f
498112:	stwu	r9,4(r6)
499	bdnz	112b
500113:	andi.	r0,r3,3
501	mtctr	r0
502	beq	120f
503114:	stb	r9,4(r6)
504	addi	r6,r6,1
505	bdnz	114b
506120:	blr
507
508	.section __ex_table,"a"
509	.align	2
510	.long	30b,108b
511	.long	31b,109b
512	.long	40b,110b
513	.long	41b,111b
514	.long	130b,132b
515	.long	131b,120b
516	.long	112b,120b
517	.long	114b,120b
518	.text
519