1/*
2 * This file contains miscellaneous low-level functions.
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 *
13 */
14
15#include <linux/config.h>
16#include <linux/sys.h>
17#include <asm/unistd.h>
18#include <asm/errno.h>
19#include <asm/processor.h>
20#include <asm/page.h>
21#include <asm/cache.h>
22#include <asm/cputable.h>
23#include <asm/mmu.h>
24#include <asm/ppc_asm.h>
25#include "ppc_defs.h"
26
27	.text
28
29	.align	5
30_GLOBAL(__delay)
31	cmpwi	0,r3,0
32	mtctr	r3
33	beqlr
341:	bdnz	1b
35	blr
36
37/*
38 * Returns (address we're running at) - (address we were linked at)
39 * for use before the text and data are mapped to KERNELBASE.
40 */
41_GLOBAL(reloc_offset)
42	mflr	r0
43	bl	1f
441:	mflr	r3
45	lis	r4,1b@ha
46	addi	r4,r4,1b@l
47	subf	r3,r4,r3
48	mtlr	r0
49	blr
50
51/*
52 * add_reloc_offset(x) returns x + reloc_offset().
53 */
54_GLOBAL(add_reloc_offset)
55	mflr	r0
56	bl	1f
571:	mflr	r5
58	lis	r4,1b@ha
59	addi	r4,r4,1b@l
60	subf	r5,r4,r5
61	add	r3,r3,r5
62	mtlr	r0
63	blr
64
65/*
66 * sub_reloc_offset(x) returns x - reloc_offset().
67 */
68_GLOBAL(sub_reloc_offset)
69	mflr	r0
70	bl	1f
711:	mflr	r5
72	lis	r4,1b@ha
73	addi	r4,r4,1b@l
74	subf	r5,r4,r5
75	subf	r3,r5,r3
76	mtlr	r0
77	blr
78
79/*
80 * reloc_got2 runs through the .got2 section adding an offset
81 * to each entry.
82 */
83_GLOBAL(reloc_got2)
84	mflr	r11
85	lis	r7,__got2_start@ha
86	addi	r7,r7,__got2_start@l
87	lis	r8,__got2_end@ha
88	addi	r8,r8,__got2_end@l
89	subf	r8,r7,r8
90	srwi.	r8,r8,2
91	beqlr
92	mtctr	r8
93	bl	1f
941:	mflr	r0
95	lis	r4,1b@ha
96	addi	r4,r4,1b@l
97	subf	r0,r4,r0
98	add	r7,r0,r7
992:	lwz	r0,0(r7)
100	add	r0,r0,r3
101	stw	r0,0(r7)
102	addi	r7,r7,4
103	bdnz	2b
104	mtlr	r11
105	blr
106
107/*
108 * identify_cpu,
109 * called with r3 = data offset and r4 = CPU number
110 * doesn't change r3
111 */
112_GLOBAL(identify_cpu)
113	addis	r8,r3,cpu_specs@ha
114	addi	r8,r8,cpu_specs@l
115	mfpvr	r7
1161:
117	lwz	r5,CPU_SPEC_PVR_MASK(r8)
118	and	r5,r5,r7
119	lwz	r6,CPU_SPEC_PVR_VALUE(r8)
120	cmplw	0,r6,r5
121	beq	1f
122	addi	r8,r8,CPU_SPEC_ENTRY_SIZE
123	b	1b
1241:
125	addis	r6,r3,cur_cpu_spec@ha
126	addi	r6,r6,cur_cpu_spec@l
127	slwi	r4,r4,2
128	sub	r8,r8,r3
129	stwx	r8,r4,r6
130	blr
131
132/*
133 * do_cpu_ftr_fixups - goes through the list of CPU feature fixups
134 * and writes nop's over sections of code that don't apply for this cpu.
135 * r3 = data offset (not changed)
136 */
137_GLOBAL(do_cpu_ftr_fixups)
138	/* Get CPU 0 features */
139	addis	r6,r3,cur_cpu_spec@ha
140	addi	r6,r6,cur_cpu_spec@l
141	lwz	r4,0(r6)
142	add	r4,r4,r3
143	lwz	r4,CPU_SPEC_FEATURES(r4)
144
145	/* Get the fixup table */
146	addis	r6,r3,__start___ftr_fixup@ha
147	addi	r6,r6,__start___ftr_fixup@l
148	addis	r7,r3,__stop___ftr_fixup@ha
149	addi	r7,r7,__stop___ftr_fixup@l
150
151	/* Do the fixup */
1521:	cmplw	0,r6,r7
153	bgelr
154	addi	r6,r6,16
155	lwz	r8,-16(r6)	/* mask */
156	and	r8,r8,r4
157	lwz	r9,-12(r6)	/* value */
158	cmplw	0,r8,r9
159	beq	1b
160	lwz	r8,-8(r6)	/* section begin */
161	lwz	r9,-4(r6)	/* section end */
162	subf.	r9,r8,r9
163	beq	1b
164	/* write nops over the section of code */
165	/* todo: if large section, add a branch at the start of it */
166	srwi	r9,r9,2
167	mtctr	r9
168	add	r8,r8,r3
169	lis	r0,0x60000000@h	/* nop */
1703:	stw	r0,0(r8)
171	andi.	r10,r4,CPU_FTR_SPLIT_ID_CACHE@l
172	beq	2f
173	dcbst	0,r8		/* suboptimal, but simpler */
174	sync
175	icbi	0,r8
1762:	addi	r8,r8,4
177	bdnz	3b
178	sync			/* additional sync needed on g4 */
179	isync
180	b	1b
181
182/*
183 * call_setup_cpu - call the setup_cpu function for this cpu
184 * r3 = data offset, r24 = cpu number
185 *
186 * Setup function is called with:
187 *   r3 = data offset
188 *   r4 = CPU number
189 *   r5 = ptr to CPU spec (relocated)
190 */
191_GLOBAL(call_setup_cpu)
192	addis	r5,r3,cur_cpu_spec@ha
193	addi	r5,r5,cur_cpu_spec@l
194	slwi	r4,r24,2
195	lwzx	r5,r4,r5
196	add	r5,r5,r3
197	lwz	r6,CPU_SPEC_SETUP(r5)
198	add	r6,r6,r3
199	mtctr	r6
200	mr	r4,r24
201	bctr
202
203/* void __save_flags_ptr(unsigned long *flags) */
204_GLOBAL(__save_flags_ptr)
205	mfmsr	r4
206	stw	r4,0(r3)
207	blr
208	/*
209	 * Need these nops here for taking over save/restore to
210	 * handle lost intrs
211	 * -- Cort
212	 */
213	nop
214	nop
215	nop
216	nop
217	nop
218	nop
219	nop
220	nop
221	nop
222	nop
223	nop
224	nop
225	nop
226	nop
227	nop
228	nop
229	nop
230_GLOBAL(__save_flags_ptr_end)
231
232/* void __restore_flags(unsigned long flags) */
233_GLOBAL(__restore_flags)
234/*
235 * Just set/clear the MSR_EE bit through restore/flags but do not
236 * change anything else.  This is needed by the RT system and makes
237 * sense anyway.
238 *    -- Cort
239 */
240	mfmsr 	r4
241	/* Copy all except the MSR_EE bit from r4 (current MSR value)
242	   to r3.  This is the sort of thing the rlwimi instruction is
243	   designed for.  -- paulus. */
244	rlwimi	r3,r4,0,17,15
245	 /* Check if things are setup the way we want _already_. */
246	cmpw	0,r3,r4
247	beqlr
2481:	SYNC
249	mtmsr	r3
250	SYNC
251	blr
252	nop
253	nop
254	nop
255	nop
256	nop
257	nop
258	nop
259	nop
260	nop
261	nop
262	nop
263	nop
264	nop
265	nop
266	nop
267	nop
268	nop
269	nop
270	nop
271_GLOBAL(__restore_flags_end)
272
273_GLOBAL(__cli)
274	mfmsr	r0		/* Get current interrupt state */
275	rlwinm	r3,r0,16+1,32-1,31	/* Extract old value of 'EE' */
276	rlwinm	r0,r0,0,17,15	/* clear MSR_EE in r0 */
277	SYNC			/* Some chip revs have problems here... */
278	mtmsr	r0		/* Update machine state */
279	blr			/* Done */
280	/*
281	 * Need these nops here for taking over save/restore to
282	 * handle lost intrs
283	 * -- Cort
284	 */
285	nop
286	nop
287	nop
288	nop
289	nop
290	nop
291	nop
292	nop
293	nop
294	nop
295	nop
296	nop
297	nop
298	nop
299	nop
300_GLOBAL(__cli_end)
301
302_GLOBAL(__sti)
303	mfmsr	r3		/* Get current state */
304	ori	r3,r3,MSR_EE	/* Turn on 'EE' bit */
305	SYNC			/* Some chip revs have problems here... */
306	mtmsr	r3		/* Update machine state */
307	blr
308	/*
309	 * Need these nops here for taking over save/restore to
310	 * handle lost intrs
311	 * -- Cort
312	 */
313	nop
314	nop
315	nop
316	nop
317	nop
318	nop
319	nop
320	nop
321	nop
322	nop
323	nop
324	nop
325	nop
326	nop
327	nop
328	nop
329_GLOBAL(__sti_end)
330
331
332/*
333 * complement mask on the msr then "or" some values on.
334 *     _nmask_and_or_msr(nmask, value_to_or)
335 */
336_GLOBAL(_nmask_and_or_msr)
337	mfmsr	r0		/* Get current msr */
338	andc	r0,r0,r3	/* And off the bits set in r3 (first parm) */
339	or	r0,r0,r4	/* Or on the bits in r4 (second parm) */
340	sync			/* Some chip revs have problems here... */
341	isync
342	mtmsr	r0		/* Update machine state */
343	isync
344	blr			/* Done */
345
346
347/*
348 * Flush MMU TLB
349 */
350_GLOBAL(_tlbia)
351#if defined(CONFIG_40x) && defined(CONFIG_PIN_TLB)
352	/* This needs to be coordinated with other pinning functions since
353	 * we don't keep a memory location of number of entries to reduce
354	 * cache pollution during these operations.
355	 */
356	lis	r3, 0
357	sync
3581:
359	tlbwe	r3, r3, TLB_TAG		/* just ensure V is clear */
360	addi	r3, r3, 1		/*   so r3 works fine for that */
361	cmpwi	0, r3, 61		/* reserve last two entries */
362	ble	1b
363	isync
364#elif defined(CONFIG_44x)
365	lis	r3,0
366	sync
3671:
368	tlbwe	r3,r3,PPC44x_TLB_PAGEID
369	addi	r3,r3,1
370	/* Load high watermark */
371	lis	r4,tlb_44x_hwater@h
372	ori	r4,r4,tlb_44x_hwater@l
373	lwz	r5,0(r4)
374	cmpw	0,r3,r5
375	ble	1b
376	isync
377#else /* !(CONFIG_40x || CONFIG_44x) */
378#if defined(CONFIG_SMP)
379	mfmsr	r10
380	SYNC
381	rlwinm	r0,r10,0,17,15		/* clear bit 16 (MSR_EE) */
382	mtmsr	r0
383	SYNC
384	lis	r9,hash_table_lock@h
385	ori	r9,r9,hash_table_lock@l
386	lwz	r8,PROCESSOR(r2)
387	oris	r8,r8,10
38810:	lwarx	r7,0,r9
389	cmpi	0,r7,0
390	bne-	10b
391	/* No 405 Erratum 77 fix needed here, because 4xx can't do SMP */
392	stwcx.	r8,0,r9
393	bne-	10b
394#endif /* CONFIG_SMP */
395	isync
396	tlbia
397	sync
398#ifdef CONFIG_SMP
399	TLBSYNC
400	li	r0,0
401	stw	r0,0(r9)		/* clear hash_table_lock */
402	mtmsr	r10
403	SYNC
404#endif /* CONFIG_SMP */
405#endif /* defined(CONFIG_40x) && defined(CONFIG_PIN_TLB) */
406	blr
407
408/*
409 * Flush MMU TLB for a particular address
410 */
411_GLOBAL(_tlbie)
412#ifdef CONFIG_40x
413	tlbsx.	r3, 0, r3
414	bne	10f
415	sync
416	/* There are only 64 TLB entries, so r3 < 64, which means bit 25, is clear.
417	 * Since 25 is the V bit in the TLB_TAG, loading this value will invalidate
418	 * the TLB entry. */
419	tlbwe	r3, r3, TLB_TAG
420	isync
42110:
422#elif defined(CONFIG_44x)
423	mfspr	r4,SPRN_MMUCR			/* Get MMUCR */
424	lis	r5,PPC44x_MMUCR_STS@h
425	ori	r5,r5,PPC44x_MMUCR_TID@l	/* Create mask */
426	andc	r4,r4,r5			/* Clear out TID/STS bits */
427	mfspr	r5,SPRN_PID			/* Get PID */
428	or	r4,r4,r5			/* Set TID bits */
429	mfmsr	r6				/* Get MSR */
430	andi.	r6,r6,MSR_IS@l			/* TS=1? */
431	beq	11f				/* If not, leave STS=0 */
432	oris	r4,r4,PPC44x_MMUCR_STS@h	/* Set STS=1 */
43311:	mtspr	SPRN_MMUCR, r4			/* Put MMUCR */
434
435	tlbsx.	r3, 0, r3
436	bne	10f
437	sync
438	/* There are only 64 TLB entries, so r3 < 64,
439	 * which means bit 22, is clear.  Since 22 is
440	 * the V bit in the TLB_PAGEID, loading this
441	 * value will invalidate the TLB entry.
442	 */
443	tlbwe	r3, r3, PPC44x_TLB_PAGEID
444	isync
44510:
446#else /* !(CONFIG_40x || CONFIG_44x) */
447#if defined(CONFIG_SMP)
448	mfmsr	r10
449	SYNC
450	rlwinm	r0,r10,0,17,15		/* clear bit 16 (MSR_EE) */
451	mtmsr	r0
452	SYNC
453	lis	r9,hash_table_lock@h
454	ori	r9,r9,hash_table_lock@l
455	lwz	r8,PROCESSOR(r2)
456	oris	r8,r8,11
45710:	lwarx	r7,0,r9
458	cmpi	0,r7,0
459	bne-	10b
460	PPC405_ERR77(0,r9)
461	stwcx.	r8,0,r9
462	bne-	10b
463#endif /* CONFIG_SMP */
464	isync
465	tlbie	r3
466	sync
467#ifdef CONFIG_SMP
468	TLBSYNC
469	li	r0,0
470	stw	r0,0(r9)		/* clear hash_table_lock */
471	mtmsr	r10
472	SYNC
473#endif
474#endif /* CONFIG_40x */
475	blr
476
477/*
478 * Flush instruction cache.
479 * This is a no-op on the 601.
480 */
481_GLOBAL(flush_instruction_cache)
482#if defined(CONFIG_8xx)
483	isync
484	lis	r5, IDC_INVALL@h
485	mtspr	IC_CST, r5
486#elif CONFIG_4xx
487#ifdef CONFIG_403GCX
488	li      r3, 512
489	mtctr   r3
490	lis     r4, KERNELBASE@h
4911:	iccci   0, r4
492	addi    r4, r4, 16
493	bdnz    1b
494#else
495	lis	r3, KERNELBASE@h
496	iccci	0,r3
497#endif
498#else
499	mfspr	r3,PVR
500	rlwinm	r3,r3,16,16,31
501	cmpi	0,r3,1
502	beqlr			/* for 601, do nothing */
503	/* 603/604 processor - use invalidate-all bit in HID0 */
504	mfspr	r3,HID0
505	ori	r3,r3,HID0_ICFI
506	mtspr	HID0,r3
507#endif /* CONFIG_8xx/4xx */
508	isync
509	blr
510
511/*
512 * Write any modified data cache blocks out to memory
513 * and invalidate the corresponding instruction cache blocks.
514 * This is a no-op on the 601.
515 *
516 * flush_icache_range(unsigned long start, unsigned long stop)
517 */
518_GLOBAL(flush_icache_range)
519	mfspr	r5,PVR
520	rlwinm	r5,r5,16,16,31
521	cmpi	0,r5,1
522	beqlr				/* for 601, do nothing */
523	li	r5,L1_CACHE_LINE_SIZE-1
524	andc	r3,r3,r5
525	subf	r4,r3,r4
526	add	r4,r4,r5
527	srwi.	r4,r4,LG_L1_CACHE_LINE_SIZE
528	beqlr
529	mtctr	r4
530	mr	r6,r3
5311:	dcbst	0,r3
532	addi	r3,r3,L1_CACHE_LINE_SIZE
533	bdnz	1b
534	sync				/* wait for dcbst's to get to ram */
535	mtctr	r4
5362:	icbi	0,r6
537	addi	r6,r6,L1_CACHE_LINE_SIZE
538	bdnz	2b
539	sync				/* additional sync needed on g4 */
540	isync
541	blr
542/*
543 * Write any modified data cache blocks out to memory.
544 * Does not invalidate the corresponding cache lines (especially for
545 * any corresponding instruction cache).
546 *
547 * clean_dcache_range(unsigned long start, unsigned long stop)
548 */
549_GLOBAL(clean_dcache_range)
550	li	r5,L1_CACHE_LINE_SIZE-1
551	andc	r3,r3,r5
552	subf	r4,r3,r4
553	add	r4,r4,r5
554	srwi.	r4,r4,LG_L1_CACHE_LINE_SIZE
555	beqlr
556	mtctr	r4
557
5581:	dcbst	0,r3
559	addi	r3,r3,L1_CACHE_LINE_SIZE
560	bdnz	1b
561	sync				/* wait for dcbst's to get to ram */
562	blr
563
564/*
565 * Write any modified data cache blocks out to memory and invalidate them.
566 * Does not invalidate the corresponding instruction cache blocks.
567 *
568 * flush_dcache_range(unsigned long start, unsigned long stop)
569 */
570_GLOBAL(flush_dcache_range)
571	li	r5,L1_CACHE_LINE_SIZE-1
572	andc	r3,r3,r5
573	subf	r4,r3,r4
574	add	r4,r4,r5
575	srwi.	r4,r4,LG_L1_CACHE_LINE_SIZE
576	beqlr
577	mtctr	r4
578
5791:	dcbf	0,r3
580	addi	r3,r3,L1_CACHE_LINE_SIZE
581	bdnz	1b
582	sync				/* wait for dcbst's to get to ram */
583	blr
584
585/*
586 * Like above, but invalidate the D-cache.  This is used by the 8xx
587 * to invalidate the cache so the PPC core doesn't get stale data
588 * from the CPM (no cache snooping here :-).
589 *
590 * invalidate_dcache_range(unsigned long start, unsigned long stop)
591 */
592_GLOBAL(invalidate_dcache_range)
593	li	r5,L1_CACHE_LINE_SIZE-1
594	andc	r3,r3,r5
595	subf	r4,r3,r4
596	add	r4,r4,r5
597	srwi.	r4,r4,LG_L1_CACHE_LINE_SIZE
598	beqlr
599	mtctr	r4
600
6011:	dcbi	0,r3
602	addi	r3,r3,L1_CACHE_LINE_SIZE
603	bdnz	1b
604	sync				/* wait for dcbi's to get to ram */
605	blr
606
607#ifdef CONFIG_NOT_COHERENT_CACHE
608/*
609 * 40x cores have 8K or 16K dcache and 32 byte line size.
610 * 44x has a 32K dcache and 32 byte line size.
611 * 8xx has 1, 2, 4, 8K variants.
612 * For now, cover the worst case of the 440.
613 * Must be called with external interrupts disabled.
614 */
615#define CACHE_NWAYS	64
616#define CACHE_NLINES	16
617
618_GLOBAL(flush_dcache_all)
619	li	r4, (2 * CACHE_NWAYS * CACHE_NLINES)
620	mtctr	r4
621	lis     r5, KERNELBASE@h
6221:	lwz	r3, 0(r5)		/* Load one word from every line */
623	addi	r5, r5, L1_CACHE_LINE_SIZE
624	bdnz    1b
625	blr
626#endif /* CONFIG_NOT_COHERENT_CACHE */
627
628/*
629 * Flush a particular page from the data cache to RAM.
630 * Note: this is necessary because the instruction cache does *not*
631 * snoop from the data cache.
632 * This is a no-op on the 601 which has a unified cache.
633 *
634 *	void __flush_dcache_icache(void *page)
635 */
636_GLOBAL(__flush_dcache_icache)
637	mfspr	r5,PVR
638	rlwinm	r5,r5,16,16,31
639	cmpi	0,r5,1
640	beqlr					/* for 601, do nothing */
641	rlwinm	r3,r3,0,0,19			/* Get page base address */
642	li	r4,4096/L1_CACHE_LINE_SIZE	/* Number of lines in a page */
643	mtctr	r4
644	mr	r6,r3
6450:	dcbst	0,r3				/* Write line to ram */
646	addi	r3,r3,L1_CACHE_LINE_SIZE
647	bdnz	0b
648	sync
649	mtctr	r4
6501:	icbi	0,r6
651	addi	r6,r6,L1_CACHE_LINE_SIZE
652	bdnz	1b
653	sync
654	isync
655	blr
656
657/*
658 * Clear a page using the dcbz instruction, which doesn't cause any
659 * memory traffic (except to write out any cache lines which get
660 * displaced).  This only works on cacheable memory.
661 */
662_GLOBAL(clear_page)
663	li	r0,4096/L1_CACHE_LINE_SIZE
664	mtctr	r0
665#ifdef CONFIG_8xx
666	li	r4, 0
6671:	stw	r4, 0(r3)
668	stw	r4, 4(r3)
669	stw	r4, 8(r3)
670	stw	r4, 12(r3)
671#else
6721:	dcbz	0,r3
673#endif
674	addi	r3,r3,L1_CACHE_LINE_SIZE
675	bdnz	1b
676	blr
677
678/*
679 * Copy a whole page.  We use the dcbz instruction on the destination
680 * to reduce memory traffic (it eliminates the unnecessary reads of
681 * the destination into cache).  This requires that the destination
682 * is cacheable.
683 */
684#define COPY_16_BYTES		\
685	lwz	r6,4(r4);	\
686	lwz	r7,8(r4);	\
687	lwz	r8,12(r4);	\
688	lwzu	r9,16(r4);	\
689	stw	r6,4(r3);	\
690	stw	r7,8(r3);	\
691	stw	r8,12(r3);	\
692	stwu	r9,16(r3)
693
694_GLOBAL(copy_page)
695	addi	r3,r3,-4
696	addi	r4,r4,-4
697
698#ifdef CONFIG_8xx
699	/* don't use prefetch on 8xx */
700    	li	r0,4096/L1_CACHE_LINE_SIZE
701	mtctr	r0
7021:	COPY_16_BYTES
703	bdnz	1b
704	blr
705
706#else	/* not 8xx, we can prefetch */
707	li	r5,4
708
709#if MAX_COPY_PREFETCH > 1
710	li	r0,MAX_COPY_PREFETCH
711	li	r11,4
712	mtctr	r0
71311:	dcbt	r11,r4
714	addi	r11,r11,L1_CACHE_LINE_SIZE
715	bdnz	11b
716#else /* MAX_COPY_PREFETCH == 1 */
717	dcbt	r5,r4
718	li	r11,L1_CACHE_LINE_SIZE+4
719#endif /* MAX_COPY_PREFETCH */
720	li	r0,4096/L1_CACHE_LINE_SIZE - MAX_COPY_PREFETCH
721	crclr	4*cr0+eq
7222:
723	mtctr	r0
7241:
725	dcbt	r11,r4
726	dcbz	r5,r3
727	COPY_16_BYTES
728#if L1_CACHE_LINE_SIZE >= 32
729	COPY_16_BYTES
730#if L1_CACHE_LINE_SIZE >= 64
731	COPY_16_BYTES
732	COPY_16_BYTES
733#if L1_CACHE_LINE_SIZE >= 128
734	COPY_16_BYTES
735	COPY_16_BYTES
736	COPY_16_BYTES
737	COPY_16_BYTES
738#endif
739#endif
740#endif
741	bdnz	1b
742	beqlr
743	crnot	4*cr0+eq,4*cr0+eq
744	li	r0,MAX_COPY_PREFETCH
745	li	r11,4
746	b	2b
747#endif	/* CONFIG_8xx */
748
749/*
750 * Atomic [test&set] exchange
751 *
752 *	unsigned long xchg_u32(void *ptr, unsigned long val)
753 * Changes the memory location '*ptr' to be val and returns
754 * the previous value stored there.
755 */
756_GLOBAL(xchg_u32)
757	mr	r5,r3		/* Save pointer */
75810:	lwarx	r3,0,r5		/* Fetch old value & reserve */
759	PPC405_ERR77(0,r5)
760	stwcx.	r4,0,r5		/* Update with new value */
761	bne-	10b		/* Retry if "reservation" (i.e. lock) lost */
762	blr
763
764/*
765 * void atomic_clear_mask(atomic_t mask, atomic_t *addr)
766 * void atomic_set_mask(atomic_t mask, atomic_t *addr);
767 */
768_GLOBAL(atomic_clear_mask)
76910:	lwarx	r5,0,r4
770	andc	r5,r5,r3
771	PPC405_ERR77(0,r4)
772	stwcx.	r5,0,r4
773	bne-	10b
774	blr
775_GLOBAL(atomic_set_mask)
77610:	lwarx	r5,0,r4
777	or	r5,r5,r3
778	PPC405_ERR77(0,r4)
779	stwcx.	r5,0,r4
780	bne-	10b
781	blr
782
783/*
784 * I/O string operations
785 *
786 * insb(port, buf, len)
787 * outsb(port, buf, len)
788 * insw(port, buf, len)
789 * outsw(port, buf, len)
790 * insl(port, buf, len)
791 * outsl(port, buf, len)
792 * insw_ns(port, buf, len)
793 * outsw_ns(port, buf, len)
794 * insl_ns(port, buf, len)
795 * outsl_ns(port, buf, len)
796 *
797 * The *_ns versions don't do byte-swapping.
798 */
799_GLOBAL(_insb)
800	cmpwi	0,r5,0
801	mtctr	r5
802	subi	r4,r4,1
803	blelr-
80400:	lbz	r5,0(r3)
805	eieio
806	stbu	r5,1(r4)
807	bdnz	00b
808	blr
809
810_GLOBAL(_outsb)
811	cmpwi	0,r5,0
812	mtctr	r5
813	subi	r4,r4,1
814	blelr-
81500:	lbzu	r5,1(r4)
816	stb	r5,0(r3)
817	eieio
818	bdnz	00b
819	blr
820
821_GLOBAL(_insw)
822	cmpwi	0,r5,0
823	mtctr	r5
824	subi	r4,r4,2
825	blelr-
82600:	lhbrx	r5,0,r3
827	eieio
828	sthu	r5,2(r4)
829	bdnz	00b
830	blr
831
832_GLOBAL(_outsw)
833	cmpwi	0,r5,0
834	mtctr	r5
835	subi	r4,r4,2
836	blelr-
83700:	lhzu	r5,2(r4)
838	eieio
839	sthbrx	r5,0,r3
840	bdnz	00b
841	blr
842
843_GLOBAL(_insl)
844	cmpwi	0,r5,0
845	mtctr	r5
846	subi	r4,r4,4
847	blelr-
84800:	lwbrx	r5,0,r3
849	eieio
850	stwu	r5,4(r4)
851	bdnz	00b
852	blr
853
854_GLOBAL(_outsl)
855	cmpwi	0,r5,0
856	mtctr	r5
857	subi	r4,r4,4
858	blelr-
85900:	lwzu	r5,4(r4)
860	stwbrx	r5,0,r3
861	eieio
862	bdnz	00b
863	blr
864
865_GLOBAL(ide_insw)
866_GLOBAL(_insw_ns)
867	cmpwi	0,r5,0
868	mtctr	r5
869	subi	r4,r4,2
870	blelr-
87100:	lhz	r5,0(r3)
872	eieio
873	sthu	r5,2(r4)
874	bdnz	00b
875	blr
876
877_GLOBAL(ide_outsw)
878_GLOBAL(_outsw_ns)
879	cmpwi	0,r5,0
880	mtctr	r5
881	subi	r4,r4,2
882	blelr-
88300:	lhzu	r5,2(r4)
884	sth	r5,0(r3)
885	eieio
886	bdnz	00b
887	blr
888
889_GLOBAL(_insl_ns)
890	cmpwi	0,r5,0
891	mtctr	r5
892	subi	r4,r4,4
893	blelr-
89400:	lwz	r5,0(r3)
895	eieio
896	stwu	r5,4(r4)
897	bdnz	00b
898	blr
899
900_GLOBAL(_outsl_ns)
901	cmpwi	0,r5,0
902	mtctr	r5
903	subi	r4,r4,4
904	blelr-
90500:	lwzu	r5,4(r4)
906	stw	r5,0(r3)
907	eieio
908	bdnz	00b
909	blr
910
911/*
912 * Extended precision shifts.
913 *
914 * Updated to be valid for shift counts from 0 to 63 inclusive.
915 * -- Gabriel
916 *
917 * R3/R4 has 64 bit value
918 * R5    has shift count
919 * result in R3/R4
920 *
921 *  ashrdi3: arithmetic right shift (sign propagation)
922 *  lshrdi3: logical right shift
923 *  ashldi3: left shift
924 */
925_GLOBAL(__ashrdi3)
926	subfic	r6,r5,32
927	srw	r4,r4,r5	# LSW = count > 31 ? 0 : LSW >> count
928	addi	r7,r5,32	# could be xori, or addi with -32
929	slw	r6,r3,r6	# t1 = count > 31 ? 0 : MSW << (32-count)
930	rlwinm	r8,r7,0,32	# t3 = (count < 32) ? 32 : 0
931	sraw	r7,r3,r7	# t2 = MSW >> (count-32)
932	or	r4,r4,r6	# LSW |= t1
933	slw	r7,r7,r8	# t2 = (count < 32) ? 0 : t2
934	sraw	r3,r3,r5	# MSW = MSW >> count
935	or	r4,r4,r7	# LSW |= t2
936	blr
937
938_GLOBAL(__ashldi3)
939	subfic	r6,r5,32
940	slw	r3,r3,r5	# MSW = count > 31 ? 0 : MSW << count
941	addi	r7,r5,32	# could be xori, or addi with -32
942	srw	r6,r4,r6	# t1 = count > 31 ? 0 : LSW >> (32-count)
943	slw	r7,r4,r7	# t2 = count < 32 ? 0 : LSW << (count-32)
944	or	r3,r3,r6	# MSW |= t1
945	slw	r4,r4,r5	# LSW = LSW << count
946	or	r3,r3,r7	# MSW |= t2
947	blr
948
949_GLOBAL(__lshrdi3)
950	subfic	r6,r5,32
951	srw	r4,r4,r5	# LSW = count > 31 ? 0 : LSW >> count
952	addi	r7,r5,32	# could be xori, or addi with -32
953	slw	r6,r3,r6	# t1 = count > 31 ? 0 : MSW << (32-count)
954	srw	r7,r3,r7	# t2 = count < 32 ? 0 : MSW >> (count-32)
955	or	r4,r4,r6	# LSW |= t1
956	srw	r3,r3,r5	# MSW = MSW >> count
957	or	r4,r4,r7	# LSW |= t2
958	blr
959
960_GLOBAL(abs)
961	srawi	r4,r3,31
962	xor	r3,r3,r4
963	sub	r3,r3,r4
964	blr
965
966_GLOBAL(_get_SP)
967	mr	r3,r1		/* Close enough */
968	blr
969
970/*
971 * These are used in the alignment trap handler when emulating
972 * single-precision loads and stores.
973 * We restore and save the fpscr so the task gets the same result
974 * and exceptions as if the cpu had performed the load or store.
975 */
976
977#ifdef CONFIG_4xx
978_GLOBAL(cvt_fd)
979	lfs	0,0(r3)
980	stfd	0,0(r4)
981	blr
982
983_GLOBAL(cvt_df)
984	lfd	0,0(r3)
985	stfs	0,0(r4)
986	blr
987#else
988_GLOBAL(cvt_fd)
989	lfd	0,-4(r5)	/* load up fpscr value */
990	mtfsf	0xff,0
991	lfs	0,0(r3)
992	stfd	0,0(r4)
993	mffs	0		/* save new fpscr value */
994	stfd	0,-4(r5)
995	blr
996
997_GLOBAL(cvt_df)
998	lfd	0,-4(r5)	/* load up fpscr value */
999	mtfsf	0xff,0
1000	lfd	0,0(r3)
1001	stfs	0,0(r4)
1002	mffs	0		/* save new fpscr value */
1003	stfd	0,-4(r5)
1004	blr
1005#endif
1006
1007/*
1008 * Create a kernel thread
1009 *   arch_kernel_thread(fn, arg, flags)
1010 */
1011_GLOBAL(arch_kernel_thread)
1012	mr	r6,r3		/* function */
1013	ori	r3,r5,CLONE_VM	/* flags */
1014	li	r0,__NR_clone
1015	sc
1016	cmpi	0,r3,0		/* parent or child? */
1017	bnelr			/* return if parent */
1018	li	r0,0		/* make top-level stack frame */
1019	stwu	r0,-16(r1)
1020	mtlr	r6              /* fn addr in lr */
1021	mr	r3,r4	        /* load arg and call fn */
1022	blrl
1023	li	r0,__NR_exit	/* exit after child exits */
1024	li	r3,0
1025	sc
1026
1027/*
1028 * This routine is just here to keep GCC happy - sigh...
1029 */
1030_GLOBAL(__main)
1031	blr
1032
1033#define SYSCALL(name) \
1034_GLOBAL(name) \
1035	li	r0,__NR_##name; \
1036	sc; \
1037	bnslr; \
1038	lis	r4,errno@ha; \
1039	stw	r3,errno@l(r4); \
1040	li	r3,-1; \
1041	blr
1042
1043#define __NR__exit __NR_exit
1044
1045SYSCALL(sync)
1046SYSCALL(setsid)
1047SYSCALL(write)
1048SYSCALL(dup)
1049SYSCALL(execve)
1050SYSCALL(open)
1051SYSCALL(close)
1052SYSCALL(waitpid)
1053SYSCALL(fork)
1054SYSCALL(delete_module)
1055SYSCALL(_exit)
1056SYSCALL(lseek)
1057SYSCALL(read)
1058
1059/* Why isn't this a) automatic, b) written in 'C'? */
1060	.data
1061	.align 4
1062_GLOBAL(sys_call_table)
1063	.long sys_ni_syscall	/* 0  -  old "setup()" system call */
1064	.long sys_exit
1065	.long sys_fork
1066	.long sys_read
1067	.long sys_write
1068	.long sys_open		/* 5 */
1069	.long sys_close
1070	.long sys_waitpid
1071	.long sys_creat
1072	.long sys_link
1073	.long sys_unlink	/* 10 */
1074	.long sys_execve
1075	.long sys_chdir
1076	.long sys_time
1077	.long sys_mknod
1078	.long sys_chmod		/* 15 */
1079	.long sys_lchown
1080	.long sys_ni_syscall			/* old break syscall holder */
1081	.long sys_stat
1082	.long sys_lseek
1083	.long sys_getpid	/* 20 */
1084	.long sys_mount
1085	.long sys_oldumount
1086	.long sys_setuid
1087	.long sys_getuid
1088	.long sys_stime		/* 25 */
1089	.long sys_ptrace
1090	.long sys_alarm
1091	.long sys_fstat
1092	.long sys_pause
1093	.long sys_utime		/* 30 */
1094	.long sys_ni_syscall			/* old stty syscall holder */
1095	.long sys_ni_syscall			/* old gtty syscall holder */
1096	.long sys_access
1097	.long sys_nice
1098	.long sys_ni_syscall	/* 35 */	/* old ftime syscall holder */
1099	.long sys_sync
1100	.long sys_kill
1101	.long sys_rename
1102	.long sys_mkdir
1103	.long sys_rmdir		/* 40 */
1104	.long sys_dup
1105	.long sys_pipe
1106	.long sys_times
1107	.long sys_ni_syscall			/* old prof syscall holder */
1108	.long sys_brk		/* 45 */
1109	.long sys_setgid
1110	.long sys_getgid
1111	.long sys_signal
1112	.long sys_geteuid
1113	.long sys_getegid	/* 50 */
1114	.long sys_acct
1115	.long sys_umount			/* recycled never used phys() */
1116	.long sys_ni_syscall			/* old lock syscall holder */
1117	.long sys_ioctl
1118	.long sys_fcntl		/* 55 */
1119	.long sys_ni_syscall			/* old mpx syscall holder */
1120	.long sys_setpgid
1121	.long sys_ni_syscall			/* old ulimit syscall holder */
1122	.long sys_olduname
1123	.long sys_umask		/* 60 */
1124	.long sys_chroot
1125	.long sys_ustat
1126	.long sys_dup2
1127	.long sys_getppid
1128	.long sys_getpgrp	/* 65 */
1129	.long sys_setsid
1130	.long sys_sigaction
1131	.long sys_sgetmask
1132	.long sys_ssetmask
1133	.long sys_setreuid	/* 70 */
1134	.long sys_setregid
1135	.long sys_sigsuspend
1136	.long sys_sigpending
1137	.long sys_sethostname
1138	.long sys_setrlimit	/* 75 */
1139	.long sys_old_getrlimit
1140	.long sys_getrusage
1141	.long sys_gettimeofday
1142	.long sys_settimeofday
1143	.long sys_getgroups	/* 80 */
1144	.long sys_setgroups
1145	.long ppc_select
1146	.long sys_symlink
1147	.long sys_lstat
1148	.long sys_readlink	/* 85 */
1149	.long sys_uselib
1150	.long sys_swapon
1151	.long sys_reboot
1152	.long old_readdir
1153	.long sys_mmap		/* 90 */
1154	.long sys_munmap
1155	.long sys_truncate
1156	.long sys_ftruncate
1157	.long sys_fchmod
1158	.long sys_fchown	/* 95 */
1159	.long sys_getpriority
1160	.long sys_setpriority
1161	.long sys_ni_syscall			/* old profil syscall holder */
1162	.long sys_statfs
1163	.long sys_fstatfs	/* 100 */
1164	.long sys_ioperm
1165	.long sys_socketcall
1166	.long sys_syslog
1167	.long sys_setitimer
1168	.long sys_getitimer	/* 105 */
1169	.long sys_newstat
1170	.long sys_newlstat
1171	.long sys_newfstat
1172	.long sys_uname
1173	.long sys_ni_syscall	/* 110 old iopl syscall */
1174	.long sys_vhangup
1175	.long sys_ni_syscall	/* old 'idle' syscall */
1176	.long sys_ni_syscall	/* old vm86 syscall */
1177	.long sys_wait4
1178	.long sys_swapoff	/* 115 */
1179	.long sys_sysinfo
1180	.long sys_ipc
1181	.long sys_fsync
1182	.long sys_sigreturn
1183	.long sys_clone		/* 120 */
1184	.long sys_setdomainname
1185	.long sys_newuname
1186	.long sys_ni_syscall	/* old modify_ldt syscall */
1187	.long sys_adjtimex
1188	.long sys_mprotect	/* 125 */
1189	.long sys_sigprocmask
1190	.long sys_create_module
1191	.long sys_init_module
1192	.long sys_delete_module
1193	.long sys_get_kernel_syms	/* 130 */
1194	.long sys_quotactl
1195	.long sys_getpgid
1196	.long sys_fchdir
1197	.long sys_bdflush
1198	.long sys_sysfs		/* 135 */
1199	.long sys_personality
1200	.long sys_ni_syscall	/* for afs_syscall */
1201	.long sys_setfsuid
1202	.long sys_setfsgid
1203	.long sys_llseek	/* 140 */
1204	.long sys_getdents
1205	.long ppc_select
1206	.long sys_flock
1207	.long sys_msync
1208	.long sys_readv		/* 145 */
1209	.long sys_writev
1210	.long sys_getsid
1211	.long sys_fdatasync
1212	.long sys_sysctl
1213	.long sys_mlock		/* 150 */
1214	.long sys_munlock
1215	.long sys_mlockall
1216	.long sys_munlockall
1217	.long sys_sched_setparam
1218	.long sys_sched_getparam	/* 155 */
1219	.long sys_sched_setscheduler
1220	.long sys_sched_getscheduler
1221	.long sys_sched_yield
1222	.long sys_sched_get_priority_max
1223	.long sys_sched_get_priority_min  /* 160 */
1224	.long sys_sched_rr_get_interval
1225	.long sys_nanosleep
1226	.long sys_mremap
1227	.long sys_setresuid
1228	.long sys_getresuid	/* 165 */
1229	.long sys_query_module
1230	.long sys_poll
1231	.long sys_nfsservctl
1232	.long sys_setresgid
1233	.long sys_getresgid	/* 170 */
1234	.long sys_prctl
1235	.long sys_rt_sigreturn
1236	.long sys_rt_sigaction
1237	.long sys_rt_sigprocmask
1238	.long sys_rt_sigpending	/* 175 */
1239	.long sys_rt_sigtimedwait
1240	.long sys_rt_sigqueueinfo
1241	.long sys_rt_sigsuspend
1242	.long sys_pread
1243	.long sys_pwrite	/* 180 */
1244	.long sys_chown
1245	.long sys_getcwd
1246	.long sys_capget
1247	.long sys_capset
1248	.long sys_sigaltstack	/* 185 */
1249	.long sys_sendfile
1250	.long sys_ni_syscall		/* streams1 */
1251	.long sys_ni_syscall		/* streams2 */
1252	.long sys_vfork
1253	.long sys_getrlimit	/* 190 */
1254	.long sys_readahead
1255	.long sys_mmap2
1256	.long sys_truncate64
1257	.long sys_ftruncate64
1258	.long sys_stat64	/* 195 */
1259	.long sys_lstat64
1260	.long sys_fstat64
1261	.long sys_pciconfig_read
1262	.long sys_pciconfig_write
1263	.long sys_pciconfig_iobase 	/* 200 */
1264	.long sys_ni_syscall		/* 201 - reserved - MacOnLinux - new */
1265	.long sys_getdents64
1266	.long sys_pivot_root
1267	.long sys_fcntl64
1268	.long sys_madvise	/* 205 */
1269	.long sys_mincore
1270	.long sys_gettid
1271	.long sys_tkill
1272	.long sys_setxattr
1273	.long sys_lsetxattr	/* 210 */
1274	.long sys_fsetxattr
1275	.long sys_getxattr
1276	.long sys_lgetxattr
1277	.long sys_fgetxattr
1278	.long sys_listxattr	/* 215 */
1279	.long sys_llistxattr
1280	.long sys_flistxattr
1281	.long sys_removexattr
1282	.long sys_lremovexattr
1283	.long sys_fremovexattr	/* 220  */
1284	.long sys_ni_syscall 	/*	reserved for sys_futex */
1285	.long sys_ni_syscall 	/*	reserved for sys_sched_setaffinity */
1286	.long sys_ni_syscall 	/*	reserved for sys_sched_getaffinity */
1287	.long sys_ni_syscall 	/*	reserved for sys_security */
1288	.long sys_ni_syscall 	/* 225	reserved for Tux */
1289	.long sys_ni_syscall 	/*	reserved for sys_sendfile64 */
1290	.long sys_ni_syscall 	/*	reserved for sys_io_setup */
1291	.long sys_ni_syscall 	/*	reserved for sys_io_destroy */
1292	.long sys_ni_syscall 	/*	reserved for sys_io_getevents */
1293	.long sys_ni_syscall 	/* 230	reserved for sys_io_submit */
1294	.long sys_ni_syscall 	/*	reserved for sys_io_cancel */
1295	.long sys_ni_syscall	/*	reserved for sys_set_tid_address */
1296	.long sys_ni_syscall	/*	reserved for sys_fadvise64 */
1297	.long sys_ni_syscall	/*	reserved for sys_exit_group */
1298	.long sys_ni_syscall	/* 235	reserved for sys_lookup_dcookie */
1299	.long sys_ni_syscall	/*	reserved for sys_epoll_create */
1300	.long sys_ni_syscall	/*	reserved for sys_epoll_ctl */
1301	.long sys_ni_syscall	/*	reserved for sys_epoll_wait */
1302	.long sys_ni_syscall	/*	reserved for sys_remap_file_pages */
1303	.long sys_ni_syscall	/* 240	reserved for sys_timer_create */
1304	.long sys_ni_syscall	/*	reserved for sys_timer_settime */
1305	.long sys_ni_syscall	/*	reserved for sys_timer_gettime */
1306	.long sys_ni_syscall	/*	reserved for sys_timer_getoverrun */
1307	.long sys_ni_syscall	/*	reserved for sys_timer_delete */
1308	.long sys_ni_syscall	/* 245	reserved for sys_clock_settime */
1309	.long sys_ni_syscall	/*	reserved for sys_clock_gettime */
1310	.long sys_ni_syscall	/*	reserved for sys_clock_getres */
1311	.long sys_ni_syscall	/*	reserved for sys_clock_nanosleep */
1312	.long sys_swapcontext
1313
1314	.rept NR_syscalls-(.-sys_call_table)/4
1315		.long sys_ni_syscall
1316	.endr
1317