1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * arch/alpha/lib/ev6-stxncpy.S
4 * 21264 version contributed by Rick Gorton <rick.gorton@api-networks.com>
5 *
6 * Copy no more than COUNT bytes of the null-terminated string from
7 * SRC to DST.
8 *
9 * This is an internal routine used by strncpy, stpncpy, and strncat.
10 * As such, it uses special linkage conventions to make implementation
11 * of these public functions more efficient.
12 *
13 * On input:
14 *	t9 = return address
15 *	a0 = DST
16 *	a1 = SRC
17 *	a2 = COUNT
18 *
19 * Furthermore, COUNT may not be zero.
20 *
21 * On output:
22 *	t0  = last word written
23 *	t10 = bitmask (with one bit set) indicating the byte position of
24 *	      the end of the range specified by COUNT
25 *	t12 = bitmask (with one bit set) indicating the last byte written
26 *	a0  = unaligned address of the last *word* written
27 *	a2  = the number of full words left in COUNT
28 *
29 * Furthermore, v0, a3-a5, t11, and $at are untouched.
30 *
31 * Much of the information about 21264 scheduling/coding comes from:
32 *	Compiler Writer's Guide for the Alpha 21264
33 *	abbreviated as 'CWG' in other comments here
34 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
35 * Scheduling notation:
36 *	E	- either cluster
37 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
38 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
39 * Try not to change the actual algorithm if possible for consistency.
40 */
41
42#include <asm/regdef.h>
43
44	.set noat
45	.set noreorder
46
47	.text
48
49/* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
50   doesn't like putting the entry point for a procedure somewhere in the
51   middle of the procedure descriptor.  Work around this by putting the
52   aligned copy in its own procedure descriptor */
53
54
55	.ent stxncpy_aligned
56	.align 4
57stxncpy_aligned:
58	.frame sp, 0, t9, 0
59	.prologue 0
60
61	/* On entry to this basic block:
62	   t0 == the first destination word for masking back in
63	   t1 == the first source word.  */
64
65	/* Create the 1st output word and detect 0's in the 1st input word.  */
66	lda	t2, -1		# E : build a mask against false zero
67	mskqh	t2, a1, t2	# U :   detection in the src word (stall)
68	mskqh	t1, a1, t3	# U :
69	ornot	t1, t2, t2	# E : (stall)
70
71	mskql	t0, a1, t0	# U : assemble the first output word
72	cmpbge	zero, t2, t8	# E : bits set iff null found
73	or	t0, t3, t0	# E : (stall)
74	beq	a2, $a_eoc	# U :
75
76	bne	t8, $a_eos	# U :
77	nop
78	nop
79	nop
80
81	/* On entry to this basic block:
82	   t0 == a source word not containing a null.  */
83
84	/*
85	 * nops here to:
86	 *	separate store quads from load quads
87	 *	limit of 1 bcond/quad to permit training
88	 */
89$a_loop:
90	stq_u	t0, 0(a0)	# L :
91	addq	a0, 8, a0	# E :
92	subq	a2, 1, a2	# E :
93	nop
94
95	ldq_u	t0, 0(a1)	# L :
96	addq	a1, 8, a1	# E :
97	cmpbge	zero, t0, t8	# E :
98	beq	a2, $a_eoc      # U :
99
100	beq	t8, $a_loop	# U :
101	nop
102	nop
103	nop
104
105	/* Take care of the final (partial) word store.  At this point
106	   the end-of-count bit is set in t8 iff it applies.
107
108	   On entry to this basic block we have:
109	   t0 == the source word containing the null
110	   t8 == the cmpbge mask that found it.  */
111
112$a_eos:
113	negq	t8, t12		# E : find low bit set
114	and	t8, t12, t12	# E : (stall)
115	/* For the sake of the cache, don't read a destination word
116	   if we're not going to need it.  */
117	and	t12, 0x80, t6	# E : (stall)
118	bne	t6, 1f		# U : (stall)
119
120	/* We're doing a partial word store and so need to combine
121	   our source and original destination words.  */
122	ldq_u	t1, 0(a0)	# L :
123	subq	t12, 1, t6	# E :
124	or	t12, t6, t8	# E : (stall)
125	zapnot	t0, t8, t0	# U : clear src bytes > null (stall)
126
127	zap	t1, t8, t1	# .. e1 : clear dst bytes <= null
128	or	t0, t1, t0	# e1    : (stall)
129	nop
130	nop
131
1321:	stq_u	t0, 0(a0)	# L :
133	ret	(t9)		# L0 : Latency=3
134	nop
135	nop
136
137	/* Add the end-of-count bit to the eos detection bitmask.  */
138$a_eoc:
139	or	t10, t8, t8	# E :
140	br	$a_eos		# L0 : Latency=3
141	nop
142	nop
143
144	.end stxncpy_aligned
145
146	.align 4
147	.ent __stxncpy
148	.globl __stxncpy
149__stxncpy:
150	.frame sp, 0, t9, 0
151	.prologue 0
152
153	/* Are source and destination co-aligned?  */
154	xor	a0, a1, t1	# E :
155	and	a0, 7, t0	# E : find dest misalignment
156	and	t1, 7, t1	# E : (stall)
157	addq	a2, t0, a2	# E : bias count by dest misalignment (stall)
158
159	subq	a2, 1, a2	# E :
160	and	a2, 7, t2	# E : (stall)
161	srl	a2, 3, a2	# U : a2 = loop counter = (count - 1)/8 (stall)
162	addq	zero, 1, t10	# E :
163
164	sll	t10, t2, t10	# U : t10 = bitmask of last count byte
165	bne	t1, $unaligned	# U :
166	/* We are co-aligned; take care of a partial first word.  */
167	ldq_u	t1, 0(a1)	# L : load first src word
168	addq	a1, 8, a1	# E :
169
170	beq	t0, stxncpy_aligned     # U : avoid loading dest word if not needed
171	ldq_u	t0, 0(a0)	# L :
172	nop
173	nop
174
175	br	stxncpy_aligned	# .. e1 :
176	nop
177	nop
178	nop
179
180
181
182/* The source and destination are not co-aligned.  Align the destination
183   and cope.  We have to be very careful about not reading too much and
184   causing a SEGV.  */
185
186	.align 4
187$u_head:
188	/* We know just enough now to be able to assemble the first
189	   full source word.  We can still find a zero at the end of it
190	   that prevents us from outputting the whole thing.
191
192	   On entry to this basic block:
193	   t0 == the first dest word, unmasked
194	   t1 == the shifted low bits of the first source word
195	   t6 == bytemask that is -1 in dest word bytes */
196
197	ldq_u	t2, 8(a1)	# L : Latency=3 load second src word
198	addq	a1, 8, a1	# E :
199	mskql	t0, a0, t0	# U : mask trailing garbage in dst
200	extqh	t2, a1, t4	# U : (3 cycle stall on t2)
201
202	or	t1, t4, t1	# E : first aligned src word complete (stall)
203	mskqh	t1, a0, t1	# U : mask leading garbage in src (stall)
204	or	t0, t1, t0	# E : first output word complete (stall)
205	or	t0, t6, t6	# E : mask original data for zero test (stall)
206
207	cmpbge	zero, t6, t8	# E :
208	beq	a2, $u_eocfin	# U :
209	lda	t6, -1		# E :
210	nop
211
212	bne	t8, $u_final	# U :
213	mskql	t6, a1, t6	# U : mask out bits already seen
214	stq_u	t0, 0(a0)	# L : store first output word
215	or      t6, t2, t2	# E : (stall)
216
217	cmpbge	zero, t2, t8	# E : find nulls in second partial
218	addq	a0, 8, a0	# E :
219	subq	a2, 1, a2	# E :
220	bne	t8, $u_late_head_exit	# U :
221
222	/* Finally, we've got all the stupid leading edge cases taken care
223	   of and we can set up to enter the main loop.  */
224	extql	t2, a1, t1	# U : position hi-bits of lo word
225	beq	a2, $u_eoc	# U :
226	ldq_u	t2, 8(a1)	# L : read next high-order source word
227	addq	a1, 8, a1	# E :
228
229	extqh	t2, a1, t0	# U : position lo-bits of hi word (stall)
230	cmpbge	zero, t2, t8	# E :
231	nop
232	bne	t8, $u_eos	# U :
233
234	/* Unaligned copy main loop.  In order to avoid reading too much,
235	   the loop is structured to detect zeros in aligned source words.
236	   This has, unfortunately, effectively pulled half of a loop
237	   iteration out into the head and half into the tail, but it does
238	   prevent nastiness from accumulating in the very thing we want
239	   to run as fast as possible.
240
241	   On entry to this basic block:
242	   t0 == the shifted low-order bits from the current source word
243	   t1 == the shifted high-order bits from the previous source word
244	   t2 == the unshifted current source word
245
246	   We further know that t2 does not contain a null terminator.  */
247
248	.align 4
249$u_loop:
250	or	t0, t1, t0	# E : current dst word now complete
251	subq	a2, 1, a2	# E : decrement word count
252	extql	t2, a1, t1	# U : extract low bits for next time
253	addq	a0, 8, a0	# E :
254
255	stq_u	t0, -8(a0)	# U : save the current word
256	beq	a2, $u_eoc	# U :
257	ldq_u	t2, 8(a1)	# U : Latency=3 load high word for next time
258	addq	a1, 8, a1	# E :
259
260	extqh	t2, a1, t0	# U : extract low bits (2 cycle stall)
261	cmpbge	zero, t2, t8	# E : test new word for eos
262	nop
263	beq	t8, $u_loop	# U :
264
265	/* We've found a zero somewhere in the source word we just read.
266	   If it resides in the lower half, we have one (probably partial)
267	   word to write out, and if it resides in the upper half, we
268	   have one full and one partial word left to write out.
269
270	   On entry to this basic block:
271	   t0 == the shifted low-order bits from the current source word
272	   t1 == the shifted high-order bits from the previous source word
273	   t2 == the unshifted current source word.  */
274$u_eos:
275	or	t0, t1, t0	# E : first (partial) source word complete
276	nop
277	cmpbge	zero, t0, t8	# E : is the null in this first bit? (stall)
278	bne	t8, $u_final	# U : (stall)
279
280	stq_u	t0, 0(a0)	# L : the null was in the high-order bits
281	addq	a0, 8, a0	# E :
282	subq	a2, 1, a2	# E :
283	nop
284
285$u_late_head_exit:
286	extql	t2, a1, t0	# U :
287	cmpbge	zero, t0, t8	# E :
288	or	t8, t10, t6	# E : (stall)
289	cmoveq	a2, t6, t8	# E : Latency=2, extra map slot (stall)
290
291	/* Take care of a final (probably partial) result word.
292	   On entry to this basic block:
293	   t0 == assembled source word
294	   t8 == cmpbge mask that found the null.  */
295$u_final:
296	negq	t8, t6		# E : isolate low bit set
297	and	t6, t8, t12	# E : (stall)
298	and	t12, 0x80, t6	# E : avoid dest word load if we can (stall)
299	bne	t6, 1f		# U : (stall)
300
301	ldq_u	t1, 0(a0)	# L :
302	subq	t12, 1, t6	# E :
303	or	t6, t12, t8	# E : (stall)
304	zapnot	t0, t8, t0	# U : kill source bytes > null
305
306	zap	t1, t8, t1	# U : kill dest bytes <= null
307	or	t0, t1, t0	# E : (stall)
308	nop
309	nop
310
3111:	stq_u	t0, 0(a0)	# L :
312	ret	(t9)		# L0 : Latency=3
313
314	  /* Got to end-of-count before end of string.
315	     On entry to this basic block:
316	     t1 == the shifted high-order bits from the previous source word  */
317$u_eoc:
318	and	a1, 7, t6	# E : avoid final load if possible
319	sll	t10, t6, t6	# U : (stall)
320	and	t6, 0xff, t6	# E : (stall)
321	bne	t6, 1f		# U : (stall)
322
323	ldq_u	t2, 8(a1)	# L : load final src word
324	nop
325	extqh	t2, a1, t0	# U : extract low bits for last word (stall)
326	or	t1, t0, t1	# E : (stall)
327
3281:	cmpbge	zero, t1, t8	# E :
329	mov	t1, t0		# E :
330
331$u_eocfin:			# end-of-count, final word
332	or	t10, t8, t8	# E :
333	br	$u_final	# L0 : Latency=3
334
335	/* Unaligned copy entry point.  */
336	.align 4
337$unaligned:
338
339	ldq_u	t1, 0(a1)	# L : load first source word
340	and	a0, 7, t4	# E : find dest misalignment
341	and	a1, 7, t5	# E : find src misalignment
342	/* Conditionally load the first destination word and a bytemask
343	   with 0xff indicating that the destination byte is sacrosanct.  */
344	mov	zero, t0	# E :
345
346	mov	zero, t6	# E :
347	beq	t4, 1f		# U :
348	ldq_u	t0, 0(a0)	# L :
349	lda	t6, -1		# E :
350
351	mskql	t6, a0, t6	# U :
352	nop
353	nop
354	subq	a1, t4, a1	# E : sub dest misalignment from src addr
355
356	/* If source misalignment is larger than dest misalignment, we need
357	   extra startup checks to avoid SEGV.  */
358
3591:	cmplt	t4, t5, t12	# E :
360	extql	t1, a1, t1	# U : shift src into place
361	lda	t2, -1		# E : for creating masks later
362	beq	t12, $u_head	# U : (stall)
363
364	extql	t2, a1, t2	# U :
365	cmpbge	zero, t1, t8	# E : is there a zero?
366	andnot	t2, t6, t2	# E : dest mask for a single word copy
367	or	t8, t10, t5	# E : test for end-of-count too
368
369	cmpbge	zero, t2, t3	# E :
370	cmoveq	a2, t5, t8	# E : Latency=2, extra map slot
371	nop			# E : keep with cmoveq
372	andnot	t8, t3, t8	# E : (stall)
373
374	beq	t8, $u_head	# U :
375	/* At this point we've found a zero in the first partial word of
376	   the source.  We need to isolate the valid source data and mask
377	   it into the original destination data.  (Incidentally, we know
378	   that we'll need at least one byte of that original dest word.) */
379	ldq_u	t0, 0(a0)	# L :
380	negq	t8, t6		# E : build bitmask of bytes <= zero
381	mskqh	t1, t4, t1	# U :
382
383	and	t6, t8, t12	# E :
384	subq	t12, 1, t6	# E : (stall)
385	or	t6, t12, t8	# E : (stall)
386	zapnot	t2, t8, t2	# U : prepare source word; mirror changes (stall)
387
388	zapnot	t1, t8, t1	# U : to source validity mask
389	andnot	t0, t2, t0	# E : zero place for source to reside
390	or	t0, t1, t0	# E : and put it there (stall both t0, t1)
391	stq_u	t0, 0(a0)	# L : (stall)
392
393	ret	(t9)		# L0 : Latency=3
394	nop
395	nop
396	nop
397
398	.end __stxncpy
399