1/* strchr/strchrnul optimized with 256-bit EVEX instructions.
2   Copyright (C) 2021-2022 Free Software Foundation, Inc.
3   This file is part of the GNU C Library.
4
5   The GNU C Library is free software; you can redistribute it and/or
6   modify it under the terms of the GNU Lesser General Public
7   License as published by the Free Software Foundation; either
8   version 2.1 of the License, or (at your option) any later version.
9
10   The GNU C Library is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13   Lesser General Public License for more details.
14
15   You should have received a copy of the GNU Lesser General Public
16   License along with the GNU C Library; if not, see
17   <https://www.gnu.org/licenses/>.  */
18
19#include <isa-level.h>
20
21#if ISA_SHOULD_BUILD (4)
22
23# include <sysdep.h>
24
25# ifndef STRCHR
26#  define STRCHR	__strchr_evex
27# endif
28
29# define VMOVU		vmovdqu64
30# define VMOVA		vmovdqa64
31
32# ifdef USE_AS_WCSCHR
33#  define VPBROADCAST	vpbroadcastd
34#  define VPCMP		vpcmpd
35#  define VPTESTN	vptestnmd
36#  define VPMINU	vpminud
37#  define CHAR_REG	esi
38#  define SHIFT_REG	ecx
39#  define CHAR_SIZE	4
40# else
41#  define VPBROADCAST	vpbroadcastb
42#  define VPCMP		vpcmpb
43#  define VPTESTN	vptestnmb
44#  define VPMINU	vpminub
45#  define CHAR_REG	sil
46#  define SHIFT_REG	edx
47#  define CHAR_SIZE	1
48# endif
49
50# define XMMZERO	xmm16
51
52# define YMMZERO	ymm16
53# define YMM0		ymm17
54# define YMM1		ymm18
55# define YMM2		ymm19
56# define YMM3		ymm20
57# define YMM4		ymm21
58# define YMM5		ymm22
59# define YMM6		ymm23
60# define YMM7		ymm24
61# define YMM8		ymm25
62
63# define VEC_SIZE 32
64# define PAGE_SIZE 4096
65# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
66
67	.section .text.evex,"ax",@progbits
68ENTRY_P2ALIGN (STRCHR, 5)
69	/* Broadcast CHAR to YMM0.	*/
70	VPBROADCAST	%esi, %YMM0
71	movl	%edi, %eax
72	andl	$(PAGE_SIZE - 1), %eax
73	/* Check if we cross page boundary with one vector load.
74	   Otherwise it is safe to use an unaligned load.  */
75	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
76	ja	L(cross_page_boundary)
77
78	/* Check the first VEC_SIZE bytes. Search for both CHAR and the
79	   null bytes.  */
80	VMOVU	(%rdi), %YMM1
81
82	/* Leaves only CHARS matching esi as 0.  */
83	vpxorq	%YMM1, %YMM0, %YMM2
84	VPMINU	%YMM2, %YMM1, %YMM2
85	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
86	VPTESTN	%YMM2, %YMM2, %k0
87	kmovd	%k0, %eax
88	testl	%eax, %eax
89	jz	L(aligned_more)
90	tzcntl	%eax, %eax
91# ifndef USE_AS_STRCHRNUL
92	/* Found CHAR or the null byte.  */
93	cmp	(%rdi, %rax, CHAR_SIZE), %CHAR_REG
94	/* NB: Use a branch instead of cmovcc here. The expectation is
95	   that with strchr the user will branch based on input being
96	   null. Since this branch will be 100% predictive of the user
97	   branch a branch miss here should save what otherwise would
98	   be branch miss in the user code. Otherwise using a branch 1)
99	   saves code size and 2) is faster in highly predictable
100	   environments.  */
101	jne	L(zero)
102# endif
103# ifdef USE_AS_WCSCHR
104	/* NB: Multiply wchar_t count by 4 to get the number of bytes.
105	 */
106	leaq	(%rdi, %rax, CHAR_SIZE), %rax
107# else
108	addq	%rdi, %rax
109# endif
110	ret
111
112
113
114	.p2align 4,, 10
115L(first_vec_x4):
116# ifndef USE_AS_STRCHRNUL
117	/* Check to see if first match was CHAR (k0) or null (k1).  */
118	kmovd	%k0, %eax
119	tzcntl	%eax, %eax
120	kmovd	%k1, %ecx
121	/* bzhil will not be 0 if first match was null.  */
122	bzhil	%eax, %ecx, %ecx
123	jne	L(zero)
124# else
125	/* Combine CHAR and null matches.  */
126	kord	%k0, %k1, %k0
127	kmovd	%k0, %eax
128	tzcntl	%eax, %eax
129# endif
130	/* NB: Multiply sizeof char type (1 or 4) to get the number of
131	   bytes.  */
132	leaq	(VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
133	ret
134
135# ifndef USE_AS_STRCHRNUL
136L(zero):
137	xorl	%eax, %eax
138	ret
139# endif
140
141
142	.p2align 4
143L(first_vec_x1):
144	/* Use bsf here to save 1-byte keeping keeping the block in 1x
145	   fetch block. eax guranteed non-zero.  */
146	bsfl	%eax, %eax
147# ifndef USE_AS_STRCHRNUL
148	/* Found CHAR or the null byte.	 */
149	cmp	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
150	jne	L(zero)
151
152# endif
153	/* NB: Multiply sizeof char type (1 or 4) to get the number of
154	   bytes.  */
155	leaq	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
156	ret
157
158	.p2align 4,, 10
159L(first_vec_x2):
160# ifndef USE_AS_STRCHRNUL
161	/* Check to see if first match was CHAR (k0) or null (k1).  */
162	kmovd	%k0, %eax
163	tzcntl	%eax, %eax
164	kmovd	%k1, %ecx
165	/* bzhil will not be 0 if first match was null.  */
166	bzhil	%eax, %ecx, %ecx
167	jne	L(zero)
168# else
169	/* Combine CHAR and null matches.  */
170	kord	%k0, %k1, %k0
171	kmovd	%k0, %eax
172	tzcntl	%eax, %eax
173# endif
174	/* NB: Multiply sizeof char type (1 or 4) to get the number of
175	   bytes.  */
176	leaq	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
177	ret
178
179	.p2align 4,, 10
180L(first_vec_x3):
181	/* Use bsf here to save 1-byte keeping keeping the block in 1x
182	   fetch block. eax guranteed non-zero.  */
183	bsfl	%eax, %eax
184# ifndef USE_AS_STRCHRNUL
185	/* Found CHAR or the null byte.	 */
186	cmp	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
187	jne	L(zero)
188# endif
189	/* NB: Multiply sizeof char type (1 or 4) to get the number of
190	   bytes.  */
191	leaq	(VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
192	ret
193
194	.p2align 4
195L(aligned_more):
196	/* Align data to VEC_SIZE.  */
197	andq	$-VEC_SIZE, %rdi
198L(cross_page_continue):
199	/* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time since
200	   data is only aligned to VEC_SIZE. Use two alternating methods
201	   for checking VEC to balance latency and port contention.  */
202
203	/* This method has higher latency but has better port
204	   distribution.  */
205	VMOVA	(VEC_SIZE)(%rdi), %YMM1
206	/* Leaves only CHARS matching esi as 0.  */
207	vpxorq	%YMM1, %YMM0, %YMM2
208	VPMINU	%YMM2, %YMM1, %YMM2
209	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
210	VPTESTN	%YMM2, %YMM2, %k0
211	kmovd	%k0, %eax
212	testl	%eax, %eax
213	jnz	L(first_vec_x1)
214
215	/* This method has higher latency but has better port
216	   distribution.  */
217	VMOVA	(VEC_SIZE * 2)(%rdi), %YMM1
218	/* Each bit in K0 represents a CHAR in YMM1.  */
219	VPCMP	$0, %YMM1, %YMM0, %k0
220	/* Each bit in K1 represents a CHAR in YMM1.  */
221	VPTESTN	%YMM1, %YMM1, %k1
222	kortestd	%k0, %k1
223	jnz	L(first_vec_x2)
224
225	VMOVA	(VEC_SIZE * 3)(%rdi), %YMM1
226	/* Leaves only CHARS matching esi as 0.  */
227	vpxorq	%YMM1, %YMM0, %YMM2
228	VPMINU	%YMM2, %YMM1, %YMM2
229	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
230	VPTESTN	%YMM2, %YMM2, %k0
231	kmovd	%k0, %eax
232	testl	%eax, %eax
233	jnz	L(first_vec_x3)
234
235	VMOVA	(VEC_SIZE * 4)(%rdi), %YMM1
236	/* Each bit in K0 represents a CHAR in YMM1.  */
237	VPCMP	$0, %YMM1, %YMM0, %k0
238	/* Each bit in K1 represents a CHAR in YMM1.  */
239	VPTESTN	%YMM1, %YMM1, %k1
240	kortestd	%k0, %k1
241	jnz	L(first_vec_x4)
242
243	/* Align data to VEC_SIZE * 4 for the loop.  */
244	addq	$VEC_SIZE, %rdi
245	andq	$-(VEC_SIZE * 4), %rdi
246
247	.p2align 4
248L(loop_4x_vec):
249	/* Check 4x VEC at a time. No penalty to imm32 offset with evex
250	   encoding.  */
251	VMOVA	(VEC_SIZE * 4)(%rdi), %YMM1
252	VMOVA	(VEC_SIZE * 5)(%rdi), %YMM2
253	VMOVA	(VEC_SIZE * 6)(%rdi), %YMM3
254	VMOVA	(VEC_SIZE * 7)(%rdi), %YMM4
255
256	/* For YMM1 and YMM3 use xor to set the CHARs matching esi to
257	   zero.  */
258	vpxorq	%YMM1, %YMM0, %YMM5
259	/* For YMM2 and YMM4 cmp not equals to CHAR and store result in
260	   k register. Its possible to save either 1 or 2 instructions
261	   using cmp no equals method for either YMM1 or YMM1 and YMM3
262	   respectively but bottleneck on p5 makes it not worth it.  */
263	VPCMP	$4, %YMM0, %YMM2, %k2
264	vpxorq	%YMM3, %YMM0, %YMM7
265	VPCMP	$4, %YMM0, %YMM4, %k4
266
267	/* Use min to select all zeros from either xor or end of string).
268	 */
269	VPMINU	%YMM1, %YMM5, %YMM1
270	VPMINU	%YMM3, %YMM7, %YMM3
271
272	/* Use min + zeromask to select for zeros. Since k2 and k4 will
273	   have 0 as positions that matched with CHAR which will set
274	   zero in the corresponding destination bytes in YMM2 / YMM4.
275	 */
276	VPMINU	%YMM1, %YMM2, %YMM2{%k2}{z}
277	VPMINU	%YMM3, %YMM4, %YMM4
278	VPMINU	%YMM2, %YMM4, %YMM4{%k4}{z}
279
280	VPTESTN	%YMM4, %YMM4, %k1
281	kmovd	%k1, %ecx
282	subq	$-(VEC_SIZE * 4), %rdi
283	testl	%ecx, %ecx
284	jz	L(loop_4x_vec)
285
286	VPTESTN	%YMM1, %YMM1, %k0
287	kmovd	%k0, %eax
288	testl	%eax, %eax
289	jnz	L(last_vec_x1)
290
291	VPTESTN	%YMM2, %YMM2, %k0
292	kmovd	%k0, %eax
293	testl	%eax, %eax
294	jnz	L(last_vec_x2)
295
296	VPTESTN	%YMM3, %YMM3, %k0
297	kmovd	%k0, %eax
298	/* Combine YMM3 matches (eax) with YMM4 matches (ecx).  */
299# ifdef USE_AS_WCSCHR
300	sall	$8, %ecx
301	orl	%ecx, %eax
302	bsfl	%eax, %eax
303# else
304	salq	$32, %rcx
305	orq	%rcx, %rax
306	bsfq	%rax, %rax
307# endif
308# ifndef USE_AS_STRCHRNUL
309	/* Check if match was CHAR or null.  */
310	cmp	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
311	jne	L(zero_end)
312# endif
313	/* NB: Multiply sizeof char type (1 or 4) to get the number of
314	   bytes.  */
315	leaq	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
316	ret
317
318	.p2align 4,, 8
319L(last_vec_x1):
320	bsfl	%eax, %eax
321# ifdef USE_AS_WCSCHR
322	/* NB: Multiply wchar_t count by 4 to get the number of bytes.
323	   */
324	leaq	(%rdi, %rax, CHAR_SIZE), %rax
325# else
326	addq	%rdi, %rax
327# endif
328
329# ifndef USE_AS_STRCHRNUL
330	/* Check if match was null.  */
331	cmp	(%rax), %CHAR_REG
332	jne	L(zero_end)
333# endif
334
335	ret
336
337	.p2align 4,, 8
338L(last_vec_x2):
339	bsfl	%eax, %eax
340# ifndef USE_AS_STRCHRNUL
341	/* Check if match was null.  */
342	cmp	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
343	jne	L(zero_end)
344# endif
345	/* NB: Multiply sizeof char type (1 or 4) to get the number of
346	   bytes.  */
347	leaq	(VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
348	ret
349
350	/* Cold case for crossing page with first load.	 */
351	.p2align 4,, 8
352L(cross_page_boundary):
353	movq	%rdi, %rdx
354	/* Align rdi.  */
355	andq	$-VEC_SIZE, %rdi
356	VMOVA	(%rdi), %YMM1
357	/* Leaves only CHARS matching esi as 0.  */
358	vpxorq	%YMM1, %YMM0, %YMM2
359	VPMINU	%YMM2, %YMM1, %YMM2
360	/* Each bit in K0 represents a CHAR or a null byte in YMM1.  */
361	VPTESTN	%YMM2, %YMM2, %k0
362	kmovd	%k0, %eax
363	/* Remove the leading bits.  */
364# ifdef USE_AS_WCSCHR
365	movl	%edx, %SHIFT_REG
366	/* NB: Divide shift count by 4 since each bit in K1 represent 4
367	   bytes.  */
368	sarl	$2, %SHIFT_REG
369	andl	$(CHAR_PER_VEC - 1), %SHIFT_REG
370# endif
371	sarxl	%SHIFT_REG, %eax, %eax
372	/* If eax is zero continue.  */
373	testl	%eax, %eax
374	jz	L(cross_page_continue)
375	bsfl	%eax, %eax
376
377# ifdef USE_AS_WCSCHR
378	/* NB: Multiply wchar_t count by 4 to get the number of
379	   bytes.  */
380	leaq	(%rdx, %rax, CHAR_SIZE), %rax
381# else
382	addq	%rdx, %rax
383# endif
384# ifndef USE_AS_STRCHRNUL
385	/* Check to see if match was CHAR or null.  */
386	cmp	(%rax), %CHAR_REG
387	je	L(cross_page_ret)
388L(zero_end):
389	xorl	%eax, %eax
390L(cross_page_ret):
391# endif
392	ret
393
394END (STRCHR)
395#endif
396