1/* Function acos vectorized with AVX2.
2   Copyright (C) 2021-2022 Free Software Foundation, Inc.
3   This file is part of the GNU C Library.
4
5   The GNU C Library is free software; you can redistribute it and/or
6   modify it under the terms of the GNU Lesser General Public
7   License as published by the Free Software Foundation; either
8   version 2.1 of the License, or (at your option) any later version.
9
10   The GNU C Library is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13   Lesser General Public License for more details.
14
15   You should have received a copy of the GNU Lesser General Public
16   License along with the GNU C Library; if not, see
17   https://www.gnu.org/licenses/.  */
18
19/*
20 * ALGORITHM DESCRIPTION:
21 *
22 *      SelMask = (|x| >= 0.5) ? 1 : 0;
23 *      R = SelMask ? sqrt(0.5 - 0.5*|x|) : |x|
24 *      acos(|x|) = SelMask ? 2*Poly(R) : (Pi/2 - Poly(R))
25 *      acos(x) = sign(x) ? (Pi - acos(|x|)) : acos(|x|)
26 *
27 */
28
29/* Offsets for data table __svml_dacos_data_internal
30 */
31#define SgnBit				0
32#define OneHalf				32
33#define SmallNorm			64
34#define MOne				96
35#define Two				128
36#define sqrt_coeff			160
37#define poly_coeff			288
38#define PiH				672
39#define Pi2H				704
40
41#include <sysdep.h>
42
43	.section .text.avx2, "ax", @progbits
44ENTRY(_ZGVdN4v_acos_avx2)
45	pushq	%rbp
46	cfi_def_cfa_offset(16)
47	movq	%rsp, %rbp
48	cfi_def_cfa(6, 16)
49	cfi_offset(6, -16)
50	andq	$-32, %rsp
51	subq	$96, %rsp
52	vmovupd	__svml_dacos_data_internal(%rip), %ymm6
53	vmovupd	OneHalf+__svml_dacos_data_internal(%rip), %ymm7
54	vmovapd	%ymm0, %ymm5
55
56	/* x = -|arg| */
57	vorpd	%ymm5, %ymm6, %ymm4
58
59	/* Y = 0.5 + 0.5*(-x) */
60	vfmadd231pd %ymm4, %ymm7, %ymm7
61
62	/* x^2 */
63	vmulpd	%ymm4, %ymm4, %ymm8
64
65	/* S ~ 2*sqrt(Y) */
66	vmovupd	sqrt_coeff+__svml_dacos_data_internal(%rip), %ymm0
67	vcmplt_oqpd SmallNorm+__svml_dacos_data_internal(%rip), %ymm7, %ymm12
68	vminpd	%ymm7, %ymm8, %ymm2
69
70	/* NaN processed in special branch (so wind test passed) */
71	vcmpnge_uqpd MOne+__svml_dacos_data_internal(%rip), %ymm4, %ymm9
72	vcvtpd2ps %ymm7, %xmm10
73	vmovupd	poly_coeff+64+__svml_dacos_data_internal(%rip), %ymm8
74	vcmpnlt_uqpd %ymm7, %ymm2, %ymm1
75	vrsqrtps %xmm10, %xmm11
76	vfmadd213pd poly_coeff+96+__svml_dacos_data_internal(%rip), %ymm2, %ymm8
77	vcvtps2pd %xmm11, %ymm13
78	vmovupd	poly_coeff+128+__svml_dacos_data_internal(%rip), %ymm11
79	vandnpd	%ymm13, %ymm12, %ymm14
80	vmulpd	%ymm14, %ymm14, %ymm15
81	vfmadd213pd poly_coeff+160+__svml_dacos_data_internal(%rip), %ymm2, %ymm11
82	vmulpd	%ymm2, %ymm2, %ymm13
83	vmovupd	poly_coeff+256+__svml_dacos_data_internal(%rip), %ymm12
84	vmulpd	%ymm13, %ymm13, %ymm10
85	vfmadd213pd poly_coeff+288+__svml_dacos_data_internal(%rip), %ymm2, %ymm12
86	vandpd	%ymm5, %ymm6, %ymm3
87	vaddpd	%ymm7, %ymm7, %ymm6
88	vmulpd	%ymm6, %ymm14, %ymm7
89	vfmsub213pd Two+__svml_dacos_data_internal(%rip), %ymm15, %ymm6
90	vmovupd	poly_coeff+320+__svml_dacos_data_internal(%rip), %ymm14
91	vfmadd213pd sqrt_coeff+32+__svml_dacos_data_internal(%rip), %ymm6, %ymm0
92	vmulpd	%ymm6, %ymm7, %ymm15
93	vfmadd213pd poly_coeff+352+__svml_dacos_data_internal(%rip), %ymm2, %ymm14
94	vfmadd213pd sqrt_coeff+64+__svml_dacos_data_internal(%rip), %ymm6, %ymm0
95	vfmadd213pd sqrt_coeff+96+__svml_dacos_data_internal(%rip), %ymm6, %ymm0
96
97	/* polynomial */
98	vmovupd	poly_coeff+__svml_dacos_data_internal(%rip), %ymm6
99	vfnmadd213pd %ymm7, %ymm15, %ymm0
100	vfmadd213pd poly_coeff+32+__svml_dacos_data_internal(%rip), %ymm2, %ymm6
101	vblendvpd %ymm1, %ymm0, %ymm4, %ymm0
102	vfmadd213pd %ymm8, %ymm13, %ymm6
103	vmovmskpd %ymm9, %edx
104	vmovupd	poly_coeff+192+__svml_dacos_data_internal(%rip), %ymm9
105	vfmadd213pd poly_coeff+224+__svml_dacos_data_internal(%rip), %ymm2, %ymm9
106	vfmadd213pd %ymm9, %ymm13, %ymm11
107	vfmadd213pd %ymm11, %ymm10, %ymm6
108	vfmadd213pd %ymm12, %ymm13, %ymm6
109	vfmadd213pd %ymm14, %ymm13, %ymm6
110	vmulpd	%ymm6, %ymm2, %ymm9
111
112	/* X<X^2 iff X<0 */
113	vcmplt_oqpd %ymm2, %ymm5, %ymm6
114	vandpd	PiH+__svml_dacos_data_internal(%rip), %ymm1, %ymm2
115	vandnpd	Pi2H+__svml_dacos_data_internal(%rip), %ymm1, %ymm7
116	vxorpd	%ymm3, %ymm0, %ymm1
117	vfmadd213pd %ymm1, %ymm1, %ymm9
118	vandpd	%ymm6, %ymm2, %ymm2
119	vaddpd	%ymm7, %ymm2, %ymm8
120	vaddpd	%ymm9, %ymm8, %ymm0
121	testl	%edx, %edx
122
123	/* Go to special inputs processing branch */
124	jne	L(SPECIAL_VALUES_BRANCH)
125	# LOE rbx r12 r13 r14 r15 edx ymm0 ymm5
126
127	/* Restore registers
128	 * and exit the function
129	 */
130
131L(EXIT):
132	movq	%rbp, %rsp
133	popq	%rbp
134	cfi_def_cfa(7, 8)
135	cfi_restore(6)
136	ret
137	cfi_def_cfa(6, 16)
138	cfi_offset(6, -16)
139
140	/* Branch to process
141	 * special inputs
142	 */
143
144L(SPECIAL_VALUES_BRANCH):
145	vmovupd	%ymm5, 32(%rsp)
146	vmovupd	%ymm0, 64(%rsp)
147	# LOE rbx r12 r13 r14 r15 edx ymm0
148
149	xorl	%eax, %eax
150	# LOE rbx r12 r13 r14 r15 eax edx
151
152	vzeroupper
153	movq	%r12, 16(%rsp)
154	/*  DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -80; DW_OP_plus)  */
155	.cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xb0, 0xff, 0xff, 0xff, 0x22
156	movl	%eax, %r12d
157	movq	%r13, 8(%rsp)
158	/*  DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -88; DW_OP_plus)  */
159	.cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa8, 0xff, 0xff, 0xff, 0x22
160	movl	%edx, %r13d
161	movq	%r14, (%rsp)
162	/*  DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -96; DW_OP_plus)  */
163	.cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa0, 0xff, 0xff, 0xff, 0x22
164	# LOE rbx r15 r12d r13d
165
166	/* Range mask
167	 * bits check
168	 */
169
170L(RANGEMASK_CHECK):
171	btl	%r12d, %r13d
172
173	/* Call scalar math function */
174	jc	L(SCALAR_MATH_CALL)
175	# LOE rbx r15 r12d r13d
176
177	/* Special inputs
178	 * processing loop
179	 */
180
181L(SPECIAL_VALUES_LOOP):
182	incl	%r12d
183	cmpl	$4, %r12d
184
185	/* Check bits in range mask */
186	jl	L(RANGEMASK_CHECK)
187	# LOE rbx r15 r12d r13d
188
189	movq	16(%rsp), %r12
190	cfi_restore(12)
191	movq	8(%rsp), %r13
192	cfi_restore(13)
193	movq	(%rsp), %r14
194	cfi_restore(14)
195	vmovupd	64(%rsp), %ymm0
196
197	/* Go to exit */
198	jmp	L(EXIT)
199	/*  DW_CFA_expression: r12 (r12) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -80; DW_OP_plus)  */
200	.cfi_escape 0x10, 0x0c, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xb0, 0xff, 0xff, 0xff, 0x22
201	/*  DW_CFA_expression: r13 (r13) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -88; DW_OP_plus)  */
202	.cfi_escape 0x10, 0x0d, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa8, 0xff, 0xff, 0xff, 0x22
203	/*  DW_CFA_expression: r14 (r14) (DW_OP_lit8; DW_OP_minus; DW_OP_const4s: -32; DW_OP_and; DW_OP_const4s: -96; DW_OP_plus)  */
204	.cfi_escape 0x10, 0x0e, 0x0e, 0x38, 0x1c, 0x0d, 0xe0, 0xff, 0xff, 0xff, 0x1a, 0x0d, 0xa0, 0xff, 0xff, 0xff, 0x22
205	# LOE rbx r12 r13 r14 r15 ymm0
206
207	/* Scalar math fucntion call
208	 * to process special input
209	 */
210
211L(SCALAR_MATH_CALL):
212	movl	%r12d, %r14d
213	vmovsd	32(%rsp, %r14, 8), %xmm0
214	call	acos@PLT
215	# LOE rbx r14 r15 r12d r13d xmm0
216
217	vmovsd	%xmm0, 64(%rsp, %r14, 8)
218
219	/* Process special inputs in loop */
220	jmp	L(SPECIAL_VALUES_LOOP)
221	# LOE rbx r15 r12d r13d
222END(_ZGVdN4v_acos_avx2)
223
224	.section .rodata, "a"
225	.align	32
226
227#ifdef __svml_dacos_data_internal_typedef
228typedef unsigned int VUINT32;
229typedef struct {
230	__declspec(align(32)) VUINT32 SgnBit[4][2];
231	__declspec(align(32)) VUINT32 OneHalf[4][2];
232	__declspec(align(32)) VUINT32 SmallNorm[4][2];
233	__declspec(align(32)) VUINT32 MOne[4][2];
234	__declspec(align(32)) VUINT32 Two[4][2];
235	__declspec(align(32)) VUINT32 sqrt_coeff[4][4][2];
236	__declspec(align(32)) VUINT32 poly_coeff[12][4][2];
237	__declspec(align(32)) VUINT32 PiH[4][2];
238	__declspec(align(32)) VUINT32 Pi2H[4][2];
239} __svml_dacos_data_internal;
240#endif
241__svml_dacos_data_internal:
242	/* SgnBit */
243	.quad	0x8000000000000000, 0x8000000000000000, 0x8000000000000000, 0x8000000000000000
244	/* OneHalf */
245	.align	32
246	.quad	0x3fe0000000000000, 0x3fe0000000000000, 0x3fe0000000000000, 0x3fe0000000000000
247	/* SmallNorm */
248	.align	32
249	.quad	0x3000000000000000, 0x3000000000000000, 0x3000000000000000, 0x3000000000000000
250	/* MOne */
251	.align	32
252	.quad	0xbff0000000000000, 0xbff0000000000000, 0xbff0000000000000, 0xbff0000000000000
253	/* Two */
254	.align	32
255	.quad	0x4000000000000000, 0x4000000000000000, 0x4000000000000000, 0x4000000000000000
256	/* sqrt_coeff[4] */
257	.align	32
258	.quad	0xbf918000993B24C3, 0xbf918000993B24C3, 0xbf918000993B24C3, 0xbf918000993B24C3 /* sqrt_coeff4 */
259	.quad	0x3fa400006F70D42D, 0x3fa400006F70D42D, 0x3fa400006F70D42D, 0x3fa400006F70D42D /* sqrt_coeff3 */
260	.quad	0xbfb7FFFFFFFFFE97, 0xbfb7FFFFFFFFFE97, 0xbfb7FFFFFFFFFE97, 0xbfb7FFFFFFFFFE97 /* sqrt_coeff2 */
261	.quad	0x3fcFFFFFFFFFFF9D, 0x3fcFFFFFFFFFFF9D, 0x3fcFFFFFFFFFFF9D, 0x3fcFFFFFFFFFFF9D /* sqrt_coeff1 */
262	/* poly_coeff[12] */
263	.align	32
264	.quad	0x3fa07520C70EB909, 0x3fa07520C70EB909, 0x3fa07520C70EB909, 0x3fa07520C70EB909 /* poly_coeff12 */
265	.quad	0xbf90FB17F7DBB0ED, 0xbf90FB17F7DBB0ED, 0xbf90FB17F7DBB0ED, 0xbf90FB17F7DBB0ED /* poly_coeff11 */
266	.quad	0x3f943F44BFBC3BAE, 0x3f943F44BFBC3BAE, 0x3f943F44BFBC3BAE, 0x3f943F44BFBC3BAE /* poly_coeff10 */
267	.quad	0x3f7A583395D45ED5, 0x3f7A583395D45ED5, 0x3f7A583395D45ED5, 0x3f7A583395D45ED5 /* poly_coeff9 */
268	.quad	0x3f88F8DC2AFCCAD6, 0x3f88F8DC2AFCCAD6, 0x3f88F8DC2AFCCAD6, 0x3f88F8DC2AFCCAD6 /* poly_coeff8 */
269	.quad	0x3f8C6DBBCB88BD57, 0x3f8C6DBBCB88BD57, 0x3f8C6DBBCB88BD57, 0x3f8C6DBBCB88BD57 /* poly_coeff7 */
270	.quad	0x3f91C6DCF538AD2E, 0x3f91C6DCF538AD2E, 0x3f91C6DCF538AD2E, 0x3f91C6DCF538AD2E /* poly_coeff6 */
271	.quad	0x3f96E89CEBDEFadd, 0x3f96E89CEBDEFadd, 0x3f96E89CEBDEFadd, 0x3f96E89CEBDEFadd /* poly_coeff5 */
272	.quad	0x3f9F1C72E13AD8BE, 0x3f9F1C72E13AD8BE, 0x3f9F1C72E13AD8BE, 0x3f9F1C72E13AD8BE /* poly_coeff4 */
273	.quad	0x3fa6DB6DB3B445F8, 0x3fa6DB6DB3B445F8, 0x3fa6DB6DB3B445F8, 0x3fa6DB6DB3B445F8 /* poly_coeff3 */
274	.quad	0x3fb333333337E0DE, 0x3fb333333337E0DE, 0x3fb333333337E0DE, 0x3fb333333337E0DE /* poly_coeff2 */
275	.quad	0x3fc555555555529C, 0x3fc555555555529C, 0x3fc555555555529C, 0x3fc555555555529C /* poly_coeff1 */
276	/* PiH */
277	.align	32
278	.quad	0x400921fb54442d18, 0x400921fb54442d18, 0x400921fb54442d18, 0x400921fb54442d18
279	/* Pi2H */
280	.align	32
281	.quad	0x3ff921fb54442d18, 0x3ff921fb54442d18, 0x3ff921fb54442d18, 0x3ff921fb54442d18
282	.align	32
283	.type	__svml_dacos_data_internal, @object
284	.size	__svml_dacos_data_internal, .-__svml_dacos_data_internal
285