1 /* x86 CPU feature tuning.
2    This file is part of the GNU C Library.
3    Copyright (C) 2017-2022 Free Software Foundation, Inc.
4 
5    The GNU C Library is free software; you can redistribute it and/or
6    modify it under the terms of the GNU Lesser General Public
7    License as published by the Free Software Foundation; either
8    version 2.1 of the License, or (at your option) any later version.
9 
10    The GNU C Library is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13    Lesser General Public License for more details.
14 
15    You should have received a copy of the GNU Lesser General Public
16    License along with the GNU C Library; if not, see
17    <https://www.gnu.org/licenses/>.  */
18 
19 #if HAVE_TUNABLES
20 # define TUNABLE_NAMESPACE cpu
21 # include <stdbool.h>
22 # include <stdint.h>
23 # include <unistd.h>		/* Get STDOUT_FILENO for _dl_printf.  */
24 # include <elf/dl-tunables.h>
25 # include <string.h>
26 # include <cpu-features.h>
27 # include <ldsodefs.h>
28 
29 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
30    since IFUNC must be set up by init_cpu_features.  */
31 # if defined USE_MULTIARCH && !defined SHARED
32 #  ifdef __x86_64__
33 /* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h.  */
34 #   include <sysdeps/x86_64/memcmp-isa-default-impl.h>
35 #  else
36 #   define DEFAULT_MEMCMP	__memcmp_ia32
37 #  endif
38 extern __typeof (memcmp) DEFAULT_MEMCMP;
39 # else
40 #  define DEFAULT_MEMCMP	memcmp
41 # endif
42 
43 # define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len)		\
44   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);	\
45   if (!DEFAULT_MEMCMP (f, #name, len))					\
46     {									\
47       CPU_FEATURE_UNSET (cpu_features, name)				\
48       break;								\
49     }
50 
51 /* Disable a preferred feature NAME.  We don't enable a preferred feature
52    which isn't available.  */
53 # define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len)	\
54   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);	\
55   if (!DEFAULT_MEMCMP (f, #name, len))					\
56     {									\
57       cpu_features->preferred[index_arch_##name]			\
58 	&= ~bit_arch_##name;						\
59       break;								\
60     }
61 
62 /* Enable/disable a preferred feature NAME.  */
63 # define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name,	\
64 					  disable, len)			\
65   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);	\
66   if (!DEFAULT_MEMCMP (f, #name, len))					\
67     {									\
68       if (disable)							\
69 	cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name;	\
70       else								\
71 	cpu_features->preferred[index_arch_##name] |= bit_arch_##name;	\
72       break;								\
73     }
74 
75 /* Enable/disable a preferred feature NAME.  Enable a preferred feature
76    only if the feature NEED is usable.  */
77 # define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name,	\
78 					       need, disable, len)	\
79   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);	\
80   if (!DEFAULT_MEMCMP (f, #name, len))					\
81     {									\
82       if (disable)							\
83 	cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name;	\
84       else if (CPU_FEATURE_USABLE_P (cpu_features, need))		\
85 	cpu_features->preferred[index_arch_##name] |= bit_arch_##name;	\
86       break;								\
87     }
88 
89 attribute_hidden
90 void
TUNABLE_CALLBACK(set_hwcaps)91 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
92 {
93   /* The current IFUNC selection is based on microbenchmarks in glibc.
94      It should give the best performance for most workloads.  But other
95      choices may have better performance for a particular workload or on
96      the hardware which wasn't available when the selection was made.
97      The environment variable:
98 
99      GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
100 
101      can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
102      yyy and zzz, where the feature name is case-sensitive and has to
103      match the ones in cpu-features.h.  It can be used by glibc developers
104      to tune for a new processor or override the IFUNC selection to
105      improve performance for a particular workload.
106 
107      NOTE: the IFUNC selection may change over time.  Please check all
108      multiarch implementations when experimenting.  */
109 
110   const char *p = valp->strval;
111   struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
112   size_t len;
113 
114   do
115     {
116       const char *c, *n;
117       bool disable;
118       size_t nl;
119 
120       for (c = p; *c != ','; c++)
121 	if (*c == '\0')
122 	  break;
123 
124       len = c - p;
125       disable = *p == '-';
126       if (disable)
127 	{
128 	  n = p + 1;
129 	  nl = len - 1;
130 	}
131       else
132 	{
133 	  n = p;
134 	  nl = len;
135 	}
136       switch (nl)
137 	{
138 	default:
139 	  break;
140 	case 3:
141 	  if (disable)
142 	    {
143 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
144 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
145 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
146 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
147 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
148 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
149 	    }
150 	  break;
151 	case 4:
152 	  if (disable)
153 	    {
154 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
155 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
156 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
157 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
158 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
159 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
160 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
161 	      CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
162 	      CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
163 	    }
164 	  break;
165 	case 5:
166 	  if (disable)
167 	    {
168 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
169 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
170 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
171 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
172 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
173 	    }
174 	  break;
175 	case 6:
176 	  if (disable)
177 	    {
178 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
179 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
180 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
181 	      if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
182 		{
183 		  /* Update xsave_state_size to XSAVE state size.  */
184 		  cpu_features->xsave_state_size
185 		    = cpu_features->xsave_state_full_size;
186 		  CPU_FEATURE_UNSET (cpu_features, XSAVEC);
187 		}
188 	    }
189 	  break;
190 	case 7:
191 	  if (disable)
192 	    {
193 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
194 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
195 	    }
196 	  break;
197 	case 8:
198 	  if (disable)
199 	    {
200 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
201 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
202 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
203 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
204 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
205 	      CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
206 	    }
207 	  CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
208 					    disable, 8);
209 	  break;
210 	case 11:
211 	    {
212 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
213 						Prefer_ERMS,
214 						disable, 11);
215 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
216 						Prefer_FSRM,
217 						disable, 11);
218 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
219 						     Slow_SSE4_2,
220 						     SSE4_2,
221 						     disable, 11);
222 	    }
223 	  break;
224 	case 15:
225 	    {
226 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
227 						Fast_Rep_String,
228 						disable, 15);
229 	    }
230 	  break;
231 	case 16:
232 	    {
233 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
234 		(n, cpu_features, Prefer_No_AVX512, AVX512F,
235 		 disable, 16);
236 	    }
237 	  break;
238 	case 18:
239 	    {
240 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
241 						Fast_Copy_Backward,
242 						disable, 18);
243 	    }
244 	  break;
245 	case 19:
246 	    {
247 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
248 						Fast_Unaligned_Load,
249 						disable, 19);
250 	      CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
251 						Fast_Unaligned_Copy,
252 						disable, 19);
253 	    }
254 	  break;
255 	case 20:
256 	    {
257 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
258 		(n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
259 		 20);
260 	    }
261 	  break;
262 	case 23:
263 	    {
264 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
265 		(n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
266 		 disable, 23);
267 	    }
268 	  break;
269 	case 24:
270 	    {
271 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
272 		(n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
273 		 disable, 24);
274 	    }
275 	  break;
276 	case 26:
277 	    {
278 	      CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
279 		(n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
280 		 disable, 26);
281 	    }
282 	  break;
283 	}
284       p += len + 1;
285     }
286   while (*p != '\0');
287 }
288 
289 # if CET_ENABLED
290 
291 attribute_hidden
292 void
TUNABLE_CALLBACK(set_x86_ibt)293 TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
294 {
295   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
296     GL(dl_x86_feature_control).ibt = cet_always_on;
297   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
298     GL(dl_x86_feature_control).ibt = cet_always_off;
299   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
300 			   sizeof ("permissive")) == 0)
301     GL(dl_x86_feature_control).ibt = cet_permissive;
302 }
303 
304 attribute_hidden
305 void
TUNABLE_CALLBACK(set_x86_shstk)306 TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
307 {
308   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
309     GL(dl_x86_feature_control).shstk = cet_always_on;
310   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
311     GL(dl_x86_feature_control).shstk = cet_always_off;
312   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
313 			   sizeof ("permissive")) == 0)
314     GL(dl_x86_feature_control).shstk = cet_permissive;
315 }
316 # endif
317 #endif
318