1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2021 ARM Limited.
4 */
5
6 #include <errno.h>
7 #include <stdbool.h>
8 #include <stddef.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <sys/auxv.h>
14 #include <sys/prctl.h>
15 #include <asm/hwcap.h>
16 #include <asm/sigcontext.h>
17 #include <asm/unistd.h>
18
19 #include "../../kselftest.h"
20
21 #include "syscall-abi.h"
22
23 #define NUM_VL ((SVE_VQ_MAX - SVE_VQ_MIN) + 1)
24
25 static int default_sme_vl;
26
27 extern void do_syscall(int sve_vl, int sme_vl);
28
fill_random(void * buf,size_t size)29 static void fill_random(void *buf, size_t size)
30 {
31 int i;
32 uint32_t *lbuf = buf;
33
34 /* random() returns a 32 bit number regardless of the size of long */
35 for (i = 0; i < size / sizeof(uint32_t); i++)
36 lbuf[i] = random();
37 }
38
39 /*
40 * We also repeat the test for several syscalls to try to expose different
41 * behaviour.
42 */
43 static struct syscall_cfg {
44 int syscall_nr;
45 const char *name;
46 } syscalls[] = {
47 { __NR_getpid, "getpid()" },
48 { __NR_sched_yield, "sched_yield()" },
49 };
50
51 #define NUM_GPR 31
52 uint64_t gpr_in[NUM_GPR];
53 uint64_t gpr_out[NUM_GPR];
54
setup_gpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)55 static void setup_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
56 uint64_t svcr)
57 {
58 fill_random(gpr_in, sizeof(gpr_in));
59 gpr_in[8] = cfg->syscall_nr;
60 memset(gpr_out, 0, sizeof(gpr_out));
61 }
62
check_gpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)63 static int check_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, uint64_t svcr)
64 {
65 int errors = 0;
66 int i;
67
68 /*
69 * GPR x0-x7 may be clobbered, and all others should be preserved.
70 */
71 for (i = 9; i < ARRAY_SIZE(gpr_in); i++) {
72 if (gpr_in[i] != gpr_out[i]) {
73 ksft_print_msg("%s SVE VL %d mismatch in GPR %d: %llx != %llx\n",
74 cfg->name, sve_vl, i,
75 gpr_in[i], gpr_out[i]);
76 errors++;
77 }
78 }
79
80 return errors;
81 }
82
83 #define NUM_FPR 32
84 uint64_t fpr_in[NUM_FPR * 2];
85 uint64_t fpr_out[NUM_FPR * 2];
86
setup_fpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)87 static void setup_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
88 uint64_t svcr)
89 {
90 fill_random(fpr_in, sizeof(fpr_in));
91 memset(fpr_out, 0, sizeof(fpr_out));
92 }
93
check_fpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)94 static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
95 uint64_t svcr)
96 {
97 int errors = 0;
98 int i;
99
100 if (!sve_vl) {
101 for (i = 0; i < ARRAY_SIZE(fpr_in); i++) {
102 if (fpr_in[i] != fpr_out[i]) {
103 ksft_print_msg("%s Q%d/%d mismatch %llx != %llx\n",
104 cfg->name,
105 i / 2, i % 2,
106 fpr_in[i], fpr_out[i]);
107 errors++;
108 }
109 }
110 }
111
112 return errors;
113 }
114
115 static uint8_t z_zero[__SVE_ZREG_SIZE(SVE_VQ_MAX)];
116 uint8_t z_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
117 uint8_t z_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
118
setup_z(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)119 static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
120 uint64_t svcr)
121 {
122 fill_random(z_in, sizeof(z_in));
123 fill_random(z_out, sizeof(z_out));
124 }
125
check_z(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)126 static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
127 uint64_t svcr)
128 {
129 size_t reg_size = sve_vl;
130 int errors = 0;
131 int i;
132
133 if (!sve_vl)
134 return 0;
135
136 /*
137 * After a syscall the low 128 bits of the Z registers should
138 * be preserved and the rest be zeroed or preserved, except if
139 * we were in streaming mode in which case the low 128 bits may
140 * also be cleared by the transition out of streaming mode.
141 */
142 for (i = 0; i < SVE_NUM_ZREGS; i++) {
143 void *in = &z_in[reg_size * i];
144 void *out = &z_out[reg_size * i];
145
146 if ((memcmp(in, out, SVE_VQ_BYTES) != 0) &&
147 !((svcr & SVCR_SM_MASK) &&
148 memcmp(z_zero, out, SVE_VQ_BYTES) == 0)) {
149 ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n",
150 cfg->name, sve_vl, i);
151 errors++;
152 }
153 }
154
155 return errors;
156 }
157
158 uint8_t p_in[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)];
159 uint8_t p_out[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)];
160
setup_p(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)161 static void setup_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
162 uint64_t svcr)
163 {
164 fill_random(p_in, sizeof(p_in));
165 fill_random(p_out, sizeof(p_out));
166 }
167
check_p(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)168 static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
169 uint64_t svcr)
170 {
171 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */
172
173 int errors = 0;
174 int i;
175
176 if (!sve_vl)
177 return 0;
178
179 /* After a syscall the P registers should be preserved or zeroed */
180 for (i = 0; i < SVE_NUM_PREGS * reg_size; i++)
181 if (p_out[i] && (p_in[i] != p_out[i]))
182 errors++;
183 if (errors)
184 ksft_print_msg("%s SVE VL %d predicate registers non-zero\n",
185 cfg->name, sve_vl);
186
187 return errors;
188 }
189
190 uint8_t ffr_in[__SVE_PREG_SIZE(SVE_VQ_MAX)];
191 uint8_t ffr_out[__SVE_PREG_SIZE(SVE_VQ_MAX)];
192
setup_ffr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)193 static void setup_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
194 uint64_t svcr)
195 {
196 /*
197 * If we are in streaming mode and do not have FA64 then FFR
198 * is unavailable.
199 */
200 if ((svcr & SVCR_SM_MASK) &&
201 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) {
202 memset(&ffr_in, 0, sizeof(ffr_in));
203 return;
204 }
205
206 /*
207 * It is only valid to set a contiguous set of bits starting
208 * at 0. For now since we're expecting this to be cleared by
209 * a syscall just set all bits.
210 */
211 memset(ffr_in, 0xff, sizeof(ffr_in));
212 fill_random(ffr_out, sizeof(ffr_out));
213 }
214
check_ffr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)215 static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
216 uint64_t svcr)
217 {
218 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */
219 int errors = 0;
220 int i;
221
222 if (!sve_vl)
223 return 0;
224
225 if ((svcr & SVCR_SM_MASK) &&
226 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64))
227 return 0;
228
229 /* After a syscall the P registers should be preserved or zeroed */
230 for (i = 0; i < reg_size; i++)
231 if (ffr_out[i] && (ffr_in[i] != ffr_out[i]))
232 errors++;
233 if (errors)
234 ksft_print_msg("%s SVE VL %d FFR non-zero\n",
235 cfg->name, sve_vl);
236
237 return errors;
238 }
239
240 uint64_t svcr_in, svcr_out;
241
setup_svcr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)242 static void setup_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
243 uint64_t svcr)
244 {
245 svcr_in = svcr;
246 }
247
check_svcr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)248 static int check_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
249 uint64_t svcr)
250 {
251 int errors = 0;
252
253 if (svcr_out & SVCR_SM_MASK) {
254 ksft_print_msg("%s Still in SM, SVCR %llx\n",
255 cfg->name, svcr_out);
256 errors++;
257 }
258
259 if ((svcr_in & SVCR_ZA_MASK) != (svcr_out & SVCR_ZA_MASK)) {
260 ksft_print_msg("%s PSTATE.ZA changed, SVCR %llx != %llx\n",
261 cfg->name, svcr_in, svcr_out);
262 errors++;
263 }
264
265 return errors;
266 }
267
268 uint8_t za_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
269 uint8_t za_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
270
setup_za(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)271 static void setup_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
272 uint64_t svcr)
273 {
274 fill_random(za_in, sizeof(za_in));
275 memset(za_out, 0, sizeof(za_out));
276 }
277
check_za(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)278 static int check_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
279 uint64_t svcr)
280 {
281 size_t reg_size = sme_vl * sme_vl;
282 int errors = 0;
283
284 if (!(svcr & SVCR_ZA_MASK))
285 return 0;
286
287 if (memcmp(za_in, za_out, reg_size) != 0) {
288 ksft_print_msg("SME VL %d ZA does not match\n", sme_vl);
289 errors++;
290 }
291
292 return errors;
293 }
294
295 typedef void (*setup_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
296 uint64_t svcr);
297 typedef int (*check_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
298 uint64_t svcr);
299
300 /*
301 * Each set of registers has a setup function which is called before
302 * the syscall to fill values in a global variable for loading by the
303 * test code and a check function which validates that the results are
304 * as expected. Vector lengths are passed everywhere, a vector length
305 * of 0 should be treated as do not test.
306 */
307 static struct {
308 setup_fn setup;
309 check_fn check;
310 } regset[] = {
311 { setup_gpr, check_gpr },
312 { setup_fpr, check_fpr },
313 { setup_z, check_z },
314 { setup_p, check_p },
315 { setup_ffr, check_ffr },
316 { setup_svcr, check_svcr },
317 { setup_za, check_za },
318 };
319
do_test(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)320 static bool do_test(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
321 uint64_t svcr)
322 {
323 int errors = 0;
324 int i;
325
326 for (i = 0; i < ARRAY_SIZE(regset); i++)
327 regset[i].setup(cfg, sve_vl, sme_vl, svcr);
328
329 do_syscall(sve_vl, sme_vl);
330
331 for (i = 0; i < ARRAY_SIZE(regset); i++)
332 errors += regset[i].check(cfg, sve_vl, sme_vl, svcr);
333
334 return errors == 0;
335 }
336
test_one_syscall(struct syscall_cfg * cfg)337 static void test_one_syscall(struct syscall_cfg *cfg)
338 {
339 int sve_vq, sve_vl;
340 int sme_vq, sme_vl;
341
342 /* FPSIMD only case */
343 ksft_test_result(do_test(cfg, 0, default_sme_vl, 0),
344 "%s FPSIMD\n", cfg->name);
345
346 if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
347 return;
348
349 for (sve_vq = SVE_VQ_MAX; sve_vq > 0; --sve_vq) {
350 sve_vl = prctl(PR_SVE_SET_VL, sve_vq * 16);
351 if (sve_vl == -1)
352 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n",
353 strerror(errno), errno);
354
355 sve_vl &= PR_SVE_VL_LEN_MASK;
356
357 if (sve_vq != sve_vq_from_vl(sve_vl))
358 sve_vq = sve_vq_from_vl(sve_vl);
359
360 ksft_test_result(do_test(cfg, sve_vl, default_sme_vl, 0),
361 "%s SVE VL %d\n", cfg->name, sve_vl);
362
363 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME))
364 continue;
365
366 for (sme_vq = SVE_VQ_MAX; sme_vq > 0; --sme_vq) {
367 sme_vl = prctl(PR_SME_SET_VL, sme_vq * 16);
368 if (sme_vl == -1)
369 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n",
370 strerror(errno), errno);
371
372 sme_vl &= PR_SME_VL_LEN_MASK;
373
374 if (sme_vq != sve_vq_from_vl(sme_vl))
375 sme_vq = sve_vq_from_vl(sme_vl);
376
377 ksft_test_result(do_test(cfg, sve_vl, sme_vl,
378 SVCR_ZA_MASK | SVCR_SM_MASK),
379 "%s SVE VL %d/SME VL %d SM+ZA\n",
380 cfg->name, sve_vl, sme_vl);
381 ksft_test_result(do_test(cfg, sve_vl, sme_vl,
382 SVCR_SM_MASK),
383 "%s SVE VL %d/SME VL %d SM\n",
384 cfg->name, sve_vl, sme_vl);
385 ksft_test_result(do_test(cfg, sve_vl, sme_vl,
386 SVCR_ZA_MASK),
387 "%s SVE VL %d/SME VL %d ZA\n",
388 cfg->name, sve_vl, sme_vl);
389 }
390 }
391 }
392
sve_count_vls(void)393 int sve_count_vls(void)
394 {
395 unsigned int vq;
396 int vl_count = 0;
397 int vl;
398
399 if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
400 return 0;
401
402 /*
403 * Enumerate up to SVE_VQ_MAX vector lengths
404 */
405 for (vq = SVE_VQ_MAX; vq > 0; --vq) {
406 vl = prctl(PR_SVE_SET_VL, vq * 16);
407 if (vl == -1)
408 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n",
409 strerror(errno), errno);
410
411 vl &= PR_SVE_VL_LEN_MASK;
412
413 if (vq != sve_vq_from_vl(vl))
414 vq = sve_vq_from_vl(vl);
415
416 vl_count++;
417 }
418
419 return vl_count;
420 }
421
sme_count_vls(void)422 int sme_count_vls(void)
423 {
424 unsigned int vq;
425 int vl_count = 0;
426 int vl;
427
428 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME))
429 return 0;
430
431 /* Ensure we configure a SME VL, used to flag if SVCR is set */
432 default_sme_vl = 16;
433
434 /*
435 * Enumerate up to SVE_VQ_MAX vector lengths
436 */
437 for (vq = SVE_VQ_MAX; vq > 0; --vq) {
438 vl = prctl(PR_SME_SET_VL, vq * 16);
439 if (vl == -1)
440 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n",
441 strerror(errno), errno);
442
443 vl &= PR_SME_VL_LEN_MASK;
444
445 if (vq != sve_vq_from_vl(vl))
446 vq = sve_vq_from_vl(vl);
447
448 vl_count++;
449 }
450
451 return vl_count;
452 }
453
main(void)454 int main(void)
455 {
456 int i;
457 int tests = 1; /* FPSIMD */
458
459 srandom(getpid());
460
461 ksft_print_header();
462 tests += sve_count_vls();
463 tests += (sve_count_vls() * sme_count_vls()) * 3;
464 ksft_set_plan(ARRAY_SIZE(syscalls) * tests);
465
466 if (getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)
467 ksft_print_msg("SME with FA64\n");
468 else if (getauxval(AT_HWCAP2) & HWCAP2_SME)
469 ksft_print_msg("SME without FA64\n");
470
471 for (i = 0; i < ARRAY_SIZE(syscalls); i++)
472 test_one_syscall(&syscalls[i]);
473
474 ksft_print_cnts();
475
476 return 0;
477 }
478