1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * FPU data structures:
4  */
5 #ifndef _ASM_X86_FPU_H
6 #define _ASM_X86_FPU_H
7 
8 /*
9  * The legacy x87 FPU state format, as saved by FSAVE and
10  * restored by the FRSTOR instructions:
11  */
12 struct fregs_state {
13 	u32			cwd;	/* FPU Control Word		*/
14 	u32			swd;	/* FPU Status Word		*/
15 	u32			twd;	/* FPU Tag Word			*/
16 	u32			fip;	/* FPU IP Offset		*/
17 	u32			fcs;	/* FPU IP Selector		*/
18 	u32			foo;	/* FPU Operand Pointer Offset	*/
19 	u32			fos;	/* FPU Operand Pointer Selector	*/
20 
21 	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
22 	u32			st_space[20];
23 
24 	/* Software status information [not touched by FSAVE]:		*/
25 	u32			status;
26 };
27 
28 /*
29  * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
30  * restored by the FXRSTOR instructions. It's similar to the FSAVE
31  * format, but differs in some areas, plus has extensions at
32  * the end for the XMM registers.
33  */
34 struct fxregs_state {
35 	u16			cwd; /* Control Word			*/
36 	u16			swd; /* Status Word			*/
37 	u16			twd; /* Tag Word			*/
38 	u16			fop; /* Last Instruction Opcode		*/
39 	union {
40 		struct {
41 			u64	rip; /* Instruction Pointer		*/
42 			u64	rdp; /* Data Pointer			*/
43 		};
44 		struct {
45 			u32	fip; /* FPU IP Offset			*/
46 			u32	fcs; /* FPU IP Selector			*/
47 			u32	foo; /* FPU Operand Offset		*/
48 			u32	fos; /* FPU Operand Selector		*/
49 		};
50 	};
51 	u32			mxcsr;		/* MXCSR Register State */
52 	u32			mxcsr_mask;	/* MXCSR Mask		*/
53 
54 	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
55 	u32			st_space[32];
56 
57 	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
58 	u32			xmm_space[64];
59 
60 	u32			padding[12];
61 
62 	union {
63 		u32		padding1[12];
64 		u32		sw_reserved[12];
65 	};
66 
67 } __attribute__((aligned(16)));
68 
69 /* Default value for fxregs_state.mxcsr: */
70 #define MXCSR_DEFAULT		0x1f80
71 
72 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
73 #define MXCSR_AND_FLAGS_SIZE sizeof(u64)
74 
75 /*
76  * Software based FPU emulation state. This is arbitrary really,
77  * it matches the x87 format to make it easier to understand:
78  */
79 struct swregs_state {
80 	u32			cwd;
81 	u32			swd;
82 	u32			twd;
83 	u32			fip;
84 	u32			fcs;
85 	u32			foo;
86 	u32			fos;
87 	/* 8*10 bytes for each FP-reg = 80 bytes: */
88 	u32			st_space[20];
89 	u8			ftop;
90 	u8			changed;
91 	u8			lookahead;
92 	u8			no_update;
93 	u8			rm;
94 	u8			alimit;
95 	struct math_emu_info	*info;
96 	u32			entry_eip;
97 };
98 
99 /*
100  * List of XSAVE features Linux knows about:
101  */
102 enum xfeature {
103 	XFEATURE_FP,
104 	XFEATURE_SSE,
105 	/*
106 	 * Values above here are "legacy states".
107 	 * Those below are "extended states".
108 	 */
109 	XFEATURE_YMM,
110 	XFEATURE_BNDREGS,
111 	XFEATURE_BNDCSR,
112 	XFEATURE_OPMASK,
113 	XFEATURE_ZMM_Hi256,
114 	XFEATURE_Hi16_ZMM,
115 	XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
116 	XFEATURE_PKRU,
117 	XFEATURE_PASID,
118 	XFEATURE_CET_USER,
119 	XFEATURE_CET_KERNEL_UNUSED,
120 	XFEATURE_RSRVD_COMP_13,
121 	XFEATURE_RSRVD_COMP_14,
122 	XFEATURE_LBR,
123 	XFEATURE_RSRVD_COMP_16,
124 	XFEATURE_XTILE_CFG,
125 	XFEATURE_XTILE_DATA,
126 
127 	XFEATURE_MAX,
128 };
129 
130 #define XFEATURE_MASK_FP		(1 << XFEATURE_FP)
131 #define XFEATURE_MASK_SSE		(1 << XFEATURE_SSE)
132 #define XFEATURE_MASK_YMM		(1 << XFEATURE_YMM)
133 #define XFEATURE_MASK_BNDREGS		(1 << XFEATURE_BNDREGS)
134 #define XFEATURE_MASK_BNDCSR		(1 << XFEATURE_BNDCSR)
135 #define XFEATURE_MASK_OPMASK		(1 << XFEATURE_OPMASK)
136 #define XFEATURE_MASK_ZMM_Hi256		(1 << XFEATURE_ZMM_Hi256)
137 #define XFEATURE_MASK_Hi16_ZMM		(1 << XFEATURE_Hi16_ZMM)
138 #define XFEATURE_MASK_PT		(1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
139 #define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
140 #define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
141 #define XFEATURE_MASK_CET_USER		(1 << XFEATURE_CET_USER)
142 #define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL_UNUSED)
143 #define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
144 #define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
145 #define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
146 
147 #define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
148 #define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
149 					 | XFEATURE_MASK_ZMM_Hi256 \
150 					 | XFEATURE_MASK_Hi16_ZMM)
151 
152 #ifdef CONFIG_X86_64
153 # define XFEATURE_MASK_XTILE		(XFEATURE_MASK_XTILE_DATA \
154 					 | XFEATURE_MASK_XTILE_CFG)
155 #else
156 # define XFEATURE_MASK_XTILE		(0)
157 #endif
158 
159 #define FIRST_EXTENDED_XFEATURE	XFEATURE_YMM
160 
161 struct reg_128_bit {
162 	u8      regbytes[128/8];
163 };
164 struct reg_256_bit {
165 	u8	regbytes[256/8];
166 };
167 struct reg_512_bit {
168 	u8	regbytes[512/8];
169 };
170 struct reg_1024_byte {
171 	u8	regbytes[1024];
172 };
173 
174 /*
175  * State component 2:
176  *
177  * There are 16x 256-bit AVX registers named YMM0-YMM15.
178  * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
179  * and are stored in 'struct fxregs_state::xmm_space[]' in the
180  * "legacy" area.
181  *
182  * The high 128 bits are stored here.
183  */
184 struct ymmh_struct {
185 	struct reg_128_bit              hi_ymm[16];
186 } __packed;
187 
188 /* Intel MPX support: */
189 
190 struct mpx_bndreg {
191 	u64				lower_bound;
192 	u64				upper_bound;
193 } __packed;
194 /*
195  * State component 3 is used for the 4 128-bit bounds registers
196  */
197 struct mpx_bndreg_state {
198 	struct mpx_bndreg		bndreg[4];
199 } __packed;
200 
201 /*
202  * State component 4 is used for the 64-bit user-mode MPX
203  * configuration register BNDCFGU and the 64-bit MPX status
204  * register BNDSTATUS.  We call the pair "BNDCSR".
205  */
206 struct mpx_bndcsr {
207 	u64				bndcfgu;
208 	u64				bndstatus;
209 } __packed;
210 
211 /*
212  * The BNDCSR state is padded out to be 64-bytes in size.
213  */
214 struct mpx_bndcsr_state {
215 	union {
216 		struct mpx_bndcsr		bndcsr;
217 		u8				pad_to_64_bytes[64];
218 	};
219 } __packed;
220 
221 /* AVX-512 Components: */
222 
223 /*
224  * State component 5 is used for the 8 64-bit opmask registers
225  * k0-k7 (opmask state).
226  */
227 struct avx_512_opmask_state {
228 	u64				opmask_reg[8];
229 } __packed;
230 
231 /*
232  * State component 6 is used for the upper 256 bits of the
233  * registers ZMM0-ZMM15. These 16 256-bit values are denoted
234  * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
235  */
236 struct avx_512_zmm_uppers_state {
237 	struct reg_256_bit		zmm_upper[16];
238 } __packed;
239 
240 /*
241  * State component 7 is used for the 16 512-bit registers
242  * ZMM16-ZMM31 (Hi16_ZMM state).
243  */
244 struct avx_512_hi16_state {
245 	struct reg_512_bit		hi16_zmm[16];
246 } __packed;
247 
248 /*
249  * State component 9: 32-bit PKRU register.  The state is
250  * 8 bytes long but only 4 bytes is used currently.
251  */
252 struct pkru_state {
253 	u32				pkru;
254 	u32				pad;
255 } __packed;
256 
257 /*
258  * State component 11 is Control-flow Enforcement user states
259  */
260 struct cet_user_state {
261 	/* user control-flow settings */
262 	u64 user_cet;
263 	/* user shadow stack pointer */
264 	u64 user_ssp;
265 };
266 
267 /*
268  * State component 15: Architectural LBR configuration state.
269  * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
270  */
271 
272 struct lbr_entry {
273 	u64 from;
274 	u64 to;
275 	u64 info;
276 };
277 
278 struct arch_lbr_state {
279 	u64 lbr_ctl;
280 	u64 lbr_depth;
281 	u64 ler_from;
282 	u64 ler_to;
283 	u64 ler_info;
284 	struct lbr_entry		entries[];
285 };
286 
287 /*
288  * State component 17: 64-byte tile configuration register.
289  */
290 struct xtile_cfg {
291 	u64				tcfg[8];
292 } __packed;
293 
294 /*
295  * State component 18: 1KB tile data register.
296  * Each register represents 16 64-byte rows of the matrix
297  * data. But the number of registers depends on the actual
298  * implementation.
299  */
300 struct xtile_data {
301 	struct reg_1024_byte		tmm;
302 } __packed;
303 
304 /*
305  * State component 10 is supervisor state used for context-switching the
306  * PASID state.
307  */
308 struct ia32_pasid_state {
309 	u64 pasid;
310 } __packed;
311 
312 struct xstate_header {
313 	u64				xfeatures;
314 	u64				xcomp_bv;
315 	u64				reserved[6];
316 } __attribute__((packed));
317 
318 /*
319  * xstate_header.xcomp_bv[63] indicates that the extended_state_area
320  * is in compacted format.
321  */
322 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
323 
324 /*
325  * This is our most modern FPU state format, as saved by the XSAVE
326  * and restored by the XRSTOR instructions.
327  *
328  * It consists of a legacy fxregs portion, an xstate header and
329  * subsequent areas as defined by the xstate header.  Not all CPUs
330  * support all the extensions, so the size of the extended area
331  * can vary quite a bit between CPUs.
332  */
333 struct xregs_state {
334 	struct fxregs_state		i387;
335 	struct xstate_header		header;
336 	u8				extended_state_area[];
337 } __attribute__ ((packed, aligned (64)));
338 
339 /*
340  * This is a union of all the possible FPU state formats
341  * put together, so that we can pick the right one runtime.
342  *
343  * The size of the structure is determined by the largest
344  * member - which is the xsave area.  The padding is there
345  * to ensure that statically-allocated task_structs (just
346  * the init_task today) have enough space.
347  */
348 union fpregs_state {
349 	struct fregs_state		fsave;
350 	struct fxregs_state		fxsave;
351 	struct swregs_state		soft;
352 	struct xregs_state		xsave;
353 	u8 __padding[PAGE_SIZE];
354 };
355 
356 struct fpstate {
357 	/* @kernel_size: The size of the kernel register image */
358 	unsigned int		size;
359 
360 	/* @user_size: The size in non-compacted UABI format */
361 	unsigned int		user_size;
362 
363 	/* @xfeatures:		xfeatures for which the storage is sized */
364 	u64			xfeatures;
365 
366 	/* @user_xfeatures:	xfeatures valid in UABI buffers */
367 	u64			user_xfeatures;
368 
369 	/* @xfd:		xfeatures disabled to trap userspace use. */
370 	u64			xfd;
371 
372 	/* @is_valloc:		Indicator for dynamically allocated state */
373 	unsigned int		is_valloc	: 1;
374 
375 	/* @is_guest:		Indicator for guest state (KVM) */
376 	unsigned int		is_guest	: 1;
377 
378 	/*
379 	 * @is_confidential:	Indicator for KVM confidential mode.
380 	 *			The FPU registers are restored by the
381 	 *			vmentry firmware from encrypted guest
382 	 *			memory. On vmexit the FPU registers are
383 	 *			saved by firmware to encrypted guest memory
384 	 *			and the registers are scrubbed before
385 	 *			returning to the host. So there is no
386 	 *			content which is worth saving and restoring.
387 	 *			The fpstate has to be there so that
388 	 *			preemption and softirq FPU usage works
389 	 *			without special casing.
390 	 */
391 	unsigned int		is_confidential	: 1;
392 
393 	/* @in_use:		State is in use */
394 	unsigned int		in_use		: 1;
395 
396 	/* @regs: The register state union for all supported formats */
397 	union fpregs_state	regs;
398 
399 	/* @regs is dynamically sized! Don't add anything after @regs! */
400 } __aligned(64);
401 
402 #define FPU_GUEST_PERM_LOCKED		BIT_ULL(63)
403 
404 struct fpu_state_perm {
405 	/*
406 	 * @__state_perm:
407 	 *
408 	 * This bitmap indicates the permission for state components, which
409 	 * are available to a thread group. The permission prctl() sets the
410 	 * enabled state bits in thread_group_leader()->thread.fpu.
411 	 *
412 	 * All run time operations use the per thread information in the
413 	 * currently active fpu.fpstate which contains the xfeature masks
414 	 * and sizes for kernel and user space.
415 	 *
416 	 * This master permission field is only to be used when
417 	 * task.fpu.fpstate based checks fail to validate whether the task
418 	 * is allowed to expand it's xfeatures set which requires to
419 	 * allocate a larger sized fpstate buffer.
420 	 *
421 	 * Do not access this field directly.  Use the provided helper
422 	 * function. Unlocked access is possible for quick checks.
423 	 */
424 	u64				__state_perm;
425 
426 	/*
427 	 * @__state_size:
428 	 *
429 	 * The size required for @__state_perm. Only valid to access
430 	 * with sighand locked.
431 	 */
432 	unsigned int			__state_size;
433 
434 	/*
435 	 * @__user_state_size:
436 	 *
437 	 * The size required for @__state_perm user part. Only valid to
438 	 * access with sighand locked.
439 	 */
440 	unsigned int			__user_state_size;
441 };
442 
443 /*
444  * Highest level per task FPU state data structure that
445  * contains the FPU register state plus various FPU
446  * state fields:
447  */
448 struct fpu {
449 	/*
450 	 * @last_cpu:
451 	 *
452 	 * Records the last CPU on which this context was loaded into
453 	 * FPU registers. (In the lazy-restore case we might be
454 	 * able to reuse FPU registers across multiple context switches
455 	 * this way, if no intermediate task used the FPU.)
456 	 *
457 	 * A value of -1 is used to indicate that the FPU state in context
458 	 * memory is newer than the FPU state in registers, and that the
459 	 * FPU state should be reloaded next time the task is run.
460 	 */
461 	unsigned int			last_cpu;
462 
463 	/*
464 	 * @avx512_timestamp:
465 	 *
466 	 * Records the timestamp of AVX512 use during last context switch.
467 	 */
468 	unsigned long			avx512_timestamp;
469 
470 	/*
471 	 * @fpstate:
472 	 *
473 	 * Pointer to the active struct fpstate. Initialized to
474 	 * point at @__fpstate below.
475 	 */
476 	struct fpstate			*fpstate;
477 
478 	/*
479 	 * @__task_fpstate:
480 	 *
481 	 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
482 	 * used only for KVM support to swap out the regular task fpstate.
483 	 */
484 	struct fpstate			*__task_fpstate;
485 
486 	/*
487 	 * @perm:
488 	 *
489 	 * Permission related information
490 	 */
491 	struct fpu_state_perm		perm;
492 
493 	/*
494 	 * @guest_perm:
495 	 *
496 	 * Permission related information for guest pseudo FPUs
497 	 */
498 	struct fpu_state_perm		guest_perm;
499 
500 	/*
501 	 * @__fpstate:
502 	 *
503 	 * Initial in-memory storage for FPU registers which are saved in
504 	 * context switch and when the kernel uses the FPU. The registers
505 	 * are restored from this storage on return to user space if they
506 	 * are not longer containing the tasks FPU register state.
507 	 */
508 	struct fpstate			__fpstate;
509 	/*
510 	 * WARNING: '__fpstate' is dynamically-sized.  Do not put
511 	 * anything after it here.
512 	 */
513 };
514 
515 /*
516  * Guest pseudo FPU container
517  */
518 struct fpu_guest {
519 	/*
520 	 * @xfeatures:			xfeature bitmap of features which are
521 	 *				currently enabled for the guest vCPU.
522 	 */
523 	u64				xfeatures;
524 
525 	/*
526 	 * @perm:			xfeature bitmap of features which are
527 	 *				permitted to be enabled for the guest
528 	 *				vCPU.
529 	 */
530 	u64				perm;
531 
532 	/*
533 	 * @xfd_err:			Save the guest value.
534 	 */
535 	u64				xfd_err;
536 
537 	/*
538 	 * @uabi_size:			Size required for save/restore
539 	 */
540 	unsigned int			uabi_size;
541 
542 	/*
543 	 * @fpstate:			Pointer to the allocated guest fpstate
544 	 */
545 	struct fpstate			*fpstate;
546 };
547 
548 /*
549  * FPU state configuration data. Initialized at boot time. Read only after init.
550  */
551 struct fpu_state_config {
552 	/*
553 	 * @max_size:
554 	 *
555 	 * The maximum size of the register state buffer. Includes all
556 	 * supported features except independent managed features.
557 	 */
558 	unsigned int		max_size;
559 
560 	/*
561 	 * @default_size:
562 	 *
563 	 * The default size of the register state buffer. Includes all
564 	 * supported features except independent managed features and
565 	 * features which have to be requested by user space before usage.
566 	 */
567 	unsigned int		default_size;
568 
569 	/*
570 	 * @max_features:
571 	 *
572 	 * The maximum supported features bitmap. Does not include
573 	 * independent managed features.
574 	 */
575 	u64 max_features;
576 
577 	/*
578 	 * @default_features:
579 	 *
580 	 * The default supported features bitmap. Does not include
581 	 * independent managed features and features which have to
582 	 * be requested by user space before usage.
583 	 */
584 	u64 default_features;
585 	/*
586 	 * @legacy_features:
587 	 *
588 	 * Features which can be reported back to user space
589 	 * even without XSAVE support, i.e. legacy features FP + SSE
590 	 */
591 	u64 legacy_features;
592 };
593 
594 /* FPU state configuration information */
595 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
596 
597 #endif /* _ASM_X86_FPU_H */
598