1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * FPU data structures:
4  */
5 #ifndef _ASM_X86_FPU_H
6 #define _ASM_X86_FPU_H
7 
8 /*
9  * The legacy x87 FPU state format, as saved by FSAVE and
10  * restored by the FRSTOR instructions:
11  */
12 struct fregs_state {
13 	u32			cwd;	/* FPU Control Word		*/
14 	u32			swd;	/* FPU Status Word		*/
15 	u32			twd;	/* FPU Tag Word			*/
16 	u32			fip;	/* FPU IP Offset		*/
17 	u32			fcs;	/* FPU IP Selector		*/
18 	u32			foo;	/* FPU Operand Pointer Offset	*/
19 	u32			fos;	/* FPU Operand Pointer Selector	*/
20 
21 	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
22 	u32			st_space[20];
23 
24 	/* Software status information [not touched by FSAVE]:		*/
25 	u32			status;
26 };
27 
28 /*
29  * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
30  * restored by the FXRSTOR instructions. It's similar to the FSAVE
31  * format, but differs in some areas, plus has extensions at
32  * the end for the XMM registers.
33  */
34 struct fxregs_state {
35 	u16			cwd; /* Control Word			*/
36 	u16			swd; /* Status Word			*/
37 	u16			twd; /* Tag Word			*/
38 	u16			fop; /* Last Instruction Opcode		*/
39 	union {
40 		struct {
41 			u64	rip; /* Instruction Pointer		*/
42 			u64	rdp; /* Data Pointer			*/
43 		};
44 		struct {
45 			u32	fip; /* FPU IP Offset			*/
46 			u32	fcs; /* FPU IP Selector			*/
47 			u32	foo; /* FPU Operand Offset		*/
48 			u32	fos; /* FPU Operand Selector		*/
49 		};
50 	};
51 	u32			mxcsr;		/* MXCSR Register State */
52 	u32			mxcsr_mask;	/* MXCSR Mask		*/
53 
54 	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
55 	u32			st_space[32];
56 
57 	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
58 	u32			xmm_space[64];
59 
60 	u32			padding[12];
61 
62 	union {
63 		u32		padding1[12];
64 		u32		sw_reserved[12];
65 	};
66 
67 } __attribute__((aligned(16)));
68 
69 /* Default value for fxregs_state.mxcsr: */
70 #define MXCSR_DEFAULT		0x1f80
71 
72 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
73 #define MXCSR_AND_FLAGS_SIZE sizeof(u64)
74 
75 /*
76  * Software based FPU emulation state. This is arbitrary really,
77  * it matches the x87 format to make it easier to understand:
78  */
79 struct swregs_state {
80 	u32			cwd;
81 	u32			swd;
82 	u32			twd;
83 	u32			fip;
84 	u32			fcs;
85 	u32			foo;
86 	u32			fos;
87 	/* 8*10 bytes for each FP-reg = 80 bytes: */
88 	u32			st_space[20];
89 	u8			ftop;
90 	u8			changed;
91 	u8			lookahead;
92 	u8			no_update;
93 	u8			rm;
94 	u8			alimit;
95 	struct math_emu_info	*info;
96 	u32			entry_eip;
97 };
98 
99 /*
100  * List of XSAVE features Linux knows about:
101  */
102 enum xfeature {
103 	XFEATURE_FP,
104 	XFEATURE_SSE,
105 	/*
106 	 * Values above here are "legacy states".
107 	 * Those below are "extended states".
108 	 */
109 	XFEATURE_YMM,
110 	XFEATURE_BNDREGS,
111 	XFEATURE_BNDCSR,
112 	XFEATURE_OPMASK,
113 	XFEATURE_ZMM_Hi256,
114 	XFEATURE_Hi16_ZMM,
115 	XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
116 	XFEATURE_PKRU,
117 	XFEATURE_PASID,
118 	XFEATURE_RSRVD_COMP_11,
119 	XFEATURE_RSRVD_COMP_12,
120 	XFEATURE_RSRVD_COMP_13,
121 	XFEATURE_RSRVD_COMP_14,
122 	XFEATURE_LBR,
123 	XFEATURE_RSRVD_COMP_16,
124 	XFEATURE_XTILE_CFG,
125 	XFEATURE_XTILE_DATA,
126 
127 	XFEATURE_MAX,
128 };
129 
130 #define XFEATURE_MASK_FP		(1 << XFEATURE_FP)
131 #define XFEATURE_MASK_SSE		(1 << XFEATURE_SSE)
132 #define XFEATURE_MASK_YMM		(1 << XFEATURE_YMM)
133 #define XFEATURE_MASK_BNDREGS		(1 << XFEATURE_BNDREGS)
134 #define XFEATURE_MASK_BNDCSR		(1 << XFEATURE_BNDCSR)
135 #define XFEATURE_MASK_OPMASK		(1 << XFEATURE_OPMASK)
136 #define XFEATURE_MASK_ZMM_Hi256		(1 << XFEATURE_ZMM_Hi256)
137 #define XFEATURE_MASK_Hi16_ZMM		(1 << XFEATURE_Hi16_ZMM)
138 #define XFEATURE_MASK_PT		(1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
139 #define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
140 #define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
141 #define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
142 #define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
143 #define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
144 
145 #define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
146 #define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
147 					 | XFEATURE_MASK_ZMM_Hi256 \
148 					 | XFEATURE_MASK_Hi16_ZMM)
149 
150 #ifdef CONFIG_X86_64
151 # define XFEATURE_MASK_XTILE		(XFEATURE_MASK_XTILE_DATA \
152 					 | XFEATURE_MASK_XTILE_CFG)
153 #else
154 # define XFEATURE_MASK_XTILE		(0)
155 #endif
156 
157 #define FIRST_EXTENDED_XFEATURE	XFEATURE_YMM
158 
159 struct reg_128_bit {
160 	u8      regbytes[128/8];
161 };
162 struct reg_256_bit {
163 	u8	regbytes[256/8];
164 };
165 struct reg_512_bit {
166 	u8	regbytes[512/8];
167 };
168 struct reg_1024_byte {
169 	u8	regbytes[1024];
170 };
171 
172 /*
173  * State component 2:
174  *
175  * There are 16x 256-bit AVX registers named YMM0-YMM15.
176  * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
177  * and are stored in 'struct fxregs_state::xmm_space[]' in the
178  * "legacy" area.
179  *
180  * The high 128 bits are stored here.
181  */
182 struct ymmh_struct {
183 	struct reg_128_bit              hi_ymm[16];
184 } __packed;
185 
186 /* Intel MPX support: */
187 
188 struct mpx_bndreg {
189 	u64				lower_bound;
190 	u64				upper_bound;
191 } __packed;
192 /*
193  * State component 3 is used for the 4 128-bit bounds registers
194  */
195 struct mpx_bndreg_state {
196 	struct mpx_bndreg		bndreg[4];
197 } __packed;
198 
199 /*
200  * State component 4 is used for the 64-bit user-mode MPX
201  * configuration register BNDCFGU and the 64-bit MPX status
202  * register BNDSTATUS.  We call the pair "BNDCSR".
203  */
204 struct mpx_bndcsr {
205 	u64				bndcfgu;
206 	u64				bndstatus;
207 } __packed;
208 
209 /*
210  * The BNDCSR state is padded out to be 64-bytes in size.
211  */
212 struct mpx_bndcsr_state {
213 	union {
214 		struct mpx_bndcsr		bndcsr;
215 		u8				pad_to_64_bytes[64];
216 	};
217 } __packed;
218 
219 /* AVX-512 Components: */
220 
221 /*
222  * State component 5 is used for the 8 64-bit opmask registers
223  * k0-k7 (opmask state).
224  */
225 struct avx_512_opmask_state {
226 	u64				opmask_reg[8];
227 } __packed;
228 
229 /*
230  * State component 6 is used for the upper 256 bits of the
231  * registers ZMM0-ZMM15. These 16 256-bit values are denoted
232  * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
233  */
234 struct avx_512_zmm_uppers_state {
235 	struct reg_256_bit		zmm_upper[16];
236 } __packed;
237 
238 /*
239  * State component 7 is used for the 16 512-bit registers
240  * ZMM16-ZMM31 (Hi16_ZMM state).
241  */
242 struct avx_512_hi16_state {
243 	struct reg_512_bit		hi16_zmm[16];
244 } __packed;
245 
246 /*
247  * State component 9: 32-bit PKRU register.  The state is
248  * 8 bytes long but only 4 bytes is used currently.
249  */
250 struct pkru_state {
251 	u32				pkru;
252 	u32				pad;
253 } __packed;
254 
255 /*
256  * State component 15: Architectural LBR configuration state.
257  * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
258  */
259 
260 struct lbr_entry {
261 	u64 from;
262 	u64 to;
263 	u64 info;
264 };
265 
266 struct arch_lbr_state {
267 	u64 lbr_ctl;
268 	u64 lbr_depth;
269 	u64 ler_from;
270 	u64 ler_to;
271 	u64 ler_info;
272 	struct lbr_entry		entries[];
273 };
274 
275 /*
276  * State component 17: 64-byte tile configuration register.
277  */
278 struct xtile_cfg {
279 	u64				tcfg[8];
280 } __packed;
281 
282 /*
283  * State component 18: 1KB tile data register.
284  * Each register represents 16 64-byte rows of the matrix
285  * data. But the number of registers depends on the actual
286  * implementation.
287  */
288 struct xtile_data {
289 	struct reg_1024_byte		tmm;
290 } __packed;
291 
292 /*
293  * State component 10 is supervisor state used for context-switching the
294  * PASID state.
295  */
296 struct ia32_pasid_state {
297 	u64 pasid;
298 } __packed;
299 
300 struct xstate_header {
301 	u64				xfeatures;
302 	u64				xcomp_bv;
303 	u64				reserved[6];
304 } __attribute__((packed));
305 
306 /*
307  * xstate_header.xcomp_bv[63] indicates that the extended_state_area
308  * is in compacted format.
309  */
310 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
311 
312 /*
313  * This is our most modern FPU state format, as saved by the XSAVE
314  * and restored by the XRSTOR instructions.
315  *
316  * It consists of a legacy fxregs portion, an xstate header and
317  * subsequent areas as defined by the xstate header.  Not all CPUs
318  * support all the extensions, so the size of the extended area
319  * can vary quite a bit between CPUs.
320  */
321 struct xregs_state {
322 	struct fxregs_state		i387;
323 	struct xstate_header		header;
324 	u8				extended_state_area[0];
325 } __attribute__ ((packed, aligned (64)));
326 
327 /*
328  * This is a union of all the possible FPU state formats
329  * put together, so that we can pick the right one runtime.
330  *
331  * The size of the structure is determined by the largest
332  * member - which is the xsave area.  The padding is there
333  * to ensure that statically-allocated task_structs (just
334  * the init_task today) have enough space.
335  */
336 union fpregs_state {
337 	struct fregs_state		fsave;
338 	struct fxregs_state		fxsave;
339 	struct swregs_state		soft;
340 	struct xregs_state		xsave;
341 	u8 __padding[PAGE_SIZE];
342 };
343 
344 struct fpstate {
345 	/* @kernel_size: The size of the kernel register image */
346 	unsigned int		size;
347 
348 	/* @user_size: The size in non-compacted UABI format */
349 	unsigned int		user_size;
350 
351 	/* @xfeatures:		xfeatures for which the storage is sized */
352 	u64			xfeatures;
353 
354 	/* @user_xfeatures:	xfeatures valid in UABI buffers */
355 	u64			user_xfeatures;
356 
357 	/* @xfd:		xfeatures disabled to trap userspace use. */
358 	u64			xfd;
359 
360 	/* @is_valloc:		Indicator for dynamically allocated state */
361 	unsigned int		is_valloc	: 1;
362 
363 	/* @is_guest:		Indicator for guest state (KVM) */
364 	unsigned int		is_guest	: 1;
365 
366 	/*
367 	 * @is_confidential:	Indicator for KVM confidential mode.
368 	 *			The FPU registers are restored by the
369 	 *			vmentry firmware from encrypted guest
370 	 *			memory. On vmexit the FPU registers are
371 	 *			saved by firmware to encrypted guest memory
372 	 *			and the registers are scrubbed before
373 	 *			returning to the host. So there is no
374 	 *			content which is worth saving and restoring.
375 	 *			The fpstate has to be there so that
376 	 *			preemption and softirq FPU usage works
377 	 *			without special casing.
378 	 */
379 	unsigned int		is_confidential	: 1;
380 
381 	/* @in_use:		State is in use */
382 	unsigned int		in_use		: 1;
383 
384 	/* @regs: The register state union for all supported formats */
385 	union fpregs_state	regs;
386 
387 	/* @regs is dynamically sized! Don't add anything after @regs! */
388 } __aligned(64);
389 
390 #define FPU_GUEST_PERM_LOCKED		BIT_ULL(63)
391 
392 struct fpu_state_perm {
393 	/*
394 	 * @__state_perm:
395 	 *
396 	 * This bitmap indicates the permission for state components, which
397 	 * are available to a thread group. The permission prctl() sets the
398 	 * enabled state bits in thread_group_leader()->thread.fpu.
399 	 *
400 	 * All run time operations use the per thread information in the
401 	 * currently active fpu.fpstate which contains the xfeature masks
402 	 * and sizes for kernel and user space.
403 	 *
404 	 * This master permission field is only to be used when
405 	 * task.fpu.fpstate based checks fail to validate whether the task
406 	 * is allowed to expand it's xfeatures set which requires to
407 	 * allocate a larger sized fpstate buffer.
408 	 *
409 	 * Do not access this field directly.  Use the provided helper
410 	 * function. Unlocked access is possible for quick checks.
411 	 */
412 	u64				__state_perm;
413 
414 	/*
415 	 * @__state_size:
416 	 *
417 	 * The size required for @__state_perm. Only valid to access
418 	 * with sighand locked.
419 	 */
420 	unsigned int			__state_size;
421 
422 	/*
423 	 * @__user_state_size:
424 	 *
425 	 * The size required for @__state_perm user part. Only valid to
426 	 * access with sighand locked.
427 	 */
428 	unsigned int			__user_state_size;
429 };
430 
431 /*
432  * Highest level per task FPU state data structure that
433  * contains the FPU register state plus various FPU
434  * state fields:
435  */
436 struct fpu {
437 	/*
438 	 * @last_cpu:
439 	 *
440 	 * Records the last CPU on which this context was loaded into
441 	 * FPU registers. (In the lazy-restore case we might be
442 	 * able to reuse FPU registers across multiple context switches
443 	 * this way, if no intermediate task used the FPU.)
444 	 *
445 	 * A value of -1 is used to indicate that the FPU state in context
446 	 * memory is newer than the FPU state in registers, and that the
447 	 * FPU state should be reloaded next time the task is run.
448 	 */
449 	unsigned int			last_cpu;
450 
451 	/*
452 	 * @avx512_timestamp:
453 	 *
454 	 * Records the timestamp of AVX512 use during last context switch.
455 	 */
456 	unsigned long			avx512_timestamp;
457 
458 	/*
459 	 * @fpstate:
460 	 *
461 	 * Pointer to the active struct fpstate. Initialized to
462 	 * point at @__fpstate below.
463 	 */
464 	struct fpstate			*fpstate;
465 
466 	/*
467 	 * @__task_fpstate:
468 	 *
469 	 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
470 	 * used only for KVM support to swap out the regular task fpstate.
471 	 */
472 	struct fpstate			*__task_fpstate;
473 
474 	/*
475 	 * @perm:
476 	 *
477 	 * Permission related information
478 	 */
479 	struct fpu_state_perm		perm;
480 
481 	/*
482 	 * @guest_perm:
483 	 *
484 	 * Permission related information for guest pseudo FPUs
485 	 */
486 	struct fpu_state_perm		guest_perm;
487 
488 	/*
489 	 * @__fpstate:
490 	 *
491 	 * Initial in-memory storage for FPU registers which are saved in
492 	 * context switch and when the kernel uses the FPU. The registers
493 	 * are restored from this storage on return to user space if they
494 	 * are not longer containing the tasks FPU register state.
495 	 */
496 	struct fpstate			__fpstate;
497 	/*
498 	 * WARNING: '__fpstate' is dynamically-sized.  Do not put
499 	 * anything after it here.
500 	 */
501 };
502 
503 /*
504  * Guest pseudo FPU container
505  */
506 struct fpu_guest {
507 	/*
508 	 * @xfeatures:			xfeature bitmap of features which are
509 	 *				currently enabled for the guest vCPU.
510 	 */
511 	u64				xfeatures;
512 
513 	/*
514 	 * @perm:			xfeature bitmap of features which are
515 	 *				permitted to be enabled for the guest
516 	 *				vCPU.
517 	 */
518 	u64				perm;
519 
520 	/*
521 	 * @xfd_err:			Save the guest value.
522 	 */
523 	u64				xfd_err;
524 
525 	/*
526 	 * @uabi_size:			Size required for save/restore
527 	 */
528 	unsigned int			uabi_size;
529 
530 	/*
531 	 * @fpstate:			Pointer to the allocated guest fpstate
532 	 */
533 	struct fpstate			*fpstate;
534 };
535 
536 /*
537  * FPU state configuration data. Initialized at boot time. Read only after init.
538  */
539 struct fpu_state_config {
540 	/*
541 	 * @max_size:
542 	 *
543 	 * The maximum size of the register state buffer. Includes all
544 	 * supported features except independent managed features.
545 	 */
546 	unsigned int		max_size;
547 
548 	/*
549 	 * @default_size:
550 	 *
551 	 * The default size of the register state buffer. Includes all
552 	 * supported features except independent managed features and
553 	 * features which have to be requested by user space before usage.
554 	 */
555 	unsigned int		default_size;
556 
557 	/*
558 	 * @max_features:
559 	 *
560 	 * The maximum supported features bitmap. Does not include
561 	 * independent managed features.
562 	 */
563 	u64 max_features;
564 
565 	/*
566 	 * @default_features:
567 	 *
568 	 * The default supported features bitmap. Does not include
569 	 * independent managed features and features which have to
570 	 * be requested by user space before usage.
571 	 */
572 	u64 default_features;
573 	/*
574 	 * @legacy_features:
575 	 *
576 	 * Features which can be reported back to user space
577 	 * even without XSAVE support, i.e. legacy features FP + SSE
578 	 */
579 	u64 legacy_features;
580 };
581 
582 /* FPU state configuration information */
583 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
584 
585 #endif /* _ASM_X86_FPU_H */
586