44 struct fpstate init_fpstate __ro_after_init;
97 	if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK)  in update_avx_timestamp()
118 		os_xsave(fpu->fpstate);  in save_fpregs_to_fpstate()
124 		fxsave(&fpu->fpstate->regs.fxsave);  in save_fpregs_to_fpstate()
132 	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave));  in save_fpregs_to_fpstate()
133 	frstor(&fpu->fpstate->regs.fsave);  in save_fpregs_to_fpstate()
136 void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)  in restore_fpregs_from_fpstate()  argument
149 			: : [addr] "m" (fpstate));  in restore_fpregs_from_fpstate()
168 		xfd_update_state(fpstate);  in restore_fpregs_from_fpstate()
181 		os_xrstor(fpstate, mask);  in restore_fpregs_from_fpstate()
184 			fxrstor(&fpstate->regs.fxsave);  in restore_fpregs_from_fpstate()
186 			frstor(&fpstate->regs.fsave);  in restore_fpregs_from_fpstate()
196 static void __fpstate_reset(struct fpstate *fpstate, u64 xfd);
220 	struct fpstate *fpstate;  in fpu_alloc_guest_fpstate()  local
223 	size = fpu_user_cfg.default_size + ALIGN(offsetof(struct fpstate, regs), 64);  in fpu_alloc_guest_fpstate()
224 	fpstate = vzalloc(size);  in fpu_alloc_guest_fpstate()
225 	if (!fpstate)  in fpu_alloc_guest_fpstate()
229 	__fpstate_reset(fpstate, 0);  in fpu_alloc_guest_fpstate()
230 	fpstate_init_user(fpstate);  in fpu_alloc_guest_fpstate()
231 	fpstate->is_valloc	= true;  in fpu_alloc_guest_fpstate()
232 	fpstate->is_guest	= true;  in fpu_alloc_guest_fpstate()
234 	gfpu->fpstate		= fpstate;  in fpu_alloc_guest_fpstate()
259 	struct fpstate *fps = gfpu->fpstate;  in fpu_free_guest_fpstate()
267 	gfpu->fpstate = NULL;  in fpu_free_guest_fpstate()
298 	guest_fpu->fpstate->xfd = xfd;  in fpu_update_guest_xfd()
299 	if (guest_fpu->fpstate->in_use)  in fpu_update_guest_xfd()
300 		xfd_update_state(guest_fpu->fpstate);  in fpu_update_guest_xfd()
319 	struct fpstate *fps = current->thread.fpu.fpstate;  in fpu_sync_guest_vmexit_xfd_state()
332 	struct fpstate *guest_fps = guest_fpu->fpstate;  in fpu_swap_kvm_fpstate()
334 	struct fpstate *cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
343 		fpu->fpstate = guest_fps;  in fpu_swap_kvm_fpstate()
347 		fpu->fpstate = fpu->__task_fpstate;  in fpu_swap_kvm_fpstate()
351 	cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
374 	struct fpstate *kstate = gfpu->fpstate;  in fpu_copy_guest_fpstate_to_uabi()
393 	struct fpstate *kstate = gfpu->fpstate;  in fpu_copy_uabi_to_guest_fpstate()
482 static inline void fpstate_init_fxstate(struct fpstate *fpstate)  in fpstate_init_fxstate()  argument
484 	fpstate->regs.fxsave.cwd = 0x37f;  in fpstate_init_fxstate()
485 	fpstate->regs.fxsave.mxcsr = MXCSR_DEFAULT;  in fpstate_init_fxstate()
491 static inline void fpstate_init_fstate(struct fpstate *fpstate)  in fpstate_init_fstate()  argument
493 	fpstate->regs.fsave.cwd = 0xffff037fu;  in fpstate_init_fstate()
494 	fpstate->regs.fsave.swd = 0xffff0000u;  in fpstate_init_fstate()
495 	fpstate->regs.fsave.twd = 0xffffffffu;  in fpstate_init_fstate()
496 	fpstate->regs.fsave.fos = 0xffff0000u;  in fpstate_init_fstate()
504 void fpstate_init_user(struct fpstate *fpstate)  in fpstate_init_user()  argument
507 		fpstate_init_soft(&fpstate->regs.soft);  in fpstate_init_user()
511 	xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures);  in fpstate_init_user()
514 		fpstate_init_fxstate(fpstate);  in fpstate_init_user()
516 		fpstate_init_fstate(fpstate);  in fpstate_init_user()
519 static void __fpstate_reset(struct fpstate *fpstate, u64 xfd)  in __fpstate_reset()  argument
522 	fpstate->size		= fpu_kernel_cfg.default_size;  in __fpstate_reset()
523 	fpstate->user_size	= fpu_user_cfg.default_size;  in __fpstate_reset()
524 	fpstate->xfeatures	= fpu_kernel_cfg.default_features;  in __fpstate_reset()
525 	fpstate->user_xfeatures	= fpu_user_cfg.default_features;  in __fpstate_reset()
526 	fpstate->xfd		= xfd;  in __fpstate_reset()
532 	fpu->fpstate = &fpu->__fpstate;  in fpstate_reset()
533 	__fpstate_reset(fpu->fpstate, init_fpstate.xfd);  in fpstate_reset()
566 	xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,  in update_fpu_shstk()
610 		memcpy(&dst_fpu->fpstate->regs, &init_fpstate.regs,  in fpu_clone()
643 		dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;  in fpu_clone()
731 	memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size());  in fpu_reset_fpregs()
758 		os_xrstor_supervisor(fpu->fpstate);  in fpu__clear_user_states()
857 			cwd = fpu->fpstate->regs.fxsave.cwd;  in fpu__exception_code()
858 			swd = fpu->fpstate->regs.fxsave.swd;  in fpu__exception_code()
860 			cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd;  in fpu__exception_code()
861 			swd = (unsigned short)fpu->fpstate->regs.fsave.swd;  in fpu__exception_code()
875 			mxcsr = fpu->fpstate->regs.fxsave.mxcsr;  in fpu__exception_code()