44 struct fpstate init_fpstate __ro_after_init;
97 	if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK)  in update_avx_timestamp()
118 		os_xsave(fpu->fpstate);  in save_fpregs_to_fpstate()
124 		fxsave(&fpu->fpstate->regs.fxsave);  in save_fpregs_to_fpstate()
132 	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave));  in save_fpregs_to_fpstate()
133 	frstor(&fpu->fpstate->regs.fsave);  in save_fpregs_to_fpstate()
136 void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)  in restore_fpregs_from_fpstate()  argument
149 			: : [addr] "m" (fpstate));  in restore_fpregs_from_fpstate()
168 		xfd_update_state(fpstate);  in restore_fpregs_from_fpstate()
181 		os_xrstor(fpstate, mask);  in restore_fpregs_from_fpstate()
184 			fxrstor(&fpstate->regs.fxsave);  in restore_fpregs_from_fpstate()
186 			frstor(&fpstate->regs.fsave);  in restore_fpregs_from_fpstate()
196 static void __fpstate_reset(struct fpstate *fpstate, u64 xfd);
220 	struct fpstate *fpstate;  in fpu_alloc_guest_fpstate()  local
223 	size = fpu_user_cfg.default_size + ALIGN(offsetof(struct fpstate, regs), 64);  in fpu_alloc_guest_fpstate()
224 	fpstate = vzalloc(size);  in fpu_alloc_guest_fpstate()
225 	if (!fpstate)  in fpu_alloc_guest_fpstate()
229 	__fpstate_reset(fpstate, 0);  in fpu_alloc_guest_fpstate()
230 	fpstate_init_user(fpstate);  in fpu_alloc_guest_fpstate()
231 	fpstate->is_valloc	= true;  in fpu_alloc_guest_fpstate()
232 	fpstate->is_guest	= true;  in fpu_alloc_guest_fpstate()
234 	gfpu->fpstate		= fpstate;  in fpu_alloc_guest_fpstate()
259 	struct fpstate *fps = gfpu->fpstate;  in fpu_free_guest_fpstate()
267 	gfpu->fpstate = NULL;  in fpu_free_guest_fpstate()
298 	guest_fpu->fpstate->xfd = xfd;  in fpu_update_guest_xfd()
299 	if (guest_fpu->fpstate->in_use)  in fpu_update_guest_xfd()
300 		xfd_update_state(guest_fpu->fpstate);  in fpu_update_guest_xfd()
319 	struct fpstate *fps = current->thread.fpu.fpstate;  in fpu_sync_guest_vmexit_xfd_state()
332 	struct fpstate *guest_fps = guest_fpu->fpstate;  in fpu_swap_kvm_fpstate()
334 	struct fpstate *cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
343 		fpu->fpstate = guest_fps;  in fpu_swap_kvm_fpstate()
347 		fpu->fpstate = fpu->__task_fpstate;  in fpu_swap_kvm_fpstate()
351 	cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
374 	struct fpstate *kstate = gfpu->fpstate;  in fpu_copy_guest_fpstate_to_uabi()
392 	struct fpstate *kstate = gfpu->fpstate;  in fpu_copy_uabi_to_guest_fpstate()
481 static inline void fpstate_init_fxstate(struct fpstate *fpstate)  in fpstate_init_fxstate()  argument
483 	fpstate->regs.fxsave.cwd = 0x37f;  in fpstate_init_fxstate()
484 	fpstate->regs.fxsave.mxcsr = MXCSR_DEFAULT;  in fpstate_init_fxstate()
490 static inline void fpstate_init_fstate(struct fpstate *fpstate)  in fpstate_init_fstate()  argument
492 	fpstate->regs.fsave.cwd = 0xffff037fu;  in fpstate_init_fstate()
493 	fpstate->regs.fsave.swd = 0xffff0000u;  in fpstate_init_fstate()
494 	fpstate->regs.fsave.twd = 0xffffffffu;  in fpstate_init_fstate()
495 	fpstate->regs.fsave.fos = 0xffff0000u;  in fpstate_init_fstate()
503 void fpstate_init_user(struct fpstate *fpstate)  in fpstate_init_user()  argument
506 		fpstate_init_soft(&fpstate->regs.soft);  in fpstate_init_user()
510 	xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures);  in fpstate_init_user()
513 		fpstate_init_fxstate(fpstate);  in fpstate_init_user()
515 		fpstate_init_fstate(fpstate);  in fpstate_init_user()
518 static void __fpstate_reset(struct fpstate *fpstate, u64 xfd)  in __fpstate_reset()  argument
521 	fpstate->size		= fpu_kernel_cfg.default_size;  in __fpstate_reset()
522 	fpstate->user_size	= fpu_user_cfg.default_size;  in __fpstate_reset()
523 	fpstate->xfeatures	= fpu_kernel_cfg.default_features;  in __fpstate_reset()
524 	fpstate->user_xfeatures	= fpu_user_cfg.default_features;  in __fpstate_reset()
525 	fpstate->xfd		= xfd;  in __fpstate_reset()
531 	fpu->fpstate = &fpu->__fpstate;  in fpstate_reset()
532 	__fpstate_reset(fpu->fpstate, init_fpstate.xfd);  in fpstate_reset()
581 		memcpy(&dst_fpu->fpstate->regs, &init_fpstate.regs,  in fpu_clone()
614 		dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;  in fpu_clone()
696 	memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size());  in fpu_reset_fpregs()
723 		os_xrstor_supervisor(fpu->fpstate);  in fpu__clear_user_states()
804 			cwd = fpu->fpstate->regs.fxsave.cwd;  in fpu__exception_code()
805 			swd = fpu->fpstate->regs.fxsave.swd;  in fpu__exception_code()
807 			cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd;  in fpu__exception_code()
808 			swd = (unsigned short)fpu->fpstate->regs.fsave.swd;  in fpu__exception_code()
822 			mxcsr = fpu->fpstate->regs.fxsave.mxcsr;  in fpu__exception_code()