44 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
95 int copy_fpregs_to_fpstate(struct fpu *fpu)  in copy_fpregs_to_fpstate()  argument
98 		copy_xregs_to_kernel(&fpu->state.xsave);  in copy_fpregs_to_fpstate()
104 		if (fpu->state.xsave.header.xfeatures & XFEATURE_MASK_AVX512)  in copy_fpregs_to_fpstate()
105 			fpu->avx512_timestamp = jiffies;  in copy_fpregs_to_fpstate()
110 		copy_fxregs_to_kernel(fpu);  in copy_fpregs_to_fpstate()
118 	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));  in copy_fpregs_to_fpstate()
140 		copy_fpregs_to_fpstate(&current->thread.fpu);  in kernel_fpu_begin()
166 void fpu__save(struct fpu *fpu)  in fpu__save()  argument
168 	WARN_ON_FPU(fpu != &current->thread.fpu);  in fpu__save()
171 	trace_x86_fpu_before_save(fpu);  in fpu__save()
174 		if (!copy_fpregs_to_fpstate(fpu)) {  in fpu__save()
175 			copy_kernel_to_fpregs(&fpu->state);  in fpu__save()
179 	trace_x86_fpu_after_save(fpu);  in fpu__save()
214 	struct fpu *dst_fpu = &dst->thread.fpu;  in fpu__copy()
215 	struct fpu *src_fpu = &src->thread.fpu;  in fpu__copy()
222 	WARN_ON_FPU(src_fpu != &current->thread.fpu);  in fpu__copy()
259 static void fpu__initialize(struct fpu *fpu)  in fpu__initialize()  argument
261 	WARN_ON_FPU(fpu != &current->thread.fpu);  in fpu__initialize()
264 	fpstate_init(&fpu->state);  in fpu__initialize()
265 	trace_x86_fpu_init_state(fpu);  in fpu__initialize()
282 void fpu__prepare_read(struct fpu *fpu)  in fpu__prepare_read()  argument
284 	if (fpu == &current->thread.fpu)  in fpu__prepare_read()
285 		fpu__save(fpu);  in fpu__prepare_read()
300 void fpu__prepare_write(struct fpu *fpu)  in fpu__prepare_write()  argument
306 	WARN_ON_FPU(fpu == &current->thread.fpu);  in fpu__prepare_write()
309 	__fpu_invalidate_fpregs_state(fpu);  in fpu__prepare_write()
321 void fpu__drop(struct fpu *fpu)  in fpu__drop()  argument
325 	if (fpu == &current->thread.fpu) {  in fpu__drop()
330 		fpregs_deactivate(fpu);  in fpu__drop()
333 	trace_x86_fpu_dropped(fpu);  in fpu__drop()
361 static void fpu__clear(struct fpu *fpu, bool user_only)  in fpu__clear()  argument
363 	WARN_ON_FPU(fpu != &current->thread.fpu);  in fpu__clear()
366 		fpu__drop(fpu);  in fpu__clear()
367 		fpu__initialize(fpu);  in fpu__clear()
374 		if (!fpregs_state_valid(fpu, smp_processor_id()) &&  in fpu__clear()
376 			copy_kernel_to_xregs(&fpu->state.xsave,  in fpu__clear()
387 void fpu__clear_user_states(struct fpu *fpu)  in fpu__clear_user_states()  argument
389 	fpu__clear(fpu, true);  in fpu__clear_user_states()
392 void fpu__clear_all(struct fpu *fpu)  in fpu__clear_all()  argument
394 	fpu__clear(fpu, false);  in fpu__clear_all()
417 	struct fpu *fpu = &current->thread.fpu;  in fpregs_assert_state_consistent()  local
422 	WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id()));  in fpregs_assert_state_consistent()
429 	struct fpu *fpu = &current->thread.fpu;  in fpregs_mark_activate()  local
431 	fpregs_activate(fpu);  in fpregs_mark_activate()
432 	fpu->last_cpu = smp_processor_id();  in fpregs_mark_activate()
441 int fpu__exception_code(struct fpu *fpu, int trap_nr)  in fpu__exception_code()  argument
458 			cwd = fpu->state.fxsave.cwd;  in fpu__exception_code()
459 			swd = fpu->state.fxsave.swd;  in fpu__exception_code()
461 			cwd = (unsigned short)fpu->state.fsave.cwd;  in fpu__exception_code()
462 			swd = (unsigned short)fpu->state.fsave.swd;  in fpu__exception_code()
476 			mxcsr = fpu->state.fxsave.mxcsr;  in fpu__exception_code()