1 // SPDX-License-Identifier: GPL-2.0-only
67  * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
87  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
92  * - the task gets scheduled in; if both the task's fpsimd_cpu field
93  *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
97  * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
103  * - the task executes an ordinary syscall; upon return to userland, the
107  * - the task executes a syscall which executes some NEON instructions; this is
109  *   register contents to memory, clears the fpsimd_last_state per-cpu variable
112  * - the task gets preempted after kernel_neon_end() is called; as we have not
125 static int __sve_default_vl = -1;
180  * The double-underscore version must only be called if you know the task
220 	kfree(task->thread.sve_state);  in __sve_free()
221 	task->thread.sve_state = NULL;  in __sve_free()
246  *    When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
247  *    corresponding Zn), P0-P15 and FFR are encoded in in
248  *    task->thread.sve_state, formatted appropriately for vector
249  *    length task->thread.sve_vl.
251  *    task->thread.sve_state must point to a valid buffer at least
263  *    When stored, FPSIMD registers V0-V31 are encoded in
264  *    task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
265  *    logically zero but not stored anywhere; P0-P15 and FFR are not
270  *    task->thread.sve_state does not need to be non-NULL, valid or any
273  *  * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
291 		sve_load_state(sve_pffr(&current->thread),  in task_fpsimd_load()
292 			       &current->thread.uw.fpsimd_state.fpsr,  in task_fpsimd_load()
293 			       sve_vq_from_vl(current->thread.sve_vl) - 1);  in task_fpsimd_load()
295 		fpsimd_load_state(&current->thread.uw.fpsimd_state);  in task_fpsimd_load()
314 			if (WARN_ON(sve_get_vl() != last->sve_vl)) {  in fpsimd_save()
317 				 * re-enter user with corrupt state.  in fpsimd_save()
324 			sve_save_state((char *)last->sve_state +  in fpsimd_save()
325 						sve_ffr_offset(last->sve_vl),  in fpsimd_save()
326 				       &last->st->fpsr);  in fpsimd_save()
328 			fpsimd_save_state(last->st);  in fpsimd_save()
334  * We're on a slow path, so some sanity-checks are included.
373 	/* Writing -1 has the special meaning "set to max": */  in sve_proc_do_default_vl()
374 	if (vl == -1)  in sve_proc_do_default_vl()
378 		return -EINVAL;  in sve_proc_do_default_vl()
397 			return -EINVAL;  in sve_sysctl_init()
407 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
434 		*p = arm64_cpu_to_le128(fst->vregs[i]);  in __fpsimd_to_sve()
439  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
440  * task->thread.sve_state.
442  * Task can be a non-runnable task, or current.  In the latter case,
445  * task->thread.sve_state must point to at least sve_state_size(task)
447  * task->thread.uw.fpsimd_state must be up to date before calling this
453 	void *sst = task->thread.sve_state;  in fpsimd_to_sve()
454 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in fpsimd_to_sve()
459 	vq = sve_vq_from_vl(task->thread.sve_vl);  in fpsimd_to_sve()
464  * Transfer the SVE state in task->thread.sve_state to
465  * task->thread.uw.fpsimd_state.
467  * Task can be a non-runnable task, or current.  In the latter case,
470  * task->thread.sve_state must point to at least sve_state_size(task)
472  * task->thread.sve_state must be up to date before calling this function.
477 	void const *sst = task->thread.sve_state;  in sve_to_fpsimd()
478 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;  in sve_to_fpsimd()
485 	vq = sve_vq_from_vl(task->thread.sve_vl);  in sve_to_fpsimd()
488 		fst->vregs[i] = arm64_le128_to_cpu(*p);  in sve_to_fpsimd()
500 	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));  in sve_state_size()
504  * Ensure that task->thread.sve_state is allocated and sufficiently large.
507  * task->thread.sve_state with new data.  The memory is always zeroed
515 	if (task->thread.sve_state) {  in sve_alloc()
516 		memset(task->thread.sve_state, 0, sve_state_size(task));  in sve_alloc()
521 	task->thread.sve_state =  in sve_alloc()
527  * Ensure that task->thread.sve_state is up to date with respect to
530  * This should only be called by ptrace.  task must be non-runnable.
531  * task->thread.sve_state must point to at least sve_state_size(task)
541  * Ensure that task->thread.uw.fpsimd_state is up to date with respect to
544  * This should only be called by ptrace.  task must be non-runnable.
545  * task->thread.sve_state must point to at least sve_state_size(task)
555  * Ensure that task->thread.sve_state is up to date with respect to
556  * the task->thread.uw.fpsimd_state.
560  * task must be non-runnable.
561  * task->thread.sve_state must point to at least sve_state_size(task)
563  * task->thread.uw.fpsimd_state must already have been initialised with
569 	void *sst = task->thread.sve_state;  in sve_sync_from_fpsimd_zeropad()
570 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;  in sve_sync_from_fpsimd_zeropad()
575 	vq = sve_vq_from_vl(task->thread.sve_vl);  in sve_sync_from_fpsimd_zeropad()
586 		return -EINVAL;  in sve_set_vector_length()
589 		return -EINVAL;  in sve_set_vector_length()
592 	 * Clamp to the maximum vector length that VL-agnostic SVE code can  in sve_set_vector_length()
603 		task->thread.sve_vl_onexec = vl;  in sve_set_vector_length()
605 		/* Reset VL to system default on next exec: */  in sve_set_vector_length()
606 		task->thread.sve_vl_onexec = 0;  in sve_set_vector_length()
612 	if (vl == task->thread.sve_vl)  in sve_set_vector_length()
618 	 * non-SVE thread.  in sve_set_vector_length()
639 	task->thread.sve_vl = vl;  in sve_set_vector_length()
659 		ret = current->thread.sve_vl_onexec;  in sve_prctl_status()
661 		ret = current->thread.sve_vl;  in sve_prctl_status()
679 		return -EINVAL;  in sve_set_current_vl()
692 		return -EINVAL;  in sve_get_current_vl()
707 	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {  in sve_probe_vqs()
708 		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */  in sve_probe_vqs()
728  * This function is called during the bring-up of early secondary CPUs only.
741  * This function is called during the bring-up of late secondary CPUs only.
754 		return -EINVAL;  in sve_verify_vq_map()
783 		return -EINVAL;  in sve_verify_vq_map()
824  * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
845 	zcr |= vq_max - 1; /* set LEN field to maximum effective value */  in read_zcr_features()
860 	 * The SVE architecture mandates support for 128-bit vectors,  in sve_setup()
871 	 * Sanity-check that the max VL we determined through CPU features  in sve_setup()
888 		/* No non-virtualisable VLs found */  in sve_setup()
890 	else if (WARN_ON(b == SVE_VQ_MAX - 1))  in sve_setup()
937 		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);  in do_sve_acc()
942 	if (!current->thread.sve_state) {  in do_sve_acc()
957 	 * disabling the trap, otherwise update our in-memory copy.  in do_sve_acc()
961 			sve_vq_from_vl(current->thread.sve_vl) - 1;  in do_sve_acc()
1024 					&next->thread.uw.fpsimd_state;  in fpsimd_thread_switch()
1025 	wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();  in fpsimd_thread_switch()
1043 	memset(&current->thread.uw.fpsimd_state, 0,  in fpsimd_flush_thread()
1044 	       sizeof(current->thread.uw.fpsimd_state));  in fpsimd_flush_thread()
1054 		 * task without an exec and hence a call to this function.  in fpsimd_flush_thread()
1061 		vl = current->thread.sve_vl_onexec ?  in fpsimd_flush_thread()
1062 			current->thread.sve_vl_onexec : get_sve_default_vl();  in fpsimd_flush_thread()
1071 		current->thread.sve_vl = vl;  in fpsimd_flush_thread()
1075 		 * length will be reset by a subsequent exec:  in fpsimd_flush_thread()
1078 			current->thread.sve_vl_onexec = 0;  in fpsimd_flush_thread()
1100  * current->thread.uw.fpsimd_state is updated so that it can be copied to
1121 	last->st = &current->thread.uw.fpsimd_state;  in fpsimd_bind_task_to_cpu()
1122 	last->sve_state = current->thread.sve_state;  in fpsimd_bind_task_to_cpu()
1123 	last->sve_vl = current->thread.sve_vl;  in fpsimd_bind_task_to_cpu()
1124 	current->thread.fpsimd_cpu = smp_processor_id();  in fpsimd_bind_task_to_cpu()
1146 	last->st = st;  in fpsimd_bind_state_to_cpu()
1147 	last->sve_state = sve_state;  in fpsimd_bind_state_to_cpu()
1148 	last->sve_vl = sve_vl;  in fpsimd_bind_state_to_cpu()
1194 	current->thread.uw.fpsimd_state = *state;  in fpsimd_update_current_state()
1212  * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1219 	t->thread.fpsimd_cpu = NR_CPUS;  in fpsimd_flush_task_state()
1263  * Kernel-side NEON support functions
1330  * Do not use them for any other purpose -- if tempted to do so, you are
1357 				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr);  in __efi_fpsimd_begin()
1382 				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr,  in __efi_fpsimd_end()
1383 				       sve_vq_from_vl(sve_get_vl()) - 1);  in __efi_fpsimd_end()
1452 		pr_notice("Floating-point is not implemented\n");  in fpsimd_init()