53 static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input)  in fproduct_copy_from_wide_()  argument
57 		output[0] = ((u64)(xi));  in fproduct_copy_from_wide_()
61 		output[1] = ((u64)(xi));  in fproduct_copy_from_wide_()
65 		output[2] = ((u64)(xi));  in fproduct_copy_from_wide_()
69 		output[3] = ((u64)(xi));  in fproduct_copy_from_wide_()
73 		output[4] = ((u64)(xi));  in fproduct_copy_from_wide_()
78 fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s)  in fproduct_sum_scalar_multiplication_()  argument
80 	output[0] += (u128)input[0] * s;  in fproduct_sum_scalar_multiplication_()
81 	output[1] += (u128)input[1] * s;  in fproduct_sum_scalar_multiplication_()
82 	output[2] += (u128)input[2] * s;  in fproduct_sum_scalar_multiplication_()
83 	output[3] += (u128)input[3] * s;  in fproduct_sum_scalar_multiplication_()
84 	output[4] += (u128)input[4] * s;  in fproduct_sum_scalar_multiplication_()
128 static __always_inline void fmul_shift_reduce(u64 *output)  in fmul_shift_reduce()  argument
130 	u64 tmp = output[4];  in fmul_shift_reduce()
134 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
135 		output[ctr] = z;  in fmul_shift_reduce()
139 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
140 		output[ctr] = z;  in fmul_shift_reduce()
144 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
145 		output[ctr] = z;  in fmul_shift_reduce()
149 		u64 z = output[ctr - 1];  in fmul_shift_reduce()
150 		output[ctr] = z;  in fmul_shift_reduce()
152 	output[0] = tmp;  in fmul_shift_reduce()
153 	b0 = output[0];  in fmul_shift_reduce()
154 	output[0] = 19 * b0;  in fmul_shift_reduce()
157 static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input,  in fmul_mul_shift_reduce_()  argument
164 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
169 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
174 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
179 		fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
184 	fproduct_sum_scalar_multiplication_(output, input, input2i);  in fmul_mul_shift_reduce_()
187 static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21)  in fmul_fmul()  argument
208 		fproduct_copy_from_wide_(output, t);  in fmul_fmul()
209 		i0 = output[0];  in fmul_fmul()
210 		i1 = output[1];  in fmul_fmul()
213 		output[0] = i0_;  in fmul_fmul()
214 		output[1] = i1_;  in fmul_fmul()
218 static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output)  in fsquare_fsquare__()  argument
220 	u64 r0 = output[0];  in fsquare_fsquare__()
221 	u64 r1 = output[1];  in fsquare_fsquare__()
222 	u64 r2 = output[2];  in fsquare_fsquare__()
223 	u64 r3 = output[3];  in fsquare_fsquare__()
224 	u64 r4 = output[4];  in fsquare_fsquare__()
247 static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output)  in fsquare_fsquare_()  argument
257 	fsquare_fsquare__(tmp, output);  in fsquare_fsquare_()
265 	fproduct_copy_from_wide_(output, tmp);  in fsquare_fsquare_()
266 	i0 = output[0];  in fsquare_fsquare_()
267 	i1 = output[1];  in fsquare_fsquare_()
270 	output[0] = i0_;  in fsquare_fsquare_()
271 	output[1] = i1_;  in fsquare_fsquare_()
274 static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp,  in fsquare_fsquare_times_()  argument
278 	fsquare_fsquare_(tmp, output);  in fsquare_fsquare_times_()
280 		fsquare_fsquare_(tmp, output);  in fsquare_fsquare_times_()
283 static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input,  in fsquare_fsquare_times()  argument
287 	memcpy(output, input, 5 * sizeof(*input));  in fsquare_fsquare_times()
288 	fsquare_fsquare_times_(output, t, count1);  in fsquare_fsquare_times()
291 static __always_inline void fsquare_fsquare_times_inplace(u64 *output,  in fsquare_fsquare_times_inplace()  argument
295 	fsquare_fsquare_times_(output, t, count1);  in fsquare_fsquare_times_inplace()
397 static __always_inline void fscalar(u64 *output, u64 *b, u64 s)  in fscalar()  argument
431 	fproduct_copy_from_wide_(output, tmp);  in fscalar()
434 static __always_inline void fmul(u64 *output, u64 *a, u64 *b)  in fmul()  argument
436 	fmul_fmul(output, a, b);  in fmul()
439 static __always_inline void crecip(u64 *output, u64 *input)  in crecip()  argument
441 	crecip_crecip(output, input);  in crecip()
473 static __always_inline void point_copy(u64 *output, u64 *input)  in point_copy()  argument
475 	memcpy(output, input, 5 * sizeof(*input));  in point_copy()
476 	memcpy(output + 5, input + 5, 5 * sizeof(*input));  in point_copy()
603 static __always_inline void format_fexpand(u64 *output, const u8 *input)  in format_fexpand()  argument
620 	output[0] = output0;  in format_fexpand()
621 	output[1] = output1;  in format_fexpand()
622 	output[2] = output2;  in format_fexpand()
623 	output[3] = output3;  in format_fexpand()
624 	output[4] = output4;  in format_fexpand()
718 static __always_inline void format_fcontract_store(u8 *output, u64 *input)  in format_fcontract_store()  argument
729 	u8 *b0 = output;  in format_fcontract_store()
730 	u8 *b1 = output + 8;  in format_fcontract_store()
731 	u8 *b2 = output + 16;  in format_fcontract_store()
732 	u8 *b3 = output + 24;  in format_fcontract_store()
739 static __always_inline void format_fcontract(u8 *output, u64 *input)  in format_fcontract()  argument
744 	format_fcontract_store(output, input);  in format_fcontract()