1 /*
2 * Cryptographic API.
3 *
4 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
5 * Supplemental SSE3 instructions.
6 *
7 * This file is based on sha1_generic.c
8 *
9 * Copyright (c) Alan Smithee.
10 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
11 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
12 * Copyright (c) Mathias Krause <minipli@googlemail.com>
13 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
14 *
15 * This program is free software; you can redistribute it and/or modify it
16 * under the terms of the GNU General Public License as published by the Free
17 * Software Foundation; either version 2 of the License, or (at your option)
18 * any later version.
19 *
20 */
21
22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23
24 #include <crypto/internal/hash.h>
25 #include <linux/init.h>
26 #include <linux/module.h>
27 #include <linux/mm.h>
28 #include <linux/cryptohash.h>
29 #include <linux/types.h>
30 #include <crypto/sha.h>
31 #include <crypto/sha1_base.h>
32 #include <asm/fpu/api.h>
33
34 typedef void (sha1_transform_fn)(u32 *digest, const char *data,
35 unsigned int rounds);
36
sha1_update(struct shash_desc * desc,const u8 * data,unsigned int len,sha1_transform_fn * sha1_xform)37 static int sha1_update(struct shash_desc *desc, const u8 *data,
38 unsigned int len, sha1_transform_fn *sha1_xform)
39 {
40 struct sha1_state *sctx = shash_desc_ctx(desc);
41
42 if (!irq_fpu_usable() ||
43 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
44 return crypto_sha1_update(desc, data, len);
45
46 /* make sure casting to sha1_block_fn() is safe */
47 BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
48
49 kernel_fpu_begin();
50 sha1_base_do_update(desc, data, len,
51 (sha1_block_fn *)sha1_xform);
52 kernel_fpu_end();
53
54 return 0;
55 }
56
sha1_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out,sha1_transform_fn * sha1_xform)57 static int sha1_finup(struct shash_desc *desc, const u8 *data,
58 unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
59 {
60 if (!irq_fpu_usable())
61 return crypto_sha1_finup(desc, data, len, out);
62
63 kernel_fpu_begin();
64 if (len)
65 sha1_base_do_update(desc, data, len,
66 (sha1_block_fn *)sha1_xform);
67 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
68 kernel_fpu_end();
69
70 return sha1_base_finish(desc, out);
71 }
72
73 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
74 unsigned int rounds);
75
sha1_ssse3_update(struct shash_desc * desc,const u8 * data,unsigned int len)76 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
77 unsigned int len)
78 {
79 return sha1_update(desc, data, len,
80 (sha1_transform_fn *) sha1_transform_ssse3);
81 }
82
sha1_ssse3_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)83 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
84 unsigned int len, u8 *out)
85 {
86 return sha1_finup(desc, data, len, out,
87 (sha1_transform_fn *) sha1_transform_ssse3);
88 }
89
90 /* Add padding and return the message digest. */
sha1_ssse3_final(struct shash_desc * desc,u8 * out)91 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
92 {
93 return sha1_ssse3_finup(desc, NULL, 0, out);
94 }
95
96 static struct shash_alg sha1_ssse3_alg = {
97 .digestsize = SHA1_DIGEST_SIZE,
98 .init = sha1_base_init,
99 .update = sha1_ssse3_update,
100 .final = sha1_ssse3_final,
101 .finup = sha1_ssse3_finup,
102 .descsize = sizeof(struct sha1_state),
103 .base = {
104 .cra_name = "sha1",
105 .cra_driver_name = "sha1-ssse3",
106 .cra_priority = 150,
107 .cra_blocksize = SHA1_BLOCK_SIZE,
108 .cra_module = THIS_MODULE,
109 }
110 };
111
register_sha1_ssse3(void)112 static int register_sha1_ssse3(void)
113 {
114 if (boot_cpu_has(X86_FEATURE_SSSE3))
115 return crypto_register_shash(&sha1_ssse3_alg);
116 return 0;
117 }
118
unregister_sha1_ssse3(void)119 static void unregister_sha1_ssse3(void)
120 {
121 if (boot_cpu_has(X86_FEATURE_SSSE3))
122 crypto_unregister_shash(&sha1_ssse3_alg);
123 }
124
125 #ifdef CONFIG_AS_AVX
126 asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
127 unsigned int rounds);
128
sha1_avx_update(struct shash_desc * desc,const u8 * data,unsigned int len)129 static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
130 unsigned int len)
131 {
132 return sha1_update(desc, data, len,
133 (sha1_transform_fn *) sha1_transform_avx);
134 }
135
sha1_avx_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)136 static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
137 unsigned int len, u8 *out)
138 {
139 return sha1_finup(desc, data, len, out,
140 (sha1_transform_fn *) sha1_transform_avx);
141 }
142
sha1_avx_final(struct shash_desc * desc,u8 * out)143 static int sha1_avx_final(struct shash_desc *desc, u8 *out)
144 {
145 return sha1_avx_finup(desc, NULL, 0, out);
146 }
147
148 static struct shash_alg sha1_avx_alg = {
149 .digestsize = SHA1_DIGEST_SIZE,
150 .init = sha1_base_init,
151 .update = sha1_avx_update,
152 .final = sha1_avx_final,
153 .finup = sha1_avx_finup,
154 .descsize = sizeof(struct sha1_state),
155 .base = {
156 .cra_name = "sha1",
157 .cra_driver_name = "sha1-avx",
158 .cra_priority = 160,
159 .cra_blocksize = SHA1_BLOCK_SIZE,
160 .cra_module = THIS_MODULE,
161 }
162 };
163
avx_usable(void)164 static bool avx_usable(void)
165 {
166 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
167 if (boot_cpu_has(X86_FEATURE_AVX))
168 pr_info("AVX detected but unusable.\n");
169 return false;
170 }
171
172 return true;
173 }
174
register_sha1_avx(void)175 static int register_sha1_avx(void)
176 {
177 if (avx_usable())
178 return crypto_register_shash(&sha1_avx_alg);
179 return 0;
180 }
181
unregister_sha1_avx(void)182 static void unregister_sha1_avx(void)
183 {
184 if (avx_usable())
185 crypto_unregister_shash(&sha1_avx_alg);
186 }
187
188 #else /* CONFIG_AS_AVX */
register_sha1_avx(void)189 static inline int register_sha1_avx(void) { return 0; }
unregister_sha1_avx(void)190 static inline void unregister_sha1_avx(void) { }
191 #endif /* CONFIG_AS_AVX */
192
193
194 #if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
195 #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */
196
197 asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
198 unsigned int rounds);
199
avx2_usable(void)200 static bool avx2_usable(void)
201 {
202 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
203 && boot_cpu_has(X86_FEATURE_BMI1)
204 && boot_cpu_has(X86_FEATURE_BMI2))
205 return true;
206
207 return false;
208 }
209
sha1_apply_transform_avx2(u32 * digest,const char * data,unsigned int rounds)210 static void sha1_apply_transform_avx2(u32 *digest, const char *data,
211 unsigned int rounds)
212 {
213 /* Select the optimal transform based on data block size */
214 if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
215 sha1_transform_avx2(digest, data, rounds);
216 else
217 sha1_transform_avx(digest, data, rounds);
218 }
219
sha1_avx2_update(struct shash_desc * desc,const u8 * data,unsigned int len)220 static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
221 unsigned int len)
222 {
223 return sha1_update(desc, data, len,
224 (sha1_transform_fn *) sha1_apply_transform_avx2);
225 }
226
sha1_avx2_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)227 static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
228 unsigned int len, u8 *out)
229 {
230 return sha1_finup(desc, data, len, out,
231 (sha1_transform_fn *) sha1_apply_transform_avx2);
232 }
233
sha1_avx2_final(struct shash_desc * desc,u8 * out)234 static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
235 {
236 return sha1_avx2_finup(desc, NULL, 0, out);
237 }
238
239 static struct shash_alg sha1_avx2_alg = {
240 .digestsize = SHA1_DIGEST_SIZE,
241 .init = sha1_base_init,
242 .update = sha1_avx2_update,
243 .final = sha1_avx2_final,
244 .finup = sha1_avx2_finup,
245 .descsize = sizeof(struct sha1_state),
246 .base = {
247 .cra_name = "sha1",
248 .cra_driver_name = "sha1-avx2",
249 .cra_priority = 170,
250 .cra_blocksize = SHA1_BLOCK_SIZE,
251 .cra_module = THIS_MODULE,
252 }
253 };
254
register_sha1_avx2(void)255 static int register_sha1_avx2(void)
256 {
257 if (avx2_usable())
258 return crypto_register_shash(&sha1_avx2_alg);
259 return 0;
260 }
261
unregister_sha1_avx2(void)262 static void unregister_sha1_avx2(void)
263 {
264 if (avx2_usable())
265 crypto_unregister_shash(&sha1_avx2_alg);
266 }
267
268 #else
register_sha1_avx2(void)269 static inline int register_sha1_avx2(void) { return 0; }
unregister_sha1_avx2(void)270 static inline void unregister_sha1_avx2(void) { }
271 #endif
272
273 #ifdef CONFIG_AS_SHA1_NI
274 asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
275 unsigned int rounds);
276
sha1_ni_update(struct shash_desc * desc,const u8 * data,unsigned int len)277 static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
278 unsigned int len)
279 {
280 return sha1_update(desc, data, len,
281 (sha1_transform_fn *) sha1_ni_transform);
282 }
283
sha1_ni_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)284 static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
285 unsigned int len, u8 *out)
286 {
287 return sha1_finup(desc, data, len, out,
288 (sha1_transform_fn *) sha1_ni_transform);
289 }
290
sha1_ni_final(struct shash_desc * desc,u8 * out)291 static int sha1_ni_final(struct shash_desc *desc, u8 *out)
292 {
293 return sha1_ni_finup(desc, NULL, 0, out);
294 }
295
296 static struct shash_alg sha1_ni_alg = {
297 .digestsize = SHA1_DIGEST_SIZE,
298 .init = sha1_base_init,
299 .update = sha1_ni_update,
300 .final = sha1_ni_final,
301 .finup = sha1_ni_finup,
302 .descsize = sizeof(struct sha1_state),
303 .base = {
304 .cra_name = "sha1",
305 .cra_driver_name = "sha1-ni",
306 .cra_priority = 250,
307 .cra_blocksize = SHA1_BLOCK_SIZE,
308 .cra_module = THIS_MODULE,
309 }
310 };
311
register_sha1_ni(void)312 static int register_sha1_ni(void)
313 {
314 if (boot_cpu_has(X86_FEATURE_SHA_NI))
315 return crypto_register_shash(&sha1_ni_alg);
316 return 0;
317 }
318
unregister_sha1_ni(void)319 static void unregister_sha1_ni(void)
320 {
321 if (boot_cpu_has(X86_FEATURE_SHA_NI))
322 crypto_unregister_shash(&sha1_ni_alg);
323 }
324
325 #else
register_sha1_ni(void)326 static inline int register_sha1_ni(void) { return 0; }
unregister_sha1_ni(void)327 static inline void unregister_sha1_ni(void) { }
328 #endif
329
sha1_ssse3_mod_init(void)330 static int __init sha1_ssse3_mod_init(void)
331 {
332 if (register_sha1_ssse3())
333 goto fail;
334
335 if (register_sha1_avx()) {
336 unregister_sha1_ssse3();
337 goto fail;
338 }
339
340 if (register_sha1_avx2()) {
341 unregister_sha1_avx();
342 unregister_sha1_ssse3();
343 goto fail;
344 }
345
346 if (register_sha1_ni()) {
347 unregister_sha1_avx2();
348 unregister_sha1_avx();
349 unregister_sha1_ssse3();
350 goto fail;
351 }
352
353 return 0;
354 fail:
355 return -ENODEV;
356 }
357
sha1_ssse3_mod_fini(void)358 static void __exit sha1_ssse3_mod_fini(void)
359 {
360 unregister_sha1_ni();
361 unregister_sha1_avx2();
362 unregister_sha1_avx();
363 unregister_sha1_ssse3();
364 }
365
366 module_init(sha1_ssse3_mod_init);
367 module_exit(sha1_ssse3_mod_fini);
368
369 MODULE_LICENSE("GPL");
370 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
371
372 MODULE_ALIAS_CRYPTO("sha1");
373 MODULE_ALIAS_CRYPTO("sha1-ssse3");
374 MODULE_ALIAS_CRYPTO("sha1-avx");
375 MODULE_ALIAS_CRYPTO("sha1-avx2");
376 #ifdef CONFIG_AS_SHA1_NI
377 MODULE_ALIAS_CRYPTO("sha1-ni");
378 #endif
379
