1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Glue Code for AVX assembler version of Twofish Cipher
4 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 *
8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 */
10
11 #include <linux/module.h>
12 #include <linux/types.h>
13 #include <linux/crypto.h>
14 #include <linux/err.h>
15 #include <crypto/algapi.h>
16 #include <crypto/internal/simd.h>
17 #include <crypto/twofish.h>
18 #include <crypto/xts.h>
19 #include <asm/crypto/glue_helper.h>
20 #include <asm/crypto/twofish.h>
21
22 #define TWOFISH_PARALLEL_BLOCKS 8
23
24 /* 8-way parallel cipher functions */
25 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
26 const u8 *src);
27 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
28 const u8 *src);
29
30 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
31 const u8 *src);
32 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
33 const u8 *src, le128 *iv);
34
35 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
36 const u8 *src, le128 *iv);
37 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
38 const u8 *src, le128 *iv);
39
twofish_setkey_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)40 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
41 const u8 *key, unsigned int keylen)
42 {
43 return twofish_setkey(&tfm->base, key, keylen);
44 }
45
twofish_enc_blk_3way(struct twofish_ctx * ctx,u8 * dst,const u8 * src)46 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
47 const u8 *src)
48 {
49 __twofish_enc_blk_3way(ctx, dst, src, false);
50 }
51
twofish_xts_enc(void * ctx,u128 * dst,const u128 * src,le128 * iv)52 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
53 {
54 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
55 GLUE_FUNC_CAST(twofish_enc_blk));
56 }
57
twofish_xts_dec(void * ctx,u128 * dst,const u128 * src,le128 * iv)58 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
59 {
60 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
61 GLUE_FUNC_CAST(twofish_dec_blk));
62 }
63
64 struct twofish_xts_ctx {
65 struct twofish_ctx tweak_ctx;
66 struct twofish_ctx crypt_ctx;
67 };
68
xts_twofish_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)69 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
70 unsigned int keylen)
71 {
72 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
73 u32 *flags = &tfm->base.crt_flags;
74 int err;
75
76 err = xts_verify_key(tfm, key, keylen);
77 if (err)
78 return err;
79
80 /* first half of xts-key is for crypt */
81 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
82 if (err)
83 return err;
84
85 /* second half of xts-key is for tweak */
86 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
87 flags);
88 }
89
90 static const struct common_glue_ctx twofish_enc = {
91 .num_funcs = 3,
92 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
93
94 .funcs = { {
95 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
96 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
97 }, {
98 .num_blocks = 3,
99 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
100 }, {
101 .num_blocks = 1,
102 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
103 } }
104 };
105
106 static const struct common_glue_ctx twofish_ctr = {
107 .num_funcs = 3,
108 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
109
110 .funcs = { {
111 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
112 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
113 }, {
114 .num_blocks = 3,
115 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
116 }, {
117 .num_blocks = 1,
118 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
119 } }
120 };
121
122 static const struct common_glue_ctx twofish_enc_xts = {
123 .num_funcs = 2,
124 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
125
126 .funcs = { {
127 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
128 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
129 }, {
130 .num_blocks = 1,
131 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
132 } }
133 };
134
135 static const struct common_glue_ctx twofish_dec = {
136 .num_funcs = 3,
137 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
138
139 .funcs = { {
140 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
141 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
142 }, {
143 .num_blocks = 3,
144 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
145 }, {
146 .num_blocks = 1,
147 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
148 } }
149 };
150
151 static const struct common_glue_ctx twofish_dec_cbc = {
152 .num_funcs = 3,
153 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
154
155 .funcs = { {
156 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
157 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
158 }, {
159 .num_blocks = 3,
160 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
161 }, {
162 .num_blocks = 1,
163 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
164 } }
165 };
166
167 static const struct common_glue_ctx twofish_dec_xts = {
168 .num_funcs = 2,
169 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
170
171 .funcs = { {
172 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
173 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
174 }, {
175 .num_blocks = 1,
176 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
177 } }
178 };
179
ecb_encrypt(struct skcipher_request * req)180 static int ecb_encrypt(struct skcipher_request *req)
181 {
182 return glue_ecb_req_128bit(&twofish_enc, req);
183 }
184
ecb_decrypt(struct skcipher_request * req)185 static int ecb_decrypt(struct skcipher_request *req)
186 {
187 return glue_ecb_req_128bit(&twofish_dec, req);
188 }
189
cbc_encrypt(struct skcipher_request * req)190 static int cbc_encrypt(struct skcipher_request *req)
191 {
192 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
193 req);
194 }
195
cbc_decrypt(struct skcipher_request * req)196 static int cbc_decrypt(struct skcipher_request *req)
197 {
198 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
199 }
200
ctr_crypt(struct skcipher_request * req)201 static int ctr_crypt(struct skcipher_request *req)
202 {
203 return glue_ctr_req_128bit(&twofish_ctr, req);
204 }
205
xts_encrypt(struct skcipher_request * req)206 static int xts_encrypt(struct skcipher_request *req)
207 {
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
210
211 return glue_xts_req_128bit(&twofish_enc_xts, req,
212 XTS_TWEAK_CAST(twofish_enc_blk),
213 &ctx->tweak_ctx, &ctx->crypt_ctx, false);
214 }
215
xts_decrypt(struct skcipher_request * req)216 static int xts_decrypt(struct skcipher_request *req)
217 {
218 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
219 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
220
221 return glue_xts_req_128bit(&twofish_dec_xts, req,
222 XTS_TWEAK_CAST(twofish_enc_blk),
223 &ctx->tweak_ctx, &ctx->crypt_ctx, true);
224 }
225
226 static struct skcipher_alg twofish_algs[] = {
227 {
228 .base.cra_name = "__ecb(twofish)",
229 .base.cra_driver_name = "__ecb-twofish-avx",
230 .base.cra_priority = 400,
231 .base.cra_flags = CRYPTO_ALG_INTERNAL,
232 .base.cra_blocksize = TF_BLOCK_SIZE,
233 .base.cra_ctxsize = sizeof(struct twofish_ctx),
234 .base.cra_module = THIS_MODULE,
235 .min_keysize = TF_MIN_KEY_SIZE,
236 .max_keysize = TF_MAX_KEY_SIZE,
237 .setkey = twofish_setkey_skcipher,
238 .encrypt = ecb_encrypt,
239 .decrypt = ecb_decrypt,
240 }, {
241 .base.cra_name = "__cbc(twofish)",
242 .base.cra_driver_name = "__cbc-twofish-avx",
243 .base.cra_priority = 400,
244 .base.cra_flags = CRYPTO_ALG_INTERNAL,
245 .base.cra_blocksize = TF_BLOCK_SIZE,
246 .base.cra_ctxsize = sizeof(struct twofish_ctx),
247 .base.cra_module = THIS_MODULE,
248 .min_keysize = TF_MIN_KEY_SIZE,
249 .max_keysize = TF_MAX_KEY_SIZE,
250 .ivsize = TF_BLOCK_SIZE,
251 .setkey = twofish_setkey_skcipher,
252 .encrypt = cbc_encrypt,
253 .decrypt = cbc_decrypt,
254 }, {
255 .base.cra_name = "__ctr(twofish)",
256 .base.cra_driver_name = "__ctr-twofish-avx",
257 .base.cra_priority = 400,
258 .base.cra_flags = CRYPTO_ALG_INTERNAL,
259 .base.cra_blocksize = 1,
260 .base.cra_ctxsize = sizeof(struct twofish_ctx),
261 .base.cra_module = THIS_MODULE,
262 .min_keysize = TF_MIN_KEY_SIZE,
263 .max_keysize = TF_MAX_KEY_SIZE,
264 .ivsize = TF_BLOCK_SIZE,
265 .chunksize = TF_BLOCK_SIZE,
266 .setkey = twofish_setkey_skcipher,
267 .encrypt = ctr_crypt,
268 .decrypt = ctr_crypt,
269 }, {
270 .base.cra_name = "__xts(twofish)",
271 .base.cra_driver_name = "__xts-twofish-avx",
272 .base.cra_priority = 400,
273 .base.cra_flags = CRYPTO_ALG_INTERNAL,
274 .base.cra_blocksize = TF_BLOCK_SIZE,
275 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
276 .base.cra_module = THIS_MODULE,
277 .min_keysize = 2 * TF_MIN_KEY_SIZE,
278 .max_keysize = 2 * TF_MAX_KEY_SIZE,
279 .ivsize = TF_BLOCK_SIZE,
280 .setkey = xts_twofish_setkey,
281 .encrypt = xts_encrypt,
282 .decrypt = xts_decrypt,
283 },
284 };
285
286 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
287
twofish_init(void)288 static int __init twofish_init(void)
289 {
290 const char *feature_name;
291
292 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
293 pr_info("CPU feature '%s' is not supported.\n", feature_name);
294 return -ENODEV;
295 }
296
297 return simd_register_skciphers_compat(twofish_algs,
298 ARRAY_SIZE(twofish_algs),
299 twofish_simd_algs);
300 }
301
twofish_exit(void)302 static void __exit twofish_exit(void)
303 {
304 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
305 twofish_simd_algs);
306 }
307
308 module_init(twofish_init);
309 module_exit(twofish_exit);
310
311 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
312 MODULE_LICENSE("GPL");
313 MODULE_ALIAS_CRYPTO("twofish");
314
