1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
4 *
5 * Copyright (c) 2018 Google, Inc
6 *
7 * Note: the NIST recommendation for XTS only specifies a 128-bit block size,
8 * but a 64-bit version (needed for Speck64) is fairly straightforward; the math
9 * is just done in GF(2^64) instead of GF(2^128), with the reducing polynomial
10 * x^64 + x^4 + x^3 + x + 1 from the original XEX paper (Rogaway, 2004:
11 * "Efficient Instantiations of Tweakable Blockciphers and Refinements to Modes
12 * OCB and PMAC"), represented as 0x1B.
13 */
14
15 #include <asm/hwcap.h>
16 #include <asm/neon.h>
17 #include <asm/simd.h>
18 #include <crypto/algapi.h>
19 #include <crypto/gf128mul.h>
20 #include <crypto/internal/skcipher.h>
21 #include <crypto/speck.h>
22 #include <crypto/xts.h>
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25
26 /* The assembly functions only handle multiples of 128 bytes */
27 #define SPECK_NEON_CHUNK_SIZE 128
28
29 /* Speck128 */
30
31 struct speck128_xts_tfm_ctx {
32 struct speck128_tfm_ctx main_key;
33 struct speck128_tfm_ctx tweak_key;
34 };
35
36 asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
37 void *dst, const void *src,
38 unsigned int nbytes, void *tweak);
39
40 asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
41 void *dst, const void *src,
42 unsigned int nbytes, void *tweak);
43
44 typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
45 u8 *, const u8 *);
46 typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
47 const void *, unsigned int, void *);
48
49 static __always_inline int
__speck128_xts_crypt(struct skcipher_request * req,speck128_crypt_one_t crypt_one,speck128_xts_crypt_many_t crypt_many)50 __speck128_xts_crypt(struct skcipher_request *req,
51 speck128_crypt_one_t crypt_one,
52 speck128_xts_crypt_many_t crypt_many)
53 {
54 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
55 const struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
56 struct skcipher_walk walk;
57 le128 tweak;
58 int err;
59
60 err = skcipher_walk_virt(&walk, req, true);
61
62 crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
63
64 while (walk.nbytes > 0) {
65 unsigned int nbytes = walk.nbytes;
66 u8 *dst = walk.dst.virt.addr;
67 const u8 *src = walk.src.virt.addr;
68
69 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
70 unsigned int count;
71
72 count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
73 kernel_neon_begin();
74 (*crypt_many)(ctx->main_key.round_keys,
75 ctx->main_key.nrounds,
76 dst, src, count, &tweak);
77 kernel_neon_end();
78 dst += count;
79 src += count;
80 nbytes -= count;
81 }
82
83 /* Handle any remainder with generic code */
84 while (nbytes >= sizeof(tweak)) {
85 le128_xor((le128 *)dst, (const le128 *)src, &tweak);
86 (*crypt_one)(&ctx->main_key, dst, dst);
87 le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
88 gf128mul_x_ble(&tweak, &tweak);
89
90 dst += sizeof(tweak);
91 src += sizeof(tweak);
92 nbytes -= sizeof(tweak);
93 }
94 err = skcipher_walk_done(&walk, nbytes);
95 }
96
97 return err;
98 }
99
speck128_xts_encrypt(struct skcipher_request * req)100 static int speck128_xts_encrypt(struct skcipher_request *req)
101 {
102 return __speck128_xts_crypt(req, crypto_speck128_encrypt,
103 speck128_xts_encrypt_neon);
104 }
105
speck128_xts_decrypt(struct skcipher_request * req)106 static int speck128_xts_decrypt(struct skcipher_request *req)
107 {
108 return __speck128_xts_crypt(req, crypto_speck128_decrypt,
109 speck128_xts_decrypt_neon);
110 }
111
speck128_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)112 static int speck128_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
113 unsigned int keylen)
114 {
115 struct speck128_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
116 int err;
117
118 err = xts_verify_key(tfm, key, keylen);
119 if (err)
120 return err;
121
122 keylen /= 2;
123
124 err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
125 if (err)
126 return err;
127
128 return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
129 }
130
131 /* Speck64 */
132
133 struct speck64_xts_tfm_ctx {
134 struct speck64_tfm_ctx main_key;
135 struct speck64_tfm_ctx tweak_key;
136 };
137
138 asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
139 void *dst, const void *src,
140 unsigned int nbytes, void *tweak);
141
142 asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
143 void *dst, const void *src,
144 unsigned int nbytes, void *tweak);
145
146 typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
147 u8 *, const u8 *);
148 typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
149 const void *, unsigned int, void *);
150
151 static __always_inline int
__speck64_xts_crypt(struct skcipher_request * req,speck64_crypt_one_t crypt_one,speck64_xts_crypt_many_t crypt_many)152 __speck64_xts_crypt(struct skcipher_request *req, speck64_crypt_one_t crypt_one,
153 speck64_xts_crypt_many_t crypt_many)
154 {
155 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
156 const struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
157 struct skcipher_walk walk;
158 __le64 tweak;
159 int err;
160
161 err = skcipher_walk_virt(&walk, req, true);
162
163 crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
164
165 while (walk.nbytes > 0) {
166 unsigned int nbytes = walk.nbytes;
167 u8 *dst = walk.dst.virt.addr;
168 const u8 *src = walk.src.virt.addr;
169
170 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
171 unsigned int count;
172
173 count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
174 kernel_neon_begin();
175 (*crypt_many)(ctx->main_key.round_keys,
176 ctx->main_key.nrounds,
177 dst, src, count, &tweak);
178 kernel_neon_end();
179 dst += count;
180 src += count;
181 nbytes -= count;
182 }
183
184 /* Handle any remainder with generic code */
185 while (nbytes >= sizeof(tweak)) {
186 *(__le64 *)dst = *(__le64 *)src ^ tweak;
187 (*crypt_one)(&ctx->main_key, dst, dst);
188 *(__le64 *)dst ^= tweak;
189 tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
190 ((tweak & cpu_to_le64(1ULL << 63)) ?
191 0x1B : 0));
192 dst += sizeof(tweak);
193 src += sizeof(tweak);
194 nbytes -= sizeof(tweak);
195 }
196 err = skcipher_walk_done(&walk, nbytes);
197 }
198
199 return err;
200 }
201
speck64_xts_encrypt(struct skcipher_request * req)202 static int speck64_xts_encrypt(struct skcipher_request *req)
203 {
204 return __speck64_xts_crypt(req, crypto_speck64_encrypt,
205 speck64_xts_encrypt_neon);
206 }
207
speck64_xts_decrypt(struct skcipher_request * req)208 static int speck64_xts_decrypt(struct skcipher_request *req)
209 {
210 return __speck64_xts_crypt(req, crypto_speck64_decrypt,
211 speck64_xts_decrypt_neon);
212 }
213
speck64_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)214 static int speck64_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
215 unsigned int keylen)
216 {
217 struct speck64_xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
218 int err;
219
220 err = xts_verify_key(tfm, key, keylen);
221 if (err)
222 return err;
223
224 keylen /= 2;
225
226 err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
227 if (err)
228 return err;
229
230 return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
231 }
232
233 static struct skcipher_alg speck_algs[] = {
234 {
235 .base.cra_name = "xts(speck128)",
236 .base.cra_driver_name = "xts-speck128-neon",
237 .base.cra_priority = 300,
238 .base.cra_blocksize = SPECK128_BLOCK_SIZE,
239 .base.cra_ctxsize = sizeof(struct speck128_xts_tfm_ctx),
240 .base.cra_alignmask = 7,
241 .base.cra_module = THIS_MODULE,
242 .min_keysize = 2 * SPECK128_128_KEY_SIZE,
243 .max_keysize = 2 * SPECK128_256_KEY_SIZE,
244 .ivsize = SPECK128_BLOCK_SIZE,
245 .walksize = SPECK_NEON_CHUNK_SIZE,
246 .setkey = speck128_xts_setkey,
247 .encrypt = speck128_xts_encrypt,
248 .decrypt = speck128_xts_decrypt,
249 }, {
250 .base.cra_name = "xts(speck64)",
251 .base.cra_driver_name = "xts-speck64-neon",
252 .base.cra_priority = 300,
253 .base.cra_blocksize = SPECK64_BLOCK_SIZE,
254 .base.cra_ctxsize = sizeof(struct speck64_xts_tfm_ctx),
255 .base.cra_alignmask = 7,
256 .base.cra_module = THIS_MODULE,
257 .min_keysize = 2 * SPECK64_96_KEY_SIZE,
258 .max_keysize = 2 * SPECK64_128_KEY_SIZE,
259 .ivsize = SPECK64_BLOCK_SIZE,
260 .walksize = SPECK_NEON_CHUNK_SIZE,
261 .setkey = speck64_xts_setkey,
262 .encrypt = speck64_xts_encrypt,
263 .decrypt = speck64_xts_decrypt,
264 }
265 };
266
speck_neon_module_init(void)267 static int __init speck_neon_module_init(void)
268 {
269 if (!(elf_hwcap & HWCAP_NEON))
270 return -ENODEV;
271 return crypto_register_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
272 }
273
speck_neon_module_exit(void)274 static void __exit speck_neon_module_exit(void)
275 {
276 crypto_unregister_skciphers(speck_algs, ARRAY_SIZE(speck_algs));
277 }
278
279 module_init(speck_neon_module_init);
280 module_exit(speck_neon_module_exit);
281
282 MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
283 MODULE_LICENSE("GPL");
284 MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
285 MODULE_ALIAS_CRYPTO("xts(speck128)");
286 MODULE_ALIAS_CRYPTO("xts-speck128-neon");
287 MODULE_ALIAS_CRYPTO("xts(speck64)");
288 MODULE_ALIAS_CRYPTO("xts-speck64-neon");
289
