1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Crypto-API module for CRC-32 algorithms implemented with the
4 * z/Architecture Vector Extension Facility.
5 *
6 * Copyright IBM Corp. 2015
7 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8 */
9 #define KMSG_COMPONENT "crc32-vx"
10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11
12 #include <linux/module.h>
13 #include <linux/cpufeature.h>
14 #include <linux/crc32.h>
15 #include <crypto/internal/hash.h>
16 #include <asm/fpu/api.h>
17
18
19 #define CRC32_BLOCK_SIZE 1
20 #define CRC32_DIGEST_SIZE 4
21
22 #define VX_MIN_LEN 64
23 #define VX_ALIGNMENT 16L
24 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
25
26 struct crc_ctx {
27 u32 key;
28 };
29
30 struct crc_desc_ctx {
31 u32 crc;
32 };
33
34 /* Prototypes for functions in assembly files */
35 u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
36 u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
37 u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
38
39 /*
40 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
41 *
42 * Creates a function to perform a particular CRC-32 computation. Depending
43 * on the message buffer, the hardware-accelerated or software implementation
44 * is used. Note that the message buffer is aligned to improve fetch
45 * operations of VECTOR LOAD MULTIPLE instructions.
46 *
47 */
48 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
49 static u32 __pure ___fname(u32 crc, \
50 unsigned char const *data, size_t datalen) \
51 { \
52 struct kernel_fpu vxstate; \
53 unsigned long prealign, aligned, remaining; \
54 \
55 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
56 return ___crc32_sw(crc, data, datalen); \
57 \
58 if ((unsigned long)data & VX_ALIGN_MASK) { \
59 prealign = VX_ALIGNMENT - \
60 ((unsigned long)data & VX_ALIGN_MASK); \
61 datalen -= prealign; \
62 crc = ___crc32_sw(crc, data, prealign); \
63 data = (void *)((unsigned long)data + prealign); \
64 } \
65 \
66 aligned = datalen & ~VX_ALIGN_MASK; \
67 remaining = datalen & VX_ALIGN_MASK; \
68 \
69 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
70 crc = ___crc32_vx(crc, data, aligned); \
71 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
72 \
73 if (remaining) \
74 crc = ___crc32_sw(crc, data + aligned, remaining); \
75 \
76 return crc; \
77 }
78
DEFINE_CRC32_VX(crc32_le_vx,crc32_le_vgfm_16,crc32_le)79 DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
80 DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
81 DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
82
83
84 static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
85 {
86 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
87
88 mctx->key = 0;
89 return 0;
90 }
91
crc32_vx_cra_init_invert(struct crypto_tfm * tfm)92 static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
93 {
94 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
95
96 mctx->key = ~0;
97 return 0;
98 }
99
crc32_vx_init(struct shash_desc * desc)100 static int crc32_vx_init(struct shash_desc *desc)
101 {
102 struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
103 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
104
105 ctx->crc = mctx->key;
106 return 0;
107 }
108
crc32_vx_setkey(struct crypto_shash * tfm,const u8 * newkey,unsigned int newkeylen)109 static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
110 unsigned int newkeylen)
111 {
112 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
113
114 if (newkeylen != sizeof(mctx->key)) {
115 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
116 return -EINVAL;
117 }
118 mctx->key = le32_to_cpu(*(__le32 *)newkey);
119 return 0;
120 }
121
crc32be_vx_setkey(struct crypto_shash * tfm,const u8 * newkey,unsigned int newkeylen)122 static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
123 unsigned int newkeylen)
124 {
125 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
126
127 if (newkeylen != sizeof(mctx->key)) {
128 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
129 return -EINVAL;
130 }
131 mctx->key = be32_to_cpu(*(__be32 *)newkey);
132 return 0;
133 }
134
crc32le_vx_final(struct shash_desc * desc,u8 * out)135 static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
136 {
137 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
138
139 *(__le32 *)out = cpu_to_le32p(&ctx->crc);
140 return 0;
141 }
142
crc32be_vx_final(struct shash_desc * desc,u8 * out)143 static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
144 {
145 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
146
147 *(__be32 *)out = cpu_to_be32p(&ctx->crc);
148 return 0;
149 }
150
crc32c_vx_final(struct shash_desc * desc,u8 * out)151 static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
152 {
153 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
154
155 /*
156 * Perform a final XOR with 0xFFFFFFFF to be in sync
157 * with the generic crc32c shash implementation.
158 */
159 *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
160 return 0;
161 }
162
__crc32le_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)163 static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
164 u8 *out)
165 {
166 *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
167 return 0;
168 }
169
__crc32be_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)170 static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
171 u8 *out)
172 {
173 *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
174 return 0;
175 }
176
__crc32c_vx_finup(u32 * crc,const u8 * data,unsigned int len,u8 * out)177 static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
178 u8 *out)
179 {
180 /*
181 * Perform a final XOR with 0xFFFFFFFF to be in sync
182 * with the generic crc32c shash implementation.
183 */
184 *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
185 return 0;
186 }
187
188
189 #define CRC32_VX_FINUP(alg, func) \
190 static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
191 unsigned int datalen, u8 *out) \
192 { \
193 return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
194 data, datalen, out); \
195 }
196
197 CRC32_VX_FINUP(crc32le, crc32_le_vx)
198 CRC32_VX_FINUP(crc32be, crc32_be_vx)
199 CRC32_VX_FINUP(crc32c, crc32c_le_vx)
200
201 #define CRC32_VX_DIGEST(alg, func) \
202 static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
203 unsigned int len, u8 *out) \
204 { \
205 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
206 data, len, out); \
207 }
208
209 CRC32_VX_DIGEST(crc32le, crc32_le_vx)
210 CRC32_VX_DIGEST(crc32be, crc32_be_vx)
211 CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
212
213 #define CRC32_VX_UPDATE(alg, func) \
214 static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
215 unsigned int datalen) \
216 { \
217 struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
218 ctx->crc = func(ctx->crc, data, datalen); \
219 return 0; \
220 }
221
222 CRC32_VX_UPDATE(crc32le, crc32_le_vx)
223 CRC32_VX_UPDATE(crc32be, crc32_be_vx)
224 CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
225
226
227 static struct shash_alg crc32_vx_algs[] = {
228 /* CRC-32 LE */
229 {
230 .init = crc32_vx_init,
231 .setkey = crc32_vx_setkey,
232 .update = crc32le_vx_update,
233 .final = crc32le_vx_final,
234 .finup = crc32le_vx_finup,
235 .digest = crc32le_vx_digest,
236 .descsize = sizeof(struct crc_desc_ctx),
237 .digestsize = CRC32_DIGEST_SIZE,
238 .base = {
239 .cra_name = "crc32",
240 .cra_driver_name = "crc32-vx",
241 .cra_priority = 200,
242 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
243 .cra_blocksize = CRC32_BLOCK_SIZE,
244 .cra_ctxsize = sizeof(struct crc_ctx),
245 .cra_module = THIS_MODULE,
246 .cra_init = crc32_vx_cra_init_zero,
247 },
248 },
249 /* CRC-32 BE */
250 {
251 .init = crc32_vx_init,
252 .setkey = crc32be_vx_setkey,
253 .update = crc32be_vx_update,
254 .final = crc32be_vx_final,
255 .finup = crc32be_vx_finup,
256 .digest = crc32be_vx_digest,
257 .descsize = sizeof(struct crc_desc_ctx),
258 .digestsize = CRC32_DIGEST_SIZE,
259 .base = {
260 .cra_name = "crc32be",
261 .cra_driver_name = "crc32be-vx",
262 .cra_priority = 200,
263 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
264 .cra_blocksize = CRC32_BLOCK_SIZE,
265 .cra_ctxsize = sizeof(struct crc_ctx),
266 .cra_module = THIS_MODULE,
267 .cra_init = crc32_vx_cra_init_zero,
268 },
269 },
270 /* CRC-32C LE */
271 {
272 .init = crc32_vx_init,
273 .setkey = crc32_vx_setkey,
274 .update = crc32c_vx_update,
275 .final = crc32c_vx_final,
276 .finup = crc32c_vx_finup,
277 .digest = crc32c_vx_digest,
278 .descsize = sizeof(struct crc_desc_ctx),
279 .digestsize = CRC32_DIGEST_SIZE,
280 .base = {
281 .cra_name = "crc32c",
282 .cra_driver_name = "crc32c-vx",
283 .cra_priority = 200,
284 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
285 .cra_blocksize = CRC32_BLOCK_SIZE,
286 .cra_ctxsize = sizeof(struct crc_ctx),
287 .cra_module = THIS_MODULE,
288 .cra_init = crc32_vx_cra_init_invert,
289 },
290 },
291 };
292
293
crc_vx_mod_init(void)294 static int __init crc_vx_mod_init(void)
295 {
296 return crypto_register_shashes(crc32_vx_algs,
297 ARRAY_SIZE(crc32_vx_algs));
298 }
299
crc_vx_mod_exit(void)300 static void __exit crc_vx_mod_exit(void)
301 {
302 crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
303 }
304
305 module_cpu_feature_match(VXRS, crc_vx_mod_init);
306 module_exit(crc_vx_mod_exit);
307
308 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
309 MODULE_LICENSE("GPL");
310
311 MODULE_ALIAS_CRYPTO("crc32");
312 MODULE_ALIAS_CRYPTO("crc32-vx");
313 MODULE_ALIAS_CRYPTO("crc32c");
314 MODULE_ALIAS_CRYPTO("crc32c-vx");
315
