1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/cpufeature.h>
9 #include <linux/crc-t10dif.h>
10 #include <linux/init.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/string.h>
14
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17
18 #include <asm/neon.h>
19 #include <asm/simd.h>
20
21 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
22
23 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
24 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
25
crct10dif_init(struct shash_desc * desc)26 static int crct10dif_init(struct shash_desc *desc)
27 {
28 u16 *crc = shash_desc_ctx(desc);
29
30 *crc = 0;
31 return 0;
32 }
33
crct10dif_update_pmull_p8(struct shash_desc * desc,const u8 * data,unsigned int length)34 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
35 unsigned int length)
36 {
37 u16 *crc = shash_desc_ctx(desc);
38
39 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
40 kernel_neon_begin();
41 *crc = crc_t10dif_pmull_p8(*crc, data, length);
42 kernel_neon_end();
43 } else {
44 *crc = crc_t10dif_generic(*crc, data, length);
45 }
46
47 return 0;
48 }
49
crct10dif_update_pmull_p64(struct shash_desc * desc,const u8 * data,unsigned int length)50 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
51 unsigned int length)
52 {
53 u16 *crc = shash_desc_ctx(desc);
54
55 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
56 kernel_neon_begin();
57 *crc = crc_t10dif_pmull_p64(*crc, data, length);
58 kernel_neon_end();
59 } else {
60 *crc = crc_t10dif_generic(*crc, data, length);
61 }
62
63 return 0;
64 }
65
crct10dif_final(struct shash_desc * desc,u8 * out)66 static int crct10dif_final(struct shash_desc *desc, u8 *out)
67 {
68 u16 *crc = shash_desc_ctx(desc);
69
70 *(u16 *)out = *crc;
71 return 0;
72 }
73
74 static struct shash_alg crc_t10dif_alg[] = {{
75 .digestsize = CRC_T10DIF_DIGEST_SIZE,
76 .init = crct10dif_init,
77 .update = crct10dif_update_pmull_p8,
78 .final = crct10dif_final,
79 .descsize = CRC_T10DIF_DIGEST_SIZE,
80
81 .base.cra_name = "crct10dif",
82 .base.cra_driver_name = "crct10dif-arm64-neon",
83 .base.cra_priority = 100,
84 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
85 .base.cra_module = THIS_MODULE,
86 }, {
87 .digestsize = CRC_T10DIF_DIGEST_SIZE,
88 .init = crct10dif_init,
89 .update = crct10dif_update_pmull_p64,
90 .final = crct10dif_final,
91 .descsize = CRC_T10DIF_DIGEST_SIZE,
92
93 .base.cra_name = "crct10dif",
94 .base.cra_driver_name = "crct10dif-arm64-ce",
95 .base.cra_priority = 200,
96 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
97 .base.cra_module = THIS_MODULE,
98 }};
99
crc_t10dif_mod_init(void)100 static int __init crc_t10dif_mod_init(void)
101 {
102 if (cpu_have_named_feature(PMULL))
103 return crypto_register_shashes(crc_t10dif_alg,
104 ARRAY_SIZE(crc_t10dif_alg));
105 else
106 /* only register the first array element */
107 return crypto_register_shash(crc_t10dif_alg);
108 }
109
crc_t10dif_mod_exit(void)110 static void __exit crc_t10dif_mod_exit(void)
111 {
112 if (cpu_have_named_feature(PMULL))
113 crypto_unregister_shashes(crc_t10dif_alg,
114 ARRAY_SIZE(crc_t10dif_alg));
115 else
116 crypto_unregister_shash(crc_t10dif_alg);
117 }
118
119 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
120 module_exit(crc_t10dif_mod_exit);
121
122 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
123 MODULE_LICENSE("GPL v2");
124 MODULE_ALIAS_CRYPTO("crct10dif");
125 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");
126
