Lines Matching refs:cryp
161 struct stm32_cryp *cryp; member
212 static inline bool is_aes(struct stm32_cryp *cryp) in is_aes() argument
214 return cryp->flags & FLG_AES; in is_aes()
217 static inline bool is_des(struct stm32_cryp *cryp) in is_des() argument
219 return cryp->flags & FLG_DES; in is_des()
222 static inline bool is_tdes(struct stm32_cryp *cryp) in is_tdes() argument
224 return cryp->flags & FLG_TDES; in is_tdes()
227 static inline bool is_ecb(struct stm32_cryp *cryp) in is_ecb() argument
229 return cryp->flags & FLG_ECB; in is_ecb()
232 static inline bool is_cbc(struct stm32_cryp *cryp) in is_cbc() argument
234 return cryp->flags & FLG_CBC; in is_cbc()
237 static inline bool is_ctr(struct stm32_cryp *cryp) in is_ctr() argument
239 return cryp->flags & FLG_CTR; in is_ctr()
242 static inline bool is_gcm(struct stm32_cryp *cryp) in is_gcm() argument
244 return cryp->flags & FLG_GCM; in is_gcm()
247 static inline bool is_ccm(struct stm32_cryp *cryp) in is_ccm() argument
249 return cryp->flags & FLG_CCM; in is_ccm()
252 static inline bool is_encrypt(struct stm32_cryp *cryp) in is_encrypt() argument
254 return cryp->flags & FLG_ENCRYPT; in is_encrypt()
257 static inline bool is_decrypt(struct stm32_cryp *cryp) in is_decrypt() argument
259 return !is_encrypt(cryp); in is_decrypt()
262 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) in stm32_cryp_read() argument
264 return readl_relaxed(cryp->regs + ofst); in stm32_cryp_read()
267 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) in stm32_cryp_write() argument
269 writel_relaxed(val, cryp->regs + ofst); in stm32_cryp_write()
272 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) in stm32_cryp_wait_busy() argument
276 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, in stm32_cryp_wait_busy()
280 static inline void stm32_cryp_enable(struct stm32_cryp *cryp) in stm32_cryp_enable() argument
282 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN, in stm32_cryp_enable()
283 cryp->regs + cryp->caps->cr); in stm32_cryp_enable()
286 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) in stm32_cryp_wait_enable() argument
290 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status, in stm32_cryp_wait_enable()
294 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) in stm32_cryp_wait_output() argument
298 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, in stm32_cryp_wait_output()
302 static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp) in stm32_cryp_key_read_enable() argument
304 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN, in stm32_cryp_key_read_enable()
305 cryp->regs + cryp->caps->cr); in stm32_cryp_key_read_enable()
308 static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp) in stm32_cryp_key_read_disable() argument
310 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN, in stm32_cryp_key_read_disable()
311 cryp->regs + cryp->caps->cr); in stm32_cryp_key_read_disable()
314 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
315 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
319 struct stm32_cryp *tmp, *cryp = NULL; in stm32_cryp_find_dev() local
322 if (!ctx->cryp) { in stm32_cryp_find_dev()
324 cryp = tmp; in stm32_cryp_find_dev()
327 ctx->cryp = cryp; in stm32_cryp_find_dev()
329 cryp = ctx->cryp; in stm32_cryp_find_dev()
334 return cryp; in stm32_cryp_find_dev()
337 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) in stm32_cryp_hw_write_iv() argument
342 stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
343 stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
345 if (is_aes(cryp)) { in stm32_cryp_hw_write_iv()
346 stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
347 stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
351 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) in stm32_cryp_get_iv() argument
353 struct skcipher_request *req = cryp->req; in stm32_cryp_get_iv()
359 if (cryp->caps->iv_protection) in stm32_cryp_get_iv()
360 stm32_cryp_key_read_enable(cryp); in stm32_cryp_get_iv()
362 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); in stm32_cryp_get_iv()
363 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); in stm32_cryp_get_iv()
365 if (is_aes(cryp)) { in stm32_cryp_get_iv()
366 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); in stm32_cryp_get_iv()
367 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); in stm32_cryp_get_iv()
370 if (cryp->caps->iv_protection) in stm32_cryp_get_iv()
371 stm32_cryp_key_read_disable(cryp); in stm32_cryp_get_iv()
493 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) in stm32_cryp_get_hw_mode() argument
495 if (is_aes(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
498 if (is_aes(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
501 if (is_aes(cryp) && is_ctr(cryp)) in stm32_cryp_get_hw_mode()
504 if (is_aes(cryp) && is_gcm(cryp)) in stm32_cryp_get_hw_mode()
507 if (is_aes(cryp) && is_ccm(cryp)) in stm32_cryp_get_hw_mode()
510 if (is_des(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
513 if (is_des(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
516 if (is_tdes(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
519 if (is_tdes(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
522 dev_err(cryp->dev, "Unknown mode\n"); in stm32_cryp_get_hw_mode()
526 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) in stm32_cryp_get_input_text_len() argument
528 return is_encrypt(cryp) ? cryp->areq->cryptlen : in stm32_cryp_get_input_text_len()
529 cryp->areq->cryptlen - cryp->authsize; in stm32_cryp_get_input_text_len()
532 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) in stm32_cryp_gcm_init() argument
538 memcpy(iv, cryp->areq->iv, 12); in stm32_cryp_gcm_init()
540 cryp->gcm_ctr = GCM_CTR_INIT; in stm32_cryp_gcm_init()
541 stm32_cryp_hw_write_iv(cryp, iv); in stm32_cryp_gcm_init()
543 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); in stm32_cryp_gcm_init()
546 ret = stm32_cryp_wait_enable(cryp); in stm32_cryp_gcm_init()
548 dev_err(cryp->dev, "Timeout (gcm init)\n"); in stm32_cryp_gcm_init()
553 if (cryp->areq->assoclen) { in stm32_cryp_gcm_init()
555 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_gcm_init()
556 } else if (stm32_cryp_get_input_text_len(cryp)) { in stm32_cryp_gcm_init()
558 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_gcm_init()
564 static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) in stm32_crypt_gcmccm_end_header() argument
570 if (!cryp->header_in) { in stm32_crypt_gcmccm_end_header()
572 err = stm32_cryp_wait_busy(cryp); in stm32_crypt_gcmccm_end_header()
574 dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); in stm32_crypt_gcmccm_end_header()
575 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_crypt_gcmccm_end_header()
576 stm32_cryp_finish_req(cryp, err); in stm32_crypt_gcmccm_end_header()
580 if (stm32_cryp_get_input_text_len(cryp)) { in stm32_crypt_gcmccm_end_header()
582 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_crypt_gcmccm_end_header()
584 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_crypt_gcmccm_end_header()
588 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_crypt_gcmccm_end_header()
599 static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) in stm32_cryp_write_ccm_first_header() argument
603 u32 alen = cryp->areq->assoclen; in stm32_cryp_write_ccm_first_header()
625 scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0); in stm32_cryp_write_ccm_first_header()
627 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_write_ccm_first_header()
629 cryp->header_in -= written; in stm32_cryp_write_ccm_first_header()
631 stm32_crypt_gcmccm_end_header(cryp); in stm32_cryp_write_ccm_first_header()
634 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) in stm32_cryp_ccm_init() argument
644 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); in stm32_cryp_ccm_init()
647 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); in stm32_cryp_ccm_init()
652 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); in stm32_cryp_ccm_init()
654 if (cryp->areq->assoclen) in stm32_cryp_ccm_init()
657 textlen = stm32_cryp_get_input_text_len(cryp); in stm32_cryp_ccm_init()
663 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); in stm32_cryp_ccm_init()
672 if (!cryp->caps->padding_wa) in stm32_cryp_ccm_init()
674 stm32_cryp_write(cryp, cryp->caps->din, xd); in stm32_cryp_ccm_init()
678 ret = stm32_cryp_wait_enable(cryp); in stm32_cryp_ccm_init()
680 dev_err(cryp->dev, "Timeout (ccm init)\n"); in stm32_cryp_ccm_init()
685 if (cryp->areq->assoclen) { in stm32_cryp_ccm_init()
687 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_ccm_init()
690 stm32_cryp_write_ccm_first_header(cryp); in stm32_cryp_ccm_init()
691 } else if (stm32_cryp_get_input_text_len(cryp)) { in stm32_cryp_ccm_init()
693 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_ccm_init()
699 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) in stm32_cryp_hw_init() argument
704 pm_runtime_get_sync(cryp->dev); in stm32_cryp_hw_init()
707 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_hw_init()
712 switch (cryp->ctx->keylen) { in stm32_cryp_hw_init()
727 hw_mode = stm32_cryp_get_hw_mode(cryp); in stm32_cryp_hw_init()
732 if (is_decrypt(cryp) && in stm32_cryp_hw_init()
735 if (cryp->caps->kp_mode) in stm32_cryp_hw_init()
736 stm32_cryp_write(cryp, cryp->caps->cr, in stm32_cryp_hw_init()
739 stm32_cryp_write(cryp, in stm32_cryp_hw_init()
740 cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE); in stm32_cryp_hw_init()
743 stm32_cryp_hw_write_key(cryp); in stm32_cryp_hw_init()
746 stm32_cryp_enable(cryp); in stm32_cryp_hw_init()
748 ret = stm32_cryp_wait_busy(cryp); in stm32_cryp_hw_init()
750 dev_err(cryp->dev, "Timeout (key preparation)\n"); in stm32_cryp_hw_init()
757 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_hw_init()
760 if (is_decrypt(cryp)) in stm32_cryp_hw_init()
764 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_hw_init()
767 stm32_cryp_hw_write_key(cryp); in stm32_cryp_hw_init()
775 ret = stm32_cryp_ccm_init(cryp, cfg); in stm32_cryp_hw_init()
777 ret = stm32_cryp_gcm_init(cryp, cfg); in stm32_cryp_hw_init()
788 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); in stm32_cryp_hw_init()
796 stm32_cryp_enable(cryp); in stm32_cryp_hw_init()
801 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) in stm32_cryp_finish_req() argument
803 if (!err && (is_gcm(cryp) || is_ccm(cryp))) in stm32_cryp_finish_req()
805 err = stm32_cryp_read_auth_tag(cryp); in stm32_cryp_finish_req()
807 if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) in stm32_cryp_finish_req()
808 stm32_cryp_get_iv(cryp); in stm32_cryp_finish_req()
810 pm_runtime_mark_last_busy(cryp->dev); in stm32_cryp_finish_req()
811 pm_runtime_put_autosuspend(cryp->dev); in stm32_cryp_finish_req()
813 if (is_gcm(cryp) || is_ccm(cryp)) in stm32_cryp_finish_req()
814 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); in stm32_cryp_finish_req()
816 crypto_finalize_skcipher_request(cryp->engine, cryp->req, in stm32_cryp_finish_req()
820 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) in stm32_cryp_cpu_start() argument
823 stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT); in stm32_cryp_cpu_start()
851 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); in stm32_cryp_crypt() local
853 if (!cryp) in stm32_cryp_crypt()
858 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); in stm32_cryp_crypt()
865 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); in stm32_cryp_aead_crypt() local
867 if (!cryp) in stm32_cryp_aead_crypt()
872 return crypto_transfer_aead_request_to_engine(cryp->engine, req); in stm32_cryp_aead_crypt()
1156 struct stm32_cryp *cryp; in stm32_cryp_prepare_req() local
1167 cryp = ctx->cryp; in stm32_cryp_prepare_req()
1172 ctx->cryp = cryp; in stm32_cryp_prepare_req()
1174 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; in stm32_cryp_prepare_req()
1175 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; in stm32_cryp_prepare_req()
1176 cryp->ctx = ctx; in stm32_cryp_prepare_req()
1179 cryp->req = req; in stm32_cryp_prepare_req()
1180 cryp->areq = NULL; in stm32_cryp_prepare_req()
1181 cryp->header_in = 0; in stm32_cryp_prepare_req()
1182 cryp->payload_in = req->cryptlen; in stm32_cryp_prepare_req()
1183 cryp->payload_out = req->cryptlen; in stm32_cryp_prepare_req()
1184 cryp->authsize = 0; in stm32_cryp_prepare_req()
1202 cryp->areq = areq; in stm32_cryp_prepare_req()
1203 cryp->req = NULL; in stm32_cryp_prepare_req()
1204 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); in stm32_cryp_prepare_req()
1205 if (is_encrypt(cryp)) { in stm32_cryp_prepare_req()
1206 cryp->payload_in = areq->cryptlen; in stm32_cryp_prepare_req()
1207 cryp->header_in = areq->assoclen; in stm32_cryp_prepare_req()
1208 cryp->payload_out = areq->cryptlen; in stm32_cryp_prepare_req()
1210 cryp->payload_in = areq->cryptlen - cryp->authsize; in stm32_cryp_prepare_req()
1211 cryp->header_in = areq->assoclen; in stm32_cryp_prepare_req()
1212 cryp->payload_out = cryp->payload_in; in stm32_cryp_prepare_req()
1217 scatterwalk_start(&cryp->in_walk, in_sg); in stm32_cryp_prepare_req()
1219 cryp->out_sg = req ? req->dst : areq->dst; in stm32_cryp_prepare_req()
1220 scatterwalk_start(&cryp->out_walk, cryp->out_sg); in stm32_cryp_prepare_req()
1222 if (is_gcm(cryp) || is_ccm(cryp)) { in stm32_cryp_prepare_req()
1224 scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2); in stm32_cryp_prepare_req()
1227 if (is_ctr(cryp)) in stm32_cryp_prepare_req()
1228 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); in stm32_cryp_prepare_req()
1230 ret = stm32_cryp_hw_init(cryp); in stm32_cryp_prepare_req()
1241 struct stm32_cryp *cryp = ctx->cryp; in stm32_cryp_cipher_one_req() local
1243 if (!cryp) in stm32_cryp_cipher_one_req()
1247 stm32_cryp_cpu_start(cryp); in stm32_cryp_cipher_one_req()
1255 struct stm32_cryp *cryp = ctx->cryp; in stm32_cryp_aead_one_req() local
1258 if (!cryp) in stm32_cryp_aead_one_req()
1265 if (unlikely(!cryp->payload_in && !cryp->header_in)) { in stm32_cryp_aead_one_req()
1267 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_aead_one_req()
1271 return stm32_cryp_cpu_start(cryp); in stm32_cryp_aead_one_req()
1274 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) in stm32_cryp_read_auth_tag() argument
1281 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_read_auth_tag()
1288 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_read_auth_tag()
1290 if (is_gcm(cryp)) { in stm32_cryp_read_auth_tag()
1292 size_bit = cryp->areq->assoclen * 8; in stm32_cryp_read_auth_tag()
1293 if (cryp->caps->swap_final) in stm32_cryp_read_auth_tag()
1296 stm32_cryp_write(cryp, cryp->caps->din, 0); in stm32_cryp_read_auth_tag()
1297 stm32_cryp_write(cryp, cryp->caps->din, size_bit); in stm32_cryp_read_auth_tag()
1299 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : in stm32_cryp_read_auth_tag()
1300 cryp->areq->cryptlen - cryp->authsize; in stm32_cryp_read_auth_tag()
1302 if (cryp->caps->swap_final) in stm32_cryp_read_auth_tag()
1305 stm32_cryp_write(cryp, cryp->caps->din, 0); in stm32_cryp_read_auth_tag()
1306 stm32_cryp_write(cryp, cryp->caps->din, size_bit); in stm32_cryp_read_auth_tag()
1313 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); in stm32_cryp_read_auth_tag()
1319 if (!cryp->caps->padding_wa) in stm32_cryp_read_auth_tag()
1321 stm32_cryp_write(cryp, cryp->caps->din, xiv); in stm32_cryp_read_auth_tag()
1326 ret = stm32_cryp_wait_output(cryp); in stm32_cryp_read_auth_tag()
1328 dev_err(cryp->dev, "Timeout (read tag)\n"); in stm32_cryp_read_auth_tag()
1332 if (is_encrypt(cryp)) { in stm32_cryp_read_auth_tag()
1336 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); in stm32_cryp_read_auth_tag()
1337 scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1); in stm32_cryp_read_auth_tag()
1342 scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0); in stm32_cryp_read_auth_tag()
1343 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); in stm32_cryp_read_auth_tag()
1345 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) in stm32_cryp_read_auth_tag()
1351 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_read_auth_tag()
1356 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) in stm32_cryp_check_ctr_counter() argument
1360 if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { in stm32_cryp_check_ctr_counter()
1365 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); in stm32_cryp_check_ctr_counter()
1367 cr = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_check_ctr_counter()
1368 stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN); in stm32_cryp_check_ctr_counter()
1370 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); in stm32_cryp_check_ctr_counter()
1372 stm32_cryp_write(cryp, cryp->caps->cr, cr); in stm32_cryp_check_ctr_counter()
1376 cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); in stm32_cryp_check_ctr_counter()
1377 cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); in stm32_cryp_check_ctr_counter()
1378 cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); in stm32_cryp_check_ctr_counter()
1379 cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); in stm32_cryp_check_ctr_counter()
1382 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) in stm32_cryp_irq_read_data() argument
1386 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_read_data()
1387 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_read_data()
1388 cryp->payload_out), 1); in stm32_cryp_irq_read_data()
1389 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_read_data()
1390 cryp->payload_out); in stm32_cryp_irq_read_data()
1393 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) in stm32_cryp_irq_write_block() argument
1397 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_block()
1398 cryp->payload_in), 0); in stm32_cryp_irq_write_block()
1399 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_block()
1400 cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); in stm32_cryp_irq_write_block()
1403 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_gcm_padded_data() argument
1412 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_irq_write_gcm_padded_data()
1413 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_write_gcm_padded_data()
1415 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1418 stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2); in stm32_cryp_irq_write_gcm_padded_data()
1423 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1427 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1430 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_gcm_padded_data()
1432 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_gcm_padded_data()
1434 dev_err(cryp->dev, "Timeout (write gcm last data)\n"); in stm32_cryp_irq_write_gcm_padded_data()
1435 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_gcm_padded_data()
1443 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_gcm_padded_data()
1445 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_gcm_padded_data()
1446 cryp->payload_out), 1); in stm32_cryp_irq_write_gcm_padded_data()
1447 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_gcm_padded_data()
1448 cryp->payload_out); in stm32_cryp_irq_write_gcm_padded_data()
1453 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1458 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1461 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_irq_write_gcm_padded_data()
1464 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_gcm_padded_data()
1466 dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); in stm32_cryp_irq_write_gcm_padded_data()
1467 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_gcm_padded_data()
1471 stm32_cryp_read(cryp, cryp->caps->dout); in stm32_cryp_irq_write_gcm_padded_data()
1474 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_irq_write_gcm_padded_data()
1477 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) in stm32_cryp_irq_set_npblb() argument
1482 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_set_npblb()
1484 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_set_npblb()
1486 cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; in stm32_cryp_irq_set_npblb()
1488 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_set_npblb()
1491 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_ccm_padded_data() argument
1502 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_irq_write_ccm_padded_data()
1504 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_write_ccm_padded_data()
1506 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
1509 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); in stm32_cryp_irq_write_ccm_padded_data()
1513 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); in stm32_cryp_irq_write_ccm_padded_data()
1516 stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp); in stm32_cryp_irq_write_ccm_padded_data()
1521 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
1525 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
1528 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_ccm_padded_data()
1530 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_ccm_padded_data()
1532 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); in stm32_cryp_irq_write_ccm_padded_data()
1533 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_ccm_padded_data()
1541 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_ccm_padded_data()
1543 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_ccm_padded_data()
1544 cryp->payload_out), 1); in stm32_cryp_irq_write_ccm_padded_data()
1545 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); in stm32_cryp_irq_write_ccm_padded_data()
1549 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); in stm32_cryp_irq_write_ccm_padded_data()
1554 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
1559 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
1565 stm32_cryp_write(cryp, cryp->caps->din, block[i]); in stm32_cryp_irq_write_ccm_padded_data()
1569 err = stm32_cryp_wait_busy(cryp); in stm32_cryp_irq_write_ccm_padded_data()
1571 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); in stm32_cryp_irq_write_ccm_padded_data()
1574 stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_ccm_padded_data()
1577 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_data() argument
1579 if (unlikely(!cryp->payload_in)) { in stm32_cryp_irq_write_data()
1580 dev_warn(cryp->dev, "No more data to process\n"); in stm32_cryp_irq_write_data()
1584 if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && in stm32_cryp_irq_write_data()
1585 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && in stm32_cryp_irq_write_data()
1586 is_encrypt(cryp))) { in stm32_cryp_irq_write_data()
1588 if (cryp->caps->padding_wa) { in stm32_cryp_irq_write_data()
1590 stm32_cryp_irq_write_gcm_padded_data(cryp); in stm32_cryp_irq_write_data()
1595 stm32_cryp_irq_set_npblb(cryp); in stm32_cryp_irq_write_data()
1598 if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && in stm32_cryp_irq_write_data()
1599 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && in stm32_cryp_irq_write_data()
1600 is_decrypt(cryp))) { in stm32_cryp_irq_write_data()
1602 if (cryp->caps->padding_wa) { in stm32_cryp_irq_write_data()
1604 stm32_cryp_irq_write_ccm_padded_data(cryp); in stm32_cryp_irq_write_data()
1609 stm32_cryp_irq_set_npblb(cryp); in stm32_cryp_irq_write_data()
1612 if (is_aes(cryp) && is_ctr(cryp)) in stm32_cryp_irq_write_data()
1613 stm32_cryp_check_ctr_counter(cryp); in stm32_cryp_irq_write_data()
1615 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_data()
1618 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) in stm32_cryp_irq_write_gcmccm_header() argument
1623 written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); in stm32_cryp_irq_write_gcmccm_header()
1625 scatterwalk_copychunks(block, &cryp->in_walk, written, 0); in stm32_cryp_irq_write_gcmccm_header()
1627 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_irq_write_gcmccm_header()
1629 cryp->header_in -= written; in stm32_cryp_irq_write_gcmccm_header()
1631 stm32_crypt_gcmccm_end_header(cryp); in stm32_cryp_irq_write_gcmccm_header()
1636 struct stm32_cryp *cryp = arg; in stm32_cryp_irq_thread() local
1638 u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc); in stm32_cryp_irq_thread()
1640 if (cryp->irq_status & MISR_OUT) in stm32_cryp_irq_thread()
1642 stm32_cryp_irq_read_data(cryp); in stm32_cryp_irq_thread()
1644 if (cryp->irq_status & MISR_IN) { in stm32_cryp_irq_thread()
1645 if (is_gcm(cryp) || is_ccm(cryp)) { in stm32_cryp_irq_thread()
1646 ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK; in stm32_cryp_irq_thread()
1649 stm32_cryp_irq_write_gcmccm_header(cryp); in stm32_cryp_irq_thread()
1652 stm32_cryp_irq_write_data(cryp); in stm32_cryp_irq_thread()
1653 if (is_gcm(cryp)) in stm32_cryp_irq_thread()
1654 cryp->gcm_ctr++; in stm32_cryp_irq_thread()
1657 stm32_cryp_irq_write_data(cryp); in stm32_cryp_irq_thread()
1662 if (!cryp->payload_in && !cryp->header_in) in stm32_cryp_irq_thread()
1664 if (!cryp->payload_out) in stm32_cryp_irq_thread()
1666 stm32_cryp_write(cryp, cryp->caps->imsc, it_mask); in stm32_cryp_irq_thread()
1668 if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) in stm32_cryp_irq_thread()
1669 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_irq_thread()
1676 struct stm32_cryp *cryp = arg; in stm32_cryp_irq() local
1678 cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis); in stm32_cryp_irq()
1970 struct stm32_cryp *cryp; in stm32_cryp_probe() local
1974 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); in stm32_cryp_probe()
1975 if (!cryp) in stm32_cryp_probe()
1978 cryp->caps = of_device_get_match_data(dev); in stm32_cryp_probe()
1979 if (!cryp->caps) in stm32_cryp_probe()
1982 cryp->dev = dev; in stm32_cryp_probe()
1984 cryp->regs = devm_platform_ioremap_resource(pdev, 0); in stm32_cryp_probe()
1985 if (IS_ERR(cryp->regs)) in stm32_cryp_probe()
1986 return PTR_ERR(cryp->regs); in stm32_cryp_probe()
1994 dev_name(dev), cryp); in stm32_cryp_probe()
2000 cryp->clk = devm_clk_get(dev, NULL); in stm32_cryp_probe()
2001 if (IS_ERR(cryp->clk)) { in stm32_cryp_probe()
2002 dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); in stm32_cryp_probe()
2004 return PTR_ERR(cryp->clk); in stm32_cryp_probe()
2007 ret = clk_prepare_enable(cryp->clk); in stm32_cryp_probe()
2009 dev_err(cryp->dev, "Failed to enable clock\n"); in stm32_cryp_probe()
2031 platform_set_drvdata(pdev, cryp); in stm32_cryp_probe()
2034 list_add(&cryp->list, &cryp_list.dev_list); in stm32_cryp_probe()
2038 cryp->engine = crypto_engine_alloc_init(dev, 1); in stm32_cryp_probe()
2039 if (!cryp->engine) { in stm32_cryp_probe()
2045 ret = crypto_engine_start(cryp->engine); in stm32_cryp_probe()
2057 if (cryp->caps->aeads_support) { in stm32_cryp_probe()
2073 crypto_engine_exit(cryp->engine); in stm32_cryp_probe()
2076 list_del(&cryp->list); in stm32_cryp_probe()
2082 clk_disable_unprepare(cryp->clk); in stm32_cryp_probe()
2089 struct stm32_cryp *cryp = platform_get_drvdata(pdev); in stm32_cryp_remove() local
2092 if (!cryp) in stm32_cryp_remove()
2095 ret = pm_runtime_resume_and_get(cryp->dev); in stm32_cryp_remove()
2099 if (cryp->caps->aeads_support) in stm32_cryp_remove()
2103 crypto_engine_exit(cryp->engine); in stm32_cryp_remove()
2106 list_del(&cryp->list); in stm32_cryp_remove()
2109 pm_runtime_disable(cryp->dev); in stm32_cryp_remove()
2110 pm_runtime_put_noidle(cryp->dev); in stm32_cryp_remove()
2112 clk_disable_unprepare(cryp->clk); in stm32_cryp_remove()
2120 struct stm32_cryp *cryp = dev_get_drvdata(dev); in stm32_cryp_runtime_suspend() local
2122 clk_disable_unprepare(cryp->clk); in stm32_cryp_runtime_suspend()
2129 struct stm32_cryp *cryp = dev_get_drvdata(dev); in stm32_cryp_runtime_resume() local
2132 ret = clk_prepare_enable(cryp->clk); in stm32_cryp_runtime_resume()
2134 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); in stm32_cryp_runtime_resume()