Lines Matching +full:byte +full:- +full:len
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Implement AES algorithm in Intel AES-NI instructions.
5 * The white paper of AES-NI instructions can be downloaded from:
6 * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
13 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
14 * interface for 64-bit kernels.
30 #include <asm/nospec-branch.h>
33 * The following macros are used to move an (un)aligned 16 byte value to/from
37 * since Nehalem (original Core i7) was released. However, the movaps is a byte
153 #define LEN %rcx macro
166 #define LEN %esi macro
248 # Clobbers rax, r10-r13 and xmm0-xmm6, %xmm13
273 # Requires the input data be at least 1 byte long because of READ_PARTIAL_BLOCK
274 # Clobbers rax, r10-r13, and xmm0-xmm15
286 and $-16, %r13 # %r13 = %r13 - (%r13 mod 16)
315 # Main loop - Encrypt/Decrypt remaining blocks
341 # Handle the last <16 Byte block separately
362 # receive the last <16 Byte block
369 # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
374 # shift right 16-r13 bytes
386 # get the appropriate mask to mask out top 16-r13 bytes of xmm0
387 pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
403 # GHASH computation for the last <16 byte block
429 # Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
444 movd %r12d, %xmm15 # len(A) in %xmm15
446 shl $3, %r12 # len(C) in bits (*128)
449 pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
450 pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
504 * Input: A and B (128-bits each, bit-reflected)
585 mov -1(\DPTR, \DLEN, 1), %al
593 # clobbers r10-11, xmm14
606 pshufb %xmm14, \TMP7 # byte-reflect the AAD data
622 pshufb %xmm14, \TMP7 # byte-reflect the AAD data
633 # Requires the input data be at least 1 byte long due to READ_PARTIAL_BLOCK
635 # Clobbers rax, r10, r12, r13, xmm0-6, xmm9-13
662 # r16-r13 is the number of bytes in plaintext mod 16)
681 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
694 # GHASH computation for the last <16 Byte block
717 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
729 # GHASH computation for the last <16 Byte block
808 pshufb %xmm14, %xmm\index # perform a 16 byte swap
813 shr $2,%eax # 128->4, 192->6, 256->8
814 add $5,%eax # 128->9, 192->11, 256->13
874 pshufb %xmm14, \XMM1 # perform a 16 byte swap
878 pshufb %xmm14, \XMM2 # perform a 16 byte swap
882 pshufb %xmm14, \XMM3 # perform a 16 byte swap
886 pshufb %xmm14, \XMM4 # perform a 16 byte swap
909 shr $2,%eax # 128->4, 192->6, 256->8
910 sub $4,%eax # 128->0, 192->2, 256->4
959 pshufb %xmm14, \XMM1 # perform a 16 byte swap
962 pshufb %xmm14, \XMM2 # perform a 16 byte swap
963 pshufb %xmm14, \XMM3 # perform a 16 byte swap
964 pshufb %xmm14, \XMM4 # perform a 16 byte swap
1000 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1002 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1003 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1004 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1096 shr $2,%eax # 128->4, 192->6, 256->8
1097 sub $4,%eax # 128->0, 192->2, 256->4
1129 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1130 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1131 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1132 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1208 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1210 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1211 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1212 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1304 shr $2,%eax # 128->4, 192->6, 256->8
1305 sub $4,%eax # 128->0, 192->2, 256->4
1341 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1342 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1343 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1344 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1456 # TMP6:XMMDst holds the result of the accumulated carry-less multiplications
1496 shr $2,%eax # 128->4, 192->6, 256->8
1497 add $5,%eax # 128->9, 192->11, 256->13
1511 * void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1514 * u8 *out, // Plaintext output. Encrypt in-place is allowed.
1517 * u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Associat…
1518 * // concatenated with 8 byte Initialisation Vector (from IPSe…
1519 * // concatenated with 0x00000001. 16-byte aligned pointer.
1520 * u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte bound…
1531 * keys are pre-expanded and aligned to 16 bytes. we are using the first
1537 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1539 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1542 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1544 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1558 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1560 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1561 * | 32-bit Sequence Number (A0) |
1562 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1564 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1566 * AAD Format with 32-bit Sequence Number
1574 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1576 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1578 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1579 * | 64-bit Extended Sequence Number {A1,A0} |
1581 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1583 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1585 * AAD Format with 64-bit Extended Sequence Number
1602 * void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1605 * u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1608 * u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Associ…
1609 * // concatenated with 8 byte Initialisation Vector (from IP…
1610 * // concatenated with 0x00000001. 16-byte aligned pointer.
1611 * u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte bou…
1621 * keys are pre-expanded and aligned to 16 bytes. we are using the
1628 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1630 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1633 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1635 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1649 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1651 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1652 * | 32-bit Sequence Number (A0) |
1653 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1655 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1657 * AAD Format with 32-bit Sequence Number
1665 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1667 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1668 * | 64-bit Extended Sequence Number {A1,A0} |
1670 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1672 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1674 * AAD Format with 64-bit Extended Sequence Number
1690 * void aesni_gcm_init(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1693 * u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Assoc…
1694 * // concatenated with 8 byte Initialisation Vector (from I…
1695 * // concatenated with 0x00000001. 16-byte aligned pointer.
1696 * u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte bo…
1708 * void aesni_gcm_enc_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1711 * u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1723 * void aesni_gcm_dec_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1726 * u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1738 * void aesni_gcm_finalize(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1919 lea 240-16(TKEYP), UKEYP
1983 movaps -0x60(TKEYP), KEY
1985 movaps -0x50(TKEYP), KEY
1989 movaps -0x40(TKEYP), KEY
1991 movaps -0x30(TKEYP), KEY
1995 movaps -0x20(TKEYP), KEY
1997 movaps -0x10(TKEYP), KEY
2049 movaps -0x60(TKEYP), KEY
2054 movaps -0x50(TKEYP), KEY
2061 movaps -0x40(TKEYP), KEY
2066 movaps -0x30(TKEYP), KEY
2073 movaps -0x20(TKEYP), KEY
2078 movaps -0x10(TKEYP), KEY
2173 movaps -0x60(TKEYP), KEY
2175 movaps -0x50(TKEYP), KEY
2179 movaps -0x40(TKEYP), KEY
2181 movaps -0x30(TKEYP), KEY
2185 movaps -0x20(TKEYP), KEY
2187 movaps -0x10(TKEYP), KEY
2239 movaps -0x60(TKEYP), KEY
2244 movaps -0x50(TKEYP), KEY
2251 movaps -0x40(TKEYP), KEY
2256 movaps -0x30(TKEYP), KEY
2263 movaps -0x20(TKEYP), KEY
2268 movaps -0x10(TKEYP), KEY
2318 * size_t len)
2323 pushl LEN
2329 movl (FRAME_OFFSET+28)(%esp), LEN # len
2331 test LEN, LEN # check length
2334 cmp $16, LEN
2336 cmp $64, LEN
2349 sub $64, LEN
2352 cmp $64, LEN
2354 cmp $16, LEN
2361 sub $16, LEN
2364 cmp $16, LEN
2370 popl LEN
2378 * size_t len);
2383 pushl LEN
2389 movl (FRAME_OFFSET+28)(%esp), LEN # len
2391 test LEN, LEN
2395 cmp $16, LEN
2397 cmp $64, LEN
2410 sub $64, LEN
2413 cmp $64, LEN
2415 cmp $16, LEN
2422 sub $16, LEN
2425 cmp $16, LEN
2431 popl LEN
2439 * size_t len, u8 *iv)
2445 pushl LEN
2451 movl (FRAME_OFFSET+32)(%esp), LEN # len
2454 cmp $16, LEN
2464 sub $16, LEN
2467 cmp $16, LEN
2474 popl LEN
2483 * size_t len, u8 *iv)
2489 pushl LEN
2495 movl (FRAME_OFFSET+32)(%esp), LEN # len
2498 cmp $16, LEN
2503 cmp $64, LEN
2541 sub $64, LEN
2544 cmp $64, LEN
2546 cmp $16, LEN
2556 sub $16, LEN
2559 cmp $16, LEN
2567 popl LEN
2576 * size_t len, u8 *iv)
2582 pushl LEN
2588 movl (FRAME_OFFSET+32)(%esp), LEN # len
2596 sub $16, LEN
2599 add LEN, T1
2600 sub LEN, IVP
2605 add LEN, INP
2614 add OUTP, LEN
2615 movups STATE, (LEN)
2624 popl LEN
2633 * size_t len, u8 *iv)
2639 pushl LEN
2645 movl (FRAME_OFFSET+32)(%esp), LEN # len
2654 sub $16, LEN
2657 add LEN, T1
2658 sub LEN, IVP
2662 add LEN, INP
2670 add OUTP, LEN
2671 movups STATE, (LEN)
2685 popl LEN
2695 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2696 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2697 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
2698 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
2699 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2700 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2703 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
2759 * size_t len, u8 *iv)
2763 cmp $16, LEN
2768 cmp $64, LEN
2793 sub $64, LEN
2796 cmp $64, LEN
2798 cmp $16, LEN
2808 sub $16, LEN
2811 cmp $16, LEN
2848 * const u8 *src, unsigned int len, le128 *iv)
2854 pushl LEN
2860 movl (FRAME_OFFSET+32)(%esp), LEN # len
2871 sub $64, LEN
2919 test LEN, LEN
2929 popl LEN
2936 add $64, LEN
2938 sub $16, LEN
2948 test LEN, LEN
2952 sub $16, LEN
2973 add LEN, INP /* rewind input pointer */
2974 add $16, LEN /* # bytes in final block */
2979 add LEN, T1
2980 sub LEN, IVP
2981 add OUTP, LEN
2986 movups STATE, (LEN)
3003 * const u8 *src, unsigned int len, le128 *iv)
3009 pushl LEN
3015 movl (FRAME_OFFSET+32)(%esp), LEN # len
3026 test $15, LEN
3028 sub $16, LEN
3031 sub $64, LEN
3079 test LEN, LEN
3089 popl LEN
3096 add $64, LEN
3103 sub $16, LEN
3111 test LEN, LEN
3135 add LEN, INP /* rewind input pointer */
3136 add $16, LEN /* # bytes in final block */
3141 add LEN, T1
3142 sub LEN, IVP
3143 add OUTP, LEN
3148 movups STATE, (LEN)