1/* 2 * Implement AES algorithm in Intel AES-NI instructions. 3 * 4 * The white paper of AES-NI instructions can be downloaded from: 5 * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf 6 * 7 * Copyright (C) 2008, Intel Corp. 8 * Author: Huang Ying <ying.huang@intel.com> 9 * Vinodh Gopal <vinodh.gopal@intel.com> 10 * Kahraman Akdemir 11 * 12 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 13 * interface for 64-bit kernels. 14 * Authors: Erdinc Ozturk (erdinc.ozturk@intel.com) 15 * Aidan O'Mahony (aidan.o.mahony@intel.com) 16 * Adrian Hoban <adrian.hoban@intel.com> 17 * James Guilford (james.guilford@intel.com) 18 * Gabriele Paoloni <gabriele.paoloni@intel.com> 19 * Tadeusz Struk (tadeusz.struk@intel.com) 20 * Wajdi Feghali (wajdi.k.feghali@intel.com) 21 * Copyright (c) 2010, Intel Corporation. 22 * 23 * Ported x86_64 version to x86: 24 * Author: Mathias Krause <minipli@googlemail.com> 25 * 26 * This program is free software; you can redistribute it and/or modify 27 * it under the terms of the GNU General Public License as published by 28 * the Free Software Foundation; either version 2 of the License, or 29 * (at your option) any later version. 30 */ 31 32#include <linux/linkage.h> 33#include <asm/inst.h> 34#include <asm/frame.h> 35#include <asm/nospec-branch.h> 36 37/* 38 * The following macros are used to move an (un)aligned 16 byte value to/from 39 * an XMM register. This can done for either FP or integer values, for FP use 40 * movaps (move aligned packed single) or integer use movdqa (move double quad 41 * aligned). It doesn't make a performance difference which instruction is used 42 * since Nehalem (original Core i7) was released. However, the movaps is a byte 43 * shorter, so that is the one we'll use for now. (same for unaligned). 44 */ 45#define MOVADQ movaps 46#define MOVUDQ movups 47 48#ifdef __x86_64__ 49 50# constants in mergeable sections, linker can reorder and merge 51.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16 52.align 16 53.Lgf128mul_x_ble_mask: 54 .octa 0x00000000000000010000000000000087 55.section .rodata.cst16.POLY, "aM", @progbits, 16 56.align 16 57POLY: .octa 0xC2000000000000000000000000000001 58.section .rodata.cst16.TWOONE, "aM", @progbits, 16 59.align 16 60TWOONE: .octa 0x00000001000000000000000000000001 61 62.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16 63.align 16 64SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F 65.section .rodata.cst16.MASK1, "aM", @progbits, 16 66.align 16 67MASK1: .octa 0x0000000000000000ffffffffffffffff 68.section .rodata.cst16.MASK2, "aM", @progbits, 16 69.align 16 70MASK2: .octa 0xffffffffffffffff0000000000000000 71.section .rodata.cst16.ONE, "aM", @progbits, 16 72.align 16 73ONE: .octa 0x00000000000000000000000000000001 74.section .rodata.cst16.F_MIN_MASK, "aM", @progbits, 16 75.align 16 76F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0 77.section .rodata.cst16.dec, "aM", @progbits, 16 78.align 16 79dec: .octa 0x1 80.section .rodata.cst16.enc, "aM", @progbits, 16 81.align 16 82enc: .octa 0x2 83 84# order of these constants should not change. 85# more specifically, ALL_F should follow SHIFT_MASK, 86# and zero should follow ALL_F 87.section .rodata, "a", @progbits 88.align 16 89SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 90ALL_F: .octa 0xffffffffffffffffffffffffffffffff 91 .octa 0x00000000000000000000000000000000 92 93.text 94 95 96#define STACK_OFFSET 8*3 97 98#define AadHash 16*0 99#define AadLen 16*1 100#define InLen (16*1)+8 101#define PBlockEncKey 16*2 102#define OrigIV 16*3 103#define CurCount 16*4 104#define PBlockLen 16*5 105#define HashKey 16*6 // store HashKey <<1 mod poly here 106#define HashKey_2 16*7 // store HashKey^2 <<1 mod poly here 107#define HashKey_3 16*8 // store HashKey^3 <<1 mod poly here 108#define HashKey_4 16*9 // store HashKey^4 <<1 mod poly here 109#define HashKey_k 16*10 // store XOR of High 64 bits and Low 64 110 // bits of HashKey <<1 mod poly here 111 //(for Karatsuba purposes) 112#define HashKey_2_k 16*11 // store XOR of High 64 bits and Low 64 113 // bits of HashKey^2 <<1 mod poly here 114 // (for Karatsuba purposes) 115#define HashKey_3_k 16*12 // store XOR of High 64 bits and Low 64 116 // bits of HashKey^3 <<1 mod poly here 117 // (for Karatsuba purposes) 118#define HashKey_4_k 16*13 // store XOR of High 64 bits and Low 64 119 // bits of HashKey^4 <<1 mod poly here 120 // (for Karatsuba purposes) 121 122#define arg1 rdi 123#define arg2 rsi 124#define arg3 rdx 125#define arg4 rcx 126#define arg5 r8 127#define arg6 r9 128#define arg7 STACK_OFFSET+8(%rsp) 129#define arg8 STACK_OFFSET+16(%rsp) 130#define arg9 STACK_OFFSET+24(%rsp) 131#define arg10 STACK_OFFSET+32(%rsp) 132#define arg11 STACK_OFFSET+40(%rsp) 133#define keysize 2*15*16(%arg1) 134#endif 135 136 137#define STATE1 %xmm0 138#define STATE2 %xmm4 139#define STATE3 %xmm5 140#define STATE4 %xmm6 141#define STATE STATE1 142#define IN1 %xmm1 143#define IN2 %xmm7 144#define IN3 %xmm8 145#define IN4 %xmm9 146#define IN IN1 147#define KEY %xmm2 148#define IV %xmm3 149 150#define BSWAP_MASK %xmm10 151#define CTR %xmm11 152#define INC %xmm12 153 154#define GF128MUL_MASK %xmm10 155 156#ifdef __x86_64__ 157#define AREG %rax 158#define KEYP %rdi 159#define OUTP %rsi 160#define UKEYP OUTP 161#define INP %rdx 162#define LEN %rcx 163#define IVP %r8 164#define KLEN %r9d 165#define T1 %r10 166#define TKEYP T1 167#define T2 %r11 168#define TCTR_LOW T2 169#else 170#define AREG %eax 171#define KEYP %edi 172#define OUTP AREG 173#define UKEYP OUTP 174#define INP %edx 175#define LEN %esi 176#define IVP %ebp 177#define KLEN %ebx 178#define T1 %ecx 179#define TKEYP T1 180#endif 181 182.macro FUNC_SAVE 183 push %r12 184 push %r13 185 push %r14 186# 187# states of %xmm registers %xmm6:%xmm15 not saved 188# all %xmm registers are clobbered 189# 190.endm 191 192 193.macro FUNC_RESTORE 194 pop %r14 195 pop %r13 196 pop %r12 197.endm 198 199# Precompute hashkeys. 200# Input: Hash subkey. 201# Output: HashKeys stored in gcm_context_data. Only needs to be called 202# once per key. 203# clobbers r12, and tmp xmm registers. 204.macro PRECOMPUTE SUBKEY TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 TMP7 205 mov \SUBKEY, %r12 206 movdqu (%r12), \TMP3 207 movdqa SHUF_MASK(%rip), \TMP2 208 PSHUFB_XMM \TMP2, \TMP3 209 210 # precompute HashKey<<1 mod poly from the HashKey (required for GHASH) 211 212 movdqa \TMP3, \TMP2 213 psllq $1, \TMP3 214 psrlq $63, \TMP2 215 movdqa \TMP2, \TMP1 216 pslldq $8, \TMP2 217 psrldq $8, \TMP1 218 por \TMP2, \TMP3 219 220 # reduce HashKey<<1 221 222 pshufd $0x24, \TMP1, \TMP2 223 pcmpeqd TWOONE(%rip), \TMP2 224 pand POLY(%rip), \TMP2 225 pxor \TMP2, \TMP3 226 movdqu \TMP3, HashKey(%arg2) 227 228 movdqa \TMP3, \TMP5 229 pshufd $78, \TMP3, \TMP1 230 pxor \TMP3, \TMP1 231 movdqu \TMP1, HashKey_k(%arg2) 232 233 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 234# TMP5 = HashKey^2<<1 (mod poly) 235 movdqu \TMP5, HashKey_2(%arg2) 236# HashKey_2 = HashKey^2<<1 (mod poly) 237 pshufd $78, \TMP5, \TMP1 238 pxor \TMP5, \TMP1 239 movdqu \TMP1, HashKey_2_k(%arg2) 240 241 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 242# TMP5 = HashKey^3<<1 (mod poly) 243 movdqu \TMP5, HashKey_3(%arg2) 244 pshufd $78, \TMP5, \TMP1 245 pxor \TMP5, \TMP1 246 movdqu \TMP1, HashKey_3_k(%arg2) 247 248 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 249# TMP5 = HashKey^3<<1 (mod poly) 250 movdqu \TMP5, HashKey_4(%arg2) 251 pshufd $78, \TMP5, \TMP1 252 pxor \TMP5, \TMP1 253 movdqu \TMP1, HashKey_4_k(%arg2) 254.endm 255 256# GCM_INIT initializes a gcm_context struct to prepare for encoding/decoding. 257# Clobbers rax, r10-r13 and xmm0-xmm6, %xmm13 258.macro GCM_INIT Iv SUBKEY AAD AADLEN 259 mov \AADLEN, %r11 260 mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length 261 xor %r11d, %r11d 262 mov %r11, InLen(%arg2) # ctx_data.in_length = 0 263 mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0 264 mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0 265 mov \Iv, %rax 266 movdqu (%rax), %xmm0 267 movdqu %xmm0, OrigIV(%arg2) # ctx_data.orig_IV = iv 268 269 movdqa SHUF_MASK(%rip), %xmm2 270 PSHUFB_XMM %xmm2, %xmm0 271 movdqu %xmm0, CurCount(%arg2) # ctx_data.current_counter = iv 272 273 PRECOMPUTE \SUBKEY, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, 274 movdqu HashKey(%arg2), %xmm13 275 276 CALC_AAD_HASH %xmm13, \AAD, \AADLEN, %xmm0, %xmm1, %xmm2, %xmm3, \ 277 %xmm4, %xmm5, %xmm6 278.endm 279 280# GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context 281# struct has been initialized by GCM_INIT. 282# Requires the input data be at least 1 byte long because of READ_PARTIAL_BLOCK 283# Clobbers rax, r10-r13, and xmm0-xmm15 284.macro GCM_ENC_DEC operation 285 movdqu AadHash(%arg2), %xmm8 286 movdqu HashKey(%arg2), %xmm13 287 add %arg5, InLen(%arg2) 288 289 xor %r11d, %r11d # initialise the data pointer offset as zero 290 PARTIAL_BLOCK %arg3 %arg4 %arg5 %r11 %xmm8 \operation 291 292 sub %r11, %arg5 # sub partial block data used 293 mov %arg5, %r13 # save the number of bytes 294 295 and $-16, %r13 # %r13 = %r13 - (%r13 mod 16) 296 mov %r13, %r12 297 # Encrypt/Decrypt first few blocks 298 299 and $(3<<4), %r12 300 jz _initial_num_blocks_is_0_\@ 301 cmp $(2<<4), %r12 302 jb _initial_num_blocks_is_1_\@ 303 je _initial_num_blocks_is_2_\@ 304_initial_num_blocks_is_3_\@: 305 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ 306%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, \operation 307 sub $48, %r13 308 jmp _initial_blocks_\@ 309_initial_num_blocks_is_2_\@: 310 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ 311%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, \operation 312 sub $32, %r13 313 jmp _initial_blocks_\@ 314_initial_num_blocks_is_1_\@: 315 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ 316%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, \operation 317 sub $16, %r13 318 jmp _initial_blocks_\@ 319_initial_num_blocks_is_0_\@: 320 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ 321%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, \operation 322_initial_blocks_\@: 323 324 # Main loop - Encrypt/Decrypt remaining blocks 325 326 cmp $0, %r13 327 je _zero_cipher_left_\@ 328 sub $64, %r13 329 je _four_cipher_left_\@ 330_crypt_by_4_\@: 331 GHASH_4_ENCRYPT_4_PARALLEL_\operation %xmm9, %xmm10, %xmm11, %xmm12, \ 332 %xmm13, %xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, \ 333 %xmm7, %xmm8, enc 334 add $64, %r11 335 sub $64, %r13 336 jne _crypt_by_4_\@ 337_four_cipher_left_\@: 338 GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \ 339%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8 340_zero_cipher_left_\@: 341 movdqu %xmm8, AadHash(%arg2) 342 movdqu %xmm0, CurCount(%arg2) 343 344 mov %arg5, %r13 345 and $15, %r13 # %r13 = arg5 (mod 16) 346 je _multiple_of_16_bytes_\@ 347 348 mov %r13, PBlockLen(%arg2) 349 350 # Handle the last <16 Byte block separately 351 paddd ONE(%rip), %xmm0 # INCR CNT to get Yn 352 movdqu %xmm0, CurCount(%arg2) 353 movdqa SHUF_MASK(%rip), %xmm10 354 PSHUFB_XMM %xmm10, %xmm0 355 356 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) 357 movdqu %xmm0, PBlockEncKey(%arg2) 358 359 cmp $16, %arg5 360 jge _large_enough_update_\@ 361 362 lea (%arg4,%r11,1), %r10 363 mov %r13, %r12 364 READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1 365 jmp _data_read_\@ 366 367_large_enough_update_\@: 368 sub $16, %r11 369 add %r13, %r11 370 371 # receive the last <16 Byte block 372 movdqu (%arg4, %r11, 1), %xmm1 373 374 sub %r13, %r11 375 add $16, %r11 376 377 lea SHIFT_MASK+16(%rip), %r12 378 # adjust the shuffle mask pointer to be able to shift 16-r13 bytes 379 # (r13 is the number of bytes in plaintext mod 16) 380 sub %r13, %r12 381 # get the appropriate shuffle mask 382 movdqu (%r12), %xmm2 383 # shift right 16-r13 bytes 384 PSHUFB_XMM %xmm2, %xmm1 385 386_data_read_\@: 387 lea ALL_F+16(%rip), %r12 388 sub %r13, %r12 389 390.ifc \operation, dec 391 movdqa %xmm1, %xmm2 392.endif 393 pxor %xmm1, %xmm0 # XOR Encrypt(K, Yn) 394 movdqu (%r12), %xmm1 395 # get the appropriate mask to mask out top 16-r13 bytes of xmm0 396 pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0 397.ifc \operation, dec 398 pand %xmm1, %xmm2 399 movdqa SHUF_MASK(%rip), %xmm10 400 PSHUFB_XMM %xmm10 ,%xmm2 401 402 pxor %xmm2, %xmm8 403.else 404 movdqa SHUF_MASK(%rip), %xmm10 405 PSHUFB_XMM %xmm10,%xmm0 406 407 pxor %xmm0, %xmm8 408.endif 409 410 movdqu %xmm8, AadHash(%arg2) 411.ifc \operation, enc 412 # GHASH computation for the last <16 byte block 413 movdqa SHUF_MASK(%rip), %xmm10 414 # shuffle xmm0 back to output as ciphertext 415 PSHUFB_XMM %xmm10, %xmm0 416.endif 417 418 # Output %r13 bytes 419 MOVQ_R64_XMM %xmm0, %rax 420 cmp $8, %r13 421 jle _less_than_8_bytes_left_\@ 422 mov %rax, (%arg3 , %r11, 1) 423 add $8, %r11 424 psrldq $8, %xmm0 425 MOVQ_R64_XMM %xmm0, %rax 426 sub $8, %r13 427_less_than_8_bytes_left_\@: 428 mov %al, (%arg3, %r11, 1) 429 add $1, %r11 430 shr $8, %rax 431 sub $1, %r13 432 jne _less_than_8_bytes_left_\@ 433_multiple_of_16_bytes_\@: 434.endm 435 436# GCM_COMPLETE Finishes update of tag of last partial block 437# Output: Authorization Tag (AUTH_TAG) 438# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15 439.macro GCM_COMPLETE AUTHTAG AUTHTAGLEN 440 movdqu AadHash(%arg2), %xmm8 441 movdqu HashKey(%arg2), %xmm13 442 443 mov PBlockLen(%arg2), %r12 444 445 cmp $0, %r12 446 je _partial_done\@ 447 448 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 449 450_partial_done\@: 451 mov AadLen(%arg2), %r12 # %r13 = aadLen (number of bytes) 452 shl $3, %r12 # convert into number of bits 453 movd %r12d, %xmm15 # len(A) in %xmm15 454 mov InLen(%arg2), %r12 455 shl $3, %r12 # len(C) in bits (*128) 456 MOVQ_R64_XMM %r12, %xmm1 457 458 pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000 459 pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C) 460 pxor %xmm15, %xmm8 461 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 462 # final GHASH computation 463 movdqa SHUF_MASK(%rip), %xmm10 464 PSHUFB_XMM %xmm10, %xmm8 465 466 movdqu OrigIV(%arg2), %xmm0 # %xmm0 = Y0 467 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0) 468 pxor %xmm8, %xmm0 469_return_T_\@: 470 mov \AUTHTAG, %r10 # %r10 = authTag 471 mov \AUTHTAGLEN, %r11 # %r11 = auth_tag_len 472 cmp $16, %r11 473 je _T_16_\@ 474 cmp $8, %r11 475 jl _T_4_\@ 476_T_8_\@: 477 MOVQ_R64_XMM %xmm0, %rax 478 mov %rax, (%r10) 479 add $8, %r10 480 sub $8, %r11 481 psrldq $8, %xmm0 482 cmp $0, %r11 483 je _return_T_done_\@ 484_T_4_\@: 485 movd %xmm0, %eax 486 mov %eax, (%r10) 487 add $4, %r10 488 sub $4, %r11 489 psrldq $4, %xmm0 490 cmp $0, %r11 491 je _return_T_done_\@ 492_T_123_\@: 493 movd %xmm0, %eax 494 cmp $2, %r11 495 jl _T_1_\@ 496 mov %ax, (%r10) 497 cmp $2, %r11 498 je _return_T_done_\@ 499 add $2, %r10 500 sar $16, %eax 501_T_1_\@: 502 mov %al, (%r10) 503 jmp _return_T_done_\@ 504_T_16_\@: 505 movdqu %xmm0, (%r10) 506_return_T_done_\@: 507.endm 508 509#ifdef __x86_64__ 510/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) 511* 512* 513* Input: A and B (128-bits each, bit-reflected) 514* Output: C = A*B*x mod poly, (i.e. >>1 ) 515* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input 516* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. 517* 518*/ 519.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5 520 movdqa \GH, \TMP1 521 pshufd $78, \GH, \TMP2 522 pshufd $78, \HK, \TMP3 523 pxor \GH, \TMP2 # TMP2 = a1+a0 524 pxor \HK, \TMP3 # TMP3 = b1+b0 525 PCLMULQDQ 0x11, \HK, \TMP1 # TMP1 = a1*b1 526 PCLMULQDQ 0x00, \HK, \GH # GH = a0*b0 527 PCLMULQDQ 0x00, \TMP3, \TMP2 # TMP2 = (a0+a1)*(b1+b0) 528 pxor \GH, \TMP2 529 pxor \TMP1, \TMP2 # TMP2 = (a0*b0)+(a1*b0) 530 movdqa \TMP2, \TMP3 531 pslldq $8, \TMP3 # left shift TMP3 2 DWs 532 psrldq $8, \TMP2 # right shift TMP2 2 DWs 533 pxor \TMP3, \GH 534 pxor \TMP2, \TMP1 # TMP2:GH holds the result of GH*HK 535 536 # first phase of the reduction 537 538 movdqa \GH, \TMP2 539 movdqa \GH, \TMP3 540 movdqa \GH, \TMP4 # copy GH into TMP2,TMP3 and TMP4 541 # in in order to perform 542 # independent shifts 543 pslld $31, \TMP2 # packed right shift <<31 544 pslld $30, \TMP3 # packed right shift <<30 545 pslld $25, \TMP4 # packed right shift <<25 546 pxor \TMP3, \TMP2 # xor the shifted versions 547 pxor \TMP4, \TMP2 548 movdqa \TMP2, \TMP5 549 psrldq $4, \TMP5 # right shift TMP5 1 DW 550 pslldq $12, \TMP2 # left shift TMP2 3 DWs 551 pxor \TMP2, \GH 552 553 # second phase of the reduction 554 555 movdqa \GH,\TMP2 # copy GH into TMP2,TMP3 and TMP4 556 # in in order to perform 557 # independent shifts 558 movdqa \GH,\TMP3 559 movdqa \GH,\TMP4 560 psrld $1,\TMP2 # packed left shift >>1 561 psrld $2,\TMP3 # packed left shift >>2 562 psrld $7,\TMP4 # packed left shift >>7 563 pxor \TMP3,\TMP2 # xor the shifted versions 564 pxor \TMP4,\TMP2 565 pxor \TMP5, \TMP2 566 pxor \TMP2, \GH 567 pxor \TMP1, \GH # result is in TMP1 568.endm 569 570# Reads DLEN bytes starting at DPTR and stores in XMMDst 571# where 0 < DLEN < 16 572# Clobbers %rax, DLEN and XMM1 573.macro READ_PARTIAL_BLOCK DPTR DLEN XMM1 XMMDst 574 cmp $8, \DLEN 575 jl _read_lt8_\@ 576 mov (\DPTR), %rax 577 MOVQ_R64_XMM %rax, \XMMDst 578 sub $8, \DLEN 579 jz _done_read_partial_block_\@ 580 xor %eax, %eax 581_read_next_byte_\@: 582 shl $8, %rax 583 mov 7(\DPTR, \DLEN, 1), %al 584 dec \DLEN 585 jnz _read_next_byte_\@ 586 MOVQ_R64_XMM %rax, \XMM1 587 pslldq $8, \XMM1 588 por \XMM1, \XMMDst 589 jmp _done_read_partial_block_\@ 590_read_lt8_\@: 591 xor %eax, %eax 592_read_next_byte_lt8_\@: 593 shl $8, %rax 594 mov -1(\DPTR, \DLEN, 1), %al 595 dec \DLEN 596 jnz _read_next_byte_lt8_\@ 597 MOVQ_R64_XMM %rax, \XMMDst 598_done_read_partial_block_\@: 599.endm 600 601# CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. 602# clobbers r10-11, xmm14 603.macro CALC_AAD_HASH HASHKEY AAD AADLEN TMP1 TMP2 TMP3 TMP4 TMP5 \ 604 TMP6 TMP7 605 MOVADQ SHUF_MASK(%rip), %xmm14 606 mov \AAD, %r10 # %r10 = AAD 607 mov \AADLEN, %r11 # %r11 = aadLen 608 pxor \TMP7, \TMP7 609 pxor \TMP6, \TMP6 610 611 cmp $16, %r11 612 jl _get_AAD_rest\@ 613_get_AAD_blocks\@: 614 movdqu (%r10), \TMP7 615 PSHUFB_XMM %xmm14, \TMP7 # byte-reflect the AAD data 616 pxor \TMP7, \TMP6 617 GHASH_MUL \TMP6, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5 618 add $16, %r10 619 sub $16, %r11 620 cmp $16, %r11 621 jge _get_AAD_blocks\@ 622 623 movdqu \TMP6, \TMP7 624 625 /* read the last <16B of AAD */ 626_get_AAD_rest\@: 627 cmp $0, %r11 628 je _get_AAD_done\@ 629 630 READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7 631 PSHUFB_XMM %xmm14, \TMP7 # byte-reflect the AAD data 632 pxor \TMP6, \TMP7 633 GHASH_MUL \TMP7, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5 634 movdqu \TMP7, \TMP6 635 636_get_AAD_done\@: 637 movdqu \TMP6, AadHash(%arg2) 638.endm 639 640# PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks 641# between update calls. 642# Requires the input data be at least 1 byte long due to READ_PARTIAL_BLOCK 643# Outputs encrypted bytes, and updates hash and partial info in gcm_data_context 644# Clobbers rax, r10, r12, r13, xmm0-6, xmm9-13 645.macro PARTIAL_BLOCK CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ 646 AAD_HASH operation 647 mov PBlockLen(%arg2), %r13 648 cmp $0, %r13 649 je _partial_block_done_\@ # Leave Macro if no partial blocks 650 # Read in input data without over reading 651 cmp $16, \PLAIN_CYPH_LEN 652 jl _fewer_than_16_bytes_\@ 653 movups (\PLAIN_CYPH_IN), %xmm1 # If more than 16 bytes, just fill xmm 654 jmp _data_read_\@ 655 656_fewer_than_16_bytes_\@: 657 lea (\PLAIN_CYPH_IN, \DATA_OFFSET, 1), %r10 658 mov \PLAIN_CYPH_LEN, %r12 659 READ_PARTIAL_BLOCK %r10 %r12 %xmm0 %xmm1 660 661 mov PBlockLen(%arg2), %r13 662 663_data_read_\@: # Finished reading in data 664 665 movdqu PBlockEncKey(%arg2), %xmm9 666 movdqu HashKey(%arg2), %xmm13 667 668 lea SHIFT_MASK(%rip), %r12 669 670 # adjust the shuffle mask pointer to be able to shift r13 bytes 671 # r16-r13 is the number of bytes in plaintext mod 16) 672 add %r13, %r12 673 movdqu (%r12), %xmm2 # get the appropriate shuffle mask 674 PSHUFB_XMM %xmm2, %xmm9 # shift right r13 bytes 675 676.ifc \operation, dec 677 movdqa %xmm1, %xmm3 678 pxor %xmm1, %xmm9 # Cyphertext XOR E(K, Yn) 679 680 mov \PLAIN_CYPH_LEN, %r10 681 add %r13, %r10 682 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling 683 sub $16, %r10 684 # Determine if if partial block is not being filled and 685 # shift mask accordingly 686 jge _no_extra_mask_1_\@ 687 sub %r10, %r12 688_no_extra_mask_1_\@: 689 690 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 691 # get the appropriate mask to mask out bottom r13 bytes of xmm9 692 pand %xmm1, %xmm9 # mask out bottom r13 bytes of xmm9 693 694 pand %xmm1, %xmm3 695 movdqa SHUF_MASK(%rip), %xmm10 696 PSHUFB_XMM %xmm10, %xmm3 697 PSHUFB_XMM %xmm2, %xmm3 698 pxor %xmm3, \AAD_HASH 699 700 cmp $0, %r10 701 jl _partial_incomplete_1_\@ 702 703 # GHASH computation for the last <16 Byte block 704 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 705 xor %eax, %eax 706 707 mov %rax, PBlockLen(%arg2) 708 jmp _dec_done_\@ 709_partial_incomplete_1_\@: 710 add \PLAIN_CYPH_LEN, PBlockLen(%arg2) 711_dec_done_\@: 712 movdqu \AAD_HASH, AadHash(%arg2) 713.else 714 pxor %xmm1, %xmm9 # Plaintext XOR E(K, Yn) 715 716 mov \PLAIN_CYPH_LEN, %r10 717 add %r13, %r10 718 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling 719 sub $16, %r10 720 # Determine if if partial block is not being filled and 721 # shift mask accordingly 722 jge _no_extra_mask_2_\@ 723 sub %r10, %r12 724_no_extra_mask_2_\@: 725 726 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 727 # get the appropriate mask to mask out bottom r13 bytes of xmm9 728 pand %xmm1, %xmm9 729 730 movdqa SHUF_MASK(%rip), %xmm1 731 PSHUFB_XMM %xmm1, %xmm9 732 PSHUFB_XMM %xmm2, %xmm9 733 pxor %xmm9, \AAD_HASH 734 735 cmp $0, %r10 736 jl _partial_incomplete_2_\@ 737 738 # GHASH computation for the last <16 Byte block 739 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 740 xor %eax, %eax 741 742 mov %rax, PBlockLen(%arg2) 743 jmp _encode_done_\@ 744_partial_incomplete_2_\@: 745 add \PLAIN_CYPH_LEN, PBlockLen(%arg2) 746_encode_done_\@: 747 movdqu \AAD_HASH, AadHash(%arg2) 748 749 movdqa SHUF_MASK(%rip), %xmm10 750 # shuffle xmm9 back to output as ciphertext 751 PSHUFB_XMM %xmm10, %xmm9 752 PSHUFB_XMM %xmm2, %xmm9 753.endif 754 # output encrypted Bytes 755 cmp $0, %r10 756 jl _partial_fill_\@ 757 mov %r13, %r12 758 mov $16, %r13 759 # Set r13 to be the number of bytes to write out 760 sub %r12, %r13 761 jmp _count_set_\@ 762_partial_fill_\@: 763 mov \PLAIN_CYPH_LEN, %r13 764_count_set_\@: 765 movdqa %xmm9, %xmm0 766 MOVQ_R64_XMM %xmm0, %rax 767 cmp $8, %r13 768 jle _less_than_8_bytes_left_\@ 769 770 mov %rax, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1) 771 add $8, \DATA_OFFSET 772 psrldq $8, %xmm0 773 MOVQ_R64_XMM %xmm0, %rax 774 sub $8, %r13 775_less_than_8_bytes_left_\@: 776 movb %al, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1) 777 add $1, \DATA_OFFSET 778 shr $8, %rax 779 sub $1, %r13 780 jne _less_than_8_bytes_left_\@ 781_partial_block_done_\@: 782.endm # PARTIAL_BLOCK 783 784/* 785* if a = number of total plaintext bytes 786* b = floor(a/16) 787* num_initial_blocks = b mod 4 788* encrypt the initial num_initial_blocks blocks and apply ghash on 789* the ciphertext 790* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers 791* are clobbered 792* arg1, %arg2, %arg3 are used as a pointer only, not modified 793*/ 794 795 796.macro INITIAL_BLOCKS_ENC_DEC TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ 797 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation 798 MOVADQ SHUF_MASK(%rip), %xmm14 799 800 movdqu AadHash(%arg2), %xmm\i # XMM0 = Y0 801 802 # start AES for num_initial_blocks blocks 803 804 movdqu CurCount(%arg2), \XMM0 # XMM0 = Y0 805 806.if (\i == 5) || (\i == 6) || (\i == 7) 807 808 MOVADQ ONE(%RIP),\TMP1 809 MOVADQ 0(%arg1),\TMP2 810.irpc index, \i_seq 811 paddd \TMP1, \XMM0 # INCR Y0 812.ifc \operation, dec 813 movdqa \XMM0, %xmm\index 814.else 815 MOVADQ \XMM0, %xmm\index 816.endif 817 PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap 818 pxor \TMP2, %xmm\index 819.endr 820 lea 0x10(%arg1),%r10 821 mov keysize,%eax 822 shr $2,%eax # 128->4, 192->6, 256->8 823 add $5,%eax # 128->9, 192->11, 256->13 824 825aes_loop_initial_\@: 826 MOVADQ (%r10),\TMP1 827.irpc index, \i_seq 828 AESENC \TMP1, %xmm\index 829.endr 830 add $16,%r10 831 sub $1,%eax 832 jnz aes_loop_initial_\@ 833 834 MOVADQ (%r10), \TMP1 835.irpc index, \i_seq 836 AESENCLAST \TMP1, %xmm\index # Last Round 837.endr 838.irpc index, \i_seq 839 movdqu (%arg4 , %r11, 1), \TMP1 840 pxor \TMP1, %xmm\index 841 movdqu %xmm\index, (%arg3 , %r11, 1) 842 # write back plaintext/ciphertext for num_initial_blocks 843 add $16, %r11 844 845.ifc \operation, dec 846 movdqa \TMP1, %xmm\index 847.endif 848 PSHUFB_XMM %xmm14, %xmm\index 849 850 # prepare plaintext/ciphertext for GHASH computation 851.endr 852.endif 853 854 # apply GHASH on num_initial_blocks blocks 855 856.if \i == 5 857 pxor %xmm5, %xmm6 858 GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 859 pxor %xmm6, %xmm7 860 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 861 pxor %xmm7, %xmm8 862 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 863.elseif \i == 6 864 pxor %xmm6, %xmm7 865 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 866 pxor %xmm7, %xmm8 867 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 868.elseif \i == 7 869 pxor %xmm7, %xmm8 870 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 871.endif 872 cmp $64, %r13 873 jl _initial_blocks_done\@ 874 # no need for precomputed values 875/* 876* 877* Precomputations for HashKey parallel with encryption of first 4 blocks. 878* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i 879*/ 880 MOVADQ ONE(%RIP),\TMP1 881 paddd \TMP1, \XMM0 # INCR Y0 882 MOVADQ \XMM0, \XMM1 883 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap 884 885 paddd \TMP1, \XMM0 # INCR Y0 886 MOVADQ \XMM0, \XMM2 887 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap 888 889 paddd \TMP1, \XMM0 # INCR Y0 890 MOVADQ \XMM0, \XMM3 891 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap 892 893 paddd \TMP1, \XMM0 # INCR Y0 894 MOVADQ \XMM0, \XMM4 895 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap 896 897 MOVADQ 0(%arg1),\TMP1 898 pxor \TMP1, \XMM1 899 pxor \TMP1, \XMM2 900 pxor \TMP1, \XMM3 901 pxor \TMP1, \XMM4 902.irpc index, 1234 # do 4 rounds 903 movaps 0x10*\index(%arg1), \TMP1 904 AESENC \TMP1, \XMM1 905 AESENC \TMP1, \XMM2 906 AESENC \TMP1, \XMM3 907 AESENC \TMP1, \XMM4 908.endr 909.irpc index, 56789 # do next 5 rounds 910 movaps 0x10*\index(%arg1), \TMP1 911 AESENC \TMP1, \XMM1 912 AESENC \TMP1, \XMM2 913 AESENC \TMP1, \XMM3 914 AESENC \TMP1, \XMM4 915.endr 916 lea 0xa0(%arg1),%r10 917 mov keysize,%eax 918 shr $2,%eax # 128->4, 192->6, 256->8 919 sub $4,%eax # 128->0, 192->2, 256->4 920 jz aes_loop_pre_done\@ 921 922aes_loop_pre_\@: 923 MOVADQ (%r10),\TMP2 924.irpc index, 1234 925 AESENC \TMP2, %xmm\index 926.endr 927 add $16,%r10 928 sub $1,%eax 929 jnz aes_loop_pre_\@ 930 931aes_loop_pre_done\@: 932 MOVADQ (%r10), \TMP2 933 AESENCLAST \TMP2, \XMM1 934 AESENCLAST \TMP2, \XMM2 935 AESENCLAST \TMP2, \XMM3 936 AESENCLAST \TMP2, \XMM4 937 movdqu 16*0(%arg4 , %r11 , 1), \TMP1 938 pxor \TMP1, \XMM1 939.ifc \operation, dec 940 movdqu \XMM1, 16*0(%arg3 , %r11 , 1) 941 movdqa \TMP1, \XMM1 942.endif 943 movdqu 16*1(%arg4 , %r11 , 1), \TMP1 944 pxor \TMP1, \XMM2 945.ifc \operation, dec 946 movdqu \XMM2, 16*1(%arg3 , %r11 , 1) 947 movdqa \TMP1, \XMM2 948.endif 949 movdqu 16*2(%arg4 , %r11 , 1), \TMP1 950 pxor \TMP1, \XMM3 951.ifc \operation, dec 952 movdqu \XMM3, 16*2(%arg3 , %r11 , 1) 953 movdqa \TMP1, \XMM3 954.endif 955 movdqu 16*3(%arg4 , %r11 , 1), \TMP1 956 pxor \TMP1, \XMM4 957.ifc \operation, dec 958 movdqu \XMM4, 16*3(%arg3 , %r11 , 1) 959 movdqa \TMP1, \XMM4 960.else 961 movdqu \XMM1, 16*0(%arg3 , %r11 , 1) 962 movdqu \XMM2, 16*1(%arg3 , %r11 , 1) 963 movdqu \XMM3, 16*2(%arg3 , %r11 , 1) 964 movdqu \XMM4, 16*3(%arg3 , %r11 , 1) 965.endif 966 967 add $64, %r11 968 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap 969 pxor \XMMDst, \XMM1 970# combine GHASHed value with the corresponding ciphertext 971 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap 972 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap 973 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap 974 975_initial_blocks_done\@: 976 977.endm 978 979/* 980* encrypt 4 blocks at a time 981* ghash the 4 previously encrypted ciphertext blocks 982* arg1, %arg3, %arg4 are used as pointers only, not modified 983* %r11 is the data offset value 984*/ 985.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \ 986TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 987 988 movdqa \XMM1, \XMM5 989 movdqa \XMM2, \XMM6 990 movdqa \XMM3, \XMM7 991 movdqa \XMM4, \XMM8 992 993 movdqa SHUF_MASK(%rip), %xmm15 994 # multiply TMP5 * HashKey using karatsuba 995 996 movdqa \XMM5, \TMP4 997 pshufd $78, \XMM5, \TMP6 998 pxor \XMM5, \TMP6 999 paddd ONE(%rip), \XMM0 # INCR CNT 1000 movdqu HashKey_4(%arg2), \TMP5 1001 PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 1002 movdqa \XMM0, \XMM1 1003 paddd ONE(%rip), \XMM0 # INCR CNT 1004 movdqa \XMM0, \XMM2 1005 paddd ONE(%rip), \XMM0 # INCR CNT 1006 movdqa \XMM0, \XMM3 1007 paddd ONE(%rip), \XMM0 # INCR CNT 1008 movdqa \XMM0, \XMM4 1009 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap 1010 PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 1011 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap 1012 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap 1013 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap 1014 1015 pxor (%arg1), \XMM1 1016 pxor (%arg1), \XMM2 1017 pxor (%arg1), \XMM3 1018 pxor (%arg1), \XMM4 1019 movdqu HashKey_4_k(%arg2), \TMP5 1020 PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) 1021 movaps 0x10(%arg1), \TMP1 1022 AESENC \TMP1, \XMM1 # Round 1 1023 AESENC \TMP1, \XMM2 1024 AESENC \TMP1, \XMM3 1025 AESENC \TMP1, \XMM4 1026 movaps 0x20(%arg1), \TMP1 1027 AESENC \TMP1, \XMM1 # Round 2 1028 AESENC \TMP1, \XMM2 1029 AESENC \TMP1, \XMM3 1030 AESENC \TMP1, \XMM4 1031 movdqa \XMM6, \TMP1 1032 pshufd $78, \XMM6, \TMP2 1033 pxor \XMM6, \TMP2 1034 movdqu HashKey_3(%arg2), \TMP5 1035 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 1036 movaps 0x30(%arg1), \TMP3 1037 AESENC \TMP3, \XMM1 # Round 3 1038 AESENC \TMP3, \XMM2 1039 AESENC \TMP3, \XMM3 1040 AESENC \TMP3, \XMM4 1041 PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 1042 movaps 0x40(%arg1), \TMP3 1043 AESENC \TMP3, \XMM1 # Round 4 1044 AESENC \TMP3, \XMM2 1045 AESENC \TMP3, \XMM3 1046 AESENC \TMP3, \XMM4 1047 movdqu HashKey_3_k(%arg2), \TMP5 1048 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1049 movaps 0x50(%arg1), \TMP3 1050 AESENC \TMP3, \XMM1 # Round 5 1051 AESENC \TMP3, \XMM2 1052 AESENC \TMP3, \XMM3 1053 AESENC \TMP3, \XMM4 1054 pxor \TMP1, \TMP4 1055# accumulate the results in TMP4:XMM5, TMP6 holds the middle part 1056 pxor \XMM6, \XMM5 1057 pxor \TMP2, \TMP6 1058 movdqa \XMM7, \TMP1 1059 pshufd $78, \XMM7, \TMP2 1060 pxor \XMM7, \TMP2 1061 movdqu HashKey_2(%arg2), \TMP5 1062 1063 # Multiply TMP5 * HashKey using karatsuba 1064 1065 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1066 movaps 0x60(%arg1), \TMP3 1067 AESENC \TMP3, \XMM1 # Round 6 1068 AESENC \TMP3, \XMM2 1069 AESENC \TMP3, \XMM3 1070 AESENC \TMP3, \XMM4 1071 PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 1072 movaps 0x70(%arg1), \TMP3 1073 AESENC \TMP3, \XMM1 # Round 7 1074 AESENC \TMP3, \XMM2 1075 AESENC \TMP3, \XMM3 1076 AESENC \TMP3, \XMM4 1077 movdqu HashKey_2_k(%arg2), \TMP5 1078 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1079 movaps 0x80(%arg1), \TMP3 1080 AESENC \TMP3, \XMM1 # Round 8 1081 AESENC \TMP3, \XMM2 1082 AESENC \TMP3, \XMM3 1083 AESENC \TMP3, \XMM4 1084 pxor \TMP1, \TMP4 1085# accumulate the results in TMP4:XMM5, TMP6 holds the middle part 1086 pxor \XMM7, \XMM5 1087 pxor \TMP2, \TMP6 1088 1089 # Multiply XMM8 * HashKey 1090 # XMM8 and TMP5 hold the values for the two operands 1091 1092 movdqa \XMM8, \TMP1 1093 pshufd $78, \XMM8, \TMP2 1094 pxor \XMM8, \TMP2 1095 movdqu HashKey(%arg2), \TMP5 1096 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1097 movaps 0x90(%arg1), \TMP3 1098 AESENC \TMP3, \XMM1 # Round 9 1099 AESENC \TMP3, \XMM2 1100 AESENC \TMP3, \XMM3 1101 AESENC \TMP3, \XMM4 1102 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 1103 lea 0xa0(%arg1),%r10 1104 mov keysize,%eax 1105 shr $2,%eax # 128->4, 192->6, 256->8 1106 sub $4,%eax # 128->0, 192->2, 256->4 1107 jz aes_loop_par_enc_done\@ 1108 1109aes_loop_par_enc\@: 1110 MOVADQ (%r10),\TMP3 1111.irpc index, 1234 1112 AESENC \TMP3, %xmm\index 1113.endr 1114 add $16,%r10 1115 sub $1,%eax 1116 jnz aes_loop_par_enc\@ 1117 1118aes_loop_par_enc_done\@: 1119 MOVADQ (%r10), \TMP3 1120 AESENCLAST \TMP3, \XMM1 # Round 10 1121 AESENCLAST \TMP3, \XMM2 1122 AESENCLAST \TMP3, \XMM3 1123 AESENCLAST \TMP3, \XMM4 1124 movdqu HashKey_k(%arg2), \TMP5 1125 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1126 movdqu (%arg4,%r11,1), \TMP3 1127 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK 1128 movdqu 16(%arg4,%r11,1), \TMP3 1129 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK 1130 movdqu 32(%arg4,%r11,1), \TMP3 1131 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK 1132 movdqu 48(%arg4,%r11,1), \TMP3 1133 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK 1134 movdqu \XMM1, (%arg3,%r11,1) # Write to the ciphertext buffer 1135 movdqu \XMM2, 16(%arg3,%r11,1) # Write to the ciphertext buffer 1136 movdqu \XMM3, 32(%arg3,%r11,1) # Write to the ciphertext buffer 1137 movdqu \XMM4, 48(%arg3,%r11,1) # Write to the ciphertext buffer 1138 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap 1139 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap 1140 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap 1141 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap 1142 1143 pxor \TMP4, \TMP1 1144 pxor \XMM8, \XMM5 1145 pxor \TMP6, \TMP2 1146 pxor \TMP1, \TMP2 1147 pxor \XMM5, \TMP2 1148 movdqa \TMP2, \TMP3 1149 pslldq $8, \TMP3 # left shift TMP3 2 DWs 1150 psrldq $8, \TMP2 # right shift TMP2 2 DWs 1151 pxor \TMP3, \XMM5 1152 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 1153 1154 # first phase of reduction 1155 1156 movdqa \XMM5, \TMP2 1157 movdqa \XMM5, \TMP3 1158 movdqa \XMM5, \TMP4 1159# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently 1160 pslld $31, \TMP2 # packed right shift << 31 1161 pslld $30, \TMP3 # packed right shift << 30 1162 pslld $25, \TMP4 # packed right shift << 25 1163 pxor \TMP3, \TMP2 # xor the shifted versions 1164 pxor \TMP4, \TMP2 1165 movdqa \TMP2, \TMP5 1166 psrldq $4, \TMP5 # right shift T5 1 DW 1167 pslldq $12, \TMP2 # left shift T2 3 DWs 1168 pxor \TMP2, \XMM5 1169 1170 # second phase of reduction 1171 1172 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 1173 movdqa \XMM5,\TMP3 1174 movdqa \XMM5,\TMP4 1175 psrld $1, \TMP2 # packed left shift >>1 1176 psrld $2, \TMP3 # packed left shift >>2 1177 psrld $7, \TMP4 # packed left shift >>7 1178 pxor \TMP3,\TMP2 # xor the shifted versions 1179 pxor \TMP4,\TMP2 1180 pxor \TMP5, \TMP2 1181 pxor \TMP2, \XMM5 1182 pxor \TMP1, \XMM5 # result is in TMP1 1183 1184 pxor \XMM5, \XMM1 1185.endm 1186 1187/* 1188* decrypt 4 blocks at a time 1189* ghash the 4 previously decrypted ciphertext blocks 1190* arg1, %arg3, %arg4 are used as pointers only, not modified 1191* %r11 is the data offset value 1192*/ 1193.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \ 1194TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation 1195 1196 movdqa \XMM1, \XMM5 1197 movdqa \XMM2, \XMM6 1198 movdqa \XMM3, \XMM7 1199 movdqa \XMM4, \XMM8 1200 1201 movdqa SHUF_MASK(%rip), %xmm15 1202 # multiply TMP5 * HashKey using karatsuba 1203 1204 movdqa \XMM5, \TMP4 1205 pshufd $78, \XMM5, \TMP6 1206 pxor \XMM5, \TMP6 1207 paddd ONE(%rip), \XMM0 # INCR CNT 1208 movdqu HashKey_4(%arg2), \TMP5 1209 PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 1210 movdqa \XMM0, \XMM1 1211 paddd ONE(%rip), \XMM0 # INCR CNT 1212 movdqa \XMM0, \XMM2 1213 paddd ONE(%rip), \XMM0 # INCR CNT 1214 movdqa \XMM0, \XMM3 1215 paddd ONE(%rip), \XMM0 # INCR CNT 1216 movdqa \XMM0, \XMM4 1217 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap 1218 PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 1219 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap 1220 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap 1221 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap 1222 1223 pxor (%arg1), \XMM1 1224 pxor (%arg1), \XMM2 1225 pxor (%arg1), \XMM3 1226 pxor (%arg1), \XMM4 1227 movdqu HashKey_4_k(%arg2), \TMP5 1228 PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) 1229 movaps 0x10(%arg1), \TMP1 1230 AESENC \TMP1, \XMM1 # Round 1 1231 AESENC \TMP1, \XMM2 1232 AESENC \TMP1, \XMM3 1233 AESENC \TMP1, \XMM4 1234 movaps 0x20(%arg1), \TMP1 1235 AESENC \TMP1, \XMM1 # Round 2 1236 AESENC \TMP1, \XMM2 1237 AESENC \TMP1, \XMM3 1238 AESENC \TMP1, \XMM4 1239 movdqa \XMM6, \TMP1 1240 pshufd $78, \XMM6, \TMP2 1241 pxor \XMM6, \TMP2 1242 movdqu HashKey_3(%arg2), \TMP5 1243 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 1244 movaps 0x30(%arg1), \TMP3 1245 AESENC \TMP3, \XMM1 # Round 3 1246 AESENC \TMP3, \XMM2 1247 AESENC \TMP3, \XMM3 1248 AESENC \TMP3, \XMM4 1249 PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 1250 movaps 0x40(%arg1), \TMP3 1251 AESENC \TMP3, \XMM1 # Round 4 1252 AESENC \TMP3, \XMM2 1253 AESENC \TMP3, \XMM3 1254 AESENC \TMP3, \XMM4 1255 movdqu HashKey_3_k(%arg2), \TMP5 1256 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1257 movaps 0x50(%arg1), \TMP3 1258 AESENC \TMP3, \XMM1 # Round 5 1259 AESENC \TMP3, \XMM2 1260 AESENC \TMP3, \XMM3 1261 AESENC \TMP3, \XMM4 1262 pxor \TMP1, \TMP4 1263# accumulate the results in TMP4:XMM5, TMP6 holds the middle part 1264 pxor \XMM6, \XMM5 1265 pxor \TMP2, \TMP6 1266 movdqa \XMM7, \TMP1 1267 pshufd $78, \XMM7, \TMP2 1268 pxor \XMM7, \TMP2 1269 movdqu HashKey_2(%arg2), \TMP5 1270 1271 # Multiply TMP5 * HashKey using karatsuba 1272 1273 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1274 movaps 0x60(%arg1), \TMP3 1275 AESENC \TMP3, \XMM1 # Round 6 1276 AESENC \TMP3, \XMM2 1277 AESENC \TMP3, \XMM3 1278 AESENC \TMP3, \XMM4 1279 PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 1280 movaps 0x70(%arg1), \TMP3 1281 AESENC \TMP3, \XMM1 # Round 7 1282 AESENC \TMP3, \XMM2 1283 AESENC \TMP3, \XMM3 1284 AESENC \TMP3, \XMM4 1285 movdqu HashKey_2_k(%arg2), \TMP5 1286 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1287 movaps 0x80(%arg1), \TMP3 1288 AESENC \TMP3, \XMM1 # Round 8 1289 AESENC \TMP3, \XMM2 1290 AESENC \TMP3, \XMM3 1291 AESENC \TMP3, \XMM4 1292 pxor \TMP1, \TMP4 1293# accumulate the results in TMP4:XMM5, TMP6 holds the middle part 1294 pxor \XMM7, \XMM5 1295 pxor \TMP2, \TMP6 1296 1297 # Multiply XMM8 * HashKey 1298 # XMM8 and TMP5 hold the values for the two operands 1299 1300 movdqa \XMM8, \TMP1 1301 pshufd $78, \XMM8, \TMP2 1302 pxor \XMM8, \TMP2 1303 movdqu HashKey(%arg2), \TMP5 1304 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1305 movaps 0x90(%arg1), \TMP3 1306 AESENC \TMP3, \XMM1 # Round 9 1307 AESENC \TMP3, \XMM2 1308 AESENC \TMP3, \XMM3 1309 AESENC \TMP3, \XMM4 1310 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 1311 lea 0xa0(%arg1),%r10 1312 mov keysize,%eax 1313 shr $2,%eax # 128->4, 192->6, 256->8 1314 sub $4,%eax # 128->0, 192->2, 256->4 1315 jz aes_loop_par_dec_done\@ 1316 1317aes_loop_par_dec\@: 1318 MOVADQ (%r10),\TMP3 1319.irpc index, 1234 1320 AESENC \TMP3, %xmm\index 1321.endr 1322 add $16,%r10 1323 sub $1,%eax 1324 jnz aes_loop_par_dec\@ 1325 1326aes_loop_par_dec_done\@: 1327 MOVADQ (%r10), \TMP3 1328 AESENCLAST \TMP3, \XMM1 # last round 1329 AESENCLAST \TMP3, \XMM2 1330 AESENCLAST \TMP3, \XMM3 1331 AESENCLAST \TMP3, \XMM4 1332 movdqu HashKey_k(%arg2), \TMP5 1333 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1334 movdqu (%arg4,%r11,1), \TMP3 1335 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK 1336 movdqu \XMM1, (%arg3,%r11,1) # Write to plaintext buffer 1337 movdqa \TMP3, \XMM1 1338 movdqu 16(%arg4,%r11,1), \TMP3 1339 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK 1340 movdqu \XMM2, 16(%arg3,%r11,1) # Write to plaintext buffer 1341 movdqa \TMP3, \XMM2 1342 movdqu 32(%arg4,%r11,1), \TMP3 1343 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK 1344 movdqu \XMM3, 32(%arg3,%r11,1) # Write to plaintext buffer 1345 movdqa \TMP3, \XMM3 1346 movdqu 48(%arg4,%r11,1), \TMP3 1347 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK 1348 movdqu \XMM4, 48(%arg3,%r11,1) # Write to plaintext buffer 1349 movdqa \TMP3, \XMM4 1350 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap 1351 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap 1352 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap 1353 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap 1354 1355 pxor \TMP4, \TMP1 1356 pxor \XMM8, \XMM5 1357 pxor \TMP6, \TMP2 1358 pxor \TMP1, \TMP2 1359 pxor \XMM5, \TMP2 1360 movdqa \TMP2, \TMP3 1361 pslldq $8, \TMP3 # left shift TMP3 2 DWs 1362 psrldq $8, \TMP2 # right shift TMP2 2 DWs 1363 pxor \TMP3, \XMM5 1364 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 1365 1366 # first phase of reduction 1367 1368 movdqa \XMM5, \TMP2 1369 movdqa \XMM5, \TMP3 1370 movdqa \XMM5, \TMP4 1371# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently 1372 pslld $31, \TMP2 # packed right shift << 31 1373 pslld $30, \TMP3 # packed right shift << 30 1374 pslld $25, \TMP4 # packed right shift << 25 1375 pxor \TMP3, \TMP2 # xor the shifted versions 1376 pxor \TMP4, \TMP2 1377 movdqa \TMP2, \TMP5 1378 psrldq $4, \TMP5 # right shift T5 1 DW 1379 pslldq $12, \TMP2 # left shift T2 3 DWs 1380 pxor \TMP2, \XMM5 1381 1382 # second phase of reduction 1383 1384 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 1385 movdqa \XMM5,\TMP3 1386 movdqa \XMM5,\TMP4 1387 psrld $1, \TMP2 # packed left shift >>1 1388 psrld $2, \TMP3 # packed left shift >>2 1389 psrld $7, \TMP4 # packed left shift >>7 1390 pxor \TMP3,\TMP2 # xor the shifted versions 1391 pxor \TMP4,\TMP2 1392 pxor \TMP5, \TMP2 1393 pxor \TMP2, \XMM5 1394 pxor \TMP1, \XMM5 # result is in TMP1 1395 1396 pxor \XMM5, \XMM1 1397.endm 1398 1399/* GHASH the last 4 ciphertext blocks. */ 1400.macro GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \ 1401TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst 1402 1403 # Multiply TMP6 * HashKey (using Karatsuba) 1404 1405 movdqa \XMM1, \TMP6 1406 pshufd $78, \XMM1, \TMP2 1407 pxor \XMM1, \TMP2 1408 movdqu HashKey_4(%arg2), \TMP5 1409 PCLMULQDQ 0x11, \TMP5, \TMP6 # TMP6 = a1*b1 1410 PCLMULQDQ 0x00, \TMP5, \XMM1 # XMM1 = a0*b0 1411 movdqu HashKey_4_k(%arg2), \TMP4 1412 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1413 movdqa \XMM1, \XMMDst 1414 movdqa \TMP2, \XMM1 # result in TMP6, XMMDst, XMM1 1415 1416 # Multiply TMP1 * HashKey (using Karatsuba) 1417 1418 movdqa \XMM2, \TMP1 1419 pshufd $78, \XMM2, \TMP2 1420 pxor \XMM2, \TMP2 1421 movdqu HashKey_3(%arg2), \TMP5 1422 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1423 PCLMULQDQ 0x00, \TMP5, \XMM2 # XMM2 = a0*b0 1424 movdqu HashKey_3_k(%arg2), \TMP4 1425 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1426 pxor \TMP1, \TMP6 1427 pxor \XMM2, \XMMDst 1428 pxor \TMP2, \XMM1 1429# results accumulated in TMP6, XMMDst, XMM1 1430 1431 # Multiply TMP1 * HashKey (using Karatsuba) 1432 1433 movdqa \XMM3, \TMP1 1434 pshufd $78, \XMM3, \TMP2 1435 pxor \XMM3, \TMP2 1436 movdqu HashKey_2(%arg2), \TMP5 1437 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1438 PCLMULQDQ 0x00, \TMP5, \XMM3 # XMM3 = a0*b0 1439 movdqu HashKey_2_k(%arg2), \TMP4 1440 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1441 pxor \TMP1, \TMP6 1442 pxor \XMM3, \XMMDst 1443 pxor \TMP2, \XMM1 # results accumulated in TMP6, XMMDst, XMM1 1444 1445 # Multiply TMP1 * HashKey (using Karatsuba) 1446 movdqa \XMM4, \TMP1 1447 pshufd $78, \XMM4, \TMP2 1448 pxor \XMM4, \TMP2 1449 movdqu HashKey(%arg2), \TMP5 1450 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 1451 PCLMULQDQ 0x00, \TMP5, \XMM4 # XMM4 = a0*b0 1452 movdqu HashKey_k(%arg2), \TMP4 1453 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) 1454 pxor \TMP1, \TMP6 1455 pxor \XMM4, \XMMDst 1456 pxor \XMM1, \TMP2 1457 pxor \TMP6, \TMP2 1458 pxor \XMMDst, \TMP2 1459 # middle section of the temp results combined as in karatsuba algorithm 1460 movdqa \TMP2, \TMP4 1461 pslldq $8, \TMP4 # left shift TMP4 2 DWs 1462 psrldq $8, \TMP2 # right shift TMP2 2 DWs 1463 pxor \TMP4, \XMMDst 1464 pxor \TMP2, \TMP6 1465# TMP6:XMMDst holds the result of the accumulated carry-less multiplications 1466 # first phase of the reduction 1467 movdqa \XMMDst, \TMP2 1468 movdqa \XMMDst, \TMP3 1469 movdqa \XMMDst, \TMP4 1470# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently 1471 pslld $31, \TMP2 # packed right shifting << 31 1472 pslld $30, \TMP3 # packed right shifting << 30 1473 pslld $25, \TMP4 # packed right shifting << 25 1474 pxor \TMP3, \TMP2 # xor the shifted versions 1475 pxor \TMP4, \TMP2 1476 movdqa \TMP2, \TMP7 1477 psrldq $4, \TMP7 # right shift TMP7 1 DW 1478 pslldq $12, \TMP2 # left shift TMP2 3 DWs 1479 pxor \TMP2, \XMMDst 1480 1481 # second phase of the reduction 1482 movdqa \XMMDst, \TMP2 1483 # make 3 copies of XMMDst for doing 3 shift operations 1484 movdqa \XMMDst, \TMP3 1485 movdqa \XMMDst, \TMP4 1486 psrld $1, \TMP2 # packed left shift >> 1 1487 psrld $2, \TMP3 # packed left shift >> 2 1488 psrld $7, \TMP4 # packed left shift >> 7 1489 pxor \TMP3, \TMP2 # xor the shifted versions 1490 pxor \TMP4, \TMP2 1491 pxor \TMP7, \TMP2 1492 pxor \TMP2, \XMMDst 1493 pxor \TMP6, \XMMDst # reduced result is in XMMDst 1494.endm 1495 1496 1497/* Encryption of a single block 1498* uses eax & r10 1499*/ 1500 1501.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 1502 1503 pxor (%arg1), \XMM0 1504 mov keysize,%eax 1505 shr $2,%eax # 128->4, 192->6, 256->8 1506 add $5,%eax # 128->9, 192->11, 256->13 1507 lea 16(%arg1), %r10 # get first expanded key address 1508 1509_esb_loop_\@: 1510 MOVADQ (%r10),\TMP1 1511 AESENC \TMP1,\XMM0 1512 add $16,%r10 1513 sub $1,%eax 1514 jnz _esb_loop_\@ 1515 1516 MOVADQ (%r10),\TMP1 1517 AESENCLAST \TMP1,\XMM0 1518.endm 1519/***************************************************************************** 1520* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1521* struct gcm_context_data *data 1522* // Context data 1523* u8 *out, // Plaintext output. Encrypt in-place is allowed. 1524* const u8 *in, // Ciphertext input 1525* u64 plaintext_len, // Length of data in bytes for decryption. 1526* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) 1527* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) 1528* // concatenated with 0x00000001. 16-byte aligned pointer. 1529* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. 1530* const u8 *aad, // Additional Authentication Data (AAD) 1531* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes 1532* u8 *auth_tag, // Authenticated Tag output. The driver will compare this to the 1533* // given authentication tag and only return the plaintext if they match. 1534* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 1535* // (most likely), 12 or 8. 1536* 1537* Assumptions: 1538* 1539* keys: 1540* keys are pre-expanded and aligned to 16 bytes. we are using the first 1541* set of 11 keys in the data structure void *aes_ctx 1542* 1543* iv: 1544* 0 1 2 3 1545* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1546* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1547* | Salt (From the SA) | 1548* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1549* | Initialization Vector | 1550* | (This is the sequence number from IPSec header) | 1551* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1552* | 0x1 | 1553* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1554* 1555* 1556* 1557* AAD: 1558* AAD padded to 128 bits with 0 1559* for example, assume AAD is a u32 vector 1560* 1561* if AAD is 8 bytes: 1562* AAD[3] = {A0, A1}; 1563* padded AAD in xmm register = {A1 A0 0 0} 1564* 1565* 0 1 2 3 1566* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1567* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1568* | SPI (A1) | 1569* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1570* | 32-bit Sequence Number (A0) | 1571* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1572* | 0x0 | 1573* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1574* 1575* AAD Format with 32-bit Sequence Number 1576* 1577* if AAD is 12 bytes: 1578* AAD[3] = {A0, A1, A2}; 1579* padded AAD in xmm register = {A2 A1 A0 0} 1580* 1581* 0 1 2 3 1582* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1583* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1584* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1585* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1586* | SPI (A2) | 1587* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1588* | 64-bit Extended Sequence Number {A1,A0} | 1589* | | 1590* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1591* | 0x0 | 1592* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1593* 1594* AAD Format with 64-bit Extended Sequence Number 1595* 1596* poly = x^128 + x^127 + x^126 + x^121 + 1 1597* 1598*****************************************************************************/ 1599ENTRY(aesni_gcm_dec) 1600 FUNC_SAVE 1601 1602 GCM_INIT %arg6, arg7, arg8, arg9 1603 GCM_ENC_DEC dec 1604 GCM_COMPLETE arg10, arg11 1605 FUNC_RESTORE 1606 ret 1607ENDPROC(aesni_gcm_dec) 1608 1609 1610/***************************************************************************** 1611* void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1612* struct gcm_context_data *data 1613* // Context data 1614* u8 *out, // Ciphertext output. Encrypt in-place is allowed. 1615* const u8 *in, // Plaintext input 1616* u64 plaintext_len, // Length of data in bytes for encryption. 1617* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) 1618* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) 1619* // concatenated with 0x00000001. 16-byte aligned pointer. 1620* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. 1621* const u8 *aad, // Additional Authentication Data (AAD) 1622* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes 1623* u8 *auth_tag, // Authenticated Tag output. 1624* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely), 1625* // 12 or 8. 1626* 1627* Assumptions: 1628* 1629* keys: 1630* keys are pre-expanded and aligned to 16 bytes. we are using the 1631* first set of 11 keys in the data structure void *aes_ctx 1632* 1633* 1634* iv: 1635* 0 1 2 3 1636* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1637* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1638* | Salt (From the SA) | 1639* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1640* | Initialization Vector | 1641* | (This is the sequence number from IPSec header) | 1642* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1643* | 0x1 | 1644* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1645* 1646* 1647* 1648* AAD: 1649* AAD padded to 128 bits with 0 1650* for example, assume AAD is a u32 vector 1651* 1652* if AAD is 8 bytes: 1653* AAD[3] = {A0, A1}; 1654* padded AAD in xmm register = {A1 A0 0 0} 1655* 1656* 0 1 2 3 1657* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1658* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1659* | SPI (A1) | 1660* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1661* | 32-bit Sequence Number (A0) | 1662* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1663* | 0x0 | 1664* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1665* 1666* AAD Format with 32-bit Sequence Number 1667* 1668* if AAD is 12 bytes: 1669* AAD[3] = {A0, A1, A2}; 1670* padded AAD in xmm register = {A2 A1 A0 0} 1671* 1672* 0 1 2 3 1673* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 1674* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1675* | SPI (A2) | 1676* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1677* | 64-bit Extended Sequence Number {A1,A0} | 1678* | | 1679* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1680* | 0x0 | 1681* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 1682* 1683* AAD Format with 64-bit Extended Sequence Number 1684* 1685* poly = x^128 + x^127 + x^126 + x^121 + 1 1686***************************************************************************/ 1687ENTRY(aesni_gcm_enc) 1688 FUNC_SAVE 1689 1690 GCM_INIT %arg6, arg7, arg8, arg9 1691 GCM_ENC_DEC enc 1692 1693 GCM_COMPLETE arg10, arg11 1694 FUNC_RESTORE 1695 ret 1696ENDPROC(aesni_gcm_enc) 1697 1698/***************************************************************************** 1699* void aesni_gcm_init(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1700* struct gcm_context_data *data, 1701* // context data 1702* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) 1703* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) 1704* // concatenated with 0x00000001. 16-byte aligned pointer. 1705* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. 1706* const u8 *aad, // Additional Authentication Data (AAD) 1707* u64 aad_len) // Length of AAD in bytes. 1708*/ 1709ENTRY(aesni_gcm_init) 1710 FUNC_SAVE 1711 GCM_INIT %arg3, %arg4,%arg5, %arg6 1712 FUNC_RESTORE 1713 ret 1714ENDPROC(aesni_gcm_init) 1715 1716/***************************************************************************** 1717* void aesni_gcm_enc_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1718* struct gcm_context_data *data, 1719* // context data 1720* u8 *out, // Ciphertext output. Encrypt in-place is allowed. 1721* const u8 *in, // Plaintext input 1722* u64 plaintext_len, // Length of data in bytes for encryption. 1723*/ 1724ENTRY(aesni_gcm_enc_update) 1725 FUNC_SAVE 1726 GCM_ENC_DEC enc 1727 FUNC_RESTORE 1728 ret 1729ENDPROC(aesni_gcm_enc_update) 1730 1731/***************************************************************************** 1732* void aesni_gcm_dec_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1733* struct gcm_context_data *data, 1734* // context data 1735* u8 *out, // Ciphertext output. Encrypt in-place is allowed. 1736* const u8 *in, // Plaintext input 1737* u64 plaintext_len, // Length of data in bytes for encryption. 1738*/ 1739ENTRY(aesni_gcm_dec_update) 1740 FUNC_SAVE 1741 GCM_ENC_DEC dec 1742 FUNC_RESTORE 1743 ret 1744ENDPROC(aesni_gcm_dec_update) 1745 1746/***************************************************************************** 1747* void aesni_gcm_finalize(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. 1748* struct gcm_context_data *data, 1749* // context data 1750* u8 *auth_tag, // Authenticated Tag output. 1751* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely), 1752* // 12 or 8. 1753*/ 1754ENTRY(aesni_gcm_finalize) 1755 FUNC_SAVE 1756 GCM_COMPLETE %arg3 %arg4 1757 FUNC_RESTORE 1758 ret 1759ENDPROC(aesni_gcm_finalize) 1760 1761#endif 1762 1763 1764.align 4 1765_key_expansion_128: 1766_key_expansion_256a: 1767 pshufd $0b11111111, %xmm1, %xmm1 1768 shufps $0b00010000, %xmm0, %xmm4 1769 pxor %xmm4, %xmm0 1770 shufps $0b10001100, %xmm0, %xmm4 1771 pxor %xmm4, %xmm0 1772 pxor %xmm1, %xmm0 1773 movaps %xmm0, (TKEYP) 1774 add $0x10, TKEYP 1775 ret 1776ENDPROC(_key_expansion_128) 1777ENDPROC(_key_expansion_256a) 1778 1779.align 4 1780_key_expansion_192a: 1781 pshufd $0b01010101, %xmm1, %xmm1 1782 shufps $0b00010000, %xmm0, %xmm4 1783 pxor %xmm4, %xmm0 1784 shufps $0b10001100, %xmm0, %xmm4 1785 pxor %xmm4, %xmm0 1786 pxor %xmm1, %xmm0 1787 1788 movaps %xmm2, %xmm5 1789 movaps %xmm2, %xmm6 1790 pslldq $4, %xmm5 1791 pshufd $0b11111111, %xmm0, %xmm3 1792 pxor %xmm3, %xmm2 1793 pxor %xmm5, %xmm2 1794 1795 movaps %xmm0, %xmm1 1796 shufps $0b01000100, %xmm0, %xmm6 1797 movaps %xmm6, (TKEYP) 1798 shufps $0b01001110, %xmm2, %xmm1 1799 movaps %xmm1, 0x10(TKEYP) 1800 add $0x20, TKEYP 1801 ret 1802ENDPROC(_key_expansion_192a) 1803 1804.align 4 1805_key_expansion_192b: 1806 pshufd $0b01010101, %xmm1, %xmm1 1807 shufps $0b00010000, %xmm0, %xmm4 1808 pxor %xmm4, %xmm0 1809 shufps $0b10001100, %xmm0, %xmm4 1810 pxor %xmm4, %xmm0 1811 pxor %xmm1, %xmm0 1812 1813 movaps %xmm2, %xmm5 1814 pslldq $4, %xmm5 1815 pshufd $0b11111111, %xmm0, %xmm3 1816 pxor %xmm3, %xmm2 1817 pxor %xmm5, %xmm2 1818 1819 movaps %xmm0, (TKEYP) 1820 add $0x10, TKEYP 1821 ret 1822ENDPROC(_key_expansion_192b) 1823 1824.align 4 1825_key_expansion_256b: 1826 pshufd $0b10101010, %xmm1, %xmm1 1827 shufps $0b00010000, %xmm2, %xmm4 1828 pxor %xmm4, %xmm2 1829 shufps $0b10001100, %xmm2, %xmm4 1830 pxor %xmm4, %xmm2 1831 pxor %xmm1, %xmm2 1832 movaps %xmm2, (TKEYP) 1833 add $0x10, TKEYP 1834 ret 1835ENDPROC(_key_expansion_256b) 1836 1837/* 1838 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 1839 * unsigned int key_len) 1840 */ 1841ENTRY(aesni_set_key) 1842 FRAME_BEGIN 1843#ifndef __x86_64__ 1844 pushl KEYP 1845 movl (FRAME_OFFSET+8)(%esp), KEYP # ctx 1846 movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key 1847 movl (FRAME_OFFSET+16)(%esp), %edx # key_len 1848#endif 1849 movups (UKEYP), %xmm0 # user key (first 16 bytes) 1850 movaps %xmm0, (KEYP) 1851 lea 0x10(KEYP), TKEYP # key addr 1852 movl %edx, 480(KEYP) 1853 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x 1854 cmp $24, %dl 1855 jb .Lenc_key128 1856 je .Lenc_key192 1857 movups 0x10(UKEYP), %xmm2 # other user key 1858 movaps %xmm2, (TKEYP) 1859 add $0x10, TKEYP 1860 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 1861 call _key_expansion_256a 1862 AESKEYGENASSIST 0x1 %xmm0 %xmm1 1863 call _key_expansion_256b 1864 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2 1865 call _key_expansion_256a 1866 AESKEYGENASSIST 0x2 %xmm0 %xmm1 1867 call _key_expansion_256b 1868 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3 1869 call _key_expansion_256a 1870 AESKEYGENASSIST 0x4 %xmm0 %xmm1 1871 call _key_expansion_256b 1872 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4 1873 call _key_expansion_256a 1874 AESKEYGENASSIST 0x8 %xmm0 %xmm1 1875 call _key_expansion_256b 1876 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5 1877 call _key_expansion_256a 1878 AESKEYGENASSIST 0x10 %xmm0 %xmm1 1879 call _key_expansion_256b 1880 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6 1881 call _key_expansion_256a 1882 AESKEYGENASSIST 0x20 %xmm0 %xmm1 1883 call _key_expansion_256b 1884 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7 1885 call _key_expansion_256a 1886 jmp .Ldec_key 1887.Lenc_key192: 1888 movq 0x10(UKEYP), %xmm2 # other user key 1889 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 1890 call _key_expansion_192a 1891 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2 1892 call _key_expansion_192b 1893 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3 1894 call _key_expansion_192a 1895 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4 1896 call _key_expansion_192b 1897 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5 1898 call _key_expansion_192a 1899 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6 1900 call _key_expansion_192b 1901 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7 1902 call _key_expansion_192a 1903 AESKEYGENASSIST 0x80 %xmm2 %xmm1 # round 8 1904 call _key_expansion_192b 1905 jmp .Ldec_key 1906.Lenc_key128: 1907 AESKEYGENASSIST 0x1 %xmm0 %xmm1 # round 1 1908 call _key_expansion_128 1909 AESKEYGENASSIST 0x2 %xmm0 %xmm1 # round 2 1910 call _key_expansion_128 1911 AESKEYGENASSIST 0x4 %xmm0 %xmm1 # round 3 1912 call _key_expansion_128 1913 AESKEYGENASSIST 0x8 %xmm0 %xmm1 # round 4 1914 call _key_expansion_128 1915 AESKEYGENASSIST 0x10 %xmm0 %xmm1 # round 5 1916 call _key_expansion_128 1917 AESKEYGENASSIST 0x20 %xmm0 %xmm1 # round 6 1918 call _key_expansion_128 1919 AESKEYGENASSIST 0x40 %xmm0 %xmm1 # round 7 1920 call _key_expansion_128 1921 AESKEYGENASSIST 0x80 %xmm0 %xmm1 # round 8 1922 call _key_expansion_128 1923 AESKEYGENASSIST 0x1b %xmm0 %xmm1 # round 9 1924 call _key_expansion_128 1925 AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10 1926 call _key_expansion_128 1927.Ldec_key: 1928 sub $0x10, TKEYP 1929 movaps (KEYP), %xmm0 1930 movaps (TKEYP), %xmm1 1931 movaps %xmm0, 240(TKEYP) 1932 movaps %xmm1, 240(KEYP) 1933 add $0x10, KEYP 1934 lea 240-16(TKEYP), UKEYP 1935.align 4 1936.Ldec_key_loop: 1937 movaps (KEYP), %xmm0 1938 AESIMC %xmm0 %xmm1 1939 movaps %xmm1, (UKEYP) 1940 add $0x10, KEYP 1941 sub $0x10, UKEYP 1942 cmp TKEYP, KEYP 1943 jb .Ldec_key_loop 1944 xor AREG, AREG 1945#ifndef __x86_64__ 1946 popl KEYP 1947#endif 1948 FRAME_END 1949 ret 1950ENDPROC(aesni_set_key) 1951 1952/* 1953 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 1954 */ 1955ENTRY(aesni_enc) 1956 FRAME_BEGIN 1957#ifndef __x86_64__ 1958 pushl KEYP 1959 pushl KLEN 1960 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx 1961 movl (FRAME_OFFSET+16)(%esp), OUTP # dst 1962 movl (FRAME_OFFSET+20)(%esp), INP # src 1963#endif 1964 movl 480(KEYP), KLEN # key length 1965 movups (INP), STATE # input 1966 call _aesni_enc1 1967 movups STATE, (OUTP) # output 1968#ifndef __x86_64__ 1969 popl KLEN 1970 popl KEYP 1971#endif 1972 FRAME_END 1973 ret 1974ENDPROC(aesni_enc) 1975 1976/* 1977 * _aesni_enc1: internal ABI 1978 * input: 1979 * KEYP: key struct pointer 1980 * KLEN: round count 1981 * STATE: initial state (input) 1982 * output: 1983 * STATE: finial state (output) 1984 * changed: 1985 * KEY 1986 * TKEYP (T1) 1987 */ 1988.align 4 1989_aesni_enc1: 1990 movaps (KEYP), KEY # key 1991 mov KEYP, TKEYP 1992 pxor KEY, STATE # round 0 1993 add $0x30, TKEYP 1994 cmp $24, KLEN 1995 jb .Lenc128 1996 lea 0x20(TKEYP), TKEYP 1997 je .Lenc192 1998 add $0x20, TKEYP 1999 movaps -0x60(TKEYP), KEY 2000 AESENC KEY STATE 2001 movaps -0x50(TKEYP), KEY 2002 AESENC KEY STATE 2003.align 4 2004.Lenc192: 2005 movaps -0x40(TKEYP), KEY 2006 AESENC KEY STATE 2007 movaps -0x30(TKEYP), KEY 2008 AESENC KEY STATE 2009.align 4 2010.Lenc128: 2011 movaps -0x20(TKEYP), KEY 2012 AESENC KEY STATE 2013 movaps -0x10(TKEYP), KEY 2014 AESENC KEY STATE 2015 movaps (TKEYP), KEY 2016 AESENC KEY STATE 2017 movaps 0x10(TKEYP), KEY 2018 AESENC KEY STATE 2019 movaps 0x20(TKEYP), KEY 2020 AESENC KEY STATE 2021 movaps 0x30(TKEYP), KEY 2022 AESENC KEY STATE 2023 movaps 0x40(TKEYP), KEY 2024 AESENC KEY STATE 2025 movaps 0x50(TKEYP), KEY 2026 AESENC KEY STATE 2027 movaps 0x60(TKEYP), KEY 2028 AESENC KEY STATE 2029 movaps 0x70(TKEYP), KEY 2030 AESENCLAST KEY STATE 2031 ret 2032ENDPROC(_aesni_enc1) 2033 2034/* 2035 * _aesni_enc4: internal ABI 2036 * input: 2037 * KEYP: key struct pointer 2038 * KLEN: round count 2039 * STATE1: initial state (input) 2040 * STATE2 2041 * STATE3 2042 * STATE4 2043 * output: 2044 * STATE1: finial state (output) 2045 * STATE2 2046 * STATE3 2047 * STATE4 2048 * changed: 2049 * KEY 2050 * TKEYP (T1) 2051 */ 2052.align 4 2053_aesni_enc4: 2054 movaps (KEYP), KEY # key 2055 mov KEYP, TKEYP 2056 pxor KEY, STATE1 # round 0 2057 pxor KEY, STATE2 2058 pxor KEY, STATE3 2059 pxor KEY, STATE4 2060 add $0x30, TKEYP 2061 cmp $24, KLEN 2062 jb .L4enc128 2063 lea 0x20(TKEYP), TKEYP 2064 je .L4enc192 2065 add $0x20, TKEYP 2066 movaps -0x60(TKEYP), KEY 2067 AESENC KEY STATE1 2068 AESENC KEY STATE2 2069 AESENC KEY STATE3 2070 AESENC KEY STATE4 2071 movaps -0x50(TKEYP), KEY 2072 AESENC KEY STATE1 2073 AESENC KEY STATE2 2074 AESENC KEY STATE3 2075 AESENC KEY STATE4 2076#.align 4 2077.L4enc192: 2078 movaps -0x40(TKEYP), KEY 2079 AESENC KEY STATE1 2080 AESENC KEY STATE2 2081 AESENC KEY STATE3 2082 AESENC KEY STATE4 2083 movaps -0x30(TKEYP), KEY 2084 AESENC KEY STATE1 2085 AESENC KEY STATE2 2086 AESENC KEY STATE3 2087 AESENC KEY STATE4 2088#.align 4 2089.L4enc128: 2090 movaps -0x20(TKEYP), KEY 2091 AESENC KEY STATE1 2092 AESENC KEY STATE2 2093 AESENC KEY STATE3 2094 AESENC KEY STATE4 2095 movaps -0x10(TKEYP), KEY 2096 AESENC KEY STATE1 2097 AESENC KEY STATE2 2098 AESENC KEY STATE3 2099 AESENC KEY STATE4 2100 movaps (TKEYP), KEY 2101 AESENC KEY STATE1 2102 AESENC KEY STATE2 2103 AESENC KEY STATE3 2104 AESENC KEY STATE4 2105 movaps 0x10(TKEYP), KEY 2106 AESENC KEY STATE1 2107 AESENC KEY STATE2 2108 AESENC KEY STATE3 2109 AESENC KEY STATE4 2110 movaps 0x20(TKEYP), KEY 2111 AESENC KEY STATE1 2112 AESENC KEY STATE2 2113 AESENC KEY STATE3 2114 AESENC KEY STATE4 2115 movaps 0x30(TKEYP), KEY 2116 AESENC KEY STATE1 2117 AESENC KEY STATE2 2118 AESENC KEY STATE3 2119 AESENC KEY STATE4 2120 movaps 0x40(TKEYP), KEY 2121 AESENC KEY STATE1 2122 AESENC KEY STATE2 2123 AESENC KEY STATE3 2124 AESENC KEY STATE4 2125 movaps 0x50(TKEYP), KEY 2126 AESENC KEY STATE1 2127 AESENC KEY STATE2 2128 AESENC KEY STATE3 2129 AESENC KEY STATE4 2130 movaps 0x60(TKEYP), KEY 2131 AESENC KEY STATE1 2132 AESENC KEY STATE2 2133 AESENC KEY STATE3 2134 AESENC KEY STATE4 2135 movaps 0x70(TKEYP), KEY 2136 AESENCLAST KEY STATE1 # last round 2137 AESENCLAST KEY STATE2 2138 AESENCLAST KEY STATE3 2139 AESENCLAST KEY STATE4 2140 ret 2141ENDPROC(_aesni_enc4) 2142 2143/* 2144 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 2145 */ 2146ENTRY(aesni_dec) 2147 FRAME_BEGIN 2148#ifndef __x86_64__ 2149 pushl KEYP 2150 pushl KLEN 2151 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx 2152 movl (FRAME_OFFSET+16)(%esp), OUTP # dst 2153 movl (FRAME_OFFSET+20)(%esp), INP # src 2154#endif 2155 mov 480(KEYP), KLEN # key length 2156 add $240, KEYP 2157 movups (INP), STATE # input 2158 call _aesni_dec1 2159 movups STATE, (OUTP) #output 2160#ifndef __x86_64__ 2161 popl KLEN 2162 popl KEYP 2163#endif 2164 FRAME_END 2165 ret 2166ENDPROC(aesni_dec) 2167 2168/* 2169 * _aesni_dec1: internal ABI 2170 * input: 2171 * KEYP: key struct pointer 2172 * KLEN: key length 2173 * STATE: initial state (input) 2174 * output: 2175 * STATE: finial state (output) 2176 * changed: 2177 * KEY 2178 * TKEYP (T1) 2179 */ 2180.align 4 2181_aesni_dec1: 2182 movaps (KEYP), KEY # key 2183 mov KEYP, TKEYP 2184 pxor KEY, STATE # round 0 2185 add $0x30, TKEYP 2186 cmp $24, KLEN 2187 jb .Ldec128 2188 lea 0x20(TKEYP), TKEYP 2189 je .Ldec192 2190 add $0x20, TKEYP 2191 movaps -0x60(TKEYP), KEY 2192 AESDEC KEY STATE 2193 movaps -0x50(TKEYP), KEY 2194 AESDEC KEY STATE 2195.align 4 2196.Ldec192: 2197 movaps -0x40(TKEYP), KEY 2198 AESDEC KEY STATE 2199 movaps -0x30(TKEYP), KEY 2200 AESDEC KEY STATE 2201.align 4 2202.Ldec128: 2203 movaps -0x20(TKEYP), KEY 2204 AESDEC KEY STATE 2205 movaps -0x10(TKEYP), KEY 2206 AESDEC KEY STATE 2207 movaps (TKEYP), KEY 2208 AESDEC KEY STATE 2209 movaps 0x10(TKEYP), KEY 2210 AESDEC KEY STATE 2211 movaps 0x20(TKEYP), KEY 2212 AESDEC KEY STATE 2213 movaps 0x30(TKEYP), KEY 2214 AESDEC KEY STATE 2215 movaps 0x40(TKEYP), KEY 2216 AESDEC KEY STATE 2217 movaps 0x50(TKEYP), KEY 2218 AESDEC KEY STATE 2219 movaps 0x60(TKEYP), KEY 2220 AESDEC KEY STATE 2221 movaps 0x70(TKEYP), KEY 2222 AESDECLAST KEY STATE 2223 ret 2224ENDPROC(_aesni_dec1) 2225 2226/* 2227 * _aesni_dec4: internal ABI 2228 * input: 2229 * KEYP: key struct pointer 2230 * KLEN: key length 2231 * STATE1: initial state (input) 2232 * STATE2 2233 * STATE3 2234 * STATE4 2235 * output: 2236 * STATE1: finial state (output) 2237 * STATE2 2238 * STATE3 2239 * STATE4 2240 * changed: 2241 * KEY 2242 * TKEYP (T1) 2243 */ 2244.align 4 2245_aesni_dec4: 2246 movaps (KEYP), KEY # key 2247 mov KEYP, TKEYP 2248 pxor KEY, STATE1 # round 0 2249 pxor KEY, STATE2 2250 pxor KEY, STATE3 2251 pxor KEY, STATE4 2252 add $0x30, TKEYP 2253 cmp $24, KLEN 2254 jb .L4dec128 2255 lea 0x20(TKEYP), TKEYP 2256 je .L4dec192 2257 add $0x20, TKEYP 2258 movaps -0x60(TKEYP), KEY 2259 AESDEC KEY STATE1 2260 AESDEC KEY STATE2 2261 AESDEC KEY STATE3 2262 AESDEC KEY STATE4 2263 movaps -0x50(TKEYP), KEY 2264 AESDEC KEY STATE1 2265 AESDEC KEY STATE2 2266 AESDEC KEY STATE3 2267 AESDEC KEY STATE4 2268.align 4 2269.L4dec192: 2270 movaps -0x40(TKEYP), KEY 2271 AESDEC KEY STATE1 2272 AESDEC KEY STATE2 2273 AESDEC KEY STATE3 2274 AESDEC KEY STATE4 2275 movaps -0x30(TKEYP), KEY 2276 AESDEC KEY STATE1 2277 AESDEC KEY STATE2 2278 AESDEC KEY STATE3 2279 AESDEC KEY STATE4 2280.align 4 2281.L4dec128: 2282 movaps -0x20(TKEYP), KEY 2283 AESDEC KEY STATE1 2284 AESDEC KEY STATE2 2285 AESDEC KEY STATE3 2286 AESDEC KEY STATE4 2287 movaps -0x10(TKEYP), KEY 2288 AESDEC KEY STATE1 2289 AESDEC KEY STATE2 2290 AESDEC KEY STATE3 2291 AESDEC KEY STATE4 2292 movaps (TKEYP), KEY 2293 AESDEC KEY STATE1 2294 AESDEC KEY STATE2 2295 AESDEC KEY STATE3 2296 AESDEC KEY STATE4 2297 movaps 0x10(TKEYP), KEY 2298 AESDEC KEY STATE1 2299 AESDEC KEY STATE2 2300 AESDEC KEY STATE3 2301 AESDEC KEY STATE4 2302 movaps 0x20(TKEYP), KEY 2303 AESDEC KEY STATE1 2304 AESDEC KEY STATE2 2305 AESDEC KEY STATE3 2306 AESDEC KEY STATE4 2307 movaps 0x30(TKEYP), KEY 2308 AESDEC KEY STATE1 2309 AESDEC KEY STATE2 2310 AESDEC KEY STATE3 2311 AESDEC KEY STATE4 2312 movaps 0x40(TKEYP), KEY 2313 AESDEC KEY STATE1 2314 AESDEC KEY STATE2 2315 AESDEC KEY STATE3 2316 AESDEC KEY STATE4 2317 movaps 0x50(TKEYP), KEY 2318 AESDEC KEY STATE1 2319 AESDEC KEY STATE2 2320 AESDEC KEY STATE3 2321 AESDEC KEY STATE4 2322 movaps 0x60(TKEYP), KEY 2323 AESDEC KEY STATE1 2324 AESDEC KEY STATE2 2325 AESDEC KEY STATE3 2326 AESDEC KEY STATE4 2327 movaps 0x70(TKEYP), KEY 2328 AESDECLAST KEY STATE1 # last round 2329 AESDECLAST KEY STATE2 2330 AESDECLAST KEY STATE3 2331 AESDECLAST KEY STATE4 2332 ret 2333ENDPROC(_aesni_dec4) 2334 2335/* 2336 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2337 * size_t len) 2338 */ 2339ENTRY(aesni_ecb_enc) 2340 FRAME_BEGIN 2341#ifndef __x86_64__ 2342 pushl LEN 2343 pushl KEYP 2344 pushl KLEN 2345 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx 2346 movl (FRAME_OFFSET+20)(%esp), OUTP # dst 2347 movl (FRAME_OFFSET+24)(%esp), INP # src 2348 movl (FRAME_OFFSET+28)(%esp), LEN # len 2349#endif 2350 test LEN, LEN # check length 2351 jz .Lecb_enc_ret 2352 mov 480(KEYP), KLEN 2353 cmp $16, LEN 2354 jb .Lecb_enc_ret 2355 cmp $64, LEN 2356 jb .Lecb_enc_loop1 2357.align 4 2358.Lecb_enc_loop4: 2359 movups (INP), STATE1 2360 movups 0x10(INP), STATE2 2361 movups 0x20(INP), STATE3 2362 movups 0x30(INP), STATE4 2363 call _aesni_enc4 2364 movups STATE1, (OUTP) 2365 movups STATE2, 0x10(OUTP) 2366 movups STATE3, 0x20(OUTP) 2367 movups STATE4, 0x30(OUTP) 2368 sub $64, LEN 2369 add $64, INP 2370 add $64, OUTP 2371 cmp $64, LEN 2372 jge .Lecb_enc_loop4 2373 cmp $16, LEN 2374 jb .Lecb_enc_ret 2375.align 4 2376.Lecb_enc_loop1: 2377 movups (INP), STATE1 2378 call _aesni_enc1 2379 movups STATE1, (OUTP) 2380 sub $16, LEN 2381 add $16, INP 2382 add $16, OUTP 2383 cmp $16, LEN 2384 jge .Lecb_enc_loop1 2385.Lecb_enc_ret: 2386#ifndef __x86_64__ 2387 popl KLEN 2388 popl KEYP 2389 popl LEN 2390#endif 2391 FRAME_END 2392 ret 2393ENDPROC(aesni_ecb_enc) 2394 2395/* 2396 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2397 * size_t len); 2398 */ 2399ENTRY(aesni_ecb_dec) 2400 FRAME_BEGIN 2401#ifndef __x86_64__ 2402 pushl LEN 2403 pushl KEYP 2404 pushl KLEN 2405 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx 2406 movl (FRAME_OFFSET+20)(%esp), OUTP # dst 2407 movl (FRAME_OFFSET+24)(%esp), INP # src 2408 movl (FRAME_OFFSET+28)(%esp), LEN # len 2409#endif 2410 test LEN, LEN 2411 jz .Lecb_dec_ret 2412 mov 480(KEYP), KLEN 2413 add $240, KEYP 2414 cmp $16, LEN 2415 jb .Lecb_dec_ret 2416 cmp $64, LEN 2417 jb .Lecb_dec_loop1 2418.align 4 2419.Lecb_dec_loop4: 2420 movups (INP), STATE1 2421 movups 0x10(INP), STATE2 2422 movups 0x20(INP), STATE3 2423 movups 0x30(INP), STATE4 2424 call _aesni_dec4 2425 movups STATE1, (OUTP) 2426 movups STATE2, 0x10(OUTP) 2427 movups STATE3, 0x20(OUTP) 2428 movups STATE4, 0x30(OUTP) 2429 sub $64, LEN 2430 add $64, INP 2431 add $64, OUTP 2432 cmp $64, LEN 2433 jge .Lecb_dec_loop4 2434 cmp $16, LEN 2435 jb .Lecb_dec_ret 2436.align 4 2437.Lecb_dec_loop1: 2438 movups (INP), STATE1 2439 call _aesni_dec1 2440 movups STATE1, (OUTP) 2441 sub $16, LEN 2442 add $16, INP 2443 add $16, OUTP 2444 cmp $16, LEN 2445 jge .Lecb_dec_loop1 2446.Lecb_dec_ret: 2447#ifndef __x86_64__ 2448 popl KLEN 2449 popl KEYP 2450 popl LEN 2451#endif 2452 FRAME_END 2453 ret 2454ENDPROC(aesni_ecb_dec) 2455 2456/* 2457 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2458 * size_t len, u8 *iv) 2459 */ 2460ENTRY(aesni_cbc_enc) 2461 FRAME_BEGIN 2462#ifndef __x86_64__ 2463 pushl IVP 2464 pushl LEN 2465 pushl KEYP 2466 pushl KLEN 2467 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx 2468 movl (FRAME_OFFSET+24)(%esp), OUTP # dst 2469 movl (FRAME_OFFSET+28)(%esp), INP # src 2470 movl (FRAME_OFFSET+32)(%esp), LEN # len 2471 movl (FRAME_OFFSET+36)(%esp), IVP # iv 2472#endif 2473 cmp $16, LEN 2474 jb .Lcbc_enc_ret 2475 mov 480(KEYP), KLEN 2476 movups (IVP), STATE # load iv as initial state 2477.align 4 2478.Lcbc_enc_loop: 2479 movups (INP), IN # load input 2480 pxor IN, STATE 2481 call _aesni_enc1 2482 movups STATE, (OUTP) # store output 2483 sub $16, LEN 2484 add $16, INP 2485 add $16, OUTP 2486 cmp $16, LEN 2487 jge .Lcbc_enc_loop 2488 movups STATE, (IVP) 2489.Lcbc_enc_ret: 2490#ifndef __x86_64__ 2491 popl KLEN 2492 popl KEYP 2493 popl LEN 2494 popl IVP 2495#endif 2496 FRAME_END 2497 ret 2498ENDPROC(aesni_cbc_enc) 2499 2500/* 2501 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2502 * size_t len, u8 *iv) 2503 */ 2504ENTRY(aesni_cbc_dec) 2505 FRAME_BEGIN 2506#ifndef __x86_64__ 2507 pushl IVP 2508 pushl LEN 2509 pushl KEYP 2510 pushl KLEN 2511 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx 2512 movl (FRAME_OFFSET+24)(%esp), OUTP # dst 2513 movl (FRAME_OFFSET+28)(%esp), INP # src 2514 movl (FRAME_OFFSET+32)(%esp), LEN # len 2515 movl (FRAME_OFFSET+36)(%esp), IVP # iv 2516#endif 2517 cmp $16, LEN 2518 jb .Lcbc_dec_just_ret 2519 mov 480(KEYP), KLEN 2520 add $240, KEYP 2521 movups (IVP), IV 2522 cmp $64, LEN 2523 jb .Lcbc_dec_loop1 2524.align 4 2525.Lcbc_dec_loop4: 2526 movups (INP), IN1 2527 movaps IN1, STATE1 2528 movups 0x10(INP), IN2 2529 movaps IN2, STATE2 2530#ifdef __x86_64__ 2531 movups 0x20(INP), IN3 2532 movaps IN3, STATE3 2533 movups 0x30(INP), IN4 2534 movaps IN4, STATE4 2535#else 2536 movups 0x20(INP), IN1 2537 movaps IN1, STATE3 2538 movups 0x30(INP), IN2 2539 movaps IN2, STATE4 2540#endif 2541 call _aesni_dec4 2542 pxor IV, STATE1 2543#ifdef __x86_64__ 2544 pxor IN1, STATE2 2545 pxor IN2, STATE3 2546 pxor IN3, STATE4 2547 movaps IN4, IV 2548#else 2549 pxor IN1, STATE4 2550 movaps IN2, IV 2551 movups (INP), IN1 2552 pxor IN1, STATE2 2553 movups 0x10(INP), IN2 2554 pxor IN2, STATE3 2555#endif 2556 movups STATE1, (OUTP) 2557 movups STATE2, 0x10(OUTP) 2558 movups STATE3, 0x20(OUTP) 2559 movups STATE4, 0x30(OUTP) 2560 sub $64, LEN 2561 add $64, INP 2562 add $64, OUTP 2563 cmp $64, LEN 2564 jge .Lcbc_dec_loop4 2565 cmp $16, LEN 2566 jb .Lcbc_dec_ret 2567.align 4 2568.Lcbc_dec_loop1: 2569 movups (INP), IN 2570 movaps IN, STATE 2571 call _aesni_dec1 2572 pxor IV, STATE 2573 movups STATE, (OUTP) 2574 movaps IN, IV 2575 sub $16, LEN 2576 add $16, INP 2577 add $16, OUTP 2578 cmp $16, LEN 2579 jge .Lcbc_dec_loop1 2580.Lcbc_dec_ret: 2581 movups IV, (IVP) 2582.Lcbc_dec_just_ret: 2583#ifndef __x86_64__ 2584 popl KLEN 2585 popl KEYP 2586 popl LEN 2587 popl IVP 2588#endif 2589 FRAME_END 2590 ret 2591ENDPROC(aesni_cbc_dec) 2592 2593#ifdef __x86_64__ 2594.pushsection .rodata 2595.align 16 2596.Lbswap_mask: 2597 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 2598.popsection 2599 2600/* 2601 * _aesni_inc_init: internal ABI 2602 * setup registers used by _aesni_inc 2603 * input: 2604 * IV 2605 * output: 2606 * CTR: == IV, in little endian 2607 * TCTR_LOW: == lower qword of CTR 2608 * INC: == 1, in little endian 2609 * BSWAP_MASK == endian swapping mask 2610 */ 2611.align 4 2612_aesni_inc_init: 2613 movaps .Lbswap_mask, BSWAP_MASK 2614 movaps IV, CTR 2615 PSHUFB_XMM BSWAP_MASK CTR 2616 mov $1, TCTR_LOW 2617 MOVQ_R64_XMM TCTR_LOW INC 2618 MOVQ_R64_XMM CTR TCTR_LOW 2619 ret 2620ENDPROC(_aesni_inc_init) 2621 2622/* 2623 * _aesni_inc: internal ABI 2624 * Increase IV by 1, IV is in big endian 2625 * input: 2626 * IV 2627 * CTR: == IV, in little endian 2628 * TCTR_LOW: == lower qword of CTR 2629 * INC: == 1, in little endian 2630 * BSWAP_MASK == endian swapping mask 2631 * output: 2632 * IV: Increase by 1 2633 * changed: 2634 * CTR: == output IV, in little endian 2635 * TCTR_LOW: == lower qword of CTR 2636 */ 2637.align 4 2638_aesni_inc: 2639 paddq INC, CTR 2640 add $1, TCTR_LOW 2641 jnc .Linc_low 2642 pslldq $8, INC 2643 paddq INC, CTR 2644 psrldq $8, INC 2645.Linc_low: 2646 movaps CTR, IV 2647 PSHUFB_XMM BSWAP_MASK IV 2648 ret 2649ENDPROC(_aesni_inc) 2650 2651/* 2652 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2653 * size_t len, u8 *iv) 2654 */ 2655ENTRY(aesni_ctr_enc) 2656 FRAME_BEGIN 2657 cmp $16, LEN 2658 jb .Lctr_enc_just_ret 2659 mov 480(KEYP), KLEN 2660 movups (IVP), IV 2661 call _aesni_inc_init 2662 cmp $64, LEN 2663 jb .Lctr_enc_loop1 2664.align 4 2665.Lctr_enc_loop4: 2666 movaps IV, STATE1 2667 call _aesni_inc 2668 movups (INP), IN1 2669 movaps IV, STATE2 2670 call _aesni_inc 2671 movups 0x10(INP), IN2 2672 movaps IV, STATE3 2673 call _aesni_inc 2674 movups 0x20(INP), IN3 2675 movaps IV, STATE4 2676 call _aesni_inc 2677 movups 0x30(INP), IN4 2678 call _aesni_enc4 2679 pxor IN1, STATE1 2680 movups STATE1, (OUTP) 2681 pxor IN2, STATE2 2682 movups STATE2, 0x10(OUTP) 2683 pxor IN3, STATE3 2684 movups STATE3, 0x20(OUTP) 2685 pxor IN4, STATE4 2686 movups STATE4, 0x30(OUTP) 2687 sub $64, LEN 2688 add $64, INP 2689 add $64, OUTP 2690 cmp $64, LEN 2691 jge .Lctr_enc_loop4 2692 cmp $16, LEN 2693 jb .Lctr_enc_ret 2694.align 4 2695.Lctr_enc_loop1: 2696 movaps IV, STATE 2697 call _aesni_inc 2698 movups (INP), IN 2699 call _aesni_enc1 2700 pxor IN, STATE 2701 movups STATE, (OUTP) 2702 sub $16, LEN 2703 add $16, INP 2704 add $16, OUTP 2705 cmp $16, LEN 2706 jge .Lctr_enc_loop1 2707.Lctr_enc_ret: 2708 movups IV, (IVP) 2709.Lctr_enc_just_ret: 2710 FRAME_END 2711 ret 2712ENDPROC(aesni_ctr_enc) 2713 2714/* 2715 * _aesni_gf128mul_x_ble: internal ABI 2716 * Multiply in GF(2^128) for XTS IVs 2717 * input: 2718 * IV: current IV 2719 * GF128MUL_MASK == mask with 0x87 and 0x01 2720 * output: 2721 * IV: next IV 2722 * changed: 2723 * CTR: == temporary value 2724 */ 2725#define _aesni_gf128mul_x_ble() \ 2726 pshufd $0x13, IV, CTR; \ 2727 paddq IV, IV; \ 2728 psrad $31, CTR; \ 2729 pand GF128MUL_MASK, CTR; \ 2730 pxor CTR, IV; 2731 2732/* 2733 * void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2734 * bool enc, u8 *iv) 2735 */ 2736ENTRY(aesni_xts_crypt8) 2737 FRAME_BEGIN 2738 cmpb $0, %cl 2739 movl $0, %ecx 2740 movl $240, %r10d 2741 leaq _aesni_enc4, %r11 2742 leaq _aesni_dec4, %rax 2743 cmovel %r10d, %ecx 2744 cmoveq %rax, %r11 2745 2746 movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK 2747 movups (IVP), IV 2748 2749 mov 480(KEYP), KLEN 2750 addq %rcx, KEYP 2751 2752 movdqa IV, STATE1 2753 movdqu 0x00(INP), INC 2754 pxor INC, STATE1 2755 movdqu IV, 0x00(OUTP) 2756 2757 _aesni_gf128mul_x_ble() 2758 movdqa IV, STATE2 2759 movdqu 0x10(INP), INC 2760 pxor INC, STATE2 2761 movdqu IV, 0x10(OUTP) 2762 2763 _aesni_gf128mul_x_ble() 2764 movdqa IV, STATE3 2765 movdqu 0x20(INP), INC 2766 pxor INC, STATE3 2767 movdqu IV, 0x20(OUTP) 2768 2769 _aesni_gf128mul_x_ble() 2770 movdqa IV, STATE4 2771 movdqu 0x30(INP), INC 2772 pxor INC, STATE4 2773 movdqu IV, 0x30(OUTP) 2774 2775 CALL_NOSPEC %r11 2776 2777 movdqu 0x00(OUTP), INC 2778 pxor INC, STATE1 2779 movdqu STATE1, 0x00(OUTP) 2780 2781 _aesni_gf128mul_x_ble() 2782 movdqa IV, STATE1 2783 movdqu 0x40(INP), INC 2784 pxor INC, STATE1 2785 movdqu IV, 0x40(OUTP) 2786 2787 movdqu 0x10(OUTP), INC 2788 pxor INC, STATE2 2789 movdqu STATE2, 0x10(OUTP) 2790 2791 _aesni_gf128mul_x_ble() 2792 movdqa IV, STATE2 2793 movdqu 0x50(INP), INC 2794 pxor INC, STATE2 2795 movdqu IV, 0x50(OUTP) 2796 2797 movdqu 0x20(OUTP), INC 2798 pxor INC, STATE3 2799 movdqu STATE3, 0x20(OUTP) 2800 2801 _aesni_gf128mul_x_ble() 2802 movdqa IV, STATE3 2803 movdqu 0x60(INP), INC 2804 pxor INC, STATE3 2805 movdqu IV, 0x60(OUTP) 2806 2807 movdqu 0x30(OUTP), INC 2808 pxor INC, STATE4 2809 movdqu STATE4, 0x30(OUTP) 2810 2811 _aesni_gf128mul_x_ble() 2812 movdqa IV, STATE4 2813 movdqu 0x70(INP), INC 2814 pxor INC, STATE4 2815 movdqu IV, 0x70(OUTP) 2816 2817 _aesni_gf128mul_x_ble() 2818 movups IV, (IVP) 2819 2820 CALL_NOSPEC %r11 2821 2822 movdqu 0x40(OUTP), INC 2823 pxor INC, STATE1 2824 movdqu STATE1, 0x40(OUTP) 2825 2826 movdqu 0x50(OUTP), INC 2827 pxor INC, STATE2 2828 movdqu STATE2, 0x50(OUTP) 2829 2830 movdqu 0x60(OUTP), INC 2831 pxor INC, STATE3 2832 movdqu STATE3, 0x60(OUTP) 2833 2834 movdqu 0x70(OUTP), INC 2835 pxor INC, STATE4 2836 movdqu STATE4, 0x70(OUTP) 2837 2838 FRAME_END 2839 ret 2840ENDPROC(aesni_xts_crypt8) 2841 2842#endif 2843
