316 	aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});  in gcm_setkey()
404 	u8 mac[AES_BLOCK_SIZE];  in gcm_final()
416 	crypto_xor(tag, mac, AES_BLOCK_SIZE);  in gcm_final()
424 	u8 iv[AES_BLOCK_SIZE];  in gcm_encrypt()
425 	u8 ks[2 * AES_BLOCK_SIZE];  in gcm_encrypt()
426 	u8 tag[AES_BLOCK_SIZE];  in gcm_encrypt()
439 	if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) {  in gcm_encrypt()
447 		pmull_gcm_encrypt_block(ks + AES_BLOCK_SIZE, iv, NULL, nrounds);  in gcm_encrypt()
451 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_encrypt()
462 					walk.nbytes % (2 * AES_BLOCK_SIZE));  in gcm_encrypt()
465 		} while (walk.nbytes >= 2 * AES_BLOCK_SIZE);  in gcm_encrypt()
470 		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {  in gcm_encrypt()
472 				walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_encrypt()
479 				crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE);  in gcm_encrypt()
480 				crypto_inc(iv, AES_BLOCK_SIZE);  in gcm_encrypt()
482 				dst += AES_BLOCK_SIZE;  in gcm_encrypt()
483 				src += AES_BLOCK_SIZE;  in gcm_encrypt()
491 						 walk.nbytes % (2 * AES_BLOCK_SIZE));  in gcm_encrypt()
495 			if (walk.nbytes > AES_BLOCK_SIZE) {  in gcm_encrypt()
496 				crypto_inc(iv, AES_BLOCK_SIZE);  in gcm_encrypt()
497 				aes_encrypt(&ctx->aes_key, ks + AES_BLOCK_SIZE, iv);  in gcm_encrypt()
544 	u8 iv[2 * AES_BLOCK_SIZE];  in gcm_decrypt()
545 	u8 tag[AES_BLOCK_SIZE];  in gcm_decrypt()
559 	if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) {  in gcm_decrypt()
567 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_decrypt()
568 			int rem = walk.total - blocks * AES_BLOCK_SIZE;  in gcm_decrypt()
578 			if (rem < (2 * AES_BLOCK_SIZE)) {  in gcm_decrypt()
579 				u8 *iv2 = iv + AES_BLOCK_SIZE;  in gcm_decrypt()
581 				if (rem > AES_BLOCK_SIZE) {  in gcm_decrypt()
582 					memcpy(iv2, iv, AES_BLOCK_SIZE);  in gcm_decrypt()
583 					crypto_inc(iv2, AES_BLOCK_SIZE);  in gcm_decrypt()
588 				if (rem > AES_BLOCK_SIZE)  in gcm_decrypt()
596 					walk.nbytes % (2 * AES_BLOCK_SIZE));  in gcm_decrypt()
599 		} while (walk.nbytes >= 2 * AES_BLOCK_SIZE);  in gcm_decrypt()
604 		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {  in gcm_decrypt()
605 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_decrypt()
615 				crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);  in gcm_decrypt()
616 				crypto_inc(iv, AES_BLOCK_SIZE);  in gcm_decrypt()
618 				dst += AES_BLOCK_SIZE;  in gcm_decrypt()
619 				src += AES_BLOCK_SIZE;  in gcm_decrypt()
623 						 walk.nbytes % (2 * AES_BLOCK_SIZE));  in gcm_decrypt()
626 			if (walk.nbytes > AES_BLOCK_SIZE) {  in gcm_decrypt()
627 				u8 *iv2 = iv + AES_BLOCK_SIZE;  in gcm_decrypt()
629 				memcpy(iv2, iv, AES_BLOCK_SIZE);  in gcm_decrypt()
630 				crypto_inc(iv2, AES_BLOCK_SIZE);  in gcm_decrypt()
678 	.chunksize		= 2 * AES_BLOCK_SIZE,
679 	.maxauthsize		= AES_BLOCK_SIZE,