53 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
57 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
61 asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[],
66 asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[],
81 static void ghash_do_update(int blocks, u64 dg[], const char *src,  in ghash_do_update()  argument
83 			    void (*simd_update)(int blocks, u64 dg[],  in ghash_do_update()  argument
90 		simd_update(blocks, dg, src, key, head);  in ghash_do_update()
100 				blocks++;  in ghash_do_update()
108 		} while (--blocks);  in ghash_do_update()
120 			  void (*simd_update)(int blocks, u64 dg[],  in __ghash_update()  argument
132 		int blocks;  in __ghash_update()  local
142 		blocks = len / GHASH_BLOCK_SIZE;  in __ghash_update()
146 			int chunk = min(blocks, MAX_BLOCKS);  in __ghash_update()
152 			blocks -= chunk;  in __ghash_update()
155 		} while (unlikely(blocks > 0));  in __ghash_update()
348 		int blocks = count / GHASH_BLOCK_SIZE;  in gcm_update_mac()  local
350 		ghash_do_update(blocks, dg, src, &ctx->ghash_key,  in gcm_update_mac()
354 		src += blocks * GHASH_BLOCK_SIZE;  in gcm_update_mac()
451 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_encrypt()  local
456 			pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr,  in gcm_encrypt()
471 			const int blocks =  in gcm_encrypt()  local
475 			int remaining = blocks;  in gcm_encrypt()
486 			ghash_do_update(blocks, dg,  in gcm_encrypt()
567 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_decrypt()  local
568 			int rem = walk.total - blocks * AES_BLOCK_SIZE;  in gcm_decrypt()
573 			pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr,  in gcm_decrypt()
605 			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;  in gcm_decrypt()  local
609 			ghash_do_update(blocks, dg, walk.src.virt.addr,  in gcm_decrypt()
620 			} while (--blocks > 0);  in gcm_decrypt()