The branch stable/15 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=ead69828206e01ee0cc8f9436def1ea76327377e
commit ead69828206e01ee0cc8f9436def1ea76327377e Author: Timothy Pearson <[email protected]> AuthorDate: 2025-09-10 16:54:24 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2025-10-15 13:23:03 +0000 ossl: Add GCM support on powerpc64/powerpc64le (POWER8+) Separate ossl's existing AES-NI GCM implementation into a common ossl_aes_gcm.c and add conditionals to switch between OpenSSL's AES-NI and POWER8 GCM routines depending on the architecture. Since the existing AVX-512 implementation is less agnostic, move it into a separate ossl_aes_gcm_avx512.c. Reviewed by: markj MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D44274 (cherry picked from commit 5daf8ed625af70ebb7e4740ab98a6054e9e52329) --- sys/conf/files.amd64 | 3 +- sys/conf/files.arm | 2 +- sys/conf/files.powerpc | 3 + sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c | 232 ++++++++++++ .../arm/{ossl_aes_gcm.c => ossl_aes_gcm_neon.c} | 0 sys/crypto/openssl/{amd64 => }/ossl_aes_gcm.c | 397 ++++++++------------- sys/crypto/openssl/ossl_ppc.c | 9 +- sys/crypto/openssl/ossl_x86.c | 6 +- sys/modules/ossl/Makefile | 7 +- 9 files changed, 410 insertions(+), 249 deletions(-) diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index c12ab9db030a..a342242ac66e 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -107,7 +107,8 @@ crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl crypto/openssl/amd64/sha256-x86_64.S optional ossl crypto/openssl/amd64/sha512-x86_64.S optional ossl -crypto/openssl/amd64/ossl_aes_gcm.c optional ossl +crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl +crypto/openssl/ossl_aes_gcm.c optional ossl dev/amdgpio/amdgpio.c optional amdgpio dev/axgbe/if_axgbe_pci.c optional axp dev/axgbe/xgbe-desc.c optional axp diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 91b01845519e..880e804b6c95 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -132,7 +132,7 @@ libkern/udivdi3.c standard libkern/umoddi3.c standard crypto/openssl/ossl_arm.c optional ossl -crypto/openssl/arm/ossl_aes_gcm.c optional ossl +crypto/openssl/arm/ossl_aes_gcm_neon.c optional ossl crypto/openssl/arm/aes-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/bsaes-armv7.S optional ossl \ diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index d2c3aa260cd9..0deada385f31 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -31,9 +31,11 @@ zfs-sha512-ppc.o optional zfs \ # openssl ppc common files crypto/openssl/ossl_ppc.c optional ossl powerpc64 | ossl powerpc64le +crypto/openssl/ossl_aes_gcm.c optional ossl powerpc64 | ossl powerpc64le # openssl assembly files (powerpc64le) crypto/openssl/powerpc64le/aes-ppc.S optional ossl powerpc64le +crypto/openssl/powerpc64le/aes-gcm-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/aesp8-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/chacha-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/ecp_nistz256-ppc64.S optional ossl powerpc64le @@ -54,6 +56,7 @@ crypto/openssl/powerpc64le/x25519-ppc64.S optional ossl powerpc64le # openssl assembly files (powerpc64) crypto/openssl/powerpc64/aes-ppc.S optional ossl powerpc64 +crypto/openssl/powerpc64/aes-gcm-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/aesp8-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/chacha-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/ecp_nistz256-ppc64.S optional ossl powerpc64 diff --git a/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c b/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c new file mode 100644 index 000000000000..694ed4fc8b32 --- /dev/null +++ b/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c @@ -0,0 +1,232 @@ +/* + * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (c) 2021, Intel Corporation. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* + * This file contains an AES-GCM wrapper implementation from OpenSSL, using + * VAES extensions. It was ported from cipher_aes_gcm_hw_vaes_avx512.inc. + */ + +#include <sys/endian.h> +#include <sys/systm.h> + +#include <crypto/openssl/ossl.h> +#include <crypto/openssl/ossl_aes_gcm.h> +#include <crypto/openssl/ossl_cipher.h> + +#include <opencrypto/cryptodev.h> + +_Static_assert( + sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context), + "ossl_gcm_context too large"); + +void aesni_set_encrypt_key(const void *key, int bits, void *ctx); + +static void +gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +{ + KASSERT(keylen == 128 || keylen == 192 || keylen == 256, + ("%s: invalid key length %zu", __func__, keylen)); + + memset(&ctx->gcm, 0, sizeof(ctx->gcm)); + memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); + aesni_set_encrypt_key(key, keylen, &ctx->aes_ks); + ctx->ops->init(ctx, key, keylen); +} + +static void +gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) +{ + (void)ctx->ops->finish(ctx, NULL, 0); + memcpy(tag, ctx->gcm.Xi.c, len); +} + +void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx); +void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); +void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, + const unsigned char *iv, size_t ivlen); +void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, + size_t len); +void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx, + unsigned int *pblocklen, const unsigned char *in, size_t len, + unsigned char *out); +void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx, + unsigned int *pblocklen, const unsigned char *in, size_t len, + unsigned char *out); +void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); + +static void +gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +{ + ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm); +} + +static void +gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv, + size_t len) +{ + KASSERT(len == AES_GCM_IV_LEN, + ("%s: invalid IV length %zu", __func__, len)); + + ctx->gcm.Yi.u[0] = 0; /* Current counter */ + ctx->gcm.Yi.u[1] = 0; + ctx->gcm.Xi.u[0] = 0; /* AAD hash */ + ctx->gcm.Xi.u[1] = 0; + ctx->gcm.len.u[0] = 0; /* AAD length */ + ctx->gcm.len.u[1] = 0; /* Message length */ + ctx->gcm.ares = 0; + ctx->gcm.mres = 0; + + ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len); +} + +static int +gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad, + size_t len) +{ + uint64_t alen = ctx->gcm.len.u[0]; + size_t lenblks; + unsigned int ares; + + /* Bad sequence: call of AAD update after message processing */ + if (ctx->gcm.len.u[1]) + return -2; + + alen += len; + /* AAD is limited by 2^64 bits, thus 2^61 bytes */ + if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) + return -1; + ctx->gcm.len.u[0] = alen; + + ares = ctx->gcm.ares; + /* Partial AAD block left from previous AAD update calls */ + if (ares > 0) { + /* + * Fill partial block buffer till full block + * (note, the hash is stored reflected) + */ + while (ares > 0 && len > 0) { + ctx->gcm.Xi.c[15 - ares] ^= *(aad++); + --len; + ares = (ares + 1) % AES_BLOCK_LEN; + } + /* Full block gathered */ + if (ares == 0) { + ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); + } else { /* no more AAD */ + ctx->gcm.ares = ares; + return 0; + } + } + + /* Bulk AAD processing */ + lenblks = len & ((size_t)(-AES_BLOCK_LEN)); + if (lenblks > 0) { + ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks); + aad += lenblks; + len -= lenblks; + } + + /* Add remaining AAD to the hash (note, the hash is stored reflected) */ + if (len > 0) { + ares = (unsigned int)len; + for (size_t i = 0; i < len; ++i) + ctx->gcm.Xi.c[15 - i] ^= aad[i]; + } + + ctx->gcm.ares = ares; + + return 0; +} + +static int +_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len, bool encrypt) +{ + uint64_t mlen = ctx->gcm.len.u[1]; + + mlen += len; + if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) + return -1; + + ctx->gcm.len.u[1] = mlen; + + /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ + if (ctx->gcm.ares > 0) { + ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); + ctx->gcm.ares = 0; + } + + if (encrypt) { + ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, + in, len, out); + } else { + ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, + in, len, out); + } + + return 0; +} + +static int +gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len) +{ + return _gcm_encrypt_avx512(ctx, in, out, len, true); +} + +static int +gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len) +{ + return _gcm_encrypt_avx512(ctx, in, out, len, false); +} + +static int +gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag, + size_t len) +{ + unsigned int *res = &ctx->gcm.mres; + + /* Finalize AAD processing */ + if (ctx->gcm.ares > 0) + res = &ctx->gcm.ares; + + ossl_aes_gcm_finalize_avx512(ctx, *res); + + ctx->gcm.ares = ctx->gcm.mres = 0; + + if (tag != NULL) + return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); + return 0; +} + +static const struct ossl_aes_gcm_ops gcm_ops_avx512 = { + .init = gcm_init_avx512, + .setiv = gcm_setiv_avx512, + .aad = gcm_aad_avx512, + .encrypt = gcm_encrypt_avx512, + .decrypt = gcm_decrypt_avx512, + .finish = gcm_finish_avx512, + .tag = gcm_tag, +}; + +int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx); + +int +ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, + void *_ctx) +{ + struct ossl_gcm_context *ctx; + + ctx = _ctx; + ctx->ops = &gcm_ops_avx512; + gcm_init(ctx, key, klen); + return (0); +} diff --git a/sys/crypto/openssl/arm/ossl_aes_gcm.c b/sys/crypto/openssl/arm/ossl_aes_gcm_neon.c similarity index 100% rename from sys/crypto/openssl/arm/ossl_aes_gcm.c rename to sys/crypto/openssl/arm/ossl_aes_gcm_neon.c diff --git a/sys/crypto/openssl/amd64/ossl_aes_gcm.c b/sys/crypto/openssl/ossl_aes_gcm.c similarity index 52% rename from sys/crypto/openssl/amd64/ossl_aes_gcm.c rename to sys/crypto/openssl/ossl_aes_gcm.c index d08b2ac8a759..e1cdc710c3aa 100644 --- a/sys/crypto/openssl/amd64/ossl_aes_gcm.c +++ b/sys/crypto/openssl/ossl_aes_gcm.c @@ -1,6 +1,7 @@ /* * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2021, Intel Corporation. All Rights Reserved. + * Copyright (c) 2023, Raptor Engineering, LLC. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,11 +10,10 @@ */ /* - * This file contains 2 AES-GCM wrapper implementations from OpenSSL, using - * AES-NI and VAES extensions respectively. These were ported from - * cipher_aes_gcm_hw_aesni.inc and cipher_aes_gcm_hw_vaes_avx512.inc. The - * AES-NI implementation makes use of a generic C implementation for partial - * blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined. + * This file contains an AES-GCM wrapper implementation from OpenSSL, using + * AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from + * cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation + * for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined. */ #include <sys/endian.h> @@ -29,225 +29,152 @@ _Static_assert( sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context), "ossl_gcm_context too large"); -void aesni_set_encrypt_key(const void *key, int bits, void *ctx); - -static void -gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) -{ - KASSERT(keylen == 128 || keylen == 192 || keylen == 256, - ("%s: invalid key length %zu", __func__, keylen)); - - memset(&ctx->gcm, 0, sizeof(ctx->gcm)); - memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); - aesni_set_encrypt_key(key, keylen, &ctx->aes_ks); - ctx->ops->init(ctx, key, keylen); -} - -static void -gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) -{ - (void)ctx->ops->finish(ctx, NULL, 0); - memcpy(tag, ctx->gcm.Xi.c, len); -} +#if defined(__amd64__) || defined(__i386__) +#define AES_set_encrypt_key aesni_set_encrypt_key +#define AES_gcm_encrypt aesni_gcm_encrypt +#define AES_gcm_decrypt aesni_gcm_decrypt +#define AES_encrypt aesni_encrypt +#define AES_ctr32_encrypt_blocks aesni_ctr32_encrypt_blocks +#define GCM_init gcm_init_avx +#define GCM_gmult gcm_gmult_avx +#define GCM_ghash gcm_ghash_avx + +void AES_set_encrypt_key(const void *key, int bits, void *ctx); +size_t AES_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +size_t AES_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks); +void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, void *ks, const unsigned char *iv); -void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx); -void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); -void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, - const unsigned char *iv, size_t ivlen); -void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, +void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]); +void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]); +void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, size_t len); -void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx, - unsigned int *pblocklen, const unsigned char *in, size_t len, - unsigned char *out); -void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx, - unsigned int *pblocklen, const unsigned char *in, size_t len, - unsigned char *out); -void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); - -static void -gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen) -{ - ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm); -} -static void -gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv, - size_t len) -{ - KASSERT(len == AES_GCM_IV_LEN, - ("%s: invalid IV length %zu", __func__, len)); +#elif defined(__powerpc64__) +#define AES_set_encrypt_key aes_p8_set_encrypt_key +#define AES_gcm_encrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,1) +#define AES_gcm_decrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,0) +#define AES_encrypt aes_p8_encrypt +#define AES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks +#define GCM_init gcm_init_p8 +#define GCM_gmult gcm_gmult_p8 +#define GCM_ghash gcm_ghash_p8 + +size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); - ctx->gcm.Yi.u[0] = 0; /* Current counter */ - ctx->gcm.Yi.u[1] = 0; - ctx->gcm.Xi.u[0] = 0; /* AAD hash */ - ctx->gcm.Xi.u[1] = 0; - ctx->gcm.len.u[0] = 0; /* AAD length */ - ctx->gcm.len.u[1] = 0; /* Message length */ - ctx->gcm.ares = 0; - ctx->gcm.mres = 0; +void AES_set_encrypt_key(const void *key, int bits, void *ctx); +void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks); +void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, void *ks, const unsigned char *iv); - ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len); -} +void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]); +void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]); +void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, + size_t len); -static int -gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad, - size_t len) +static size_t +ppc_aes_gcm_crypt(const unsigned char *in, unsigned char *out, + size_t len, const void *key, unsigned char ivec_[16], uint64_t *Xi, + int encrypt) { - uint64_t alen = ctx->gcm.len.u[0]; - size_t lenblks; - unsigned int ares; - - /* Bad sequence: call of AAD update after message processing */ - if (ctx->gcm.len.u[1]) - return -2; - - alen += len; - /* AAD is limited by 2^64 bits, thus 2^61 bytes */ - if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) - return -1; - ctx->gcm.len.u[0] = alen; + union { + uint32_t d[4]; + uint8_t c[16]; + } *ivec = (void *)ivec_; + int s = 0; + int ndone = 0; + int ctr_reset = 0; + uint32_t ivec_val; + uint64_t blocks_unused; + uint64_t nb = len / 16; + uint64_t next_ctr = 0; + unsigned char ctr_saved[12]; + + memcpy(ctr_saved, ivec, 12); + + while (nb) { + ivec_val = ivec->d[3]; +#if BYTE_ORDER == LITTLE_ENDIAN + ivec_val = bswap32(ivec_val); +#endif - ares = ctx->gcm.ares; - /* Partial AAD block left from previous AAD update calls */ - if (ares > 0) { - /* - * Fill partial block buffer till full block - * (note, the hash is stored reflected) - */ - while (ares > 0 && len > 0) { - ctx->gcm.Xi.c[15 - ares] ^= *(aad++); - --len; - ares = (ares + 1) % AES_BLOCK_LEN; - } - /* Full block gathered */ - if (ares == 0) { - ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); - } else { /* no more AAD */ - ctx->gcm.ares = ares; - return 0; + blocks_unused = (uint64_t)0xffffffffU + 1 - (uint64_t)ivec_val; + if (nb > blocks_unused) { + len = blocks_unused * 16; + nb -= blocks_unused; + next_ctr = blocks_unused; + ctr_reset = 1; + } else { + len = nb * 16; + next_ctr = nb; + nb = 0; } - } - /* Bulk AAD processing */ - lenblks = len & ((size_t)(-AES_BLOCK_LEN)); - if (lenblks > 0) { - ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks); - aad += lenblks; - len -= lenblks; - } + s = encrypt ? ppc_aes_gcm_encrypt(in, out, len, key, ivec->c, Xi) : + ppc_aes_gcm_decrypt(in, out, len, key, ivec->c, Xi); - /* Add remaining AAD to the hash (note, the hash is stored reflected) */ - if (len > 0) { - ares = (unsigned int)len; - for (size_t i = 0; i < len; ++i) - ctx->gcm.Xi.c[15 - i] ^= aad[i]; + /* add counter to ivec */ +#if BYTE_ORDER == LITTLE_ENDIAN + ivec->d[3] = bswap32(ivec_val + next_ctr); +#else + ivec->d[3] += next_ctr; +#endif + if (ctr_reset) { + ctr_reset = 0; + in += len; + out += len; + } + memcpy(ivec, ctr_saved, 12); + ndone += s; } - ctx->gcm.ares = ares; - - return 0; + return ndone; } -static int -_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len, bool encrypt) -{ - uint64_t mlen = ctx->gcm.len.u[1]; - - mlen += len; - if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) - return -1; - - ctx->gcm.len.u[1] = mlen; - - /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ - if (ctx->gcm.ares > 0) { - ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); - ctx->gcm.ares = 0; - } - - if (encrypt) { - ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, - in, len, out); - } else { - ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, - in, len, out); - } - - return 0; -} +#else +#error "Unsupported architecture!" +#endif -static int -gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len) +static void +gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) { - return _gcm_encrypt_avx512(ctx, in, out, len, true); -} + KASSERT(keylen == 128 || keylen == 192 || keylen == 256, + ("%s: invalid key length %zu", __func__, keylen)); -static int -gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len) -{ - return _gcm_encrypt_avx512(ctx, in, out, len, false); + memset(&ctx->gcm, 0, sizeof(ctx->gcm)); + memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); + AES_set_encrypt_key(key, keylen, &ctx->aes_ks); + ctx->ops->init(ctx, key, keylen); } -static int -gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag, - size_t len) +static void +gcm_tag_op(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) { - unsigned int *res = &ctx->gcm.mres; - - /* Finalize AAD processing */ - if (ctx->gcm.ares > 0) - res = &ctx->gcm.ares; - - ossl_aes_gcm_finalize_avx512(ctx, *res); - - ctx->gcm.ares = ctx->gcm.mres = 0; - - if (tag != NULL) - return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); - return 0; + (void)ctx->ops->finish(ctx, NULL, 0); + memcpy(tag, ctx->gcm.Xi.c, len); } -static const struct ossl_aes_gcm_ops gcm_ops_avx512 = { - .init = gcm_init_avx512, - .setiv = gcm_setiv_avx512, - .aad = gcm_aad_avx512, - .encrypt = gcm_encrypt_avx512, - .decrypt = gcm_decrypt_avx512, - .finish = gcm_finish_avx512, - .tag = gcm_tag, -}; - -size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi); -size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi); -void aesni_encrypt(const unsigned char *in, unsigned char *out, void *ks); -void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, - size_t blocks, void *ks, const unsigned char *iv); - -void gcm_init_avx(__uint128_t Htable[16], uint64_t Xi[2]); -void gcm_gmult_avx(uint64_t Xi[2], const __uint128_t Htable[16]); -void gcm_ghash_avx(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, - size_t len); - static void -gcm_init_aesni(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +gcm_init_op(struct ossl_gcm_context *ctx, const void *key, size_t keylen) { - aesni_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks); #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.H.u[0] = bswap64(ctx->gcm.H.u[0]); ctx->gcm.H.u[1] = bswap64(ctx->gcm.H.u[1]); #endif - gcm_init_avx(ctx->gcm.Htable, ctx->gcm.H.u); + GCM_init(ctx->gcm.Htable, ctx->gcm.H.u); } static void -gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv, +gcm_setiv_op(struct ossl_gcm_context *ctx, const unsigned char *iv, size_t len) { uint32_t ctr; @@ -269,7 +196,7 @@ gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv, ctx->gcm.Xi.u[0] = 0; ctx->gcm.Xi.u[1] = 0; - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks); ctr++; #if BYTE_ORDER == LITTLE_ENDIAN @@ -280,7 +207,7 @@ gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv, } static int -gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad, +gcm_aad_op(struct ossl_gcm_context *ctx, const unsigned char *aad, size_t len) { size_t i; @@ -303,14 +230,14 @@ gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad, n = (n + 1) % 16; } if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); else { ctx->gcm.ares = n; return 0; } } if ((i = (len & (size_t)-AES_BLOCK_LEN))) { - gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i); + GCM_ghash(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i); aad += i; len -= i; } @@ -341,7 +268,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in, if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } @@ -354,7 +281,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in, n = mres % 16; for (i = 0; i < len; ++i) { if (n == 0) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN @@ -366,7 +293,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in, ctx->gcm.Xi.c[n] ^= out[i] = in[i] ^ ctx->gcm.EKi.c[n]; mres = n = (n + 1) % 16; if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); } ctx->gcm.mres = mres; @@ -390,7 +317,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } @@ -408,7 +335,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, n = (n + 1) % 16; } if (n == 0) { - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); mres = 0; } else { ctx->gcm.mres = n; @@ -418,7 +345,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, if ((i = (len & (size_t)-16))) { size_t j = i / 16; - aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); + AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); ctr += (unsigned int)j; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); @@ -430,12 +357,12 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, while (j--) { for (i = 0; i < 16; ++i) ctx->gcm.Xi.c[i] ^= out[i]; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); out += 16; } } if (len) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); @@ -453,7 +380,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, } static int -gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, +gcm_encrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { size_t bulk = 0, res; @@ -463,7 +390,7 @@ gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, if ((error = gcm_encrypt(ctx, in, out, res)) != 0) return error; - bulk = aesni_gcm_encrypt(in + res, out + res, len - res, + bulk = AES_gcm_encrypt(in + res, out + res, len - res, &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u); ctx->gcm.len.u[1] += bulk; bulk += res; @@ -492,7 +419,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in, if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } @@ -506,7 +433,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in, for (i = 0; i < len; ++i) { uint8_t c; if (n == 0) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN @@ -520,7 +447,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in, ctx->gcm.Xi.c[n] ^= c; mres = n = (n + 1) % 16; if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); } ctx->gcm.mres = mres; @@ -544,7 +471,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, if (ctx->gcm.ares) { /* First call to decrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } @@ -564,7 +491,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, n = (n + 1) % 16; } if (n == 0) { - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); mres = 0; } else { ctx->gcm.mres = n; @@ -578,12 +505,12 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, size_t k; for (k = 0; k < 16; ++k) ctx->gcm.Xi.c[k] ^= in[k]; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); in += 16; } j = i / 16; in -= i; - aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); + AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); ctr += (unsigned int)j; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); @@ -595,7 +522,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, len -= i; } if (len) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); @@ -615,7 +542,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, } static int -gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, +gcm_decrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { size_t bulk = 0, res; @@ -625,7 +552,7 @@ gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, if ((error = gcm_decrypt(ctx, in, out, res)) != 0) return error; - bulk = aesni_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks, + bulk = AES_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u); ctx->gcm.len.u[1] += bulk; bulk += res; @@ -637,14 +564,14 @@ gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, } static int -gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag, +gcm_finish_op(struct ossl_gcm_context *ctx, const unsigned char *tag, size_t len) { uint64_t alen = ctx->gcm.len.u[0] << 3; uint64_t clen = ctx->gcm.len.u[1] << 3; if (ctx->gcm.mres || ctx->gcm.ares) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); #if BYTE_ORDER == LITTLE_ENDIAN alen = bswap64(alen); @@ -653,7 +580,7 @@ gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag, ctx->gcm.Xi.u[0] ^= alen; ctx->gcm.Xi.u[1] ^= clen; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0]; ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1]; @@ -663,40 +590,26 @@ gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag, return 0; } -static const struct ossl_aes_gcm_ops gcm_ops_aesni = { - .init = gcm_init_aesni, - .setiv = gcm_setiv_aesni, - .aad = gcm_aad_aesni, - .encrypt = gcm_encrypt_aesni, - .decrypt = gcm_decrypt_aesni, - .finish = gcm_finish_aesni, - .tag = gcm_tag, +static const struct ossl_aes_gcm_ops gcm_ops = { + .init = gcm_init_op, + .setiv = gcm_setiv_op, + .aad = gcm_aad_op, + .encrypt = gcm_encrypt_op, + .decrypt = gcm_decrypt_op, + .finish = gcm_finish_op, + .tag = gcm_tag_op, }; -int ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, void *_ctx); - -int -ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, - void *_ctx) -{ - struct ossl_gcm_context *ctx; - - ctx = _ctx; - ctx->ops = &gcm_ops_aesni; - gcm_init(ctx, key, klen); - return (0); -} - -int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx); +int ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx); int -ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, +ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx) { struct ossl_gcm_context *ctx; ctx = _ctx; - ctx->ops = &gcm_ops_avx512; + ctx->ops = &gcm_ops; gcm_init(ctx, key, klen); return (0); } diff --git a/sys/crypto/openssl/ossl_ppc.c b/sys/crypto/openssl/ossl_ppc.c index 0951745c4b43..980211f46a76 100644 *** 91 LINES SKIPPED ***
