* cipher/sm4.c (ocb_crypt_fn_t, ocb_auth_fn_t) [ASM_FUNC_ABI]: New. (_gcry_sm4_aesni_avx_ocb_enc, _gcry_sm4_aesni_avx_ocb_dec) (_gcry_sm4_aesni_avx_ocb_auth, _gcry_sm4_aesni_avx2_ocb_enc) (_gcry_sm4_aesni_avx2_ocb_dec, _gcry_sm4_aesni_avx2_ocb_auth) (_gcry_sm4_intel_avx2_ocb_enc, _gcry_sm4_intel_avx2_ocb_dec) (_gcry_sm4_intel_avx2_ocb_auth, _gcry_sm4_intel_avx512_ocb_enc_blk32) (_gcry_sm4_intel_avx512_ocb_dec_blk32, _gcry_sm4_gfni_avx2_ocb_enc) (_gcry_sm4_gfni_avx2_ocb_dec, _gcry_sm4_gfni_avx2_ocb_auth) (_gcry_sm4_gfni_avx512_ocb_enc, _gcry_sm4_gfni_avx512_ocb_dec) (_gcry_sm4_gfni_avx512_ocb_auth, _gcry_sm4_gfni_avx512_ocb_enc_blk32) (_gcry_sm4_gfni_avx512_ocb_dec_blk32): Make 'Ls' an unsized array parameter. (_gcry_sm4_ocb_crypt, _gcry_sm4_ocb_auth) [ASM_FUNC_ABI]: Select the bulk implementation through a function pointer and share the 8/16/32-block processing loops. --
The bulk OCB encrypt/decrypt and authentication paths repeated the same block-chunk processing loop for each implementation. Select the implementation's bulk function through a function pointer instead, so the loops are written only once. Signed-off-by: Jussi Kivilinna <[email protected]> --- cipher/sm4.c | 338 ++++++++++++++++----------------------------------- 1 file changed, 103 insertions(+), 235 deletions(-) diff --git a/cipher/sm4.c b/cipher/sm4.c index 37e1a6c3..3c60dafc 100644 --- a/cipher/sm4.c +++ b/cipher/sm4.c @@ -172,6 +172,16 @@ static size_t _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, typedef bulk_crypt_fn_t crypt_blk1_16_fn_t; +#ifdef ASM_FUNC_ABI +typedef void (*ocb_crypt_fn_t)(const u32 *rk_enc, unsigned char *out, + const unsigned char *in, unsigned char *offset, + unsigned char *checksum, + const u64 Ls[]) ASM_FUNC_ABI; +typedef void (*ocb_auth_fn_t)(const u32 *rk_enc, const unsigned char *abuf, + unsigned char *offset, unsigned char *checksum, + const u64 Ls[]) ASM_FUNC_ABI; +#endif + typedef struct { u32 rkey_enc[32]; @@ -1934,79 +1944,71 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; u64 blkn = c->u_mode.ocb.data_nblocks; + u32 *rk = encrypt ? ctx->rkey_enc : ctx->rkey_dec; int burn_stack_depth = 0; +#ifdef ASM_FUNC_ABI + int bulk_func_available = 0; + ocb_crypt_fn_t crypt_blk32 = NULL; + ocb_crypt_fn_t crypt_blk16 = NULL; + ocb_crypt_fn_t crypt_blk8 = NULL; -#ifdef USE_INTEL_SM4_AVX512 - if (ctx->use_intel_sm4_avx512) +#ifdef USE_AESNI_AVX + if (ctx->use_aesni_avx) { - u64 Ls[32]; - u64 *l; - - if (nblocks >= 32) - { - l = bulk_ocb_prepare_L_pointers_array_blk32 (c, Ls, blkn); - - /* Process data in 32 block chunks. */ - while (nblocks >= 32) - { - blkn += 32; - *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 32); - - if (encrypt) - _gcry_sm4_intel_avx512_ocb_enc_blk32 (ctx->rkey_enc, outbuf, - inbuf, c->u_iv.iv, - c->u_ctr.ctr, Ls); - else - _gcry_sm4_intel_avx512_ocb_dec_blk32 (ctx->rkey_dec, outbuf, - inbuf, c->u_iv.iv, - c->u_ctr.ctr, Ls); - - nblocks -= 32; - outbuf += 32 * 16; - inbuf += 32 * 16; - } - } + bulk_func_available = 1; + crypt_blk8 = encrypt ? _gcry_sm4_aesni_avx_ocb_enc : + _gcry_sm4_aesni_avx_ocb_dec; + } +#endif +#ifdef USE_AESNI_AVX2 + if (ctx->use_aesni_avx2) + { + bulk_func_available = 1; + crypt_blk16 = encrypt ? _gcry_sm4_aesni_avx2_ocb_enc : + _gcry_sm4_aesni_avx2_ocb_dec; + } +#endif +#ifdef USE_GFNI_AVX2 + if (ctx->use_gfni_avx2) + { + bulk_func_available = 1; + crypt_blk16 = encrypt ? _gcry_sm4_gfni_avx2_ocb_enc : + _gcry_sm4_gfni_avx2_ocb_dec; + } +#endif +#ifdef USE_GFNI_AVX512 + if (ctx->use_gfni_avx512) + { + bulk_func_available = 1; + crypt_blk16 = encrypt ? _gcry_sm4_gfni_avx512_ocb_enc : + _gcry_sm4_gfni_avx512_ocb_dec; + crypt_blk32 = encrypt ? _gcry_sm4_gfni_avx512_ocb_enc_blk32 : + _gcry_sm4_gfni_avx512_ocb_dec_blk32; } #endif - #ifdef USE_INTEL_SM4_AVX2 if (ctx->use_intel_sm4_avx2) { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - while (nblocks >= 16) - { - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - - if (encrypt) - _gcry_sm4_intel_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - else - _gcry_sm4_intel_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - - nblocks -= 16; - outbuf += 16 * 16; - inbuf += 16 * 16; - } - } + bulk_func_available = 1; + crypt_blk16 = encrypt ? _gcry_sm4_intel_avx2_ocb_enc : + _gcry_sm4_intel_avx2_ocb_dec; + } +#endif +#ifdef USE_INTEL_SM4_AVX512 + if (ctx->use_intel_sm4_avx512) + { + bulk_func_available = 1; + crypt_blk32 = encrypt ? _gcry_sm4_intel_avx512_ocb_enc_blk32 : + _gcry_sm4_intel_avx512_ocb_dec_blk32; } #endif -#ifdef USE_GFNI_AVX512 - if (ctx->use_gfni_avx512) + if (bulk_func_available) { u64 Ls[32]; u64 *l; - if (nblocks >= 32) + if (crypt_blk32 != NULL && nblocks >= 32) { l = bulk_ocb_prepare_L_pointers_array_blk32 (c, Ls, blkn); @@ -2016,14 +2018,8 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, blkn += 32; *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 32); - if (encrypt) - _gcry_sm4_gfni_avx512_ocb_enc_blk32 (ctx->rkey_enc, outbuf, - inbuf, c->u_iv.iv, - c->u_ctr.ctr, Ls); - else - _gcry_sm4_gfni_avx512_ocb_dec_blk32 (ctx->rkey_dec, outbuf, - inbuf, c->u_iv.iv, - c->u_ctr.ctr, Ls); + crypt_blk32 (rk, outbuf, inbuf, c->u_iv.iv, + c->u_ctr.ctr, Ls); nblocks -= 32; outbuf += 32 * 16; @@ -2031,66 +2027,7 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, } } - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - - if (encrypt) - _gcry_sm4_gfni_avx512_ocb_enc(ctx->rkey_enc, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - else - _gcry_sm4_gfni_avx512_ocb_dec(ctx->rkey_dec, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - - nblocks -= 16; - outbuf += 16 * 16; - inbuf += 16 * 16; - } - } -#endif - -#ifdef USE_GFNI_AVX2 - if (ctx->use_gfni_avx2) - { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - while (nblocks >= 16) - { - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - - if (encrypt) - _gcry_sm4_gfni_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - else - _gcry_sm4_gfni_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - - nblocks -= 16; - outbuf += 16 * 16; - inbuf += 16 * 16; - } - } - } -#endif - -#ifdef USE_AESNI_AVX2 - if (ctx->use_aesni_avx2) - { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) + if (crypt_blk16 != NULL && nblocks >= 16) { l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); @@ -2100,28 +2037,16 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, blkn += 16; *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - if (encrypt) - _gcry_sm4_aesni_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - else - _gcry_sm4_aesni_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); + crypt_blk16 (rk, outbuf, inbuf, c->u_iv.iv, + c->u_ctr.ctr, Ls); nblocks -= 16; outbuf += 16 * 16; inbuf += 16 * 16; } } - } -#endif - -#ifdef USE_AESNI_AVX - if (ctx->use_aesni_avx) - { - u64 Ls[8]; - u64 *l; - if (nblocks >= 8) + if (crypt_blk8 != NULL && nblocks >= 8) { l = bulk_ocb_prepare_L_pointers_array_blk8 (c, Ls, blkn); @@ -2131,12 +2056,8 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, blkn += 8; *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 8); - if (encrypt) - _gcry_sm4_aesni_avx_ocb_enc(ctx->rkey_enc, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); - else - _gcry_sm4_aesni_avx_ocb_dec(ctx->rkey_dec, outbuf, inbuf, - c->u_iv.iv, c->u_ctr.ctr, Ls); + crypt_blk8 (rk, outbuf, inbuf, c->u_iv.iv, + c->u_ctr.ctr, Ls); nblocks -= 8; outbuf += 8 * 16; @@ -2150,7 +2071,6 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg, if (nblocks) { crypt_blk1_16_fn_t crypt_blk1_16 = ctx->crypt_blk1_16; - u32 *rk = encrypt ? ctx->rkey_enc : ctx->rkey_dec; unsigned char tmpbuf[16 * 16]; unsigned int tmp_used = 16; size_t nburn; @@ -2179,95 +2099,53 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks) const unsigned char *abuf = abuf_arg; u64 blkn = c->u_mode.ocb.aad_nblocks; int burn_stack_depth = 0; +#ifdef ASM_FUNC_ABI + int bulk_func_available = 0; + ocb_auth_fn_t auth_blk16 = NULL; + ocb_auth_fn_t auth_blk8 = NULL; -#ifdef USE_INTEL_SM4_AVX2 - if (ctx->use_intel_sm4_avx2) +#ifdef USE_AESNI_AVX + if (ctx->use_aesni_avx) { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - while (nblocks >= 16) - { - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - - _gcry_sm4_intel_avx2_ocb_auth(ctx->rkey_enc, abuf, - c->u_mode.ocb.aad_offset, - c->u_mode.ocb.aad_sum, Ls); - - nblocks -= 16; - abuf += 16 * 16; - } - } + bulk_func_available = 1; + auth_blk8 = _gcry_sm4_aesni_avx_ocb_auth; } #endif - -#ifdef USE_GFNI_AVX512 - if (ctx->use_gfni_avx512) +#ifdef USE_AESNI_AVX2 + if (ctx->use_aesni_avx2) { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - while (nblocks >= 16) - { - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 16); - - _gcry_sm4_gfni_avx512_ocb_auth (ctx->rkey_enc, abuf, - c->u_mode.ocb.aad_offset, - c->u_mode.ocb.aad_sum, Ls); - - nblocks -= 16; - abuf += 16 * 16; - } - } + bulk_func_available = 1; + auth_blk16 = _gcry_sm4_aesni_avx2_ocb_auth; } #endif - #ifdef USE_GFNI_AVX2 if (ctx->use_gfni_avx2) { - u64 Ls[16]; - u64 *l; - - if (nblocks >= 16) - { - l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); - - /* Process data in 16 block chunks. */ - while (nblocks >= 16) - { - blkn += 16; - *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - - _gcry_sm4_gfni_avx2_ocb_auth(ctx->rkey_enc, abuf, - c->u_mode.ocb.aad_offset, - c->u_mode.ocb.aad_sum, Ls); - - nblocks -= 16; - abuf += 16 * 16; - } - } + bulk_func_available = 1; + auth_blk16 = _gcry_sm4_gfni_avx2_ocb_auth; + } +#endif +#ifdef USE_GFNI_AVX512 + if (ctx->use_gfni_avx512) + { + bulk_func_available = 1; + auth_blk16 = _gcry_sm4_gfni_avx512_ocb_auth; + } +#endif +#ifdef USE_INTEL_SM4_AVX2 + if (ctx->use_intel_sm4_avx2) + { + bulk_func_available = 1; + auth_blk16 = _gcry_sm4_intel_avx2_ocb_auth; } #endif -#ifdef USE_AESNI_AVX2 - if (ctx->use_aesni_avx2) + if (bulk_func_available) { u64 Ls[16]; u64 *l; - if (nblocks >= 16) + if (auth_blk16 != NULL && nblocks >= 16) { l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn); @@ -2277,24 +2155,15 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks) blkn += 16; *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16); - _gcry_sm4_aesni_avx2_ocb_auth(ctx->rkey_enc, abuf, - c->u_mode.ocb.aad_offset, - c->u_mode.ocb.aad_sum, Ls); + auth_blk16(ctx->rkey_enc, abuf, c->u_mode.ocb.aad_offset, + c->u_mode.ocb.aad_sum, Ls); nblocks -= 16; abuf += 16 * 16; } } - } -#endif -#ifdef USE_AESNI_AVX - if (ctx->use_aesni_avx) - { - u64 Ls[8]; - u64 *l; - - if (nblocks >= 8) + if (auth_blk8 != NULL && nblocks >= 8) { l = bulk_ocb_prepare_L_pointers_array_blk8 (c, Ls, blkn); @@ -2304,9 +2173,8 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks) blkn += 8; *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 8); - _gcry_sm4_aesni_avx_ocb_auth(ctx->rkey_enc, abuf, - c->u_mode.ocb.aad_offset, - c->u_mode.ocb.aad_sum, Ls); + auth_blk8(ctx->rkey_enc, abuf, c->u_mode.ocb.aad_offset, + c->u_mode.ocb.aad_sum, Ls); nblocks -= 8; abuf += 8 * 16; -- 2.53.0 _______________________________________________ Gcrypt-devel mailing list [email protected] https://lists.gnupg.org/mailman/listinfo/gcrypt-devel
