* cipher/twofish.c (twofish_setkey): Assign ctr16be_enc bulk op. (_gcry_twofish_ctr_enc): Use cipher_block_add_be16. * cipher/twofish-amd64.S (RT0bl, RT1bl): New. (_gcry_twofish_amd64_ctr_enc): Increment low 16 counter bits in place, drop full-width carry and byteswap. * cipher/twofish-avx2-amd64.S (inc_le128): Remove. (_gcry_twofish_avx2_ctr_enc): Add to low 16 counter bits only, drop full-width carry handling. --
Twofish CTR built IVs with full 128-bit increment and byteswap round-trip. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Scalar amd64 path now increments low counter bytes in place, dropping byteswap. AVX2 path uses 16-bit stepping and drops full-width carry. Moves twofish off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna <[email protected]> --- cipher/twofish-amd64.S | 53 +++++++++++++-------------- cipher/twofish-avx2-amd64.S | 71 +++++++------------------------------ cipher/twofish.c | 4 +-- 3 files changed, 39 insertions(+), 89 deletions(-) diff --git a/cipher/twofish-amd64.S b/cipher/twofish-amd64.S index 913b252d..beafd10f 100644 --- a/cipher/twofish-amd64.S +++ b/cipher/twofish-amd64.S @@ -359,6 +359,9 @@ ELF(.size _gcry_twofish_amd64_encrypt_block,.-_gcry_twofish_amd64_encrypt_block; #define RT0d %edx #define RT1d %esi +#define RT0bl %dl +#define RT1bl %sil + #define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \ movzbl ab ## bl, tmp2 ## d; \ movzbl ab ## bh, tmp1 ## d; \ @@ -650,35 +653,27 @@ _gcry_twofish_amd64_ctr_enc: movq %rdx, (7 * 8)(%rsp); movq %rcx, RX0; - /* load IV and byteswap */ - movq 8(RX0), RT0; - movq 0(RX0), RT1; - movq RT0, RCD0; - movq RT1, RAB0; - bswapq RT0; - bswapq RT1; - - /* construct IVs */ - movq RT0, RCD1; - movq RT1, RAB1; - movq RT0, RCD2; - movq RT1, RAB2; - addq $1, RCD1; - adcq $0, RAB1; - bswapq RCD1; - bswapq RAB1; - addq $2, RCD2; - adcq $0, RAB2; - bswapq RCD2; - bswapq RAB2; - addq $3, RT0; - adcq $0, RT1; - bswapq RT0; - bswapq RT1; - - /* store new IV */ - movq RT0, 8(RX0); - movq RT1, 0(RX0); + /* Construct and update IV */ + movzbl 15(RX0), RT0d; + movzbl 14(RX0), RT1d; + movq 8(RX0), RCD0; + movq 0(RX0), RAB0; + movq RAB0, RAB1; + movq RAB0, RAB2; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); + movq 8(RX0), RCD1; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); + movq 8(RX0), RCD2; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); call __twofish_enc_blk3; diff --git a/cipher/twofish-avx2-amd64.S b/cipher/twofish-avx2-amd64.S index 67c1e52e..72a39d17 100644 --- a/cipher/twofish-avx2-amd64.S +++ b/cipher/twofish-avx2-amd64.S @@ -553,12 +553,6 @@ _gcry_twofish_avx2_blk16: CFI_ENDPROC(); ELF(.size _gcry_twofish_avx2_blk16,.-_gcry_twofish_avx2_blk16;) -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl _gcry_twofish_avx2_ctr_enc ELF(.type _gcry_twofish_avx2_ctr_enc,@function;) @@ -571,79 +565,40 @@ _gcry_twofish_avx2_ctr_enc: */ CFI_STARTPROC(); - movq 8(%rcx), %rax; - bswapq %rax; - vzeroupper; vbroadcasti128 .Lbswap128_mask rRIP, RTMP3; vpcmpeqd RNOT, RNOT, RNOT; - vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ - vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ + vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ + vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ /* load IV and byteswap */ vmovdqu (%rcx), RTMP4x; vpshufb RTMP3x, RTMP4x, RTMP4x; vmovdqa RTMP4x, RTMP0x; - inc_le128(RTMP4x, RNOTx, RTMP1x); + vpsubw RNOTx, RTMP4x, RTMP4x; vinserti128 $1, RTMP4x, RTMP0, RTMP0; vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */ - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 16), %rax; - ja .Lhandle_ctr_carry; + /* Update IV */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); /* construct IVs */ - vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ vpshufb RTMP3, RTMP0, RB0; - vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ vpshufb RTMP3, RTMP0, RC0; - vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ vpshufb RTMP3, RTMP0, RD0; - vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ vpshufb RTMP3, RTMP0, RA1; - vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ vpshufb RTMP3, RTMP0, RB1; - vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ vpshufb RTMP3, RTMP0, RC1; - vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ vpshufb RTMP3, RTMP0, RD1; - vpsubq RTMP2, RTMP0, RTMP0; /* +16 */ - vpshufb RTMP3x, RTMP0x, RTMP0x; - - jmp .Lctr_carry_done; - -.Lhandle_ctr_carry: - /* construct IVs */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB0; /* +3 ; +2 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RC0; /* +5 ; +4 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RD0; /* +7 ; +6 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA1; /* +9 ; +8 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RC1; /* +13 ; +12 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RD1; /* +15 ; +14 */ - inc_le128(RTMP0, RNOT, RTMP1); - vextracti128 $1, RTMP0, RTMP0x; - vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */ - -.align 4 -.Lctr_carry_done: - /* store new IV */ - vmovdqu RTMP0x, (%rcx); call __twofish_enc_blk16; diff --git a/cipher/twofish.c b/cipher/twofish.c index 11a6e251..4341a7fb 100644 --- a/cipher/twofish.c +++ b/cipher/twofish.c @@ -774,7 +774,7 @@ twofish_setkey (void *context, const byte *key, unsigned int keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_twofish_cbc_dec; bulk_ops->cfb_dec = _gcry_twofish_cfb_dec; - bulk_ops->ctr_enc = _gcry_twofish_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_twofish_ctr_enc; bulk_ops->ocb_crypt = _gcry_twofish_ocb_crypt; bulk_ops->ocb_auth = _gcry_twofish_ocb_auth; bulk_ops->xts_crypt = _gcry_twofish_xts_crypt; @@ -1197,7 +1197,7 @@ _gcry_twofish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, outbuf += TWOFISH_BLOCKSIZE; inbuf += TWOFISH_BLOCKSIZE; /* Increment the counter. */ - cipher_block_add(ctr, 1, TWOFISH_BLOCKSIZE); + cipher_block_add_be16(ctr, 1, TWOFISH_BLOCKSIZE); } wipememory(tmpbuf, sizeof(tmpbuf)); -- 2.53.0 _______________________________________________ Gcrypt-devel mailing list [email protected] https://lists.gnupg.org/mailman/listinfo/gcrypt-devel
