The branch master has been updated
via b4f2a462b752213135f6a64b22c8085901d2cb53 (commit)
via edbc681d2240c4f5c9ae097347bddbea2f5ba088 (commit)
via b547aba9546aeea06b55905e22144ef350fa9afa (commit)
via 54f8f9a1edfcf1a2d2df801728e462841e2c316e (commit)
from a95d7574dbcd91c734c1542a423e1cac34dc18b5 (commit)
- Log -----------------------------------------------------------------
commit b4f2a462b752213135f6a64b22c8085901d2cb53
Author: Andy Polyakov <[email protected]>
Date: Fri Jun 30 13:35:59 2017 +0200
sha/keccak1600.c: internalize KeccakF1600 and simplify SHA3_absorb.
Reviewed-by: Bernd Edlinger <[email protected]>
commit edbc681d2240c4f5c9ae097347bddbea2f5ba088
Author: Andy Polyakov <[email protected]>
Date: Fri Jun 30 11:45:34 2017 +0200
sha/asm/keccak1600-x86_64.pl: close gap with Keccak Code Package.
[Also typo and readability fixes. Ryzen result is added.]
Reviewed-by: Bernd Edlinger <[email protected]>
commit b547aba9546aeea06b55905e22144ef350fa9afa
Author: Andy Polyakov <[email protected]>
Date: Fri Jun 30 11:42:58 2017 +0200
sha/asm/keccak1600-s390x.pl: typo and readability, minor size optimization.
Reviewed-by: Bernd Edlinger <[email protected]>
commit 54f8f9a1edfcf1a2d2df801728e462841e2c316e
Author: Andy Polyakov <[email protected]>
Date: Fri Jun 30 11:42:00 2017 +0200
x86_64 assembly pack: fill some blanks in Ryzen results.
Reviewed-by: Bernd Edlinger <[email protected]>
-----------------------------------------------------------------------
Summary of changes:
crypto/aes/asm/aesni-x86_64.pl | 2 +-
crypto/chacha/asm/chacha-x86_64.pl | 2 +-
crypto/poly1305/asm/poly1305-x86_64.pl | 2 +-
crypto/sha/asm/keccak1600-s390x.pl | 23 +++++--------
crypto/sha/asm/keccak1600-x86_64.pl | 63 +++++++++++++++++-----------------
crypto/sha/asm/sha1-x86_64.pl | 2 +-
crypto/sha/asm/sha512-x86_64.pl | 2 +-
crypto/sha/keccak1600.c | 52 +++++++++-------------------
8 files changed, 61 insertions(+), 87 deletions(-)
diff --git a/crypto/aes/asm/aesni-x86_64.pl b/crypto/aes/asm/aesni-x86_64.pl
index 004bf02..ea5dc14 100644
--- a/crypto/aes/asm/aesni-x86_64.pl
+++ b/crypto/aes/asm/aesni-x86_64.pl
@@ -181,7 +181,7 @@
# Silvermont 5.75/3.54 3.56 4.12 3.87(*) 4.11
# Goldmont 3.82/1.26 1.26 1.29 1.29 1.50
# Bulldozer 5.77/0.70 0.72 0.90 0.70 0.95
-# Ryzen 2.71/0.35 0.35 0.44 ? ?
+# Ryzen 2.71/0.35 0.35 0.44 0.38 0.49
#
# (*) Atom Silvermont ECB result is suboptimal because of penalties
# incurred by operations on %xmm8-15. As ECB is not considered
diff --git a/crypto/chacha/asm/chacha-x86_64.pl
b/crypto/chacha/asm/chacha-x86_64.pl
index 3f1003a..e2c6a32 100755
--- a/crypto/chacha/asm/chacha-x86_64.pl
+++ b/crypto/chacha/asm/chacha-x86_64.pl
@@ -37,7 +37,7 @@
# Goldmont 10.6/+17% 5.10/- 3.28
# Sledgehammer 7.28/+52% -/14.2(ii) -
# Bulldozer 9.66/+28% 9.85/11.1 3.06(iv)
-# Ryzen 5.96 ? 2.40 2.09
+# Ryzen 5.96/+50% 5.19/- 2.40 2.09
# VIA Nano 10.5/+46% 6.72/8.60 6.05
#
# (i) compared to older gcc 3.x one can observe >2x improvement on
diff --git a/crypto/poly1305/asm/poly1305-x86_64.pl
b/crypto/poly1305/asm/poly1305-x86_64.pl
index 242e322..1dce5d6 100755
--- a/crypto/poly1305/asm/poly1305-x86_64.pl
+++ b/crypto/poly1305/asm/poly1305-x86_64.pl
@@ -39,7 +39,7 @@
# VIA Nano 1.82/+150% -
# Sledgehammer 1.38/+160% -
# Bulldozer 2.30/+130% 0.97
-# Ryzen 1.15/? 1.08 1.18
+# Ryzen 1.15/+200% 1.08 1.18
#
# (*) improvement coefficients relative to clang are more modest and
# are ~50% on most processors, in both cases we are comparing to
diff --git a/crypto/sha/asm/keccak1600-s390x.pl
b/crypto/sha/asm/keccak1600-s390x.pl
index b150abe..4bd3619 100755
--- a/crypto/sha/asm/keccak1600-s390x.pl
+++ b/crypto/sha/asm/keccak1600-s390x.pl
@@ -60,7 +60,7 @@ my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
-{ my @C = @C; # copy, because we mess the up...
+{ my @C = @C; # copy, because we mess them up...
my @D = @D;
$code.=<<___;
@@ -75,6 +75,7 @@ __KeccakF1600:
lg @C[2],$A[4][2]($src)
lg @C[3],$A[4][3]($src)
lg @C[4],$A[4][4]($src)
+ larl $iotas,iotas
j .Loop
.align 16
@@ -125,9 +126,7 @@ __KeccakF1600:
rllg @C[4],@C[4],1
xgr @C[4],@T[0] # D[3] = ROL64(C[4], 1) ^ C[2]
___
- my @E = @D;
- @D = (@C[1],@C[2],@C[3],@C[4],@C[0]);
- @C = @E;
+ (@D[0..4], @C) = (@C[1..4,0], @D);
$code.=<<___;
xgr @C[1],@D[1]
xgr @C[2],@D[2]
@@ -298,7 +297,7 @@ $code.=<<___;
xgr $dst,$src
rllg @D[4],@D[4],$rhotates[2][4]
___
- @C = (@D[2],@D[3],@D[4],@D[0],@D[1]);
+ @C = @D[2..4,0,1];
$code.=<<___;
lgr @T[0],@C[0]
ngr @C[0],@C[1]
@@ -359,15 +358,13 @@ KeccakF1600:
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
- stg @D[0],$A[0][1]($src)
- stg @D[1],$A[0][2]($src)
+ stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
stg @T[0],$A[4][0]($src)
la $dst,$stdframe($sp)
- larl $iotas,iotas
bras %r14,__KeccakF1600
@@ -383,8 +380,7 @@ KeccakF1600:
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
- stg @D[0],$A[0][1]($src)
- stg @D[1],$A[0][2]($src)
+ stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
@@ -420,8 +416,7 @@ SHA3_absorb:
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
- stg @D[0],$A[0][1]($src)
- stg @D[1],$A[0][2]($src)
+ stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
@@ -445,7 +440,6 @@ SHA3_absorb:
stm${g} $inp,$len,$frame+3*$SIZE_T($sp)
la $dst,$stdframe($sp)
- larl $iotas,iotas
bras %r14,__KeccakF1600
lm${g} $inp,$bsz,$frame+3*$SIZE_T($sp)
j .Loop_absorb
@@ -464,8 +458,7 @@ SHA3_absorb:
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
- stg @D[0],$A[0][1]($src)
- stg @D[1],$A[0][2]($src)
+ stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
diff --git a/crypto/sha/asm/keccak1600-x86_64.pl
b/crypto/sha/asm/keccak1600-x86_64.pl
index eb12c99..94c9c10 100755
--- a/crypto/sha/asm/keccak1600-x86_64.pl
+++ b/crypto/sha/asm/keccak1600-x86_64.pl
@@ -13,7 +13,7 @@
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
-# Keccak-1600 for x86_86.
+# Keccak-1600 for x86_64.
#
# June 2017.
#
@@ -22,9 +22,8 @@
# instead of actually unrolling the loop pair-wise I simply flip
# pointers to T[][] and A[][] at the end of round. Since number of
# rounds is even, last round writes to A[][] and everything works out.
-# How does it compare to assembly module in Keccak Code Package? KCP
-# is faster on couple of processors, VIA Nano and Goldmont by 4-6%,
-# otherwise this module is either as fast or faster by up to 15%...
+# How does it compare to x86_64 assembly module in Keccak Code Package?
+# Depending on processor it's either as fast or faster by up to 15%...
#
########################################################################
# Numbers are cycles per processed byte out of large message.
@@ -32,16 +31,17 @@
# r=1088(*)
#
# P4 25.8
-# Core 2 13.0
+# Core 2 12.9
# Westmere 13.7
# Sandy Bridge 12.9(**)
-# Haswell 9.7
+# Haswell 9.6
# Skylake 9.4
# Silvermont 22.8
-# Goldmont 16.4
-# VIA Nano 18.0
+# Goldmont 15.8
+# VIA Nano 17.3
# Sledgehammer 13.3
# Bulldozer 16.5
+# Ryzen 8.8
#
# (*) Corresponds to SHA3-256. Improvement over compiler-generate
# varies a lot, most commont coefficient is 15% in comparison to
@@ -138,9 +138,7 @@ __KeccakF1600:
rol \$1,@C[4]
xor @T[0],@C[4] # D[3] = ROL64(C[4], 1) ^ C[2]
___
- my @E = @D;
- @D = (@C[1],@C[2],@C[3],@C[4],@C[0]);
- @C = @E;
+ (@D[0..4], @C) = (@C[1..4,0], @D);
$code.=<<___;
xor @D[1],@C[1]
xor @D[2],@C[2]
@@ -166,24 +164,23 @@ $code.=<<___;
mov @C[4],$A[0][2](%rsi) # R[0][2] = C[2] ^ ( C[4] & C[3])
or @C[3],@C[2]
+ mov $A[4][2](%rdi),@C[4]
xor @T[0],@C[2] # C[1] ^ (~C[2] | C[3])
mov @C[2],$A[0][1](%rsi) # R[0][1] = C[1] ^ (~C[2] | C[3])
and @C[0],@T[0]
+ mov $A[1][4](%rdi),@C[1]
xor @T[1],@T[0] # C[4] ^ ( C[1] & C[0])
+ mov $A[2][0](%rdi),@C[2]
mov @T[0],$A[0][4](%rsi) # R[0][4] = C[4] ^ ( C[1] & C[0])
or @C[0],@T[1]
+ mov $A[0][3](%rdi),@C[0]
xor @C[3],@T[1] # C[3] ^ ( C[4] | C[0])
+ mov $A[3][1](%rdi),@C[3]
mov @T[1],$A[0][3](%rsi) # R[0][3] = C[3] ^ ( C[4] | C[0])
- mov $A[0][3](%rdi),@C[0]
- mov $A[4][2](%rdi),@C[4]
- mov $A[3][1](%rdi),@C[3]
- mov $A[1][4](%rdi),@C[1]
- mov $A[2][0](%rdi),@C[2]
-
xor @D[3],@C[0]
xor @D[2],@C[4]
rol \$$rhotates[0][3],@C[0]
@@ -202,29 +199,28 @@ $code.=<<___;
mov @C[1],@T[1]
and @T[0],@C[1]
+ mov $A[0][1](%rdi),@C[0]
xor @C[4],@C[1] # C[4] ^ (C[1] & C[0])
not @C[4]
mov @C[1],$A[1][4](%rsi) # R[1][4] = C[4] ^ (C[1] & C[0])
or @C[3],@C[4]
+ mov $A[1][2](%rdi),@C[1]
xor @C[2],@C[4] # C[2] ^ (~C[4] | C[3])
mov @C[4],$A[1][2](%rsi) # R[1][2] = C[2] ^ (~C[4] | C[3])
and @C[2],@C[3]
+ mov $A[4][0](%rdi),@C[4]
xor @T[1],@C[3] # C[1] ^ (C[3] & C[2])
mov @C[3],$A[1][1](%rsi) # R[1][1] = C[1] ^ (C[3] & C[2])
or @C[2],@T[1]
+ mov $A[2][3](%rdi),@C[2]
xor @T[0],@T[1] # C[0] ^ (C[1] | C[2])
+ mov $A[3][4](%rdi),@C[3]
mov @T[1],$A[1][0](%rsi) # R[1][0] = C[0] ^ (C[1] | C[2])
- mov $A[2][3](%rdi),@C[2]
- mov $A[3][4](%rdi),@C[3]
- mov $A[1][2](%rdi),@C[1]
- mov $A[4][0](%rdi),@C[4]
- mov $A[0][1](%rdi),@C[0]
-
xor @D[3],@C[2]
xor @D[4],@C[3]
rol \$$rhotates[2][3],@C[2]
@@ -244,10 +240,12 @@ $code.=<<___;
mov @C[4],@T[1]
and @C[3],@C[4]
+ mov $A[2][1](%rdi),@C[2]
xor @T[0],@C[4] # C[2] ^ ( C[4] & ~C[3])
mov @C[4],$A[2][2](%rsi) # R[2][2] = C[2] ^ ( C[4] & ~C[3])
or @C[1],@T[0]
+ mov $A[4][3](%rdi),@C[4]
xor @C[0],@T[0] # C[0] ^ ( C[2] | C[1])
mov @T[0],$A[2][0](%rsi) # R[2][0] = C[0] ^ ( C[2] | C[1])
@@ -255,15 +253,13 @@ $code.=<<___;
xor @T[1],@C[1] # C[4] ^ ( C[1] & C[0])
mov @C[1],$A[2][4](%rsi) # R[2][4] = C[4] ^ ( C[1] & C[0])
- or @T[1],@C[0]
- xor @C[3],@C[0] # ~C[3] ^ ( C[0] | C[4])
- mov @C[0],$A[2][3](%rsi) # R[2][3] = ~C[3] ^ ( C[0] | C[4])
+ or @C[0],@T[1]
+ mov $A[1][0](%rdi),@C[1]
+ xor @C[3],@T[1] # ~C[3] ^ ( C[0] | C[4])
+ mov $A[3][2](%rdi),@C[3]
+ mov @T[1],$A[2][3](%rsi) # R[2][3] = ~C[3] ^ ( C[0] | C[4])
- mov $A[2][1](%rdi),@C[2]
- mov $A[3][2](%rdi),@C[3]
- mov $A[1][0](%rdi),@C[1]
- mov $A[4][3](%rdi),@C[4]
mov $A[0][4](%rdi),@C[0]
xor @D[1],@C[2]
@@ -313,7 +309,7 @@ $code.=<<___;
rol \$$rhotates[2][4],@D[4]
rol \$$rhotates[3][0],@D[0]
___
- @C = (@D[2],@D[3],@D[4],@D[0],@D[1]);
+ @C = @D[2..4,0,1];
$code.=<<___;
mov @C[0],@T[0]
and @C[1],@C[0]
@@ -599,9 +595,12 @@ iotas:
___
foreach (split("\n",$code)) {
- # Below replacement results in 11.3 on Sandy Bridge, 9.4 on
+ # Below replacement results in 11.2 on Sandy Bridge, 9.4 on
# Haswell, but it hurts other processors by up to 2-3-4x...
#s/rol\s+(\$[0-9]+),(%[a-z][a-z0-9]+)/shld\t$1,$2,$2/;
+ # Below replacement results in 9.3 on Haswell [as well as
+ # on Ryzen, i.e. it *hurts* Ryzen]...
+ #s/rol\s+\$([0-9]+),(%[a-z][a-z0-9]+)/rorx\t\$64-$1,$2,$2/;
print $_, "\n";
}
diff --git a/crypto/sha/asm/sha1-x86_64.pl b/crypto/sha/asm/sha1-x86_64.pl
index 8528e54..73ac8ee 100755
--- a/crypto/sha/asm/sha1-x86_64.pl
+++ b/crypto/sha/asm/sha1-x86_64.pl
@@ -82,7 +82,7 @@
# Haswell 5.45 4.15/+31% 3.57/+53%
# Skylake 5.18 4.06/+28% 3.54/+46%
# Bulldozer 9.11 5.95/+53%
-# Ryzen 4.75 ? 1.93/+150%(**)
+# Ryzen 4.75 3.80/+24% 1.93/+150%(**)
# VIA Nano 9.32 7.15/+30%
# Atom 10.3 9.17/+12%
# Silvermont 13.1(*) 9.37/+40%
diff --git a/crypto/sha/asm/sha512-x86_64.pl b/crypto/sha/asm/sha512-x86_64.pl
index d13f3a9..bb7561e 100755
--- a/crypto/sha/asm/sha512-x86_64.pl
+++ b/crypto/sha/asm/sha512-x86_64.pl
@@ -95,7 +95,7 @@
# Haswell 12.2 9.28(+31%) 7.80(+56%) 7.66 5.40(+42%)
# Skylake 11.4 9.03(+26%) 7.70(+48%) 7.25 5.20(+40%)
# Bulldozer 21.1 13.6(+54%) 13.6(+54%(***)) 13.5 8.58(+57%)
-# Ryzen 11.0 ? 2.05(+440%) 7.05 5.67(+20%)
+# Ryzen 11.0 9.02(+22%) 2.05(+440%) 7.05 5.67(+20%)
# VIA Nano 23.0 16.5(+39%) - 14.7 -
# Atom 23.0 18.9(+22%) - 14.7 -
# Silvermont 27.4 20.6(+33%) - 17.5 -
diff --git a/crypto/sha/keccak1600.c b/crypto/sha/keccak1600.c
index b0ee159..0ea9818 100644
--- a/crypto/sha/keccak1600.c
+++ b/crypto/sha/keccak1600.c
@@ -212,7 +212,7 @@ static void Iota(uint64_t A[5][5], size_t i)
A[0][0] ^= iotas[i];
}
-void KeccakF1600(uint64_t A[5][5])
+static void KeccakF1600(uint64_t A[5][5])
{
size_t i;
@@ -347,7 +347,7 @@ static void Round(uint64_t A[5][5], size_t i)
A[4][4] = C[4] ^ (~C[0] & C[1]);
}
-void KeccakF1600(uint64_t A[5][5])
+static void KeccakF1600(uint64_t A[5][5])
{
size_t i;
@@ -490,7 +490,7 @@ static void Round(uint64_t A[5][5], size_t i)
A[0][0] ^= iotas[i];
}
-void KeccakF1600(uint64_t A[5][5])
+static void KeccakF1600(uint64_t A[5][5])
{
size_t i;
@@ -628,7 +628,7 @@ static void Round(uint64_t R[5][5], uint64_t A[5][5],
size_t i)
#endif
}
-void KeccakF1600(uint64_t A[5][5])
+static void KeccakF1600(uint64_t A[5][5])
{
uint64_t T[5][5];
size_t i;
@@ -946,7 +946,7 @@ static void FourRounds(uint64_t A[5][5], size_t i)
/* C[4] ^= */ A[4][4] = B[4] ^ (~B[0] & B[1]);
}
-void KeccakF1600(uint64_t A[5][5])
+static void KeccakF1600(uint64_t A[5][5])
{
size_t i;
@@ -1071,14 +1071,22 @@ size_t SHA3_absorb(uint64_t A[5][5], const unsigned
char *inp, size_t len,
void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r)
{
uint64_t *A_flat = (uint64_t *)A;
- size_t i, rem, w = r / 8;
+ size_t i, w = r / 8;
assert(r < (25 * sizeof(A[0][0])) && (r % 8) == 0);
- while (len >= r) {
- for (i = 0; i < w; i++) {
+ while (len != 0) {
+ for (i = 0; i < w && len != 0; i++) {
uint64_t Ai = BitDeinterleave(A_flat[i]);
+ if (len < 8) {
+ for (i = 0; i < len; i++) {
+ *out++ = (unsigned char)Ai;
+ Ai >>= 8;
+ }
+ return;
+ }
+
out[0] = (unsigned char)(Ai);
out[1] = (unsigned char)(Ai >> 8);
out[2] = (unsigned char)(Ai >> 16);
@@ -1088,37 +1096,11 @@ void SHA3_squeeze(uint64_t A[5][5], unsigned char *out,
size_t len, size_t r)
out[6] = (unsigned char)(Ai >> 48);
out[7] = (unsigned char)(Ai >> 56);
out += 8;
+ len -= 8;
}
- len -= r;
if (len)
KeccakF1600(A);
}
-
- rem = len % 8;
- len /= 8;
-
- for (i = 0; i < len; i++) {
- uint64_t Ai = BitDeinterleave(A_flat[i]);
-
- out[0] = (unsigned char)(Ai);
- out[1] = (unsigned char)(Ai >> 8);
- out[2] = (unsigned char)(Ai >> 16);
- out[3] = (unsigned char)(Ai >> 24);
- out[4] = (unsigned char)(Ai >> 32);
- out[5] = (unsigned char)(Ai >> 40);
- out[6] = (unsigned char)(Ai >> 48);
- out[7] = (unsigned char)(Ai >> 56);
- out += 8;
- }
-
- if (rem) {
- uint64_t Ai = BitDeinterleave(A_flat[i]);
-
- for (i = 0; i < rem; i++) {
- *out++ = (unsigned char)Ai;
- Ai >>= 8;
- }
- }
}
#else
size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len,
_____
openssl-commits mailing list
To unsubscribe: https://mta.openssl.org/mailman/listinfo/openssl-commits
