Author: mav
Date: Wed Sep 11 23:45:58 2019
New Revision: 352235
URL: https://svnweb.freebsd.org/changeset/base/352235
Log:
MFC r348268 (by sef), r348293 (by cem):
Add an AESNI-optimized version of the CCM/CBC cryptographic and authentication
code. The primary client of this is probably going to be ZFS encryption.
Added:
stable/12/sys/crypto/aesni/aesni_ccm.c
- copied, changed from r348268, head/sys/crypto/aesni/aesni_ccm.c
Modified:
stable/12/sys/conf/files.amd64
stable/12/sys/conf/files.i386
stable/12/sys/crypto/aesni/aesni.c
stable/12/sys/crypto/aesni/aesni.h
stable/12/sys/crypto/aesni/aesni_wrap.c
stable/12/sys/modules/aesni/Makefile
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/conf/files.amd64
==============================================================================
--- stable/12/sys/conf/files.amd64 Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/conf/files.amd64 Wed Sep 11 23:45:58 2019
(r352235)
@@ -175,6 +175,11 @@ aesni_ghash.o optional aesni
\
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \
no-implicit-rule \
clean "aesni_ghash.o"
+aesni_ccm.o optional aesni \
+ dependency "$S/crypto/aesni/aesni_ccm.c" \
+ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aesni_ccm.o"
aesni_wrap.o optional aesni \
dependency "$S/crypto/aesni/aesni_wrap.c" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \
Modified: stable/12/sys/conf/files.i386
==============================================================================
--- stable/12/sys/conf/files.i386 Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/conf/files.i386 Wed Sep 11 23:45:58 2019
(r352235)
@@ -127,6 +127,11 @@ aesni_ghash.o optional aesni
\
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \
no-implicit-rule \
clean "aesni_ghash.o"
+aesni_ccm.o optional aesni \
+ dependency "$S/crypto/aesni/aesni_ccm.c" \
+ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "aesni_ccm.o"
aesni_wrap.o optional aesni \
dependency "$S/crypto/aesni/aesni_wrap.c" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR}
${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \
Modified: stable/12/sys/crypto/aesni/aesni.c
==============================================================================
--- stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/crypto/aesni/aesni.c Wed Sep 11 23:45:58 2019
(r352235)
@@ -131,9 +131,10 @@ aesni_probe(device_t dev)
return (EINVAL);
} else if (has_aes && has_sha)
device_set_desc(dev,
- "AES-CBC,AES-XTS,AES-GCM,AES-ICM,SHA1,SHA256");
+ "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256");
else if (has_aes)
- device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM");
+ device_set_desc(dev,
+ "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS");
else
device_set_desc(dev, "SHA1,SHA256");
@@ -193,6 +194,8 @@ aesni_attach(device_t dev)
crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0);
+ crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
}
if (sc->has_sha) {
crypto_register(sc->cid, CRYPTO_SHA1, 0, 0);
@@ -226,6 +229,7 @@ aesni_newsession(device_t dev, crypto_session_t cses,
struct aesni_session *ses;
struct cryptoini *encini, *authini;
bool gcm_hash, gcm;
+ bool cbc_hash, ccm;
int error;
KASSERT(cses != NULL, ("EDOOFUS"));
@@ -242,10 +246,17 @@ aesni_newsession(device_t dev, crypto_session_t cses,
encini = NULL;
gcm = false;
gcm_hash = false;
+ ccm = cbc_hash = false;
+
for (; cri != NULL; cri = cri->cri_next) {
switch (cri->cri_alg) {
case CRYPTO_AES_NIST_GCM_16:
- gcm = true;
+ case CRYPTO_AES_CCM_16:
+ if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) {
+ gcm = true;
+ } else if (cri->cri_alg == CRYPTO_AES_CCM_16) {
+ ccm = true;
+ }
/* FALLTHROUGH */
case CRYPTO_AES_CBC:
case CRYPTO_AES_ICM:
@@ -258,6 +269,10 @@ aesni_newsession(device_t dev, crypto_session_t cses,
}
encini = cri;
break;
+ case CRYPTO_AES_CCM_CBC_MAC:
+ cbc_hash = true;
+ authini = cri;
+ break;
case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:
@@ -265,7 +280,12 @@ aesni_newsession(device_t dev, crypto_session_t cses,
* nothing to do here, maybe in the future cache some
* values for GHASH
*/
+ if (authini != NULL) {
+ CRYPTDEB("authini already set");
+ return (EINVAL);
+ }
gcm_hash = true;
+ authini = cri;
break;
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
@@ -295,9 +315,16 @@ unhandled:
* GMAC algorithms are only supported with simultaneous GCM. Likewise
* GCM is not supported without GMAC.
*/
- if (gcm_hash != gcm)
+ if (gcm_hash != gcm) {
+ CRYPTDEB("gcm_hash != gcm");
return (EINVAL);
+ }
+ if (cbc_hash != ccm) {
+ CRYPTDEB("cbc_hash != ccm");
+ return (EINVAL);
+ }
+
if (encini != NULL)
ses->algo = encini->cri_alg;
if (authini != NULL)
@@ -338,6 +365,7 @@ aesni_process(device_t dev, struct cryptop *crp, int h
for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
switch (crd->crd_alg) {
case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
needauth = 1;
/* FALLTHROUGH */
case CRYPTO_AES_CBC:
@@ -353,6 +381,7 @@ aesni_process(device_t dev, struct cryptop *crp, int h
case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:
+ case CRYPTO_AES_CCM_CBC_MAC:
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_224:
@@ -647,6 +676,7 @@ aesni_cipher_process(struct aesni_session *ses, struct
if (enccrd != NULL) {
if ((enccrd->crd_alg == CRYPTO_AES_ICM ||
+ enccrd->crd_alg == CRYPTO_AES_CCM_16 ||
enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) &&
(enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
return (EINVAL);
@@ -700,8 +730,9 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
int error, ivlen;
bool encflag, allocated, authallocated;
- KASSERT(ses->algo != CRYPTO_AES_NIST_GCM_16 || authcrd != NULL,
- ("AES_NIST_GCM_16 must include MAC descriptor"));
+ KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 &&
+ ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL,
+ ("AES_NIST_GCM_16/AES_CCM_16 must include MAC descriptor"));
ivlen = 0;
authbuf = NULL;
@@ -711,7 +742,8 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
return (ENOMEM);
authallocated = false;
- if (ses->algo == CRYPTO_AES_NIST_GCM_16) {
+ if (ses->algo == CRYPTO_AES_NIST_GCM_16 ||
+ ses->algo == CRYPTO_AES_CCM_16) {
authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
if (authbuf == NULL) {
error = ENOMEM;
@@ -737,6 +769,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
ivlen = 8;
break;
case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
ivlen = 12; /* should support arbitarily larger */
break;
}
@@ -786,7 +819,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
case CRYPTO_AES_NIST_GCM_16:
if (!encflag)
crypto_copydata(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
+ authcrd->crd_inject, sizeof(tag), tag);
else
bzero(tag, sizeof tag);
@@ -797,7 +830,7 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
if (authcrd != NULL)
crypto_copyback(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
+ authcrd->crd_inject, sizeof(tag), tag);
} else {
if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag,
enccrd->crd_len, authcrd->crd_len, ivlen,
@@ -805,9 +838,28 @@ aesni_cipher_crypt(struct aesni_session *ses, struct c
error = EBADMSG;
}
break;
+ case CRYPTO_AES_CCM_16:
+ if (!encflag)
+ crypto_copydata(crp->crp_flags, crp->crp_buf,
+ authcrd->crd_inject, sizeof(tag), tag);
+ else
+ bzero(tag, sizeof tag);
+ if (encflag) {
+ AES_CCM_encrypt(buf, buf, authbuf, iv, tag,
+ enccrd->crd_len, authcrd->crd_len, ivlen,
+ ses->enc_schedule, ses->rounds);
+ if (authcrd != NULL)
+ crypto_copyback(crp->crp_flags, crp->crp_buf,
+ authcrd->crd_inject, sizeof(tag), tag);
+ } else {
+ if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag,
+ enccrd->crd_len, authcrd->crd_len, ivlen,
+ ses->enc_schedule, ses->rounds))
+ error = EBADMSG;
+ }
+ break;
}
-
- if (allocated)
+ if (allocated && error == 0)
crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
enccrd->crd_len, buf);
Modified: stable/12/sys/crypto/aesni/aesni.h
==============================================================================
--- stable/12/sys/crypto/aesni/aesni.h Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/crypto/aesni/aesni.h Wed Sep 11 23:45:58 2019
(r352235)
@@ -111,6 +111,15 @@ int AES_GCM_decrypt(const unsigned char *in, unsigned
const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
const unsigned char *key, int nr);
+/* CCM + CBC-MAC functions */
+void AES_CCM_encrypt(const unsigned char *in, unsigned char *out,
+ const unsigned char *addt, const unsigned char *ivec,
+ unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
+ const unsigned char *key, int nr);
+int AES_CCM_decrypt(const unsigned char *in, unsigned char *out,
+ const unsigned char *addt, const unsigned char *ivec,
+ const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
+ const unsigned char *key, int nr);
int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
int keylen);
Copied and modified: stable/12/sys/crypto/aesni/aesni_ccm.c (from r348268,
head/sys/crypto/aesni/aesni_ccm.c)
==============================================================================
--- head/sys/crypto/aesni/aesni_ccm.c Sat May 25 07:26:30 2019
(r348268, copy source)
+++ stable/12/sys/crypto/aesni/aesni_ccm.c Wed Sep 11 23:45:58 2019
(r352235)
@@ -58,7 +58,7 @@
static inline __m128i
xor_and_encrypt(__m128i a, __m128i b, const unsigned char *k, int nr)
{
- __m128 retval = _mm_xor_si128(a, b);
+ __m128i retval = _mm_xor_si128(a, b);
retval = AESNI_ENC(retval, k, nr);
return (retval);
Modified: stable/12/sys/crypto/aesni/aesni_wrap.c
==============================================================================
--- stable/12/sys/crypto/aesni/aesni_wrap.c Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/crypto/aesni/aesni_wrap.c Wed Sep 11 23:45:58 2019
(r352235)
@@ -446,6 +446,7 @@ aesni_cipher_setup_common(struct aesni_session *ses, c
switch (ses->algo) {
case CRYPTO_AES_ICM:
case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
decsched = 0;
/* FALLTHROUGH */
case CRYPTO_AES_CBC:
Modified: stable/12/sys/modules/aesni/Makefile
==============================================================================
--- stable/12/sys/modules/aesni/Makefile Wed Sep 11 23:41:09 2019
(r352234)
+++ stable/12/sys/modules/aesni/Makefile Wed Sep 11 23:45:58 2019
(r352235)
@@ -8,16 +8,20 @@ SRCS= aesni.c
SRCS+= aeskeys_${MACHINE_CPUARCH}.S
SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h
-OBJS+= aesni_ghash.o aesni_wrap.o
+OBJS+= aesni_ghash.o aesni_wrap.o aesni_ccm.o
OBJS+= intel_sha1.o intel_sha256.o
# Remove -nostdinc so we can get the intrinsics.
aesni_ghash.o: aesni_ghash.c
- # XXX - gcc won't understand -mpclmul
${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \
-mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}
${CTFCONVERT_CMD}
+aesni_ccm.o: aesni_ccm.c
+ ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \
+ -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}
+ ${CTFCONVERT_CMD}
+
aesni_wrap.o: aesni_wrap.c
${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \
-mmmx -msse -msse4 -maes ${.IMPSRC}
@@ -35,6 +39,7 @@ intel_sha256.o: intel_sha256.c
aesni_ghash.o: aesni.h
aesni_wrap.o: aesni.h
+aesni_ccm.o: aesni.h
intel_sha1.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h
intel_sha256.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
