From: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
Built tested. Please double check it, especially that part where
required_pkey_type vanished.
This patch / contribution is MIT licensed.
Applies as of r2830.
Signed-off-by: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
---
Reposted, been told that I need to either sign the copyright agreement
or post under the MIT license.
src/VBox/RDP/client-1.8.3/ssl.c | 70 +++++++++++++++++---------
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++++++++++------
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32
modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len,
uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid ==
NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server
certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm =
OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x10100000
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+ const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32
max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int
msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
- HMAC_CTX_cleanup(&ctx);
}
diff --git a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
index ddfccec..fa19fa7 100644
--- a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
+++ b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
@@ -76,26 +76,28 @@ PFNRT g_VBoxRTDeps[] =
(PFNRT)i2d_X509,
(PFNRT)i2d_X509,
(PFNRT)i2d_PublicKey,
+#if OPENSSL_VERSION_NUMBER < 0x10100000
(PFNRT)RSA_generate_key,
- (PFNRT)RSA_generate_key_ex,
(PFNRT)DH_generate_parameters,
- (PFNRT)DH_generate_parameters_ex,
- (PFNRT)RAND_load_file,
(PFNRT)CRYPTO_set_dynlock_create_callback,
(PFNRT)CRYPTO_set_dynlock_lock_callback,
(PFNRT)CRYPTO_set_dynlock_destroy_callback,
+ (PFNRT)SSL_library_init,
+ (PFNRT)SSL_load_error_strings,
+ (PFNRT)TLSv1_server_method,
+#endif
+ (PFNRT)RSA_generate_key_ex,
+ (PFNRT)DH_generate_parameters_ex,
+ (PFNRT)RAND_load_file,
(PFNRT)RTAssertShouldPanic,
(PFNRT)ASMAtomicReadU64,
(PFNRT)ASMAtomicCmpXchgU64,
(PFNRT)ASMBitFirstSet,
(PFNRT)RTBldCfgRevision,
(PFNRT)SSL_free,
- (PFNRT)SSL_library_init,
- (PFNRT)SSL_load_error_strings,
(PFNRT)SSL_CTX_free,
(PFNRT)SSL_CTX_use_certificate_file,
(PFNRT)SSLv23_method,
- (PFNRT)TLSv1_server_method,
NULL
};
diff --git a/src/VBox/Runtime/common/crypto/pkix-verify.cpp
b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
index ba537d5..924005b 100644
--- a/src/VBox/Runtime/common/crypto/pkix-verify.cpp
+++ b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
@@ -124,28 +124,39 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID
pAlgorithm, PCRTASN1DYNT
"EVP_get_digestbyname failed on %s (%s)",
pszAlogSn, pAlgorithm->szObjId);
/* Initialize the EVP message digest context. */
- EVP_MD_CTX EvpMdCtx;
- EVP_MD_CTX_init(&EvpMdCtx);
- if (!EVP_VerifyInit_ex(&EvpMdCtx, pEvpMdType, NULL /*engine*/))
+ EVP_MD_CTX *EvpMdCtx;
+
+ EvpMdCtx = EVP_MD_CTX_create();
+
+ if (!EvpMdCtx)
+ return RTErrInfoSetF(pErrInfo,
VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
+ "EVP_MD_CTX_create failed");
+
+ if (!EVP_VerifyInit_ex(EvpMdCtx, pEvpMdType, NULL /*engine*/)) {
+ EVP_MD_CTX_destroy(EvpMdCtx);
return RTErrInfoSetF(pErrInfo,
VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
"EVP_VerifyInit_ex failed (algorithm type is %s /
%s)", pszAlogSn, pAlgorithm->szObjId);
+ }
/* Create an EVP public key. */
int rcOssl;
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey =
RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey,
&puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey,
RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Digest the data. */
- EVP_VerifyUpdate(&EvpMdCtx, pvData, cbData);
+ EVP_VerifyUpdate(EvpMdCtx, pvData, cbData);
/* Verify the signature. */
- if (EVP_VerifyFinal(&EvpMdCtx,
+ if (EVP_VerifyFinal(EvpMdCtx,
RTASN1BITSTRING_GET_BIT0_PTR(pSignatureValue),
RTASN1BITSTRING_GET_BYTE_SIZE(pSignatureValue),
pEvpPublicKey) > 0)
@@ -158,13 +169,13 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID
pAlgorithm, PCRTASN1DYNT
}
else
rcOssl = RTErrInfoSetF(pErrInfo,
VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed",
pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d)
failed", pEvpMdType->required_pkey_type[0]);
- EVP_MD_CTX_cleanup(&EvpMdCtx);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d)
failed", iAlgoNid);
+ EVP_MD_CTX_destroy(EvpMdCtx);
/*
* Check the result.
@@ -261,11 +272,14 @@ RTDECL(int)
RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey =
RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey,
&puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey,
RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Create an EVP public key context we can use to validate the
digest. */
EVP_PKEY_CTX *pEvpPKeyCtx = EVP_PKEY_CTX_new(pEvpPublicKey,
NULL);
@@ -306,12 +320,12 @@ RTDECL(int)
RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
}
else
rcOssl = RTErrInfoSetF(pErrInfo,
VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed",
pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d)
failed", pEvpMdType->required_pkey_type[0]);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d)
failed", iAlgoNid);
/*
* Check the result.
--
2.9.3
_______________________________________________
vbox-dev mailing list
vbox-dev@virtualbox.org
https://www.virtualbox.org/mailman/listinfo/vbox-dev
