On 2016-06-26 12:24:40 [+0200], Kurt Roeckx wrote:
> If you have problems making things work, feel free to contact us.
HALP!
It builds against old & new ssl and I am proud what I managed in
rdssl_cert_to_rkey(). However they dereference
EVP_MD->required_pkey_type which vanished in 1.1 and I have no idea what
it was and its purpose was. They don't even assign anything to it in
1.0.2h.
> Kurt
Sebastian
>From 0e3d6cc69959fdf4f4fd56f65353b76897c0daad Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
Date: Wed, 7 Sep 2016 20:22:54 +0000
Subject: [PATCH] virtualbox 5.1.4: get it built against openssl-1.1
Signed-off-by: Sebastian Andrzej Siewior <sebast...@breakpoint.cc>
---
src/VBox/RDP/client-1.8.3/ssl.c | 70 +++++++++++++++++---------
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++++++++++------
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x10100000
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+ const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
- HMAC_CTX_cleanup(&ctx);
}
diff --git a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
index ddfccec..fa19fa7 100644
--- a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
+++ b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
@@ -76,26 +76,28 @@ PFNRT g_VBoxRTDeps[] =
(PFNRT)i2d_X509,
(PFNRT)i2d_X509,
(PFNRT)i2d_PublicKey,
+#if OPENSSL_VERSION_NUMBER < 0x10100000
(PFNRT)RSA_generate_key,
- (PFNRT)RSA_generate_key_ex,
(PFNRT)DH_generate_parameters,
- (PFNRT)DH_generate_parameters_ex,
- (PFNRT)RAND_load_file,
(PFNRT)CRYPTO_set_dynlock_create_callback,
(PFNRT)CRYPTO_set_dynlock_lock_callback,
(PFNRT)CRYPTO_set_dynlock_destroy_callback,
+ (PFNRT)SSL_library_init,
+ (PFNRT)SSL_load_error_strings,
+ (PFNRT)TLSv1_server_method,
+#endif
+ (PFNRT)RSA_generate_key_ex,
+ (PFNRT)DH_generate_parameters_ex,
+ (PFNRT)RAND_load_file,
(PFNRT)RTAssertShouldPanic,
(PFNRT)ASMAtomicReadU64,
(PFNRT)ASMAtomicCmpXchgU64,
(PFNRT)ASMBitFirstSet,
(PFNRT)RTBldCfgRevision,
(PFNRT)SSL_free,
- (PFNRT)SSL_library_init,
- (PFNRT)SSL_load_error_strings,
(PFNRT)SSL_CTX_free,
(PFNRT)SSL_CTX_use_certificate_file,
(PFNRT)SSLv23_method,
- (PFNRT)TLSv1_server_method,
NULL
};
diff --git a/src/VBox/Runtime/common/crypto/pkix-verify.cpp b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
index ba537d5..924005b 100644
--- a/src/VBox/Runtime/common/crypto/pkix-verify.cpp
+++ b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
@@ -124,28 +124,39 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
"EVP_get_digestbyname failed on %s (%s)", pszAlogSn, pAlgorithm->szObjId);
/* Initialize the EVP message digest context. */
- EVP_MD_CTX EvpMdCtx;
- EVP_MD_CTX_init(&EvpMdCtx);
- if (!EVP_VerifyInit_ex(&EvpMdCtx, pEvpMdType, NULL /*engine*/))
+ EVP_MD_CTX *EvpMdCtx;
+
+ EvpMdCtx = EVP_MD_CTX_create();
+
+ if (!EvpMdCtx)
+ return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
+ "EVP_MD_CTX_create failed");
+
+ if (!EVP_VerifyInit_ex(EvpMdCtx, pEvpMdType, NULL /*engine*/)) {
+ EVP_MD_CTX_destroy(EvpMdCtx);
return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
"EVP_VerifyInit_ex failed (algorithm type is %s / %s)", pszAlogSn, pAlgorithm->szObjId);
+ }
/* Create an EVP public key. */
int rcOssl;
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, pEvpMdType->required_pkey_type[0]);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Digest the data. */
- EVP_VerifyUpdate(&EvpMdCtx, pvData, cbData);
+ EVP_VerifyUpdate(EvpMdCtx, pvData, cbData);
/* Verify the signature. */
- if (EVP_VerifyFinal(&EvpMdCtx,
+ if (EVP_VerifyFinal(EvpMdCtx,
RTASN1BITSTRING_GET_BIT0_PTR(pSignatureValue),
RTASN1BITSTRING_GET_BYTE_SIZE(pSignatureValue),
pEvpPublicKey) > 0)
@@ -158,13 +169,13 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
- EVP_MD_CTX_cleanup(&EvpMdCtx);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
+ EVP_MD_CTX_destroy(EvpMdCtx);
/*
* Check the result.
@@ -261,11 +272,14 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, pEvpMdType->required_pkey_type[0]);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Create an EVP public key context we can use to validate the digest. */
EVP_PKEY_CTX *pEvpPKeyCtx = EVP_PKEY_CTX_new(pEvpPublicKey, NULL);
@@ -306,12 +320,12 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
/*
* Check the result.
--
2.9.3
