http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/cert.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/cert.h b/be/src/kudu/security/cert.h new file mode 100644 index 0000000..4629883 --- /dev/null +++ b/be/src/kudu/security/cert.h @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <memory> +#include <string> +#include <vector> + +#include <openssl/asn1.h> + +#include "kudu/gutil/port.h" +#include "kudu/security/openssl_util.h" + +typedef struct X509_name_st X509_NAME; + +namespace boost { +template <class T> +class optional; +} + +namespace kudu { + +class Status; + +namespace security { + +class PrivateKey; +class PublicKey; + +// Convert an X509_NAME object to a human-readable string. +std::string X509NameToString(X509_NAME* name); + +// Return the OpenSSL NID for the custom X509 extension where we store +// our Kerberos principal in IPKI certs. +int GetKuduKerberosPrincipalOidNid(); + +// A wrapper class around the STACK_OF(X509) object. This can either hold one certificate or +// a chain of certificates. +// TODO(unknown): Currently, there isn't a mechanism to add to the chain. Implement it when needed. +class Cert : public RawDataWrapper<STACK_OF(X509)> { + public: + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT; + + int chain_len() const { return sk_X509_num(data_.get()); } + + std::string SubjectName() const; + std::string IssuerName() const; + + // Return DNS names from the SAN extension field of the end-user cert. + std::vector<std::string> Hostnames() const; + + // Return the 'userId' extension of the end-user cert, if set. + boost::optional<std::string> UserId() const; + + // Return the Kerberos principal encoded in the end-user certificate, if set. + boost::optional<std::string> KuduKerberosPrincipal() const; + + // Check whether the specified private key matches the end-user certificate. + // Return Status::OK() if key match the end-user certificate. + Status CheckKeyMatch(const PrivateKey& key) const WARN_UNUSED_RESULT; + + // Returns the 'tls-server-end-point' channel bindings for the end-user certificate as + // specified in RFC 5929. + Status GetServerEndPointChannelBindings(std::string* channel_bindings) const WARN_UNUSED_RESULT; + + // Adopts the provided STACK_OF(X509), and increments the reference count of the X509 cert + // contained within it. Currently, only one certificate should be contained in the stack. + void AdoptAndAddRefRawData(RawDataType* data); + + // Adopts the provided X509 certificate, and replaces the current underlying STACK_OF(X509). + void AdoptX509(X509* cert); + + // Adopts the provided X509 certificate, increments its reference count and replaces the current + // underlying STACK_OF(X509). + void AdoptAndAddRefX509(X509* cert); + + // Returns the end-user certificate's public key. + Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT; + + // Get the first certificate in the chain, otherwise known as the 'end-user' certificate. + X509* GetTopOfChainX509() const; +}; + +class CertSignRequest : public RawDataWrapper<X509_REQ> { + public: + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT; + + // Returns a clone of the CSR. + // + // Whether this clone is deep or shallow (i.e. only a reference count is + // incremented) depends on the version of OpenSSL. Either way, the right + // thing happens when the clone goes out of scope. + CertSignRequest Clone() const; + + // Returns the CSR's public key. + Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT; +}; + +} // namespace security +} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/crypto-test.cc b/be/src/kudu/security/crypto-test.cc new file mode 100644 index 0000000..c1e32df --- /dev/null +++ b/be/src/kudu/security/crypto-test.cc @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <cstring> +#include <string> +#include <utility> +#include <vector> + +#include <gtest/gtest.h> + +#include "kudu/gutil/strings/strip.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/security/crypto.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/test/test_certs.h" +#include "kudu/util/env.h" +#include "kudu/util/path_util.h" +#include "kudu/util/slice.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" +#include "kudu/util/url-coding.h" + +using std::pair; +using std::string; +using std::vector; +using strings::Substitute; + +namespace kudu { +namespace security { + +// Test for various crypto-related functionality in the security library. +class CryptoTest : public KuduTest { + public: + CryptoTest() : + pem_dir_(GetTestPath("pem")), + private_key_file_(JoinPathSegments(pem_dir_, "private_key.pem")), + public_key_file_(JoinPathSegments(pem_dir_, "public_key.pem")), + corrupted_private_key_file_(JoinPathSegments(pem_dir_, + "corrupted.private_key.pem")), + corrupted_public_key_file_(JoinPathSegments(pem_dir_, + "corrupted.public_key.pem")) { + } + + void SetUp() override { + ASSERT_OK(env_->CreateDir(pem_dir_)); + ASSERT_OK(WriteStringToFile(env_, kCaPrivateKey, private_key_file_)); + ASSERT_OK(WriteStringToFile(env_, kCaPublicKey, public_key_file_)); + ASSERT_OK(WriteStringToFile(env_, + string(kCaPrivateKey, strlen(kCaPrivateKey) / 2), + corrupted_private_key_file_)); + ASSERT_OK(WriteStringToFile(env_, + string(kCaPublicKey, strlen(kCaPublicKey) / 2), + corrupted_public_key_file_)); + } + + protected: + template<typename Key> + void CheckToAndFromString(const Key& key_ref, DataFormat format) { + SCOPED_TRACE(Substitute("ToAndFromString for $0 format", + DataFormatToString(format))); + string key_ref_str; + ASSERT_OK(key_ref.ToString(&key_ref_str, format)); + Key key; + ASSERT_OK(key.FromString(key_ref_str, format)); + string key_str; + ASSERT_OK(key.ToString(&key_str, format)); + ASSERT_EQ(key_ref_str, key_str); + } + + const string pem_dir_; + + const string private_key_file_; + const string public_key_file_; + const string corrupted_private_key_file_; + const string corrupted_public_key_file_; +}; + +// Check input/output of RSA private keys in PEM format. +TEST_F(CryptoTest, RsaPrivateKeyInputOutputPEM) { + PrivateKey key; + ASSERT_OK(key.FromFile(private_key_file_, DataFormat::PEM)); + string key_str; + ASSERT_OK(key.ToString(&key_str, DataFormat::PEM)); + RemoveExtraWhitespace(&key_str); + + string ref_key_str(kCaPrivateKey); + RemoveExtraWhitespace(&ref_key_str); + EXPECT_EQ(ref_key_str, key_str); +} + +// Check input of corrupted RSA private keys in PEM format. +TEST_F(CryptoTest, CorruptedRsaPrivateKeyInputPEM) { + static const string kFiles[] = { + corrupted_private_key_file_, + public_key_file_, + corrupted_public_key_file_, + "/bin/sh" + }; + for (const auto& file : kFiles) { + PrivateKey key; + const Status s = key.FromFile(file, DataFormat::PEM); + EXPECT_TRUE(s.IsRuntimeError()) << s.ToString(); + } +} + +// Check input/output of RSA public keys in PEM format. +TEST_F(CryptoTest, RsaPublicKeyInputOutputPEM) { + PublicKey key; + ASSERT_OK(key.FromFile(public_key_file_, DataFormat::PEM)); + string key_str; + ASSERT_OK(key.ToString(&key_str, DataFormat::PEM)); + RemoveExtraWhitespace(&key_str); + + string ref_key_str(kCaPublicKey); + RemoveExtraWhitespace(&ref_key_str); + EXPECT_EQ(ref_key_str, key_str); +} + +// Check input of corrupted RSA public keys in PEM format. +TEST_F(CryptoTest, CorruptedRsaPublicKeyInputPEM) { + static const string kFiles[] = { + corrupted_public_key_file_, + private_key_file_, + corrupted_private_key_file_, + "/bin/sh" + }; + for (const auto& file : kFiles) { + PublicKey key; + const Status s = key.FromFile(file, DataFormat::PEM); + EXPECT_TRUE(s.IsRuntimeError()) << s.ToString(); + } +} + +// Check extraction of the public part from RSA private keys par. +TEST_F(CryptoTest, RsaExtractPublicPartFromPrivateKey) { + // Load the reference RSA private key. + PrivateKey private_key; + ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM)); + + PublicKey public_key; + ASSERT_OK(private_key.GetPublicKey(&public_key)); + string str_public_key; + ASSERT_OK(public_key.ToString(&str_public_key, DataFormat::PEM)); + RemoveExtraWhitespace(&str_public_key); + + string ref_str_public_key(kCaPublicKey); + RemoveExtraWhitespace(&ref_str_public_key); + EXPECT_EQ(ref_str_public_key, str_public_key); +} + +class CryptoKeySerDesTest : + public CryptoTest, + public ::testing::WithParamInterface<DataFormat> { +}; + +// Check the transformation chains for RSA public/private keys: +// internal -> PEM -> internal -> PEM +// internal -> DER -> internal -> DER +TEST_P(CryptoKeySerDesTest, ToAndFromString) { + const auto format = GetParam(); + + // Generate private RSA key. + PrivateKey private_key; + ASSERT_OK(GeneratePrivateKey(2048, &private_key)); + NO_FATALS(CheckToAndFromString(private_key, format)); + + // Extract public part of the key. + PublicKey public_key; + ASSERT_OK(private_key.GetPublicKey(&public_key)); + NO_FATALS(CheckToAndFromString(public_key, format)); +} + +INSTANTIATE_TEST_CASE_P( + DataFormats, CryptoKeySerDesTest, + ::testing::Values(DataFormat::DER, DataFormat::PEM)); + +// Check making crypto signatures against the reference data. +TEST_F(CryptoTest, MakeVerifySignatureRef) { + static const vector<pair<string, string>> kRefElements = { + { kDataTiny, kSignatureTinySHA512 }, + { kDataShort, kSignatureShortSHA512 }, + { kDataLong, kSignatureLongSHA512 }, + }; + + // Load the reference RSA private key. + PrivateKey private_key; + ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM)); + + // Load the reference RSA public key. + PublicKey public_key; + ASSERT_OK(public_key.FromString(kCaPublicKey, DataFormat::PEM)); + + for (const auto& e : kRefElements) { + string sig; + ASSERT_OK(private_key.MakeSignature(DigestType::SHA512, e.first, &sig)); + + // Ad-hoc verification: check the produced signature matches the reference. + string sig_base64; + Base64Encode(sig, &sig_base64); + EXPECT_EQ(e.second, sig_base64); + + // Verify the signature cryptographically. + EXPECT_OK(public_key.VerifySignature(DigestType::SHA512, e.first, sig)); + } +} + +TEST_F(CryptoTest, VerifySignatureWrongData) { + static const vector<string> kRefSignatures = { + kSignatureTinySHA512, + kSignatureShortSHA512, + kSignatureLongSHA512, + }; + + // Load the reference RSA public key. + PublicKey key; + ASSERT_OK(key.FromString(kCaPublicKey, DataFormat::PEM)); + + for (const auto& e : kRefSignatures) { + string signature; + ASSERT_TRUE(Base64Decode(e, &signature)); + Status s = key.VerifySignature(DigestType::SHA512, + "non-expected-data", signature); + EXPECT_TRUE(s.IsCorruption()) << s.ToString(); + } +} + +TEST_F(CryptoTest, TestGenerateNonce) { + string nonce; + ASSERT_OK(GenerateNonce(&nonce)); + + // Do some basic validation on the returned nonce. + ASSERT_EQ(kNonceSize, nonce.size()); + ASSERT_NE(string(kNonceSize, '\0'), nonce); + + // Nonces should be unique, by definition. + string another_nonce; + ASSERT_OK(GenerateNonce(&another_nonce)); + ASSERT_NE(nonce, another_nonce); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/crypto.cc b/be/src/kudu/security/crypto.cc new file mode 100644 index 0000000..234d193 --- /dev/null +++ b/be/src/kudu/security/crypto.cc @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/crypto.h" + +#include <memory> +#include <ostream> +#include <string> + +#include <glog/logging.h> +#include <openssl/bio.h> +#include <openssl/bn.h> +#include <openssl/err.h> +#include <openssl/evp.h> +#include <openssl/opensslv.h> +#include <openssl/ossl_typ.h> +#include <openssl/pem.h> +#include <openssl/rand.h> +#include <openssl/rsa.h> +#include <openssl/x509.h> + +#include "kudu/gutil/strings/substitute.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/openssl_util_bio.h" +#include "kudu/util/status.h" + +using std::string; +using strings::Substitute; + +namespace kudu { +namespace security { + +const size_t kNonceSize = 16; + +namespace { + +// Writing the private key from an EVP_PKEY has a different +// signature than the rest of the write functions, so we +// have to provide this wrapper. +int PemWritePrivateKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return PEM_write_bio_RSAPrivateKey( + bio, rsa.get(), nullptr, nullptr, 0, nullptr, nullptr); +} + +int PemWritePublicKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return PEM_write_bio_RSA_PUBKEY(bio, rsa.get()); +} + +int DerWritePublicKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return i2d_RSA_PUBKEY_bio(bio, rsa.get()); +} + +} // anonymous namespace + +template<> struct SslTypeTraits<BIGNUM> { + static constexpr auto kFreeFunc = &BN_free; +}; +struct RsaPrivateKeyTraits : public SslTypeTraits<EVP_PKEY> { + static constexpr auto kReadPemFunc = &PEM_read_bio_PrivateKey; + static constexpr auto kReadDerFunc = &d2i_PrivateKey_bio; + static constexpr auto kWritePemFunc = &PemWritePrivateKey; + static constexpr auto kWriteDerFunc = &i2d_PrivateKey_bio; +}; +struct RsaPublicKeyTraits : public SslTypeTraits<EVP_PKEY> { + static constexpr auto kReadPemFunc = &PEM_read_bio_PUBKEY; + static constexpr auto kReadDerFunc = &d2i_PUBKEY_bio; + static constexpr auto kWritePemFunc = &PemWritePublicKey; + static constexpr auto kWriteDerFunc = &DerWritePublicKey; +}; +template<> struct SslTypeTraits<RSA> { + static constexpr auto kFreeFunc = &RSA_free; +}; +template<> struct SslTypeTraits<EVP_MD_CTX> { +#if OPENSSL_VERSION_NUMBER < 0x10100000L + static constexpr auto kFreeFunc = &EVP_MD_CTX_destroy; +#else + static constexpr auto kFreeFunc = &EVP_MD_CTX_free; +#endif +}; + +namespace { + +const EVP_MD* GetMessageDigest(DigestType digest_type) { + switch (digest_type) { + case DigestType::SHA256: return EVP_sha256(); + case DigestType::SHA512: return EVP_sha512(); + } + LOG(FATAL) << "unknown digest type"; +} + +} // anonymous namespace + + +Status PublicKey::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString<RawDataType, RsaPublicKeyTraits>( + data, format, &data_); +} + +Status PublicKey::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString<RawDataType, RsaPublicKeyTraits>( + data, format, data_.get()); +} + +Status PublicKey::FromFile(const std::string& fpath, DataFormat format) { + return ::kudu::security::FromFile<RawDataType, RsaPublicKeyTraits>( + fpath, format, &data_); +} + +Status PublicKey::FromBIO(BIO* bio, DataFormat format) { + return ::kudu::security::FromBIO<RawDataType, RsaPublicKeyTraits>( + bio, format, &data_); +} + +// Modeled after code in $OPENSSL_ROOT/apps/dgst.c +Status PublicKey::VerifySignature(DigestType digest, + const std::string& data, + const std::string& signature) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + const EVP_MD* md = GetMessageDigest(digest); + auto md_ctx = ssl_make_unique(EVP_MD_CTX_create()); + + OPENSSL_RET_NOT_OK(EVP_DigestVerifyInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()), + "error initializing verification digest"); + OPENSSL_RET_NOT_OK(EVP_DigestVerifyUpdate(md_ctx.get(), data.data(), data.size()), + "error verifying data signature"); +#if OPENSSL_VERSION_NUMBER < 0x10002000L + unsigned char* sig_data = reinterpret_cast<unsigned char*>( + const_cast<char*>(signature.data())); +#else + const unsigned char* sig_data = reinterpret_cast<const unsigned char*>( + signature.data()); +#endif + // The success is indicated by return code 1. All other values means + // either wrong signature or error while performing signature verification. + const int rc = EVP_DigestVerifyFinal(md_ctx.get(), sig_data, signature.size()); + if (rc < 0 || rc > 1) { + return Status::RuntimeError( + Substitute("error verifying data signature: $0", GetOpenSSLErrors())); + } + if (rc == 0) { + // No sense stringifying the internal OpenSSL error, since a bad verification + // is self-explanatory. + ERR_clear_error(); + return Status::Corruption("data signature verification failed"); + } + + return Status::OK(); +} + +Status PublicKey::Equals(const PublicKey& other, bool* equals) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + int cmp = EVP_PKEY_cmp(data_.get(), other.data_.get()); + switch (cmp) { + case -2: + return Status::NotSupported("failed to compare public keys"); + case -1: // Key types are different; treat this as not equal + case 0: // Keys are not equal + *equals = false; + return Status::OK(); + case 1: + *equals = true; + return Status::OK(); + default: + return Status::RuntimeError("unexpected public key comparison result", std::to_string(cmp)); + } +} + +Status PrivateKey::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString<RawDataType, RsaPrivateKeyTraits>( + data, format, &data_); +} + +Status PrivateKey::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString<RawDataType, RsaPrivateKeyTraits>( + data, format, data_.get()); +} + +Status PrivateKey::FromFile(const std::string& fpath, DataFormat format, + const PasswordCallback& password_cb) { + return ::kudu::security::FromFile<RawDataType, RsaPrivateKeyTraits>( + fpath, format, &data_, password_cb); +} + +// The code is modeled after $OPENSSL_ROOT/apps/rsa.c code: there is +// corresponding functionality to read public part from RSA private/public +// keypair. +Status PrivateKey::GetPublicKey(PublicKey* public_key) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(public_key); + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(CHECK_NOTNULL(data_.get()))); + if (PREDICT_FALSE(!rsa)) { + return Status::RuntimeError(GetOpenSSLErrors()); + } + auto tmp = ssl_make_unique(BIO_new(BIO_s_mem())); + CHECK(tmp); + // Export public key in DER format into the temporary buffer. + OPENSSL_RET_NOT_OK(i2d_RSA_PUBKEY_bio(tmp.get(), rsa.get()), + "error extracting public RSA key"); + // Read the public key into the result placeholder. + RETURN_NOT_OK(public_key->FromBIO(tmp.get(), DataFormat::DER)); + + return Status::OK(); +} + +// Modeled after code in $OPENSSL_ROOT/apps/dgst.c +Status PrivateKey::MakeSignature(DigestType digest, + const std::string& data, + std::string* signature) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(signature); + const EVP_MD* md = GetMessageDigest(digest); + auto md_ctx = ssl_make_unique(EVP_MD_CTX_create()); + + OPENSSL_RET_NOT_OK(EVP_DigestSignInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()), + "error initializing signing digest"); + OPENSSL_RET_NOT_OK(EVP_DigestSignUpdate(md_ctx.get(), data.data(), data.size()), + "error signing data"); + size_t sig_len = EVP_PKEY_size(GetRawData()); + static const size_t kSigBufSize = 4 * 1024; + CHECK(sig_len <= kSigBufSize); + unsigned char buf[kSigBufSize]; + OPENSSL_RET_NOT_OK(EVP_DigestSignFinal(md_ctx.get(), buf, &sig_len), + "error finalizing data signature"); + *signature = string(reinterpret_cast<char*>(buf), sig_len); + + return Status::OK(); +} + +Status GeneratePrivateKey(int num_bits, PrivateKey* ret) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(ret); + InitializeOpenSSL(); + auto key = ssl_make_unique(EVP_PKEY_new()); + { + auto bn = ssl_make_unique(BN_new()); + OPENSSL_CHECK_OK(BN_set_word(bn.get(), RSA_F4)); + auto rsa = ssl_make_unique(RSA_new()); + OPENSSL_RET_NOT_OK( + RSA_generate_key_ex(rsa.get(), num_bits, bn.get(), nullptr), + "error generating RSA key"); + OPENSSL_RET_NOT_OK( + EVP_PKEY_set1_RSA(key.get(), rsa.get()), "error assigning RSA key"); + } + ret->AdoptRawData(key.release()); + + return Status::OK(); +} + +Status GenerateNonce(string* s) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK_NOTNULL(s); + unsigned char buf[kNonceSize]; + OPENSSL_RET_NOT_OK(RAND_bytes(buf, sizeof(buf)), "failed to generate nonce"); + s->assign(reinterpret_cast<char*>(buf), kNonceSize); + return Status::OK(); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/crypto.h b/be/src/kudu/security/crypto.h new file mode 100644 index 0000000..145c405 --- /dev/null +++ b/be/src/kudu/security/crypto.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <cstddef> +#include <string> + +#include <openssl/bio.h> +#include <openssl/rsa.h> + +#include "kudu/gutil/port.h" +#include "kudu/security/openssl_util.h" + +// Forward declarations for the OpenSSL typedefs. +typedef struct rsa_st RSA; +typedef struct bio_st BIO; + +namespace kudu { + +class Status; + +namespace security { + +extern const size_t kNonceSize; + +// Supported message digests for data signing and signature verification. +enum class DigestType { + SHA256, + SHA512, +}; + +// A class with generic public key interface, but actually it represents +// an RSA key. +class PublicKey : public RawDataWrapper<EVP_PKEY> { + public: + ~PublicKey() {} + + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT; + + Status FromBIO(BIO* bio, DataFormat format) WARN_UNUSED_RESULT; + + // Using the key, verify data signature using the specified message + // digest algorithm for signature verification. + // The input signature should be in in raw format (i.e. no base64 encoding). + Status VerifySignature(DigestType digest, + const std::string& data, + const std::string& signature) const WARN_UNUSED_RESULT; + + // Sets 'equals' to true if the other public key equals this. + Status Equals(const PublicKey& other, bool* equals) const WARN_UNUSED_RESULT; +}; + +// A class with generic private key interface, but actually it represents +// an RSA private key. It's important to have PrivateKey and PublicKey +// be different types to avoid accidental leakage of private keys. +class PrivateKey : public RawDataWrapper<EVP_PKEY> { + public: + ~PrivateKey() {} + + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + + // If 'cb' is set, it will be called to obtain the password necessary to decrypt + // the private key file in 'fpath'. + Status FromFile(const std::string& fpath, DataFormat format, + const PasswordCallback& password_cb = PasswordCallback()) WARN_UNUSED_RESULT; + + // Output the public part of the keypair into the specified placeholder. + Status GetPublicKey(PublicKey* public_key) const WARN_UNUSED_RESULT; + + // Using the key, generate data signature using the specified + // message digest algorithm. The result signature is in raw format + // (i.e. no base64 encoding). + Status MakeSignature(DigestType digest, + const std::string& data, + std::string* signature) const WARN_UNUSED_RESULT; +}; + +// Utility method to generate private keys. +Status GeneratePrivateKey(int num_bits, PrivateKey* ret) WARN_UNUSED_RESULT; + +// Generates a nonce of size kNonceSize, and writes it to the provided string. +Status GenerateNonce(std::string* s) WARN_UNUSED_RESULT; + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/init.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/init.cc b/be/src/kudu/security/init.cc new file mode 100644 index 0000000..ee754eb --- /dev/null +++ b/be/src/kudu/security/init.cc @@ -0,0 +1,465 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/init.h" + +#include <algorithm> +#include <cctype> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <ctime> +#include <functional> +#include <memory> +#include <mutex> +#include <ostream> +#include <random> +#include <string> +#include <type_traits> + +#include <boost/optional/optional.hpp> +#include <gflags/gflags.h> +#include <glog/logging.h> +#include <krb5/krb5.h> + +#include "kudu/gutil/macros.h" +#include "kudu/gutil/ref_counted.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/gutil/strings/util.h" +#include "kudu/util/flag_tags.h" +#include "kudu/util/monotime.h" +#include "kudu/util/net/net_util.h" +#include "kudu/util/rw_mutex.h" +#include "kudu/util/scoped_cleanup.h" +#include "kudu/util/status.h" +#include "kudu/util/thread.h" + +#ifndef __APPLE__ +static constexpr bool kDefaultSystemAuthToLocal = true; +#else +// macOS's Heimdal library has a no-op implementation of +// krb5_aname_to_localname, so instead we just use the simple +// implementation. +static constexpr bool kDefaultSystemAuthToLocal = false; +#endif +DEFINE_bool(use_system_auth_to_local, kDefaultSystemAuthToLocal, + "When enabled, use the system krb5 library to map Kerberos principal " + "names to local (short) usernames. If not enabled, the first component " + "of the principal will be used as the short name. For example, " + "'kudu/foo.example.com@EXAMPLE' will map to 'kudu'."); +TAG_FLAG(use_system_auth_to_local, advanced); + + +using std::mt19937; +using std::random_device; +using std::string; +using std::uniform_int_distribution; +using std::uniform_real_distribution; +using strings::Substitute; + +namespace kudu { +namespace security { + +namespace { + +class KinitContext; + +// Global context for usage of the Krb5 library. +krb5_context g_krb5_ctx; + +// Global instance of the context used by the kinit/reacquire thread. +KinitContext* g_kinit_ctx; + +// This lock is used to avoid a race while reacquiring the kerberos ticket. +// The race can occur between the time we reinitialize the cache and the +// time when we actually store the new credentials back in the cache. +RWMutex* g_kerberos_reinit_lock; + +class KinitContext { + public: + KinitContext(); + + // Equivalent implementation of 'kinit -kt <keytab path> <principal>'. + // + // This logs in from the given keytab as the given principal, returning + // RuntimeError if any part of this process fails. + // + // If the log-in is successful, then the default ticket cache is overwritten + // with the credentials of the newly logged-in principal. + Status Kinit(const string& keytab_path, const string& principal); + + // Acquires a new Ticket Granting Ticket (TGT). + // + // Renews the existing ticket if possible, or acquires a new Ticket Granting + // Ticket (TGT). + Status DoRenewal(); + + // Calculates the next sleep interval based on the 'ticket_end_timestamp_' and + // adds some jitter so that all the nodes do not hit the KDC at the same time. + // + // If 'num_retries' > 0, it calls GetBackedOffRenewInterval() to return a backed + // off interval. + int32_t GetNextRenewInterval(uint32_t num_retries); + + // Returns a value based on 'time_remaining' that increases exponentially with + // 'num_retries', with a random jitter of +/- 0%-50% of that value. + int32_t GetBackedOffRenewInterval(int32_t time_remaining, uint32_t num_retries); + + const string& principal_str() const { return principal_str_; } + const string& username_str() const { return username_str_; } + + private: + krb5_principal principal_; + krb5_keytab keytab_; + krb5_ccache ccache_; + krb5_get_init_creds_opt* opts_; + + // The stringified principal and username that we are logged in as. + string principal_str_, username_str_; + + // This is the time that the current TGT in use expires. + int32_t ticket_end_timestamp_; +}; + +Status Krb5CallToStatus(krb5_context ctx, krb5_error_code code) { + if (code == 0) return Status::OK(); + + std::unique_ptr<const char, std::function<void(const char*)>> err_msg( + krb5_get_error_message(ctx, code), + std::bind(krb5_free_error_message, ctx, std::placeholders::_1)); + return Status::RuntimeError(err_msg.get()); +} +#define KRB5_RETURN_NOT_OK_PREPEND(call, prepend) \ + RETURN_NOT_OK_PREPEND(Krb5CallToStatus(g_krb5_ctx, (call)), (prepend)) + + +void InitKrb5Ctx() { + static std::once_flag once; + std::call_once(once, [&]() { + CHECK_EQ(krb5_init_context(&g_krb5_ctx), 0); + }); +} + +KinitContext::KinitContext() {} + +// Port of the data_eq() implementation from krb5/k5-int.h +inline int data_eq(krb5_data d1, krb5_data d2) { + return (d1.length == d2.length && !memcmp(d1.data, d2.data, d1.length)); +} + +// Port of the data_eq_string() implementation from krb5/k5-int.h +inline int data_eq_string(krb5_data d, const char *s) { + return (d.length == strlen(s) && !memcmp(d.data, s, d.length)); +} + +Status Krb5UnparseName(krb5_principal princ, string* name) { + char* c_name; + KRB5_RETURN_NOT_OK_PREPEND(krb5_unparse_name(g_krb5_ctx, princ, &c_name), + "krb5_unparse_name"); + SCOPED_CLEANUP({ + krb5_free_unparsed_name(g_krb5_ctx, c_name); + }); + *name = c_name; + return Status::OK(); +} + +// Periodically calls DoRenewal(). +void RenewThread() { + uint32_t failure_retries = 0; + while (true) { + // This thread is run immediately after the first Kinit, so sleep first. + SleepFor(MonoDelta::FromSeconds(g_kinit_ctx->GetNextRenewInterval(failure_retries))); + + Status s = g_kinit_ctx->DoRenewal(); + WARN_NOT_OK(s, "Kerberos reacquire error: "); + if (!s.ok()) { + ++failure_retries; + } else { + failure_retries = 0; + } + } +} + +int32_t KinitContext::GetNextRenewInterval(uint32_t num_retries) { + int32_t time_remaining = ticket_end_timestamp_ - time(nullptr); + + // If the last ticket reacqusition was a failure, we back off our retry attempts exponentially. + if (num_retries > 0) return GetBackedOffRenewInterval(time_remaining, num_retries); + + // If the time remaining between now and ticket expiry is: + // * > 10 minutes: We attempt to reacquire the ticket between 5 seconds and 5 minutes before the + // ticket expires. + // * 5 - 10 minutes: We attempt to reacquire the ticket betwen 5 seconds and 1 minute before the + // ticket expires. + // * < 5 minutes: Attempt to reacquire the ticket every 'time_remaining'. + // The jitter is added to make sure that every server doesn't flood the KDC at the same time. + random_device rd; + mt19937 generator(rd()); + if (time_remaining > 600) { + uniform_int_distribution<> dist(5, 300); + return time_remaining - dist(generator); + } else if (time_remaining > 300) { + uniform_int_distribution<> dist(5, 60); + return time_remaining - dist(generator); + } + return time_remaining; +} + +int32_t KinitContext::GetBackedOffRenewInterval(int32_t time_remaining, uint32_t num_retries) { + // The minimum sleep interval after a failure will be 60 seconds. + int32_t next_interval = std::max(time_remaining, 60); + int32_t base_time = std::min(next_interval * (1 << num_retries), INT32_MAX); + random_device rd; + mt19937 generator(rd()); + uniform_real_distribution<> dist(0.5, 1.5); + return static_cast<int32_t>(base_time * dist(generator)); +} + +Status KinitContext::DoRenewal() { + + krb5_cc_cursor cursor; + // Setup a cursor to iterate through the credential cache. + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_start_seq_get(g_krb5_ctx, ccache_, &cursor), + "Failed to peek into ccache"); + SCOPED_CLEANUP({ + krb5_cc_end_seq_get(g_krb5_ctx, ccache_, &cursor); }); + + krb5_creds creds; + memset(&creds, 0, sizeof(krb5_creds)); + + krb5_error_code rc; + // Iterate through the credential cache. + while (!(rc = krb5_cc_next_cred(g_krb5_ctx, ccache_, &cursor, &creds))) { + SCOPED_CLEANUP({ + krb5_free_cred_contents(g_krb5_ctx, &creds); }); + if (krb5_is_config_principal(g_krb5_ctx, creds.server)) continue; + + // We only want to reacquire the TGT (Ticket Granting Ticket). Ignore all other tickets. + // This follows the same format as is_local_tgt() from krb5:src/clients/klist/klist.c + if (creds.server->length != 2 || + data_eq(creds.server->data[1], principal_->realm) == 0 || + data_eq_string(creds.server->data[0], KRB5_TGS_NAME) == 0 || + data_eq(creds.server->realm, principal_->realm) == 0) { + continue; + } + + krb5_creds new_creds; + memset(&new_creds, 0, sizeof(krb5_creds)); + SCOPED_CLEANUP({ + krb5_free_cred_contents(g_krb5_ctx, &new_creds); }); + // Acquire a new ticket using the keytab. This ticket will automatically be put into the + // credential cache. + { + std::lock_guard<RWMutex> l(*g_kerberos_reinit_lock); + KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_keytab(g_krb5_ctx, &new_creds, principal_, + keytab_, 0 /* valid from now */, + nullptr /* TKT service name */, + opts_), + "Reacquire error: unable to login from keytab"); +#if !defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE) + // Heimdal krb5 doesn't have the 'krb5_get_init_creds_opt_set_out_ccache' option, + // so use this alternate route. + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_initialize(g_krb5_ctx, ccache_, principal_), + "Reacquire error: could not init ccache"); + + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_store_cred(g_krb5_ctx, ccache_, &new_creds), + "Reacquire error: could not store creds in cache"); +#endif + } + LOG(INFO) << "Successfully reacquired a new kerberos TGT"; + ticket_end_timestamp_ = new_creds.times.endtime; + break; + } + return Status::OK(); +} + +Status KinitContext::Kinit(const string& keytab_path, const string& principal) { + InitKrb5Ctx(); + + // Parse the principal + KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal.c_str(), &principal_), + "could not parse principal"); + + KRB5_RETURN_NOT_OK_PREPEND(krb5_kt_resolve(g_krb5_ctx, keytab_path.c_str(), &keytab_), + "unable to resolve keytab"); + + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_default(g_krb5_ctx, &ccache_), + "unable to get default credentials cache"); + + KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_opt_alloc(g_krb5_ctx, &opts_), + "unable to allocate get_init_creds_opt struct"); + +#if defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE) + KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_opt_set_out_ccache(g_krb5_ctx, opts_, ccache_), + "unable to set init_creds options"); +#endif + + krb5_creds creds; + KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_keytab(g_krb5_ctx, &creds, principal_, keytab_, + 0 /* valid from now */, + nullptr /* TKT service name */, opts_), + "unable to login from keytab"); + SCOPED_CLEANUP({ + krb5_free_cred_contents(g_krb5_ctx, &creds); }); + + ticket_end_timestamp_ = creds.times.endtime; + +#if !defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE) + // Heimdal krb5 doesn't have the 'krb5_get_init_creds_opt_set_out_ccache' option, + // so use this alternate route. + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_initialize(g_krb5_ctx, ccache_, principal_), + "could not init ccache"); + + KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_store_cred(g_krb5_ctx, ccache_, &creds), + "could not store creds in cache"); +#endif + + // Convert the logged-in principal back to a string. This may be different than + // 'principal', since the default realm will be filled in based on the Kerberos + // configuration if not originally specified. + RETURN_NOT_OK_PREPEND(Krb5UnparseName(principal_, &principal_str_), + "could not stringify the logged-in principal"); + RETURN_NOT_OK_PREPEND(MapPrincipalToLocalName(principal_str_, &username_str_), + "could not map own logged-in principal to a short username"); + + LOG(INFO) << "Logged in from keytab as " << principal_str_ + << " (short username " << username_str_ << ")"; + + return Status::OK(); +} + +// 'in_principal' is the user specified principal to use with Kerberos. It may have a token +// in the string of the form '_HOST', which if present, needs to be replaced with the FQDN of the +// current host. +// 'out_principal' has the final principal with which one may Kinit. +Status GetConfiguredPrincipal(const std::string& in_principal, string* out_principal) { + *out_principal = in_principal; + const auto& kHostToken = "_HOST"; + if (in_principal.find(kHostToken) != string::npos) { + string hostname; + // Try to fill in either the FQDN or hostname. + if (!GetFQDN(&hostname).ok()) { + RETURN_NOT_OK(GetHostname(&hostname)); + } + // Hosts in principal names are canonicalized to lower-case. + std::transform(hostname.begin(), hostname.end(), hostname.begin(), tolower); + GlobalReplaceSubstring(kHostToken, hostname, out_principal); + } + return Status::OK(); +} +} // anonymous namespace + + +RWMutex* KerberosReinitLock() { + return g_kerberos_reinit_lock; +} + +Status CanonicalizeKrb5Principal(std::string* principal) { + InitKrb5Ctx(); + krb5_principal princ; + KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal->c_str(), &princ), + "could not parse principal"); + SCOPED_CLEANUP({ + krb5_free_principal(g_krb5_ctx, princ); + }); + RETURN_NOT_OK_PREPEND(Krb5UnparseName(princ, principal), + "failed to convert principal back to string"); + return Status::OK(); +} + +Status MapPrincipalToLocalName(const std::string& principal, std::string* local_name) { + InitKrb5Ctx(); + krb5_principal princ; + KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal.c_str(), &princ), + "could not parse principal"); + SCOPED_CLEANUP({ + krb5_free_principal(g_krb5_ctx, princ); + }); + char buf[1024]; + krb5_error_code rc = KRB5_LNAME_NOTRANS; + if (FLAGS_use_system_auth_to_local) { + rc = krb5_aname_to_localname(g_krb5_ctx, princ, arraysize(buf), buf); + } + if (rc == KRB5_LNAME_NOTRANS || rc == KRB5_PLUGIN_NO_HANDLE) { + // No name mapping specified, or krb5-based name mapping is disabled. + // + // We fall back to simply taking the first component of the principal, for + // compatibility with the default behavior of Hadoop. + // + // NOTE: KRB5_PLUGIN_NO_HANDLE isn't typically expected here, but works around + // a bug in SSSD's auth_to_local implementation: https://pagure.io/SSSD/sssd/issue/3459 + // + // TODO(todd): we should support custom configured auth-to-local mapping, since + // most Hadoop ecosystem components do not load them from krb5.conf. + if (princ->length > 0) { + local_name->assign(princ->data[0].data, princ->data[0].length); + return Status::OK(); + } + return Status::NotFound("unable to find first component of principal"); + } + if (rc == KRB5_CONFIG_NOTENUFSPACE) { + return Status::InvalidArgument("mapped username too large"); + } + KRB5_RETURN_NOT_OK_PREPEND(rc, "krb5_aname_to_localname"); + if (strlen(buf) == 0) { + return Status::InvalidArgument("principal mapped to empty username"); + } + local_name->assign(buf); + return Status::OK(); +} + +boost::optional<string> GetLoggedInPrincipalFromKeytab() { + if (!g_kinit_ctx) return boost::none; + return g_kinit_ctx->principal_str(); +} + +boost::optional<string> GetLoggedInUsernameFromKeytab() { + if (!g_kinit_ctx) return boost::none; + return g_kinit_ctx->username_str(); +} + +Status InitKerberosForServer(const std::string& raw_principal, const std::string& keytab_file, + const std::string& krb5ccname, bool disable_krb5_replay_cache) { + if (keytab_file.empty()) return Status::OK(); + + setenv("KRB5CCNAME", krb5ccname.c_str(), 1); + setenv("KRB5_KTNAME", keytab_file.c_str(), 1); + + if (disable_krb5_replay_cache) { + // KUDU-1897: disable the Kerberos replay cache. The KRPC protocol includes a + // per-connection server-generated nonce to protect against replay attacks + // when authenticating via Kerberos. The replay cache has many performance and + // implementation issues. + setenv("KRB5RCACHETYPE", "none", 1); + } + + g_kinit_ctx = new KinitContext(); + string configured_principal; + RETURN_NOT_OK(GetConfiguredPrincipal(raw_principal, &configured_principal)); + RETURN_NOT_OK_PREPEND(g_kinit_ctx->Kinit( + keytab_file, configured_principal), "unable to kinit"); + + g_kerberos_reinit_lock = new RWMutex(RWMutex::Priority::PREFER_WRITING); + scoped_refptr<Thread> reacquire_thread; + // Start the reacquire thread. + RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread", &RenewThread, &reacquire_thread)); + + return Status::OK(); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/init.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/init.h b/be/src/kudu/security/init.h new file mode 100644 index 0000000..8b1519a --- /dev/null +++ b/be/src/kudu/security/init.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include <string> + +namespace boost { +template <class T> +class optional; +} + +namespace kudu { + +class RWMutex; +class Status; + +namespace security { + +// The default kerberos credential cache name. +// Have the daemons use an in-memory ticket cache, so they don't accidentally +// pick up credentials from test cases or any other daemon. +static const std::string kKrb5CCName = "MEMORY:kudu"; + +// Initializes Kerberos for a server. In particular, this processes +// the '--keytab_file' command line flag. +// 'raw_principal' is the principal to Kinit with after calling GetConfiguredPrincipal() +// on it. +// 'keytab_file' is the path to the kerberos keytab file. If it's an empty string, kerberos +// will not be initialized. +// 'krb5ccname' is passed into the KRB5CCNAME env var. +// 'disable_krb5_replay_cache' if set to true, disables the kerberos replay cache by setting +// the KRB5RCACHETYPE env var to "none". +Status InitKerberosForServer(const std::string& raw_principal, + const std::string& keytab_file, + const std::string& krb5ccname = kKrb5CCName, + bool disable_krb5_replay_cache = true); + +// Returns the process lock 'kerberos_reinit_lock' +// This lock is taken in write mode while the ticket is being reacquired, and +// taken in read mode before using the SASL library which might require a ticket. +RWMutex* KerberosReinitLock(); + +// Return the full principal (user/host@REALM) that the server has used to +// log in from the keytab. +// +// If the server has not logged in from a keytab, returns boost::none. +boost::optional<std::string> GetLoggedInPrincipalFromKeytab(); + +// Same, but returns the mapped short username. +boost::optional<std::string> GetLoggedInUsernameFromKeytab(); + +// Canonicalize the given principal name by adding '@DEFAULT_REALM' in the case that +// the principal has no realm. +// +// TODO(todd): move to kerberos_util.h in the later patch in this series (the file doesn't +// exist yet, and trying to avoid rebase pain). +Status CanonicalizeKrb5Principal(std::string* principal); + +// Map the given Kerberos principal 'principal' to a short username (i.e. with no realm or +// host component). +// +// This respects the "auth-to-local" mappings from the system krb5.conf. However, if no such +// mapping can be found, we fall back to simply taking the first component of the principal. +// +// TODO(todd): move to kerberos_util.h in the later patch in this series (the file doesn't +// exist yet, and trying to avoid rebase pain). +Status MapPrincipalToLocalName(const std::string& principal, std::string* local_name); + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/kerberos_util.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/kerberos_util.cc b/be/src/kudu/security/kerberos_util.cc new file mode 100644 index 0000000..5e6d9b8 --- /dev/null +++ b/be/src/kudu/security/kerberos_util.cc @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/kerberos_util.h" +#include "kudu/gutil/strings/split.h" +#include "kudu/gutil/strings/stringpiece.h" + +#include <array> +#include <utility> + +namespace kudu { +namespace security { + +std::array<StringPiece, 3> SplitKerberosPrincipal(StringPiece principal) { + + std::pair<StringPiece, StringPiece> user_realm = strings::Split(principal, "@"); + std::pair<StringPiece, StringPiece> princ_host = strings::Split(user_realm.first, "/"); + return {{princ_host.first, princ_host.second, user_realm.second}}; +} + + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/kerberos_util.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/kerberos_util.h b/be/src/kudu/security/kerberos_util.h new file mode 100644 index 0000000..4c27a86 --- /dev/null +++ b/be/src/kudu/security/kerberos_util.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include <array> + +class StringPiece; + +namespace kudu { +namespace security { + +std::array<StringPiece, 3> SplitKerberosPrincipal(StringPiece principal); + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/krb5_realm_override.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/krb5_realm_override.cc b/be/src/kudu/security/krb5_realm_override.cc new file mode 100644 index 0000000..05e8af8 --- /dev/null +++ b/be/src/kudu/security/krb5_realm_override.cc @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This file provides a workaround for tests running with Kerberos 1.11 or earlier. +// These versions of Kerberos are missing a fix which allows service principals +// to use IP addresses in their host component: +// +// http://krbdev.mit.edu/rt/Ticket/Display.html?id=7603 +// +// We use such principals in external minicluster tests, where servers have IP addresses +// like 127.x.y.z that have no corresponding reverse DNS. +// +// The file contains an implementation of krb5_get_host_realm which wraps the one +// in the Kerberos library. It detects the return code that indicates the +// above problem and falls back to the default realm/ +// +// The wrapper is injected via linking it into tests as well as the +// "security" library. The linkage invocation uses the '-Wl,--undefined' +// linker flag to force linking even though no symbol here is explicitly +// referenced. + +#include <dlfcn.h> + +#include <cerrno> +#include <cstdlib> +#include <cstring> + +#include <krb5/krb5.h> +#include <glog/logging.h> + +extern "C" { + +// This symbol is exported from the static library so that other static-linked binaries +// can reference it and force this compilation unit to be linked. Otherwise the linker +// thinks it's unused and doesn't link it. +int krb5_realm_override_loaded = 1; + +// Save the original function from the Kerberos library itself. +// We use dlsym() to load all of them, since this file gets linked into +// some test binaries that themselves may not link against libkrb5.so at all. +static void* g_orig_krb5_get_host_realm; +static void* g_orig_krb5_get_default_realm; +static void* g_orig_krb5_free_default_realm; + +// We only enable our workaround if this environment variable is set. +constexpr static const char* kEnvVar = "KUDU_ENABLE_KRB5_REALM_FIX"; + +#define CALL_ORIG(func_name, ...) \ + ((decltype(&func_name))g_orig_ ## func_name)(__VA_ARGS__) + +__attribute__((constructor)) +static void init_orig_func() { + g_orig_krb5_get_host_realm = dlsym(RTLD_NEXT, "krb5_get_host_realm"); + g_orig_krb5_get_default_realm = dlsym(RTLD_NEXT, "krb5_get_default_realm"); + g_orig_krb5_free_default_realm = dlsym(RTLD_NEXT, "krb5_free_default_realm"); +} + +krb5_error_code krb5_get_host_realm(krb5_context context, const char* host, char*** realmsp) { + CHECK(g_orig_krb5_get_host_realm); + CHECK(g_orig_krb5_get_default_realm); + CHECK(g_orig_krb5_free_default_realm); + + krb5_error_code rc = CALL_ORIG(krb5_get_host_realm, context, host, realmsp); + if (rc != KRB5_ERR_NUMERIC_REALM || getenv(kEnvVar) == nullptr) { + return rc; + } + // If we get KRB5_ERR_NUMERIC_REALM, this is indicative of a Kerberos version + // which has not provided support for numeric addresses as service host names + // So, we fill in the default realm instead. + char* default_realm; + rc = CALL_ORIG(krb5_get_default_realm, context, &default_realm); + if (rc != 0) { + return rc; + } + + char** ret_realms; + ret_realms = static_cast<char**>(malloc(2 * sizeof(*ret_realms))); + if (ret_realms == nullptr) return ENOMEM; + ret_realms[0] = strdup(default_realm); + if (ret_realms[0] == nullptr) { + free(ret_realms); + return ENOMEM; + } + ret_realms[1] = 0; + *realmsp = ret_realms; + + CALL_ORIG(krb5_free_default_realm, context, default_realm); + return 0; +} + +} // extern "C" http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/openssl_util.cc b/be/src/kudu/security/openssl_util.cc new file mode 100644 index 0000000..a32140f --- /dev/null +++ b/be/src/kudu/security/openssl_util.cc @@ -0,0 +1,322 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/openssl_util.h" + +#include <cerrno> +#include <cstdint> +#include <cstdio> +#include <mutex> +#include <string> +#include <vector> + +#include <glog/logging.h> +#include <openssl/crypto.h> +#include <openssl/err.h> +#include <openssl/rand.h> + +#include "kudu/gutil/strings/split.h" +#include "kudu/gutil/strings/strip.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/debug/leakcheck_disabler.h" +#include "kudu/util/errno.h" +#include "kudu/util/mutex.h" +#include "kudu/util/scoped_cleanup.h" +#include "kudu/util/status.h" +#include "kudu/util/subprocess.h" + +using std::ostringstream; +using std::string; +using std::vector; + +namespace kudu { +namespace security { + +namespace { + +// Determine whether initialization was ever called. +// +// Thread safety: +// - written by DoInitializeOpenSSL (single-threaded, due to std::call_once) +// - read by DisableOpenSSLInitialization (must not be concurrent with above) +bool g_ssl_is_initialized = false; + +// If true, then we expect someone else has initialized SSL. +// +// Thread safety: +// - read by DoInitializeOpenSSL (single-threaded, due to std::call_once) +// - written by DisableOpenSSLInitialization (must not be concurrent with above) +bool g_disable_ssl_init = false; + +// Array of locks used by OpenSSL. +// We use an intentionally-leaked C-style array here to avoid non-POD static data. +// +// As of OpenSSL 1.1, locking callbacks are no longer used. +#if OPENSSL_VERSION_NUMBER < 0x10100000L +Mutex* kCryptoLocks = nullptr; + +// Lock/Unlock the nth lock. Only to be used by OpenSSL. +void LockingCB(int mode, int type, const char* /*file*/, int /*line*/) { + DCHECK(kCryptoLocks); + Mutex* m = &kCryptoLocks[type]; + if (mode & CRYPTO_LOCK) { + m->lock(); + } else { + m->unlock(); + } +} +#endif + +Status CheckOpenSSLInitialized() { + if (!CRYPTO_get_locking_callback()) { + return Status::RuntimeError("Locking callback not initialized"); + } + auto ctx = ssl_make_unique(SSL_CTX_new(SSLv23_method())); + if (!ctx) { + ERR_clear_error(); + return Status::RuntimeError("SSL library appears uninitialized (cannot create SSL_CTX)"); + } + return Status::OK(); +} + +void DoInitializeOpenSSL() { +#if OPENSSL_VERSION_NUMBER > 0x10100000L + // The OPENSSL_init_ssl manpage [1] says "As of version 1.1.0 OpenSSL will + // automatically allocate all resources it needs so no explicit initialisation + // is required." However, eliding library initialization leads to a memory + // leak in some versions of OpenSSL 1.1 when the first OpenSSL is + // ERR_peek_error [2]. In Kudu this is often the + // case due to prolific application of SCOPED_OPENSSL_NO_PENDING_ERRORS. + // + // Rather than determine whether this particular OpenSSL instance is + // leak-free, we'll initialize the library explicitly. + // + // 1. https://www.openssl.org/docs/man1.1.0/ssl/OPENSSL_init_ssl.html + // 2. https://github.com/openssl/openssl/issues/5899 + if (g_disable_ssl_init) { + VLOG(2) << "Not initializing OpenSSL (disabled by application)"; + return; + } + CHECK_EQ(1, OPENSSL_init_ssl(0, nullptr)); + SCOPED_OPENSSL_NO_PENDING_ERRORS; +#else + // In case the user's thread has left some error around, clear it. + ERR_clear_error(); + SCOPED_OPENSSL_NO_PENDING_ERRORS; + if (g_disable_ssl_init) { + VLOG(2) << "Not initializing OpenSSL (disabled by application)"; + return; + } + + // Check that OpenSSL isn't already initialized. If it is, it's likely + // we are embedded in (or embedding) another application/library which + // initializes OpenSSL, and we risk installing conflicting callbacks + // or crashing due to concurrent initialization attempts. In that case, + // log a warning. + auto ctx = ssl_make_unique(SSL_CTX_new(SSLv23_method())); + if (ctx) { + LOG(WARNING) << "It appears that OpenSSL has been previously initialized by " + << "code outside of Kudu. Please use kudu::client::DisableOpenSSLInitialization() " + << "to avoid potential crashes due to conflicting initialization."; + // Continue anyway; all of the below is idempotent, except for the locking callback, + // which we check before overriding. They aren't thread-safe, however -- that's why + // we try to get embedding applications to do the right thing here rather than risk a + // potential initialization race. + } else { + // As expected, SSL is not initialized, so SSL_CTX_new() failed. Make sure + // it didn't leave anything in our error queue. + ERR_clear_error(); + } + + SSL_load_error_strings(); + SSL_library_init(); + OpenSSL_add_all_algorithms(); + RAND_poll(); + + if (!CRYPTO_get_locking_callback()) { + // Initialize the OpenSSL mutexes. We intentionally leak these, so ignore + // LSAN warnings. + debug::ScopedLeakCheckDisabler d; + int num_locks = CRYPTO_num_locks(); + CHECK(!kCryptoLocks); + kCryptoLocks = new Mutex[num_locks]; + + // Callbacks used by OpenSSL required in a multi-threaded setting. + CRYPTO_set_locking_callback(LockingCB); + } +#endif + + g_ssl_is_initialized = true; +} + +} // anonymous namespace + +// Reads a STACK_OF(X509) from the BIO and returns it. +STACK_OF(X509)* PEM_read_STACK_OF_X509(BIO* bio, void* /* unused */, pem_password_cb* /* unused */, + void* /* unused */) { + // Extract information from the chain certificate. + STACK_OF(X509_INFO)* info = PEM_X509_INFO_read_bio(bio, nullptr, nullptr, nullptr); + if (!info) return nullptr; + SCOPED_CLEANUP({ + sk_X509_INFO_pop_free(info, X509_INFO_free); + }); + + // Initialize the Stack. + STACK_OF(X509)* sk = sk_X509_new_null(); + + // Iterate through the chain certificate and add each one to the stack. + for (int i = 0; i < sk_X509_INFO_num(info); ++i) { + X509_INFO *stack_item = sk_X509_INFO_value(info, i); + sk_X509_push(sk, stack_item->x509); + // We don't want the ScopedCleanup to free the x509 certificates as well since we will + // use it as a part of the STACK_OF(X509) object to be returned, so we set it to nullptr. + // We will take the responsibility of freeing it when we are done with the STACK_OF(X509). + stack_item->x509 = nullptr; + } + return sk; +} + +// Writes a STACK_OF(X509) to the BIO. +int PEM_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj) { + int chain_len = sk_X509_num(obj); + // Iterate through the stack and add each one to the BIO. + for (int i = 0; i < chain_len; ++i) { + X509* cert_item = sk_X509_value(obj, i); + int ret = PEM_write_bio_X509(bio, cert_item); + if (ret <= 0) return ret; + } + return 1; +} + +// Reads a single X509 certificate and returns a STACK_OF(X509) with the single certificate. +STACK_OF(X509)* DER_read_STACK_OF_X509(BIO* bio, void* /* unused */) { + // We don't support chain certificates written in DER format. + auto x = ssl_make_unique(d2i_X509_bio(bio, nullptr)); + if (!x) return nullptr; + STACK_OF(X509)* sk = sk_X509_new_null(); + if (sk_X509_push(sk, x.get()) == 0) { + return nullptr; + } + x.release(); + return sk; +} + +// Writes a single X509 certificate that it gets from the STACK_OF(X509) 'obj'. +int DER_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj) { + int chain_len = sk_X509_num(obj); + // We don't support chain certificates written in DER format. + DCHECK_EQ(chain_len, 1); + X509* cert_item = sk_X509_value(obj, 0); + if (cert_item == nullptr) return 0; + return i2d_X509_bio(bio, cert_item); +} + +void free_STACK_OF_X509(STACK_OF(X509)* sk) { + sk_X509_pop_free(sk, X509_free); +} + +Status DisableOpenSSLInitialization() { + if (g_disable_ssl_init) return Status::OK(); + if (g_ssl_is_initialized) { + return Status::IllegalState("SSL already initialized. Initialization can only be disabled " + "before first usage."); + } + RETURN_NOT_OK(CheckOpenSSLInitialized()); + g_disable_ssl_init = true; + return Status::OK(); +} + +void InitializeOpenSSL() { + static std::once_flag ssl_once; + std::call_once(ssl_once, DoInitializeOpenSSL); +} + +string GetOpenSSLErrors() { + ostringstream serr; + uint32_t l; + int line, flags; + const char *file, *data; + bool is_first = true; + while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) { + if (is_first) { + is_first = false; + } else { + serr << " "; + } + + char buf[256]; + ERR_error_string_n(l, buf, sizeof(buf)); + serr << buf << ":" << file << ":" << line; + if (flags & ERR_TXT_STRING) { + serr << ":" << data; + } + } + return serr.str(); +} + +string GetSSLErrorDescription(int error_code) { + switch (error_code) { + case SSL_ERROR_NONE: return ""; + case SSL_ERROR_ZERO_RETURN: return "SSL_ERROR_ZERO_RETURN"; + case SSL_ERROR_WANT_READ: return "SSL_ERROR_WANT_READ"; + case SSL_ERROR_WANT_WRITE: return "SSL_ERROR_WANT_WRITE"; + case SSL_ERROR_WANT_CONNECT: return "SSL_ERROR_WANT_CONNECT"; + case SSL_ERROR_WANT_ACCEPT: return "SSL_ERROR_WANT_ACCEPT"; + case SSL_ERROR_WANT_X509_LOOKUP: return "SSL_ERROR_WANT_X509_LOOKUP"; + case SSL_ERROR_SYSCALL: { + string queued_error = GetOpenSSLErrors(); + if (!queued_error.empty()) { + return queued_error; + } + return kudu::ErrnoToString(errno); + }; + default: return GetOpenSSLErrors(); + } +} + +const string& DataFormatToString(DataFormat fmt) { + static const string kStrFormatUnknown = "UNKNOWN"; + static const string kStrFormatDer = "DER"; + static const string kStrFormatPem = "PEM"; + switch (fmt) { + case DataFormat::DER: + return kStrFormatDer; + case DataFormat::PEM: + return kStrFormatPem; + default: + return kStrFormatUnknown; + } +} + +Status GetPasswordFromShellCommand(const string& cmd, string* password) { + vector<string> argv = strings::Split(cmd, " ", strings::SkipEmpty()); + if (argv.empty()) { + return Status::RuntimeError("invalid empty private key password command"); + } + string stderr, stdout; + Status s = Subprocess::Call(argv, "" /* stdin */, &stdout, &stderr); + if (!s.ok()) { + return Status::RuntimeError(strings::Substitute( + "failed to run private key password command: $0", s.ToString()), stderr); + } + StripTrailingWhitespace(&stdout); + *password = stdout; + return Status::OK(); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/openssl_util.h b/be/src/kudu/security/openssl_util.h new file mode 100644 index 0000000..00c4ebf --- /dev/null +++ b/be/src/kudu/security/openssl_util.h @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <functional> +#include <memory> +#include <ostream> +#include <string> + +#include <glog/logging.h> +#include <openssl/err.h> +#include <openssl/pem.h> +#include <openssl/ssl.h> +#include <openssl/x509.h> + +#include "kudu/gutil/port.h" +#include "kudu/util/status.h" + +// Forward declarations for the OpenSSL typedefs. +typedef struct X509_req_st X509_REQ; +typedef struct bio_st BIO; +typedef struct evp_pkey_st EVP_PKEY; +typedef struct ssl_ctx_st SSL_CTX; +typedef struct ssl_st SSL; +typedef struct x509_st X509; + +#define OPENSSL_CHECK_OK(call) \ + CHECK_GT((call), 0) + +#define OPENSSL_RET_NOT_OK(call, msg) \ + if ((call) <= 0) { \ + return Status::RuntimeError((msg), GetOpenSSLErrors()); \ + } + +#define OPENSSL_RET_IF_NULL(call, msg) \ + if ((call) == nullptr) { \ + return Status::RuntimeError((msg), GetOpenSSLErrors()); \ + } + +// Scoped helper which DCHECKs that on both scope entry and exit, there are no +// pending OpenSSL errors for the current thread. +// +// This allows us to avoid calling ERR_clear_error() defensively before every +// OpenSSL call, but rather call it only when we get an error code indicating +// there may be some pending error. +// +// Example usage: +// +// void MyFunc() { +// SCOPED_OPENSSL_NO_PENDING_ERRORS; +// ... use OpenSSL APIs ... +// } +#define SCOPED_OPENSSL_NO_PENDING_ERRORS \ + kudu::security::internal::ScopedCheckNoPendingSSLErrors _no_ssl_errors(__PRETTY_FUNCTION__) + +namespace kudu { +namespace security { + +using PasswordCallback = std::function<std::string(void)>; + +// Disable initialization of OpenSSL. Must be called before +// any call to InitializeOpenSSL(). +Status DisableOpenSSLInitialization() WARN_UNUSED_RESULT; + +// Initializes static state required by the OpenSSL library. +// This is a no-op if DisableOpenSSLInitialization() has been called. +// +// Safe to call multiple times. +void InitializeOpenSSL(); + +// Fetches errors from the OpenSSL error error queue, and stringifies them. +// +// The error queue will be empty after this method returns. +// +// See man(3) ERR_get_err for more discussion. +std::string GetOpenSSLErrors(); + +// Returns a string representation of the provided error code, which must be +// from a prior call to the SSL_get_error function. +// +// If necessary, the OpenSSL error queue may be inspected and emptied as part of +// this call, and/or 'errno' may be inspected. As a result, this method should +// only be used directly after the error occurs, and from the same thread. +// +// See man(3) SSL_get_error for more discussion. +std::string GetSSLErrorDescription(int error_code); + +// Runs the shell command 'cmd' which should give a password to a private key file +// as the output. +// +// 'password' is populated with the password string if the command was a success. +// An error Status object is returned otherwise. +Status GetPasswordFromShellCommand(const std::string& cmd, std::string* password); + +// A generic wrapper for OpenSSL structures. +template <typename T> +using c_unique_ptr = std::unique_ptr<T, std::function<void(T*)>>; + +// For each SSL type, the Traits class provides the important OpenSSL +// API functions. +template<typename SSL_TYPE> +struct SslTypeTraits {}; + +template<> struct SslTypeTraits<X509> { + static constexpr auto kFreeFunc = &X509_free; + static constexpr auto kReadPemFunc = &PEM_read_bio_X509; + static constexpr auto kReadDerFunc = &d2i_X509_bio; + static constexpr auto kWritePemFunc = &PEM_write_bio_X509; + static constexpr auto kWriteDerFunc = &i2d_X509_bio; +}; + +// SslTypeTraits functions for Type STACK_OF(X509) +STACK_OF(X509)* PEM_read_STACK_OF_X509(BIO* bio, void* /* unused */, + pem_password_cb* /* unused */, void* /* unused */); +int PEM_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj); +STACK_OF(X509)* DER_read_STACK_OF_X509(BIO* bio, void* /* unused */); +int DER_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj); +void free_STACK_OF_X509(STACK_OF(X509)* sk); + +template<> struct SslTypeTraits<STACK_OF(X509)> { + static constexpr auto kFreeFunc = &free_STACK_OF_X509; + static constexpr auto kReadPemFunc = &PEM_read_STACK_OF_X509; + static constexpr auto kReadDerFunc = &DER_read_STACK_OF_X509; + static constexpr auto kWritePemFunc = &PEM_write_STACK_OF_X509; + static constexpr auto kWriteDerFunc = &DER_write_STACK_OF_X509; +}; +template<> struct SslTypeTraits<X509_EXTENSION> { + static constexpr auto kFreeFunc = &X509_EXTENSION_free; +}; +template<> struct SslTypeTraits<X509_REQ> { + static constexpr auto kFreeFunc = &X509_REQ_free; + static constexpr auto kReadPemFunc = &PEM_read_bio_X509_REQ; + static constexpr auto kReadDerFunc = &d2i_X509_REQ_bio; + static constexpr auto kWritePemFunc = &PEM_write_bio_X509_REQ; + static constexpr auto kWriteDerFunc = &i2d_X509_REQ_bio; +}; +template<> struct SslTypeTraits<EVP_PKEY> { + static constexpr auto kFreeFunc = &EVP_PKEY_free; +}; +template<> struct SslTypeTraits<SSL_CTX> { + static constexpr auto kFreeFunc = &SSL_CTX_free; +}; + +template<typename SSL_TYPE, typename Traits = SslTypeTraits<SSL_TYPE>> +c_unique_ptr<SSL_TYPE> ssl_make_unique(SSL_TYPE* d) { + return {d, Traits::kFreeFunc}; +} + +// Acceptable formats for keys, X509 certificates and X509 CSRs. +enum class DataFormat { + DER = 0, // DER/ASN1 format (binary): for representing object on the wire + PEM = 1, // PEM format (ASCII): for storing on filesystem, printing, etc. +}; + +// Data format representation as a string. +const std::string& DataFormatToString(DataFormat fmt); + +// Template wrapper for dynamically allocated entities with custom deleter. +// Mostly, using it for xxx_st types from the OpenSSL crypto library. +template<typename Type> +class RawDataWrapper { + public: + typedef Type RawDataType; + + RawDataType* GetRawData() const { + return data_.get(); + } + + void AdoptRawData(RawDataType* d) { + data_ = ssl_make_unique(d); + } + + protected: + c_unique_ptr<RawDataType> data_; +}; + + +namespace internal { + +// Implementation of SCOPED_OPENSSL_NO_PENDING_ERRORS. Use the macro form +// instead of directly instantiating the implementation class. +struct ScopedCheckNoPendingSSLErrors { + public: + explicit ScopedCheckNoPendingSSLErrors(const char* func) + : func_(func) { + DCHECK_EQ(ERR_peek_error(), 0) + << "Expected no pending OpenSSL errors on " << func_ + << " entry, but had: " << GetOpenSSLErrors(); + } + ~ScopedCheckNoPendingSSLErrors() { + DCHECK_EQ(ERR_peek_error(), 0) + << "Expected no pending OpenSSL errors on " << func_ + << " exit, but had: " << GetOpenSSLErrors(); + } + + private: + const char* const func_; +}; + +} // namespace internal +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util_bio.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/openssl_util_bio.h b/be/src/kudu/security/openssl_util_bio.h new file mode 100644 index 0000000..c935b0b --- /dev/null +++ b/be/src/kudu/security/openssl_util_bio.h @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "kudu/security/openssl_util.h" + +#include <string> + +#include <glog/logging.h> +#include <openssl/bio.h> +#include <openssl/buffer.h> + +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/status.h" + +namespace kudu { +namespace security { + +template<> struct SslTypeTraits<BIO> { + static constexpr auto kFreeFunc = &BIO_free; +}; + +template<typename TYPE, typename Traits = SslTypeTraits<TYPE>> +Status ToBIO(BIO* bio, DataFormat format, TYPE* obj) { + CHECK(bio); + CHECK(obj); + switch (format) { + case DataFormat::DER: + OPENSSL_RET_NOT_OK(Traits::kWriteDerFunc(bio, obj), + "error exporting data in DER format"); + break; + case DataFormat::PEM: + OPENSSL_RET_NOT_OK(Traits::kWritePemFunc(bio, obj), + "error exporting data in PEM format"); + break; + } + OPENSSL_RET_NOT_OK(BIO_flush(bio), "error flushing BIO"); + return Status::OK(); +} + +// The callback which is called by the OpenSSL library when trying to decrypt +// a password protected private key. +inline int TLSPasswordCB(char* buf, int size, int /* rwflag */, void* userdata) { + const auto* cb = reinterpret_cast<const PasswordCallback*>(userdata); + std::string pw = (*cb)(); + if (pw.size() >= size) { + LOG(ERROR) << "Provided key password is longer than maximum length " + << size; + return -1; + } + strncpy(buf, pw.c_str(), size); + return pw.size(); +} + +template<typename TYPE, typename Traits = SslTypeTraits<TYPE>> +Status FromBIO(BIO* bio, DataFormat format, c_unique_ptr<TYPE>* ret, + const PasswordCallback& cb = PasswordCallback()) { + CHECK(bio); + switch (format) { + case DataFormat::DER: + *ret = ssl_make_unique(Traits::kReadDerFunc(bio, nullptr)); + break; + case DataFormat::PEM: + *ret = ssl_make_unique(Traits::kReadPemFunc(bio, nullptr, &TLSPasswordCB, + const_cast<PasswordCallback*>(&cb))); + break; + } + if (PREDICT_FALSE(!*ret)) { + return Status::RuntimeError(GetOpenSSLErrors()); + } + return Status::OK(); +} + +template<typename Type, typename Traits = SslTypeTraits<Type>> +Status FromString(const std::string& data, DataFormat format, + c_unique_ptr<Type>* ret) { + const void* mdata = reinterpret_cast<const void*>(data.data()); + auto bio = ssl_make_unique(BIO_new_mem_buf( +#if OPENSSL_VERSION_NUMBER < 0x10002000L + const_cast<void*>(mdata), +#else + mdata, +#endif + data.size())); + RETURN_NOT_OK_PREPEND((FromBIO<Type, Traits>(bio.get(), format, ret)), + "unable to load data from memory"); + return Status::OK(); +} + +template<typename Type, typename Traits = SslTypeTraits<Type>> +Status ToString(std::string* data, DataFormat format, Type* obj) { + CHECK(data); + auto bio = ssl_make_unique(BIO_new(BIO_s_mem())); + RETURN_NOT_OK_PREPEND((ToBIO<Type, Traits>(bio.get(), format, obj)), + "error serializing data"); + BUF_MEM* membuf; + OPENSSL_CHECK_OK(BIO_get_mem_ptr(bio.get(), &membuf)); + data->assign(membuf->data, membuf->length); + return Status::OK(); +} + +template<typename Type, typename Traits = SslTypeTraits<Type>> +Status FromFile(const std::string& fpath, DataFormat format, + c_unique_ptr<Type>* ret, const PasswordCallback& cb = PasswordCallback()) { + auto bio = ssl_make_unique(BIO_new(BIO_s_file())); + OPENSSL_RET_NOT_OK(BIO_read_filename(bio.get(), fpath.c_str()), + strings::Substitute("could not read data from file '$0'", fpath)); + RETURN_NOT_OK_PREPEND((FromBIO<Type, Traits>(bio.get(), format, ret, cb)), + strings::Substitute("unable to load data from file '$0'", fpath)); + return Status::OK(); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/security-test-util.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/security-test-util.cc b/be/src/kudu/security/security-test-util.cc new file mode 100644 index 0000000..40b0938 --- /dev/null +++ b/be/src/kudu/security/security-test-util.cc @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/security-test-util.h" + +#include <cstdint> +#include <string> + +#include <boost/optional/optional.hpp> + +#include "kudu/security/ca/cert_management.h" +#include "kudu/security/cert.h" +#include "kudu/security/crypto.h" +#include "kudu/security/test/test_certs.h" +#include "kudu/security/tls_context.h" +#include "kudu/util/test_util.h" + +namespace kudu { +namespace security { + +using ca::CaCertRequestGenerator; +using ca::CertSigner; + +Status GenerateSelfSignedCAForTests(PrivateKey* ca_key, Cert* ca_cert) { + static const int64_t kRootCaCertExpirationSeconds = 24 * 60 * 60; + // Create a key for the self-signed CA. + // + // OpenSSL has a concept of "security levels" which, amongst other things, + // place certain restrictions on key strength. OpenSSL 1.0 defaults to level + // 0 (no restrictions) while 1.1 defaults to level 1, which requires RSA keys + // to have at least 1024 bits. For simplicity, we'll just use 1024 bits here, + // even though shorter keys would decrease test running time. + // + // See https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_get_security_level.html + // for more details. + RETURN_NOT_OK(GeneratePrivateKey(1024, ca_key)); + + CaCertRequestGenerator::Config config = { "test-ca-cn" }; + RETURN_NOT_OK(CertSigner::SelfSignCA(*ca_key, + config, + kRootCaCertExpirationSeconds, + ca_cert)); + return Status::OK(); +} + +std::ostream& operator<<(std::ostream& o, PkiConfig c) { + switch (c) { + case PkiConfig::NONE: o << "NONE"; break; + case PkiConfig::SELF_SIGNED: o << "SELF_SIGNED"; break; + case PkiConfig::TRUSTED: o << "TRUSTED"; break; + case PkiConfig::SIGNED: o << "SIGNED"; break; + case PkiConfig::EXTERNALLY_SIGNED: o << "EXTERNALLY_SIGNED"; break; + } + return o; +} + +Status ConfigureTlsContext(PkiConfig config, + const Cert& ca_cert, + const PrivateKey& ca_key, + TlsContext* tls_context) { + switch (config) { + case PkiConfig::NONE: break; + case PkiConfig::SELF_SIGNED: + RETURN_NOT_OK(tls_context->GenerateSelfSignedCertAndKey()); + break; + case PkiConfig::TRUSTED: + RETURN_NOT_OK(tls_context->AddTrustedCertificate(ca_cert)); + break; + case PkiConfig::SIGNED: { + RETURN_NOT_OK(tls_context->AddTrustedCertificate(ca_cert)); + RETURN_NOT_OK(tls_context->GenerateSelfSignedCertAndKey()); + Cert cert; + RETURN_NOT_OK(CertSigner(&ca_cert, &ca_key).Sign(*tls_context->GetCsrIfNecessary(), &cert)); + RETURN_NOT_OK(tls_context->AdoptSignedCert(cert)); + break; + }; + case PkiConfig::EXTERNALLY_SIGNED: { + std::string cert_path, key_path; + // Write certificate and private key to file. + RETURN_NOT_OK(CreateTestSSLCertWithPlainKey(GetTestDataDirectory(), &cert_path, &key_path)); + RETURN_NOT_OK(tls_context->LoadCertificateAndKey(cert_path, key_path)); + RETURN_NOT_OK(tls_context->LoadCertificateAuthority(cert_path)); + }; + } + return Status::OK(); +} + +} // namespace security +} // namespace kudu