IMPALA-4669: [SECURITY] Import Kudu security library from kudu@314c9d8 The security library provides Kerberos and TLS facilities to the rpc library.
Change-Id: I76daeead00f672aa468f5ab6de4d70eac2078cb2 Reviewed-on: http://gerrit.cloudera.org:8080/5716 Reviewed-by: Henry Robinson <[email protected]> Tested-by: Henry Robinson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/84b8155c Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/84b8155c Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/84b8155c Branch: refs/heads/master Commit: 84b8155cc340f7235cc2915abbeba13a081a7890 Parents: 6757b62 Author: Henry Robinson <[email protected]> Authored: Fri Jan 13 03:29:38 2017 -0800 Committer: Henry Robinson <[email protected]> Committed: Tue Aug 15 00:45:44 2017 +0000 ---------------------------------------------------------------------- LICENSE.txt | 152 +++++ be/src/kudu/security/CMakeLists.txt | 117 ++++ be/src/kudu/security/ca/cert_management-test.cc | 290 ++++++++ be/src/kudu/security/ca/cert_management.cc | 422 ++++++++++++ be/src/kudu/security/ca/cert_management.h | 221 ++++++ be/src/kudu/security/cert-test.cc | 162 +++++ be/src/kudu/security/cert.cc | 251 +++++++ be/src/kudu/security/cert.h | 90 +++ be/src/kudu/security/crypto-test.cc | 256 +++++++ be/src/kudu/security/crypto.cc | 265 ++++++++ be/src/kudu/security/crypto.h | 94 +++ be/src/kudu/security/init.cc | 473 +++++++++++++ be/src/kudu/security/init.h | 66 ++ be/src/kudu/security/kerberos_util.cc | 36 + be/src/kudu/security/kerberos_util.h | 29 + be/src/kudu/security/krb5_realm_override.cc | 104 +++ be/src/kudu/security/openssl_util.cc | 211 ++++++ be/src/kudu/security/openssl_util.h | 190 ++++++ be/src/kudu/security/openssl_util_bio.h | 113 ++++ be/src/kudu/security/security-test-util.cc | 94 +++ be/src/kudu/security/security-test-util.h | 130 ++++ be/src/kudu/security/simple_acl.cc | 87 +++ be/src/kudu/security/simple_acl.h | 60 ++ be/src/kudu/security/test/mini_kdc-test.cc | 149 +++++ be/src/kudu/security/test/mini_kdc.cc | 390 +++++++++++ be/src/kudu/security/test/mini_kdc.h | 133 ++++ be/src/kudu/security/test/test_certs.cc | 396 +++++++++++ be/src/kudu/security/test/test_certs.h | 67 ++ be/src/kudu/security/test/test_pass.cc | 40 ++ be/src/kudu/security/test/test_pass.h | 33 + be/src/kudu/security/tls_context.cc | 459 +++++++++++++ be/src/kudu/security/tls_context.h | 182 +++++ be/src/kudu/security/tls_handshake-test.cc | 385 +++++++++++ be/src/kudu/security/tls_handshake.cc | 257 +++++++ be/src/kudu/security/tls_handshake.h | 166 +++++ be/src/kudu/security/tls_socket.cc | 157 +++++ be/src/kudu/security/tls_socket.h | 56 ++ be/src/kudu/security/token-test.cc | 666 +++++++++++++++++++ be/src/kudu/security/token.proto | 97 +++ be/src/kudu/security/token_signer.cc | 297 +++++++++ be/src/kudu/security/token_signer.h | 317 +++++++++ be/src/kudu/security/token_signing_key.cc | 109 +++ be/src/kudu/security/token_signing_key.h | 102 +++ be/src/kudu/security/token_verifier.cc | 164 +++++ be/src/kudu/security/token_verifier.h | 119 ++++ be/src/kudu/security/x509_check_host.cc | 441 ++++++++++++ be/src/kudu/security/x509_check_host.h | 48 ++ bin/rat_exclude_files.txt | 2 + cmake_modules/FindKerberos.cmake | 30 + 49 files changed, 9175 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt index da12783..01359d4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -768,3 +768,155 @@ Some portions of this module are derived from code from LevelDB -------------------------------------------------------------------------------- SHA-1 code from be/src/thirdparty/squeasel/squeasel.c: public domain + +-------------------------------------------------------------------------------- + +be/src/kudu/security/init.cc (some portions): MIT license + +Some portions of this module are derived from code from krb5 +(https://github.com/krb5/krb5/). + + Copyright 2000, 2007, 2008 by the Massachusetts Institute of Technology. + All Rights Reserved. + + Export of this software from the United States of America may require a specific license + from the United States Government. It is the responsibility of any person or + organization contemplating export to obtain such a license before exporting. + + WITHIN THAT CONSTRAINT, permission to use, copy, modify, and distribute this software + and its documentation for any purpose and without fee is hereby granted, provided that + the above copyright notice appear in all copies and that both that copyright notice and + this permission notice appear in supporting documentation, and that the name of + M.I.T. not be used in advertising or publicity pertaining to distribution of the + software without specific, written prior permission. Furthermore if you modify this + software you must label your software as modified software and not distribute it in such + a fashion that it might be confused with the original M.I.T. software. M.I.T. makes no + representations about the suitability of this software for any purpose. It is provided + "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +be/src/kudu/util/x509_check_host.*: OpenSSL software license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a dual license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + ==================================================================== + Copyright (c) 1998-2016 The OpenSSL Project. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. All advertising materials mentioning features or use of this + software must display the following acknowledgment: + "This product includes software developed by the OpenSSL Project + for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + + 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + endorse or promote products derived from this software without + prior written permission. For written permission, please contact + [email protected]. + + 5. Products derived from this software may not be called "OpenSSL" + nor may "OpenSSL" appear in their names without prior written + permission of the OpenSSL Project. + + 6. Redistributions of any form whatsoever must retain the following + acknowledgment: + "This product includes software developed by the OpenSSL Project + for use in the OpenSSL Toolkit (http://www.openssl.org/)" + + THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + ==================================================================== + + This product includes cryptographic software written by Eric Young + ([email protected]). This product includes software written by Tim + Hudson ([email protected]). + + + Original SSLeay License + ----------------------- + + Copyright (C) 1995-1998 Eric Young ([email protected]) + All rights reserved. + + This package is an SSL implementation written + by Eric Young ([email protected]). + The implementation was written so as to conform with Netscapes SSL. + + This library is free for commercial and non-commercial use as long as + the following conditions are aheared to. The following conditions + apply to all code found in this distribution, be it the RC4, RSA, + lhash, DES, etc., code; not just the SSL code. The SSL documentation + included with this distribution is covered by the same copyright terms + except that the holder is Tim Hudson ([email protected]). + + Copyright remains Eric Young's, and as such any Copyright notices in + the code are not to be removed. + If this package is used in a product, Eric Young should be given attribution + as the author of the parts of the library used. + This can be in the form of a textual message at program startup or + in documentation (online or textual) provided with the package. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. All advertising materials mentioning features or use of this software + must display the following acknowledgement: + "This product includes cryptographic software written by + Eric Young ([email protected])" + The word 'cryptographic' can be left out if the rouines from the library + being used are not cryptographic related :-). + 4. If you include any Windows specific code (or a derivative thereof) from + the apps directory (application code) you must include an acknowledgement: + "This product includes software written by Tim Hudson ([email protected])" + + THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + + The licence and distribution terms for any publically available version or + derivative of this code cannot be changed. i.e. this code cannot simply be + copied and put under another distribution licence + [including the GNU Public Licence.] + +-------------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/CMakeLists.txt b/be/src/kudu/security/CMakeLists.txt new file mode 100644 index 0000000..0dc7d0f --- /dev/null +++ b/be/src/kudu/security/CMakeLists.txt @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# See the comment in krb5_realm_override.cc for details on this library's usage. +# The top-level CMakeLists sets a ${KRB5_REALM_OVERRIDE} variable which should +# be linked first into all Kudu binaries. + +############################## +# krb5_realm_override +############################## + +add_library(krb5_realm_override STATIC krb5_realm_override.cc) +target_link_libraries(krb5_realm_override glog) +if(NOT APPLE) + target_link_libraries(krb5_realm_override dl) +endif() + +############################## +# token_proto +############################## + +PROTOBUF_GENERATE_CPP( + TOKEN_PROTO_SRCS TOKEN_PROTO_HDRS TOKEN_PROTO_TGTS + SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../.. + BINARY_ROOT ${CMAKE_CURRENT_BINARY_DIR}/../.. + PROTO_FILES token.proto) +set(TOKEN_PROTO_LIBS protobuf pb_util_proto) +ADD_EXPORTABLE_LIBRARY(token_proto + SRCS ${TOKEN_PROTO_SRCS} + DEPS ${TOKEN_PROTO_LIBS} + NONLINK_DEPS ${TOKEN_PROTO_TGTS}) + + +############################## +# security +############################## + +# Fall back to using the ported functionality if we're using an older version of OpenSSL. +if (${OPENSSL_VERSION} VERSION_LESS "1.0.2") + set(PORTED_X509_CHECK_HOST_CC "x509_check_host.cc") +endif() + +set(SECURITY_SRCS + ca/cert_management.cc + cert.cc + crypto.cc + kerberos_util.cc + init.cc + openssl_util.cc + ${PORTED_X509_CHECK_HOST_CC} + simple_acl.cc + tls_context.cc + tls_handshake.cc + tls_socket.cc + token_verifier.cc + token_signer.cc + token_signing_key.cc + ) + +set(SECURITY_LIBS + gutil + kudu_util + token_proto + + krb5 + openssl_crypto + openssl_ssl) + +ADD_EXPORTABLE_LIBRARY(security + SRCS ${SECURITY_SRCS} + DEPS ${SECURITY_LIBS}) + + +############################## +# security-test +############################## + +if (NOT NO_TESTS) + set(SECURITY_TEST_SRCS + security-test-util.cc + test/mini_kdc.cc + test/test_certs.cc test/test_pass.cc) + + add_library(security-test ${SECURITY_TEST_SRCS}) + target_link_libraries(security-test + gutil + kudu_test_util + kudu_util + security) + + # Tests + set(KUDU_TEST_LINK_LIBS + security + security-test + ${KUDU_MIN_TEST_LIBS}) + + ADD_KUDU_TEST(ca/cert_management-test) + ADD_KUDU_TEST(cert-test) + ADD_KUDU_TEST(crypto-test) + ADD_KUDU_TEST(test/mini_kdc-test) + ADD_KUDU_TEST(tls_handshake-test) + ADD_KUDU_TEST(token-test) +endif() http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/ca/cert_management-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/ca/cert_management-test.cc b/be/src/kudu/security/ca/cert_management-test.cc new file mode 100644 index 0000000..6e4662d --- /dev/null +++ b/be/src/kudu/security/ca/cert_management-test.cc @@ -0,0 +1,290 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/ca/cert_management.h" + +#include <memory> +#include <utility> +#include <vector> + +#include "kudu/gutil/strings/substitute.h" +#include "kudu/gutil/strings/util.h" +#include "kudu/security/cert.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/security-test-util.h" +#include "kudu/security/test/test_certs.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" + +using std::string; +using std::vector; +using strings::Substitute; + +namespace kudu { +namespace security { +namespace ca { + +class CertManagementTest : public KuduTest { + public: + void SetUp() override { + ASSERT_OK(ca_cert_.FromString(kCaCert, DataFormat::PEM)); + ASSERT_OK(ca_private_key_.FromString(kCaPrivateKey, DataFormat::PEM)); + ASSERT_OK(ca_public_key_.FromString(kCaPublicKey, DataFormat::PEM)); + ASSERT_OK(ca_exp_cert_.FromString(kCaExpiredCert, DataFormat::PEM)); + ASSERT_OK(ca_exp_private_key_.FromString(kCaExpiredPrivateKey, DataFormat::PEM)); + // Sanity checks. + ASSERT_OK(ca_cert_.CheckKeyMatch(ca_private_key_)); + ASSERT_OK(ca_exp_cert_.CheckKeyMatch(ca_exp_private_key_)); + } + + protected: + CertRequestGenerator::Config PrepareConfig( + const string& hostname = "localhost.localdomain") { + return { hostname }; + } + + CaCertRequestGenerator::Config PrepareCaConfig(const string& cn) { + return { cn }; + } + + // Create a new private key in 'key' and return a CSR associated with that + // key. + template<class CSRGen = CertRequestGenerator> + CertSignRequest PrepareTestCSR(typename CSRGen::Config config, + PrivateKey* key) { + CHECK_OK(GeneratePrivateKey(512, key)); + CSRGen gen(std::move(config)); + CHECK_OK(gen.Init()); + CertSignRequest req; + CHECK_OK(gen.GenerateRequest(*key, &req)); + return req; + } + + Cert ca_cert_; + PrivateKey ca_private_key_; + PublicKey ca_public_key_; + + Cert ca_exp_cert_; + PrivateKey ca_exp_private_key_; +}; + +// Check for basic constraints while initializing CertRequestGenerator objects. +TEST_F(CertManagementTest, RequestGeneratorConstraints) { + const CertRequestGenerator::Config gen_config = PrepareConfig(""); + CertRequestGenerator gen(gen_config); + const Status s = gen.Init(); + const string err_msg = s.ToString(); + ASSERT_TRUE(s.IsInvalidArgument()) << err_msg; + ASSERT_STR_CONTAINS(err_msg, "hostname must not be empty"); +} + +// Check for the basic functionality of the CertRequestGenerator class: +// check it's able to generate keys of expected number of bits and that it +// reports an error if trying to generate a key of unsupported number of bits. +TEST_F(CertManagementTest, RequestGeneratorBasics) { + const CertRequestGenerator::Config gen_config = PrepareConfig(); + + PrivateKey key; + ASSERT_OK(GeneratePrivateKey(1024, &key)); + CertRequestGenerator gen(gen_config); + ASSERT_OK(gen.Init()); + string key_str; + ASSERT_OK(key.ToString(&key_str, DataFormat::PEM)); + // Check for non-supported number of bits for the key. + Status s = GeneratePrivateKey(7, &key); + ASSERT_TRUE(s.IsRuntimeError()); +} + +// Check that CertSigner behaves in a predictable way if given non-matching +// CA private key and certificate. +TEST_F(CertManagementTest, SignerInitWithMismatchedCertAndKey) { + PrivateKey key; + const auto& csr = PrepareTestCSR(PrepareConfig(), &key); + { + Cert cert; + Status s = CertSigner(&ca_cert_, &ca_exp_private_key_) + .Sign(csr, &cert); + + const string err_msg = s.ToString(); + ASSERT_TRUE(s.IsRuntimeError()) << err_msg; + ASSERT_STR_CONTAINS(err_msg, "certificate does not match private key"); + } + { + Cert cert; + Status s = CertSigner(&ca_exp_cert_, &ca_private_key_) + .Sign(csr, &cert); + const string err_msg = s.ToString(); + ASSERT_TRUE(s.IsRuntimeError()) << err_msg; + ASSERT_STR_CONTAINS(err_msg, "certificate does not match private key"); + } +} + +// Check how CertSigner behaves if given expired CA certificate +// and corresponding private key. +TEST_F(CertManagementTest, SignerInitWithExpiredCert) { + const CertRequestGenerator::Config gen_config = PrepareConfig(); + PrivateKey key; + CertSignRequest req = PrepareTestCSR(gen_config, &key); + + // Signer works fine even with expired CA certificate. + Cert cert; + ASSERT_OK(CertSigner(&ca_exp_cert_, &ca_exp_private_key_).Sign(req, &cert)); + ASSERT_OK(cert.CheckKeyMatch(key)); +} + +// Generate X509 CSR and issue corresponding certificate putting the specified +// hostname into the SAN X509v3 extension field. The fix for KUDU-1981 addresses +// the issue of enabling Kudu server components on systems with FQDN longer than +// 64 characters. This test is a regression for KUDU-1981, so let's verify that +// CSRs and the result X509 cerificates with long hostnames in SAN are handled +// properly. +TEST_F(CertManagementTest, SignCertLongHostnameInSan) { + for (auto const& hostname : + { + "foo.bar.com", + + "222222222222222222222222222222222222222222222222222222222222222." + "555555555555555555555555555555555555555555555555555555555555555." + "555555555555555555555555555555555555555555555555555555555555555." + "chaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaars", + }) { + CertRequestGenerator::Config gen_config; + gen_config.hostname = hostname; + gen_config.user_id = "test-uid"; + PrivateKey key; + const auto& csr = PrepareTestCSR(gen_config, &key); + Cert cert; + ASSERT_OK(CertSigner(&ca_cert_, &ca_private_key_).Sign(csr, &cert)); + ASSERT_OK(cert.CheckKeyMatch(key)); + + EXPECT_EQ("C = US, ST = CA, O = MyCompany, CN = MyName, emailAddress = [email protected]", + cert.IssuerName()); + EXPECT_EQ("UID = test-uid", cert.SubjectName()); + vector<string> hostnames = cert.Hostnames(); + ASSERT_EQ(1, hostnames.size()); + EXPECT_EQ(hostname, hostnames[0]); + } +} + +// Generate X509 CSR and issues corresponding certificate. +TEST_F(CertManagementTest, SignCert) { + CertRequestGenerator::Config gen_config; + gen_config.hostname = "foo.bar.com"; + gen_config.user_id = "test-uid"; + gen_config.kerberos_principal = "kudu/[email protected]"; + PrivateKey key; + const auto& csr = PrepareTestCSR(gen_config, &key); + Cert cert; + ASSERT_OK(CertSigner(&ca_cert_, &ca_private_key_).Sign(csr, &cert)); + ASSERT_OK(cert.CheckKeyMatch(key)); + + EXPECT_EQ("C = US, ST = CA, O = MyCompany, CN = MyName, emailAddress = [email protected]", + cert.IssuerName()); + EXPECT_EQ("UID = test-uid", cert.SubjectName()); + EXPECT_EQ(gen_config.user_id, *cert.UserId()); + EXPECT_EQ(gen_config.kerberos_principal, *cert.KuduKerberosPrincipal()); + vector<string> hostnames = cert.Hostnames(); + ASSERT_EQ(1, hostnames.size()); + EXPECT_EQ("foo.bar.com", hostnames[0]); +} + +// Generate X509 CA CSR and sign the result certificate. +TEST_F(CertManagementTest, SignCaCert) { + const CaCertRequestGenerator::Config gen_config(PrepareCaConfig("self-ca")); + PrivateKey key; + const auto& csr = PrepareTestCSR<CaCertRequestGenerator>(gen_config, &key); + Cert cert; + ASSERT_OK(CertSigner(&ca_cert_, &ca_private_key_).Sign(csr, &cert)); + ASSERT_OK(cert.CheckKeyMatch(key)); +} + +// Test the creation and use of a CA which uses a self-signed CA cert +// generated on the fly. +TEST_F(CertManagementTest, TestSelfSignedCA) { + PrivateKey ca_key; + Cert ca_cert; + ASSERT_OK(GenerateSelfSignedCAForTests(&ca_key, &ca_cert)); + + // Create a key and CSR for the tablet server. + const auto& config = PrepareConfig(); + PrivateKey ts_key; + CertSignRequest ts_csr = PrepareTestCSR(config, &ts_key); + + // Sign it using the self-signed CA. + Cert ts_cert; + ASSERT_OK(CertSigner(&ca_cert, &ca_key).Sign(ts_csr, &ts_cert)); + ASSERT_OK(ts_cert.CheckKeyMatch(ts_key)); +} + +// Check the transformation chains for X509 CSRs: +// internal -> PEM -> internal -> PEM +// internal -> DER -> internal -> DER +TEST_F(CertManagementTest, X509CsrFromAndToString) { + static const DataFormat kFormats[] = { DataFormat::PEM, DataFormat::DER }; + + PrivateKey key; + ASSERT_OK(GeneratePrivateKey(1024, &key)); + CertRequestGenerator gen(PrepareConfig()); + ASSERT_OK(gen.Init()); + CertSignRequest req_ref; + ASSERT_OK(gen.GenerateRequest(key, &req_ref)); + + for (auto format : kFormats) { + SCOPED_TRACE(Substitute("X509 CSR format: $0", DataFormatToString(format))); + string str_req_ref; + ASSERT_OK(req_ref.ToString(&str_req_ref, format)); + CertSignRequest req; + ASSERT_OK(req.FromString(str_req_ref, format)); + string str_req; + ASSERT_OK(req.ToString(&str_req, format)); + ASSERT_EQ(str_req_ref, str_req); + } +} + +// Check the transformation chains for X509 certs: +// internal -> PEM -> internal -> PEM +// internal -> DER -> internal -> DER +TEST_F(CertManagementTest, X509FromAndToString) { + static const DataFormat kFormats[] = { DataFormat::PEM, DataFormat::DER }; + + PrivateKey key; + ASSERT_OK(GeneratePrivateKey(1024, &key)); + CertRequestGenerator gen(PrepareConfig()); + ASSERT_OK(gen.Init()); + CertSignRequest req; + ASSERT_OK(gen.GenerateRequest(key, &req)); + + Cert cert_ref; + ASSERT_OK(CertSigner(&ca_cert_, &ca_private_key_) + .Sign(req, &cert_ref)); + + for (auto format : kFormats) { + SCOPED_TRACE(Substitute("X509 format: $0", DataFormatToString(format))); + string str_cert_ref; + ASSERT_OK(cert_ref.ToString(&str_cert_ref, format)); + Cert cert; + ASSERT_OK(cert.FromString(str_cert_ref, format)); + string str_cert; + ASSERT_OK(cert.ToString(&str_cert, format)); + ASSERT_EQ(str_cert_ref, str_cert); + } +} + +} // namespace ca +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/ca/cert_management.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/ca/cert_management.cc b/be/src/kudu/security/ca/cert_management.cc new file mode 100644 index 0000000..a6d7f7b --- /dev/null +++ b/be/src/kudu/security/ca/cert_management.cc @@ -0,0 +1,422 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/ca/cert_management.h" + +#include <cstdio> +#include <cstdlib> +#include <functional> +#include <iostream> +#include <memory> +#include <sstream> +#include <string> +#include <type_traits> + +#include <glog/logging.h> +#include <openssl/conf.h> +#ifndef OPENSSL_NO_ENGINE +#include <openssl/engine.h> +#endif +#include <openssl/pem.h> +#include <openssl/x509.h> +#include <openssl/x509v3.h> + +#include "kudu/gutil/strings/substitute.h" +#include "kudu/security/cert.h" +#include "kudu/security/init.h" +#include "kudu/security/openssl_util.h" +#include "kudu/util/scoped_cleanup.h" +#include "kudu/util/status.h" + +using std::lock_guard; +using std::move; +using std::ostringstream; +using std::string; +using strings::Substitute; + +namespace kudu { +namespace security { + +template<> struct SslTypeTraits<ASN1_INTEGER> { + static constexpr auto free = &ASN1_INTEGER_free; +}; +template<> struct SslTypeTraits<BIGNUM> { + static constexpr auto free = &BN_free; +}; + +namespace ca { + +namespace { + +Status SetSubjectNameField(X509_NAME* name, + const char* field_code, + const string& field_value) { + CHECK(name); + CHECK(field_code); + OPENSSL_RET_NOT_OK(X509_NAME_add_entry_by_txt( + name, field_code, MBSTRING_ASC, + reinterpret_cast<const unsigned char*>(field_value.c_str()), -1, -1, 0), + Substitute("error setting subject field $0", field_code)); + return Status::OK(); +} + +} // anonymous namespace + +CertRequestGenerator::~CertRequestGenerator() { + sk_X509_EXTENSION_pop_free(extensions_, X509_EXTENSION_free); +} + +Status CertRequestGeneratorBase::GenerateRequest(const PrivateKey& key, + CertSignRequest* ret) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(ret); + CHECK(Initialized()); + auto req = ssl_make_unique(X509_REQ_new()); + OPENSSL_RET_NOT_OK(X509_REQ_set_pubkey(req.get(), key.GetRawData()), + "error setting X509 public key"); + + // Populate the subject field of the request. + RETURN_NOT_OK(SetSubject(req.get())); + + // Set necessary extensions into the request. + RETURN_NOT_OK(SetExtensions(req.get())); + + // And finally sign the result. + OPENSSL_RET_NOT_OK(X509_REQ_sign(req.get(), key.GetRawData(), EVP_sha256()), + "error signing X509 request"); + ret->AdoptRawData(req.release()); + + return Status::OK(); +} + +Status CertRequestGeneratorBase::PushExtension(stack_st_X509_EXTENSION* st, + int32_t nid, StringPiece value) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + auto ex = ssl_make_unique( + X509V3_EXT_conf_nid(nullptr, nullptr, nid, const_cast<char*>(value.data()))); + OPENSSL_RET_IF_NULL(ex, "error configuring extension"); + OPENSSL_RET_NOT_OK(sk_X509_EXTENSION_push(st, ex.release()), + "error pushing extension into the stack"); + return Status::OK(); +} + +CertRequestGenerator::CertRequestGenerator(Config config) + : CertRequestGeneratorBase(), + config_(std::move(config)) { +} + +Status CertRequestGenerator::Init() { + InitializeOpenSSL(); + SCOPED_OPENSSL_NO_PENDING_ERRORS; + + CHECK(!is_initialized_); + + // Build the SAN field using the specified hostname. In general, it might be + // multiple DNS hostnames in the field, but in our use-cases it's always one. + if (config_.hostname.empty()) { + return Status::InvalidArgument("hostname must not be empty"); + } + const string san_hosts = Substitute("DNS.0:$0", config_.hostname); + + extensions_ = sk_X509_EXTENSION_new_null(); + + // Permitted usages for the generated keys is set via X509 V3 + // standard/extended key usage attributes. + // See https://www.openssl.org/docs/man1.0.1/apps/x509v3_config.html + // for details. + + // The generated certificates are for using as TLS certificates for + // both client and server. + string usage = "critical,digitalSignature,keyEncipherment"; + if (for_self_signing_) { + // If we are generating a CSR for self-signing, then we need to + // add this keyUsage attribute. See https://s.apache.org/BFHk + usage += ",keyCertSign"; + } + + RETURN_NOT_OK(PushExtension(extensions_, NID_key_usage, usage)); + // The generated certificates should be good for authentication + // of a server to a client and vice versa: the intended users of the + // certificates are tablet servers which are going to talk to master + // and other tablet servers via TLS channels. + RETURN_NOT_OK(PushExtension(extensions_, NID_ext_key_usage, + "critical,serverAuth,clientAuth")); + + // The generated certificates are not intended to be used as CA certificates + // (i.e. they cannot be used to sign/issue certificates). + RETURN_NOT_OK(PushExtension(extensions_, NID_basic_constraints, + "critical,CA:FALSE")); + + if (config_.kerberos_principal) { + int nid = GetKuduKerberosPrincipalOidNid(); + RETURN_NOT_OK(PushExtension(extensions_, nid, + Substitute("ASN1:UTF8:$0", *config_.kerberos_principal))); + } + RETURN_NOT_OK(PushExtension(extensions_, NID_subject_alt_name, san_hosts)); + + is_initialized_ = true; + + return Status::OK(); +} + +bool CertRequestGenerator::Initialized() const { + return is_initialized_; +} + +Status CertRequestGenerator::SetSubject(X509_REQ* req) const { + if (config_.user_id) { + RETURN_NOT_OK(SetSubjectNameField(X509_REQ_get_subject_name(req), + "UID", *config_.user_id)); + } + return Status::OK(); +} + +Status CertRequestGenerator::SetExtensions(X509_REQ* req) const { + OPENSSL_RET_NOT_OK(X509_REQ_add_extensions(req, extensions_), + "error setting X509 request extensions"); + return Status::OK(); +} + +CaCertRequestGenerator::CaCertRequestGenerator(Config config) + : config_(std::move(config)), + extensions_(nullptr), + is_initialized_(false) { +} + +CaCertRequestGenerator::~CaCertRequestGenerator() { + sk_X509_EXTENSION_pop_free(extensions_, X509_EXTENSION_free); +} + +Status CaCertRequestGenerator::Init() { + InitializeOpenSSL(); + SCOPED_OPENSSL_NO_PENDING_ERRORS; + + lock_guard<simple_spinlock> guard(lock_); + if (is_initialized_) { + return Status::OK(); + } + if (config_.cn.empty()) { + return Status::InvalidArgument("missing CA service UUID/name"); + } + + extensions_ = sk_X509_EXTENSION_new_null(); + + // Permitted usages for the generated keys is set via X509 V3 + // standard/extended key usage attributes. + // See https://www.openssl.org/docs/man1.0.1/apps/x509v3_config.html + // for details. + + // The target ceritifcate is a CA certificate: it's for signing X509 certs. + RETURN_NOT_OK(PushExtension(extensions_, NID_key_usage, + "critical,keyCertSign")); + // The generated certificates are for the private CA service. + RETURN_NOT_OK(PushExtension(extensions_, NID_basic_constraints, + "critical,CA:TRUE")); + is_initialized_ = true; + + return Status::OK(); +} + +bool CaCertRequestGenerator::Initialized() const { + lock_guard<simple_spinlock> guard(lock_); + return is_initialized_; +} + +Status CaCertRequestGenerator::SetSubject(X509_REQ* req) const { + return SetSubjectNameField(X509_REQ_get_subject_name(req), "CN", config_.cn); +} + +Status CaCertRequestGenerator::SetExtensions(X509_REQ* req) const { + OPENSSL_RET_NOT_OK(X509_REQ_add_extensions(req, extensions_), + "error setting X509 request extensions"); + return Status::OK(); +} + +Status CertSigner::SelfSignCA(const PrivateKey& key, + CaCertRequestGenerator::Config config, + int64_t cert_expiration_seconds, + Cert* cert) { + // Generate a CSR for the CA. + CertSignRequest ca_csr; + { + CaCertRequestGenerator gen(std::move(config)); + RETURN_NOT_OK(gen.Init()); + RETURN_NOT_OK(gen.GenerateRequest(key, &ca_csr)); + } + + // Self-sign the CA's CSR. + return CertSigner(nullptr, &key) + .set_expiration_interval(MonoDelta::FromSeconds(cert_expiration_seconds)) + .Sign(ca_csr, cert); +} + +Status CertSigner::SelfSignCert(const PrivateKey& key, + CertRequestGenerator::Config config, + Cert* cert) { + // Generate a CSR. + CertSignRequest csr; + { + CertRequestGenerator gen(std::move(config)); + gen.enable_self_signing(); + RETURN_NOT_OK(gen.Init()); + RETURN_NOT_OK(gen.GenerateRequest(key, &csr)); + } + + // Self-sign the CSR with the key. + return CertSigner(nullptr, &key).Sign(csr, cert); +} + + +CertSigner::CertSigner(const Cert* ca_cert, + const PrivateKey* ca_private_key) + : ca_cert_(ca_cert), + ca_private_key_(ca_private_key) { + // Private key is required. + CHECK(ca_private_key_ && ca_private_key_->GetRawData()); + // The cert is optional, but if we have it, it should be initialized. + CHECK(!ca_cert_ || ca_cert_->GetRawData()); +} + +Status CertSigner::Sign(const CertSignRequest& req, Cert* ret) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + InitializeOpenSSL(); + CHECK(ret); + + // If we are not self-signing, then make sure that the provided CA + // cert and key match each other. Technically this would be programmer + // error since we're always using internally-generated CA certs, but + // this isn't a hot path so we'll keep the extra safety. + if (ca_cert_) { + RETURN_NOT_OK(ca_cert_->CheckKeyMatch(*ca_private_key_)); + } + auto x509 = ssl_make_unique(X509_new()); + RETURN_NOT_OK(FillCertTemplateFromRequest(req.GetRawData(), x509.get())); + RETURN_NOT_OK(DoSign(EVP_sha256(), exp_interval_sec_, x509.get())); + ret->AdoptRawData(x509.release()); + + return Status::OK(); +} + +// This is modeled after code in copy_extensions() function from +// $OPENSSL_ROOT/apps/apps.c with OpenSSL 1.0.2. +Status CertSigner::CopyExtensions(X509_REQ* req, X509* x) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(req); + CHECK(x); + STACK_OF(X509_EXTENSION)* exts = X509_REQ_get_extensions(req); + auto exts_cleanup = MakeScopedCleanup([&exts]() { + sk_X509_EXTENSION_pop_free(exts, X509_EXTENSION_free); + }); + for (size_t i = 0; i < sk_X509_EXTENSION_num(exts); ++i) { + X509_EXTENSION* ext = sk_X509_EXTENSION_value(exts, i); + ASN1_OBJECT* obj = X509_EXTENSION_get_object(ext); + int32_t idx = X509_get_ext_by_OBJ(x, obj, -1); + if (idx != -1) { + // If extension exits, delete all extensions of same type. + do { + auto tmpext = ssl_make_unique(X509_get_ext(x, idx)); + X509_delete_ext(x, idx); + idx = X509_get_ext_by_OBJ(x, obj, -1); + } while (idx != -1); + } + OPENSSL_RET_NOT_OK(X509_add_ext(x, ext, -1), "error adding extension"); + } + + return Status::OK(); +} + +Status CertSigner::FillCertTemplateFromRequest(X509_REQ* req, X509* tmpl) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(req); + if (!req->req_info || + !req->req_info->pubkey || + !req->req_info->pubkey->public_key || + !req->req_info->pubkey->public_key->data) { + return Status::RuntimeError("corrupted CSR: no public key"); + } + auto pub_key = ssl_make_unique(X509_REQ_get_pubkey(req)); + OPENSSL_RET_IF_NULL(pub_key, "error unpacking public key from CSR"); + const int rc = X509_REQ_verify(req, pub_key.get()); + if (rc < 0) { + return Status::RuntimeError("CSR signature verification error", + GetOpenSSLErrors()); + } + if (rc == 0) { + return Status::RuntimeError("CSR signature mismatch", + GetOpenSSLErrors()); + } + OPENSSL_RET_NOT_OK(X509_set_subject_name(tmpl, X509_REQ_get_subject_name(req)), + "error setting cert subject name"); + RETURN_NOT_OK(CopyExtensions(req, tmpl)); + OPENSSL_RET_NOT_OK(X509_set_pubkey(tmpl, pub_key.get()), + "error setting cert public key"); + return Status::OK(); +} + +Status CertSigner::DigestSign(const EVP_MD* md, EVP_PKEY* pkey, X509* x) { + OPENSSL_RET_NOT_OK(X509_sign(x, pkey, md), "error signing certificate"); + return Status::OK(); +} + +Status CertSigner::GenerateSerial(c_unique_ptr<ASN1_INTEGER>* ret) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + auto btmp = ssl_make_unique(BN_new()); + OPENSSL_RET_NOT_OK(BN_pseudo_rand(btmp.get(), 64, 0, 0), + "error generating random number"); + auto serial = ssl_make_unique(ASN1_INTEGER_new()); + OPENSSL_RET_IF_NULL(BN_to_ASN1_INTEGER(btmp.get(), serial.get()), + "error converting number into ASN1 representation"); + if (ret) { + ret->swap(serial); + } + return Status::OK(); +} + +Status CertSigner::DoSign(const EVP_MD* digest, int32_t exp_seconds, + X509* ret) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(ret); + + // Version 3 (v3) of X509 certificates. The integer value is one less + // than the version it represents. This is not a typo. :) + static const int kX509V3 = 2; + + // If we have a CA cert, then the CA is the issuer. + // Otherwise, we are self-signing so the target cert is also the issuer. + X509* issuer_cert = ca_cert_ ? ca_cert_->GetRawData() : ret; + X509_NAME* issuer_name = X509_get_subject_name(issuer_cert); + OPENSSL_RET_NOT_OK(X509_set_issuer_name(ret, issuer_name), + "error setting issuer name"); + c_unique_ptr<ASN1_INTEGER> serial; + RETURN_NOT_OK(GenerateSerial(&serial)); + // set version to v3 + OPENSSL_RET_NOT_OK(X509_set_version(ret, kX509V3), + "error setting cert version"); + OPENSSL_RET_NOT_OK(X509_set_serialNumber(ret, serial.get()), + "error setting cert serial"); + OPENSSL_RET_IF_NULL(X509_gmtime_adj(X509_get_notBefore(ret), 0L), + "error setting cert validity time"); + OPENSSL_RET_IF_NULL(X509_gmtime_adj(X509_get_notAfter(ret), exp_seconds), + "error setting cert expiration time"); + RETURN_NOT_OK(DigestSign(digest, ca_private_key_->GetRawData(), ret)); + + return Status::OK(); +} + +} // namespace ca +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/ca/cert_management.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/ca/cert_management.h b/be/src/kudu/security/ca/cert_management.h new file mode 100644 index 0000000..919caff --- /dev/null +++ b/be/src/kudu/security/ca/cert_management.h @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <functional> +#include <map> +#include <memory> +#include <mutex> +#include <string> +#include <vector> + +#include <boost/optional.hpp> + +#include "kudu/gutil/macros.h" +#include "kudu/gutil/strings/stringpiece.h" +#include "kudu/security/crypto.h" +#include "kudu/security/openssl_util.h" +#include "kudu/util/locks.h" +#include "kudu/util/monotime.h" +#include "kudu/util/status.h" + +// Forward declarations for the relevant OpenSSL typedefs +// in addition to openssl_util.h. +typedef struct asn1_string_st ASN1_INTEGER; +typedef struct env_md_st EVP_MD; +typedef struct rsa_st RSA; +typedef struct x509_st X509; +typedef struct X509_req_st X509_REQ; +struct stack_st_X509_EXTENSION; // STACK_OF(X509_EXTENSION) + +namespace kudu { +namespace security { + +class Cert; +class CertSignRequest; + +namespace ca { + +// Base utility class for issuing X509 CSRs. +class CertRequestGeneratorBase { + public: + CertRequestGeneratorBase() = default; + virtual ~CertRequestGeneratorBase() = default; + + virtual Status Init() = 0; + virtual bool Initialized() const = 0; + + // Generate X509 CSR using the specified key. To obtain the key, + // call the GeneratePrivateKey() function. + Status GenerateRequest(const PrivateKey& key, CertSignRequest* ret) const WARN_UNUSED_RESULT; + + protected: + // Push the specified extension into the stack provided. + static Status PushExtension(stack_st_X509_EXTENSION* st, + int32_t nid, + StringPiece value) WARN_UNUSED_RESULT; + + // Set the certificate-specific subject fields into the specified request. + virtual Status SetSubject(X509_REQ* req) const = 0; + + // Set the certificate-specific extensions into the specified request. + virtual Status SetExtensions(X509_REQ* req) const = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(CertRequestGeneratorBase); +}; + +// An utility class that facilitates issuing certificate signing requests +// (a.k.a. X509 CSRs). +class CertRequestGenerator : public CertRequestGeneratorBase { + public: + // Properties for the generated X509 CSR. The 'hostname' is for the name of + // the machine the requestor is to use the certificate at. Valid configuration + // should contain non-empty 'hostname' field. + struct Config { + // FQDN name to put into the 'DNS' fields of the subjectAltName extension. + std::string hostname; + // userId (UID) + boost::optional<std::string> user_id; + // Our custom extension which stores the full Kerberos principal for IPKI certs. + boost::optional<std::string> kerberos_principal; + }; + + // 'config' contains the properties to fill into the X509 attributes of the + // CSR. + explicit CertRequestGenerator(Config config); + ~CertRequestGenerator(); + + Status Init() override WARN_UNUSED_RESULT; + bool Initialized() const override; + + CertRequestGenerator& enable_self_signing() { + CHECK(!is_initialized_); + for_self_signing_ = true; + return *this; + } + + protected: + Status SetSubject(X509_REQ* req) const override WARN_UNUSED_RESULT; + Status SetExtensions(X509_REQ* req) const override WARN_UNUSED_RESULT; + + private: + const Config config_; + stack_st_X509_EXTENSION* extensions_ = nullptr; + bool is_initialized_ = false; + bool for_self_signing_ = false; +}; + +// An utility class that facilitates issuing of root CA self-signed certificate +// signing requests. +class CaCertRequestGenerator : public CertRequestGeneratorBase { + public: + // Properties for the generated X509 CA CSR. + struct Config { + // Common name (CN); e.g. 'master 239D6D2F-BDD2-4463-8933-78D9559C2124'. + // Don't put hostname/FQDN in here: for CA cert it does not make sense and + // it might be longer than 64 characters which is the limit specified + // by RFC5280. The limit is enforced by the OpenSSL library. + std::string cn; + }; + + explicit CaCertRequestGenerator(Config config); + ~CaCertRequestGenerator(); + + Status Init() override WARN_UNUSED_RESULT; + bool Initialized() const override; + + protected: + Status SetSubject(X509_REQ* req) const override WARN_UNUSED_RESULT; + Status SetExtensions(X509_REQ* req) const override WARN_UNUSED_RESULT; + + private: + const Config config_; + stack_st_X509_EXTENSION* extensions_; + mutable simple_spinlock lock_; + bool is_initialized_; // protected by lock_ +}; + +// An utility class for issuing and signing certificates. +// +// This is used in "fluent" style. For example: +// +// CHECK_OK(CertSigner(&my_ca_cert, &my_ca_key) +// .set_expiration_interval(MonoDelta::FromSeconds(3600)) +// .Sign(csr, &cert)); +// +// As such, this class is not guaranteed thread-safe. +class CertSigner { + public: + // Generate a self-signed certificate authority using the given key + // and CSR configuration. + static Status SelfSignCA(const PrivateKey& key, + CaCertRequestGenerator::Config config, + int64_t cert_expiration_seconds, + Cert* cert) WARN_UNUSED_RESULT; + + // Generate a self-signed certificate using the given key and CSR + // configuration. + static Status SelfSignCert(const PrivateKey& key, + CertRequestGenerator::Config config, + Cert* cert) WARN_UNUSED_RESULT; + + // Create a CertSigner. + // + // The given cert and key must stay valid for the lifetime of the + // cert signer. See class documentation above for recommended usage. + // + // 'ca_cert' may be nullptr in order to perform self-signing (though + // the SelfSignCA() static method above is recommended). + CertSigner(const Cert* ca_cert, const PrivateKey* ca_private_key); + ~CertSigner() = default; + + // Set the expiration interval for certs signed by this signer. + // This may be changed at any point. + CertSigner& set_expiration_interval(MonoDelta expiration) { + exp_interval_sec_ = expiration.ToSeconds(); + return *this; + } + + Status Sign(const CertSignRequest& req, Cert* ret) const WARN_UNUSED_RESULT; + + private: + + static Status CopyExtensions(X509_REQ* req, X509* x) WARN_UNUSED_RESULT; + static Status FillCertTemplateFromRequest(X509_REQ* req, X509* tmpl) WARN_UNUSED_RESULT; + static Status DigestSign(const EVP_MD* md, EVP_PKEY* pkey, X509* x) WARN_UNUSED_RESULT; + static Status GenerateSerial(c_unique_ptr<ASN1_INTEGER>* ret) WARN_UNUSED_RESULT; + + Status DoSign(const EVP_MD* digest, int32_t exp_seconds, X509 *ret) const WARN_UNUSED_RESULT; + + // The expiration interval of certs signed by this signer. + int32_t exp_interval_sec_ = 24 * 60 * 60; + + // The CA cert. null if this CertSigner is configured for self-signing. + const Cert* const ca_cert_; + + // The CA private key. If configured for self-signing, this is the + // private key associated with the target cert. + const PrivateKey* const ca_private_key_; + + DISALLOW_COPY_AND_ASSIGN(CertSigner); +}; + +} // namespace ca +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/cert-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/cert-test.cc b/be/src/kudu/security/cert-test.cc new file mode 100644 index 0000000..acd0f74 --- /dev/null +++ b/be/src/kudu/security/cert-test.cc @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <thread> +#include <utility> +#include <vector> + +#include <boost/optional.hpp> +#include <boost/optional/optional_io.hpp> + +#include "kudu/gutil/strings/strip.h" +#include "kudu/security/cert.h" +#include "kudu/security/crypto.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/test/test_certs.h" +#include "kudu/util/barrier.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" + +using std::pair; +using std::thread; +using std::vector; + +namespace kudu { +namespace security { + +// Test for various certificate-related functionality in the security library. +// These do not cover CA certificate mananagement part; check +// cert_management-test.cc for those. +class CertTest : public KuduTest { + public: + void SetUp() override { + ASSERT_OK(ca_cert_.FromString(kCaCert, DataFormat::PEM)); + ASSERT_OK(ca_private_key_.FromString(kCaPrivateKey, DataFormat::PEM)); + ASSERT_OK(ca_public_key_.FromString(kCaPublicKey, DataFormat::PEM)); + ASSERT_OK(ca_exp_cert_.FromString(kCaExpiredCert, DataFormat::PEM)); + ASSERT_OK(ca_exp_private_key_.FromString(kCaExpiredPrivateKey, + DataFormat::PEM)); + // Sanity checks. + ASSERT_OK(ca_cert_.CheckKeyMatch(ca_private_key_)); + ASSERT_OK(ca_exp_cert_.CheckKeyMatch(ca_exp_private_key_)); + } + + protected: + Cert ca_cert_; + PrivateKey ca_private_key_; + PublicKey ca_public_key_; + + Cert ca_exp_cert_; + PrivateKey ca_exp_private_key_; +}; + +// Regression test to make sure that GetKuduKerberosPrincipalOidNid is thread +// safe. OpenSSL 1.0.0's OBJ_create method is not thread safe. +TEST_F(CertTest, GetKuduKerberosPrincipalOidNidConcurrent) { + int kConcurrency = 16; + Barrier barrier(kConcurrency); + + vector<thread> threads; + for (int i = 0; i < kConcurrency; i++) { + threads.emplace_back([&] () { + barrier.Wait(); + CHECK_NE(NID_undef, GetKuduKerberosPrincipalOidNid()); + }); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +// Check input/output of the X509 certificates in PEM format. +TEST_F(CertTest, CertInputOutputPEM) { + const Cert& cert = ca_cert_; + string cert_str; + ASSERT_OK(cert.ToString(&cert_str, DataFormat::PEM)); + RemoveExtraWhitespace(&cert_str); + + string ca_input_cert(kCaCert); + RemoveExtraWhitespace(&ca_input_cert); + EXPECT_EQ(ca_input_cert, cert_str); +} + +// Check that Cert behaves in a predictable way if given invalid PEM data. +TEST_F(CertTest, CertInvalidInput) { + // Providing files which guaranteed to exists, but do not contain valid data. + // This is to make sure the init handles that situation correctly and + // does not choke on the wrong input data. + Cert c; + ASSERT_FALSE(c.FromFile("/bin/sh", DataFormat::PEM).ok()); +} + +// Check X509 certificate/private key matching: match cases. +TEST_F(CertTest, CertMatchesRsaPrivateKey) { + const pair<const Cert*, const PrivateKey*> cases[] = { + { &ca_cert_, &ca_private_key_ }, + { &ca_exp_cert_, &ca_exp_private_key_ }, + }; + for (const auto& e : cases) { + EXPECT_OK(e.first->CheckKeyMatch(*e.second)); + } +} + +// Check X509 certificate/private key matching: mismatch cases. +TEST_F(CertTest, CertMismatchesRsaPrivateKey) { + const pair<const Cert*, const PrivateKey*> cases[] = { + { &ca_cert_, &ca_exp_private_key_ }, + { &ca_exp_cert_, &ca_private_key_ }, + }; + for (const auto& e : cases) { + const Status s = e.first->CheckKeyMatch(*e.second); + EXPECT_TRUE(s.IsRuntimeError()) << s.ToString(); + ASSERT_STR_CONTAINS(s.ToString(), "certificate does not match private key"); + } +} + +TEST_F(CertTest, TestGetKuduSpecificFieldsWhenMissing) { + EXPECT_EQ(boost::none, ca_cert_.UserId()); + EXPECT_EQ(boost::none, ca_cert_.KuduKerberosPrincipal()); +} + +TEST_F(CertTest, DnsHostnameInSanField) { + const string hostname_foo_bar = "foo.bar.com"; + const string hostname_mega_giga = "mega.giga.io"; + const string hostname_too_long = + "toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo." + "looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo" + "oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo" + "oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo" + "oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo" + "oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo" + "ng.hostname.io"; + + Cert cert; + ASSERT_OK(cert.FromString(kCertDnsHostnamesInSan, DataFormat::PEM)); + + EXPECT_EQ("C = US, ST = CA, O = MyCompany, CN = MyName, emailAddress = [email protected]", + cert.IssuerName()); + vector<string> hostnames = cert.Hostnames(); + ASSERT_EQ(3, hostnames.size()); + EXPECT_EQ(hostname_mega_giga, hostnames[0]); + EXPECT_EQ(hostname_foo_bar, hostnames[1]); + EXPECT_EQ(hostname_too_long, hostnames[2]); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/cert.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/cert.cc b/be/src/kudu/security/cert.cc new file mode 100644 index 0000000..fa4c753 --- /dev/null +++ b/be/src/kudu/security/cert.cc @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/cert.h" + +#include <mutex> +#include <string> + +#include <openssl/evp.h> +#include <openssl/pem.h> +#include <openssl/x509.h> +#include <openssl/x509v3.h> + +#include <boost/optional.hpp> + +#include "kudu/security/crypto.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/openssl_util_bio.h" +#include "kudu/util/scoped_cleanup.h" +#include "kudu/util/status.h" + +using std::string; + +namespace kudu { +namespace security { + +template<> struct SslTypeTraits<GENERAL_NAMES> { + static constexpr auto free = &GENERAL_NAMES_free; +}; + +// This OID is generated via the UUID method. +static const char* kKuduKerberosPrincipalOidStr = "2.25.243346677289068076843480765133256509912"; + +string X509NameToString(X509_NAME* name) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(name); + auto bio = ssl_make_unique(BIO_new(BIO_s_mem())); + OPENSSL_CHECK_OK(X509_NAME_print_ex(bio.get(), name, 0, XN_FLAG_ONELINE)); + + BUF_MEM* membuf; + OPENSSL_CHECK_OK(BIO_get_mem_ptr(bio.get(), &membuf)); + return string(membuf->data, membuf->length); +} + +int GetKuduKerberosPrincipalOidNid() { + InitializeOpenSSL(); + static std::once_flag flag; + static int nid; + std::call_once(flag, [&] () { + nid = OBJ_create(kKuduKerberosPrincipalOidStr, "kuduPrinc", "kuduKerberosPrincipal"); + CHECK_NE(nid, NID_undef) << "failed to create kuduPrinc oid: " << GetOpenSSLErrors(); + }); + return nid; +} + +Status Cert::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString(data, format, &data_); +} + +Status Cert::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString(data, format, data_.get()); +} + +Status Cert::FromFile(const std::string& fpath, DataFormat format) { + return ::kudu::security::FromFile(fpath, format, &data_); +} + +string Cert::SubjectName() const { + return X509NameToString(X509_get_subject_name(data_.get())); +} + +string Cert::IssuerName() const { + return X509NameToString(X509_get_issuer_name(data_.get())); +} + +boost::optional<string> Cert::UserId() const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + X509_NAME* name = X509_get_subject_name(data_.get()); + char buf[1024]; + int len = X509_NAME_get_text_by_NID(name, NID_userId, buf, arraysize(buf)); + if (len < 0) return boost::none; + return string(buf, len); +} + +vector<string> Cert::Hostnames() const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + vector<string> result; + auto gens = ssl_make_unique(reinterpret_cast<GENERAL_NAMES*>(X509_get_ext_d2i( + data_.get(), NID_subject_alt_name, nullptr, nullptr))); + if (gens) { + for (int i = 0; i < sk_GENERAL_NAME_num(gens.get()); ++i) { + GENERAL_NAME* gen = sk_GENERAL_NAME_value(gens.get(), i); + if (gen->type != GEN_DNS) { + continue; + } + const ASN1_STRING* cstr = gen->d.dNSName; + if (cstr->type != V_ASN1_IA5STRING || cstr->data == nullptr) { + LOG(DFATAL) << "invalid DNS name in the SAN field"; + return {}; + } + result.emplace_back(reinterpret_cast<char*>(cstr->data), cstr->length); + } + } + return result; +} + +boost::optional<string> Cert::KuduKerberosPrincipal() const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + int idx = X509_get_ext_by_NID(data_.get(), GetKuduKerberosPrincipalOidNid(), -1); + if (idx < 0) return boost::none; + X509_EXTENSION* ext = X509_get_ext(data_.get(), idx); + ASN1_OCTET_STRING* octet_str = X509_EXTENSION_get_data(ext); + const unsigned char* octet_str_data = octet_str->data; + long len; // NOLINT(runtime/int) + int tag, xclass; + if (ASN1_get_object(&octet_str_data, &len, &tag, &xclass, octet_str->length) != 0 || + tag != V_ASN1_UTF8STRING) { + LOG(DFATAL) << "invalid extension value in cert " << SubjectName(); + return boost::none; + } + + return string(reinterpret_cast<const char*>(octet_str_data), len); +} + +Status Cert::CheckKeyMatch(const PrivateKey& key) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + OPENSSL_RET_NOT_OK(X509_check_private_key(data_.get(), key.GetRawData()), + "certificate does not match private key"); + return Status::OK(); +} + +Status Cert::GetServerEndPointChannelBindings(string* channel_bindings) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + // Find the signature type of the certificate. This corresponds to the digest + // (hash) algorithm, and the public key type which signed the cert. + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + int signature_nid = X509_get_signature_nid(data_.get()); +#else + // Older version of OpenSSL appear not to have a public way to get the + // signature digest method from a certificate. Instead, we reach into the + // 'private' internals. + int signature_nid = OBJ_obj2nid(data_->sig_alg->algorithm); +#endif + + // Retrieve the digest algorithm type. + int digest_nid; + int public_key_nid; + OBJ_find_sigid_algs(signature_nid, &digest_nid, &public_key_nid); + + // RFC 5929: if the certificate's signatureAlgorithm uses no hash functions or + // uses multiple hash functions, then this channel binding type's channel + // bindings are undefined at this time (updates to is channel binding type may + // occur to address this issue if it ever arises). + // + // TODO(dan): can the multiple hash function scenario actually happen? What + // does OBJ_find_sigid_algs do in that scenario? + if (digest_nid == NID_undef) { + return Status::NotSupported("server certificate has no signature digest (hash) algorithm"); + } + + // RFC 5929: if the certificate's signatureAlgorithm uses a single hash + // function, and that hash function is either MD5 [RFC1321] or SHA-1 + // [RFC3174], then use SHA-256 [FIPS-180-3]; + if (digest_nid == NID_md5 || digest_nid == NID_sha1) { + digest_nid = NID_sha256; + } + + const EVP_MD* md = EVP_get_digestbynid(digest_nid); + OPENSSL_RET_IF_NULL(md, "digest for nid not found"); + + // Create a digest BIO. All data written to the BIO will be sent through the + // digest (hash) function. The digest BIO requires a null BIO to writethrough to. + auto null_bio = ssl_make_unique(BIO_new(BIO_s_null())); + auto md_bio = ssl_make_unique(BIO_new(BIO_f_md())); + OPENSSL_RET_NOT_OK(BIO_set_md(md_bio.get(), md), "failed to set digest for BIO"); + BIO_push(md_bio.get(), null_bio.get()); + + // Write the cert to the digest BIO. + RETURN_NOT_OK(ToBIO(md_bio.get(), DataFormat::DER, data_.get())); + + // Read the digest from the BIO and append it to 'channel_bindings'. + char buf[EVP_MAX_MD_SIZE]; + int digest_len = BIO_gets(md_bio.get(), buf, sizeof(buf)); + OPENSSL_RET_NOT_OK(digest_len, "failed to get cert digest from BIO"); + channel_bindings->assign(buf, digest_len); + return Status::OK(); +} + +void Cert::AdoptAndAddRefRawData(X509* data) { +#if OPENSSL_VERSION_NUMBER < 0x10100000L + CHECK_GT(CRYPTO_add(&data->references, 1, CRYPTO_LOCK_X509), 1) << "X509 use-after-free detected"; +#else + OPENSSL_CHECK_OK(X509_up_ref(data)) << "X509 use-after-free detected: " << GetOpenSSLErrors(); +#endif + AdoptRawData(data); +} + +Status Cert::GetPublicKey(PublicKey* key) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + EVP_PKEY* raw_key = X509_get_pubkey(data_.get()); + OPENSSL_RET_IF_NULL(raw_key, "unable to get certificate public key"); + key->AdoptRawData(raw_key); + return Status::OK(); +} + +Status CertSignRequest::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString(data, format, &data_); +} + +Status CertSignRequest::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString(data, format, data_.get()); +} + +Status CertSignRequest::FromFile(const std::string& fpath, DataFormat format) { + return ::kudu::security::FromFile(fpath, format, &data_); +} + +CertSignRequest CertSignRequest::Clone() const { + CHECK_GT(CRYPTO_add(&data_->references, 1, CRYPTO_LOCK_X509_REQ), 1) + << "X509_REQ use-after-free detected"; + + CertSignRequest clone; + clone.AdoptRawData(GetRawData()); + return clone; +} + +Status CertSignRequest::GetPublicKey(PublicKey* key) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + EVP_PKEY* raw_key = X509_REQ_get_pubkey(data_.get()); + OPENSSL_RET_IF_NULL(raw_key, "unable to get CSR public key"); + key->AdoptRawData(raw_key); + return Status::OK(); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/cert.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/cert.h b/be/src/kudu/security/cert.h new file mode 100644 index 0000000..2facb2c --- /dev/null +++ b/be/src/kudu/security/cert.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <string> +#include <vector> + +#include <boost/optional/optional_fwd.hpp> + +#include "kudu/security/openssl_util.h" + +namespace kudu { + +class Status; + +namespace security { + +class PrivateKey; +class PublicKey; + +// Convert an X509_NAME object to a human-readable string. +std::string X509NameToString(X509_NAME* name); + +// Return the OpenSSL NID for the custom X509 extension where we store +// our Kerberos principal in IPKI certs. +int GetKuduKerberosPrincipalOidNid(); + +class Cert : public RawDataWrapper<X509> { + public: + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT; + + std::string SubjectName() const; + std::string IssuerName() const; + + // Return DNS names from the SAN extension field. + std::vector<std::string> Hostnames() const; + + // Return the 'userId' extension of this cert, if set. + boost::optional<std::string> UserId() const; + + // Return the Kerberos principal encoded in this certificate, if set. + boost::optional<std::string> KuduKerberosPrincipal() const; + + // Check whether the specified private key matches the certificate. + // Return Status::OK() if key match the certificate. + Status CheckKeyMatch(const PrivateKey& key) const WARN_UNUSED_RESULT; + + // Returns the 'tls-server-end-point' channel bindings for the certificate as + // specified in RFC 5929. + Status GetServerEndPointChannelBindings(std::string* channel_bindings) const WARN_UNUSED_RESULT; + + // Adopts the provided X509 certificate, and increments the reference count. + void AdoptAndAddRefRawData(X509* data); + + // Returns the certificate's public key. + Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT; +}; + +class CertSignRequest : public RawDataWrapper<X509_REQ> { + public: + Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT; + Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT; + Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT; + + // Returns a shallow clone of the CSR (only a reference count is incremented). + CertSignRequest Clone() const; + + // Returns the CSR's public key. + Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT; +}; + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/crypto-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/crypto-test.cc b/be/src/kudu/security/crypto-test.cc new file mode 100644 index 0000000..87399e4 --- /dev/null +++ b/be/src/kudu/security/crypto-test.cc @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <cstring> +#include <functional> +#include <mutex> +#include <utility> +#include <vector> + +#include "kudu/gutil/strings/strip.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/gutil/strings/util.h" +#include "kudu/security/crypto.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/test/test_certs.h" +#include "kudu/util/env.h" +#include "kudu/util/path_util.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" +#include "kudu/util/url-coding.h" + +using std::pair; +using std::string; +using std::vector; +using strings::Substitute; + +namespace kudu { +namespace security { + +// Test for various crypto-related functionality in the security library. +class CryptoTest : public KuduTest { + public: + CryptoTest() : + pem_dir_(GetTestPath("pem")), + private_key_file_(JoinPathSegments(pem_dir_, "private_key.pem")), + public_key_file_(JoinPathSegments(pem_dir_, "public_key.pem")), + corrupted_private_key_file_(JoinPathSegments(pem_dir_, + "corrupted.private_key.pem")), + corrupted_public_key_file_(JoinPathSegments(pem_dir_, + "corrupted.public_key.pem")) { + } + + void SetUp() override { + ASSERT_OK(env_->CreateDir(pem_dir_)); + ASSERT_OK(WriteStringToFile(env_, kCaPrivateKey, private_key_file_)); + ASSERT_OK(WriteStringToFile(env_, kCaPublicKey, public_key_file_)); + ASSERT_OK(WriteStringToFile(env_, + string(kCaPrivateKey, strlen(kCaPrivateKey) / 2), + corrupted_private_key_file_)); + ASSERT_OK(WriteStringToFile(env_, + string(kCaPublicKey, strlen(kCaPublicKey) / 2), + corrupted_public_key_file_)); + } + + protected: + template<typename Key> + void CheckToAndFromString(const Key& key_ref, DataFormat format) { + SCOPED_TRACE(Substitute("ToAndFromString for $0 format", + DataFormatToString(format))); + string key_ref_str; + ASSERT_OK(key_ref.ToString(&key_ref_str, format)); + Key key; + ASSERT_OK(key.FromString(key_ref_str, format)); + string key_str; + ASSERT_OK(key.ToString(&key_str, format)); + ASSERT_EQ(key_ref_str, key_str); + } + + const string pem_dir_; + + const string private_key_file_; + const string public_key_file_; + const string corrupted_private_key_file_; + const string corrupted_public_key_file_; +}; + +// Check input/output of RSA private keys in PEM format. +TEST_F(CryptoTest, RsaPrivateKeyInputOutputPEM) { + PrivateKey key; + ASSERT_OK(key.FromFile(private_key_file_, DataFormat::PEM)); + string key_str; + ASSERT_OK(key.ToString(&key_str, DataFormat::PEM)); + RemoveExtraWhitespace(&key_str); + + string ref_key_str(kCaPrivateKey); + RemoveExtraWhitespace(&ref_key_str); + EXPECT_EQ(ref_key_str, key_str); +} + +// Check input of corrupted RSA private keys in PEM format. +TEST_F(CryptoTest, CorruptedRsaPrivateKeyInputPEM) { + static const string kFiles[] = { + corrupted_private_key_file_, + public_key_file_, + corrupted_public_key_file_, + "/bin/sh" + }; + for (const auto& file : kFiles) { + PrivateKey key; + const Status s = key.FromFile(file, DataFormat::PEM); + EXPECT_TRUE(s.IsRuntimeError()) << s.ToString(); + } +} + +// Check input/output of RSA public keys in PEM format. +TEST_F(CryptoTest, RsaPublicKeyInputOutputPEM) { + PublicKey key; + ASSERT_OK(key.FromFile(public_key_file_, DataFormat::PEM)); + string key_str; + ASSERT_OK(key.ToString(&key_str, DataFormat::PEM)); + RemoveExtraWhitespace(&key_str); + + string ref_key_str(kCaPublicKey); + RemoveExtraWhitespace(&ref_key_str); + EXPECT_EQ(ref_key_str, key_str); +} + +// Check input of corrupted RSA public keys in PEM format. +TEST_F(CryptoTest, CorruptedRsaPublicKeyInputPEM) { + static const string kFiles[] = { + corrupted_public_key_file_, + private_key_file_, + corrupted_private_key_file_, + "/bin/sh" + }; + for (const auto& file : kFiles) { + PublicKey key; + const Status s = key.FromFile(file, DataFormat::PEM); + EXPECT_TRUE(s.IsRuntimeError()) << s.ToString(); + } +} + +// Check extraction of the public part from RSA private keys par. +TEST_F(CryptoTest, RsaExtractPublicPartFromPrivateKey) { + // Load the reference RSA private key. + PrivateKey private_key; + ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM)); + + PublicKey public_key; + ASSERT_OK(private_key.GetPublicKey(&public_key)); + string str_public_key; + ASSERT_OK(public_key.ToString(&str_public_key, DataFormat::PEM)); + RemoveExtraWhitespace(&str_public_key); + + string ref_str_public_key(kCaPublicKey); + RemoveExtraWhitespace(&ref_str_public_key); + EXPECT_EQ(ref_str_public_key, str_public_key); +} + +class CryptoKeySerDesTest : + public CryptoTest, + public ::testing::WithParamInterface<DataFormat> { +}; + +// Check the transformation chains for RSA public/private keys: +// internal -> PEM -> internal -> PEM +// internal -> DER -> internal -> DER +TEST_P(CryptoKeySerDesTest, ToAndFromString) { + const auto format = GetParam(); + + // Generate private RSA key. + PrivateKey private_key; + ASSERT_OK(GeneratePrivateKey(2048, &private_key)); + NO_FATALS(CheckToAndFromString(private_key, format)); + + // Extract public part of the key. + PublicKey public_key; + ASSERT_OK(private_key.GetPublicKey(&public_key)); + NO_FATALS(CheckToAndFromString(public_key, format)); +} + +INSTANTIATE_TEST_CASE_P( + DataFormats, CryptoKeySerDesTest, + ::testing::Values(DataFormat::DER, DataFormat::PEM)); + +// Check making crypto signatures against the reference data. +TEST_F(CryptoTest, MakeVerifySignatureRef) { + static const vector<pair<string, string>> kRefElements = { + { kDataTiny, kSignatureTinySHA512 }, + { kDataShort, kSignatureShortSHA512 }, + { kDataLong, kSignatureLongSHA512 }, + }; + + // Load the reference RSA private key. + PrivateKey private_key; + ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM)); + + // Load the reference RSA public key. + PublicKey public_key; + ASSERT_OK(public_key.FromString(kCaPublicKey, DataFormat::PEM)); + + for (const auto& e : kRefElements) { + string sig; + ASSERT_OK(private_key.MakeSignature(DigestType::SHA512, e.first, &sig)); + + // Ad-hoc verification: check the produced signature matches the reference. + string sig_base64; + Base64Encode(sig, &sig_base64); + EXPECT_EQ(e.second, sig_base64); + + // Verify the signature cryptographically. + EXPECT_OK(public_key.VerifySignature(DigestType::SHA512, e.first, sig)); + } +} + +TEST_F(CryptoTest, VerifySignatureWrongData) { + static const vector<string> kRefSignatures = { + kSignatureTinySHA512, + kSignatureShortSHA512, + kSignatureLongSHA512, + }; + + // Load the reference RSA public key. + PublicKey key; + ASSERT_OK(key.FromString(kCaPublicKey, DataFormat::PEM)); + + for (const auto& e : kRefSignatures) { + string signature; + ASSERT_TRUE(Base64Decode(e, &signature)); + Status s = key.VerifySignature(DigestType::SHA512, + "non-expected-data", signature); + EXPECT_TRUE(s.IsCorruption()) << s.ToString(); + } +} + +TEST_F(CryptoTest, TestGenerateNonce) { + string nonce; + ASSERT_OK(GenerateNonce(&nonce)); + + // Do some basic validation on the returned nonce. + ASSERT_EQ(kNonceSize, nonce.size()); + ASSERT_NE(string(kNonceSize, '\0'), nonce); + + // Nonces should be unique, by definition. + string another_nonce; + ASSERT_OK(GenerateNonce(&another_nonce)); + ASSERT_NE(nonce, another_nonce); +} + +} // namespace security +} // namespace kudu http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/84b8155c/be/src/kudu/security/crypto.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/security/crypto.cc b/be/src/kudu/security/crypto.cc new file mode 100644 index 0000000..1aab6b1 --- /dev/null +++ b/be/src/kudu/security/crypto.cc @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/crypto.h" + +#include <cstdio> +#include <cstdlib> +#include <string> + +#include <glog/logging.h> +#include <openssl/bio.h> +#include <openssl/evp.h> +#include <openssl/pem.h> +#include <openssl/rand.h> + +#include "kudu/gutil/strings/substitute.h" +#include "kudu/security/openssl_util.h" +#include "kudu/security/openssl_util_bio.h" +#include "kudu/util/status.h" + +using std::string; +using strings::Substitute; + +namespace kudu { +namespace security { + +const size_t kNonceSize = 16; + +namespace { + +// Writing the private key from an EVP_PKEY has a different +// signature than the rest of the write functions, so we +// have to provide this wrapper. +int PemWritePrivateKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return PEM_write_bio_RSAPrivateKey( + bio, rsa.get(), nullptr, nullptr, 0, nullptr, nullptr); +} + +int PemWritePublicKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return PEM_write_bio_RSA_PUBKEY(bio, rsa.get()); +} + +int DerWritePublicKey(BIO* bio, EVP_PKEY* key) { + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key)); + return i2d_RSA_PUBKEY_bio(bio, rsa.get()); +} + +} // anonymous namespace + +template<> struct SslTypeTraits<BIGNUM> { + static constexpr auto free = &BN_free; +}; +struct RsaPrivateKeyTraits : public SslTypeTraits<EVP_PKEY> { + static constexpr auto read_pem = &PEM_read_bio_PrivateKey; + static constexpr auto read_der = &d2i_PrivateKey_bio; + static constexpr auto write_pem = &PemWritePrivateKey; + static constexpr auto write_der = &i2d_PrivateKey_bio; +}; +struct RsaPublicKeyTraits : public SslTypeTraits<EVP_PKEY> { + static constexpr auto read_pem = &PEM_read_bio_PUBKEY; + static constexpr auto read_der = &d2i_PUBKEY_bio; + static constexpr auto write_pem = &PemWritePublicKey; + static constexpr auto write_der = &DerWritePublicKey; +}; +template<> struct SslTypeTraits<RSA> { + static constexpr auto free = &RSA_free; +}; +template<> struct SslTypeTraits<EVP_MD_CTX> { + static constexpr auto free = &EVP_MD_CTX_destroy; +}; + +namespace { + +const EVP_MD* GetMessageDigest(DigestType digest_type) { + switch (digest_type) { + case DigestType::SHA256: return EVP_sha256(); + case DigestType::SHA512: return EVP_sha512(); + } + LOG(FATAL) << "unknown digest type"; +} + +} // anonymous namespace + + +Status PublicKey::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString<RawDataType, RsaPublicKeyTraits>( + data, format, &data_); +} + +Status PublicKey::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString<RawDataType, RsaPublicKeyTraits>( + data, format, data_.get()); +} + +Status PublicKey::FromFile(const std::string& fpath, DataFormat format) { + return ::kudu::security::FromFile<RawDataType, RsaPublicKeyTraits>( + fpath, format, &data_); +} + +Status PublicKey::FromBIO(BIO* bio, DataFormat format) { + return ::kudu::security::FromBIO<RawDataType, RsaPublicKeyTraits>( + bio, format, &data_); +} + +// Modeled after code in $OPENSSL_ROOT/apps/dgst.c +Status PublicKey::VerifySignature(DigestType digest, + const std::string& data, + const std::string& signature) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + const EVP_MD* md = GetMessageDigest(digest); + auto md_ctx = ssl_make_unique(EVP_MD_CTX_create()); + + OPENSSL_RET_NOT_OK(EVP_DigestVerifyInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()), + "error initializing verification digest"); + OPENSSL_RET_NOT_OK(EVP_DigestVerifyUpdate(md_ctx.get(), data.data(), data.size()), + "error verifying data signature"); +#if OPENSSL_VERSION_NUMBER < 0x10002000L + unsigned char* sig_data = reinterpret_cast<unsigned char*>( + const_cast<char*>(signature.data())); +#else + const unsigned char* sig_data = reinterpret_cast<const unsigned char*>( + signature.data()); +#endif + // The success is indicated by return code 1. All other values means + // either wrong signature or error while performing signature verification. + const int rc = EVP_DigestVerifyFinal(md_ctx.get(), sig_data, signature.size()); + if (rc < 0 || rc > 1) { + return Status::RuntimeError( + Substitute("error verifying data signature: $0", GetOpenSSLErrors())); + } + if (rc == 0) { + // No sense stringifying the internal OpenSSL error, since a bad verification + // is self-explanatory. + ERR_clear_error(); + return Status::Corruption("data signature verification failed"); + } + + return Status::OK(); +} + +Status PublicKey::Equals(const PublicKey& other, bool* equals) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + int cmp = EVP_PKEY_cmp(data_.get(), other.data_.get()); + switch (cmp) { + case -2: + return Status::NotSupported("failed to compare public keys"); + case -1: // Key types are different; treat this as not equal + case 0: // Keys are not equal + *equals = false; + return Status::OK(); + case 1: + *equals = true; + return Status::OK(); + default: + return Status::RuntimeError("unexpected public key comparison result", std::to_string(cmp)); + } +} + +Status PrivateKey::FromString(const std::string& data, DataFormat format) { + return ::kudu::security::FromString<RawDataType, RsaPrivateKeyTraits>( + data, format, &data_); +} + +Status PrivateKey::ToString(std::string* data, DataFormat format) const { + return ::kudu::security::ToString<RawDataType, RsaPrivateKeyTraits>( + data, format, data_.get()); +} + +Status PrivateKey::FromFile(const std::string& fpath, DataFormat format) { + return ::kudu::security::FromFile<RawDataType, RsaPrivateKeyTraits>( + fpath, format, &data_); +} + +// The code is modeled after $OPENSSL_ROOT/apps/rsa.c code: there is +// corresponding functionality to read public part from RSA private/public +// keypair. +Status PrivateKey::GetPublicKey(PublicKey* public_key) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(public_key); + auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(CHECK_NOTNULL(data_.get()))); + if (PREDICT_FALSE(!rsa)) { + return Status::RuntimeError(GetOpenSSLErrors()); + } + auto tmp = ssl_make_unique(BIO_new(BIO_s_mem())); + CHECK(tmp); + // Export public key in DER format into the temporary buffer. + OPENSSL_RET_NOT_OK(i2d_RSA_PUBKEY_bio(tmp.get(), rsa.get()), + "error extracting public RSA key"); + // Read the public key into the result placeholder. + RETURN_NOT_OK(public_key->FromBIO(tmp.get(), DataFormat::DER)); + + return Status::OK(); +} + +// Modeled after code in $OPENSSL_ROOT/apps/dgst.c +Status PrivateKey::MakeSignature(DigestType digest, + const std::string& data, + std::string* signature) const { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(signature); + const EVP_MD* md = GetMessageDigest(digest); + auto md_ctx = ssl_make_unique(EVP_MD_CTX_create()); + + OPENSSL_RET_NOT_OK(EVP_DigestSignInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()), + "error initializing signing digest"); + OPENSSL_RET_NOT_OK(EVP_DigestSignUpdate(md_ctx.get(), data.data(), data.size()), + "error signing data"); + size_t sig_len = EVP_PKEY_size(GetRawData()); + static const size_t kSigBufSize = 4 * 1024; + CHECK(sig_len <= kSigBufSize); + unsigned char buf[kSigBufSize]; + OPENSSL_RET_NOT_OK(EVP_DigestSignFinal(md_ctx.get(), buf, &sig_len), + "error finalizing data signature"); + *signature = string(reinterpret_cast<char*>(buf), sig_len); + + return Status::OK(); +} + +Status GeneratePrivateKey(int num_bits, PrivateKey* ret) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK(ret); + InitializeOpenSSL(); + auto key = ssl_make_unique(EVP_PKEY_new()); + { + auto bn = ssl_make_unique(BN_new()); + OPENSSL_CHECK_OK(BN_set_word(bn.get(), RSA_F4)); + auto rsa = ssl_make_unique(RSA_new()); + OPENSSL_RET_NOT_OK( + RSA_generate_key_ex(rsa.get(), num_bits, bn.get(), nullptr), + "error generating RSA key"); + OPENSSL_RET_NOT_OK( + EVP_PKEY_set1_RSA(key.get(), rsa.get()), "error assigning RSA key"); + } + ret->AdoptRawData(key.release()); + + return Status::OK(); +} + +Status GenerateNonce(string* s) { + SCOPED_OPENSSL_NO_PENDING_ERRORS; + CHECK_NOTNULL(s); + unsigned char buf[kNonceSize]; + OPENSSL_RET_NOT_OK(RAND_bytes(buf, sizeof(buf)), "failed to generate nonce"); + s->assign(reinterpret_cast<char*>(buf), kNonceSize); + return Status::OK(); +} + +} // namespace security +} // namespace kudu
