Repository: kudu Updated Branches: refs/heads/master 27992c120 -> 7138468a5
MiniKdc for C++ Change-Id: I63fc53eeaa1e40b217030adc1ca0c132f43a076c Reviewed-on: http://gerrit.cloudera.org:8080/4752 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7138468a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7138468a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7138468a Branch: refs/heads/master Commit: 7138468a567962dd0bbf9419df4c0b853660b913 Parents: 27992c1 Author: Dan Burkert <[email protected]> Authored: Fri Oct 14 14:58:42 2016 -0700 Committer: Adar Dembo <[email protected]> Committed: Thu Oct 20 21:53:26 2016 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 33 +- cmake_modules/FindKdc.cmake | 36 +++ src/kudu/integration-tests/CMakeLists.txt | 2 +- src/kudu/integration-tests/delete_table-test.cc | 2 +- .../integration-tests/master_failover-itest.cc | 2 +- .../integration-tests/master_migration-itest.cc | 2 +- src/kudu/security/CMakeLists.txt | 33 ++ src/kudu/security/mini_kdc-test.cc | 51 ++++ src/kudu/security/mini_kdc.cc | 304 +++++++++++++++++++ src/kudu/security/mini_kdc.h | 103 +++++++ src/kudu/tools/kudu-admin-test.cc | 8 +- src/kudu/tools/kudu-tool-test.cc | 2 +- src/kudu/tools/kudu-ts-cli-test.cc | 6 +- src/kudu/util/net/net_util.cc | 2 +- src/kudu/util/subprocess-test.cc | 17 +- src/kudu/util/subprocess.cc | 12 +- src/kudu/util/subprocess.h | 6 +- 17 files changed, 588 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 60e67d4..2d3391e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -902,6 +902,12 @@ if (NOT APPLE) SHARED_LIB "${DL_LIB_PATH}") endif() +## Kerberos +if (NOT NO_TESTS) + ## We rely on the Kerberos KDC binary for testing security. + find_package(Kdc) +endif() + ## Boost # The boost-cmake project hasn't been maintained for years. Let's make sure we @@ -1045,22 +1051,23 @@ endif (UNIX) # Subdirectories ############################################################ -# Google util libraries borrowed from supersonic, tcmalloc, Chromium, etc. -add_subdirectory(src/kudu/gutil) -add_subdirectory(src/kudu/util) -add_subdirectory(src/kudu/common) +add_subdirectory(src/kudu/benchmarks) add_subdirectory(src/kudu/cfile) +add_subdirectory(src/kudu/client) +add_subdirectory(src/kudu/codegen) +add_subdirectory(src/kudu/common) +add_subdirectory(src/kudu/consensus) +add_subdirectory(src/kudu/experiments) add_subdirectory(src/kudu/fs) +# Google util libraries borrowed from supersonic, tcmalloc, Chromium, etc. +add_subdirectory(src/kudu/gutil) +add_subdirectory(src/kudu/integration-tests) +add_subdirectory(src/kudu/master) +add_subdirectory(src/kudu/rpc) +add_subdirectory(src/kudu/security) add_subdirectory(src/kudu/server) add_subdirectory(src/kudu/tablet) -add_subdirectory(src/kudu/rpc) +add_subdirectory(src/kudu/tools) add_subdirectory(src/kudu/tserver) -add_subdirectory(src/kudu/consensus) -add_subdirectory(src/kudu/master) -add_subdirectory(src/kudu/client) -add_subdirectory(src/kudu/integration-tests) -add_subdirectory(src/kudu/experiments) -add_subdirectory(src/kudu/benchmarks) add_subdirectory(src/kudu/twitter-demo) -add_subdirectory(src/kudu/tools) -add_subdirectory(src/kudu/codegen) +add_subdirectory(src/kudu/util) http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/cmake_modules/FindKdc.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindKdc.cmake b/cmake_modules/FindKdc.cmake new file mode 100644 index 0000000..6d0d4a8 --- /dev/null +++ b/cmake_modules/FindKdc.cmake @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# - Find Kerberos KDC +# This module defines +# KDC_BIN - the Kerberos KDC binary +# KDC_BIN_FOUND- set to true if the Kerberos KDC binary is found + +find_program(KDC_BIN krb5kdc + PATHS + # Linux install location. + /usr/sbin + # Homebrew install location. + /usr/local/opt/krb5/sbin + # Macports install location. + /opt/local/sbin + # SLES + /usr/lib/mit/sbin) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Kdc REQUIRED_VARS KDC_BIN + FAIL_MESSAGE "Kerberos not found: security tests will fail") http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/integration-tests/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 9e45d95..3095d81 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -18,9 +18,9 @@ set(INTEGRATION_TESTS_SRCS cluster_itest_util.cc cluster_verifier.cc - log_verifier.cc external_mini_cluster.cc external_mini_cluster_fs_inspector.cc + log_verifier.cc mini_cluster.cc test_workload.cc ) http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/integration-tests/delete_table-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/delete_table-test.cc b/src/kudu/integration-tests/delete_table-test.cc index d331d43..e48d917 100644 --- a/src/kudu/integration-tests/delete_table-test.cc +++ b/src/kudu/integration-tests/delete_table-test.cc @@ -940,7 +940,7 @@ vector<string> ListOpenFiles(pid_t pid) { string cmd = strings::Substitute("export PATH=$$PATH:/usr/bin:/usr/sbin; lsof -n -p $0", pid); vector<string> argv = { "bash", "-c", cmd }; string out; - CHECK_OK(Subprocess::Call(argv, &out)); + CHECK_OK(Subprocess::Call(argv, "", &out)); vector<string> lines = strings::Split(out, "\n"); return lines; } http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/integration-tests/master_failover-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/master_failover-itest.cc b/src/kudu/integration-tests/master_failover-itest.cc index 5063350..3534978 100644 --- a/src/kudu/integration-tests/master_failover-itest.cc +++ b/src/kudu/integration-tests/master_failover-itest.cc @@ -387,7 +387,7 @@ TEST_F(MasterFailoverTest, TestMasterPermanentFailure) { master::SysCatalogTable::kSysCatalogTabletId }; string output; - ASSERT_OK(Subprocess::Call(args, &output)); + ASSERT_OK(Subprocess::Call(args, "", &output)); StripWhiteSpace(&output); LOG(INFO) << "UUIDS: " << output; set<string> uuids = Split(output, " "); http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/integration-tests/master_migration-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/master_migration-itest.cc b/src/kudu/integration-tests/master_migration-itest.cc index c1d66e2..20336f9 100644 --- a/src/kudu/integration-tests/master_migration-itest.cc +++ b/src/kudu/integration-tests/master_migration-itest.cc @@ -130,7 +130,7 @@ TEST_F(MasterMigrationTest, TestEndToEndMigration) { "--fs_data_dirs=" + data_root }; string uuid; - ASSERT_OK(Subprocess::Call(args, &uuid)); + ASSERT_OK(Subprocess::Call(args, "", &uuid)); StripWhiteSpace(&uuid); master_uuids_and_ports.emplace_back(uuid, kMasterRpcPorts[i]); } http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/security/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/security/CMakeLists.txt b/src/kudu/security/CMakeLists.txt new file mode 100644 index 0000000..3386f3e --- /dev/null +++ b/src/kudu/security/CMakeLists.txt @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(SECURITY_TEST_SRCS + mini_kdc.cc +) + +add_library(security-test ${SECURITY_TEST_SRCS}) +target_link_libraries(security-test + gutil + kudu_test_util + kudu_util) + +# Tests +set(KUDU_TEST_LINK_LIBS + security-test + ${KUDU_MIN_TEST_LIBS}) + +ADD_KUDU_TEST(mini_kdc-test) http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/security/mini_kdc-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/security/mini_kdc-test.cc b/src/kudu/security/mini_kdc-test.cc new file mode 100644 index 0000000..26a3621 --- /dev/null +++ b/src/kudu/security/mini_kdc-test.cc @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <string> + +#include <gtest/gtest.h> + +#include "kudu/security/mini_kdc.h" +#include "kudu/util/test_util.h" + +using std::string; + +namespace kudu { + +TEST(MiniKdcTest, TestBasicOperation) { + MiniKdcOptions options; + MiniKdc kdc(options); + ASSERT_OK(kdc.Start()); + ASSERT_GT(kdc.port(), 0); + ASSERT_OK(kdc.CreateUserPrincipal("alice")); + ASSERT_OK(kdc.Kinit("alice")); + + ASSERT_OK(kdc.Stop()); + ASSERT_OK(kdc.Start()); + + ASSERT_OK(kdc.CreateUserPrincipal("bob")); + ASSERT_OK(kdc.Kinit("bob")); + + string klist; + ASSERT_OK(kdc.Klist(&klist)); + SCOPED_TRACE(klist); + ASSERT_STR_CONTAINS(klist, "[email protected]"); + ASSERT_STR_CONTAINS(klist, "[email protected]"); + ASSERT_STR_CONTAINS(klist, "krbtgt/[email protected]"); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/security/mini_kdc.cc ---------------------------------------------------------------------- diff --git a/src/kudu/security/mini_kdc.cc b/src/kudu/security/mini_kdc.cc new file mode 100644 index 0000000..2546e16 --- /dev/null +++ b/src/kudu/security/mini_kdc.cc @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/mini_kdc.h" + +#include <csignal> + +#include <limits> +#include <memory> +#include <string> + +#include <glog/logging.h> + +#include "kudu/gutil/gscoped_ptr.h" +#include "kudu/gutil/strings/numbers.h" +#include "kudu/gutil/strings/split.h" +#include "kudu/gutil/strings/strip.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/env.h" +#include "kudu/util/monotime.h" +#include "kudu/util/path_util.h" +#include "kudu/util/subprocess.h" +#include "kudu/util/test_util.h" + +using std::string; +using std::unique_ptr; + +namespace kudu { + +string MiniKdcOptions::ToString() const { + return strings::Substitute("{ realm: $0, port: $1, data_root: $2 }", realm, port, data_root); +} + +MiniKdc::MiniKdc() + : MiniKdc(MiniKdcOptions()) { +} + +MiniKdc::MiniKdc(const MiniKdcOptions& options) + : options_(options) { + if (options_.realm.empty()) { + options_.realm = "KRBTEST.COM"; + } + if (options_.data_root.empty()) { + options_.data_root = JoinPathSegments(GetTestDataDirectory(), "krb5kdc"); + } +} + +MiniKdc::~MiniKdc() { + WARN_NOT_OK(Stop(), "Unable to stop MiniKdc"); +} + +vector<string> MiniKdc::MakeArgv(const vector<string>& in_argv) { + string krb5_config = + strings::Substitute("KRB5_CONFIG=$0", JoinPathSegments(options_.data_root, "krb5.conf")); + string krb5_kdc_profile = + strings::Substitute("KRB5_KDC_PROFILE=$0", JoinPathSegments(options_.data_root, "kdc.conf")); + + vector<string> real_argv = { "env", krb5_config, krb5_kdc_profile }; + for (const string& a : in_argv) { + real_argv.push_back(a); + } + return real_argv; +} + +namespace { +// Attempts to find the path to the specified Kerberos binary, storing it in 'path'. +Status GetBinaryPath(const string& binary, + const vector<string>& search, + string* path) { + string p; + + // First, check specified locations which are sometimes not on the PATH. + // This is necessary to check first so that the system Heimdal kerberos + // binaries won't be found first on OS X. + for (const auto& location : search) { + p = JoinPathSegments(location, binary); + if (Env::Default()->FileExists(p)) { + *path = p; + return Status::OK(); + } + } + + // Next check if the binary is on the PATH. + Status s = Subprocess::Call({ "which", binary }, "", &p); + if (s.ok()) { + StripTrailingNewline(&p); + *path = p; + return Status::OK(); + } + + return Status::NotFound("Unable to find binary", binary); +} + +// Attempts to find the path to the specified Kerberos binary, storing it in 'path'. +Status GetBinaryPath(const string& binary, string* path) { + static const vector<string> kCommonLocations = { + "/usr/local/opt/krb5/sbin", // Homebrew + "/usr/local/opt/krb5/bin", // Homebrew + "/opt/local/sbin", // Macports + "/opt/local/bin", // Macports + "/usr/lib/mit/sbin", // SLES + "/usr/sbin", // Linux + }; + return GetBinaryPath(binary, kCommonLocations, path); +} +} // namespace + + +Status MiniKdc::Start() { + CHECK(!kdc_process_); + VLOG(1) << "Starting Kerberos KDC: " << options_.ToString(); + + if (!Env::Default()->FileExists(options_.data_root)) { + RETURN_NOT_OK(Env::Default()->CreateDir(options_.data_root)); + + RETURN_NOT_OK(CreateKdcConf()); + RETURN_NOT_OK(CreateKrb5Conf()); + + // Create the KDC database using the kdb5_util tool. + string kdb5_util_bin; + RETURN_NOT_OK(GetBinaryPath("kdb5_util", &kdb5_util_bin)); + + RETURN_NOT_OK(Subprocess::Call(MakeArgv({ + kdb5_util_bin, "create", + "-s", // Stash the master password. + "-P", "masterpw", // Set a password. + "-W", // Use weak entropy (since we don't need real security). + }))); + } + + // Start the Kerberos KDC. + string krb5kdc_bin; + RETURN_NOT_OK(GetBinaryPath("krb5kdc", &krb5kdc_bin)); + + kdc_process_.reset(new Subprocess( + "env", MakeArgv({ + krb5kdc_bin, + "-n", // Do not daemonize. + }))); + + RETURN_NOT_OK(kdc_process_->Start()); + + // If we asked for an ephemeral port, grab the actual ports and + // rewrite the configuration so that clients can connect. + if (options_.port == 0) { + RETURN_NOT_OK(WaitForKdcPorts()); + RETURN_NOT_OK(CreateKrb5Conf()); + RETURN_NOT_OK(CreateKdcConf()); + } + + return Status::OK(); +} + +Status MiniKdc::Stop() { + CHECK(kdc_process_); + VLOG(1) << "Stopping KDC"; + unique_ptr<Subprocess> proc(kdc_process_.release()); + RETURN_NOT_OK(proc->Kill(SIGKILL)); + RETURN_NOT_OK(proc->Wait()); + + return Status::OK(); +} + +// Creates a kdc.conf file according to the provided options. +Status MiniKdc::CreateKdcConf() const { + static const string kFileTemplate = R"( +[kdcdefaults] +kdc_ports = "" +kdc_tcp_ports = $2 + +[realms] +$1 = { + max_renewable_life = 7d 0h 0m 0s + acl_file = $0/kadm5.acl + admin_keytab = $0/kadm5.keytab + + database_name = $0/principal + key_stash_file = $0/.k5.$1 + acl_file = $0/kadm5.acl +} + )"; + string file_contents = strings::Substitute(kFileTemplate, options_.data_root, + options_.realm, options_.port); + return WriteStringToFile(Env::Default(), file_contents, + JoinPathSegments(options_.data_root, "kdc.conf")); +} + +// Creates a krb5.conf file according to the provided options. +Status MiniKdc::CreateKrb5Conf() const { + static const string kFileTemplate = R"( +[logging] + kdc = STDERR + +[libdefaults] + default_realm = $1 + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + default_ccache_name = $2 + + # The KDC is configured to only use TCP, so the client should not prefer UDP. + udp_preference_limit = 0 + +[realms] + $1 = { + kdc = 127.0.0.1:$0 + } + )"; + string ccache = "DIR:" + JoinPathSegments(options_.data_root, "krb5cc"); + string file_contents = strings::Substitute(kFileTemplate, options_.port, options_.realm, ccache); + return WriteStringToFile(Env::Default(), file_contents, + JoinPathSegments(options_.data_root, "krb5.conf")); +} + +Status MiniKdc::WaitForKdcPorts() { + // We have to use 'lsof' to figure out which ports the KDC bound to if we + // requested ephemeral ones. The KDC doesn't log the bound port or expose it + // in any other fashion, and re-implementing lsof involves parsing a lot of + // files in /proc/. So, requiring lsof for tests and parsing its output seems + // more straight-forward. We call lsof in a loop in case the kdc is slow to + // bind to the ports. + + string lsof; + RETURN_NOT_OK(GetBinaryPath("lsof", {"/sbin"}, &lsof)); + + vector<string> cmd = { + lsof, "-wbn", "-Fn", + "-p", std::to_string(kdc_process_->pid()), + "-a", "-i", "4TCP"}; + + string lsof_out; + for (int i = 1; ; i++) { + lsof_out.clear(); + Status s = Subprocess::Call(cmd, "", &lsof_out); + + if (s.ok()) { + StripTrailingNewline(&lsof_out); + break; + } else if (i > 10) { + return s; + } + + SleepFor(MonoDelta::FromMilliseconds(i * i)); + } + + // The '-Fn' flag gets lsof to output something like: + // p19730 + // n*:41254 + // The first line is the pid, which we already know. The second has the + // bind address and port. + vector<string> lines = strings::Split(lsof_out, "\n"); + int32_t port = -1; + if (lines.size() != 2 || + lines[1].substr(0, 3) != "n*:" || + !safe_strto32(lines[1].substr(3), &port) || + port <= 0) { + return Status::RuntimeError("unexpected lsof output", lsof_out); + } + CHECK(port > 0 && port < std::numeric_limits<uint16_t>::max()) + << "parsed invalid port: " << port; + options_.port = port; + VLOG(1) << "Determined bound KDC port: " << options_.port; + return Status::OK(); +} + +Status MiniKdc::CreateUserPrincipal(const string& username) { + string kadmin; + RETURN_NOT_OK(GetBinaryPath("kadmin.local", &kadmin)); + RETURN_NOT_OK(Subprocess::Call(MakeArgv({ + kadmin, "-q", strings::Substitute("add_principal -pw $0 $0", username, username)}))); + return Status::OK(); +} + +Status MiniKdc::Kinit(const string& username) { + string kinit; + RETURN_NOT_OK(GetBinaryPath("kinit", &kinit)); + Subprocess::Call(MakeArgv({ kinit, username }), username); + return Status::OK(); +} + +Status MiniKdc::Klist(string* output) { + string klist; + RETURN_NOT_OK(GetBinaryPath("klist", &klist)); + RETURN_NOT_OK(Subprocess::Call(MakeArgv({ klist, "-A" }), "", output)); + return Status::OK(); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/security/mini_kdc.h ---------------------------------------------------------------------- diff --git a/src/kudu/security/mini_kdc.h b/src/kudu/security/mini_kdc.h new file mode 100644 index 0000000..84ca610 --- /dev/null +++ b/src/kudu/security/mini_kdc.h @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <memory> +#include <string> +#include <vector> + +#include <glog/logging.h> + +#include "kudu/util/status.h" + +namespace kudu { + +class Env; +class Subprocess; + +struct MiniKdcOptions { + + // Kerberos Realm. + // Default: "KRBTEST.COM" + std::string realm; + + // Directory in which to store data. + // Default: "", which auto-generates a unique path for this KDC. + // The default may only be used from a gtest unit test. + std::string data_root; + + // KDC port. + // Default: 0 (ephemeral port). + uint16_t port = 0; + + // Returns a string representation of the options suitable for debug printing. + std::string ToString() const; +}; + +class MiniKdc { + public: + // Creates a new MiniKdc with the default options. + MiniKdc(); + + // Creates a new MiniKdc with the provided options. + explicit MiniKdc(const MiniKdcOptions& options); + + ~MiniKdc(); + + // Starts the mini Kerberos KDC. + Status Start() WARN_UNUSED_RESULT; + + // Stops the mini Kerberos KDC. + Status Stop() WARN_UNUSED_RESULT; + + uint16_t port() const { + CHECK(kdc_process_) << "must start first"; + return options_.port; + } + + // Creates a new user with the given username. + // The password is the same as the username. + Status CreateUserPrincipal(const std::string& username) WARN_UNUSED_RESULT; + + // Kinit a user to the mini KDC. + Status Kinit(const std::string& username) WARN_UNUSED_RESULT; + + // Call the 'klist' utility. This is useful for logging the local ticket + // cache state. + Status Klist(std::string* output) WARN_UNUSED_RESULT; + + private: + + // Prepends required Kerberos environment variables to the process arguments. + std::vector<std::string> MakeArgv(const std::vector<std::string>& in_argv); + + // Creates a kdc.conf in the data root. + Status CreateKrb5Conf() const WARN_UNUSED_RESULT; + + // Creates a krb5.conf in the data root. + Status CreateKdcConf() const WARN_UNUSED_RESULT; + + // Determine the ports that the KDC bound to. Will wait for the KDC if it is + // still initializing. + Status WaitForKdcPorts() WARN_UNUSED_RESULT; + + std::unique_ptr<Subprocess> kdc_process_; + MiniKdcOptions options_; +}; + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/tools/kudu-admin-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc index b5c5f83..0272550 100644 --- a/src/kudu/tools/kudu-admin-test.cc +++ b/src/kudu/tools/kudu-admin-test.cc @@ -199,7 +199,7 @@ TEST_F(AdminCliTest, TestLeaderStepDown) { Status s = Subprocess::Call({GetKuduCtlAbsolutePath(), "tablet", "leader_step_down", cluster_->master()->bound_rpc_addr().ToString(), - tablet_id_}, nullptr, &stderr); + tablet_id_}, "", nullptr, &stderr); bool not_currently_leader = stderr.find( Status::IllegalState("").CodeAsString()) != string::npos; ASSERT_TRUE(s.ok() || not_currently_leader); @@ -238,7 +238,7 @@ TEST_F(AdminCliTest, TestLeaderStepDownWhenNotPresent) { "leader_step_down", cluster_->master()->bound_rpc_addr().ToString(), tablet_id_ - }, &stdout)); + }, "", &stdout)); ASSERT_STR_CONTAINS(stdout, Substitute("No leader replica found for tablet $0", tablet_id_)); @@ -280,7 +280,7 @@ TEST_F(AdminCliTest, TestListTables) { "table", "list", cluster_->master()->bound_rpc_addr().ToString() - }, &stdout, nullptr)); + }, "", &stdout, nullptr)); vector<string> stdout_lines = strings::Split(stdout, ",", strings::SkipEmpty()); @@ -318,7 +318,7 @@ TEST_F(AdminCliTest, TestListTablesDetail) { "list", "--list_tablets", cluster_->master()->bound_rpc_addr().ToString() - }, &stdout, nullptr)); + }, "", &stdout, nullptr)); vector<string> stdout_lines = strings::Split(stdout, "\n", strings::SkipEmpty()); http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/tools/kudu-tool-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 94c5160..6ad0d08 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -107,7 +107,7 @@ class ToolTest : public KuduTest { string out; string err; - Status s = Subprocess::Call(args, &out, &err); + Status s = Subprocess::Call(args, "", &out, &err); if (stdout) { *stdout = out; StripWhiteSpace(stdout); http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/tools/kudu-ts-cli-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/kudu-ts-cli-test.cc b/src/kudu/tools/kudu-ts-cli-test.cc index b3e36a2..bbbf73d 100644 --- a/src/kudu/tools/kudu-ts-cli-test.cc +++ b/src/kudu/tools/kudu-ts-cli-test.cc @@ -69,7 +69,7 @@ TEST_F(KuduTsCliTest, TestDeleteTablet) { cluster_->tablet_server(0)->bound_rpc_addr().ToString(), tablet_id, "Deleting for kudu-ts-cli-test" - }, &out)); + }, "", &out)); ASSERT_EQ("", out); ASSERT_OK(inspect_->WaitForTabletDataStateOnTS(0, tablet_id, { tablet::TABLET_DATA_TOMBSTONED })); @@ -107,7 +107,7 @@ TEST_F(KuduTsCliTest, TestDumpTablet) { "dump", cluster_->tablet_server(0)->bound_rpc_addr().ToString(), tablet_id - }, &out)); + }, "", &out)); ASSERT_EQ("", out); // Insert very little data and dump_tablet again. @@ -123,7 +123,7 @@ TEST_F(KuduTsCliTest, TestDumpTablet) { "dump", cluster_->tablet_server(0)->bound_rpc_addr().ToString(), tablet_id - }, &out)); + }, "", &out)); // Split the output into multiple rows and check format of each row, // and also check total number of rows are at least kNumRows. http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/util/net/net_util.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/net/net_util.cc b/src/kudu/util/net/net_util.cc index ac5fb62..d67375c 100644 --- a/src/kudu/util/net/net_util.cc +++ b/src/kudu/util/net/net_util.cc @@ -296,7 +296,7 @@ void TryRunLsof(const Sockaddr& addr, vector<string>* log) { LOG_STRING(INFO, log) << "$ " << cmd; vector<string> argv = { "bash", "-c", cmd }; string results; - Status s = Subprocess::Call(argv, &results); + Status s = Subprocess::Call(argv, "", &results); if (PREDICT_FALSE(!s.ok())) { LOG_STRING(WARNING, log) << s.ToString(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/util/subprocess-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/subprocess-test.cc b/src/kudu/util/subprocess-test.cc index 76778c6..f08802f 100644 --- a/src/kudu/util/subprocess-test.cc +++ b/src/kudu/util/subprocess-test.cc @@ -128,7 +128,16 @@ TEST_F(SubprocessTest, TestReadFromStdoutAndStderr) { "dd if=/dev/urandom of=/dev/stdout bs=512 count=2048 &" "dd if=/dev/urandom of=/dev/stderr bs=512 count=2048 &" "wait" - }, &stdout, &stderr)); + }, "", &stdout, &stderr)); +} + +// Tests writing to the subprocess stdin. +TEST_F(SubprocessTest, TestCallWithStdin) { + string stdout; + ASSERT_OK(Subprocess::Call({ "/bin/bash" }, + "echo \"quick brown fox\"", + &stdout)); + EXPECT_EQ("quick brown fox\n", stdout); } // Test KUDU-1674: '/bin/bash -c "echo"' command below is expected to @@ -139,15 +148,15 @@ TEST_F(SubprocessTest, TestReadSingleFD) { string stderr; const string str = "ApacheKudu"; const string cmd_str = Substitute("/bin/echo -n $0 1>&2", str); - ASSERT_OK(Subprocess::Call({"/bin/sh", "-c", cmd_str}, nullptr, &stderr)); + ASSERT_OK(Subprocess::Call({"/bin/sh", "-c", cmd_str}, "", nullptr, &stderr)); ASSERT_EQ(stderr, str); // Also sanity check other combinations. string stdout; - ASSERT_OK(Subprocess::Call({"/bin/ls", "/dev/null"}, &stdout, nullptr)); + ASSERT_OK(Subprocess::Call({"/bin/ls", "/dev/null"}, "", &stdout, nullptr)); ASSERT_STR_CONTAINS(stdout, "/dev/null"); - ASSERT_OK(Subprocess::Call({"/bin/ls", "/dev/zero"}, nullptr, nullptr)); + ASSERT_OK(Subprocess::Call({"/bin/ls", "/dev/zero"}, "", nullptr, nullptr)); } TEST_F(SubprocessTest, TestGetExitStatusExitSuccess) { http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/util/subprocess.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/subprocess.cc b/src/kudu/util/subprocess.cc index 89854cb..603cf99 100644 --- a/src/kudu/util/subprocess.cc +++ b/src/kudu/util/subprocess.cc @@ -20,6 +20,7 @@ #include <dirent.h> #include <fcntl.h> #include <signal.h> +#include <stdio.h> #if defined(__linux__) #include <sys/prctl.h> #endif @@ -308,6 +309,7 @@ static int pipe2(int pipefd[2], int flags) { #endif Status Subprocess::Start() { + VLOG(2) << "Invoking command: " << argv_; if (state_ != kNotStarted) { const string err_str = Substitute("$0: illegal sub-process state", state_); LOG(DFATAL) << err_str; @@ -520,13 +522,13 @@ Status Subprocess::GetExitStatus(int* exit_status, string* info_str) const { Status Subprocess::Call(const string& arg_str) { vector<string> argv = Split(arg_str, " "); - return Call(argv, nullptr, nullptr); + return Call(argv, "", nullptr, nullptr); } Status Subprocess::Call(const vector<string>& argv, + const string& stdin_in, string* stdout_out, string* stderr_out) { - VLOG(2) << "Invoking command: " << argv; Subprocess p(argv[0], argv); if (stdout_out) { @@ -537,6 +539,12 @@ Status Subprocess::Call(const vector<string>& argv, } RETURN_NOT_OK_PREPEND(p.Start(), "Unable to fork " + argv[0]); + + if (!stdin_in.empty() && + write(p.to_child_stdin_fd(), stdin_in.data(), stdin_in.size()) < stdin_in.size()) { + return Status::IOError("Unable to write to child process stdin", ErrnoToString(errno), errno); + } + int err = close(p.ReleaseChildStdinFd()); if (PREDICT_FALSE(err != 0)) { return Status::IOError("Unable to close child process stdin", ErrnoToString(errno), errno); http://git-wip-us.apache.org/repos/asf/kudu/blob/7138468a/src/kudu/util/subprocess.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/subprocess.h b/src/kudu/util/subprocess.h index 640f4aa..2ec9468 100644 --- a/src/kudu/util/subprocess.h +++ b/src/kudu/util/subprocess.h @@ -61,7 +61,7 @@ class Subprocess { // Start the subprocess. Can only be called once. // - // Thie returns a bad Status if the fork() fails. However, + // This returns a bad Status if the fork() fails. However, // note that if the executable path was incorrect such that // exec() fails, this will still return Status::OK. You must // use Wait() to check for failure. @@ -103,9 +103,13 @@ class Subprocess { // Same as above, but accepts a vector that includes the path to the // executable as argv[0] and the arguments to the program in argv[1..n]. // + // Writes the value of 'stdin_in' to the subprocess' stdin. The length of + // 'stdin_in' should be limited to 64kib. + // // Also collects the output from the child process stdout and stderr into // 'stdout_out' and 'stderr_out' respectively. static Status Call(const std::vector<std::string>& argv, + const std::string& stdin_in = "", std::string* stdout_out = nullptr, std::string* stderr_out = nullptr);
