This is an automated email from the ASF dual-hosted git repository. laiyingchun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 8e8a397415c819c5a454460028b4f4397fd18ae8 Author: kedeng <[email protected]> AuthorDate: Fri Mar 10 15:58:14 2023 +0800 KUDU-3413 add tenant info in metadata for multi-tenancy If we want to introduce multi-tenancy feature, it is important to add relevant metadata information first. In this patch, I added definition of tenant information in metadata, and refactored some metadata loading to distinguish between data at rest encryption feature and multi-tenancy feature. In the subsequent use, if we encounter a situation where both server key and tenant key are present in the metadata, we need to pay attention to the fact that tenant key will have a higher priority. That is, we will consider it as enabling the multi-tenancy feature and ignore the processing of server key. Of course, it is also necessary to provide a way to upgrade server key to tenant key, which I plan to implement in the next patch. In this commit, I added a new flag '--enable_multi_tenancy' to indicate whether we enable multi-tenancy feature. However, considering that the development of the entire feature has not been completed, it is not recommended to set this flag temporarily. We need to wait until the entire feature is developed before considering whether to set it. And I also added unit tests for the new open process to ensure the open logic is effective. Change-Id: I9e450d73940eb1dbaac6f905a46d6ccd084f15cf Reviewed-on: http://gerrit.cloudera.org:8080/19622 Tested-by: Kudu Jenkins Reviewed-by: Yingchun Lai <[email protected]> --- src/kudu/consensus/raft_consensus_quorum-test.cc | 16 +- src/kudu/fs/dir_manager.h | 3 + src/kudu/fs/fs.proto | 31 +++ src/kudu/fs/fs_manager-test.cc | 86 +++++++ src/kudu/fs/fs_manager.cc | 246 +++++++++++++++++++-- src/kudu/fs/fs_manager.h | 94 +++++++- src/kudu/integration-tests/raft_consensus-itest.cc | 4 +- src/kudu/mini-cluster/external_mini_cluster.cc | 30 ++- src/kudu/mini-cluster/external_mini_cluster.h | 4 +- src/kudu/mini-cluster/internal_mini_cluster.cc | 21 +- src/kudu/ranger-kms/ranger_kms_client.cc | 1 + src/kudu/server/server_base.cc | 21 +- src/kudu/server/server_base_options.cc | 51 ++++- src/kudu/server/server_base_options.h | 25 ++- src/kudu/tools/kudu-tool-test.cc | 32 ++- src/kudu/tools/tool_action_common.cc | 70 ++++-- src/kudu/tools/tool_action_fs.cc | 55 ++++- src/kudu/tserver/tablet_copy_client-test.cc | 19 +- src/kudu/tserver/tablet_server-test-base.cc | 4 +- src/kudu/util/env-test.cc | 3 +- src/kudu/util/env_posix.cc | 19 ++ src/kudu/util/flag_validators-test.cc | 2 +- src/kudu/util/test_util.cc | 16 +- src/kudu/util/test_util.h | 5 +- 24 files changed, 765 insertions(+), 93 deletions(-) diff --git a/src/kudu/consensus/raft_consensus_quorum-test.cc b/src/kudu/consensus/raft_consensus_quorum-test.cc index 214d2282b..6275fd863 100644 --- a/src/kudu/consensus/raft_consensus_quorum-test.cc +++ b/src/kudu/consensus/raft_consensus_quorum-test.cc @@ -135,14 +135,26 @@ class RaftConsensusQuorumTest : public KuduTest { opts.wal_root = test_path; opts.data_roots = { test_path }; unique_ptr<FsManager> fs_manager(new FsManager(env_, opts)); + string tenant_name; + string tenant_id; string encryption_key; string encryption_key_iv; string encryption_key_version; - GetEncryptionKey(&encryption_key, &encryption_key_iv, &encryption_key_version); - if (encryption_key.empty()) { + GetEncryptionKey(&tenant_name, &tenant_id, &encryption_key, + &encryption_key_iv, &encryption_key_version); + if (tenant_name.empty() && encryption_key.empty()) { RETURN_NOT_OK(fs_manager->CreateInitialFileSystemLayout()); + } else if (tenant_name.empty()) { + RETURN_NOT_OK(fs_manager->CreateInitialFileSystemLayout(nullopt, + nullopt, + nullopt, + encryption_key, + encryption_key_iv, + encryption_key_version)); } else { RETURN_NOT_OK(fs_manager->CreateInitialFileSystemLayout(nullopt, + tenant_name, + tenant_id, encryption_key, encryption_key_iv, encryption_key_version)); diff --git a/src/kudu/fs/dir_manager.h b/src/kudu/fs/dir_manager.h index 88a17bdd2..b7a54fe40 100644 --- a/src/kudu/fs/dir_manager.h +++ b/src/kudu/fs/dir_manager.h @@ -53,6 +53,9 @@ typedef std::vector<CanonicalizedRootAndStatus> CanonicalizedRootsList; namespace fs { +static const std::string kDefaultTenantName = "default_tenant_kudu"; +static const std::string kDefaultTenantID = "00000000000000000000000000000000"; + typedef std::unordered_map<int, std::string> UuidByUuidIndexMap; typedef std::unordered_map<std::string, int> UuidIndexByUuidMap; class DirInstanceMetadataFile; diff --git a/src/kudu/fs/fs.proto b/src/kudu/fs/fs.proto index 849aa591a..c257e284a 100644 --- a/src/kudu/fs/fs.proto +++ b/src/kudu/fs/fs.proto @@ -44,6 +44,37 @@ message InstanceMetadataPB { // Server key version. optional string server_key_version = 5; + // The tenant_key works on the base of the server_key, but the server_key is not + // always upgraded to tenant_key, which means that the server_key may exist in + // the latest version. We provide an upgrading tool to facilitate the upgrade of + // server_key to tenant_key. During the upgrade process, if we detect that the server_key + // related configuration items have values, we read them out and assign them to + // the default tenant. Then we don't clear the server_key related configuration items + // for compatibility guarantee. And finally update the metadata file with new + // configuration values. + + message TenantMetadataPB { + // Tenant name. + optional string tenant_name = 1; + + // Tenant uuid. + optional string tenant_id = 2; + + // Encrypted tenant key used to encrypt/decrypt file keys for this tenant. + optional string tenant_key = 3; + + // Initialization vector for the tenant key. + optional string tenant_key_iv = 4; + + // Tenant key version. + optional string tenant_key_version = 5; + } + + // NOTE: + // The priority of tenant key is higher than that of server key, which means that if + // both server key info and tenants info exist in the metadata, we will use the tenants. + repeated TenantMetadataPB tenants = 6; + // TODO: add a "node type" (TS/Master?) } diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc index b0766dbb2..9124f38ee 100644 --- a/src/kudu/fs/fs_manager-test.cc +++ b/src/kudu/fs/fs_manager-test.cc @@ -28,6 +28,7 @@ #include <memory> #include <set> #include <string> +#include <tuple> #include <unordered_map> #include <unordered_set> #include <utility> @@ -77,6 +78,8 @@ DECLARE_string(block_manager); DECLARE_string(env_inject_eio_globs); DECLARE_string(env_inject_lock_failure_globs); DECLARE_string(umask); +DECLARE_bool(encrypt_data_at_rest); +DECLARE_bool(enable_multi_tenancy); namespace kudu { namespace fs { @@ -1193,5 +1196,88 @@ TEST_P(FsManagerTestBase, TestAncillaryDirsReported) { ASSERT_STR_CONTAINS(report_str, "metadata directory: " + opts.metadata_root); } +class OpenFsTypeTest : public KuduTest, + public ::testing::WithParamInterface<std::tuple<string, bool, bool>> { + public: + OpenFsTypeTest() + : fs_root_(GetTestPath("fs_root")) { + } + + void SetUp() override { + KuduTest::SetUp(); + FLAGS_block_manager = std::get<0>(GetParam()); + FLAGS_encrypt_data_at_rest = std::get<1>(GetParam()); + FLAGS_enable_multi_tenancy = std::get<2>(GetParam()); + + // '--enable_multi_tenancy' should set with '--encrypt_data_at_rest'. + if (!FLAGS_encrypt_data_at_rest && FLAGS_enable_multi_tenancy) { + GTEST_SKIP(); + } + + FsManagerOpts opts; + opts.wal_root = GetTestPath("wal"); + opts.data_roots = { GetTestPath("data") }; + opts.metadata_root = GetTestPath("metadata"); + ReinitFsManagerWithOpts(opts); + + ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout()); + ASSERT_OK(fs_manager_->Open()); + } + + FsManager *fs_manager() const { return fs_manager_.get(); } + + static bool is_multi_tenancy_enable() { + return FLAGS_encrypt_data_at_rest && FLAGS_enable_multi_tenancy; + } + + static bool is_only_encryption_enable() { + return FLAGS_encrypt_data_at_rest && !FLAGS_enable_multi_tenancy; + } + + static bool is_encryption_disable() { + return !FLAGS_encrypt_data_at_rest && !FLAGS_enable_multi_tenancy; + } + + protected: + void ReinitFsManagerWithPaths(string wal_path, vector<string> data_paths) { + FsManagerOpts opts; + opts.wal_root = std::move(wal_path); + opts.data_roots = std::move(data_paths); + ReinitFsManagerWithOpts(std::move(opts)); + } + + void ReinitFsManagerWithOpts(FsManagerOpts opts) { + fs_manager_.reset(new FsManager(env_, std::move(opts))); + } + + private: + const string fs_root_; + unique_ptr<FsManager> fs_manager_; +}; + +INSTANTIATE_TEST_SUITE_P(, OpenFsTypeTest, + ::testing::Combine( + ::testing::ValuesIn(BlockManager::block_manager_types()), + ::testing::Bool() /* --encrypt_data_at_rest */, + ::testing::Bool() /* --enable_multi_tenancy */)); + +TEST_P(OpenFsTypeTest, TestDifferentTypesOpen) { + if (is_encryption_disable()) { + ASSERT_TRUE(fs_manager()->server_key().empty()); + ASSERT_FALSE(fs_manager()->is_tenants_exist()); + } + + if (is_only_encryption_enable()) { + ASSERT_FALSE(fs_manager()->server_key().empty()); + ASSERT_FALSE(fs_manager()->is_tenants_exist()); + } + + if (is_multi_tenancy_enable()) { + // This isn't upgrade case, so no server key exist. + ASSERT_TRUE(fs_manager()->server_key().empty()); + ASSERT_TRUE(fs_manager()->is_tenants_exist()); + } +} + } // namespace fs } // namespace kudu diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc index c78cda902..6eb35c48f 100644 --- a/src/kudu/fs/fs_manager.cc +++ b/src/kudu/fs/fs_manager.cc @@ -22,6 +22,7 @@ #include <functional> #include <initializer_list> #include <iostream> +#include <mutex> #include <unordered_map> #include <unordered_set> #include <utility> @@ -160,6 +161,7 @@ bool ValidateRangerKMSFlags() { GROUP_FLAG_VALIDATOR(validate_ranger_kms_flags, ValidateRangerKMSFlags); +DECLARE_bool(enable_multi_tenancy); DECLARE_bool(encrypt_data_at_rest); DECLARE_int32(encryption_key_length); @@ -504,12 +506,38 @@ Status FsManager::Open(FsReport* report, Timer* read_instance_metadata_files, read_instance_metadata_files->Stop(); } - if (!server_key().empty() && key_provider_) { - string decrypted_key; + string decrypted_key; + bool tenants_exist = is_tenants_exist(); + if (tenants_exist && key_provider_) { + if (!FLAGS_enable_multi_tenancy) { + return Status::IllegalState( + "The '--enable_multi_tenancy' should set for the existed tenants."); + } + + // TODO(kedeng) : + // After implementing tenant management, different tenants need to be handled here. + // + // The priority of tenant key is higher than that of server key. + RETURN_NOT_OK( + key_provider_->DecryptEncryptionKey(this->tenant_key(fs::kDefaultTenantName), + this->tenant_key_iv(fs::kDefaultTenantName), + this->tenant_key_version(fs::kDefaultTenantName), + &decrypted_key)); + } else if (!server_key().empty() && key_provider_) { + // Just check whether the upgrade operation is needed for '--enable_multi_tenancy'. + if (FLAGS_enable_multi_tenancy) { + return Status::IllegalState( + "We should do server key upgrade with Kudu CLI tool for '--enable_multi_tenancy'."); + } + + string server_key; RETURN_NOT_OK(key_provider_->DecryptEncryptionKey(this->server_key(), this->server_key_iv(), this->server_key_version(), &decrypted_key)); + } + + if (!decrypted_key.empty()) { // 'decrypted_key' is a hexadecimal string and SetEncryptionKey expects bits // (hex / 2 = bytes * 8 = bits). env_->SetEncryptionKey(reinterpret_cast<const uint8_t*>( @@ -596,7 +624,69 @@ Status FsManager::Open(FsReport* report, Timer* read_instance_metadata_files, return Status::OK(); } +Status FsManager::UpdateMetadata(unique_ptr<InstanceMetadataPB> metadata) { + // In the event of failure, rollback everything we changed. + // <string, string> <=> <old instance file, backup instance file> + unordered_map<string, string> changed_dirs; + auto rollbacker = MakeScopedCleanup([&]() { + // Delete new files first so that the backup files could do rollback. + for (const auto& changed : changed_dirs) { + WARN_NOT_OK(env_->DeleteFile(changed.first), + Substitute("Could not delete file $0 for rollback.", changed.first)); + VLOG(1) << "Delete file: " << changed.first << " for rollback."; + + WARN_NOT_OK(env_->RenameFile(changed.second, changed.first), + Substitute("Could not rename file $0 for rollback.", changed.second)); + VLOG(1) << "Rename file: " << changed.second << " to " << changed.first << " for rollback."; + } + }); + + for (const auto& root : canonicalized_all_fs_roots_) { + if (!root.status.ok()) { + continue; + } + // Backup the metadata at first. + const string old_path = GetInstanceMetadataPath(root.path); + string tmp_path = Substitute("$0-$1", old_path, GetCurrentTimeMicros()); + WARN_NOT_OK(env_->RenameFile(old_path, tmp_path), + Substitute("Could not rename file $0, ", old_path)); + changed_dirs[old_path] = tmp_path; + // Write the instance metadata with latest data. + RETURN_NOT_OK_PREPEND(WriteInstanceMetadata(*metadata, root.path), + Substitute("Fail to rewrite the instance metadata for path: $0, " + "now try to rollback all the instance metadata file.", old_path)); + } + + { + // Update the records in memory. + std::lock_guard<percpu_rwlock> lock(metadata_rwlock_); + metadata_ = std::move(metadata); + } + + // If all op success, remove the backup data. + for (const auto& changed : changed_dirs) { + WARN_NOT_OK(env_->DeleteFile(changed.second), Substitute("Could not delete file $0, ", + changed.second)); + } + + // Success: don't rollback any files. + rollbacker.cancel(); + return Status::OK(); +} + +void FsManager::UpdateMetadataFormatAndStampUnlock(InstanceMetadataPB* metadata) { + string time_str; + StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false); + string hostname; + if (!GetHostname(&hostname).ok()) { + hostname = "<unknown host>"; + } + metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname)); +} + Status FsManager::CreateInitialFileSystemLayout(optional<string> uuid, + optional<string> tenant_name, + optional<string> tenant_id, optional<string> encryption_key, optional<string> encryption_key_iv, optional<string> encryption_key_version) { @@ -624,6 +714,8 @@ Status FsManager::CreateInitialFileSystemLayout(optional<string> uuid, // Files/directories created will NOT be synchronized to disk. InstanceMetadataPB metadata; RETURN_NOT_OK_PREPEND(CreateInstanceMetadata(std::move(uuid), + std::move(tenant_name), + std::move(tenant_id), std::move(encryption_key), std::move(encryption_key_iv), std::move(encryption_key_version), @@ -722,6 +814,8 @@ Status FsManager::CreateFileSystemRoots( } Status FsManager::CreateInstanceMetadata(optional<string> uuid, + optional<string> tenant_name, + optional<string> tenant_id, optional<string> encryption_key, optional<string> encryption_key_iv, optional<string> encryption_key_version, @@ -733,31 +827,71 @@ Status FsManager::CreateInstanceMetadata(optional<string> uuid, } else { metadata->set_uuid(oid_generator_.Next()); } - if (encryption_key && encryption_key_iv && encryption_key_version) { + + if (tenant_name && tenant_id && encryption_key && encryption_key_iv && encryption_key_version) { + // The tenant key info exist. + // + // If all the fileds, which means tenant_name/tenant_id/encryption_key/encryption_key_iv/ + // encryption_key_version, are specified, we treat it as tenant key info of multi-tenancy. + InstanceMetadataPB::TenantMetadataPB* tenant_metadata = metadata->add_tenants(); + tenant_metadata->set_tenant_name(*tenant_name); + tenant_metadata->set_tenant_id(*tenant_id); + tenant_metadata->set_tenant_key(*encryption_key); + tenant_metadata->set_tenant_key_iv(*encryption_key_iv); + tenant_metadata->set_tenant_key_version(*encryption_key_version); + } else if (!tenant_name && !tenant_id && + encryption_key && encryption_key_iv && encryption_key_version) { + // The server key info exist. + // + // If all the fileds(encryption_key/encryption_key_iv/encryption_key_version) are specified + // except tenant_name/tenant_id, we treat it as server key info for encryption. metadata->set_server_key(*encryption_key); metadata->set_server_key_iv(*encryption_key_iv); metadata->set_server_key_version(*encryption_key_version); - } else if (encryption_key || encryption_key_iv || encryption_key_version) { - return Status::InvalidArgument( - "'encryption_key', 'encryption_key_iv', and 'encryption_key_version' must be specified " - "together (either all of them must be specified, or none of them)."); - } else if (FLAGS_encrypt_data_at_rest) { - string key_version; - RETURN_NOT_OK_PREPEND( - key_provider_->GenerateEncryptionKey(metadata->mutable_server_key(), - metadata->mutable_server_key_iv(), - &key_version), - "failed to generate encrypted server key"); - metadata->set_server_key_version(key_version); - } + } else { + if (encryption_key || encryption_key_iv || encryption_key_version) { + // There is incomplete encrypted key information. + // (Tenant name or tenant id is not important in this case.) + return Status::InvalidArgument( + "'encryption_key', 'encryption_key_iv', and 'encryption_key_version' must be specified " + "together (either all of them must be specified, or none of them)."); + } - string time_str; - StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false); - string hostname; - if (!GetHostname(&hostname).ok()) { - hostname = "<unknown host>"; + // No encrypted key information exist. + if (FLAGS_encrypt_data_at_rest && FLAGS_enable_multi_tenancy) { + // The multi_tenancy enabled. + // + // Use kDefaultTenant to encrypt data if '--enable_multi_tenancy' set with + // '--encrypt_data_at_rest'. + string key; + string key_iv; + string key_version; + RETURN_NOT_OK_PREPEND( + key_provider_->GenerateEncryptionKey(&key, + &key_iv, + &key_version), + "failed to generate encrypted default tenant key"); + InstanceMetadataPB::TenantMetadataPB* tenant_metadata = metadata->add_tenants(); + tenant_metadata->set_tenant_name(fs::kDefaultTenantName); + tenant_metadata->set_tenant_id(fs::kDefaultTenantID); + tenant_metadata->set_tenant_key(key); + tenant_metadata->set_tenant_key_iv(key_iv); + tenant_metadata->set_tenant_key_version(key_version); + } else if (FLAGS_encrypt_data_at_rest) { + // The encrypt_data_at_rest enabled. + // + // Generate encrypted server key to encrypt data if only set '--encrypt_data_at_rest'. + string key_version; + RETURN_NOT_OK_PREPEND( + key_provider_->GenerateEncryptionKey(metadata->mutable_server_key(), + metadata->mutable_server_key_iv(), + &key_version), + "failed to generate encrypted server key"); + metadata->set_server_key_version(key_version); + } } - metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname)); + + UpdateMetadataFormatAndStampUnlock(metadata); return Status::OK(); } @@ -793,6 +927,74 @@ const string& FsManager::server_key_version() const { return CHECK_NOTNULL(metadata_.get())->server_key_version(); } +bool FsManager::is_tenants_exist() const { + shared_lock<rw_spinlock> md_lock(metadata_rwlock_.get_lock()); + return metadata_->tenants_size() > 0; +} + +bool FsManager::is_tenant_exist(const string& tenant_name) const { + return GetTenant(tenant_name) != nullptr; +} + +const InstanceMetadataPB_TenantMetadataPB* FsManager::GetTenantUnlock( + const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = nullptr; + for (const auto& tdata : metadata_->tenants()) { + if (tdata.tenant_name() == tenant_name) { + tenant = &tdata; + break; + } + } + + return tenant; +} + +const InstanceMetadataPB_TenantMetadataPB* FsManager::GetTenant( + const string& tenant_name) const { + shared_lock<rw_spinlock> md_lock(metadata_rwlock_.get_lock()); + return GetTenantUnlock(tenant_name); +} + +string FsManager::tenant_id_unlock(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenantUnlock(tenant_name); + return tenant ? tenant->tenant_id() : string(""); +} + +string FsManager::tenant_id(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenant(tenant_name); + return tenant ? tenant->tenant_id() : string(""); +} + +string FsManager::tenant_key_unlock(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenantUnlock(tenant_name); + return tenant ? tenant->tenant_key() : string(""); +} + +string FsManager::tenant_key(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenant(tenant_name); + return tenant ? tenant->tenant_key() : string(""); +} + +string FsManager::tenant_key_iv_unlock(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenantUnlock(tenant_name); + return tenant ? tenant->tenant_key_iv() : string(""); +} + +string FsManager::tenant_key_iv(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenant(tenant_name); + return tenant ? tenant->tenant_key_iv() : string(""); +} + +string FsManager::tenant_key_version_unlock(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenantUnlock(tenant_name); + return tenant ? tenant->tenant_key_version() : string(""); +} + +string FsManager::tenant_key_version(const string& tenant_name) const { + const InstanceMetadataPB::TenantMetadataPB* tenant = GetTenant(tenant_name); + return tenant ? tenant->tenant_key_version() : string(""); +} + vector<string> FsManager::GetDataRootDirs() const { // Get the data subdirectory for each data root. return dd_manager_->GetDirs(); diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h index f38d8a2c3..b003e6a80 100644 --- a/src/kudu/fs/fs_manager.h +++ b/src/kudu/fs/fs_manager.h @@ -33,6 +33,7 @@ #include "kudu/fs/error_manager.h" #include "kudu/gutil/macros.h" #include "kudu/gutil/ref_counted.h" +#include "kudu/util/locks.h" // for percpu_rwlock #include "kudu/util/env.h" #include "kudu/util/metrics.h" #include "kudu/util/oid_generator.h" @@ -52,6 +53,7 @@ namespace kudu { class BlockId; class FileCache; class InstanceMetadataPB; +class InstanceMetadataPB_TenantMetadataPB; class MemTracker; class Timer; @@ -205,14 +207,19 @@ class FsManager { std::atomic<int>* containers_total = nullptr ); // Create the initial filesystem layout. If 'uuid' is provided, uses it as - // uuid of the filesystem. Otherwise generates one at random. If 'encryption_key', - // 'encryption_key_iv', and 'encryption_key_version' are provided, they are used as - // the server key of the filesystem. Otherwise, if encryption is enabled, - // generates one at random. + // uuid of the filesystem. Otherwise generates one at random. If 'tenant_name', + // 'tenant_id', 'encryption_key', 'encryption_key_iv', and 'encryption_key_version' are + // provided, they are used as the tenant info of the filesystem. If 'encryption_key', + // 'encryption_key_iv', and 'encryption_key_version' are provided, they are used as the + // server key info of the filesystem. Otherwise, if only '--encrypt_data_at_rest' + // is enabled, generates one server key at random. If both '--enable_multi_tenancy' + // and '--encrypt_data_at_rest' set at same time, generates one default tenant at random. // // Returns an error if the file system is already initialized. Status CreateInitialFileSystemLayout( std::optional<std::string> uuid = std::nullopt, + std::optional<std::string> tenant_name = std::nullopt, + std::optional<std::string> tenant_id = std::nullopt, std::optional<std::string> encryption_key = std::nullopt, std::optional<std::string> encryption_key_iv = std::nullopt, std::optional<std::string> encryption_key_version = std::nullopt); @@ -300,10 +307,45 @@ class FsManager { // Open() have not been called, this will crash. const std::string& uuid() const; - // Return the server key persisted on the local filesystem. After the server - // key is decrypted, it can be used to encrypt/decrypt file keys on the - // filesystem. If PartialOpen() or Open() have not been called, this will - // crash. If the file system is not encrypted, it returns an empty string. + // ========================================================================== + // tenant helpers + // ========================================================================== + + // Use to confirm whether there is tenants information in metadata. + bool is_tenants_exist() const; + + // Use to get the existence of the specific tenant. + bool is_tenant_exist(const std::string& tenant_name) const; + + // Return the initialization uuid for the tenant. + // If PartialOpen() or Open() have not been called, this will + // crash. If the tenant does not exist, it returns an empty string. + std::string tenant_id(const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the tenant key persisted on the local filesystem corresponding to the + // tenant_name. After the tenant key is decrypted, it can be used to encrypt/decrypt + // file keys on the filesystem. If PartialOpen() or Open() have not been called, this + // will crash. If the tenant does not exist, it returns an empty string. + std::string tenant_key(const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the initialization vector for the tenant key. + // If PartialOpen() or Open() have not been called, this will + // crash. If the tenant does not exist, it returns an empty string. + std::string tenant_key_iv(const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the version of the tenant key. + // If PartialOpen() or Open() have not been called, this will + // crash. If the tenant does not exist, it returns an empty string. + std::string tenant_key_version( + const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the server key persisted on the local filesystem. + // + // NOTE : + // During the first upgrade to the multi-tenant version, if it is detected + // that the relevant configuration items of server key have a value, we will + // use it to initialize the default tenant, and the original server key related + // configuration items will be retained at the same time. const std::string& server_key() const; // Return the initialization vector for the server key. @@ -372,6 +414,8 @@ class FsManager { // Create a new InstanceMetadataPB. Status CreateInstanceMetadata(std::optional<std::string> uuid, + std::optional<std::string> tenant_name, + std::optional<std::string> tenant_id, std::optional<std::string> encryption_key, std::optional<std::string> encryption_key_iv, std::optional<std::string> encryption_key_version, @@ -382,6 +426,37 @@ class FsManager { Status WriteInstanceMetadata(const InstanceMetadataPB& metadata, const std::string& root); + // Update metadata after adding tenant or removing tenant. + Status UpdateMetadata( + std::unique_ptr<InstanceMetadataPB> metadata); + + // Search tenant for metadata by tenant name. + const InstanceMetadataPB_TenantMetadataPB* GetTenant(const std::string& tenant_name) const; + // Except that the caller must hold metadata_rwlock_. + const InstanceMetadataPB_TenantMetadataPB* GetTenantUnlock(const std::string& tenant_name) const; + + // Return the tenant id persisted on the local filesystem. + // Except that the caller must hold metadata_rwlock_. + std::string tenant_id_unlock(const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the tenant key persisted on the local filesystem. + // Except that the caller must hold metadata_rwlock_. + std::string tenant_key_unlock(const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the initialization vector for the tenant key unlock. + // Except that the caller must hold metadata_rwlock_. + std::string tenant_key_iv_unlock( + const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Return the version of the tenant key unlock. + // Except that the caller must hold metadata_rwlock_. + std::string tenant_key_version_unlock( + const std::string& tenant_name = fs::kDefaultTenantName) const; + + // Use for update the format and the stamp in the metadata. + // If you need to ensure call security, you must lock it by yourself. + static void UpdateMetadataFormatAndStampUnlock(InstanceMetadataPB* metadata); + // ========================================================================== // file-system helpers // ========================================================================== @@ -428,6 +503,9 @@ class FsManager { CanonicalizedRootsList canonicalized_data_fs_roots_; CanonicalizedRootsList canonicalized_all_fs_roots_; + // Lock protecting the 'metadata_' below. + mutable percpu_rwlock metadata_rwlock_; + // Belongs to the default tenant. std::unique_ptr<InstanceMetadataPB> metadata_; std::unique_ptr<fs::FsErrorManager> error_manager_; diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc index ee1055710..07eff7105 100644 --- a/src/kudu/integration-tests/raft_consensus-itest.cc +++ b/src/kudu/integration-tests/raft_consensus-itest.cc @@ -832,7 +832,9 @@ TEST_P(RaftConsensusParamEncryptionITest, TestCatchupAfterOpsEvicted) { string encryption_key; string encryption_key_iv; string encryption_key_version; - GetEncryptionKey(&encryption_key, &encryption_key_iv, &encryption_key_version); + // TODO(kedeng) : add tenant key info test + GetEncryptionKey(nullptr, nullptr, &encryption_key, + &encryption_key_iv, &encryption_key_version); kTsFlags.emplace_back("--test_server_key=" + encryption_key); kTsFlags.emplace_back("--test_server_key_iv=" + encryption_key_iv); kTsFlags.emplace_back("--test_server_key_version=" + encryption_key_version); diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc b/src/kudu/mini-cluster/external_mini_cluster.cc index f16224968..b863ff7e6 100644 --- a/src/kudu/mini-cluster/external_mini_cluster.cc +++ b/src/kudu/mini-cluster/external_mini_cluster.cc @@ -1374,19 +1374,37 @@ Env* ExternalDaemon::env() const { return Env::Default(); } -Status ExternalDaemon::SetEncryptionKey() { +Status ExternalDaemon::SetEncryptionKey(const string& tenant_name) { string path = JoinPathSegments(this->wal_dir(), "instance");; LOG(INFO) << "Reading " << path; InstanceMetadataPB instance; RETURN_NOT_OK(pb_util::ReadPBContainerFromPath(env(), path, &instance, pb_util::NOT_SENSITIVE)); - if (!instance.server_key().empty()) { - string key; + string decrypted_key; + if (!instance.tenants().empty()) { + const InstanceMetadataPB::TenantMetadataPB* tenant = nullptr; + for (const auto& tdata : instance.tenants()) { + if (tdata.tenant_name() == tenant_name) { + tenant = &tdata; + break; + } + } + if (tenant && !tenant->tenant_key().empty()) { + RETURN_NOT_OK(key_provider_->DecryptEncryptionKey(tenant->tenant_key(), + tenant->tenant_key_iv(), + tenant->tenant_key_version(), + &decrypted_key)); + } + } else if (!instance.server_key().empty()) { RETURN_NOT_OK(key_provider_->DecryptEncryptionKey(instance.server_key(), instance.server_key_iv(), instance.server_key_version(), - &key)); - LOG(INFO) << "Setting key " << key; - env()->SetEncryptionKey(reinterpret_cast<const uint8_t*>(a2b_hex(key).c_str()), key.size() * 4); + &decrypted_key)); + } + + if (!decrypted_key.empty()) { + LOG(INFO) << "Setting key " << decrypted_key; + env()->SetEncryptionKey(reinterpret_cast<const uint8_t*>(a2b_hex(decrypted_key).c_str()), + decrypted_key.size() * 4); } return Status::OK(); } diff --git a/src/kudu/mini-cluster/external_mini_cluster.h b/src/kudu/mini-cluster/external_mini_cluster.h index f715b3feb..90dcccafe 100644 --- a/src/kudu/mini-cluster/external_mini_cluster.h +++ b/src/kudu/mini-cluster/external_mini_cluster.h @@ -667,6 +667,8 @@ struct ExternalDaemonOptions { std::string ranger_cluster_key; }; +static const std::string kDefaultTenantName = "default_tenant_kudu"; + class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> { public: explicit ExternalDaemon(ExternalDaemonOptions opts); @@ -722,7 +724,7 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> { const std::string& principal_base, const std::string& bind_host); - Status SetEncryptionKey(); + Status SetEncryptionKey(const std::string& tenant_name = kDefaultTenantName); // Sends a SIGSTOP signal to the daemon. Status Pause() WARN_UNUSED_RESULT; diff --git a/src/kudu/mini-cluster/internal_mini_cluster.cc b/src/kudu/mini-cluster/internal_mini_cluster.cc index 932453ac2..0c146398f 100644 --- a/src/kudu/mini-cluster/internal_mini_cluster.cc +++ b/src/kudu/mini-cluster/internal_mini_cluster.cc @@ -20,6 +20,7 @@ #include <cstdint> #include <memory> #include <ostream> +#include <type_traits> #include <unordered_set> #include <utility> @@ -33,9 +34,9 @@ #include "kudu/master/master.proxy.h" #include "kudu/master/master_options.h" #include "kudu/master/mini_master.h" -#include "kudu/master/ts_descriptor.h" #include "kudu/master/ts_manager.h" #include "kudu/rpc/messenger.h" +#include "kudu/server/server_base_options.h" #include "kudu/tablet/tablet.h" #include "kudu/tablet/tablet_replica.h" #include "kudu/tserver/mini_tablet_server.h" @@ -157,9 +158,12 @@ Status InternalMiniCluster::StartMasters() { auto mini_master(std::make_shared<MiniMaster>( GetMasterFsRoot(i), master_rpc_addrs[i])); auto* options = mini_master->mutable_options(); - KuduTest::GetEncryptionKey(&options->server_key, - &options->server_key_iv, - &options->server_key_version); + // TODO(kedeng) : add tenant key info test + KuduTest::GetEncryptionKey(nullptr, + nullptr, + &options->server_key_info.server_key, + &options->server_key_info.server_key_iv, + &options->server_key_info.server_key_version); if (num_masters > 1 || opts_.supply_single_master_addr) { mini_master->SetMasterAddresses(master_rpc_addrs); } @@ -218,9 +222,12 @@ Status InternalMiniCluster::AddTabletServer(const HostPort& hp) { new MiniTabletServer(GetTabletServerFsRoot(new_idx), hp, opts_.num_data_dirs)); tablet_server->options()->master_addresses = master_rpc_addrs(); auto* options = tablet_server->options(); - KuduTest::GetEncryptionKey(&options->server_key, - &options->server_key_iv, - &options->server_key_version); + // TODO(kedeng) : add tenant key info test + KuduTest::GetEncryptionKey(nullptr, + nullptr, + &options->server_key_info.server_key, + &options->server_key_info.server_key_iv, + &options->server_key_info.server_key_version); RETURN_NOT_OK(tablet_server->Start()); mini_tablet_servers_.emplace_back(std::move(tablet_server)); diff --git a/src/kudu/ranger-kms/ranger_kms_client.cc b/src/kudu/ranger-kms/ranger_kms_client.cc index 279735c31..ad6ab3010 100644 --- a/src/kudu/ranger-kms/ranger_kms_client.cc +++ b/src/kudu/ranger-kms/ranger_kms_client.cc @@ -68,6 +68,7 @@ Status RangerKMSClient::DecryptEncryptionKey(const string& encryption_key, RETURN_NOT_OK_PREPEND( curl.PostToURL(urls, payload.ToString(), &resp, {"Content-Type: application/json"}), "failed to decrypt encryption key"); + JsonReader r(resp.ToString()); RETURN_NOT_OK(r.Init()); string dek_b64; diff --git a/src/kudu/server/server_base.cc b/src/kudu/server/server_base.cc index 1dcae38dd..af3022137 100644 --- a/src/kudu/server/server_base.cc +++ b/src/kudu/server/server_base.cc @@ -766,14 +766,27 @@ Status ServerBase::Init() { if (s.IsNotFound()) { LOG(INFO) << "This appears to be a new deployment of Kudu; creating new FS layout"; is_first_run_ = true; - if (options_.server_key.empty()) { + + if (options_.server_key_info.server_key.empty() && + options_.tenant_key_info.tenant_key.empty()) { s = fs_manager_->CreateInitialFileSystemLayout(); + } else if (!options_.tenant_key_info.tenant_key.empty()) { + // The priority of tenant key is higher than that of server key. + s = fs_manager_->CreateInitialFileSystemLayout(std::nullopt, + options_.tenant_key_info.tenant_name, + options_.tenant_key_info.tenant_id, + options_.tenant_key_info.tenant_key, + options_.tenant_key_info.tenant_key_iv, + options_.tenant_key_info.tenant_key_version); } else { s = fs_manager_->CreateInitialFileSystemLayout(std::nullopt, - options_.server_key, - options_.server_key_iv, - options_.server_key_version); + std::nullopt, + std::nullopt, + options_.server_key_info.server_key, + options_.server_key_info.server_key_iv, + options_.server_key_info.server_key_version); } + if (s.IsAlreadyPresent()) { return s.CloneAndPrepend("FS layout already exists; not overwriting existing layout"); } diff --git a/src/kudu/server/server_base_options.cc b/src/kudu/server/server_base_options.cc index d2d984b19..8de5e60ac 100644 --- a/src/kudu/server/server_base_options.cc +++ b/src/kudu/server/server_base_options.cc @@ -60,17 +60,60 @@ DEFINE_string(test_server_key_version, "", "It is only used when creating the file system, it's disregarded on " "consecutive startups. It should only be used in tests."); TAG_FLAG(test_server_key_version, hidden); + +DEFINE_string(test_tenant_name, "", + "Tenant name in plain-text to be persisted into the instance file. " + "It is only used when creating the file system, it's disregarded on " + "consecutive startups. It should only be used in tests."); +TAG_FLAG(test_tenant_name, hidden); + +DEFINE_string(test_tenant_id, "", + "Tenant id in plain-text to be persisted into the instance file. " + "It is only used when creating the file system, it's disregarded on " + "consecutive startups. It should only be used in tests."); +TAG_FLAG(test_tenant_id, hidden); + +DEFINE_string(test_tenant_key, "", + "Tenant key in plain-text to be persisted into the instance file. " + "It is only used when creating the file system, it's disregarded on " + "consecutive startups. It should only be used in tests."); +TAG_FLAG(test_tenant_key, hidden); + +DEFINE_string(test_tenant_key_iv, "", + "Tenant key IV in plain-text to be persisted into the instance file. " + "It is only used when creating the file system, it's disregarded on " + "consecutive startups. It should only be used in tests."); +TAG_FLAG(test_tenant_key_iv, hidden); + + +DEFINE_string(test_tenant_key_version, "", + "Tenant key version in plain-text to be persisted into the instance file. " + "It is only used when creating the file system, it's disregarded on " + "consecutive startups. It should only be used in tests."); +TAG_FLAG(test_tenant_key_version, hidden); + namespace kudu { namespace server { +ServerKeyInfo::ServerKeyInfo() + : server_key(FLAGS_test_server_key), + server_key_iv(FLAGS_test_server_key_iv), + server_key_version(FLAGS_test_server_key_version) { +} + +TenantKeyInfo::TenantKeyInfo() + : tenant_name(FLAGS_test_tenant_name), + tenant_id(FLAGS_test_tenant_id), + tenant_key(FLAGS_test_tenant_key), + tenant_key_iv(FLAGS_test_tenant_key_iv), + tenant_key_version(FLAGS_test_tenant_key_version) { +} + ServerBaseOptions::ServerBaseOptions() : env(Env::Default()), dump_info_path(FLAGS_server_dump_info_path), dump_info_format(FLAGS_server_dump_info_format), - metrics_log_interval_ms(FLAGS_metrics_log_interval_ms), - server_key(FLAGS_test_server_key), - server_key_iv(FLAGS_test_server_key_iv), - server_key_version(FLAGS_test_server_key_version) { + metrics_log_interval_ms(FLAGS_metrics_log_interval_ms) { } } // namespace server diff --git a/src/kudu/server/server_base_options.h b/src/kudu/server/server_base_options.h index 133cf83da..ebed818a4 100644 --- a/src/kudu/server/server_base_options.h +++ b/src/kudu/server/server_base_options.h @@ -30,6 +30,26 @@ class Env; namespace server { +struct ServerKeyInfo { + std::string server_key; + std::string server_key_iv; + std::string server_key_version; + + public: + ServerKeyInfo(); +}; + +struct TenantKeyInfo { + std::string tenant_name; + std::string tenant_id; + std::string tenant_key; + std::string tenant_key_iv; + std::string tenant_key_version; + + public: + TenantKeyInfo(); +}; + // Options common to both types of servers. // The subclass constructor should fill these in with defaults from // server-specific flags. @@ -45,9 +65,8 @@ struct ServerBaseOptions { int32_t metrics_log_interval_ms; - std::string server_key; - std::string server_key_iv; - std::string server_key_version; + ServerKeyInfo server_key_info; + TenantKeyInfo tenant_key_info; protected: ServerBaseOptions(); diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 60c707ce9..00c0183d0 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -1871,6 +1871,32 @@ TEST_F(ToolTest, TestFsFormatWithServerKey) { ASSERT_EQ(server_key_version, fs.server_key_version()); } +TEST_F(ToolTest, TestFsFormatWithTenantKey) { + const string kTestDir = GetTestPath("test"); + ObjectIdGenerator generator; + string original_uuid = generator.Next(); + string tenant_name = "default_tenant_kudu"; + string tenant_id = "00000000000000000000000000000000"; + string tenant_key = "00010203040506070809101112131442"; + string tenant_key_iv = "42141312111009080706050403020100"; + string tenant_key_version = "kudutenantkey@0"; + NO_FATALS(RunActionStdoutNone(Substitute( + "fs format --fs_wal_dir=$0 --encryption_key_type=$1 --uuid=$2 --tenant_name=$3 " + "--tenant_id=$4 --tenant_key=$5 --tenant_key_iv=$6 --tenant_key_version=$7", + kTestDir, "tenant_key", original_uuid, tenant_name, tenant_id, tenant_key, tenant_key_iv, + tenant_key_version))); + FsManager fs(env_, FsManagerOpts(kTestDir)); + ASSERT_OK(fs.Open()); + + string canonicalized_uuid; + ASSERT_OK(generator.Canonicalize(fs.uuid(), &canonicalized_uuid)); + ASSERT_EQ(canonicalized_uuid, fs.uuid()); + ASSERT_EQ(original_uuid, fs.uuid()); + ASSERT_EQ(tenant_key, fs.tenant_key()); + ASSERT_EQ(tenant_key_iv, fs.tenant_key_iv()); + ASSERT_EQ(tenant_key_version, fs.tenant_key_version()); +} + TEST_F(ToolTest, TestFsDumpUuid) { const string kTestDir = GetTestPath("test"); string uuid; @@ -1891,8 +1917,8 @@ TEST_F(ToolTest, TestPbcTools) { // It's pointless to run these tests in an encrypted environment, as it uses // instance files to test pbc tools, which are not encrypted anyway. The tests // also make assumptions about the contents of the instance files, which are - // different on encrypted servers, as they contain an extra server_key field, - // which would make these tests break. + // different on encrypted servers, as they contain an extra server_key/tenant_key + // field, which would make these tests break. if (FLAGS_encrypt_data_at_rest) { GTEST_SKIP(); } @@ -9034,7 +9060,7 @@ TEST_F(ToolTest, TestLocalReplicaCopyLocal) { // replica copy shares an Env between the source and the destination and they // use different instance files. This shouldn't be a problem in real life, as // its meant to copy tablets between disks on the same server, which would - // share UUIDs and server keys. + // share UUIDs and encryption keys. if (FLAGS_encrypt_data_at_rest) { GTEST_SKIP(); } diff --git a/src/kudu/tools/tool_action_common.cc b/src/kudu/tools/tool_action_common.cc index 95407aff8..de08344ed 100644 --- a/src/kudu/tools/tool_action_common.cc +++ b/src/kudu/tools/tool_action_common.cc @@ -171,10 +171,30 @@ DECLARE_bool(show_values); DEFINE_string(instance_file, "", "Path to the instance file containing the encrypted encryption key."); +DEFINE_string(encryption_key_type, + "server_key", + "The type of encryption. Must be one of 'tenant_key' or 'server_key'." + "To compatible with older versions, the default value is 'server_key'."); + +static bool ValidateEncryptionKeyType(const char* flag_name, const std::string& value) { + if (kudu::iequals(value, "server_key") || + kudu::iequals(value, "tenant_key")) { + return true; + } + LOG(ERROR) << strings::Substitute("unknown value for --$0 flag: '$1' " + "(expected one of 'tenant_key' or 'server_key').", + flag_name, value); + return false; +} +DEFINE_validator(encryption_key_type, &ValidateEncryptionKeyType); + DECLARE_string(encryption_key_provider); DECLARE_string(ranger_kms_url); DECLARE_string(encryption_cluster_key_name); +DEFINE_string(tenant_name, "", + "The encrypted tenant name to use in the filesystem."); + bool ValidateTimeoutSettings() { if (FLAGS_timeout_ms < FLAGS_negotiation_timeout_ms) { LOG(ERROR) << strings::Substitute( @@ -965,25 +985,45 @@ Status SetEncryptionKey() { &instance, pb_util::NOT_SENSITIVE), "Could not open instance file"); - if (string key = instance.server_key(); - !key.empty()) { - unique_ptr<security::KeyProvider> key_provider; - if (FLAGS_encryption_key_provider == "ranger-kms" - || FLAGS_encryption_key_provider == "ranger_kms") { - key_provider.reset(new RangerKMSKeyProvider(FLAGS_ranger_kms_url, - FLAGS_encryption_cluster_key_name)); - } else { - key_provider.reset(new DefaultKeyProvider()); + string decrypted_key; + unique_ptr<security::KeyProvider> key_provider; + if (FLAGS_encryption_key_provider == "ranger-kms" + || FLAGS_encryption_key_provider == "ranger_kms") { + key_provider.reset(new RangerKMSKeyProvider(FLAGS_ranger_kms_url, + FLAGS_encryption_cluster_key_name)); + } else { + key_provider.reset(new DefaultKeyProvider()); + } + + if (iequals(FLAGS_encryption_key_type, "server_key")) { + const string& server_key = instance.server_key(); + if (!server_key.empty()) { + RETURN_NOT_OK(key_provider->DecryptEncryptionKey(instance.server_key(), + instance.server_key_iv(), + instance.server_key_version(), + &decrypted_key)); + } + } else { + InstanceMetadataPB::TenantMetadataPB tenant; + for (const auto& tdata : instance.tenants()) { + if (tdata.tenant_name() == FLAGS_tenant_name) { + tenant.CopyFrom(tdata); + break; + } + } + + if (tenant.has_tenant_key()) { + RETURN_NOT_OK(key_provider->DecryptEncryptionKey(tenant.tenant_key(), + tenant.tenant_key_iv(), + tenant.tenant_key_version(), + &decrypted_key)); } + } - string decrypted_key; - RETURN_NOT_OK(key_provider->DecryptEncryptionKey(instance.server_key(), - instance.server_key_iv(), - instance.server_key_version(), - &decrypted_key)); + if (!decrypted_key.empty()) { Env::Default()->SetEncryptionKey(reinterpret_cast<const uint8_t*>( a2b_hex(decrypted_key).c_str()), - key.length() * 4); + decrypted_key.size() * 4); } return Status::OK(); diff --git a/src/kudu/tools/tool_action_fs.cc b/src/kudu/tools/tool_action_fs.cc index 505e0da13..48f0f7cd8 100644 --- a/src/kudu/tools/tool_action_fs.cc +++ b/src/kudu/tools/tool_action_fs.cc @@ -105,6 +105,33 @@ bool ServerKeySetTogether() { GROUP_FLAG_VALIDATOR(server_key_set_together, ServerKeySetTogether); +DECLARE_string(encryption_key_type); +DECLARE_string(tenant_name); + +DEFINE_string(tenant_id, "", + "The encrypted tenant id to use in the filesystem."); +DEFINE_string(tenant_key, "", + "The encrypted tenant key to use in the filesystem."); +DEFINE_string(tenant_key_iv, "", + "The tenant key IV to use in the filesystem."); +DEFINE_string(tenant_key_version, "", + "The tenant key version to use in the filesystem."); + +bool TenantKeySetTogether() { + if (FLAGS_tenant_name.empty() != FLAGS_tenant_id.empty() + || FLAGS_tenant_name.empty() != FLAGS_tenant_key.empty() + || FLAGS_tenant_name.empty() != FLAGS_tenant_key_iv.empty() + || FLAGS_tenant_name.empty() != FLAGS_tenant_key_version.empty()) { + LOG(ERROR) << "'tenant_name', 'tenant_id', 'tenant_key', 'tenant_key_iv', and " + "'tenant_key_version' must either all be set, or none of them " + "must be set."; + return false; + } + return true; +} + +GROUP_FLAG_VALIDATOR(tenant_key_set_together, TenantKeySetTogether); + DEFINE_bool(repair, false, "Repair any inconsistencies in the filesystem."); @@ -247,20 +274,33 @@ Status Check(const RunnerContext& /*context*/) { Status Format(const RunnerContext& /*context*/) { FsManager fs_manager(Env::Default()); optional<string> uuid; + optional<string> tenant_name; + optional<string> tenant_id; optional<string> encryption_key; optional<string> encryption_key_iv; optional<string> encryption_key_version; if (!FLAGS_uuid.empty()) { uuid = FLAGS_uuid; } - if (!FLAGS_server_key.empty() - && !FLAGS_server_key_iv.empty() - && !FLAGS_server_key_version.empty()) { - encryption_key = FLAGS_server_key; - encryption_key_iv = FLAGS_server_key_iv; - encryption_key_version = FLAGS_server_key_version; + if (iequals(FLAGS_encryption_key_type, "server_key")) { + if (!FLAGS_server_key.empty() + && !FLAGS_server_key_iv.empty() + && !FLAGS_server_key_version.empty()) { + encryption_key = FLAGS_server_key; + encryption_key_iv = FLAGS_server_key_iv; + encryption_key_version = FLAGS_server_key_version; + } + } else { + if (!FLAGS_tenant_name.empty()) { + tenant_name = FLAGS_tenant_name; + tenant_id = FLAGS_tenant_id; + encryption_key = FLAGS_tenant_key; + encryption_key_iv = FLAGS_tenant_key_iv; + encryption_key_version = FLAGS_tenant_key_version; + } } - return fs_manager.CreateInitialFileSystemLayout(uuid, encryption_key, + + return fs_manager.CreateInitialFileSystemLayout(uuid, tenant_name, tenant_id, encryption_key, encryption_key_iv, encryption_key_version); } @@ -910,6 +950,7 @@ unique_ptr<Mode> BuildFsMode() { .AddOptionalParameter("fs_metadata_dir") .AddOptionalParameter("fs_wal_dir") .AddOptionalParameter("uuid") + .AddOptionalParameter("encryption_key_type") .Build(); unique_ptr<Action> update = diff --git a/src/kudu/tserver/tablet_copy_client-test.cc b/src/kudu/tserver/tablet_copy_client-test.cc index ed1f87380..56abb4a5a 100644 --- a/src/kudu/tserver/tablet_copy_client-test.cc +++ b/src/kudu/tserver/tablet_copy_client-test.cc @@ -91,6 +91,7 @@ using kudu::consensus::GetRaftConfigLeader; using kudu::consensus::RaftPeerPB; using kudu::fs::DataDirManager; using kudu::tablet::TabletMetadata; +using std::nullopt; using std::shared_ptr; using std::string; using std::thread; @@ -124,14 +125,26 @@ class TabletCopyClientTest : public TabletCopyTest { metric_entity_ = METRIC_ENTITY_server.Instantiate(&metric_registry_, "test"); opts.metric_entity = metric_entity_; fs_manager_.reset(new FsManager(Env::Default(), opts)); + string tenant_name; + string tenant_id; string encryption_key; string encryption_key_iv; string encryption_key_version; - GetEncryptionKey(&encryption_key, &encryption_key_iv, &encryption_key_version); - if (encryption_key.empty()) { + GetEncryptionKey(&tenant_name, &tenant_id, &encryption_key, + &encryption_key_iv, &encryption_key_version); + if (tenant_name.empty() && encryption_key.empty()) { ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout()); + } else if (tenant_name.empty()) { + ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout(nullopt, + nullopt, + nullopt, + encryption_key, + encryption_key_iv, + encryption_key_version)); } else { - ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout(std::nullopt, + ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout(nullopt, + tenant_name, + tenant_id, encryption_key, encryption_key_iv, encryption_key_version)); diff --git a/src/kudu/tserver/tablet_server-test-base.cc b/src/kudu/tserver/tablet_server-test-base.cc index c28f9ebac..6f9aef844 100644 --- a/src/kudu/tserver/tablet_server-test-base.cc +++ b/src/kudu/tserver/tablet_server-test-base.cc @@ -99,7 +99,9 @@ TabletServerTestBase::TabletServerTestBase() // purposefully specify non-running Master servers. FLAGS_heartbeat_rpc_timeout_ms = 1000; - GetEncryptionKey(&FLAGS_test_server_key, + GetEncryptionKey(nullptr, + nullptr, + &FLAGS_test_server_key, &FLAGS_test_server_key_iv, &FLAGS_test_server_key_version); } diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc index c5fad1d06..e7c69f186 100644 --- a/src/kudu/util/env-test.cc +++ b/src/kudu/util/env-test.cc @@ -39,10 +39,12 @@ #include <cstdint> #include <cstdlib> #include <functional> +#include <map> #include <memory> #include <ostream> #include <string> #include <thread> +#include <type_traits> #include <unordered_set> #include <utility> #include <vector> @@ -80,7 +82,6 @@ DECLARE_int32(env_inject_short_read_bytes); DECLARE_int32(env_inject_short_write_bytes); DECLARE_int32(encryption_key_length); DECLARE_string(env_inject_eio_globs); -DECLARE_string(encryption_server_key); namespace kudu { diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc index 3f8302971..519699c77 100644 --- a/src/kudu/util/env_posix.cc +++ b/src/kudu/util/env_posix.cc @@ -57,6 +57,7 @@ #include "kudu/util/errno.h" #include "kudu/util/fault_injection.h" #include "kudu/util/flag_tags.h" +#include "kudu/util/flag_validators.h" #include "kudu/util/flags.h" #include "kudu/util/logging.h" #include "kudu/util/malloc.h" @@ -209,6 +210,24 @@ TAG_FLAG(encryption_key_length, advanced); DEFINE_validator(encryption_key_length, [](const char* /*n*/, int32 v) { return v == 128 || v == 192 || v == 256; }); +DEFINE_bool(enable_multi_tenancy, false, + "Whether enable the multi tenancy feature." + "Should set together with --encrypt_data_at_rest"); +TAG_FLAG(enable_multi_tenancy, advanced); +TAG_FLAG(enable_multi_tenancy, experimental); + +bool ValidateMultiTenancySettings() { + if (FLAGS_enable_multi_tenancy && !FLAGS_encrypt_data_at_rest) { + LOG(ERROR) << strings::Substitute( + "The --enable_multi_tenancy can be set 'true' only when --encrypt_data_at_rest " + "is set 'true'. Current settings are $0 and $1 correspondingly", + FLAGS_enable_multi_tenancy, FLAGS_encrypt_data_at_rest); + return false; + } + return true; +} +GROUP_FLAG_VALIDATOR(enable_multi_tenancy, ValidateMultiTenancySettings); + static __thread uint64_t thread_local_id; static Atomic64 cur_thread_local_id_; diff --git a/src/kudu/util/flag_validators-test.cc b/src/kudu/util/flag_validators-test.cc index 77efab3a7..6b2a5fca8 100644 --- a/src/kudu/util/flag_validators-test.cc +++ b/src/kudu/util/flag_validators-test.cc @@ -77,7 +77,7 @@ class FlagsValidatorsBasicTest : public KuduTest { TEST_F(FlagsValidatorsBasicTest, Grouped) { const auto& validators = GetFlagValidators(); - ASSERT_EQ(2, validators.size()); + ASSERT_EQ(3, validators.size()); const auto& it = validators.find("test_group_validator01"); ASSERT_NE(validators.end(), it); const auto& validator = it->second; diff --git a/src/kudu/util/test_util.cc b/src/kudu/util/test_util.cc index f13e68ab3..cd551bf1c 100644 --- a/src/kudu/util/test_util.cc +++ b/src/kudu/util/test_util.cc @@ -67,6 +67,7 @@ DEFINE_string(test_leave_files, "on_failure", DEFINE_int32(test_random_seed, 0, "Random seed to use for randomized tests"); DECLARE_string(time_source); +DECLARE_bool(enable_multi_tenancy); DECLARE_bool(encrypt_data_at_rest); using std::string; @@ -88,7 +89,9 @@ static const uint8_t kEncryptionKey[kEncryptionKeySize] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42}; static const uint8_t kEncryptionKeyIv[kEncryptionKeySize] = {42, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; -static const char* const kEncryptionKeyVersion = "kuduclusterkey@0"; +static const char* const kEncryptionKeyVersion = "kudutenantkey@0"; +static const char* const kEncryptionTenantName = "default_tenant_kudu"; +static const char* const kEncryptionTenantID = "00000000000000000000000000000000"; static const uint64_t kTestBeganAtMicros = Env::Default()->NowMicros(); @@ -208,12 +211,21 @@ void KuduTest::SetEncryptionFlags(bool enable_encryption) { } } -void KuduTest::GetEncryptionKey(string* key, string* iv, string* version) { +void KuduTest::GetEncryptionKey(string* name, string* id, string* key, string* iv, + string* version) { if (FLAGS_encrypt_data_at_rest) { + if (FLAGS_enable_multi_tenancy && name && id) { + *name = kEncryptionTenantName; + *id = kEncryptionTenantID; + } strings::b2a_hex(kEncryptionKey, key, kEncryptionKeySize); strings::b2a_hex(kEncryptionKeyIv, iv, kEncryptionKeySize); *version = kEncryptionKeyVersion; } else { + if (name && id) { + *name = ""; + *id = ""; + } *key = ""; *iv = ""; *version = ""; diff --git a/src/kudu/util/test_util.h b/src/kudu/util/test_util.h index b54bd83ad..2e8e077b7 100644 --- a/src/kudu/util/test_util.h +++ b/src/kudu/util/test_util.h @@ -73,8 +73,9 @@ class KuduTest : public ::testing::Test { // variables so that we don't pick up the user's credentials. static void OverrideKrb5Environment(); - // Returns the encryption key, IV, and version used by the test. - static void GetEncryptionKey(std::string* key, std::string* iv, std::string* version); + // Returns the encryption tenant name, tenant id, key, IV, and version used by the test. + static void GetEncryptionKey(std::string* name, std::string* id, + std::string* key, std::string* iv, std::string* version); protected: // Returns absolute path based on a unit test-specific work directory, given
