This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new f33757512 [multi-tenancy] add a server key upgrade tool
f33757512 is described below
commit f337575127c8cfb73b684998866478bc7f0a8e1c
Author: kedeng <[email protected]>
AuthorDate: Wed Jun 21 20:14:44 2023 +0800
[multi-tenancy] add a server key upgrade tool
For the purpose of easily upgrading the server key to the
tenant key, I have implemented the corresponding function
in this patch.
With the help of a command-line tool, such as
`kudu fs upgrade_encryption_key --fs_data_dirs=xx --fs_wal_dir=xx
--fs_metadata_dir=xx`,
we can easily complete the metadata upgrade for a single
tserver. After all tservers have been upgraded, the previous
server key info will be converted into the encrypted information
of the default tenant, and the metadata will no longer retain
the previous server key info.
In addition, after the upgrade is completed, the multi-tenant
enable flag '--enable_multi_tenancy' needs to be set to true to
complete the entire upgrade process.
I have also added unit tests for this process to ensure that
this logic can be executed correctly.
Change-Id: I2fe50fdcdf294b5955756dfcb92d3b627534bcc7
Reviewed-on: http://gerrit.cloudera.org:8080/20103
Reviewed-by: Yingchun Lai <[email protected]>
Tested-by: Yingchun Lai <[email protected]>
---
src/kudu/fs/fs_manager.cc | 11 +++++
src/kudu/fs/fs_manager.h | 12 +++++
src/kudu/tools/kudu-tool-test.cc | 104 ++++++++++++++++++++++++++++++++++++++-
src/kudu/tools/tool_action_fs.cc | 71 ++++++++++++++++++++++++++
4 files changed, 197 insertions(+), 1 deletion(-)
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 6eb35c48f..1efe2ce43 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -624,6 +624,12 @@ Status FsManager::Open(FsReport* report, Timer*
read_instance_metadata_files,
return Status::OK();
}
+void FsManager::CopyMetadata(
+ unique_ptr<InstanceMetadataPB>* metadata) {
+ shared_lock<rw_spinlock> md_lock(metadata_rwlock_.get_lock());
+ (*metadata)->CopyFrom(*metadata_);
+}
+
Status FsManager::UpdateMetadata(unique_ptr<InstanceMetadataPB> metadata) {
// In the event of failure, rollback everything we changed.
// <string, string> <=> <old instance file, backup instance file>
@@ -927,6 +933,11 @@ const string& FsManager::server_key_version() const {
return CHECK_NOTNULL(metadata_.get())->server_key_version();
}
+const int32_t FsManager::tenants_count() const {
+ shared_lock<rw_spinlock> md_lock(metadata_rwlock_.get_lock());
+ return metadata_->tenants_size();
+}
+
bool FsManager::is_tenants_exist() const {
shared_lock<rw_spinlock> md_lock(metadata_rwlock_.get_lock());
return metadata_->tenants_size() > 0;
diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h
index b003e6a80..3313d7d19 100644
--- a/src/kudu/fs/fs_manager.h
+++ b/src/kudu/fs/fs_manager.h
@@ -82,6 +82,10 @@ namespace tserver {
class MiniTabletServerTest_TestFsLayoutEndToEnd_Test;
} // namespace tserver
+namespace tools {
+Status UpdateEncryptionKeyInfo(Env* env);
+} // namespace tools
+
// Options that control the behavior of FsManager.
struct FsManagerOpts {
// Creates a new FsManagerOpts with default values.
@@ -307,10 +311,17 @@ class FsManager {
// Open() have not been called, this will crash.
const std::string& uuid() const;
+ // Copy the metadata_ to metadata.
+ void CopyMetadata(
+ std::unique_ptr<InstanceMetadataPB>* metadata);
+
// ==========================================================================
// tenant helpers
// ==========================================================================
+ // Use to get the total count of all the tenants.
+ const int32_t tenants_count() const;
+
// Use to confirm whether there is tenants information in metadata.
bool is_tenants_exist() const;
@@ -390,6 +401,7 @@ class FsManager {
FRIEND_TEST(fs::FsManagerTestBase, TestOpenWithDuplicateInstanceFiles);
FRIEND_TEST(tserver::MiniTabletServerTest, TestFsLayoutEndToEnd);
friend class itest::MiniClusterFsInspector; // for access to directory names
+ friend Status tools::UpdateEncryptionKeyInfo(Env* env); // for update the
metadata
// Initializes, sanitizes, and canonicalizes the filesystem roots.
// Determines the correct filesystem root for tablet-specific metadata.
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 9bcbd91ab..ddcac9dbe 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -70,6 +70,7 @@
#include "kudu/consensus/ref_counted_replicate.h"
#include "kudu/fs/block_id.h"
#include "kudu/fs/block_manager.h"
+#include "kudu/fs/dir_manager.h"
#include "kudu/fs/fs_manager.h"
#include "kudu/fs/fs_report.h"
#include "kudu/fs/log_block_manager.h"
@@ -145,6 +146,7 @@ DECLARE_bool(allow_unsafe_replication_factor);
DECLARE_bool(catalog_manager_check_ts_count_for_create_table);
DECLARE_bool(enable_tablet_orphaned_block_deletion);
DECLARE_bool(encrypt_data_at_rest);
+DECLARE_bool(enable_multi_tenancy);
DECLARE_bool(disable_gflag_filter_logic_for_testing);
DECLARE_bool(fs_data_dirs_consider_available_space);
DECLARE_bool(hive_metastore_sasl_enabled);
@@ -1361,8 +1363,9 @@ TEST_F(ToolTest, TestModeHelp) {
"check.*Kudu filesystem for inconsistencies",
"dump.*Dump a Kudu filesystem",
"format.*new Kudu filesystem",
- "list.*List metadata for on-disk tablets, rowsets, blocks",
+ "list.*List metadata for on-disk tablets, rowsets",
"update_dirs.*Updates the set of data directories",
+ "upgrade_encryption_key.*Upgrade the encryption key info",
};
NO_FATALS(RunTestHelp("fs", kFsModeRegexes));
NO_FATALS(RunTestHelp("fs not_a_mode", kFsModeRegexes,
@@ -8089,6 +8092,105 @@ TEST_F(ToolTest,
TestFsSwappingDirectoriesFailsGracefully) {
wal_root, JoinStrings(new_data_roots, ","))));
}
+TEST_F(ToolTest,
TestFsUpgradeEncryptionKeyFromServerKeyInfoAndFromTenantKeyInfo) {
+ // Create a cluster with server key info.
+ FLAGS_encrypt_data_at_rest = true;
+
+ NO_FATALS(StartMiniCluster());
+ MiniTabletServer* mts = mini_cluster_->mini_tablet_server(0);
+ const string& wal_root = mts->options()->fs_opts.wal_root;
+ vector<string> data_roots = mts->options()->fs_opts.data_roots;
+ const string& metadata_root = mts->options()->fs_opts.metadata_root;
+
+ // Obtain source encryption server key info.
+ FsManager* fs_manager = mts->server()->fs_manager();
+ ASSERT_FALSE(fs_manager->server_key().empty());
+ const string source_key = fs_manager->server_key();
+ ASSERT_FALSE(fs_manager->server_key_iv().empty());
+ const string source_key_iv = fs_manager->server_key_iv();
+ ASSERT_FALSE(fs_manager->server_key_version().empty());
+ const string source_key_version = fs_manager->server_key_version();
+ // No tenant key info exist.
+ ASSERT_FALSE(fs_manager->is_tenants_exist());
+
+ mts->Shutdown();
+
+ // Do the encryption key info upgrade with the kudu CLI tool.
+ // Providing the necessary arguments, the tool should work.
+ string stdout;
+ NO_FATALS(RunActionStdoutString(Substitute(
+ "fs upgrade_encryption_key --fs_data_dirs=$0 --fs_wal_dir=$1
--fs_metadata_dir=$2",
+ JoinStrings(data_roots, ","), wal_root, metadata_root), &stdout));
+ SCOPED_TRACE(stdout);
+
+ // This flag needs to set true after the upgrading.
+ FLAGS_enable_multi_tenancy = true;
+ // Start the tserver and check the encryption key info.
+ ASSERT_OK(mts->Start());
+ ASSERT_OK(mts->WaitStarted());
+
+ {
+ FsManager* fs_manager = mts->server()->fs_manager();
+ // The default tenant key info corresponds to the previous server key
information.
+ ASSERT_TRUE(fs_manager->is_tenants_exist());
+ ASSERT_TRUE(fs_manager->is_tenant_exist(fs::kDefaultTenantName));
+ // Make sure there is only one tenant exist.
+ ASSERT_EQ(1, fs_manager->tenants_count());
+ ASSERT_FALSE(fs_manager->tenant_id(fs::kDefaultTenantName).empty());
+ ASSERT_EQ(fs::kDefaultTenantID,
fs_manager->tenant_id(fs::kDefaultTenantName));
+ ASSERT_FALSE(fs_manager->tenant_key(fs::kDefaultTenantName).empty());
+ ASSERT_EQ(source_key, fs_manager->tenant_key(fs::kDefaultTenantName));
+ ASSERT_FALSE(fs_manager->tenant_key_iv(fs::kDefaultTenantName).empty());
+ ASSERT_EQ(source_key_iv,
fs_manager->tenant_key_iv(fs::kDefaultTenantName));
+
ASSERT_FALSE(fs_manager->tenant_key_version(fs::kDefaultTenantName).empty());
+ ASSERT_EQ(source_key_version,
fs_manager->tenant_key_version(fs::kDefaultTenantName));
+
+ // There is no server key in the metadata.
+ ASSERT_TRUE(fs_manager->server_key().empty());
+ ASSERT_TRUE(fs_manager->server_key_iv().empty());
+ ASSERT_TRUE(fs_manager->server_key_version().empty());
+ }
+
+ // We are in tenant key version, try to 'upgrade_encryption_key' from tenant
key info.
+ mts->Shutdown();
+
+ string err;
+ RunActionStderrString(Substitute(
+ "fs upgrade_encryption_key --fs_data_dirs=$0 --fs_wal_dir=$1
--fs_metadata_dir=$2",
+ JoinStrings(data_roots, ","), wal_root, metadata_root), &err);
+ // This upgrade tool only works on the cluster with server key info exist.
+ ASSERT_STR_CONTAINS(err, "We should not do server key upgrade on a cluster");
+}
+
+TEST_F(ToolTest, TestFsUpgradeEncryptionKeyFromNoEncryptionKeyInfo) {
+ // Create a cluster without server key info.
+ FLAGS_encrypt_data_at_rest = false;
+
+ NO_FATALS(StartMiniCluster());
+ MiniTabletServer* mts = mini_cluster_->mini_tablet_server(0);
+ const string& wal_root = mts->options()->fs_opts.wal_root;
+ vector<string> data_roots = mts->options()->fs_opts.data_roots;
+ const string& metadata_root = mts->options()->fs_opts.metadata_root;
+
+ // No server key info exist.
+ FsManager* fs_manager = mts->server()->fs_manager();
+ ASSERT_TRUE(fs_manager->server_key().empty());
+ ASSERT_TRUE(fs_manager->server_key_iv().empty());
+ ASSERT_TRUE(fs_manager->server_key_version().empty());
+ // No tenant key info exist.
+ ASSERT_FALSE(fs_manager->is_tenants_exist());
+
+ mts->Shutdown();
+
+ // Do the encryption key info upgrade for a tserver without server key with
the kudu CLI tool.
+ string err;
+ RunActionStderrString(Substitute(
+ "fs upgrade_encryption_key --fs_data_dirs=$0 --fs_wal_dir=$1
--fs_metadata_dir=$2",
+ JoinStrings(data_roots, ","), wal_root, metadata_root), &err);
+ // This upgrade tool only works on the cluster with server key info exist.
+ ASSERT_STR_CONTAINS(err, "We should not do server key upgrade on a cluster");
+}
+
TEST_F(ToolTest, TestStartEndMaintenanceMode) {
NO_FATALS(StartMiniCluster());
// Perform the steps on a tserver that exists and one that doesn't.
diff --git a/src/kudu/tools/tool_action_fs.cc b/src/kudu/tools/tool_action_fs.cc
index 48f0f7cd8..b0900ec85 100644
--- a/src/kudu/tools/tool_action_fs.cc
+++ b/src/kudu/tools/tool_action_fs.cc
@@ -73,6 +73,7 @@
#include "kudu/util/flag_validators.h"
#include "kudu/util/memory/arena.h"
#include "kudu/util/pb_util.h"
+#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
#include "kudu/util/string_case.h"
@@ -161,6 +162,7 @@ using kudu::tablet::TabletDataState;
using kudu::tablet::TabletMetadata;
using std::cout;
using std::endl;
+using std::move;
using std::nullopt;
using std::optional;
using std::shared_ptr;
@@ -173,6 +175,45 @@ using strings::Substitute;
namespace kudu {
namespace tools {
+Status UpdateEncryptionKeyInfo(Env* env) {
+ FsManagerOpts fs_opts;
+ fs_opts.update_instances =
UpdateInstanceBehavior::UPDATE_AND_ERROR_ON_FAILURE;
+ fs_opts.skip_block_manager = true;
+ FsManager fs_manager(env, std::move(fs_opts));
+ RETURN_NOT_OK(fs_manager.Open());
+
+ if (fs_manager.server_key().empty()) {
+ return Status::IllegalState(
+ "We should not do server key upgrade on a cluster with data rest
encryption disabled.");
+ }
+
+ if (fs_manager.is_tenants_exist()) {
+ cout << "This tserver has already in tenant key version, and there is no
need to upgrade again."
+ << endl;
+ return Status::OK();
+ }
+
+ unique_ptr<InstanceMetadataPB> metadata(new InstanceMetadataPB);
+ fs_manager.CopyMetadata(&metadata);
+
+ // Add the default tenant key with the server key.
+ InstanceMetadataPB::TenantMetadataPB* tenant_metadata =
metadata->add_tenants();
+ tenant_metadata->set_tenant_name(fs::kDefaultTenantName);
+ tenant_metadata->set_tenant_id(fs::kDefaultTenantID);
+ tenant_metadata->set_tenant_key(fs_manager.server_key());
+ tenant_metadata->set_tenant_key_iv(fs_manager.server_key_iv());
+ tenant_metadata->set_tenant_key_version(fs_manager.server_key_version());
+ // Clear the server key info.
+ metadata->clear_server_key();
+ metadata->clear_server_key_iv();
+ metadata->clear_server_key_version();
+
+ // Write the new metadata to disk.
+ RETURN_NOT_OK(fs_manager.UpdateMetadata(move(metadata)));
+
+ return Status::OK();
+}
+
namespace {
Status Check(const RunnerContext& /*context*/) {
@@ -452,6 +493,22 @@ Status Update(const RunnerContext& /*context*/) {
return fs.Open();
}
+Status UpgradeEncryptionKey(const RunnerContext& /*context*/) {
+ Env* env = Env::Default();
+ const string& filename = "/tmp/kudu_tserver-metadata-upgrade-instance.lock";
+ FileLock* file_lock;
+ KUDU_RETURN_NOT_OK_PREPEND(env->LockFile(filename, &file_lock),
+ "Could not lock instance file. Make sure that "
+ "this tool is not already running.");
+ auto unlock = MakeScopedCleanup([&]() {
+ env->UnlockFile(file_lock);
+ });
+
+ RETURN_NOT_OK(UpdateEncryptionKeyInfo(env));
+
+ return Status::OK();
+}
+
namespace {
// The 'kudu fs list' column fields.
@@ -999,6 +1056,19 @@ unique_ptr<Mode> BuildFsMode() {
.AddOptionalParameter("h")
.Build();
+ unique_ptr<Action> upgrade_encryption_key =
+ ActionBuilder("upgrade_encryption_key", &UpgradeEncryptionKey)
+ .Description("Upgrade the encryption key info in metadata")
+ .ExtraDescription("Upgrade the server key to the tenant key which
belongs to the default "
+ "tenant. This feature only works on a cluster with
data rest encryption "
+ "enabled.\n\n"
+ "Note: this function is exclusively for use with the
Kudu CLI tool and can "
+ "only be used once per tserver.")
+ .AddOptionalParameter("fs_data_dirs")
+ .AddOptionalParameter("fs_metadata_dir")
+ .AddOptionalParameter("fs_wal_dir")
+ .Build();
+
return ModeBuilder("fs")
.Description("Operate on a local Kudu filesystem")
.AddMode(BuildFsDumpMode())
@@ -1006,6 +1076,7 @@ unique_ptr<Mode> BuildFsMode() {
.AddAction(std::move(format))
.AddAction(std::move(list))
.AddAction(std::move(update))
+ .AddAction(std::move(upgrade_encryption_key))
.Build();
}
