This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 89e1ee320 KUDU-3326 [master] add flag
FLAGS_deleted_table_reserve_seconds to change the cluster-wide behavior of the
DeleteTable() RPC.
89e1ee320 is described below
commit 89e1ee3202b47e0f0e247ae7beb91ac344a012a6
Author: kedeng <[email protected]>
AuthorDate: Fri Aug 19 11:03:25 2022 +0800
KUDU-3326 [master] add flag FLAGS_deleted_table_reserve_seconds to change
the cluster-wide
behavior of the DeleteTable() RPC.
This patch helps to change the default behavior of the DeleteTable RPC at
the master side.
Value 0 means DeleteTable() works the regular way, i.e. dropping the table
and purging its
data immediately. If it's set to anything greater than 0, then all
DeleteTable() RPCs are
turned into SoftDeleteTable(..., FLAGS_deleted_table_reserve_seconds)").
NOTE: this flag is useless if HMS is enabled.
Change-Id: I7cf3021fa0e9a425d5f8091ea3336c55a3c14bbc
Reviewed-on: http://gerrit.cloudera.org:8080/18864
Tested-by: Kudu Jenkins
Reviewed-by: Yingchun Lai <[email protected]>
Reviewed-by: Alexey Serbin <[email protected]>
---
src/kudu/client/client-test.cc | 126 +++++++++++++++++++++++++++++++++++++
src/kudu/master/catalog_manager.cc | 41 ++++++++++--
2 files changed, 162 insertions(+), 5 deletions(-)
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index d97d87ebe..f8eef831d 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -187,6 +187,7 @@ DECLARE_uint32(txn_keepalive_interval_ms);
DECLARE_uint32(txn_staleness_tracker_interval_ms);
DECLARE_uint32(txn_manager_status_table_num_replicas);
DEFINE_int32(test_scan_num_rows, 1000, "Number of rows to insert and scan");
+DECLARE_uint32(default_deleted_table_reserve_seconds);
METRIC_DECLARE_counter(block_manager_total_bytes_read);
METRIC_DECLARE_counter(location_mapping_cache_hits);
@@ -5239,6 +5240,131 @@ TEST_F(ClientTest,
TestSoftDeleteAndRecallAfterReserveTimeTable) {
ASSERT_TRUE(tables.empty());
}
+TEST_F(ClientTest, TestDeleteWithDeletedTableReserveSeconds) {
+ // Open the table before deleting it.
+ ASSERT_OK(client_->OpenTable(kTableName, &client_table_));
+
+ // Insert a few rows, and scan them back. This is to populate the MetaCache.
+ NO_FATALS(InsertTestRows(client_.get(), client_table_.get(), 10));
+ vector<string> rows;
+ ScanTableToStrings(client_table_.get(), &rows);
+ ASSERT_EQ(10, rows.size());
+
+ ASSERT_EQ(0, FLAGS_default_deleted_table_reserve_seconds);
+ FLAGS_default_deleted_table_reserve_seconds = 60;
+
+ // Delete the table, the table won't be purged immediately if
+ // FLAGS_default_deleted_table_reserve_seconds set to anything greater than
0.
+ ASSERT_OK(client_->DeleteTable(kTableName));
+ CatalogManager *catalog_manager =
cluster_->mini_master()->master()->catalog_manager();
+ {
+ CatalogManager::ScopedLeaderSharedLock l(catalog_manager);
+ ASSERT_OK(l.first_failed_status());
+ bool exists;
+ ASSERT_OK(catalog_manager->TableNameExists(kTableName, &exists));
+ ASSERT_TRUE(exists);
+ }
+
+ // The table is in soft-deleted state.
+ vector<string> tables;
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_EQ(1, tables.size());
+ ASSERT_EQ(kTableName, tables[0]);
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_EQ(0, tables.size());
+
+ // Try to recall the table.
+ ASSERT_OK(client_->RecallTable(client_table_->id()));
+
+ // Check the data in the table.
+ ScanTableToStrings(client_table_.get(), &rows);
+ ASSERT_EQ(10, rows.size());
+
+ // Force to delete the table with
FLAGS_default_deleted_table_reserve_seconds set to 0.
+ FLAGS_default_deleted_table_reserve_seconds = 0;
+ ASSERT_OK(client_->DeleteTable(kTableName));
+
+ // No tables left.
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_TRUE(tables.empty());
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_TRUE(tables.empty());
+}
+
+TEST_F(ClientTest, TestDeleteWithDeletedTableReserveSecondsWorks) {
+ SKIP_IF_SLOW_NOT_ALLOWED();
+ // Open the table before deleting it.
+ ASSERT_OK(client_->OpenTable(kTableName, &client_table_));
+
+ ASSERT_EQ(0, FLAGS_default_deleted_table_reserve_seconds);
+ FLAGS_default_deleted_table_reserve_seconds = 5;
+
+ // Delete the table, the table won't be purged immediately if
+ // FLAGS_default_deleted_table_reserve_seconds set to anything greater than
0.
+ ASSERT_OK(client_->DeleteTable(kTableName));
+ CatalogManager *catalog_manager =
cluster_->mini_master()->master()->catalog_manager();
+ {
+ CatalogManager::ScopedLeaderSharedLock l(catalog_manager);
+ ASSERT_OK(l.first_failed_status());
+ bool exists;
+ ASSERT_OK(catalog_manager->TableNameExists(kTableName, &exists));
+ ASSERT_TRUE(exists);
+ }
+
+ // The table is in soft-deleted state.
+ vector<string> tables;
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_EQ(1, tables.size());
+ ASSERT_EQ(kTableName, tables[0]);
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_TRUE(tables.empty());
+
+ // Test FLAGS_table_reserve_seconds.
+ SleepFor(MonoDelta::FromMilliseconds(5 * 1000));
+
+ // No tables left.
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_TRUE(tables.empty());
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_TRUE(tables.empty());
+}
+
+TEST_F(ClientTest, TestDeleteSoftDeleteStatusFromDeletedTableReserveSeconds) {
+ // Open the table before deleting it.
+ ASSERT_OK(client_->OpenTable(kTableName, &client_table_));
+
+ ASSERT_EQ(0, FLAGS_default_deleted_table_reserve_seconds);
+ FLAGS_default_deleted_table_reserve_seconds = 600;
+
+ // Get a soft-deleted table.
+ ASSERT_OK(client_->SoftDeleteTable(kTableName, 60));
+ CatalogManager *catalog_manager =
cluster_->mini_master()->master()->catalog_manager();
+ {
+ CatalogManager::ScopedLeaderSharedLock l(catalog_manager);
+ ASSERT_OK(l.first_failed_status());
+ bool exists;
+ ASSERT_OK(catalog_manager->TableNameExists(kTableName, &exists));
+ ASSERT_TRUE(exists);
+ }
+
+ // The table is in soft-deleted state.
+ vector<string> tables;
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_EQ(1, tables.size());
+ ASSERT_EQ(kTableName, tables[0]);
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_TRUE(tables.empty());
+
+ // purge the soft-deleted table
+ ASSERT_OK(client_->DeleteTable(kTableName));
+
+ // No tables left.
+ ASSERT_OK(client_->ListTables(&tables));
+ ASSERT_TRUE(tables.empty());
+ ASSERT_OK(client_->ListSoftDeletedTables(&tables));
+ ASSERT_TRUE(tables.empty());
+}
+
TEST_F(ClientTest, TestGetTableSchema) {
KuduSchema schema;
diff --git a/src/kudu/master/catalog_manager.cc
b/src/kudu/master/catalog_manager.cc
index 76397ee3e..d16447acd 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -407,6 +407,32 @@
DEFINE_bool(require_new_spec_for_custom_hash_schema_range_bound, false,
TAG_FLAG(require_new_spec_for_custom_hash_schema_range_bound, experimental);
TAG_FLAG(require_new_spec_for_custom_hash_schema_range_bound, runtime);
+DEFINE_uint32(default_deleted_table_reserve_seconds, 0,
+ "Time in seconds to be reserved before purging a deleted table
for the table "
+ "from DeleteTable request. Value 0 means DeleteTable() works the
regular way, "
+ "i.e. dropping the table and purging its data immediately. If
it's set to "
+ "anything greater than 0, then all DeleteTable() RPCs are turned
into "
+ "SoftDeleteTable(...,
FLAGS_default_deleted_table_reserve_seconds). "
+ "NOTE : this flag make no sense for soft-delete function because
the "
+ "reserve_seconds has been specified by user while calling
SoftDeleteTable.");
+TAG_FLAG(default_deleted_table_reserve_seconds, advanced);
+TAG_FLAG(default_deleted_table_reserve_seconds, runtime);
+
+DECLARE_string(hive_metastore_uris);
+
+bool ValidateDeletedTableReserveSeconds() {
+ if (FLAGS_default_deleted_table_reserve_seconds > 0 &&
+ !FLAGS_hive_metastore_uris.empty()) {
+ LOG(ERROR) << "If enabling HMS,
FLAGS_default_deleted_table_reserve_seconds "
+ "makes no sense.";
+ return false;
+ }
+ return true;
+}
+
+GROUP_FLAG_VALIDATOR(default_deleted_table_reserve_seconds,
+ ValidateDeletedTableReserveSeconds);
+
DECLARE_bool(raft_prepare_replacement_before_eviction);
DECLARE_int64(tsk_rotation_seconds);
DECLARE_string(ranger_config_path);
@@ -2391,8 +2417,11 @@ Status CatalogManager::SoftDeleteTableRpc(const
DeleteTableRequestPB& req,
resp, MasterErrorPB::TABLE_SOFT_DELETED);
}
- // Reserve seconds equal 0 means delete it directly.
- if (req.reserve_seconds() == 0) {
+ // Reserve seconds equal 0 means delete it directly if
default_deleted_table_reserve_seconds
+ // set to 0, which means the cluster-wide behavior of the DeleteTable() RPC
is keep default,
+ // or the table is in soft-deleted status.
+ if (req.reserve_seconds() == 0 &&
+ (FLAGS_default_deleted_table_reserve_seconds == 0 ||
is_soft_deleted_table)) {
return DeleteTableRpc(req, resp, rpc);
}
@@ -2401,8 +2430,8 @@ Status CatalogManager::SoftDeleteTableRpc(const
DeleteTableRequestPB& req,
}
Status CatalogManager::SoftDeleteTable(const DeleteTableRequestPB& req,
- DeleteTableResponsePB* resp,
- rpc::RpcContext* rpc) {
+ DeleteTableResponsePB* resp,
+ rpc::RpcContext* rpc) {
leader_lock_.AssertAcquiredForReading();
// TODO(kedeng) : soft_deleted state need sync to HMS.
@@ -2437,7 +2466,9 @@ Status CatalogManager::SoftDeleteTable(const
DeleteTableRequestPB& req,
// soft delete state change
l.mutable_data()->set_state(SysTablesEntryPB::SOFT_DELETED, deletion_msg);
l.mutable_data()->set_delete_timestamp(WallTime_Now());
- l.mutable_data()->set_soft_deleted_reserved_seconds(req.reserve_seconds());
+ uint32_t reserve_seconds = req.reserve_seconds() == 0 ?
+ FLAGS_default_deleted_table_reserve_seconds : req.reserve_seconds();
+ l.mutable_data()->set_soft_deleted_reserved_seconds(reserve_seconds);
// 2. Look up the tablets, lock them, and mark them as soft deleted.
{
