This is an automated email from the ASF dual-hosted git repository.
deardeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e87bc301743 [fix](cloud) Update node decommission status atomically
(#70224)
e87bc301743 is described below
commit e87bc301743969fdc123b6117c8077771ffc84f7
Author: deardeng <[email protected]>
AuthorDate: Tue Nov 17 15:35:39 2026 +0800
[fix](cloud) Update node decommission status atomically (#70224)
Problem Summary: Both start-decommission and completion updated node
status by deleting the node and re-adding it in separate transactions.
Each transaction independently committed and refreshed the MS cache,
exposing a state with the target node missing. A concurrent rowset
commit could then fail with "failed to commit rowset: empty
instance_id". Update the existing node's status in one transaction and
add deterministic regression coverage for both transitions.
---
cloud/src/meta-service/injection_point_http.cpp | 17 +-
cloud/src/meta-service/meta_service.cpp | 1 +
cloud/src/meta-service/meta_service_resource.cpp | 188 ++++++++-----------
cloud/src/resource-manager/resource_manager.cpp | 7 +-
cloud/src/resource-manager/resource_manager.h | 4 +-
cloud/test/meta_service_http_test.cpp | 97 ++++++++--
cloud/test/mock_resource_manager.h | 3 +-
cloud/test/resource_test.cpp | 201 +++++++++++++++++++++
.../cloud_p0/node_mgr/test_cloud_decommission.out | 7 +
.../node_mgr/test_cloud_decommission.groovy | 115 +++++++++++-
10 files changed, 501 insertions(+), 139 deletions(-)
diff --git a/cloud/src/meta-service/injection_point_http.cpp
b/cloud/src/meta-service/injection_point_http.cpp
index 3c2ae4fee33..fdca4f83bc7 100644
--- a/cloud/src/meta-service/injection_point_http.cpp
+++ b/cloud/src/meta-service/injection_point_http.cpp
@@ -268,6 +268,17 @@ HttpResponse set_sleep(const std::string& point, const
brpc::URI& uri) {
return http_json_reply(MetaServiceCode::OK, "OK");
}
+HttpResponse set_wait(const std::string& point) {
+ auto* sp = SyncPoint::get_instance();
+ sp->set_call_back(point, [point, sp](auto&&) {
+ LOG(INFO) << "injection point waiting, point=" << point;
+ while (sp->get_enable() && sp->has_point(point)) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+ });
+ return http_json_reply(MetaServiceCode::OK, "OK");
+}
+
HttpResponse set_return(const std::string& point, const brpc::URI& uri) {
auto sp = SyncPoint::get_instance();
sp->set_call_back(point, [point](auto&& args) {
@@ -293,7 +304,9 @@ HttpResponse handle_set(const brpc::URI& uri) {
if (behavior.empty()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, "empty
behavior");
}
- if (behavior == "sleep") {
+ if (behavior == "wait") {
+ return set_wait(point);
+ } else if (behavior == "sleep") {
return set_sleep(point, uri);
} else if (behavior == "return") {
return set_return(point, uri);
@@ -349,6 +362,8 @@ HttpResponse handle_disable(const brpc::URI& uri) {
// curl
"ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=disable"
// ```
+// behavior=wait blocks at a point until it is cleared or injection is
disabled.
+//
// clear all injection points
// ```
// curl
"ms_ip:port/MetaService/http/v1/injection_point?token=greedisgood9999&op=clear"
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index dac3e2e6e2a..38e759e5374 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -2728,6 +2728,7 @@ void
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
msg = "no rowset meta";
return;
}
+ TEST_INJECTION_POINT("commit_rowset::before_get_instance_id:" +
request->cloud_unique_id());
instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
if (instance_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index b0b5b1db74d..df7adab3090 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -3314,133 +3314,95 @@ void handle_drop_node(const std::string& instance_id,
const AlterClusterRequest*
msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}
-void handle_decommission_node(const std::string& instance_id, const
AlterClusterRequest* request,
- std::shared_ptr<ResourceManager> resource_mgr,
std::string& msg,
- MetaServiceCode& code) {
- resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false,
false);
- if (msg != "") {
- LOG(WARNING) << msg;
- return;
+static bool matches_node(const NodeInfoPB& node, const NodeInfoPB& requested) {
+ if (node.cloud_unique_id() != requested.cloud_unique_id() ||
+ node.heartbeat_port() != requested.heartbeat_port()) {
+ return false;
+ }
+ if (node.has_ip() && requested.has_ip()) {
+ return node.ip() == requested.ip();
}
+ return node.has_host() && requested.has_host() && node.host() ==
requested.host();
+}
- std::string be_unique_id =
(request->cluster().nodes())[0].cloud_unique_id();
+static std::string resolve_cluster_from_node(ResourceManager& resource_mgr,
+ const NodeInfoPB& requested,
ClusterPB& cluster) {
std::vector<NodeInfo> nodes;
- std::string err = resource_mgr->get_node(be_unique_id, &nodes);
- if (!err.empty()) {
- LOG(INFO) << "failed to check instance info, err=" << err;
- msg = err;
- return;
- }
-
- std::vector<NodeInfo> decomission_nodes;
- for (auto& node : nodes) {
- for (auto req_node : request->cluster().nodes()) {
- bool ip_processed = false;
- if (node.node_info.has_ip() && req_node.has_ip()) {
- std::string endpoint =
- node.node_info.ip() + ":" +
std::to_string(node.node_info.heartbeat_port());
- std::string req_endpoint =
- req_node.ip() + ":" +
std::to_string(req_node.heartbeat_port());
- if (endpoint == req_endpoint) {
- decomission_nodes.push_back(node);
-
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
- }
- ip_processed = true;
- }
-
- if (!ip_processed && node.node_info.has_host() &&
req_node.has_host()) {
- std::string endpoint = node.node_info.host() + ":" +
-
std::to_string(node.node_info.heartbeat_port());
- std::string req_endpoint =
- req_node.host() + ":" +
std::to_string(req_node.heartbeat_port());
- if (endpoint == req_endpoint) {
- decomission_nodes.push_back(node);
-
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
- }
- }
+ auto msg = resource_mgr.get_node(requested.cloud_unique_id(), &nodes);
+ if (!msg.empty()) {
+ return msg;
+ }
+ for (const auto& node : nodes) {
+ if (matches_node(node.node_info, requested)) {
+ cluster.set_cluster_id(node.cluster_id);
+ cluster.set_cluster_name(node.cluster_name);
+ return "";
}
}
+ return "failed to find node to update: " + requested.ShortDebugString();
+}
- {
- std::vector<NodeInfo> to_add;
- std::vector<NodeInfo>& to_del = decomission_nodes;
- msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
- }
- {
- std::vector<NodeInfo>& to_add = decomission_nodes;
- std::vector<NodeInfo> to_del;
- for (auto& node : to_add) {
-
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
- LOG(INFO) << "decomission node, "
- << "size: " << to_add.size() << " " <<
node.node_info.DebugString() << " "
- << node.cluster_id << " " << node.cluster_name;
+static std::string update_cluster_node_statuses(ClusterPB& cluster,
+ const ClusterPB&
requested_cluster,
+ NodeStatusPB status) {
+ const auto mtime =
duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+ auto* nodes = cluster.mutable_nodes();
+ for (const auto& requested : requested_cluster.nodes()) {
+ NodeInfoPB* matched = nullptr;
+ for (auto& node : *nodes) {
+ if (matches_node(node, requested)) {
+ matched = &node;
+ break;
+ }
+ }
+ if (matched == nullptr) {
+ return "failed to find node to update: " +
requested.ShortDebugString();
}
- msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
+ matched->set_status(status);
+ matched->set_mtime(mtime);
}
+ return "";
}
-void handle_notify_decommissioned(const std::string& instance_id,
- const AlterClusterRequest* request,
- std::shared_ptr<ResourceManager>
resource_mgr, std::string& msg,
- MetaServiceCode& code) {
- resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false,
false);
- if (msg != "") {
+void handle_update_node_status(const std::string& instance_id, const
AlterClusterRequest* request,
+ std::shared_ptr<ResourceManager> resource_mgr,
NodeStatusPB status,
+ std::string& msg) {
+ const auto& requested_cluster = request->cluster();
+ resource_mgr->check_cluster_params_valid(requested_cluster, &msg, false,
false);
+ if (!msg.empty()) {
LOG(WARNING) << msg;
return;
}
-
- std::string be_unique_id =
(request->cluster().nodes())[0].cloud_unique_id();
- std::vector<NodeInfo> nodes;
- std::string err = resource_mgr->get_node(be_unique_id, &nodes);
- if (!err.empty()) {
- LOG(INFO) << "failed to check instance info, err=" << err;
- msg = err;
- return;
- }
-
- std::vector<NodeInfo> decomission_nodes;
- for (auto& node : nodes) {
- for (auto req_node : request->cluster().nodes()) {
- bool ip_processed = false;
- if (node.node_info.has_ip() && req_node.has_ip()) {
- std::string endpoint =
- node.node_info.ip() + ":" +
std::to_string(node.node_info.heartbeat_port());
- std::string req_endpoint =
- req_node.ip() + ":" +
std::to_string(req_node.heartbeat_port());
- if (endpoint == req_endpoint) {
- decomission_nodes.push_back(node);
- }
- ip_processed = true;
- }
-
- if (!ip_processed && node.node_info.has_host() &&
req_node.has_host()) {
- std::string endpoint = node.node_info.host() + ":" +
-
std::to_string(node.node_info.heartbeat_port());
- std::string req_endpoint =
- req_node.host() + ":" +
std::to_string(req_node.heartbeat_port());
- if (endpoint == req_endpoint) {
- decomission_nodes.push_back(node);
- }
- }
- }
+ if (requested_cluster.nodes().empty()) {
+ msg = "no nodes to update";
+ return;
}
- {
- std::vector<NodeInfo> to_add;
- std::vector<NodeInfo>& to_del = decomission_nodes;
- msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
- }
- {
- std::vector<NodeInfo>& to_add = decomission_nodes;
- std::vector<NodeInfo> to_del;
- for (auto& node : to_add) {
-
node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
- LOG(INFO) << "notify node decomissioned, "
- << " size: " << to_add.size() << " " <<
node.node_info.DebugString() << " "
- << node.cluster_id << " " << node.cluster_name;
+ ClusterInfo cluster;
+ cluster.cluster.CopyFrom(requested_cluster);
+ if (cluster.cluster.cluster_id().empty()) {
+ // Older HTTP callers can identify the cluster through the node alone.
+ msg = resolve_cluster_from_node(*resource_mgr,
requested_cluster.nodes(0), cluster.cluster);
+ if (!msg.empty()) {
+ return;
}
- msg = resource_mgr->modify_nodes(instance_id, to_add, to_del);
}
+
+ const auto& cluster_id = cluster.cluster.cluster_id();
+ const auto& cluster_name = cluster.cluster.cluster_name();
+ // HTTP-created clusters without a stored ID are distinguished by name.
+ auto match_cluster = [&cluster_id, &cluster_name](const ClusterPB&
candidate) {
+ return candidate.cluster_id() == cluster_id &&
+ (!cluster_id.empty() || candidate.cluster_name() ==
cluster_name);
+ };
+ auto update_nodes = [&requested_cluster, status](ClusterPB& current,
std::vector<ClusterPB>&) {
+ return update_cluster_node_statuses(current, requested_cluster,
status);
+ };
+
+ // Publish only the final state: removing and re-adding nodes breaks
instance lookup.
+ msg = resource_mgr->update_cluster(instance_id, cluster, match_cluster,
update_nodes,
+
/*replace_if_existing_empty_target_cluster=*/false,
+ /*allow_cluster_name_fallback=*/true);
}
void handle_rename_cluster(const std::string& instance_id, const ClusterInfo&
cluster,
@@ -3738,10 +3700,12 @@ void
MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller,
handle_drop_node(instance_id, request, resource_mgr(), msg, code);
break;
case AlterClusterRequest::DECOMMISSION_NODE:
- handle_decommission_node(instance_id, request, resource_mgr(), msg,
code);
+ handle_update_node_status(instance_id, request, resource_mgr(),
+ NodeStatusPB::NODE_STATUS_DECOMMISSIONING,
msg);
break;
case AlterClusterRequest::NOTIFY_DECOMMISSIONED:
- handle_notify_decommissioned(instance_id, request, resource_mgr(),
msg, code);
+ handle_update_node_status(instance_id, request, resource_mgr(),
+ NodeStatusPB::NODE_STATUS_DECOMMISSIONED,
msg);
break;
case AlterClusterRequest::RENAME_CLUSTER: {
// SQL mode, cluster cluster name eq empty cluster name, need drop
empty cluster first.
diff --git a/cloud/src/resource-manager/resource_manager.cpp
b/cloud/src/resource-manager/resource_manager.cpp
index fafd4ae09e8..3c6487e4fff 100644
--- a/cloud/src/resource-manager/resource_manager.cpp
+++ b/cloud/src/resource-manager/resource_manager.cpp
@@ -781,14 +781,15 @@ std::string ResourceManager::update_cluster(
const std::string& instance_id, const ClusterInfo& cluster,
std::function<bool(const ClusterPB&)> filter,
std::function<std::string(ClusterPB&, std::vector<ClusterPB>&
clusters_in_instance)> action,
- bool replace_if_existing_empty_target_cluster) {
+ bool replace_if_existing_empty_target_cluster, bool
allow_cluster_name_fallback) {
std::stringstream ss;
std::string msg;
std::string cluster_id = cluster.cluster.has_cluster_id() ?
cluster.cluster.cluster_id() : "";
std::string cluster_name =
cluster.cluster.has_cluster_name() ?
cluster.cluster.cluster_name() : "";
- if (cluster_id.empty()) {
+ const bool can_fallback_to_name = allow_cluster_name_fallback &&
!cluster_name.empty();
+ if (cluster_id.empty() && !can_fallback_to_name) {
ss << "missing cluster_id=" << cluster_id;
msg = ss.str();
LOG(INFO) << msg;
@@ -916,6 +917,7 @@ std::string ResourceManager::update_cluster(
<< " instance json=" << proto_to_json(instance);
refresh_instance(instance_id, instance);
+ TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");
return msg;
}
@@ -1383,6 +1385,7 @@ std::string ResourceManager::modify_nodes(const
std::string& instance_id,
}
refresh_instance(instance_id, instance);
+ TEST_INJECTION_POINT("ResourceManager::node_update:after_refresh");
return "";
}
diff --git a/cloud/src/resource-manager/resource_manager.h
b/cloud/src/resource-manager/resource_manager.h
index ca33a3f0983..f8a3a75da80 100644
--- a/cloud/src/resource-manager/resource_manager.h
+++ b/cloud/src/resource-manager/resource_manager.h
@@ -89,6 +89,7 @@ public:
* @param cluster cluster to update, only cluster name and cluster id are
concered
* @param action update operation code snippet
* @param replace_if_existing_empty_target_cluster, find
cluster.cluster_name is a empty cluster(no node), drop it
+ * @param allow_cluster_name_fallback allow an empty ID when the cluster
name is set
* @filter filter condition
* @return empty string for success, otherwise failure reason returned
*/
@@ -97,7 +98,8 @@ public:
std::function<bool(const ClusterPB&)> filter,
std::function<std::string(ClusterPB&, std::vector<ClusterPB>&
clusters_in_instance)>
action,
- bool replace_if_existing_empty_target_cluster = false);
+ bool replace_if_existing_empty_target_cluster = false,
+ bool allow_cluster_name_fallback = false);
/**
* Get instance from underlying storage with given transaction.
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 9b7c05c125c..3d0199b1e68 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -954,49 +954,45 @@ TEST(MetaServiceHttpTest, AlterClusterTest) {
{
AlterClusterRequest req;
req.set_instance_id(mock_instance);
- req.mutable_cluster()->set_cluster_name(mock_cluster_name);
+ req.mutable_cluster()->set_cluster_name("rename_cluster_name");
req.mutable_cluster()->set_cluster_id(mock_cluster_id);
req.mutable_cluster()->set_type(ClusterPB::COMPUTE);
auto node = req.mutable_cluster()->add_nodes();
node->set_ip("127.0.0.1");
node->set_heartbeat_port(9999);
node->set_cloud_unique_id("cloud_unique_id");
- auto& meta_service = ctx.meta_service_;
- NodeInfoPB npb;
- npb.set_heartbeat_port(9999);
- npb.set_ip("127.0.0.1");
- npb.set_cloud_unique_id("cloud_unique_id");
- meta_service->resource_mgr()->node_info_.insert(
- {"cloud_unique_id", NodeInfo {Role::COMPUTE_NODE,
mock_instance,
- "rename_cluster_name",
mock_cluster_id, npb}});
+ auto [add_status_code, add_resp] =
ctx.forward<MetaServiceResponseStatus>("add_node", req);
+ ASSERT_EQ(add_status_code, 200);
+ ASSERT_EQ(add_resp.code(), MetaServiceCode::OK);
auto [status_code, resp] =
ctx.forward<MetaServiceResponseStatus>("decommission_node", req);
ASSERT_EQ(status_code, 200);
ASSERT_EQ(resp.code(), MetaServiceCode::OK);
+
+ auto instance = ctx.get_instance_info(mock_instance);
+ ASSERT_EQ(instance.clusters(0).nodes_size(), 1);
+ EXPECT_EQ(instance.clusters(0).nodes(0).status(),
+ NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
}
// notify_decommissioned
{
AlterClusterRequest req;
req.set_instance_id(mock_instance);
- req.mutable_cluster()->set_cluster_name(mock_cluster_name);
+ req.mutable_cluster()->set_cluster_name("rename_cluster_name");
req.mutable_cluster()->set_cluster_id(mock_cluster_id);
req.mutable_cluster()->set_type(ClusterPB::COMPUTE);
auto node = req.mutable_cluster()->add_nodes();
node->set_ip("127.0.0.1");
- node->set_heartbeat_port(9996);
+ node->set_heartbeat_port(9999);
node->set_cloud_unique_id("cloud_unique_id");
- auto& meta_service = ctx.meta_service_;
- NodeInfoPB npb;
- npb.set_heartbeat_port(9996);
- npb.set_ip("127.0.0.1");
- npb.set_cloud_unique_id("cloud_unique_id");
- meta_service->resource_mgr()->node_info_.insert(
- {"cloud_unique_id", NodeInfo {Role::COMPUTE_NODE,
mock_instance,
- "rename_cluster_name",
mock_cluster_id, npb}});
auto [status_code, resp] =
ctx.forward<MetaServiceResponseStatus>("notify_decommissioned", req);
ASSERT_EQ(status_code, 200);
ASSERT_EQ(resp.code(), MetaServiceCode::OK);
+
+ auto instance = ctx.get_instance_info(mock_instance);
+ ASSERT_EQ(instance.clusters(0).nodes_size(), 1);
+ EXPECT_EQ(instance.clusters(0).nodes(0).status(),
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
}
// update_cluster_endpoint
@@ -1014,6 +1010,69 @@ TEST(MetaServiceHttpTest, AlterClusterTest) {
}
}
+class NodeStatusWithoutClusterIdHttpTest : public
testing::TestWithParam<std::tuple<bool, bool>> {};
+
+TEST_P(NodeStatusWithoutClusterIdHttpTest,
PreservesNamedClusterWithoutStoredId) {
+ const auto [explicit_empty_id, notify_decommissioned] = GetParam();
+ HttpContext ctx;
+ CreateInstanceRequest create_req;
+ create_req.set_instance_id("legacy_node_status_instance");
+ create_req.set_user_id("test_user");
+ create_req.set_name("test_name");
+ auto [create_http_status, create_resp] =
+ ctx.forward<MetaServiceResponseStatus>("create_instance",
create_req);
+ ASSERT_EQ(create_http_status, 200);
+ ASSERT_EQ(create_resp.code(), MetaServiceCode::OK) << create_resp.msg();
+
+ AlterClusterRequest req;
+ req.set_instance_id(create_req.instance_id());
+ auto* cluster = req.mutable_cluster();
+ cluster->set_type(ClusterPB::COMPUTE);
+ cluster->set_cluster_name("legacy_named_cluster");
+ if (explicit_empty_id) {
+ cluster->set_cluster_id("");
+ }
+ auto* node = cluster->add_nodes();
+ node->set_cloud_unique_id("legacy_node_uid");
+ node->set_ip("127.0.0.1");
+ node->set_heartbeat_port(9050);
+ node->set_status(NodeStatusPB::NODE_STATUS_RUNNING);
+ auto [add_http_status, add_resp] =
ctx.forward<MetaServiceResponseStatus>("add_cluster", req);
+ ASSERT_EQ(add_http_status, 200);
+ ASSERT_EQ(add_resp.code(), MetaServiceCode::OK) << add_resp.msg();
+ auto persisted = ctx.get_instance_info(create_req.instance_id());
+ ASSERT_EQ(persisted.clusters_size(), 1);
+ ASSERT_TRUE(persisted.clusters(0).cluster_id().empty());
+ ASSERT_EQ(persisted.clusters(0).has_cluster_id(), explicit_empty_id);
+
+ // Seed an old timestamp so the refresh is observable even within the same
second.
+ constexpr int64_t old_mtime = 200;
+ persisted.mutable_clusters(0)->mutable_nodes(0)->set_mtime(old_mtime);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(ctx.meta_service_->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ txn->put(instance_key({create_req.instance_id()}),
persisted.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ auto expected = persisted.clusters(0);
+
+ const auto* operation = notify_decommissioned ? "notify_decommissioned" :
"decommission_node";
+ auto [http_status, resp] =
ctx.forward<MetaServiceResponseStatus>(operation, req);
+ ASSERT_EQ(http_status, 200) << resp.ShortDebugString();
+ ASSERT_EQ(resp.code(), MetaServiceCode::OK) << resp.msg();
+ persisted = ctx.get_instance_info(create_req.instance_id());
+ ASSERT_EQ(persisted.clusters_size(), 1);
+ ASSERT_EQ(persisted.clusters(0).nodes_size(), 1);
+ const auto mtime = persisted.clusters(0).nodes(0).mtime();
+ EXPECT_GT(mtime, old_mtime);
+ auto* expected_node = expected.mutable_nodes(0);
+ expected_node->set_mtime(mtime);
+ expected_node->set_status(notify_decommissioned ?
NodeStatusPB::NODE_STATUS_DECOMMISSIONED
+ :
NodeStatusPB::NODE_STATUS_DECOMMISSIONING);
+ EXPECT_EQ(persisted.clusters(0).SerializeAsString(),
expected.SerializeAsString());
+}
+
+INSTANTIATE_TEST_SUITE_P(StoredClusterId, NodeStatusWithoutClusterIdHttpTest,
+ testing::Combine(testing::Bool(), testing::Bool()));
+
TEST(MetaServiceHttpTest, GetClusterTest) {
HttpContext ctx(true);
diff --git a/cloud/test/mock_resource_manager.h
b/cloud/test/mock_resource_manager.h
index 59d0365c8e6..a072b0578b5 100644
--- a/cloud/test/mock_resource_manager.h
+++ b/cloud/test/mock_resource_manager.h
@@ -61,7 +61,8 @@ public:
std::function<bool(const ClusterPB&)> filter,
std::function<std::string(ClusterPB&, std::vector<ClusterPB>&
clusters_in_instance)>
action,
- bool replace_if_existing_empty_target_cluster) override {
+ bool replace_if_existing_empty_target_cluster,
+ bool allow_cluster_name_fallback) override {
return "";
}
diff --git a/cloud/test/resource_test.cpp b/cloud/test/resource_test.cpp
index 206f563a20a..75dbfd3288b 100644
--- a/cloud/test/resource_test.cpp
+++ b/cloud/test/resource_test.cpp
@@ -28,6 +28,7 @@
#include <atomic>
#include <condition_variable>
#include <cstdint>
+#include <functional>
#include <memory>
#include <optional>
#include <random>
@@ -217,6 +218,206 @@ static void drop_cluster(MetaServiceProxy* ms, const
std::string& instance_id,
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
}
+class NodeStatusResourceManager : public ResourceManager {
+public:
+ using ResourceManager::ResourceManager;
+
+ void refresh_instance(const std::string& instance_id, const
InstanceInfoPB& instance) override {
+ ResourceManager::refresh_instance(instance_id, instance);
+ after_refresh();
+ }
+
+ std::function<void()> after_refresh = [] {};
+};
+
+class NodeStatusTest : public testing::TestWithParam<bool> {
+protected:
+ static constexpr int64_t initial_node_mtime = 200;
+
+ void SetUp() override {
+ txn_kv = create_txn_kv();
+ resource_mgr = std::make_shared<NodeStatusResourceManager>(txn_kv);
+ ASSERT_EQ(resource_mgr->init(), 0);
+ auto rate_limiter = std::make_shared<RateLimiter>();
+ auto snapshot = std::make_shared<SnapshotManager>(txn_kv);
+ meta_service = std::make_unique<MetaServiceProxy>(
+ std::make_unique<MetaServiceImpl>(txn_kv, resource_mgr,
rate_limiter, snapshot));
+
+ instance.set_instance_id("node_status_instance");
+ for (int cluster_idx = 0; cluster_idx < 2; ++cluster_idx) {
+ auto* cluster = instance.add_clusters();
+ cluster->set_cluster_id("cluster_" + std::to_string(cluster_idx));
+ cluster->set_cluster_name("cluster_" +
std::to_string(cluster_idx));
+ cluster->set_type(ClusterPB::COMPUTE);
+ for (int node_idx = 0; node_idx < 3; ++node_idx) {
+ auto* node = cluster->add_nodes();
+ node->set_cloud_unique_id(node_idx == 0
+ ? "unique_node_" +
std::to_string(cluster_idx)
+ : "shared_node");
+ GetParam() ? node->set_host("host_" +
std::to_string(cluster_idx))
+ : node->set_ip("127.0.0." +
std::to_string(cluster_idx + 1));
+ node->set_heartbeat_port(10000 + node_idx);
+ node->set_status(NodeStatusPB::NODE_STATUS_RUNNING);
+ node->set_ctime(100);
+ node->set_mtime(initial_node_mtime);
+ }
+ }
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(instance_key({instance.instance_id()}),
instance.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ resource_mgr->refresh_instance(instance.instance_id(), instance);
+ }
+
+ AlterClusterRequest request_for_node(int node_idx) {
+ AlterClusterRequest req;
+ req.set_instance_id(instance.instance_id());
+ req.mutable_cluster()->CopyFrom(instance.clusters(0));
+ req.mutable_cluster()->clear_nodes();
+
req.mutable_cluster()->add_nodes()->CopyFrom(instance.clusters(0).nodes(node_idx));
+ return req;
+ }
+
+ void alter_node(const AlterClusterRequest& req, MetaServiceCode
expected_code) {
+ brpc::Controller cntl;
+ AlterClusterResponse resp;
+ meta_service->alter_cluster(&cntl, &req, &resp, brpc::DoNothing());
+ ASSERT_FALSE(cntl.Failed());
+ ASSERT_EQ(resp.status().code(), expected_code) << resp.status().msg();
+ }
+
+ void check_nodes(NodeInfoPB* updated_node = nullptr) {
+ InstanceInfoPB persisted;
+ get_instance_info(meta_service.get(), &persisted,
instance.instance_id());
+ ASSERT_EQ(persisted.clusters_size(), instance.clusters_size());
+ for (int i = 0; i < instance.clusters_size(); ++i) {
+ ASSERT_EQ(persisted.clusters(i).nodes_size(),
instance.clusters(i).nodes_size());
+ for (int j = 0; j < instance.clusters(i).nodes_size(); ++j) {
+ const auto& expected_node = instance.clusters(i).nodes(j);
+ const auto& persisted_node = persisted.clusters(i).nodes(j);
+ if (&expected_node == updated_node) {
+ EXPECT_GT(persisted_node.mtime(), initial_node_mtime);
+ updated_node->set_mtime(persisted_node.mtime());
+ }
+ EXPECT_EQ(persisted_node.SerializeAsString(),
expected_node.SerializeAsString());
+ EXPECT_EQ(get_instance_id(resource_mgr,
expected_node.cloud_unique_id()),
+ instance.instance_id());
+ }
+ }
+ }
+
+ std::shared_ptr<TxnKv> txn_kv;
+ InstanceInfoPB instance;
+ std::shared_ptr<NodeStatusResourceManager> resource_mgr;
+ std::unique_ptr<MetaServiceProxy> meta_service;
+};
+
+TEST_P(NodeStatusTest, KeepsNodesResolvableAtEveryPublishedUpdate) {
+ int refreshes = 0;
+ NodeInfoPB* updated_node = nullptr;
+ // Observe a reader immediately after each committed metadata/cache update.
+ resource_mgr->after_refresh = [&] {
+ ++refreshes;
+ EXPECT_EQ(get_instance_id(resource_mgr, "unique_node_0"),
instance.instance_id());
+ check_nodes(updated_node);
+ };
+ for (int node_idx : {0, 1}) {
+ for (auto op :
+ {AlterClusterRequest::DECOMMISSION_NODE,
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+ auto req = request_for_node(node_idx);
+ req.set_op(op);
+ updated_node =
instance.mutable_clusters(0)->mutable_nodes(node_idx);
+ updated_node->set_status(op ==
AlterClusterRequest::DECOMMISSION_NODE
+ ?
NodeStatusPB::NODE_STATUS_DECOMMISSIONING
+ :
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
+ int before_refreshes = refreshes;
+ alter_node(req, MetaServiceCode::OK);
+ EXPECT_EQ(refreshes, before_refreshes + 1);
+ check_nodes();
+ // Preserve HTTP requests that identify a cluster by the node
rather than its ID.
+ req.mutable_cluster()->clear_cluster_id();
+ alter_node(req, MetaServiceCode::OK);
+ check_nodes();
+ }
+ }
+ resource_mgr->after_refresh = [] {};
+}
+
+TEST_P(NodeStatusTest, UpdatesNodesWithoutStoredClusterIdAtomically) {
+ for (bool explicit_empty_id : {false, true}) {
+ SCOPED_TRACE(explicit_empty_id);
+ auto* cluster = instance.mutable_clusters(0);
+ cluster->clear_cluster_id();
+ if (explicit_empty_id) {
+ cluster->set_cluster_id("");
+ }
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(instance_key({instance.instance_id()}),
instance.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ resource_mgr->refresh_instance(instance.instance_id(), instance);
+
+ int refreshes = 0;
+ auto* updated_node = cluster->mutable_nodes(1);
+ resource_mgr->after_refresh = [&] {
+ ++refreshes;
+ check_nodes(updated_node);
+ };
+ for (auto op :
+ {AlterClusterRequest::DECOMMISSION_NODE,
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+ SCOPED_TRACE(op);
+ // A shared UID must resolve this named cluster without changing
the other cluster.
+ auto req = request_for_node(1);
+ req.mutable_cluster()->clear_cluster_name();
+ req.set_op(op);
+ updated_node->set_status(op ==
AlterClusterRequest::DECOMMISSION_NODE
+ ?
NodeStatusPB::NODE_STATUS_DECOMMISSIONING
+ :
NodeStatusPB::NODE_STATUS_DECOMMISSIONED);
+ int before_refreshes = refreshes;
+ alter_node(req, MetaServiceCode::OK);
+ EXPECT_EQ(refreshes, before_refreshes + 1);
+ check_nodes();
+ InstanceInfoPB persisted;
+ get_instance_info(meta_service.get(), &persisted,
instance.instance_id());
+ EXPECT_EQ(persisted.clusters(0).SerializeAsString(),
+ instance.clusters(0).SerializeAsString());
+ }
+ resource_mgr->after_refresh = [] {};
+ }
+}
+
+TEST_P(NodeStatusTest, RejectsMissingNodesWithoutPublishingPartialUpdates) {
+ int refreshes = 0;
+ resource_mgr->after_refresh = [&] { ++refreshes; };
+ for (auto op :
+ {AlterClusterRequest::DECOMMISSION_NODE,
AlterClusterRequest::NOTIFY_DECOMMISSIONED}) {
+ auto req = request_for_node(0);
+ req.set_op(op);
+
req.mutable_cluster()->add_nodes()->CopyFrom(instance.clusters(0).nodes(1));
+ req.mutable_cluster()->mutable_nodes(1)->set_heartbeat_port(12345);
+ alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(refreshes, 0);
+ check_nodes();
+
+ req = request_for_node(0);
+ req.set_op(op);
+
req.mutable_cluster()->mutable_nodes(0)->set_cloud_unique_id("missing_uid");
+ alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(refreshes, 0);
+ check_nodes();
+
+ req = request_for_node(0);
+ req.set_op(op);
+ req.mutable_cluster()->set_cluster_id("missing_cluster");
+ alter_node(req, MetaServiceCode::INVALID_ARGUMENT);
+ EXPECT_EQ(refreshes, 0);
+ check_nodes();
+ }
+ resource_mgr->after_refresh = [] {};
+}
+
+INSTANTIATE_TEST_SUITE_P(NodeAddress, NodeStatusTest, testing::Bool());
+
// test cluster's node addr use ip
TEST(ResourceTest, ModifyNodesIpTest) {
auto meta_service = get_meta_service();
diff --git a/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out
b/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out
new file mode 100644
index 00000000000..84e20211dba
--- /dev/null
+++ b/regression-test/data/cloud_p0/node_mgr/test_cloud_decommission.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !decommission_atomic --
+1 1
+
+-- !decommissioned_atomic --
+1 1
+
diff --git
a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
index d2d4947f859..0033d06d11c 100644
--- a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
+++ b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy
@@ -17,6 +17,7 @@
import groovy.json.JsonSlurper
import groovy.json.JsonOutput
import org.apache.doris.regression.suite.ClusterOptions
+import java.util.concurrent.TimeUnit
suite("cloud_decommission", 'p0, docker') {
if (!isCloudMode()) {
@@ -51,7 +52,7 @@ suite("cloud_decommission", 'p0, docker') {
}
}
- def check = { Closure beforeDecommissionActionSupplier, Closure
afterDecommissionActionSupplier, int beNum ->
+ def checkDecommission = { Closure beforeDecommissionActionSupplier,
Closure afterDecommissionActionSupplier, int beNum ->
def begin = System.currentTimeMillis()
setFeConfig("cloud_balance_tablet_percent_per_run", 0.5)
@@ -160,7 +161,7 @@ suite("cloud_decommission", 'p0, docker') {
def checkDifferentAction = { Closure beforeDecommissionActionSupplier,
Closure afterDecommissionActionSupplier, int atLeastCost, int waitTime, int
beNum ->
def begin = System.currentTimeMillis()
- def cost = check.call(beforeDecommissionActionSupplier,
afterDecommissionActionSupplier, beNum)
+ def cost = checkDecommission.call(beforeDecommissionActionSupplier,
afterDecommissionActionSupplier, beNum)
log.info("in check, inner cost {}", cost)
cost = System.currentTimeMillis() - begin
log.info("in check, outter cost {}", cost)
@@ -251,6 +252,114 @@ suite("cloud_decommission", 'p0, docker') {
assertEquals(2, ret.size())
}
+ // Keep the target RPC in flight while a node-state update is published.
+ // Both builds use the same gates: the old one publishes a missing node,
+ // while the fixed one publishes the same node with its new status.
+ for (String operation : ["decommission_node", "notify_decommissioned"]) {
+ def atomicOptions = new ClusterOptions()
+ atomicOptions.setFeNum(1)
+ atomicOptions.setBeNum(1)
+ atomicOptions.cloudMode = true
+ atomicOptions.feConfigs +=
['cloud_tablet_rebalancer_interval_second=3600']
+ docker(atomicOptions) {
+ sql "USE @compute_cluster"
+ sql "DROP TABLE IF EXISTS decommission_atomic_table"
+ sql """CREATE TABLE decommission_atomic_table (id INT)
+ DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES
("replication_num"="1")"""
+ // Wait for the automatic decommission checker to finish its first
round.
+ // Its next round is an hour away, so only this test changes node
status.
+ def feLog = new File(cluster.getFeByIndex(1).getLogFilePath())
+ awaitUntil(60) { feLog.text.contains("rebalancer sche interval
3600 s") }
+
+ def be = (sql_return_maparray("SHOW BACKENDS"))[0]
+ def tag = new JsonSlurper().parseText(be.Tag as String)
+ def ms = cluster.getAllMetaservices().get(0)
+ def msLog = new File(ms.getLogFilePath())
+ def nodePoint = "ResourceManager::node_update:after_refresh"
+ def commitPoint =
"commit_rowset::before_get_instance_id:${tag.cloud_unique_id}"
+ def inject = { String action, String point = "" ->
+ httpTest {
+ op "get"
+ endpoint "${ms.host}:${ms.httpPort}"
+ uri
"/MetaService/http/v1/injection_point?token=${token}&op=${action}&name=${point}&behavior=wait"
+ check { code, response ->
+ assertEquals(200, code)
+ assertEquals("OK", parseJson(response).code)
+ }
+ }
+ }
+ def waitForPoint = { String point, long offset ->
+ awaitUntil(60) {
+ def log = new RandomAccessFile(msLog, "r")
+ try {
+ log.seek(offset)
+ byte[] bytes = new byte[(int) (log.length() - offset)]
+ log.readFully(bytes)
+ new String(bytes, "UTF-8").contains("injection point
waiting, point=${point}")
+ } finally {
+ log.close()
+ }
+ }
+ logger.info("Reached MS injection point {}", point)
+ }
+ def alterStatus = { String action ->
+ httpTest {
+ endpoint "${ms.host}:${ms.httpPort}"
+ uri "/MetaService/http/${action}?token=${token}"
+ body JsonOutput.toJson([
+ instance_id: context.config.instanceId,
+ cluster: [
+ type: "COMPUTE",
+ cluster_id: tag.compute_group_id,
+ cluster_name: tag.compute_group_name,
+ nodes: [[cloud_unique_id: tag.cloud_unique_id,
+ ip: be.Host, heartbeat_port:
be.HeartbeatPort as int]]
+ ]
+ ])
+ check { code, response ->
+ assertEquals(200, code)
+ assertEquals("OK", parseJson(response).code)
+ }
+ }
+ }
+ def load = null
+ def alter = null
+ inject("enable")
+ try {
+ long loadOffset = msLog.length()
+ inject("set", commitPoint)
+ load = thread { sql "INSERT INTO decommission_atomic_table
VALUES (1)" }
+ waitForPoint(commitPoint, loadOffset)
+ if (operation == "notify_decommissioned") {
+ alterStatus("decommission_node")
+ }
+
+ long updateOffset = msLog.length()
+ inject("set", nodePoint)
+ alter = thread { alterStatus(operation) }
+ waitForPoint(nodePoint, updateOffset)
+ // The node-state request stays blocked until the load
finishes.
+ inject("clear", commitPoint)
+ load.get(60, TimeUnit.SECONDS)
+ if (operation == "decommission_node") {
+ qt_decommission_atomic "SELECT count(*), sum(id) FROM
decommission_atomic_table"
+ } else {
+ qt_decommissioned_atomic "SELECT count(*), sum(id) FROM
decommission_atomic_table"
+ }
+ } finally {
+ inject("clear", commitPoint)
+ inject("clear", nodePoint)
+ inject("disable")
+ if (load != null && !load.isDone()) {
+ load.get(60, TimeUnit.SECONDS)
+ }
+ if (alter != null) {
+ alter.get(60, TimeUnit.SECONDS)
+ }
+ }
+ }
+ }
+
def clusterOptions = [
new ClusterOptions(),
new ClusterOptions(),
@@ -279,7 +388,7 @@ suite("cloud_decommission", 'p0, docker') {
def noWalCheckCost = 0
docker(clusterOptions[i]) {
createTestTable.call()
- noWalCheckCost = check(null, null, beNum)
+ noWalCheckCost = checkDecommission(null, null, beNum)
checkFENormalAfterRestart.call()
}
log.info("no wal check cost {}", noWalCheckCost)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]