This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new e789cf3f24a branch-4.1: [improvement](cloud) Add dry-run observability
for MetaService rate limiting #66969 (#67086)
e789cf3f24a is described below
commit e789cf3f24ac108652bf3a41f3cdbae8b981851a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Aug 26 10:40:33 2026 +0800
branch-4.1: [improvement](cloud) Add dry-run observability for MetaService
rate limiting #66969 (#67086)
Cherry-picked from #66969
Co-authored-by: Yixuan Wang <[email protected]>
---
cloud/src/common/bvars.cpp | 10 ++++++++++
cloud/src/common/bvars.h | 6 ++++++
cloud/src/common/config.h | 4 +++-
cloud/src/meta-service/meta_service_helper.h | 11 ++++++-----
.../meta-service/meta_service_rate_limit_helper.cpp | 18 ++++++++++++++++++
.../src/meta-service/meta_service_rate_limit_helper.h | 8 ++++++++
6 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index f22495e0ecc..004dd1a8cc2 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -111,6 +111,16 @@ BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot("ms",
"list_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_clone_instance("ms", "clone_instance");
BvarLatencyRecorderWithTag g_bvar_ms_compact_snapshot("ms",
"compact_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info("ms",
"update_packed_file_info");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_cluster(
+ "ms", "rate_limit_trigger_fdb_cluster");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_client_thread(
+ "ms", "rate_limit_trigger_fdb_client_thread");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_ms_resource(
+ "ms", "rate_limit_trigger_ms_resource");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_test_injection(
+ "ms", "rate_limit_trigger_test_injection");
+bvar::Status<int64_t>
g_bvar_ms_cpu_usage_percent("ms_process_cpu_usage_percent", -1);
+bvar::Status<int64_t>
g_bvar_ms_memory_usage_percent("ms_process_memory_usage_percent", -1);
bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
bvar::Window<bvar::Adder<int64_t> >
g_bvar_update_delete_bitmap_fail_counter_minute("ms",
"update_delete_bitmap_fail", &g_bvar_update_delete_bitmap_fail_counter, 60);
bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index d1a7bb6c8c7..b488283670e 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -623,6 +623,12 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_clone_instance;
extern BvarLatencyRecorderWithTag g_bvar_ms_compact_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_cluster;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_client_thread;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_ms_resource;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_test_injection;
+extern bvar::Status<int64_t> g_bvar_ms_cpu_usage_percent;
+extern bvar::Status<int64_t> g_bvar_ms_memory_usage_percent;
extern bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
extern bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
extern BvarLatencyRecorderWithStatus<60>
g_bvar_ms_txn_commit_with_tablet_count;
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 5af6e31a308..238cf59b50d 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -204,7 +204,9 @@ CONF_Int64(default_max_qps_limit, "1000000");
CONF_String(specific_max_qps_limit, "get_cluster:5000000;begin_txn:5000000");
CONF_Bool(enable_rate_limit, "true");
CONF_Int64(bvar_qps_update_second, "5");
-CONF_mBool(enable_ms_rate_limit, "false");
+CONF_mBool(enable_ms_rate_limit, "true");
+// Collect rate-limit trigger metrics independently of rate-limit enforcement.
+CONF_mBool(enable_ms_rate_limit_dry_run, "true");
// Fault injection: randomly return meta service rate limit error for testing.
// ms_rate_limit_injection_probability is the probability (0-100) of injecting
a rate limit error.
CONF_mBool(enable_ms_rate_limit_injection, "false");
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index ab8605c20a1..2dd1a000e3a 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -345,11 +345,12 @@ inline MetaServiceCode cast_as(TxnErrorCode code) {
[[maybe_unused]] std::string instance_id;
\
[[maybe_unused]] bool drop_request = false;
\
[[maybe_unused]] KVStats stats;
\
- [[maybe_unused]] MsStressDecision ms_stress_decision;
\
- if (config::enable_ms_rate_limit ||
config::enable_ms_rate_limit_injection) { \
- ms_stress_decision = get_ms_stress_decision();
\
- if (RpcRateLimitWhitelist::instance().should_rate_limit(#func_name) &&
\
- ms_stress_decision.under_great_stress()) {
\
+ [[maybe_unused]] MsStressDecision ms_stress_decision =
get_ms_stress_decision(); \
+ if (config::enable_ms_rate_limit &&
\
+ RpcRateLimitWhitelist::instance().should_rate_limit(#func_name) &&
\
+ ms_stress_decision.under_great_stress()) {
\
+ record_ms_rate_limit_triggers(ms_stress_decision);
\
+ if (!config::enable_ms_rate_limit_dry_run) {
\
drop_request = true;
\
msg = ms_stress_decision.debug_string();
\
code = MetaServiceCode::MS_TOO_BUSY;
\
diff --git a/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
b/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
index 9af2aead05c..5418db7f68e 100644
--- a/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
+++ b/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
@@ -564,6 +564,9 @@ public:
// Compute decision from metrics and store it in latest_decision_.
// Called by the background thread or synchronously in tests.
void update(int64_t now_ms, const MsStressMetrics& metrics) {
+ g_bvar_ms_cpu_usage_percent.set_value(metrics.ms_cpu_usage_percent);
+
g_bvar_ms_memory_usage_percent.set_value(metrics.ms_memory_usage_percent);
+
MsStressDecision decision;
decision.fdb_commit_latency_ns = metrics.fdb_commit_latency_ns;
decision.fdb_read_latency_ns = metrics.fdb_read_latency_ns;
@@ -800,6 +803,21 @@ MsStressDecision get_ms_stress_decision() {
return decision;
}
+void record_ms_rate_limit_triggers(const MsStressDecision& decision) {
+ if (decision.fdb_cluster_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_fdb_cluster << 1;
+ }
+ if (decision.fdb_client_thread_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_fdb_client_thread << 1;
+ }
+ if (decision.ms_resource_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_ms_resource << 1;
+ }
+ if (decision.rate_limit_injected_for_test) {
+ g_bvar_ms_rate_limit_trigger_test_injection << 1;
+ }
+}
+
MsStressDecision update_ms_stress_detector_for_test(int64_t now_ms, const
MsStressMetrics& metrics,
bool reset,
int32_t
rate_limit_injected_random_value) {
diff --git a/cloud/src/meta-service/meta_service_rate_limit_helper.h
b/cloud/src/meta-service/meta_service_rate_limit_helper.h
index 365c53e9aaa..0b3d67e81dd 100644
--- a/cloud/src/meta-service/meta_service_rate_limit_helper.h
+++ b/cloud/src/meta-service/meta_service_rate_limit_helper.h
@@ -61,6 +61,14 @@ struct MsStressDecision {
};
MsStressDecision get_ms_stress_decision();
+
+// Record each triggered reason independently:
+// 1. FDB cluster pressure
+// 2. FDB client-thread pressure,
+// 3. MetaService resource pressure
+// 4. test injection.
+void record_ms_rate_limit_triggers(const MsStressDecision& decision);
+
MsStressDecision update_ms_stress_detector_for_test(int64_t now_ms, const
MsStressMetrics& metrics,
bool reset = false,
int32_t
rate_limit_injected_random_value = -1);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
