This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4f79307b744 [improvement](cloud) Add dry-run observability for
MetaService rate limiting (#66969)
4f79307b744 is described below
commit 4f79307b74456ee0264361c08c600831c8ca73c2
Author: Yixuan Wang <[email protected]>
AuthorDate: Tue Aug 25 10:00:15 2026 +0800
[improvement](cloud) Add dry-run observability for MetaService rate
limiting (#66969)
### What problem does this PR solve?
Problem Summary:
MetaService rate limiting lacks sufficient observability before
enforcement is enabled, making it difficult to evaluate trigger
frequency and validate stress thresholds safely.
Introduce `enable_ms_rate_limit_dry_run` to evaluate rate-limit
conditions without rejecting requests. Record trigger metrics by
pressure reason and export the sampled MetaService CPU and memory usage
through bvars.
Rate-limit enforcement remains controlled independently by
`enable_ms_rate_limit` and `enable_ms_rate_limit_injection`.
curl '<meta_service_host>:<meta_service_port>/brpc_metrics' | grep -E
'ms.*trigger' | grep -v '#'
```
ms_rate_limit_trigger_fdb_client_thread 10
ms_rate_limit_trigger_fdb_cluster 25
ms_rate_limit_trigger_ms_resource 12
ms_rate_limit_trigger_test_injection 0
```
---
cloud/src/common/bvars.cpp | 10 ++++++++++
cloud/src/common/bvars.h | 6 ++++++
cloud/src/common/config.h | 4 +++-
cloud/src/meta-service/meta_service_helper.h | 11 ++++++-----
.../meta-service/meta_service_rate_limit_helper.cpp | 18 ++++++++++++++++++
.../src/meta-service/meta_service_rate_limit_helper.h | 8 ++++++++
6 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index a70ffec1d51..18da953b1c1 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -113,6 +113,16 @@ BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot("ms",
"list_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_clone_instance("ms", "clone_instance");
BvarLatencyRecorderWithTag g_bvar_ms_compact_snapshot("ms",
"compact_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info("ms",
"update_packed_file_info");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_cluster(
+ "ms", "rate_limit_trigger_fdb_cluster");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_client_thread(
+ "ms", "rate_limit_trigger_fdb_client_thread");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_ms_resource(
+ "ms", "rate_limit_trigger_ms_resource");
+bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_test_injection(
+ "ms", "rate_limit_trigger_test_injection");
+bvar::Status<int64_t>
g_bvar_ms_cpu_usage_percent("ms_process_cpu_usage_percent", -1);
+bvar::Status<int64_t>
g_bvar_ms_memory_usage_percent("ms_process_memory_usage_percent", -1);
bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
bvar::Window<bvar::Adder<int64_t> >
g_bvar_update_delete_bitmap_fail_counter_minute("ms",
"update_delete_bitmap_fail", &g_bvar_update_delete_bitmap_fail_counter, 60);
bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index 7aa56bdd101..8c40c131666 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -624,6 +624,12 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_clone_instance;
extern BvarLatencyRecorderWithTag g_bvar_ms_compact_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_cluster;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_fdb_client_thread;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_ms_resource;
+extern bvar::Adder<int64_t> g_bvar_ms_rate_limit_trigger_test_injection;
+extern bvar::Status<int64_t> g_bvar_ms_cpu_usage_percent;
+extern bvar::Status<int64_t> g_bvar_ms_memory_usage_percent;
extern bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
extern bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
extern BvarLatencyRecorderWithStatus<60>
g_bvar_ms_txn_commit_with_tablet_count;
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index c147f910d97..faae59ad7e3 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -208,7 +208,9 @@ CONF_Int64(default_max_qps_limit, "1000000");
CONF_String(specific_max_qps_limit, "get_cluster:5000000;begin_txn:5000000");
CONF_Bool(enable_rate_limit, "true");
CONF_Int64(bvar_qps_update_second, "5");
-CONF_mBool(enable_ms_rate_limit, "false");
+CONF_mBool(enable_ms_rate_limit, "true");
+// Collect rate-limit trigger metrics independently of rate-limit enforcement.
+CONF_mBool(enable_ms_rate_limit_dry_run, "true");
// Fault injection: randomly return meta service rate limit error for testing.
// ms_rate_limit_injection_probability is the probability (0-100) of injecting
a rate limit error.
CONF_mBool(enable_ms_rate_limit_injection, "false");
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index ab8605c20a1..2dd1a000e3a 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -345,11 +345,12 @@ inline MetaServiceCode cast_as(TxnErrorCode code) {
[[maybe_unused]] std::string instance_id;
\
[[maybe_unused]] bool drop_request = false;
\
[[maybe_unused]] KVStats stats;
\
- [[maybe_unused]] MsStressDecision ms_stress_decision;
\
- if (config::enable_ms_rate_limit ||
config::enable_ms_rate_limit_injection) { \
- ms_stress_decision = get_ms_stress_decision();
\
- if (RpcRateLimitWhitelist::instance().should_rate_limit(#func_name) &&
\
- ms_stress_decision.under_great_stress()) {
\
+ [[maybe_unused]] MsStressDecision ms_stress_decision =
get_ms_stress_decision(); \
+ if (config::enable_ms_rate_limit &&
\
+ RpcRateLimitWhitelist::instance().should_rate_limit(#func_name) &&
\
+ ms_stress_decision.under_great_stress()) {
\
+ record_ms_rate_limit_triggers(ms_stress_decision);
\
+ if (!config::enable_ms_rate_limit_dry_run) {
\
drop_request = true;
\
msg = ms_stress_decision.debug_string();
\
code = MetaServiceCode::MS_TOO_BUSY;
\
diff --git a/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
b/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
index 9af2aead05c..5418db7f68e 100644
--- a/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
+++ b/cloud/src/meta-service/meta_service_rate_limit_helper.cpp
@@ -564,6 +564,9 @@ public:
// Compute decision from metrics and store it in latest_decision_.
// Called by the background thread or synchronously in tests.
void update(int64_t now_ms, const MsStressMetrics& metrics) {
+ g_bvar_ms_cpu_usage_percent.set_value(metrics.ms_cpu_usage_percent);
+
g_bvar_ms_memory_usage_percent.set_value(metrics.ms_memory_usage_percent);
+
MsStressDecision decision;
decision.fdb_commit_latency_ns = metrics.fdb_commit_latency_ns;
decision.fdb_read_latency_ns = metrics.fdb_read_latency_ns;
@@ -800,6 +803,21 @@ MsStressDecision get_ms_stress_decision() {
return decision;
}
+void record_ms_rate_limit_triggers(const MsStressDecision& decision) {
+ if (decision.fdb_cluster_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_fdb_cluster << 1;
+ }
+ if (decision.fdb_client_thread_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_fdb_client_thread << 1;
+ }
+ if (decision.ms_resource_under_pressure) {
+ g_bvar_ms_rate_limit_trigger_ms_resource << 1;
+ }
+ if (decision.rate_limit_injected_for_test) {
+ g_bvar_ms_rate_limit_trigger_test_injection << 1;
+ }
+}
+
MsStressDecision update_ms_stress_detector_for_test(int64_t now_ms, const
MsStressMetrics& metrics,
bool reset,
int32_t
rate_limit_injected_random_value) {
diff --git a/cloud/src/meta-service/meta_service_rate_limit_helper.h
b/cloud/src/meta-service/meta_service_rate_limit_helper.h
index 365c53e9aaa..0b3d67e81dd 100644
--- a/cloud/src/meta-service/meta_service_rate_limit_helper.h
+++ b/cloud/src/meta-service/meta_service_rate_limit_helper.h
@@ -61,6 +61,14 @@ struct MsStressDecision {
};
MsStressDecision get_ms_stress_decision();
+
+// Record each triggered reason independently:
+// 1. FDB cluster pressure
+// 2. FDB client-thread pressure,
+// 3. MetaService resource pressure
+// 4. test injection.
+void record_ms_rate_limit_triggers(const MsStressDecision& decision);
+
MsStressDecision update_ms_stress_detector_for_test(int64_t now_ms, const
MsStressMetrics& metrics,
bool reset = false,
int32_t
rate_limit_injected_random_value = -1);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
