This is an automated email from the ASF dual-hosted git repository. wu-sheng pushed a commit to branch swip-15-banyandb-queue-batch-metrics in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit a2a85cc9ab2bf08759d882bcf1064b22af58ccc0 Author: Wu Sheng <[email protected]> AuthorDate: Thu Jun 11 15:29:22 2026 +0800 SWIP-15: add BanyanDB queue batch/message metrics (BanyanDB #1169) --- docs/en/banyandb/dashboards-banyandb.md | 5 ++ docs/en/changes/changes.md | 8 +++ .../banyandb/banyandb-endpoint.data.yaml | 36 ++++++++++ .../banyandb/banyandb-instance.data.yaml | 83 ++++++++++++++++++++++ .../otel-rules/banyandb/banyandb-endpoint.yaml | 11 +++ .../otel-rules/banyandb/banyandb-instance.yaml | 14 ++++ test/e2e-v2/cases/banyandb/banyandb-cases.yaml | 12 ++++ test/e2e-v2/script/env | 2 +- 8 files changed, 170 insertions(+), 1 deletion(-) diff --git a/docs/en/banyandb/dashboards-banyandb.md b/docs/en/banyandb/dashboards-banyandb.md index 78e9094144..96771f9183 100644 --- a/docs/en/banyandb/dashboards-banyandb.md +++ b/docs/en/banyandb/dashboards-banyandb.md @@ -126,8 +126,10 @@ stay in lockstep with the BanyanDB catalog. The rule files are | r/s | `non_query_op_rate` | Registry / non-query operation rate | | w/s | `write_rate` | Write rate seen at the front door | | ops | `publish_throughput` | Tier-2 publish throughput by operation | +| ops | `publish_batch_throughput` | Tier-2 publish batch throughput by operation | | Bytes/s | `publish_bytes` | Publish bytes | | s | `publish_latency_p99` | Publish send latency p99 | +| s | `publish_batch_latency_p99` | Publish batch send latency p99 | | Count | `wqueue_pending` / `wqueue_file_parts` / `wqueue_mem_part` | Write-queue depth | **Data** (backend; the dashboard gates these on `container_name == 'data'`): @@ -143,6 +145,7 @@ stay in lockstep with the BanyanDB catalog. The rule files are | o/s | `stream_tst_write_rate` / `stream_tst_term_search_rate` | Stream tst index write / term-search rate | | Count | `stream_tst_total_docs` | Stream tst index documents | | ops | `queue_sub_throughput` | Subscribe-queue throughput by operation | +| ops | `queue_sub_message_throughput` | Subscribe-queue per-message throughput by operation | | s | `queue_sub_latency_p99` | Subscribe-queue latency p99 | | percent | `retention_measure_disk_usage_percent` / `retention_stream_disk_usage_percent` / `retention_trace_disk_usage_percent` | Retention disk-usage % per scope | @@ -167,6 +170,8 @@ stay in lockstep with the BanyanDB catalog. The rule files are | o/s | `series_write_rate` | Inverted-index write rate for the group | | Count | `total_series` | Inverted-index documents for the group | | ops | `queue_throughput` | Subscribe-queue throughput by operation for the group | +| ops | `queue_batch_throughput` | Subscribe-queue batch throughput by operation for the group | +| ops | `queue_message_throughput` | Subscribe-queue per-message throughput by operation for the group | | s | `queue_latency_p99` | Publish-queue latency p99 for the group | | Bytes/s | `publish_bytes` | Publish bytes for the group | diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 3cc1f13e18..3b9d0167f0 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -250,6 +250,14 @@ `banyandb-instance.yaml` redesigned to mirror the upstream FODC-proxy Grafana boards. The stale single-node `host_name` model and the removed `etcd_operation_rate` / `up`-derived `active_instance` metrics are gone. +* SWIP-15: add BanyanDB queue batch / message granularity metrics (requires BanyanDB 0.11.x with the + `queue_pub_total_batch_*` / `queue_sub_total_message_*` families, BanyanDB #1169). Instance scope gains + `publish_batch_throughput` / `publish_batch_latency_p99` (liaison publish-side batch granularity) and + `queue_sub_message_throughput` (data per-record subscribe rate); endpoint scope gains + `queue_batch_throughput` / `queue_message_throughput` per storage group (subscribe-side counters that + carry a real `group`). The subscribe-side batch counters and the publish-side batch-latency histogram + are not surfaced where they do not populate per their modeled scope (the data node ingests via the + per-message path, and `queue_pub_total_batch_latency` is emitted with an empty `group`). * Runtime MAL/LAL hot-update rules can declare `layerDefinitions:` to introduce new layers. Ordinals are operator-pinned in the `100_000+` tier; the layer is refcount-tracked and unregistered when the last declaring rule is removed. See diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-endpoint.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-endpoint.data.yaml index 0cf20a0121..47a39d5ea9 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-endpoint.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-endpoint.data.yaml @@ -167,6 +167,18 @@ input: remote_role: 'data' remote_tier: 'hot' value: 12.0 + banyandb_queue_sub_total_batch_finished: + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + remote_node: 'n1' + remote_role: 'data' + value: 100.0 banyandb_queue_sub_total_finished: - labels: cluster: 'test-cluster' @@ -179,6 +191,18 @@ input: remote_node: 'n1' remote_role: 'data' value: 100.0 + banyandb_queue_sub_total_message_finished: + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + remote_node: 'n1' + remote_role: 'data' + value: 100.0 banyandb_stream_storage_inverted_index_total_doc_count: - labels: cluster: 'test-cluster' @@ -374,6 +398,18 @@ expected: service: test-cluster endpoint: test-group layer: BANYANDB + meter_banyandb_endpoint_queue_batch_throughput: + entities: + - scope: ENDPOINT + service: test-cluster + endpoint: test-group + layer: BANYANDB + meter_banyandb_endpoint_queue_message_throughput: + entities: + - scope: ENDPOINT + service: test-cluster + endpoint: test-group + layer: BANYANDB meter_banyandb_endpoint_publish_bytes: entities: - scope: ENDPOINT diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-instance.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-instance.data.yaml index 357a109c84..e190e3f9dc 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-instance.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/banyandb/banyandb-instance.data.yaml @@ -220,6 +220,59 @@ input: remote_role: 'data' remote_tier: 'hot' value: 100.0 + banyandb_queue_pub_total_batch_finished: + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + remote_node: 'n1' + remote_role: 'data' + remote_tier: 'hot' + value: 100.0 + banyandb_queue_pub_total_batch_latency: + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + le: '50' + remote_node: 'n1' + remote_role: 'data' + remote_tier: 'hot' + value: 5.0 + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + le: '100' + remote_node: 'n1' + remote_role: 'data' + remote_tier: 'hot' + value: 10.0 + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + le: '500' + remote_node: 'n1' + remote_role: 'data' + remote_tier: 'hot' + value: 12.0 banyandb_queue_pub_total_finished: - labels: cluster: 'test-cluster' @@ -322,6 +375,18 @@ input: remote_node: 'n1' remote_role: 'data' value: 12.0 + banyandb_queue_sub_total_message_finished: + - labels: + cluster: 'test-cluster' + pod_name: 'test-pod' + container_name: 'data' + node_role: 'ROLE_DATA' + node_type: 'hot' + operation: 'batch-write' + group: 'test-group' + remote_node: 'n1' + remote_role: 'data' + value: 100.0 banyandb_storage_retention_measure_disk_usage_percent: - labels: cluster: 'test-cluster' @@ -834,6 +899,18 @@ expected: service: test-cluster instance: test-pod@data layer: BANYANDB + meter_banyandb_instance_publish_batch_throughput: + entities: + - scope: SERVICE_INSTANCE + service: test-cluster + instance: test-pod@data + layer: BANYANDB + meter_banyandb_instance_publish_batch_latency_p99: + entities: + - scope: SERVICE_INSTANCE + service: test-cluster + instance: test-pod@data + layer: BANYANDB meter_banyandb_instance_wqueue_pending: entities: - scope: SERVICE_INSTANCE @@ -924,6 +1001,12 @@ expected: service: test-cluster instance: test-pod@data layer: BANYANDB + meter_banyandb_instance_queue_sub_message_throughput: + entities: + - scope: SERVICE_INSTANCE + service: test-cluster + instance: test-pod@data + layer: BANYANDB meter_banyandb_instance_retention_measure_disk_usage_percent: entities: - scope: SERVICE_INSTANCE diff --git a/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-endpoint.yaml b/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-endpoint.yaml index 13ecb57fb3..76c2537ca7 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-endpoint.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-endpoint.yaml @@ -90,6 +90,17 @@ metricsRules: - name: queue_latency_p99 exp: banyandb_queue_pub_total_latency.sum(['le', 'cluster', 'group', 'operation']).histogram().histogram_percentile([99]) + # batch + message granularity per group (BanyanDB #1169). Both are subscribe-side counters that + # carry a real `group`, so they roll up per Endpoint: batch throughput is the per-group rate of + # write-batches finished (the liaison's OAP-facing receive hop contributes these), message + # throughput is the per-group per-record rate. (The publish-side batch latency histogram is NOT + # modeled here: queue_pub_total_batch_latency is emitted with group="" so it cannot roll up per + # group -- it is an instance-scope metric only, see banyandb-instance.yaml publish_batch_latency_p99.) + - name: queue_batch_throughput + exp: banyandb_queue_sub_total_batch_finished.sum(['cluster', 'group', 'operation']).rate('PT1M') + - name: queue_message_throughput + exp: banyandb_queue_sub_total_message_finished.sum(['cluster', 'group', 'operation']).rate('PT1M') + # publish bytes/s for the group. Wire family is banyandb_queue_pub_sent_bytes -- NO `total` # infix (unlike queue_pub_total_started/_finished). Liaison-only; sum collapses # operation/remote_node/remote_role/remote_tier before the rate. diff --git a/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-instance.yaml b/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-instance.yaml index 6604455b81..71a9e2b8f4 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-instance.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/banyandb/banyandb-instance.yaml @@ -100,6 +100,12 @@ metricsRules: exp: banyandb_queue_pub_sent_bytes.sum(['cluster','pod_name','container_name','node_role','node_type']).rate('PT15S') - name: publish_latency_p99 exp: banyandb_queue_pub_total_latency.sum(['cluster','pod_name','container_name','node_role','node_type','operation','le']).histogram().histogram_percentile([99]) + # tier-2 publish, batch granularity (BanyanDB #1169): batches published/s by operation and the batch + # send-latency p99. total_batch_latency uses a coarser bucket set than the per-operation total_latency. + - name: publish_batch_throughput + exp: banyandb_queue_pub_total_batch_finished.sum(['cluster','pod_name','container_name','node_role','node_type','operation']).rate('PT15S') + - name: publish_batch_latency_p99 + exp: banyandb_queue_pub_total_batch_latency.sum(['cluster','pod_name','container_name','node_role','node_type','operation','le']).histogram().histogram_percentile([99]) # write-queue (wqueue) depth: pending records, on-disk file parts, in-memory parts. On the liaison # these reflect the write buffer; the same families on data containers reflect storage parts (the # dashboard gates on container_name). Gauges, summed to the instance. @@ -145,6 +151,14 @@ metricsRules: exp: banyandb_queue_sub_total_finished.sum(['cluster','pod_name','container_name','node_role','node_type','operation']).rate('PT15S') - name: queue_sub_latency_p99 exp: banyandb_queue_sub_total_latency.sum(['cluster','pod_name','container_name','node_role','node_type','operation','le']).histogram().histogram_percentile([99]) + # subscribe-side per-message throughput (BanyanDB #1169). A data node ingests writes via the + # per-message dispatch path, which ticks only the message counter (sub_total_message_finished), + # NOT the batch counter -- the sub-side batch counters fire on the liaison's OAP-facing receive + # hop (handleEOF), not on the data node, so a per-node `queue_sub_batch_*` is empty on data and is + # intentionally not modeled here. Batch-level granularity lives on the liaison's publish side + # (publish_batch_throughput / publish_batch_latency_p99 above). + - name: queue_sub_message_throughput + exp: banyandb_queue_sub_total_message_finished.sum(['cluster','pod_name','container_name','node_role','node_type','operation']).rate('PT15S') # retention disk-usage % per data-model scope (0-100 gauge). Kept per scope rather than summed (a sum # of three percentages is meaningless). Not in the upstream Grafana boards; a SkyWalking addition. - name: retention_measure_disk_usage_percent diff --git a/test/e2e-v2/cases/banyandb/banyandb-cases.yaml b/test/e2e-v2/cases/banyandb/banyandb-cases.yaml index 83367a20c9..308ba6a4bb 100644 --- a/test/e2e-v2/cases/banyandb/banyandb-cases.yaml +++ b/test/e2e-v2/cases/banyandb/banyandb-cases.yaml @@ -54,6 +54,10 @@ cases: expected: expected/metrics-has-label-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_instance_queue_sub_throughput --service-name=e2e-banyandb --instance-name=banyandb-data-hot-0@data expected: expected/metrics-has-label-value.yml + # per-message throughput on the data node (BanyanDB #1169). The data node ingests via the per-message + # dispatch path, so the sub-side message counter ticks (batch counters fire on the liaison hop, not here). + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_instance_queue_sub_message_throughput --service-name=e2e-banyandb --instance-name=banyandb-data-hot-0@data + expected: expected/metrics-has-label-value.yml # liaison node (banyandb-liaison-0@liaison) - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_instance_node_uptime --service-name=e2e-banyandb --instance-name=banyandb-liaison-0@liaison expected: expected/metrics-has-value.yml @@ -61,6 +65,9 @@ cases: expected: expected/metrics-has-label-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_instance_publish_throughput --service-name=e2e-banyandb --instance-name=banyandb-liaison-0@liaison expected: expected/metrics-has-label-value.yml + # publish-side batch throughput on the liaison (BanyanDB #1169). + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_instance_publish_batch_throughput --service-name=e2e-banyandb --instance-name=banyandb-liaison-0@liaison + expected: expected/metrics-has-label-value.yml # ---- Endpoint scope (storage group sw_metricsMinute) ---- - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_endpoint_write_rate --service-name=e2e-banyandb --endpoint-name=sw_metricsMinute @@ -69,3 +76,8 @@ cases: expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_endpoint_queue_throughput --service-name=e2e-banyandb --endpoint-name=sw_metricsMinute expected: expected/metrics-has-label-value.yml + # batch + message granularity per group (BanyanDB #1169). + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_endpoint_queue_batch_throughput --service-name=e2e-banyandb --endpoint-name=sw_metricsMinute + expected: expected/metrics-has-label-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_banyandb_endpoint_queue_message_throughput --service-name=e2e-banyandb --endpoint-name=sw_metricsMinute + expected: expected/metrics-has-label-value.yml diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 5a87ea4991..8f9401faa2 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=f08776d909eb1d9bc79c600e493030651b97e491 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=da0e267f877b9b8e5f7728ae4ea7dc7723a2a073 SW_ROVER_COMMIT=79292fe07f17f98f486e0c4471213e1961fb2d1d -SW_BANYANDB_COMMIT=8a1936ce96653e89d3d13250a42abc6e3d42fae7 +SW_BANYANDB_COMMIT=c2d925e4eae4d77edda94e1fd438243483960150 SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994
