This is an automated email from the ASF dual-hosted git repository.
wankai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 0a0890c148 MQE: Support `top_n_of` function for merging multiple
metrics topn query. (#12937)
0a0890c148 is described below
commit 0a0890c148e3139df33142c2270db4d098570872
Author: Wan Kai <[email protected]>
AuthorDate: Wed Jan 8 12:53:34 2025 +0800
MQE: Support `top_n_of` function for merging multiple metrics topn query.
(#12937)
---
docs/en/api/metrics-query-expression.md | 20 ++++++
docs/en/changes/changes.md | 1 +
.../apache/skywalking/mqe/rt/grammar/MQELexer.g4 | 1 +
.../apache/skywalking/mqe/rt/grammar/MQEParser.g4 | 6 +-
.../apache/skywalking/mqe/rt/MQEVisitorBase.java | 23 ++++++-
.../skywalking/mqe/rt/operation/TopNOfOp.java | 71 ++++++++++++++++++++++
.../org/apache/skywalking/mqe/rt/TopNOfOpTest.java | 59 ++++++++++++++++++
.../oap/query/graphql/mqe/rt/MQEVisitor.java | 8 +--
test/e2e-v2/cases/mqe/mqe-cases.yaml | 5 +-
9 files changed, 186 insertions(+), 8 deletions(-)
diff --git a/docs/en/api/metrics-query-expression.md
b/docs/en/api/metrics-query-expression.md
index 28e28f367d..1446368fa7 100644
--- a/docs/en/api/metrics-query-expression.md
+++ b/docs/en/api/metrics-query-expression.md
@@ -217,6 +217,7 @@ round(service_cpm / 60 , 2)
The different operators could impact the `ExpressionResultType`, please refer
to the above table.
## TopN Operation
+### TopN Query
TopN Operation takes an expression and performs calculation to get the TopN of
Services/Instances/Endpoints.
The result depends on the `entity` condition in the query.
- Global TopN:
@@ -264,6 +265,25 @@ top_n(service_instance_cpm, 10, des)
### Result Type
According to the type of the metric, the `ExpressionResultType` of the
expression will be `SORTED_LIST` or `RECORD_LIST`.
+### Multiple TopNs Merging
+As the difference between agent and ebpf, some metrics would be separated,
e.g. service cpm and k8s service cpm.
+If you want to merge the topN results of these metrics, you can use the
`ton_n_of` operation to merge the results.
+
+expression:
+```text
+ton_n_of(<top_n>, <top_n>, ...,<top_number>, <order>)
+```
+
+- `<top_n>` is the [topN](#topn-query) expression. The result type of those
tonN expression should be same, can be `SORTED_LIST` or `RECORD_LIST`, `but can
not be mixed`.
+- `<top_number>` is the number of the merged top results, should be a positive
integer.
+- `<order>` is the order of the merged top results. The value of `<order>` can
be `asc` or `des`.
+
+for example:
+If we want to get the top 10 services with the highest `service_cpm` and
`k8s_service_cpm`, we can use the following expression:
+```text
+ton_n_of(top_n(service_cpm, 10, des), top_n(k8s_service_cpm, 10, des), 10, des)
+```
+
## Relabel Operation
Relabel Operation takes an expression and replaces the label with new label on
its results.
Since v10.0.0, SkyWalking supports relabel multiple labels.
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 3001b77ee1..ea5ee772b0 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -53,6 +53,7 @@
* Add `bydb.dependencies.properties` config file to define server dependency
versions.
* Fix `AvgHistogramPercentileFunction` doesn't have proper field definition
for `ranks`.
* BanyanDB: Support the new Property data module.
+* MQE: Support `top_n_of` function for merging multiple metrics topn query.
#### UI
diff --git
a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQELexer.g4
b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQELexer.g4
index 0ea98355b1..dbea0ce0bf 100644
---
a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQELexer.g4
+++
b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQELexer.g4
@@ -64,6 +64,7 @@ RATE: 'rate';
// TopN
TOP_N: 'top_n';
+TOP_N_OF: 'top_n_of';
// ViewAsSeq
VIEW_AS_SEQ: 'view_as_seq';
diff --git
a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4
b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4
index d85edcde40..d1a23a2bd5 100644
---
a/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4
+++
b/oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4
@@ -33,7 +33,8 @@ expression
| mathematical_operator1 L_PAREN expression COMMA parameter R_PAREN
#mathematicalOperator1OP
| trend L_PAREN metric COMMA INTEGER R_PAREN #trendOP
| logical_operator L_PAREN expressionList R_PAREN #logicalOperatorOP
- | topN L_PAREN metric COMMA INTEGER COMMA order (COMMA attributeList)?
R_PAREN #topNOP
+ | topN
#topNOP
+ | topNOf L_PAREN topN (COMMA topN)* COMMA INTEGER COMMA order R_PAREN
#topNOfOP
| relabels L_PAREN expression COMMA label COMMA replaceLabel R_PAREN
#relablesOP
| aggregateLabels L_PAREN expression COMMA aggregateLabelsFunc R_PAREN
#aggregateLabelsOp
| sort_values L_PAREN expression (COMMA INTEGER)? COMMA order R_PAREN
#sortValuesOP
@@ -75,7 +76,8 @@ mathematical_operator1:
trend:
INCREASE | RATE;
-topN: TOP_N;
+topN: TOP_N L_PAREN metric COMMA INTEGER COMMA order (COMMA attributeList)?
R_PAREN;
+topNOf: TOP_N_OF;
logical_operator:
VIEW_AS_SEQ | IS_PRESENT;
diff --git
a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
index fcc41dc194..d15cb11943 100644
---
a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
+++
b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
@@ -34,6 +34,7 @@ import
org.apache.skywalking.mqe.rt.operation.LogicalFunctionOp;
import org.apache.skywalking.mqe.rt.operation.MathematicalFunctionOp;
import org.apache.skywalking.mqe.rt.operation.SortLabelValuesOp;
import org.apache.skywalking.mqe.rt.operation.SortValuesOp;
+import org.apache.skywalking.mqe.rt.operation.TopNOfOp;
import org.apache.skywalking.mqe.rt.operation.TrendOp;
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
@@ -240,7 +241,27 @@ public abstract class MQEVisitorBase extends
MQEParserBaseVisitor<ExpressionResu
DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
DebuggingSpan span = traceContext.createSpan("MQE TopN OP: " +
ctx.getText());
try {
- return visit(ctx.metric());
+ return visit(ctx.topN().metric());
+ } finally {
+ traceContext.stopSpan(span);
+ }
+ }
+
+ @Override
+ public ExpressionResult visitTopNOfOP(MQEParser.TopNOfOPContext ctx) {
+ DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
+ DebuggingSpan span = traceContext.createSpan("MQE TopNOf OP: " +
ctx.getText());
+ try {
+ List<MQEParser.TopNContext> topNContexts = ctx.topN();
+ List<ExpressionResult> topNResults = new ArrayList<>();
+ for (MQEParser.TopNContext topNContext : topNContexts) {
+ topNResults.add(visit(topNContext.metric()));
+ }
+ try {
+ return TopNOfOp.doMergeTopNResult(topNResults,
Integer.parseInt(ctx.INTEGER().getText()), ctx.order().getStart().getType());
+ } catch (IllegalExpressionException e) {
+ return getErrorResult(e.getMessage());
+ }
} finally {
traceContext.stopSpan(span);
}
diff --git
a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/TopNOfOp.java
b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/TopNOfOp.java
new file mode 100644
index 0000000000..478e6cbdae
--- /dev/null
+++
b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/TopNOfOp.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.mqe.rt.operation;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
+import org.apache.skywalking.mqe.rt.grammar.MQEParser;
+import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
+import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResultType;
+import org.apache.skywalking.oap.server.core.query.mqe.MQEValue;
+import org.apache.skywalking.oap.server.core.query.mqe.MQEValues;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
+
+public class TopNOfOp {
+ public static ExpressionResult doMergeTopNResult(List<ExpressionResult>
topNResults,
+ int limit,
+ int order) throws
IllegalExpressionException {
+ ExpressionResultType type = null;
+ List<MQEValue> allValues = new ArrayList<>();
+ for (ExpressionResult topNResult : topNResults) {
+ if (StringUtil.isNotEmpty(topNResult.getError())) {
+ return topNResult;
+ }
+ // check the type of topNResults
+ if (type != null && type != topNResult.getType()) {
+ throw new IllegalExpressionException("TopN type is not
consistent, one is " + type + ", another is " +
+
topNResult.getType());
+ }
+ type = topNResult.getType();
+ // topN result should have values without label
+ allValues.addAll(topNResult.getResults().get(0).getValues());
+ }
+ if (limit > allValues.size()) {
+ limit = allValues.size();
+ }
+ List<MQEValue> mergedValues = allValues.stream()
+ // Filter out empty values
+ .filter(mqeValue ->
!mqeValue.isEmptyValue())
+ .sorted(MQEParser.ASC == order
? Comparator.comparingDouble(
+ MQEValue::getDoubleValue) :
+
Comparator.comparingDouble(MQEValue::getDoubleValue)
+
.reversed())
+
.limit(limit).collect(Collectors.toList());
+
+ ExpressionResult result = new ExpressionResult();
+ MQEValues mqeValues = new MQEValues();
+ mqeValues.setValues(mergedValues);
+ result.getResults().add(mqeValues);
+ result.setType(type);
+ return result;
+ }
+}
diff --git
a/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/TopNOfOpTest.java
b/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/TopNOfOpTest.java
new file mode 100644
index 0000000000..ec5c77fe05
--- /dev/null
+++
b/oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/TopNOfOpTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.mqe.rt;
+
+import java.util.List;
+import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
+import org.apache.skywalking.mqe.rt.grammar.MQEParser;
+import org.apache.skywalking.mqe.rt.operation.TopNOfOp;
+import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
+import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResultType;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TopNOfOpTest {
+
+ @Test
+ public void mergeTopNResultTest() throws IllegalExpressionException {
+ MockData mockData = new MockData();
+ List<ExpressionResult> topNResults = List.of(
+ mockData.newListResult(1000, 100),
+ mockData.newListResult(600, 500),
+ mockData.newListResult(300, 2000)
+ );
+ ExpressionResult topNResult = TopNOfOp.doMergeTopNResult(topNResults,
2, MQEParser.DES);
+ Assertions.assertEquals(ExpressionResultType.SORTED_LIST,
topNResult.getType());
+ Assertions.assertEquals(2,
topNResult.getResults().get(0).getValues().size());
+ Assertions.assertEquals(2000,
topNResult.getResults().get(0).getValues().get(0).getDoubleValue());
+ Assertions.assertEquals("service_B",
topNResult.getResults().get(0).getValues().get(0).getId());
+ Assertions.assertEquals(1000,
topNResult.getResults().get(0).getValues().get(1).getDoubleValue());
+ Assertions.assertEquals("service_A",
topNResult.getResults().get(0).getValues().get(1).getId());
+
+ ExpressionResult topNResultAsc =
TopNOfOp.doMergeTopNResult(topNResults, 8, MQEParser.ASC);
+ Assertions.assertEquals(6,
topNResultAsc.getResults().get(0).getValues().size());
+ Assertions.assertEquals(100,
topNResultAsc.getResults().get(0).getValues().get(0).getDoubleValue());
+ Assertions.assertEquals("service_B",
topNResultAsc.getResults().get(0).getValues().get(0).getId());
+ Assertions.assertEquals(2000,
topNResultAsc.getResults().get(0).getValues().get(5).getDoubleValue());
+ Assertions.assertEquals("service_B",
topNResultAsc.getResults().get(0).getValues().get(5).getId());
+ topNResults.get(2).setType(ExpressionResultType.RECORD_LIST);
+ Assertions.assertThrows(IllegalExpressionException.class, () -> {
+ TopNOfOp.doMergeTopNResult(topNResults, 2, MQEParser.DES);
+ });
+ }
+}
diff --git
a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java
b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java
index ed7290e905..0d09ecf0e1 100644
---
a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java
+++
b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java
@@ -124,8 +124,8 @@ public class MQEVisitor extends MQEVisitorBase {
Column.ValueDataType dataType = valueColumn.get().getDataType();
try {
if (Column.ValueDataType.COMMON_VALUE == dataType) {
- if (ctx.parent instanceof MQEParser.TopNOPContext) {
- MQEParser.TopNOPContext parent =
(MQEParser.TopNOPContext) ctx.parent;
+ if (ctx.parent instanceof MQEParser.TopNContext) {
+ MQEParser.TopNContext parent = (MQEParser.TopNContext)
ctx.parent;
int topN =
Integer.parseInt(parent.INTEGER().getText());
if (topN <= 0) {
throw new IllegalExpressionException("TopN value
must be > 0.");
@@ -170,8 +170,8 @@ public class MQEVisitor extends MQEVisitorBase {
queryLabeledMetrics(metricName, queryLabels,
this.duration, result);
}
} else if (Column.ValueDataType.SAMPLED_RECORD == dataType) {
- if (ctx.parent instanceof MQEParser.TopNOPContext) {
- MQEParser.TopNOPContext parent =
(MQEParser.TopNOPContext) ctx.parent;
+ if (ctx.parent instanceof MQEParser.TopNContext) {
+ MQEParser.TopNContext parent = (MQEParser.TopNContext)
ctx.parent;
int topN =
Integer.parseInt(parent.INTEGER().getText());
if (topN <= 0) {
throw new IllegalExpressionException("TopN value
must be > 0.");
diff --git a/test/e2e-v2/cases/mqe/mqe-cases.yaml
b/test/e2e-v2/cases/mqe/mqe-cases.yaml
index 0805f2f34b..53a231ad50 100644
--- a/test/e2e-v2/cases/mqe/mqe-cases.yaml
+++ b/test/e2e-v2/cases/mqe/mqe-cases.yaml
@@ -45,10 +45,13 @@ cases:
- query: swctl --display yaml
--base-url=http://${oap_host}:${oap_12800}/graphql metrics exec
--expression="top_n(service_resp_time,3,des,attr0!='Not_GENERAL')/100"
expected: expected/topN-OP-service.yml
- # topN-OP-isntance
+ # topN-OP-instance
- query: swctl --display yaml
--base-url=http://${oap_host}:${oap_12800}/graphql metrics exec
--expression="top_n(service_instance_resp_time,3,des)/100"
--service-name=e2e-service-provider
expected: expected/topN-OP-instance.yml
+ # topN-Of-OP
+ - query: swctl --display yaml
--base-url=http://${oap_host}:${oap_12800}/graphql metrics exec
--expression="top_n_of(top_n(service_resp_time,3,des,attr0='GENERAL'),
top_n(service_cpm,3,des,attr0='GENERAL'), 2, des)"
+ expected: expected/topN-OP-service.yml
# select labels and relabels
- query: swctl --display yaml
--base-url=http://${oap_host}:${oap_12800}/graphql metrics exec
--expression="relabels(relabels(service_percentile{p='50,75,90'},p='50,75',p='P50,P75'),p='90',p='P90')"
--service-name=e2e-service-provider
expected: expected/relabels-OP.yml
