This is an automated email from the ASF dual-hosted git repository. hanahmily pushed a commit to branch query-labeled in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e35868394e1d2caee27051e1a629115f6c808734 Author: Gao Hongtao <[email protected]> AuthorDate: Wed Jul 29 22:42:06 2020 +0800 Query all of labels once selected labels condition is absent. --- .../server/core/analysis/metrics/DataTable.java | 5 + .../core/storage/query/IMetricsQueryDAO.java | 51 +++++++++ .../core/storage/query/MetricsQueryUtilTest.java | 114 +++++++++++++++++++++ .../elasticsearch/query/MetricsQueryEsDAO.java | 41 ++------ .../plugin/influxdb/query/MetricsQuery.java | 30 +----- .../plugin/jdbc/h2/dao/H2MetricsQueryDAO.java | 30 +----- 6 files changed, 182 insertions(+), 89 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java index a104ee6..2fca2da 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -94,6 +95,10 @@ public class DataTable implements StorageDataComplexObject<DataTable> { return values; } + public Set<String> keys() { + return data.keySet(); + } + public boolean hasData() { return !data.isEmpty(); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java index 79be899..803c6bb 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java @@ -18,8 +18,14 @@ package org.apache.skywalking.oap.server.core.storage.query; +import io.vavr.Tuple; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; import org.apache.skywalking.oap.server.core.query.type.HeatMap; @@ -27,6 +33,11 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues; import org.apache.skywalking.oap.server.core.query.type.KVInt; import org.apache.skywalking.oap.server.core.query.type.MetricsValues; import org.apache.skywalking.oap.server.core.storage.DAO; +import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; /** * Query metrics values in different ways. @@ -76,5 +87,45 @@ public interface IMetricsQueryDAO extends DAO { } return origin; } + + /** + * Compose the multiple metric result based on conditions. + */ + public static List<MetricsValues> composeLabelValue(final MetricsCondition condition, + final List<String> labels, + final List<String> ids, + final Map<String, DataTable> idMap) { + List<String> allLabels; + if (Objects.isNull(labels) || labels.size() < 1) { + allLabels = idMap.values().stream() + .flatMap(dataTable -> dataTable.keys().stream()) + .distinct().collect(Collectors.toList()); + } else { + allLabels = labels; + } + final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); + return allLabels.stream() + .flatMap(label -> ids.stream() + .map(id -> Tuple.of( + label, + id, + Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse(0L)))) + .collect(groupingBy(t -> t._1, mapping(t -> { + KVInt kv = new KVInt(); + kv.setId(t._2); + kv.setValue(t._3); + return kv; + }, toList()))) + .entrySet().stream() + .map(entry -> { + MetricsValues labelValue = new MetricsValues(); + labelValue.setLabel(entry.getKey()); + IntValues values = new IntValues(); + entry.getValue().forEach(values::addKVInt); + labelValue.setValues(sortValues(values, ids, defaultValue)); + return labelValue; + }) + .collect(toList()); + } } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java new file mode 100644 index 0000000..3283e1e --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.query; + +import com.google.gson.Gson; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; +import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; +import org.apache.skywalking.oap.server.core.query.sql.Function; +import org.apache.skywalking.oap.server.core.query.type.MetricsValues; +import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static com.google.common.collect.ImmutableMap.of; +import static java.util.Arrays.asList; +import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +@RunWith(Parameterized.class) +@RequiredArgsConstructor +public class MetricsQueryUtilTest { + + private static final String MODULE_NAME = "meter-test"; + + private final List<String> queryConditionLabels; + + private final List<String> datePoints; + + private final Map<String, DataTable> valueColumnData; + + private final String expectedResult; + + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + { + asList("200", "400"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]" + }, + { + Collections.emptyList(), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]" + }, + { + Collections.singletonList("200"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]" + }, + { + asList("200", "400", "500"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," + + "{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":0},{\"id\":\"202007291426\",\"value\":0}]}}]" + }, + { + asList("200", "400"), + asList("202007291425", "202007291426"), + of("202007291425", new DataTable("200,1|400,2")), + "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":0}]}}," + + "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":0}]}}]" + }, + }); + } + + @Before + public void setup() { + ValueColumnMetadata.INSTANCE.putIfAbsent( + MODULE_NAME, "value", LABELED_VALUE, Function.None, 0 + ); + } + + @Test + public void testComposeLabelValue() { + MetricsCondition condition = new MetricsCondition(); + condition.setName(MODULE_NAME); + List<MetricsValues> result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData); + assertThat(new Gson().toJson(result), is(expectedResult)); + } + +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java index c49ca9a..5145de5 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; @@ -138,42 +139,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { }); SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0])); - Map<String, Map<String, Object>> idMap = toMap(response); - - Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); - for (String id : ids) { - if (idMap.containsKey(id)) { - Map<String, Object> source = idMap.get(id); - DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, "")); - - labels.forEach(label -> { - final IntValues values = labeledValues.get(label).getValues(); - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - values.addKVInt(kv); - }); - } - + Map<String, DataTable> idMap = new LinkedHashMap<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, ""))); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java index dc8842a..f250d4f 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO { log.debug("SQL: {} result set: {}", query.getCommand(), series); } - Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); + Map<String, DataTable> idMap = new HashMap<>(); if (!CollectionUtils.isEmpty(series)) { series.get(0).getValues().forEach(values -> { final String id = (String) values.get(1); DataTable multipleValues = new DataTable(5); multipleValues.toObject((String) values.get(2)); - - labels.forEach(label -> { - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - final IntValues intValues = labeledValues.get(label).getValues(); - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - intValues.addKVInt(kv); - }); + idMap.put(id, multipleValues); }); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java index 84d0ca0..f2b0f32 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java @@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO } sql.append(")"); - Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size()); - labels.forEach(label -> { - MetricsValues labelValue = new MetricsValues(); - labelValue.setLabel(label); - - labeledValues.put(label, labelValue); - }); - - final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName()); - + Map<String, DataTable> idMap = new HashMap<>(); try (Connection connection = h2Client.getConnection()) { try (ResultSet resultSet = h2Client.executeQuery( connection, sql.toString(), parameters.toArray(new Object[0]))) { @@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO DataTable multipleValues = new DataTable(5); multipleValues.toObject(resultSet.getString(valueColumnName)); - labels.forEach(label -> { - Long data = multipleValues.get(label); - if (data == null) { - data = (long) defaultValue; - } - final IntValues values = labeledValues.get(label).getValues(); - KVInt kv = new KVInt(); - kv.setId(id); - kv.setValue(data); - values.addKVInt(kv); - }); + idMap.put(id, multipleValues); } } } catch (SQLException e) { throw new IOException(e); } - - return Util.sortValues( - new ArrayList<>(labeledValues.values()), - ids, - defaultValue - ); + return Util.composeLabelValue(condition, labels, ids, idMap); } @Override
