This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a39e06d652 [IOTDB-5225] Remove getAllMeasurement() method in
ISchemaTree (#8479)
a39e06d652 is described below
commit a39e06d6527125f2b7c3af712437686c68e62c16
Author: liuminghui233 <[email protected]>
AuthorDate: Sat Dec 17 09:02:10 2022 +0800
[IOTDB-5225] Remove getAllMeasurement() method in ISchemaTree (#8479)
[IOTDB-5225] Remove getAllMeasurement() method in ISchemaTree (#8479)
---
.../db/metadata/cache/DataNodeSchemaCache.java | 3 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 1 -
.../common/schematree/DeviceGroupSchemaTree.java | 5 -
.../db/mpp/common/schematree/ISchemaTree.java | 2 -
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 187 +++++++--------------
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 1 +
.../db/mpp/plan/statement/crud/QueryStatement.java | 15 +-
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 6 +-
8 files changed, 84 insertions(+), 136 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCache.java
b/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCache.java
index c5614e34eb..c8a0f68b0b 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCache.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCache.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.common.schematree.ClusterSchemaTree;
-import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -137,7 +136,7 @@ public class DataNodeSchemaCache {
return schemaTree;
}
- public void put(ISchemaTree schemaTree) {
+ public void put(ClusterSchemaTree schemaTree) {
for (MeasurementPath measurementPath : schemaTree.getAllMeasurement()) {
putSingleMeasurementPath(schemaTree.getBelongedDatabase(measurementPath),
measurementPath);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
index d1581176ca..b51247fcda 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
@@ -90,7 +90,6 @@ public class ClusterSchemaTree implements ISchemaTree {
return new Pair<>(visitor.getAllResult(), visitor.getNextOffset());
}
- @Override
public List<MeasurementPath> getAllMeasurement() {
return searchMeasurementPaths(ALL_MATCH_PATTERN, 0, 0, false).left;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
index a8a64f38f8..389c38f4d4 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/DeviceGroupSchemaTree.java
@@ -67,11 +67,6 @@ public class DeviceGroupSchemaTree implements ISchemaTree {
throw new UnsupportedOperationException();
}
- @Override
- public List<MeasurementPath> getAllMeasurement() {
- throw new UnsupportedOperationException();
- }
-
@Override
public List<DeviceSchemaInfo> getMatchedDevices(PartialPath pathPattern,
boolean isPrefixMatch) {
throw new UnsupportedOperationException();
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ISchemaTree.java
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ISchemaTree.java
index 010443a63a..61e9bf0226 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ISchemaTree.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ISchemaTree.java
@@ -39,8 +39,6 @@ public interface ISchemaTree {
Pair<List<MeasurementPath>, Integer> searchMeasurementPaths(PartialPath
pathPattern);
- List<MeasurementPath> getAllMeasurement();
-
/**
* Get all device matching the path pattern.
*
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index 81910ef650..8bc1b3fa45 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -144,6 +144,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TimeZone;
+import java.util.TreeSet;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkState;
@@ -229,7 +230,19 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
throw new SemanticException("Only time filters are supported in LAST
query");
}
analyzeOrderBy(analysis, queryStatement);
- return analyzeLast(analysis, schemaTree.getAllMeasurement(),
schemaTree);
+
+ List<Expression> selectExpressions = new ArrayList<>();
+ for (ResultColumn resultColumn :
queryStatement.getSelectComponent().getResultColumns()) {
+ selectExpressions.add(resultColumn.getExpression());
+ }
+ analyzeLastSource(analysis, selectExpressions, schemaTree);
+
+
analysis.setRespDatasetHeader(DatasetHeaderFactory.getLastQueryHeader());
+
+ // fetch partition information
+ analyzeDataPartition(analysis, queryStatement, schemaTree);
+
+ return analysis;
}
List<Pair<Expression, String>> outputExpressions;
@@ -253,11 +266,15 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
} else {
Map<Integer, List<Pair<Expression, String>>> outputExpressionMap =
analyzeSelect(analysis, queryStatement, schemaTree);
+
outputExpressions = new ArrayList<>();
outputExpressionMap.values().forEach(outputExpressions::addAll);
+
analyzeHaving(analysis, queryStatement, schemaTree);
+
analyzeGroupByLevel(analysis, queryStatement, outputExpressionMap,
outputExpressions);
- analyzeGroupByTag(analysis, queryStatement, outputExpressions,
schemaTree);
+ analyzeGroupByTag(analysis, queryStatement, outputExpressions);
+
Set<Expression> selectExpressions =
outputExpressions.stream()
.map(Pair::getLeft)
@@ -328,71 +345,32 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
analysis.setHasValueFilter(hasValueFilter);
}
- private Analysis analyzeLast(
- Analysis analysis, List<MeasurementPath> allSelectedPath, ISchemaTree
schemaTree) {
+ private void analyzeLastSource(
+ Analysis analysis, List<Expression> selectExpressions, ISchemaTree
schemaTree) {
Set<Expression> sourceExpressions;
- List<SortItem> sortItemList =
analysis.getMergeOrderParameter().getSortItemList();
- if (sortItemList.size() > 0) {
+
+ OrderByParameter orderByParameter = analysis.getMergeOrderParameter();
+ if (orderByParameter != null && orderByParameter.getSortItemList().size()
> 0) {
+ List<SortItem> sortItemList = orderByParameter.getSortItemList();
checkState(
sortItemList.size() == 1 && sortItemList.get(0).getSortKey() ==
SortKey.TIMESERIES,
"Last queries only support sorting by timeseries now.");
boolean isAscending = sortItemList.get(0).getOrdering() == Ordering.ASC;
sourceExpressions =
- allSelectedPath.stream()
- .map(TimeSeriesOperand::new)
- .sorted(
- (o1, o2) ->
- isAscending
- ?
o1.getExpressionString().compareTo(o2.getExpressionString())
- :
o2.getExpressionString().compareTo(o1.getExpressionString()))
- .collect(Collectors.toCollection(LinkedHashSet::new));
+ new TreeSet<>(
+ (e1, e2) ->
+ isAscending
+ ? e1.toString().compareTo(e2.toString())
+ : e2.toString().compareTo(e1.toString()));
} else {
- sourceExpressions =
- allSelectedPath.stream()
- .map(TimeSeriesOperand::new)
- .collect(Collectors.toCollection(LinkedHashSet::new));
+ sourceExpressions = new LinkedHashSet<>();
}
- analysis.setSourceExpressions(sourceExpressions);
-
- analysis.setRespDatasetHeader(DatasetHeaderFactory.getLastQueryHeader());
-
- Set<String> deviceSet =
-
allSelectedPath.stream().map(MeasurementPath::getDevice).collect(Collectors.toSet());
-
- Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(analysis.getGlobalTimeFilter());
-
- DataPartition dataPartition;
-
- // there is no satisfied time range
- if (res.left.isEmpty() && !res.right.left) {
- dataPartition =
- new DataPartition(
- Collections.emptyMap(),
- CONFIG.getSeriesPartitionExecutorClass(),
- CONFIG.getSeriesPartitionSlotNum());
- } else {
- Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new
HashMap<>();
- for (String devicePath : deviceSet) {
- DataPartitionQueryParam queryParam =
- new DataPartitionQueryParam(devicePath, res.left, res.right.left,
res.right.right);
- sgNameToQueryParamsMap
- .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key
-> new ArrayList<>())
- .add(queryParam);
- }
-
- if (res.right.left || res.right.right) {
- dataPartition =
-
partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap);
- } else {
- dataPartition =
partitionFetcher.getDataPartition(sgNameToQueryParamsMap);
- }
+ for (Expression selectExpression : selectExpressions) {
+ sourceExpressions.addAll(
+ ExpressionAnalyzer.removeWildcardInExpression(selectExpression,
schemaTree));
}
-
- analysis.setDataPartitionInfo(dataPartition);
-
- return analysis;
+ analysis.setSourceExpressions(sourceExpressions);
}
private Map<Integer, List<Pair<Expression, String>>> analyzeSelect(
@@ -756,64 +734,43 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
private void analyzeGroupByTag(
Analysis analysis,
QueryStatement queryStatement,
- List<Pair<Expression, String>> outputExpressions,
- ISchemaTree schemaTree) {
+ List<Pair<Expression, String>> outputExpressions) {
if (!queryStatement.isGroupByTag()) {
return;
}
if (analysis.hasValueFilter()) {
throw new SemanticException("Only time filters are supported in GROUP BY
TAGS query");
}
- if (queryStatement.hasHaving()) {
- throw new SemanticException("Having clause is not supported yet in GROUP
BY TAGS query");
- }
+
+ List<String> tagKeys =
queryStatement.getGroupByTagComponent().getTagKeys();
Map<List<String>, LinkedHashMap<Expression, List<Expression>>>
tagValuesToGroupedTimeseriesOperands = new HashMap<>();
- LinkedHashMap<Expression, Set<Expression>> groupByTagOutputExpressions =
new LinkedHashMap<>();
- List<String> tagKeys =
queryStatement.getGroupByTagComponent().getTagKeys();
- List<MeasurementPath> allSelectedPath = schemaTree.getAllMeasurement();
- Map<MeasurementPath, Map<String, String>> queriedTagMap = new HashMap<>();
- allSelectedPath.forEach(v -> queriedTagMap.put(v, v.getTagMap()));
+ LinkedHashMap<Expression, Set<Expression>>
outputExpressionToRawExpressionsMap =
+ new LinkedHashMap<>();
for (Pair<Expression, String> outputExpressionAndAlias :
outputExpressions) {
- if (!(outputExpressionAndAlias.getLeft() instanceof FunctionExpression
- && outputExpressionAndAlias.getLeft().getExpressions().get(0)
instanceof TimeSeriesOperand
- &&
outputExpressionAndAlias.getLeft().isBuiltInAggregationFunctionExpression())) {
- throw new SemanticException(
- outputExpressionAndAlias.getLeft()
- + " can't be used in group by tag. It will be supported in the
future.");
- }
- FunctionExpression outputExpression = (FunctionExpression)
outputExpressionAndAlias.getLeft();
- MeasurementPath measurementPath =
- (MeasurementPath)
- ((TimeSeriesOperand)
outputExpression.getExpressions().get(0)).getPath();
- MeasurementPath fakePath = null;
- try {
- fakePath =
- new MeasurementPath(measurementPath.getMeasurement(),
measurementPath.getSeriesType());
- } catch (IllegalPathException e) {
- // do nothing
- }
- Expression measurementExpression = new TimeSeriesOperand(fakePath);
- Expression groupedExpression =
- new FunctionExpression(
- outputExpression.getFunctionName(),
- outputExpression.getFunctionAttributes(),
- Collections.singletonList(measurementExpression));
- groupByTagOutputExpressions
- .computeIfAbsent(groupedExpression, v -> new HashSet<>())
- .add(outputExpression);
- Map<String, String> tagMap = queriedTagMap.get(measurementPath);
+ FunctionExpression rawExpression = (FunctionExpression)
outputExpressionAndAlias.getLeft();
+ FunctionExpression measurementExpression =
+ (FunctionExpression)
ExpressionAnalyzer.getMeasurementExpression(rawExpression);
+ outputExpressionToRawExpressionsMap
+ .computeIfAbsent(measurementExpression, v -> new HashSet<>())
+ .add(rawExpression);
+
+ Map<String, String> tagMap =
+ ((MeasurementPath)
+ ((TimeSeriesOperand)
measurementExpression.getExpressions().get(0)).getPath())
+ .getTagMap();
List<String> tagValues = new ArrayList<>();
for (String tagKey : tagKeys) {
tagValues.add(tagMap.get(tagKey));
}
tagValuesToGroupedTimeseriesOperands
.computeIfAbsent(tagValues, key -> new LinkedHashMap<>())
- .computeIfAbsent(groupedExpression, key -> new ArrayList<>())
- .add(outputExpression.getExpressions().get(0));
+ .computeIfAbsent(measurementExpression, key -> new ArrayList<>())
+ .add(rawExpression.getExpressions().get(0));
}
+ // update outputExpressions
outputExpressions.clear();
for (String tagKey : tagKeys) {
Expression tagKeyExpression =
@@ -821,14 +778,14 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
analyzeExpression(analysis, tagKeyExpression);
outputExpressions.add(new Pair<>(tagKeyExpression, null));
}
- for (Expression groupByTagOutputExpression :
groupByTagOutputExpressions.keySet()) {
+ for (Expression outputExpression :
outputExpressionToRawExpressionsMap.keySet()) {
// TODO: support alias
- analyzeExpression(analysis, groupByTagOutputExpression);
- outputExpressions.add(new Pair<>(groupByTagOutputExpression, null));
+ analyzeExpression(analysis, outputExpression);
+ outputExpressions.add(new Pair<>(outputExpression, null));
}
analysis.setTagKeys(queryStatement.getGroupByTagComponent().getTagKeys());
analysis.setTagValuesToGroupedTimeseriesOperands(tagValuesToGroupedTimeseriesOperands);
- analysis.setCrossGroupByExpressions(groupByTagOutputExpressions);
+ analysis.setCrossGroupByExpressions(outputExpressionToRawExpressionsMap);
}
private void analyzeDeviceToAggregation(
@@ -2099,27 +2056,13 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
logger.debug("[StartFetchSchema]");
ISchemaTree schemaTree = schemaFetcher.fetchSchema(patternTree);
logger.debug("[EndFetchSchema]]");
- List<MeasurementPath> allSelectedPath = schemaTree.getAllMeasurement();
-
- Set<Expression> sourceExpressions =
- allSelectedPath.stream()
- .map(TimeSeriesOperand::new)
- .collect(Collectors.toCollection(LinkedHashSet::new));
- analysis.setSourceExpressions(sourceExpressions);
- sourceExpressions.forEach(expression -> analyzeExpression(analysis,
expression));
-
- Set<String> deviceSet =
-
allSelectedPath.stream().map(MeasurementPath::getDevice).collect(Collectors.toSet());
- Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new
HashMap<>();
- for (String devicePath : deviceSet) {
- DataPartitionQueryParam queryParam = new DataPartitionQueryParam();
- queryParam.setDevicePath(devicePath);
- sgNameToQueryParamsMap
- .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key
-> new ArrayList<>())
- .add(queryParam);
- }
- DataPartition dataPartition =
partitionFetcher.getDataPartition(sgNameToQueryParamsMap);
- analysis.setDataPartitionInfo(dataPartition);
+
+ analyzeLastSource(
+ analysis,
+ Collections.singletonList(
+ new TimeSeriesOperand(showTimeSeriesStatement.getPathPattern())),
+ schemaTree);
+ analyzeDataPartition(analysis, new QueryStatement(), schemaTree);
}
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader());
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index d0be42611a..cd8143ea08 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -1185,6 +1185,7 @@ public class ExpressionAnalyzer {
if (rawPath.isMeasurementAliasExists()) {
measurementWithSchema.setMeasurementAlias(rawPath.getMeasurementAlias());
}
+ measurementWithSchema.setTagMap(rawPath.getTagMap());
return new TimeSeriesOperand(measurementWithSchema);
} else if (expression instanceof LeafOperand) {
return expression;
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
index f8c84f38e5..a1e164c7d8 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
@@ -25,6 +25,7 @@ import
org.apache.iotdb.db.mpp.plan.analyze.ExpressionAnalyzer;
import org.apache.iotdb.db.mpp.plan.constant.StatementType;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
+import org.apache.iotdb.db.mpp.plan.expression.multi.FunctionExpression;
import org.apache.iotdb.db.mpp.plan.statement.Statement;
import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor;
import org.apache.iotdb.db.mpp.plan.statement.component.FillComponent;
@@ -355,6 +356,9 @@ public class QueryStatement extends Statement {
: resultColumn.getExpression().getExpressionString());
}
if (isGroupByTag()) {
+ if (hasHaving()) {
+ throw new SemanticException("Having clause is not supported yet in
GROUP BY TAGS query");
+ }
for (String s : getGroupByTagComponent().getTagKeys()) {
if (outputColumn.contains(s)) {
throw new SemanticException("Output column is duplicated with the
tag key: " + s);
@@ -363,6 +367,15 @@ public class QueryStatement extends Statement {
if (rowLimit > 0 || rowOffset > 0 || seriesLimit > 0 || seriesOffset >
0) {
throw new SemanticException("Limit or slimit are not supported yet
in GROUP BY TAGS");
}
+ for (ResultColumn resultColumn : selectComponent.getResultColumns()) {
+ Expression expression = resultColumn.getExpression();
+ if (!(expression instanceof FunctionExpression
+ && expression.getExpressions().get(0) instanceof
TimeSeriesOperand
+ && expression.isBuiltInAggregationFunctionExpression())) {
+ throw new SemanticException(
+ expression + " can't be used in group by tag. It will be
supported in the future.");
+ }
+ }
}
} else {
if (isGroupByTime() || isGroupByLevel()) {
@@ -371,7 +384,7 @@ public class QueryStatement extends Statement {
}
}
- if (getHavingCondition() != null) {
+ if (hasHaving()) {
Expression havingExpression = getHavingCondition().getPredicate();
if (ExpressionAnalyzer.identifyOutputColumnType(havingExpression, true)
!= ResultColumn.ColumnType.AGGREGATION) {
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java
b/server/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java
index bee7c16e6b..88148499a6 100644
---
a/server/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java
@@ -62,7 +62,7 @@ public class DataNodeSchemaCacheTest {
measurements[1] = "s2";
measurements[2] = "s3";
- dataNodeSchemaCache.put(generateSchemaTree1());
+ dataNodeSchemaCache.put((ClusterSchemaTree) generateSchemaTree1());
Map<PartialPath, SchemaCacheEntry> schemaCacheEntryMap =
dataNodeSchemaCache.get(device1,
measurements).getAllMeasurement().stream()
@@ -95,7 +95,7 @@ public class DataNodeSchemaCacheTest {
otherMeasurements[1] = "s4";
otherMeasurements[2] = "s5";
- dataNodeSchemaCache.put(generateSchemaTree2());
+ dataNodeSchemaCache.put((ClusterSchemaTree) generateSchemaTree2());
schemaCacheEntryMap =
dataNodeSchemaCache.get(device1,
otherMeasurements).getAllMeasurement().stream()
@@ -133,7 +133,7 @@ public class DataNodeSchemaCacheTest {
Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2));
Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3));
// test no last cache
- dataNodeSchemaCache.put(generateSchemaTree1());
+ dataNodeSchemaCache.put((ClusterSchemaTree) generateSchemaTree1());
Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath1));
Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2));
Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3));
