This is an automated email from the ASF dual-hosted git repository. xiangweiwei pushed a commit to branch new_vector_query in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bb16ca9353a4585f0159c2ea107e0d2ec74b1b71 Author: Alima777 <[email protected]> AuthorDate: Tue Nov 2 10:52:20 2021 +0800 query physical plan first version --- .../org/apache/iotdb/db/metadata/MManager.java | 4 ++- .../org/apache/iotdb/db/metadata/mtree/MTree.java | 12 +++++++-- .../iotdb/db/qp/logical/crud/QueryOperator.java | 15 ++++++----- .../iotdb/db/qp/physical/crud/QueryPlan.java | 4 +-- .../db/qp/physical/crud/RawDataQueryPlan.java | 29 +--------------------- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 3 +-- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 20 ++++++++------- .../apache/iotdb/db/qp/utils/WildcardsRemover.java | 6 +++-- .../query/expression/unary/TimeSeriesOperand.java | 3 ++- .../org/apache/iotdb/db/service/TSServiceImpl.java | 6 ++--- .../org/apache/iotdb/db/utils/SchemaUtils.java | 5 +--- 11 files changed, 46 insertions(+), 61 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 7e90243..760aa66 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; import org.apache.iotdb.db.metadata.mtree.MTree; import org.apache.iotdb.db.metadata.path.AlignedPath; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.metadata.tag.TagManager; import org.apache.iotdb.db.metadata.template.Template; @@ -1070,7 +1071,8 @@ public class MManager { * * @param pathPattern can be a pattern or a full path of timeseries. */ - public List<PartialPath> getMeasurementPaths(PartialPath pathPattern) throws MetadataException { + public List<MeasurementPath> getMeasurementPaths(PartialPath pathPattern) + throws MetadataException { return mtree.getMeasurementPaths(pathPattern); } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java index c29502f..513cf0b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java @@ -35,7 +35,14 @@ import org.apache.iotdb.db.metadata.MManager.StorageGroupFilter; import org.apache.iotdb.db.metadata.MetadataConstant; import org.apache.iotdb.db.metadata.logfile.MLogReader; import org.apache.iotdb.db.metadata.logfile.MLogWriter; -import org.apache.iotdb.db.metadata.mnode.*; +import org.apache.iotdb.db.metadata.mnode.IEntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.mnode.InternalMNode; +import org.apache.iotdb.db.metadata.mnode.MNodeUtils; +import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode; import org.apache.iotdb.db.metadata.mtree.traverser.collector.EntityCollector; import org.apache.iotdb.db.metadata.mtree.traverser.collector.MNodeCollector; import org.apache.iotdb.db.metadata.mtree.traverser.collector.MeasurementCollector; @@ -892,7 +899,8 @@ public class MTree implements Serializable { * * @param pathPattern a path pattern or a full path, may contain wildcard. */ - public List<PartialPath> getMeasurementPaths(PartialPath pathPattern) throws MetadataException { + public List<MeasurementPath> getMeasurementPaths(PartialPath pathPattern) + throws MetadataException { return getMeasurementPathsWithAlias(pathPattern, 0, 0).left; } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java index 29e36dd..f60b8da 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.query.LogicalOptimizeException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.index.common.IndexType; import org.apache.iotdb.db.metadata.path.AlignedPath; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.constant.SQLConstant; import org.apache.iotdb.db.qp.logical.Operator; @@ -199,6 +200,7 @@ public class QueryOperator extends Operator { FilterOperator filterOperator = whereComponent.getFilterOperator(); List<PartialPath> filterPaths = new ArrayList<>(filterOperator.getPathSet()); try { + // TODO List<TSDataType> seriesTypes = generator.getSeriesTypes(filterPaths); HashMap<PartialPath, TSDataType> pathTSDataTypeHashMap = new HashMap<>(); for (int i = 0; i < filterPaths.size(); i++) { @@ -219,6 +221,7 @@ public class QueryOperator extends Operator { } try { + // TODO rawDataQueryPlan.setDataTypes(generator.getSeriesTypes(selectComponent.getPaths())); queryPlan.deduplicate(generator); } catch (MetadataException e) { @@ -264,9 +267,8 @@ public class QueryOperator extends Operator { PartialPath fullPath = device.concatPath(suffixPath); try { // remove stars in SELECT to get actual paths - List<PartialPath> actualPaths = getMatchedTimeseries(fullPath); - if (suffixPath.getNodes().length > 1 - && (actualPaths.isEmpty() || !(actualPaths.get(0) instanceof AlignedPath))) { + List<MeasurementPath> actualPaths = getMatchedTimeseries(fullPath); + if (suffixPath.getNodes().length > 1) { throw new QueryProcessException(AlignByDevicePlan.MEASUREMENT_ERROR_MESSAGE); } if (resultColumn.hasAlias() && actualPaths.size() >= 2) { @@ -280,9 +282,9 @@ public class QueryOperator extends Operator { nonExistMeasurement, new MeasurementInfo(MeasurementType.NonExist)); } } else { - for (PartialPath path : actualPaths) { + for (MeasurementPath path : actualPaths) { String measurementName = getMeasurementName(path, aggregation); - TSDataType measurementDataType = IoTDB.metaManager.getSeriesType(path); + TSDataType measurementDataType = path.getMeasurementSchema().getType(); TSDataType columnDataType = getAggregationType(aggregation); columnDataType = columnDataType == null ? measurementDataType : columnDataType; MeasurementInfo measurementInfo = @@ -443,6 +445,7 @@ public class QueryOperator extends Operator { // transform to a list so it can be indexed List<PartialPath> filterPathList = new ArrayList<>(filterPaths); try { + // TODO List<TSDataType> seriesTypes = generator.getSeriesTypes(filterPathList); Map<PartialPath, TSDataType> pathTSDataTypeHashMap = new HashMap<>(); for (int i = 0; i < filterPathList.size(); i++) { @@ -500,7 +503,7 @@ public class QueryOperator extends Operator { return IoTDB.metaManager.getMatchedDevices(path); } - protected List<PartialPath> getMatchedTimeseries(PartialPath path) throws MetadataException { + protected List<MeasurementPath> getMatchedTimeseries(PartialPath path) throws MetadataException { return IoTDB.metaManager.getMeasurementPaths(path); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java index c00f479..1641580 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.qp.physical.crud; import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; @@ -33,8 +34,7 @@ import java.util.Map; public abstract class QueryPlan extends PhysicalPlan { protected List<ResultColumn> resultColumns = null; - protected List<PartialPath> paths = null; - protected List<TSDataType> dataTypes = null; + protected List<MeasurementPath> paths = null; private boolean alignByTime = true; // for disable align sql private int rowLimit = 0; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java index 5658caf..1903cf9 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/RawDataQueryPlan.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.strategy.PhysicalGenerator; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.expression.IExpression; import org.apache.iotdb.tsfile.utils.Pair; @@ -40,7 +39,6 @@ import java.util.Set; public class RawDataQueryPlan extends QueryPlan { private List<PartialPath> deduplicatedPaths = new ArrayList<>(); - private List<TSDataType> deduplicatedDataTypes = new ArrayList<>(); private IExpression expression = null; private Map<String, Set<String>> deviceToMeasurements = new HashMap<>(); @@ -48,8 +46,6 @@ public class RawDataQueryPlan extends QueryPlan { /** used to group all the sub sensors of one vector into VectorPartialPath */ private List<PartialPath> deduplicatedVectorPaths = new ArrayList<>(); - private List<TSDataType> deduplicatedVectorDataTypes = new ArrayList<>(); - public RawDataQueryPlan() { super(); } @@ -140,18 +136,6 @@ public class RawDataQueryPlan extends QueryPlan { this.deduplicatedPaths = deduplicatedPaths; } - public List<TSDataType> getDeduplicatedDataTypes() { - return deduplicatedDataTypes; - } - - public void addDeduplicatedDataTypes(TSDataType dataType) { - this.deduplicatedDataTypes.add(dataType); - } - - public void setDeduplicatedDataTypes(List<TSDataType> deduplicatedDataTypes) { - this.deduplicatedDataTypes = deduplicatedDataTypes; - } - public Set<String> getAllMeasurementsInDevice(String device) { return deviceToMeasurements.getOrDefault(device, new HashSet<>()); } @@ -173,12 +157,10 @@ public class RawDataQueryPlan extends QueryPlan { * the raw query has value filter here. */ public void groupVectorPaths(PhysicalGenerator physicalGenerator) throws MetadataException { + // TODO List<PartialPath> vectorizedDeduplicatedPaths = physicalGenerator.groupVectorPaths(getDeduplicatedPaths()); - List<TSDataType> vectorizedDeduplicatedDataTypes = - new ArrayList<>(physicalGenerator.getSeriesTypes(vectorizedDeduplicatedPaths)); setDeduplicatedVectorPaths(vectorizedDeduplicatedPaths); - setDeduplicatedVectorDataTypes(vectorizedDeduplicatedDataTypes); } public List<PartialPath> getDeduplicatedVectorPaths() { @@ -189,14 +171,6 @@ public class RawDataQueryPlan extends QueryPlan { this.deduplicatedVectorPaths = deduplicatedVectorPaths; } - public List<TSDataType> getDeduplicatedVectorDataTypes() { - return deduplicatedVectorDataTypes; - } - - public void setDeduplicatedVectorDataTypes(List<TSDataType> deduplicatedVectorDataTypes) { - this.deduplicatedVectorDataTypes = deduplicatedVectorDataTypes; - } - /** * RawQueryWithoutValueFilter should call this method to use grouped vector partial path to * replace the previous deduplicatedPaths and deduplicatedDataTypes @@ -204,7 +178,6 @@ public class RawDataQueryPlan extends QueryPlan { public void transformToVector() { if (!this.deduplicatedVectorPaths.isEmpty()) { this.deduplicatedPaths = this.deduplicatedVectorPaths; - this.deduplicatedDataTypes = this.deduplicatedVectorDataTypes; } } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java index 8e9816b..c16a182 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java @@ -27,7 +27,6 @@ import org.apache.iotdb.db.qp.logical.sys.LoadConfigurationOperator.LoadConfigur import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.sys.LoadConfigurationPlan; import org.apache.iotdb.db.qp.physical.sys.LoadConfigurationPlan.LoadConfigurationPlanType; -import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import java.util.List; @@ -55,7 +54,7 @@ public class PhysicalGenerator { } public List<TSDataType> getSeriesTypes(List<PartialPath> paths) throws MetadataException { - return SchemaUtils.getSeriesTypesByPaths(paths); + throw new RuntimeException("f**k"); } public List<PartialPath> groupVectorPaths(List<PartialPath> paths) throws MetadataException { diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java index f540fe5..9ddcdd4 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.LogicalOptimizeException; import org.apache.iotdb.db.exception.query.PathNumOverLimitException; import org.apache.iotdb.db.exception.runtime.SQLParserException; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.constant.FilterConstant.FilterType; import org.apache.iotdb.db.qp.constant.SQLConstant; @@ -151,7 +152,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer { } private FilterOperator concatFilterAndRemoveWildcards( - List<PartialPath> fromPaths, FilterOperator operator, Set<PartialPath> filterPaths) + List<PartialPath> fromPaths, FilterOperator operator, Set<MeasurementPath> filterPaths) throws LogicalOptimizeException { if (!operator.isLeaf()) { List<FilterOperator> newFilterList = new ArrayList<>(); @@ -161,17 +162,18 @@ public class ConcatPathOptimizer implements ILogicalOptimizer { operator.setChildren(newFilterList); return operator; } + List<PartialPath> concatPaths = new ArrayList<>(); FunctionOperator functionOperator = (FunctionOperator) operator; PartialPath filterPath = functionOperator.getSinglePath(); // do nothing in the cases of "where time > 5" or "where root.d1.s1 > 5" - if (SQLConstant.isReservedPath(filterPath) - || filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) { - filterPaths.add(filterPath); + if (SQLConstant.isReservedPath(filterPath)) { return operator; + } else if (filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) { + concatPaths.add(filterPath); + } else { + fromPaths.forEach(fromPath -> concatPaths.add(fromPath.concatPath(filterPath))); } - List<PartialPath> concatPaths = new ArrayList<>(); - fromPaths.forEach(fromPath -> concatPaths.add(fromPath.concatPath(filterPath))); - List<PartialPath> noStarPaths = removeWildcardsInConcatPaths(concatPaths); + List<MeasurementPath> noStarPaths = removeWildcardsInConcatPaths(concatPaths); filterPaths.addAll(noStarPaths); if (noStarPaths.size() == 1) { // Transform "select s1 from root.car.* where s1 > 10" to @@ -232,12 +234,12 @@ public class ConcatPathOptimizer implements ILogicalOptimizer { return filterBinaryTree; } - private List<PartialPath> removeWildcardsInConcatPaths(List<PartialPath> originalPaths) + private List<MeasurementPath> removeWildcardsInConcatPaths(List<PartialPath> originalPaths) throws LogicalOptimizeException { HashSet<PartialPath> actualPaths = new HashSet<>(); try { for (PartialPath originalPath : originalPaths) { - List<PartialPath> all = + List<MeasurementPath> all = IoTDB.metaManager.getMeasurementPathsWithAlias(originalPath, 0, 0).left; if (all.isEmpty()) { throw new LogicalOptimizeException( diff --git a/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java index 51f3ae3..725e301 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.query.LogicalOptimizeException; import org.apache.iotdb.db.exception.query.PathNumOverLimitException; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.logical.crud.QueryOperator; import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer; @@ -56,9 +57,10 @@ public class WildcardsRemover { public WildcardsRemover() {} - public List<PartialPath> removeWildcardFrom(PartialPath path) throws LogicalOptimizeException { + public List<MeasurementPath> removeWildcardFrom(PartialPath path) + throws LogicalOptimizeException { try { - Pair<List<PartialPath>, Integer> pair = + Pair<List<MeasurementPath>, Integer> pair = IoTDB.metaManager.getMeasurementPathsWithAlias(path, currentLimit, currentOffset); consumed += pair.right; diff --git a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java index eb08fd4..a4b4896 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java +++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.expression.unary; import org.apache.iotdb.db.exception.query.LogicalOptimizeException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.crud.UDTFPlan; import org.apache.iotdb.db.qp.utils.WildcardsRemover; @@ -65,7 +66,7 @@ public class TimeSeriesOperand extends Expression { @Override public void removeWildcards(WildcardsRemover wildcardsRemover, List<Expression> resultExpressions) throws LogicalOptimizeException { - for (PartialPath actualPath : wildcardsRemover.removeWildcardFrom(path)) { + for (MeasurementPath actualPath : wildcardsRemover.removeWildcardFrom(path)) { resultExpressions.add(new TimeSeriesOperand(actualPath)); } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 3583ce2..1cf3622 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -970,6 +970,7 @@ public class TSServiceImpl implements TSIService.Iface { } else { respColumns.add(resultColumns.get(i).getResultColumnName()); } + // TODO seriesTypes.add(getSeriesTypeByPath(paths.get(i))); } break; @@ -985,6 +986,7 @@ public class TSServiceImpl implements TSIService.Iface { for (ResultColumn resultColumn : resultColumns) { respColumns.add(resultColumn.getResultColumnName()); } + // TODO seriesTypes = SchemaUtils.getSeriesTypesByPaths(paths, aggregations); break; case UDTF: @@ -2186,10 +2188,6 @@ public class TSServiceImpl implements TSIService.Iface { return executor.processNonQuery(plan); } - protected TSDataType getSeriesTypeByPath(PartialPath path) throws MetadataException { - return SchemaUtils.getSeriesTypeByPath(path); - } - private TSStatus onQueryException(Exception e, String operation) { TSStatus status = tryCatchQueryException(e); return status != null diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 295930d..c01f2f6 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -152,10 +152,6 @@ public class SchemaUtils { return measurementDataType; } - public static TSDataType getSeriesTypeByPath(PartialPath path) throws MetadataException { - return IoTDB.metaManager.getSeriesType(path); - } - public static List<TSDataType> getSeriesTypesByPaths( List<PartialPath> paths, List<String> aggregations) throws MetadataException { List<TSDataType> tsDataTypes = new ArrayList<>(); @@ -166,6 +162,7 @@ public class SchemaUtils { tsDataTypes.add(dataType); } else { PartialPath path = paths.get(i); + // TODO tsDataTypes.add(path == null ? null : IoTDB.metaManager.getSeriesType(path)); } }
