This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch pushdown_limit in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 6e8499355e2771186817bd366d196962793eee4e Author: qiaojialin <[email protected]> AuthorDate: Thu Apr 23 10:57:05 2020 +0800 push limit from MManager to MTree --- .../org/apache/iotdb/db/metadata/MManager.java | 31 ++++---------- .../java/org/apache/iotdb/db/metadata/MTree.java | 48 ++++++++++++++++++---- .../db/qp/physical/sys/ShowTimeSeriesPlan.java | 15 ++++--- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +- 4 files changed, 59 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index a18f3d5..356992a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -704,9 +704,11 @@ public class MManager { int offset = plan.getOffset(); for (LeafMNode leaf : allMatchedNodes) { if (match(leaf.getFullPath(), prefixNodes)) { - curOffset ++; - if (curOffset < offset) { - continue; + if (limit != 0 || offset != 0) { + curOffset ++; + if (curOffset < offset || count == limit) { + continue; + } } try { Pair<Map<String, String>, Map<String, String>> pair = @@ -717,9 +719,8 @@ public class MManager { getStorageGroupName(leaf.getFullPath()), measurementSchema.getType().toString(), measurementSchema.getEncodingType().toString(), measurementSchema.getCompressor().toString(), pair.left)); - count ++; - if (count == limit) { - return res; + if (limit != 0 || offset != 0) { + count++; } } catch (IOException e) { throw new MetadataException( @@ -757,17 +758,9 @@ public class MManager { public List<ShowTimeSeriesResult> showTimeseries(ShowTimeSeriesPlan plan) throws MetadataException { lock.readLock().lock(); try { - List<String[]> ans = mtree.getAllMeasurementSchema(plan.getPath().getFullPath()); - int count = 0; - int offset = plan.getOffset(); + List<String[]> ans = mtree.getAllMeasurementSchema(plan); List<ShowTimeSeriesResult> res = new LinkedList<>(); - for (int i = 0; i < ans.size(); i++) { - if (i < offset) { - continue; - } - - String[] ansString = ans.get(i); - + for (String[] ansString : ans) { long tagFileOffset = Long.parseLong(ansString[6]); try { if (tagFileOffset < 0) { @@ -782,12 +775,6 @@ public class MManager { res.add(new ShowTimeSeriesResult(ansString[0], ansString[1], ansString[2], ansString[3], ansString[4], ansString[5], pair.left)); } - - count ++; - if (count == plan.getLimit()) { - return res; - } - } catch (IOException e) { throw new MetadataException( "Something went wrong while deserialize tag info of " + ansString[0], e); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java index 3949aee..80e5d99 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java @@ -28,10 +28,12 @@ import org.apache.iotdb.db.metadata.mnode.InternalMNode; import org.apache.iotdb.db.metadata.mnode.LeafMNode; import org.apache.iotdb.db.metadata.mnode.MNode; import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -51,6 +53,11 @@ public class MTree implements Serializable { private static final long serialVersionUID = -4200394435237291964L; private MNode root; + private int limit = 0; + private int offset = 0; + private int count = 0; + private int curOffset = -1; + MTree() { this.root = new InternalMNode(null, IoTDBConstant.PATH_ROOT); } @@ -466,7 +473,8 @@ public class MTree implements Serializable { * @param prefixPath a prefix path or a full path, may contain '*'. */ List<String> getAllTimeseriesName(String prefixPath) throws MetadataException { - List<String[]> res = getAllMeasurementSchema(prefixPath); + ShowTimeSeriesPlan plan = new ShowTimeSeriesPlan(new Path(prefixPath)); + List<String[]> res = getAllMeasurementSchema(plan); List<String> paths = new ArrayList<>(); for (String[] p : res) { paths.add(p[0]); @@ -477,27 +485,45 @@ public class MTree implements Serializable { /** * Get all time series schema under the given path * - * MeasurementSchema: [name, alias, storage group, dataType, encoding, compression, offset] + * result: [name, alias, storage group, dataType, encoding, compression, offset] */ - List<String[]> getAllMeasurementSchema(String prefixPath) throws MetadataException { + List<String[]> getAllMeasurementSchema(ShowTimeSeriesPlan plan) throws MetadataException { List<String[]> res = new ArrayList<>(); - String[] nodes = MetaUtils.getNodeNames(prefixPath); + String[] nodes = MetaUtils.getNodeNames(plan.getPath().getFullPath()); if (nodes.length == 0 || !nodes[0].equals(root.getName())) { - throw new IllegalPathException(prefixPath); + throw new IllegalPathException(plan.getPath().getFullPath()); + } + this.limit = plan.getLimit(); + this.offset = plan.getOffset(); + if (offset != 0 || limit != 0) { + findPath(root, nodes, 1, "", res, true); + resetCountCurOffset(); + } else { + findPath(root, nodes, 1, "", res, false); } - findPath(root, nodes, 1, "", res); return res; } + private void resetCountCurOffset() { + curOffset = -1; + count = 0; + } + /** * Iterate through MTree to fetch metadata info of all leaf nodes under the given seriesPath * * @param timeseriesSchemaList List<timeseriesSchema> */ private void findPath(MNode node, String[] nodes, int idx, String parent, - List<String[]> timeseriesSchemaList) throws MetadataException { + List<String[]> timeseriesSchemaList, boolean hasLimit) throws MetadataException { if (node instanceof LeafMNode) { if (nodes.length <= idx) { + if (hasLimit) { + curOffset++; + if (curOffset < offset || count == limit) { + return; + } + } String nodeName; if (node.getName().contains(TsFileConstant.PATH_SEPARATOR)) { nodeName = "\"" + node + "\""; @@ -515,6 +541,10 @@ public class MTree implements Serializable { tsRow[5] = measurementSchema.getCompressor().toString(); tsRow[6] = String.valueOf(((LeafMNode) node).getOffset()); timeseriesSchemaList.add(tsRow); + + if (hasLimit) { + count++; + } } return; } @@ -522,7 +552,7 @@ public class MTree implements Serializable { if (!nodeReg.contains(PATH_WILDCARD)) { if (node.hasChild(nodeReg)) { findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, - timeseriesSchemaList); + timeseriesSchemaList, hasLimit); } } else { for (MNode child : node.getChildren().values()) { @@ -530,7 +560,7 @@ public class MTree implements Serializable { continue; } findPath(child, nodes, idx + 1, parent + node.getName() + PATH_SEPARATOR, - timeseriesSchemaList); + timeseriesSchemaList, hasLimit); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java index 48a6491..fbf8fcb 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/ShowTimeSeriesPlan.java @@ -28,12 +28,17 @@ public class ShowTimeSeriesPlan extends ShowPlan { private boolean isContains; private String key; private String value; - private int limit; - private int offset; + private int limit = 0; + private int offset = 0; - public ShowTimeSeriesPlan(ShowContentType showContentType, Path path, boolean isContains, - String key, String value, int limit, int offset) { - super(showContentType); + public ShowTimeSeriesPlan(Path path) { + super(ShowContentType.TIMESERIES); + this.path = path; + } + + public ShowTimeSeriesPlan(Path path, boolean isContains, String key, String value, int limit, + int offset) { + super(ShowContentType.TIMESERIES); this.path = path; this.isContains = isContains; this.key = key; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java index 5cfc547..ff80539 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java @@ -126,7 +126,7 @@ public class PhysicalGenerator { return new ShowPlan(ShowContentType.VERSION); case SQLConstant.TOK_TIMESERIES: ShowTimeSeriesOperator showTimeSeriesOperator = (ShowTimeSeriesOperator) operator; - return new ShowTimeSeriesPlan(ShowContentType.TIMESERIES, + return new ShowTimeSeriesPlan( showTimeSeriesOperator.getPath(), showTimeSeriesOperator.isContains(), showTimeSeriesOperator.getKey(), showTimeSeriesOperator.getValue(), showTimeSeriesOperator.getLimit(), showTimeSeriesOperator.getOffset());
