This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch add_TsFileFilter in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 9355a1fa650e0dedfb4425be093d6cba6e9e102d Author: jt2594838 <[email protected]> AuthorDate: Fri Feb 21 10:10:18 2020 +0800 add TsFileFilter --- .gitignore | 3 +++ .../iotdb/db/query/executor/EngineExecutor.java | 4 +-- .../java/org/apache/iotdb/db/query/fill/IFill.java | 3 ++- .../apache/iotdb/db/query/filter/TsFileFilter.java | 31 ++++++++++++++++++++++ .../seriesRelated/SeriesReaderWithValueFilter.java | 6 +++-- .../SeriesReaderWithoutValueFilter.java | 6 ++++- .../query/timegenerator/EngineNodeConstructor.java | 3 ++- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 14 ++++++++++ 8 files changed, 63 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 70913bb..34f0e12 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,6 @@ tsfile/src/test/resources/*.ts local-snapshots-dir/ venv/ +partitions.tmp +partitions +node_identifier diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java index 0a10375..456eeb3 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java @@ -78,7 +78,7 @@ public class EngineExecutor { TSDataType dataType = deduplicatedDataTypes.get(i); ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, - true); + true, null); readersOfSelectedSeries.add(reader); } @@ -104,7 +104,7 @@ public class EngineExecutor { TSDataType dataType = deduplicatedDataTypes.get(i); ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, - true); + true, null); readersOfSelectedSeries.add(reader); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java index 5db57eb..4fb17ed 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java +++ b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java @@ -54,7 +54,8 @@ public abstract class IFill { void constructReaders(Path path, QueryContext context, long beforeRange) throws IOException, StorageEngineException { Filter timeFilter = constructFilter(beforeRange); - allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true); + allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true, + null); } public abstract IPointReader getFillResult() throws IOException, UnSupportedFillTypeException; diff --git a/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java new file mode 100644 index 0000000..97711ef --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.filter; + +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + +/** + * TsFileFilter is used to filter unwanted TsFiles in a QueryDataSource, to better support data + * partitioning in the distributed version and other features. + */ +@FunctionalInterface +public interface TsFileFilter { + boolean fileNotSatisfy(TsFileResource resource); +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java index 83f0d3c..dda88cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.seriesRelated; import java.io.IOException; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.reader.IBatchReader; @@ -48,9 +49,10 @@ public class SeriesReaderWithValueFilter extends SeriesReaderWithoutValueFilter private TimeValuePair timeValuePair; private BatchData batchData; - public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter, QueryContext context) + public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter, + QueryContext context, TsFileFilter fileFilter) throws StorageEngineException, IOException { - super(seriesPath, dataType, filter, context, false); + super(seriesPath, dataType, filter, context, false, fileFilter); this.filter = filter; } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java index 27bd19c..0ab2692 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java @@ -23,9 +23,11 @@ import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.QueryResourceManager; +import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.db.query.reader.ManagedSeriesReader; import org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader; import org.apache.iotdb.db.query.reader.resourceRelated.NewUnseqResourceMergeReader; +import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.db.utils.TimeValuePair; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.BatchData; @@ -83,9 +85,11 @@ public class SeriesReaderWithoutValueFilter implements ManagedSeriesReader { * to. We do not push down value filter to unsequence readers */ public SeriesReaderWithoutValueFilter(Path seriesPath, TSDataType dataType, Filter timeFilter, - QueryContext context, boolean pushdownUnseq) throws StorageEngineException, IOException { + QueryContext context, boolean pushdownUnseq, TsFileFilter fileFilter) throws StorageEngineException, + IOException { QueryDataSource queryDataSource = QueryResourceManager.getInstance() .getQueryDataSource(seriesPath, context); + QueryUtils.filterQueryDataSource(queryDataSource, fileFilter); timeFilter = queryDataSource.updateTimeFilter(timeFilter); // reader for sequence resources diff --git a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java index 41fc515..0a9891d 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java @@ -56,7 +56,8 @@ public class EngineNodeConstructor extends AbstractNodeConstructor { Filter filter = ((SingleSeriesExpression) expression).getFilter(); Path path = ((SingleSeriesExpression) expression).getSeriesPath(); TSDataType dataType = MManager.getInstance().getSeriesType(path.getFullPath()); - return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context)); + return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context + , null)); } catch (IOException | PathException e) { throw new StorageEngineException(e.getMessage()); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java index 03e90e3..e681832 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java @@ -22,6 +22,9 @@ package org.apache.iotdb.db.utils; import java.util.List; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.Modification; +import org.apache.iotdb.db.engine.querycontext.QueryDataSource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.filter.TsFileFilter; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; public class QueryUtils { @@ -73,4 +76,15 @@ public class QueryUtils { } return false; } + + // remove files that do not satisfy the filter + public static void filterQueryDataSource(QueryDataSource queryDataSource, TsFileFilter fileFilter) { + if (fileFilter == null) { + return; + } + List<TsFileResource> seqResources = queryDataSource.getSeqResources(); + List<TsFileResource> unseqResources = queryDataSource.getUnseqResources(); + seqResources.removeIf(fileFilter::fileNotSatisfy); + unseqResources.removeIf(fileFilter::fileNotSatisfy); + } }
