This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 40c19d2b368901558a47233625febe123636ac61 Merge: 3ded47f f0e59f1 Author: Jialin Qiao <[email protected]> AuthorDate: Sun Feb 23 15:06:28 2020 +0800 Merge pull request #713 from apache/new_series_reader [IOTDB-418] New series reader .travis.yml | 6 +- .../org/apache/iotdb/client/AbstractClient.java | 104 ++-- .../SystemDesign/1-TsFile/4-Read.md | 541 ++++++++++++++++++++- .../SystemDesign/5-DataQuery/1-DataQuery.md | 52 +- .../SystemDesign/5-DataQuery/2-SeriesReader.md | 414 ++++++++++++++++ .../SystemDesign/5-DataQuery/3-RawDataQuery.md | 301 ++++++++++++ .../SystemDesign/5-DataQuery/4-AggregationQuery.md | 118 +++++ .../SystemDesign/5-DataQuery/5-GroupByQuery.md | 192 ++++++++ .../2-DML (Data Manipulation Language).md | 4 +- .../5-Operation Manual/4-SQL Reference.md | 58 +-- docs/Documentation/SystemDesign/0-Content.md | 4 + docs/Documentation/SystemDesign/1-TsFile/4-Read.md | 537 ++++++++++++++++++++ .../2-DML (Data Manipulation Language).md | 20 +- .../5-Operation Manual/4-SQL Reference.md | 58 +-- hadoop/src/test/resources/logback.xml | 2 +- .../src/test/resources/logback.xml | 4 +- .../resources/conf/iotdb-engine.properties | 18 +- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 20 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 52 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 84 ++-- .../db/conf/adapter/ActiveTimeSeriesCounter.java | 19 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 26 +- .../db/engine/cache/CacheHitRatioMonitor.java | 5 + .../engine/cache/CacheHitRatioMonitorMXBean.java | 2 + .../apache/iotdb/db/engine/cache/ChunkCache.java | 150 ++++++ .../iotdb/db/engine/cache/DeviceMetaDataCache.java | 13 +- .../iotdb/db/engine/cache/RamUsageEstimator.java | 34 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 33 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 42 +- .../apache/iotdb/db/engine/memtable/IMemTable.java | 8 +- .../db/engine/memtable/IWritableMemChunk.java | 16 +- .../db/engine/memtable/MemSeriesLazyMerger.java | 50 -- .../engine/memtable/TimeValuePairInMemTable.java | 45 -- .../db/engine/memtable/TimeValuePairSorter.java | 55 --- .../iotdb/db/engine/memtable/WritableMemChunk.java | 66 +-- .../iotdb/db/engine/merge/manage/MergeManager.java | 40 ++ .../db/engine/merge/manage/MergeResource.java | 6 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 6 +- .../db/engine/merge/task/MergeMultiChunkTask.java | 17 +- .../db/engine/querycontext/QueryDataSource.java | 13 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 131 ++--- .../engine/storagegroup/StorageGroupProcessor.java | 101 ++-- .../db/engine/storagegroup/TsFileProcessor.java | 274 +++++++---- .../db/engine/storagegroup/TsFileResource.java | 20 +- .../db/exception/ConfigAdjusterException.java | 5 +- .../exception/DiskSpaceInsufficientException.java | 4 +- .../{ProcessException.java => IoTDBException.java} | 7 +- .../apache/iotdb/db/exception/MergeException.java | 5 +- .../exception/QueryInBatchStatementException.java | 6 +- .../iotdb/db/exception/StartupException.java | 12 +- .../iotdb/db/exception/StorageEngineException.java | 14 +- .../db/exception/SyncConnectionException.java | 11 +- .../SyncDeviceOwnerConflictException.java | 8 +- .../iotdb/db/exception/SystemCheckException.java | 6 +- .../db/exception/TsFileProcessorException.java | 8 +- .../db/exception/metadata/MetadataException.java | 19 +- .../metadata/TimeseriesAlreadyExistException.java | 4 +- .../db/exception/path/MTreePathException.java | 1 - .../exception/path/NotStorageGroupException.java | 4 +- .../iotdb/db/exception/path/PathException.java | 10 +- .../exception/query/LogicalOperatorException.java | 16 +- .../exception/query/LogicalOptimizeException.java | 6 +- .../db/exception/query/OutOfTTLException.java | 4 +- .../db/exception/query/QueryProcessException.java | 16 +- .../query/UnSupportedFillTypeException.java | 4 +- .../JDBCServiceException.java} | 13 +- .../storageGroup/StorageGroupException.java | 7 +- .../StorageGroupProcessorException.java | 10 +- .../org/apache/iotdb/db/metadata/MManager.java | 11 + .../db/qp/{QueryProcessor.java => Planner.java} | 30 +- .../apache/iotdb/db/qp/constant/SQLConstant.java | 3 +- .../qp/executor/AbstractQueryProcessExecutor.java | 390 --------------- ...ueryProcessExecutor.java => IPlanExecutor.java} | 48 +- ...QueryProcessExecutor.java => PlanExecutor.java} | 478 ++++++++++++++---- .../db/qp/logical/crud/BasicFunctionOperator.java | 8 +- .../iotdb/db/qp/logical/crud/FilterOperator.java | 17 +- .../iotdb/db/qp/logical/crud/InOperator.java | 8 +- .../iotdb/db/qp/logical/crud/QueryOperator.java | 20 +- .../iotdb/db/qp/physical/crud/AggregationPlan.java | 2 +- .../db/qp/physical/crud/AlignByDevicePlan.java | 176 +++++++ .../iotdb/db/qp/physical/crud/FillQueryPlan.java | 2 +- .../iotdb/db/qp/physical/crud/GroupByPlan.java | 13 +- .../iotdb/db/qp/physical/crud/QueryPlan.java | 188 +------ .../db/qp/physical/crud/RawDataQueryPlan.java | 76 +++ .../iotdb/db/qp/strategy/LogicalGenerator.java | 15 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 177 +++---- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 20 +- .../db/qp/strategy/optimizer/IFilterOptimizer.java | 3 +- .../db/query/aggregation/AggreResultData.java | 192 -------- .../db/query/aggregation/AggregateFunction.java | 122 ----- .../db/query/aggregation/AggregateResult.java | 212 ++++++++ .../db/query/aggregation/impl/AvgAggrFunc.java | 173 ------- .../db/query/aggregation/impl/AvgAggrResult.java | 115 +++++ .../db/query/aggregation/impl/CountAggrFunc.java | 156 ------ .../db/query/aggregation/impl/CountAggrResult.java | 89 ++++ .../query/aggregation/impl/FirstValueAggrFunc.java | 154 ------ .../aggregation/impl/FirstValueAggrResult.java | 97 ++++ .../query/aggregation/impl/LastValueAggrFunc.java | 151 ------ .../aggregation/impl/LastValueAggrResult.java | 90 ++++ .../db/query/aggregation/impl/MaxTimeAggrFunc.java | 135 ----- .../query/aggregation/impl/MaxTimeAggrResult.java | 93 ++++ .../query/aggregation/impl/MaxValueAggrFunc.java | 170 ------- .../query/aggregation/impl/MaxValueAggrResult.java | 94 ++++ .../db/query/aggregation/impl/MinTimeAggrFunc.java | 156 ------ .../query/aggregation/impl/MinTimeAggrResult.java | 85 ++++ .../query/aggregation/impl/MinValueAggrFunc.java | 166 ------- .../query/aggregation/impl/MinValueAggrResult.java | 90 ++++ .../db/query/aggregation/impl/SumAggrResult.java | 107 ++++ .../iotdb/db/query/control/FileReaderManager.java | 21 +- .../db/query/control/QueryResourceManager.java | 29 +- .../query/dataset/AggreResultDataPointReader.java | 56 --- ...erateDataSet.java => AlignByDeviceDataSet.java} | 139 +++--- .../db/query/dataset/NonAlignEngineDataSet.java | 45 +- .../OldEngineDataSetWithoutValueFilter.java | 153 ------ ...er.java => RawQueryDataSetWithValueFilter.java} | 28 +- ...java => RawQueryDataSetWithoutValueFilter.java} | 103 ++-- .../iotdb/db/query/dataset/SingleDataSet.java | 3 +- .../dataset/groupby/GroupByEngineDataSet.java | 86 +--- .../groupby/GroupByWithValueFilterDataSet.java | 71 +-- .../groupby/GroupByWithoutValueFilterDataSet.java | 385 +++++++-------- .../db/query/executor/AggregateEngineExecutor.java | 336 ------------- .../db/query/executor/AggregationExecutor.java | 289 +++++++++++ ...lEngineExecutor.java => FillQueryExecutor.java} | 57 ++- .../{IEngineQueryRouter.java => IQueryRouter.java} | 6 +- .../{EngineQueryRouter.java => QueryRouter.java} | 123 +++-- ...gineExecutor.java => RawDataQueryExecutor.java} | 98 ++-- .../db/query/externalsort/ExternalSortJob.java | 2 +- .../query/externalsort/ExternalSortJobEngine.java | 7 +- .../db/query/externalsort/ExternalSortJobPart.java | 2 +- .../iotdb/db/query/externalsort/LineMerger.java | 6 +- .../MultiSourceExternalSortJobPart.java | 2 +- .../externalsort/SimpleExternalSortEngine.java | 15 +- .../SingleSourceExternalSortJobPart.java | 28 +- .../adapter/ByTimestampReaderAdapter.java | 67 ++- .../serialize/IExternalSortFileDeserializer.java | 6 +- .../serialize/IExternalSortFileSerializer.java | 2 +- .../FixLengthIExternalSortFileDeserializer.java | 6 +- .../impl/FixLengthTimeValuePairSerializer.java | 2 +- ...greFuncFactory.java => AggreResultFactory.java} | 44 +- .../java/org/apache/iotdb/db/query/fill/IFill.java | 71 +-- .../org/apache/iotdb/db/query/fill/LinearFill.java | 51 +- .../apache/iotdb/db/query/fill/PreviousFill.java | 39 +- .../ChunkDataIterator.java} | 44 +- .../{chunkRelated => chunk}/ChunkReaderWrap.java | 14 +- .../db/query/reader/chunk/DiskChunkLoader.java | 36 +- .../DiskChunkReaderByTimestamp.java | 64 +-- .../db/query/reader/chunk/MemChunkLoader.java | 41 +- .../{chunkRelated => chunk}/MemChunkReader.java | 58 +-- .../chunk/MemPageReader.java} | 31 +- .../reader/chunkRelated/CachedDiskChunkReader.java | 88 ---- .../chunkRelated/MemChunkReaderByTimestamp.java | 78 --- .../FileSeriesReaderByTimestampAdapter.java | 49 -- .../fileRelated/UnSealedTsFileIterateReader.java | 130 ----- .../UnSealedTsFileReaderByTimestamp.java | 96 ---- .../CachedUnseqResourceMergeReader.java | 11 +- .../NewUnseqResourceMergeReader.java | 199 -------- .../OldUnseqResourceMergeReader.java | 140 ------ .../resourceRelated/ResourceRelatedUtil.java | 48 -- .../resourceRelated/SeqResourceIterateReader.java | 157 ------ .../SeqResourceReaderByTimestamp.java | 190 -------- .../UnseqResourceReaderByTimestamp.java | 101 ---- .../db/query/reader/series}/IAggregateReader.java | 36 +- .../reader/{ => series}/IReaderByTimestamp.java | 9 +- .../reader/{ => series}/ManagedSeriesReader.java | 8 +- .../query/reader/series/SeriesAggregateReader.java | 102 ++++ .../reader/series/SeriesRawDataBatchReader.java | 132 +++++ .../reader/series/SeriesRawDataPointReader.java | 100 ++++ .../iotdb/db/query/reader/series/SeriesReader.java | 471 ++++++++++++++++++ .../reader/series/SeriesReaderByTimestamp.java | 93 ++++ .../seriesRelated/SeriesReaderByTimestamp.java | 57 --- .../seriesRelated/SeriesReaderWithValueFilter.java | 115 ----- .../SeriesReaderWithoutValueFilter.java | 270 ---------- .../universal/CachedPriorityMergeReader.java | 8 +- .../db/query/reader/universal/IterateReader.java | 90 ---- .../reader/universal/PriorityMergeReader.java | 48 +- .../universal/PriorityMergeReaderByTimestamp.java | 75 --- .../timegenerator/AbstractNodeConstructor.java | 75 --- .../query/timegenerator/EngineNodeConstructor.java | 69 --- .../query/timegenerator/EngineTimeGenerator.java | 67 --- .../query/timegenerator/ServerTimeGenerator.java | 76 +++ .../java/org/apache/iotdb/db/service/IService.java | 2 + .../org/apache/iotdb/db/service/JDBCService.java | 7 +- .../org/apache/iotdb/db/service/JMXService.java | 10 +- .../apache/iotdb/db/service/MetricsService.java | 6 +- .../apache/iotdb/db/service/RegisterManager.java | 2 +- .../org/apache/iotdb/db/service/StartupChecks.java | 2 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 321 ++++++------ .../java/org/apache/iotdb/db/utils/FileUtils.java | 8 +- .../java/org/apache/iotdb/db/utils/MergeUtils.java | 1 + .../apache/iotdb/db/utils/TimeValuePairUtils.java | 66 +-- .../iotdb/db/utils/datastructure/BinaryTVList.java | 26 +- .../db/utils/datastructure/BooleanTVList.java | 27 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 31 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 31 +- .../iotdb/db/utils/datastructure/IntTVList.java | 27 +- .../iotdb/db/utils/datastructure/LongTVList.java | 26 +- .../iotdb/db/utils/datastructure/TVList.java | 122 ++++- .../db/engine/cache/DeviceMetaDataCacheTest.java | 6 +- .../db/engine/memtable/PrimitiveMemTableTest.java | 80 +-- .../iotdb/db/engine/merge/MergeOverLapTest.java | 39 +- .../iotdb/db/engine/merge/MergeTaskTest.java | 190 ++++---- .../engine/modification/DeletionFileNodeTest.java | 68 +-- .../db/engine/modification/DeletionQueryTest.java | 27 +- .../storagegroup/StorageGroupProcessorTest.java | 44 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 96 ++-- .../engine/storagegroup/TsFileProcessorTest.java | 144 +++--- .../iotdb/db/integration/IOTDBGroupByIT.java | 43 +- .../integration/IOTDBGroupByInnerIntervalIT.java | 26 +- .../iotdb/db/integration/IoTDBAggregationIT.java | 170 +++++-- .../integration/IoTDBAggregationSmallDataIT.java | 129 +++-- ...upbyDeviceIT.java => IoTDBAlignByDeviceIT.java} | 67 ++- .../iotdb/db/integration/IoTDBDisableAlignIT.java | 4 +- .../db/integration/IoTDBEngineTimeGeneratorIT.java | 10 +- .../apache/iotdb/db/integration/IoTDBFillIT.java | 2 +- .../iotdb/db/integration/IoTDBMergeTest.java | 3 + .../iotdb/db/integration/IoTDBRecoverIT.java | 2 + .../db/integration/IoTDBSequenceDataQueryIT.java | 23 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 53 +- .../{QueryProcessorTest.java => PlannerTest.java} | 7 +- .../iotdb/db/qp/bench/QueryParseBenchmark.java | 11 +- .../iotdb/db/qp/other/TSPlanContextAuthorTest.java | 9 +- .../db/qp/other/TSPlanContextPropertyTest.java | 9 +- .../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 38 +- .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 288 +++++------ .../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 206 -------- .../iotdb/db/qp/plan/TestConcatOptimizer.java | 103 ++-- .../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 196 -------- .../dataset/EngineDataSetWithValueFilterTest.java | 14 +- .../query/executor/GroupByEngineDataSetTest.java | 27 +- .../query/externalsort/ExternalSortEngineTest.java | 40 +- .../db/query/externalsort/FakeChunkReaderWrap.java | 6 +- ...ExternalSortFileSerializerDeserializerTest.java | 6 +- .../iotdb/db/query/reader/ReaderTestHelper.java | 2 + .../fileRelated/UnSealedTsFileReaderTest.java | 112 ----- .../NewUnseqResourceMergeReaderTest.java | 109 ----- .../resourceRelated/SeqResourceReaderTest.java | 116 ----- .../resourceRelated/UnseqResourceReaderTest.java | 146 ------ .../reader/series/SeriesAggregateReaderTest.java | 116 +++++ .../reader/series/SeriesReaderByTimestampTest.java | 84 ++++ .../db/query/reader/series/SeriesReaderTest.java | 127 +++++ .../query/reader/series/SeriesReaderTestUtil.java | 177 +++++++ .../reader/seriesRelated/FakedIBatchPoint.java | 110 ----- .../reader/seriesRelated/FakedIPointReader.java | 79 --- .../SeriesReaderWithValueFilterTest.java | 59 --- .../SeriesReaderWithoutValueFilterTest.java | 74 --- .../query/reader/universal/FakedSeriesReader.java | 10 +- .../universal/FakedSeriesReaderByTimestamp.java | 79 --- .../PriorityMergeReaderByTimestampTest.java | 172 ------- .../reader/universal/PriorityMergeReaderTest.java | 8 +- .../reader/universal/PriorityMergeReaderTest2.java | 13 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 69 ++- .../db/utils/datastructure/LongTVListTest.java | 2 +- .../iotdb/db/writelog/recover/LogReplayerTest.java | 21 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 9 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 25 +- server/src/test/resources/logback.xml | 13 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + .../org/apache/iotdb/session/IoTDBSessionIT.java | 60 ++- .../apache/iotdb/tsfile/common/cache/LRUCache.java | 8 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 11 + .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 5 +- .../iotdb/tsfile/file/header/ChunkHeader.java | 8 +- .../iotdb/tsfile/file/metadata/ChunkMetaData.java | 412 ++++++++-------- .../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 4 +- .../apache/iotdb/tsfile/read}/TimeValuePair.java | 2 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 11 +- .../apache/iotdb/tsfile/read/common/BatchData.java | 52 +- .../org/apache/iotdb/tsfile/read/common/Field.java | 40 +- .../org/apache/iotdb/tsfile/read/common/Path.java | 3 +- .../apache/iotdb/tsfile/read/common/RowRecord.java | 5 + .../iotdb/tsfile/read/common/SignalBatchData.java | 5 + .../common/{RowRecord.java => TimeColumn.java} | 67 ++- ...kLoaderImpl.java => CachedChunkLoaderImpl.java} | 16 +- .../iotdb/tsfile/read/controller/IChunkLoader.java | 8 +- .../iotdb/tsfile/read/filter/GroupByFilter.java | 22 +- .../tsfile/read/filter/basic/UnaryFilter.java | 4 + .../query/dataset/DataSetWithTimeGenerator.java | 4 +- .../tsfile/read/query/dataset/QueryDataSet.java | 34 -- .../query/executor/ExecutorWithTimeGenerator.java | 4 +- .../read/query/timegenerator/TimeGenerator.java | 99 +++- .../query/timegenerator/TimeGeneratorImpl.java | 130 ----- .../query/timegenerator/TsFileTimeGenerator.java | 51 ++ .../read/query/timegenerator/node/AndNode.java | 86 ++-- .../read/query/timegenerator/node/LeafNode.java | 54 +- .../tsfile/read/query/timegenerator/node/Node.java | 5 +- .../read/query/timegenerator/node/OrNode.java | 121 +++-- .../tsfile/read/reader/BatchDataIterator.java | 44 +- .../iotdb/tsfile/read/reader/IChunkReader.java | 17 +- .../{IDataReader.java => reader/IPageReader.java} | 13 +- .../iotdb/tsfile/read}/reader/IPointReader.java | 12 +- .../tsfile/read/reader/chunk/ChunkReader.java | 87 ++-- .../read/reader/chunk/ChunkReaderByTimestamp.java | 4 +- .../iotdb/tsfile/read/reader/page/PageReader.java | 42 +- .../reader/series/AbstractFileSeriesReader.java | 37 +- .../read/reader/series/EmptyFileSeriesReader.java | 5 - .../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 13 - .../apache/iotdb/tsfile/write/TsFileWriter.java | 23 +- .../write/writer/ForceAppendTsFileWriter.java | 8 +- .../write/writer/RestorableTsFileIOWriter.java | 5 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +- .../decoder/delta/DeltaBinaryEncoderLongTest.java | 18 +- .../apache/iotdb/tsfile/read/ExpressionTest.java | 60 +++ .../org/apache/iotdb/tsfile/read/ReadTest.java | 18 +- .../iotdb/tsfile/read/common/BatchDataTest.java | 82 ++++ .../tsfile/read/controller/ChunkLoaderTest.java | 2 +- .../read/filter/MinTimeMaxTimeFilterTest.java | 6 + .../read/query/executor/QueryExecutorTest.java | 4 +- .../tsfile/read/query/timegenerator/NodeTest.java | 38 +- .../query/timegenerator/ReaderByTimestampTest.java | 4 +- .../query/timegenerator/TimeGeneratorTest.java | 9 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 6 +- .../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 56 +-- .../iotdb/tsfile/utils/TsPrimitiveTypeTest.java | 60 +++ .../iotdb/tsfile/write/TsFileWriterTest.java | 26 +- .../iotdb/tsfile/write/writer/PageWriterTest.java | 18 +- 315 files changed, 10368 insertions(+), 10471 deletions(-)
