This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch master_with_chunk_size in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b4a14db6832722a2ba323e78e11c9a4b82cc4ce7 Merge: 7190bb2 e1edeb7 Author: HTHou <[email protected]> AuthorDate: Thu Oct 29 09:25:21 2020 +0800 Merge branch 'master' of https://github.com/apache/iotdb into master_with_chunk_size .travis.yml | 50 +- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 5 + cli/pom.xml | 13 + cli/src/assembly/resources/sbin/start-cli.bat | 8 +- cli/src/assembly/resources/sbin/start-cli.sh | 8 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 14 +- .../org/apache/iotdb/cli/StartClientScriptIT.java | 26 +- client-cpp/src/main/IOTDBSession.h | 2 +- client-py/src/utils/IoTDBConstants.py | 1 + .../tests/tools/importCsv/AbstractScript.java | 29 +- .../tests/tools/importCsv/ImportCsvTestIT.java | 65 +-- docs/SystemDesign/TsFile/Format.md | 1 + docs/UserGuide/Comparison/TSDB-Comparison.md | 28 +- docs/UserGuide/Concept/Compression.md | 4 +- docs/UserGuide/Get Started/QuickStart.md | 2 +- .../DDL Data Definition Language.md | 2 +- docs/UserGuide/Server/Cluster Setup.md | 207 ++++++- docs/UserGuide/Server/Config Manual.md | 2 +- docs/UserGuide/Server/ServerFileList.md | 157 ++++++ docs/UserGuide/System Tools/NodeTool.md | 289 ++++++++++ .../System Tools/Performance Tracing Tool.md | 18 +- docs/zh/SystemDesign/TsFile/Format.md | 1 + docs/zh/UserGuide/Concept/Compression.md | 3 +- .../DDL Data Definition Language.md | 2 +- docs/zh/UserGuide/Server/Cluster Setup.md | 197 ++++++- docs/zh/UserGuide/Server/ServerFileList.md | 153 ++++++ docs/zh/UserGuide/System Tools/NodeTool.md | 246 +++++++++ .../System Tools/Performance Tracing Tool.md | 16 +- example/rocketmq/pom.xml | 6 + grafana/pom.xml | 6 +- .../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 19 +- .../main/java/org/apache/iotdb/jdbc/Config.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 2 +- .../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 8 +- .../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 46 +- .../resources/conf/iotdb-engine.properties | 52 +- server/src/assembly/resources/conf/iotdb-env.bat | 70 ++- .../src/assembly/resources/sbin/start-server.bat | 3 +- server/src/assembly/resources/sbin/stop-server.bat | 17 +- server/src/assembly/resources/sbin/stop-server.sh | 11 +- .../iotdb/db/auth/authorizer/BasicAuthorizer.java | 1 - .../iotdb/db/auth/authorizer/OpenIdAuthorizer.java | 23 +- .../iotdb/db/auth/user/LocalFileUserAccessor.java | 21 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 197 ++++++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 77 ++- .../org/apache/iotdb/db/engine/StorageEngine.java | 137 ++++- .../db/engine/fileSystem/SystemFileFactory.java | 5 +- .../flush/CloseFileListener.java} | 13 +- .../flush/FlushListener.java} | 28 +- .../iotdb/db/engine/flush/TsFileFlushPolicy.java | 2 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 31 +- .../apache/iotdb/db/engine/memtable/IMemTable.java | 7 +- .../db/engine/memtable/PrimitiveMemTable.java | 1 - .../iotdb/db/engine/merge/manage/MergeManager.java | 33 +- .../db/engine/merge/manage/MergeResource.java | 2 - .../engine/modification/io/ModificationReader.java | 1 - .../engine/modification/io/ModificationWriter.java | 1 - .../db/engine/querycontext/ReadOnlyMemChunk.java | 5 +- .../engine/storagegroup/StorageGroupProcessor.java | 376 ++++--------- .../db/engine/storagegroup/TsFileProcessor.java | 80 ++- .../db/engine/storagegroup/TsFileResource.java | 50 +- .../HotCompactionMergeTaskPoolManager.java | 9 +- .../engine/tsfilemanagement/TsFileManagement.java | 238 ++++++++ .../level/LevelTsFileManagement.java | 177 ++---- .../tsfilemanagement/utils/HotCompactionUtils.java | 268 +++++---- .../apache/iotdb/db/engine/upgrade/UpgradeLog.java | 9 +- .../version/SimpleFileVersionController.java | 4 +- .../iotdb/db/engine/version/VersionController.java | 8 +- ...ception.java => DataTypeMismatchException.java} | 14 +- .../exception/metadata/PathNotExistException.java | 14 +- .../exception/query/PathNumOverLimitException.java | 35 ++ .../db/exception/runtime/SQLParserException.java | 5 +- .../org/apache/iotdb/db/metadata/MLogWriter.java | 102 ++-- .../org/apache/iotdb/db/metadata/MManager.java | 96 ++-- .../java/org/apache/iotdb/db/metadata/MTree.java | 42 +- .../org/apache/iotdb/db/metadata/MetaUtils.java | 2 +- .../org/apache/iotdb/db/metadata/PartialPath.java | 10 + .../org/apache/iotdb/db/metadata/TagLogFile.java | 15 +- .../org/apache/iotdb/db/metadata/mnode/MNode.java | 2 +- .../apache/iotdb/db/metrics/server/JettyUtil.java | 4 +- .../iotdb/db/metrics/server/MetricsSystem.java | 2 +- .../iotdb/db/metrics/server/ServerArgument.java | 2 +- .../apache/iotdb/db/metrics/sink/ConsoleSink.java | 2 +- .../iotdb/db/metrics/sink/MetricsServletSink.java | 6 +- .../apache/iotdb/db/metrics/source/JvmSource.java | 2 +- .../iotdb/db/metrics/source/MetricsSource.java | 2 +- .../apache/iotdb/db/metrics/ui/MetricsPage.java | 2 +- .../apache/iotdb/db/metrics/ui/MetricsWebUI.java | 2 +- .../org/apache/iotdb/db/monitor/IStatistic.java | 1 - .../org/apache/iotdb/db/monitor/StatMonitor.java | 17 +- .../apache/iotdb/db/mqtt/BrokerAuthenticator.java | 2 +- .../apache/iotdb/db/mqtt/JSONPayloadFormatter.java | 1 - .../apache/iotdb/db/mqtt/PayloadFormatManager.java | 1 - .../org/apache/iotdb/db/mqtt/PayloadFormatter.java | 1 - .../main/java/org/apache/iotdb/db/qp/Planner.java | 53 +- .../apache/iotdb/db/qp/constant/SQLConstant.java | 3 - .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 3 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 114 ++-- .../db/qp/logical/crud/BasicOperatorType.java | 3 +- .../iotdb/db/qp/logical/crud/QueryOperator.java | 3 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 26 +- .../db/qp/physical/crud/DeletePartitionPlan.java | 3 +- .../iotdb/db/qp/physical/crud/DeletePlan.java | 9 +- .../db/qp/physical/crud/GroupByTimeFillPlan.java | 3 +- .../iotdb/db/qp/physical/crud/InsertPlan.java | 53 +- .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 51 +- .../db/qp/physical/crud/InsertTabletPlan.java | 177 +++++- .../iotdb/db/qp/physical/crud/QueryPlan.java | 1 + .../iotdb/db/qp/physical/crud/UpdatePlan.java | 17 +- .../db/qp/physical/sys/AlterTimeSeriesPlan.java | 144 ++++- .../iotdb/db/qp/physical/sys/AuthorPlan.java | 15 +- .../iotdb/db/qp/physical/sys/ClearCachePlan.java | 4 +- .../apache/iotdb/db/qp/physical/sys/CountPlan.java | 1 - .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 21 +- .../db/qp/physical/sys/CreateSnapshotPlan.java | 5 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 12 +- .../iotdb/db/qp/physical/sys/DataAuthPlan.java | 7 +- .../db/qp/physical/sys/DeleteStorageGroupPlan.java | 6 + .../db/qp/physical/sys/DeleteTimeSeriesPlan.java | 11 + .../apache/iotdb/db/qp/physical/sys/FlushPlan.java | 209 +++++++- .../db/qp/physical/sys/LoadConfigurationPlan.java | 6 +- .../iotdb/db/qp/physical/sys/LoadDataPlan.java | 10 +- .../apache/iotdb/db/qp/physical/sys/MergePlan.java | 5 +- .../iotdb/db/qp/physical/sys/OperateFilePlan.java | 3 +- .../db/qp/physical/sys/SetStorageGroupPlan.java | 15 +- .../iotdb/db/qp/physical/sys/SetTTLPlan.java | 9 +- .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 3 +- .../db/qp/physical/sys/ShowTimeSeriesPlan.java | 4 + .../iotdb/db/qp/physical/sys/TracingPlan.java | 4 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 57 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 50 +- .../qp/strategy/optimizer/ILogicalOptimizer.java | 4 +- .../optimizer/MergeSingleFilterOptimizer.java | 7 +- .../db/query/aggregation/AggregateResult.java | 38 +- .../db/query/aggregation/impl/AvgAggrResult.java | 6 +- .../db/query/aggregation/impl/CountAggrResult.java | 2 +- .../aggregation/impl/FirstValueAggrResult.java | 20 +- ...grResult.java => FirstValueDescAggrResult.java} | 66 +-- .../aggregation/impl/LastValueAggrResult.java | 12 +- ...ggrResult.java => LastValueDescAggrResult.java} | 70 +-- .../query/aggregation/impl/MaxTimeAggrResult.java | 25 +- ...eAggrResult.java => MaxTimeDescAggrResult.java} | 75 +-- .../query/aggregation/impl/MaxValueAggrResult.java | 9 +- .../query/aggregation/impl/MinTimeAggrResult.java | 23 +- ...tAggrResult.java => MinTimeDescAggrResult.java} | 63 +-- .../query/aggregation/impl/MinValueAggrResult.java | 6 +- .../db/query/aggregation/impl/SumAggrResult.java | 2 +- .../iotdb/db/query/control/FileReaderManager.java | 15 +- .../iotdb/db/query/control/QueryFileManager.java | 9 +- .../db/query/control/QueryResourceManager.java | 52 +- .../iotdb/db/query/control/TracingManager.java | 63 ++- .../db/query/dataset/AlignByDeviceDataSet.java | 3 +- .../db/query/dataset/ShowTimeseriesDataSet.java | 6 +- .../dataset/groupby/GroupByEngineDataSet.java | 3 +- .../db/query/dataset/groupby/GroupByExecutor.java | 5 +- .../query/dataset/groupby/GroupByFillDataSet.java | 86 ++- .../groupby/GroupByWithValueFilterDataSet.java | 13 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 2 +- .../dataset/groupby/LocalGroupByExecutor.java | 19 +- .../db/query/executor/AggregationExecutor.java | 89 +-- .../iotdb/db/query/executor/IQueryRouter.java | 10 +- .../db/query/executor/RawDataQueryExecutor.java | 11 +- .../iotdb/db/query/executor/fill/LinearFill.java | 12 +- .../query/externalsort/ExternalSortJobEngine.java | 7 +- .../iotdb/db/query/externalsort/LineMerger.java | 2 +- .../externalsort/SimpleExternalSortEngine.java | 11 +- .../SingleSourceExternalSortJobPart.java | 2 +- .../adapter/ByTimestampReaderAdapter.java | 2 +- .../serialize/IExternalSortFileDeserializer.java | 2 +- .../FixLengthIExternalSortFileDeserializer.java | 4 +- .../impl/FixLengthTimeValuePairSerializer.java | 2 +- .../db/query/factory/AggregateResultFactory.java | 54 +- .../db/query/reader/chunk/ChunkDataIterator.java | 5 +- .../db/query/reader/chunk/ChunkReaderWrap.java | 3 +- .../reader/chunk/DiskChunkReaderByTimestamp.java | 3 +- .../db/query/reader/chunk/MemChunkLoader.java | 3 +- .../db/query/reader/chunk/MemChunkReader.java | 7 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 4 +- .../resource/CachedUnseqResourceMergeReader.java | 5 +- .../reader/series/DescSeriesReaderByTimestamp.java | 53 -- .../db/query/reader/series/IAggregateReader.java | 4 +- .../query/reader/series/SeriesAggregateReader.java | 4 + .../reader/series/SeriesRawDataPointReader.java | 3 +- .../iotdb/db/query/reader/series/SeriesReader.java | 262 +++++++-- .../reader/series/SeriesReaderByTimestamp.java | 25 +- .../universal/CachedPriorityMergeReader.java | 2 +- .../reader/universal/PriorityMergeReader.java | 4 +- .../org/apache/iotdb/db/rescon/MemTablePool.java | 5 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 4 + .../org/apache/iotdb/db/service/MQTTService.java | 5 +- .../org/apache/iotdb/db/service/RPCService.java | 1 - .../apache/iotdb/db/service/RegisterManager.java | 1 - .../org/apache/iotdb/db/service/StaticResps.java | 1 + .../org/apache/iotdb/db/service/TSServiceImpl.java | 109 ++-- .../db/service/thrift/ThriftServiceThread.java | 10 +- .../receiver/recover/SyncReceiverLogAnalyzer.java | 2 +- .../db/sync/receiver/transfer/SyncServiceImpl.java | 19 +- .../db/sync/sender/manage/SyncFileManager.java | 22 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 1 - .../java/org/apache/iotdb/db/tools/WalChecker.java | 1 - .../iotdb/db/tools/memestimation/MemEstTool.java | 1 - .../org/apache/iotdb/db/utils/CommonUtils.java | 16 +- .../iotdb/db/utils/EncodingInferenceUtils.java | 57 ++ .../java/org/apache/iotdb/db/utils/MathUtils.java | 1 - .../apache/iotdb/db/utils/QueryDataSetUtils.java | 13 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 66 ++- .../org/apache/iotdb/db/utils/SerializeUtils.java | 2 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 9 + .../iotdb/db/utils/datastructure/TVList.java | 2 +- .../WALFlushListener.java} | 40 +- .../iotdb/db/writelog/recover/LogReplayer.java | 14 +- .../db/engine/cache/ChunkMetadataCacheTest.java | 2 - .../db/engine/memtable/PrimitiveMemTableTest.java | 9 +- .../iotdb/db/engine/merge/MergeManagerTest.java | 6 +- .../engine/modification/DeletionFileNodeTest.java | 40 +- .../db/engine/modification/DeletionQueryTest.java | 71 ++- .../storagegroup/StorageGroupProcessorTest.java | 105 ++-- .../iotdb/db/engine/storagegroup/TTLTest.java | 6 +- .../iotdb/db/integration/IOTDBGroupByIT.java | 200 ++++++- .../integration/IOTDBGroupByInnerIntervalIT.java | 262 +++++---- .../iotdb/db/integration/IoTDBAddSubDeviceIT.java | 11 +- .../IoTDBAggregationAlignByDeviceIT.java | 113 ---- .../iotdb/db/integration/IoTDBAggregationIT.java | 199 ++++++- .../integration/IoTDBAggregationLargeDataIT.java | 398 ++++++++++---- .../apache/iotdb/db/integration/IoTDBAliasIT.java | 10 +- .../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 102 ++++ .../iotdb/db/integration/IoTDBDeletionIT.java | 29 + .../iotdb/db/integration/IoTDBGroupByFillIT.java | 596 +++++++++++++-------- .../integration/IoTDBGroupByFillWithRangeIT.java | 21 +- .../apache/iotdb/db/integration/IoTDBLastIT.java | 52 +- .../db/integration/IoTDBLoadExternalTsfileIT.java | 1 + .../iotdb/db/integration/IoTDBMergeTest.java | 14 +- .../db/integration/IoTDBSequenceDataQueryIT.java | 9 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 11 +- .../integration/IoTDBSortedShowTimeseriesIT.java | 11 +- .../apache/iotdb/db/integration/IoTDBTtlIT.java | 4 +- .../iotdb/db/metadata/MManagerBasicTest.java | 70 ++- .../org/apache/iotdb/db/metadata/MTreeTest.java | 8 +- .../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 2 +- .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 34 +- .../apache/iotdb/db/qp/plan/SerializationTest.java | 68 ++- .../db/query/aggregation/AggregateResultTest.java | 94 ++-- .../query/executor/GroupByEngineDataSetTest.java | 2 + .../reader/series/SeriesAggregateReaderTest.java | 19 +- .../reader/series/SeriesReaderByTimestampTest.java | 19 +- .../db/query/reader/series/SeriesReaderTest.java | 17 +- .../apache/iotdb/db/sql/CheckPathValidityTest.java | 53 -- .../db/sync/receiver/load/FileLoaderTest.java | 3 + .../apache/iotdb/db/utils/EnvironmentUtils.java | 88 +-- .../iotdb/db/utils/TypeInferenceUtilsTest.java | 4 +- .../apache/iotdb/db/writelog/WriteLogNodeTest.java | 3 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 23 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + .../main/java/org/apache/iotdb/session/Config.java | 3 +- .../java/org/apache/iotdb/session/Session.java | 131 ++++- .../iotdb/session/IoTDBSessionComplexIT.java | 54 +- .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 554 ++++++++++--------- .../apache/iotdb/spark/db/EnvironmentUtils.java | 14 +- thrift/src/main/thrift/cluster.thrift | 17 + tsfile/pom.xml | 20 - .../iotdb/tsfile/common/conf/TSFileConfig.java | 3 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 14 + .../tsfile/file/metadata/TimeseriesMetadata.java | 11 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 28 + .../apache/iotdb/tsfile/read/common/BatchData.java | 16 - .../iotdb/tsfile/read/common/DescBatchData.java | 13 +- .../iotdb/tsfile/read/filter/GroupByFilter.java | 6 +- .../iotdb/tsfile/read/filter/TimeFilter.java | 2 +- .../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 5 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 111 ++-- .../apache/iotdb/tsfile/write/page/PageWriter.java | 4 +- .../apache/iotdb/tsfile/write/record/Tablet.java | 125 +++-- .../tsfile/write/schema/MeasurementSchema.java | 5 + .../tsfile/read/TsFileSequenceReaderTest.java | 40 +- .../tsfile/read/filter/GroupByFilterTest.java | 2 +- .../iotdb/tsfile/utils/ReadWriteIOUtilsTest.java | 140 +++++ 276 files changed, 7567 insertions(+), 3342 deletions(-) diff --cc server/src/assembly/resources/conf/iotdb-engine.properties index a931ce7,82d83f0..5df28f8 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@@ -231,11 -236,15 +236,15 @@@ enable_parameter_adapter=tru # Memory Allocation Ratio: Write, Read, and Free Memory. # The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:3:1 -write_read_free_memory_proportion=6:3:1 +write_read_free_memory_proportion=4:3:3 # primitive array size (length of each array) in array pool -primitive_array_size=128 +primitive_array_size=32 + # allowed max numbers of deduplicated path in one query + # it's just an advised value, the real limitation will be the smaller one between this and the one we calculated + max_deduplicated_path_num=1000 + #################### ### Upgrade Configurations ####################
