This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch change_rpc_port in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a7345e3747536d1241fa64576f67bc63871b16b6 Merge: db3e595 946d4b8 Author: xiangdong huang <[email protected]> AuthorDate: Sun Jan 31 14:47:58 2021 +0800 merge with master .../upgrade/config.properties => .dockerignore | 10 +- .github/pull_request_template.md | 37 +- .github/workflows/client.yml | 98 ++ .github/workflows/e2e.yml | 52 + .github/workflows/main-linux.yml | 70 ++ .github/workflows/main-mac.yml | 46 + .github/workflows/{main-ci.yml => main-win.yml} | 69 +- .github/workflows/sonar_and_coverall.yml | 76 -- .travis.yml | 192 ---- Jenkinsfile | 4 +- LICENSE-binary | 20 +- NOTICE | 2 +- NOTICE-binary | 2 +- README.md | 15 + README_ZH.md | 14 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 31 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 138 +-- cli/src/main/java/org/apache/iotdb/cli/Cli.java | 2 + cli/src/main/java/org/apache/iotdb/cli/WinCli.java | 3 + .../org/apache/iotdb/cli/utils/IoTPrinter.java | 107 ++ .../main/java/org/apache/iotdb/tool/ImportCsv.java | 63 +- client-cpp/README.md | 31 + client-cpp/pom.xml | 41 +- client-cpp/src/assembly/client-cpp.xml | 16 +- client-cpp/src/main/CMakeLists.txt | 23 +- client-cpp/src/main/Session.cpp | 191 +++- client-cpp/src/main/Session.h | 23 +- client-cpp/src/test/CMakeLists.txt | 40 +- client-cpp/src/test/cpp/sessionIT.cpp | 155 ++- client-py/readme.md | 6 + client-py/src/SessionExample.py | 14 +- .../src/{SessionExample.py => SessionTest.py} | 79 +- client-py/src/iotdb/Session.py | 143 ++- client-py/src/iotdb/utils/IoTDBRpcDataSet.py | 5 +- cluster/pom.xml | 43 +- cluster/src/assembly/cluster.xml | 4 +- cluster/src/assembly/resources/sbin/add-node.sh | 2 +- cluster/src/assembly/resources/sbin/start-node.sh | 2 +- .../iotdb/cluster/ClusterFileFlushPolicy.java | 2 +- .../java/org/apache/iotdb/cluster/ClusterMain.java | 4 +- .../apache/iotdb/cluster/RemoteTsFileResource.java | 41 +- .../cluster/client/async/AsyncClientFactory.java | 6 +- .../cluster/client/async/AsyncClientPool.java | 81 +- .../cluster/client/async/AsyncDataClient.java | 11 +- .../iotdb/cluster/client/sync/SyncClientPool.java | 24 +- .../iotdb/cluster/config/ClusterConstant.java | 31 +- .../iotdb/cluster/coordinator/Coordinator.java | 167 ++-- .../apache/iotdb/cluster/log/LogDispatcher.java | 16 +- .../cluster/log/applier/AsyncDataLogApplier.java | 17 +- .../iotdb/cluster/log/applier/BaseApplier.java | 2 +- .../iotdb/cluster/log/applier/DataLogApplier.java | 13 +- .../iotdb/cluster/log/catchup/CatchUpTask.java | 2 +- .../cluster/log/manage/CommittedEntryManager.java | 5 +- .../iotdb/cluster/log/manage/RaftLogManager.java | 26 +- .../log/manage/UnCommittedEntryManager.java | 10 +- .../iotdb/cluster/log/snapshot/FileSnapshot.java | 38 +- .../apache/iotdb/cluster/metadata/CMManager.java | 28 +- .../apache/iotdb/cluster/metadata/MetaPuller.java | 5 +- .../iotdb/cluster/partition/PartitionTable.java | 3 +- .../cluster/query/ClusterPhysicalGenerator.java | 15 +- .../iotdb/cluster/query/ClusterPlanExecutor.java | 53 +- .../iotdb/cluster/query/ClusterPlanRouter.java | 74 ++ .../cluster/query/filter/SlotTsFileFilter.java | 16 +- .../cluster/query/manage/QueryCoordinator.java | 115 +-- .../apache/iotdb/cluster/server/ClientServer.java | 10 +- .../iotdb/cluster/server/DataClusterServer.java | 2 +- .../iotdb/cluster/server/MetaClusterServer.java | 10 + .../apache/iotdb/cluster/server/RaftServer.java | 7 +- .../handlers/caller/AppendNodeEntryHandler.java | 6 +- .../server/handlers/caller/HeartbeatHandler.java | 2 +- .../cluster/server/heartbeat/HeartbeatThread.java | 8 +- .../cluster/server/member/DataGroupMember.java | 19 +- .../cluster/server/member/MetaGroupMember.java | 73 +- .../iotdb/cluster/server/member/RaftMember.java | 126 +-- .../cluster/server/{ => monitor}/NodeReport.java | 3 +- .../manage => server/monitor}/NodeStatus.java | 41 +- .../monitor/NodeStatusManager.java} | 87 +- .../iotdb/cluster/server/{ => monitor}/Peer.java | 2 +- .../iotdb/cluster/server/{ => monitor}/Timer.java | 2 +- .../cluster/server/service/MetaAsyncService.java | 6 + .../cluster/server/service/MetaSyncService.java | 5 + .../apache/iotdb/cluster/utils/ClusterUtils.java | 9 +- .../cluster/utils/nodetool/ClusterMonitor.java | 2 +- .../utils/nodetool/function/NodeToolCmd.java | 22 +- .../iotdb/cluster/common/EnvironmentUtils.java | 218 ----- .../org/apache/iotdb/cluster/common/IoTDBTest.java | 4 +- .../org/apache/iotdb/cluster/common/TestUtils.java | 6 +- .../cluster/integration/BaseSingleNodeTest.java | 2 +- .../iotdb/cluster/integration/SingleNodeTest.java | 7 +- .../iotdb/cluster/log/CommitLogCallbackTest.java | 2 +- .../iotdb/cluster/log/CommitLogTaskTest.java | 2 +- .../iotdb/cluster/log/LogDispatcherTest.java | 2 +- .../log/applier/AsyncDataLogApplierTest.java | 2 +- .../cluster/log/applier/DataLogApplierTest.java | 11 +- .../iotdb/cluster/log/catchup/CatchUpTaskTest.java | 9 +- .../cluster/log/catchup/LogCatchUpTaskTest.java | 8 +- .../log/catchup/SnapshotCatchUpTaskTest.java | 8 +- .../FilePartitionedSnapshotLogManagerTest.java | 1 + .../cluster/log/manage/RaftLogManagerTest.java | 22 +- .../cluster/log/snapshot/DataSnapshotTest.java | 7 +- .../cluster/log/snapshot/PullSnapshotTaskTest.java | 25 +- .../iotdb/cluster/partition/SlotManagerTest.java | 2 +- .../cluster/partition/SlotPartitionTableTest.java | 2 +- .../apache/iotdb/cluster/query/BaseQueryTest.java | 17 +- .../query/ClusterAggregateExecutorTest.java | 48 +- .../query/ClusterDataQueryExecutorTest.java | 22 +- .../cluster/query/ClusterFillExecutorTest.java | 72 +- .../query/ClusterPhysicalGeneratorTest.java | 1 + .../cluster/query/ClusterPlanExecutorTest.java | 9 +- .../cluster/query/ClusterQueryRouterTest.java | 216 +++-- .../iotdb/cluster/query/LoadConfigurationTest.java | 122 +++ .../ClusterGroupByNoVFilterDataSetTest.java | 64 +- .../groupby/ClusterGroupByVFilterDataSetTest.java | 74 +- .../query/groupby/MergeGroupByExecutorTest.java | 83 +- .../query/groupby/RemoteGroupByExecutorTest.java | 146 +-- .../cluster/query/manage/QueryCoordinatorTest.java | 15 +- .../query/reader/ClusterTimeGeneratorTest.java | 34 +- .../cluster/query/reader/DatasourceInfoTest.java | 16 +- .../reader/RemoteSeriesReaderByTimestampTest.java | 122 +-- .../query/reader/RemoteSimpleSeriesReaderTest.java | 136 +-- .../caller/AppendGroupEntryHandlerTest.java | 2 +- .../caller/AppendNodeEntryHandlerTest.java | 4 +- .../handlers/caller/ElectionHandlerTest.java | 2 +- .../handlers/caller/HeartbeatHandlerTest.java | 2 +- .../handlers/caller/LogCatchUpHandlerTest.java | 2 +- .../server/heartbeat/DataHeartbeatThreadTest.java | 5 + .../server/heartbeat/HeartbeatThreadTest.java | 10 +- .../server/heartbeat/MetaHeartbeatThreadTest.java | 5 + .../cluster/server/member/DataGroupMemberTest.java | 156 +-- .../iotdb/cluster/server/member/MemberTest.java | 18 +- .../cluster/server/member/MetaGroupMemberTest.java | 56 +- .../README.md | 12 +- .../copy-code-coverage-sources.sh | 35 +- code-coverage/pom.xml | 132 +++ compile-tools/README.md | 85 +- compile-tools/boost/pom.xml | 145 --- compile-tools/boost/src/assembly/bundle.xml | 35 - compile-tools/pom.xml | 7 +- compile-tools/thrift/pom.xml | 51 +- docker/src/main/Dockerfile | 46 +- docker/src/main/Dockerfile-0.10.0 | 4 +- docker/src/main/Dockerfile-0.10.1 | 4 +- docker/src/main/Dockerfile-0.11.0 | 4 +- .../main/{Dockerfile-0.11.0 => Dockerfile-0.11.1} | 10 +- .../main/{Dockerfile-0.11.0 => Dockerfile-0.11.2} | 10 +- docs/Download/README.md | 33 +- docs/UserGuide/Client/Programming - Native API.md | 26 + docs/UserGuide/Client/Status Codes.md | 2 + .../Concept/Data Model and Terminology.md | 15 +- docs/UserGuide/Concept/SDT.md | 21 +- docs/UserGuide/Operation Manual/Administration.md | 2 + .../DDL Data Definition Language.md | 5 + .../DML Data Manipulation Language.md | 192 +++- docs/UserGuide/Operation Manual/Kill Query.md | 60 ++ docs/UserGuide/Operation Manual/SQL Reference.md | 25 +- .../Operation Manual/UDF User Defined Function.md | 72 ++ docs/UserGuide/Server/Cluster Setup.md | 3 - docs/UserGuide/Server/Config Manual.md | 18 + docs/UserGuide/System Tools/CSV Tool.md | 1 + docs/UserGuide/System Tools/NodeTool.md | 9 +- docs/zh/Download/README.md | 30 +- docs/zh/SystemDesign/StorageEngine/Compaction.md | 40 +- .../UserGuide/Client/Programming - Native API.md | 24 +- docs/zh/UserGuide/Client/Status Codes.md | 2 + .../Concept/Data Model and Terminology.md | 16 +- docs/zh/UserGuide/Concept/SDT.md | 17 +- .../UserGuide/Operation Manual/Administration.md | 2 + .../DDL Data Definition Language.md | 6 + .../DML Data Manipulation Language.md | 198 +++- docs/zh/UserGuide/Operation Manual/Kill Query.md | 61 ++ .../zh/UserGuide/Operation Manual/SQL Reference.md | 31 +- .../Operation Manual/UDF User Defined Function.md | 74 ++ docs/zh/UserGuide/Server/Cluster Setup.md | 3 - docs/zh/UserGuide/Server/Config Manual.md | 17 + docs/zh/UserGuide/System Tools/CSV Tool.md | 1 + docs/zh/UserGuide/System Tools/NodeTool.md | 10 +- .../client-cpp-example}/README.md | 28 +- {client-cpp => example}/client-cpp-example/pom.xml | 12 +- .../client-cpp-example/src/CMakeLists.txt | 19 +- .../client-cpp-example/src/SessionExample.cpp | 13 + .../main/java/org/apache/iotdb/JDBCExample.java | 25 +- .../apache/iotdb/kafka/KafkaConsumerThread.java | 1 + .../main/java/org/apache/iotdb/SessionExample.java | 40 +- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 62 +- example/udf/pom.xml | 48 +- .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 1 + hadoop/pom.xml | 47 +- .../apache/iotdb/hadoop/fileSystem/HDFSInput.java | 11 + .../apache/iotdb/hadoop/fileSystem/HDFSOutput.java | 6 + .../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 4 +- hive-connector/pom.xml | 53 +- jdbc/pom.xml | 47 +- .../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 4 +- .../main/java/org/apache/iotdb/jdbc/Activator.java | 2 + .../main/java/org/apache/iotdb/jdbc/Config.java | 20 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 99 +- .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 18 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 4 +- .../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 5 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 8 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 69 +- .../src/main/java/org/apache/iotdb/jdbc/Utils.java | 7 + .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 16 +- .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 17 +- pom.xml | 119 ++- server/file-changelists/TsFileMods-changelist.md | 11 + .../file-changelists/TsFileResource-changelist.md | 8 +- .../system.properties-changelist.md | 9 + server/pom.xml | 19 + .../resources/conf/iotdb-engine.properties | 24 +- server/src/assembly/resources/conf/iotdb-env.bat | 1 - server/src/assembly/resources/conf/iotdb-env.sh | 10 +- server/src/assembly/resources/conf/logback.xml | 69 +- .../src/assembly/resources/sbin/start-server.bat | 4 +- server/src/assembly/resources/sbin/start-server.sh | 4 +- .../assembly/resources/tools/start-WalChecker.sh | 2 +- .../resources/tools/upgrade/offline-upgrade.bat | 67 -- .../iotdb/db/concurrent/WrappedRunnable.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 67 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 207 +++- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 10 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 + .../org/apache/iotdb/db/engine/StorageEngine.java | 628 ++++++------ .../apache/iotdb/db/engine/cache/ChunkCache.java | 5 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 33 +- .../compaction/CompactionMergeTaskPoolManager.java | 2 +- .../db/engine/compaction/TsFileManagement.java | 16 + .../level/LevelCompactionTsFileManagement.java | 77 +- .../no/NoCompactionTsFileManagement.java | 5 + .../engine/compaction/utils/CompactionLogger.java | 2 +- .../engine/compaction/utils/CompactionUtils.java | 8 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 24 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 219 +++-- .../iotdb/db/engine/memtable/AbstractMemTable.java | 80 +- .../apache/iotdb/db/engine/memtable/IMemTable.java | 17 +- .../db/engine/memtable/PrimitiveMemTable.java | 8 - .../iotdb/db/engine/memtable/WritableMemChunk.java | 18 +- .../merge/selector/MaxFileMergeFileSelector.java | 15 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 151 ++- .../db/engine/merge/task/MergeMultiChunkTask.java | 6 +- .../iotdb/db/engine/merge/task/MergeTask.java | 33 +- .../iotdb/db/engine/modification/Deletion.java | 8 +- .../iotdb/db/engine/modification/Modification.java | 18 +- .../db/engine/modification/ModificationFile.java | 1 + .../io/LocalTextModificationAccessor.java | 10 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 9 +- .../db/engine/storagegroup/StorageGroupInfo.java | 51 +- .../engine/storagegroup/StorageGroupProcessor.java | 868 ++++++++++------- .../db/engine/storagegroup/TsFileProcessor.java | 162 ++-- .../db/engine/storagegroup/TsFileResource.java | 393 +++----- .../storagegroup/timeindex/DeviceTimeIndex.java | 308 ++++++ .../storagegroup/timeindex/FileTimeIndex.java | 193 ++++ .../engine/storagegroup/timeindex/ITimeIndex.java | 138 +++ .../storagegroup/timeindex/TimeIndexLevel.java} | 43 +- .../virtualSg/HashVirtualPartitioner.java | 67 ++ .../virtualSg/VirtualPartitioner.java} | 26 +- .../virtualSg/VirtualStorageGroupManager.java | 434 +++++++++ .../iotdb/db/engine/upgrade/UpgradeTask.java | 108 +-- .../apache/iotdb/db/exception/IoTDBException.java | 21 + .../db/exception/PartitionViolationException.java | 8 +- ...xception.java => QueryIdNotExsitException.java} | 10 +- .../iotdb/db/exception/StorageEngineException.java | 2 +- .../db/exception/UDFRegistrationException.java | 7 +- .../iotdb/db/exception/WriteProcessException.java | 4 + .../metadata/AliasAlreadyExistException.java | 1 + ...n.java => IllegalParameterOfPathException.java} | 11 +- .../exception/metadata/IllegalPathException.java | 1 + .../db/exception/metadata/MetadataException.java | 8 + .../metadata/PathAlreadyExistException.java | 1 + .../exception/metadata/PathNotExistException.java | 20 +- .../metadata/StorageGroupNotSetException.java | 5 + .../db/exception/query/OutOfTTLException.java | 2 +- .../db/exception/query/QueryProcessException.java | 6 +- .../QueryTimeoutRuntimeException.java} | 68 +- .../org/apache/iotdb/db/metadata/MManager.java | 138 +-- .../java/org/apache/iotdb/db/metadata/MTree.java | 144 ++- .../org/apache/iotdb/db/metadata/PartialPath.java | 1 + .../iotdb/db/metadata/logfile/MLogWriter.java | 148 +-- .../org/apache/iotdb/db/metadata/mnode/MNode.java | 27 +- .../iotdb/db/metrics/source/MetricsSource.java | 10 + .../apache/iotdb/db/metrics/ui/MetricsPage.java | 2 +- .../apache/iotdb/db/monitor/MonitorConstants.java | 5 - .../org/apache/iotdb/db/monitor/StatMonitor.java | 20 +- .../org/apache/iotdb/db/mqtt/PublishHandler.java | 123 +-- .../main/java/org/apache/iotdb/db/qp/Planner.java | 1 + .../apache/iotdb/db/qp/constant/SQLConstant.java | 2 + .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 16 + .../apache/iotdb/db/qp/executor/PlanExecutor.java | 158 ++- .../org/apache/iotdb/db/qp/logical/Operator.java | 5 +- ...TracingOperator.java => KillQueryOperator.java} | 20 +- .../db/qp/logical/sys/RemoveFileOperator.java | 5 - .../db/qp/logical/sys/ShowDevicesOperator.java | 18 + .../iotdb/db/qp/logical/sys/TracingOperator.java | 10 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 13 +- .../db/qp/physical/crud/InsertMultiTabletPlan.java | 326 +++++++ .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 29 +- .../physical/crud/InsertRowsOfOneDevicePlan.java | 154 +++ .../db/qp/physical/crud/InsertTabletPlan.java | 10 +- .../iotdb/db/qp/physical/crud/LastQueryPlan.java | 1 + .../apache/iotdb/db/qp/physical/crud/UDFPlan.java | 3 +- .../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 3 +- .../db/qp/physical/sys/AlterTimeSeriesPlan.java | 2 +- .../iotdb/db/qp/physical/sys/AuthorPlan.java | 2 +- .../apache/iotdb/db/qp/physical/sys/CountPlan.java | 1 + .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 6 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 14 +- ...howStorageGroupPlan.java => KillQueryPlan.java} | 80 +- .../db/qp/physical/sys/ShowChildPathsPlan.java | 1 + .../iotdb/db/qp/physical/sys/ShowDevicesPlan.java | 13 +- .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 51 +- .../qp/physical/sys/ShowQueryProcesslistPlan.java | 13 +- .../db/qp/physical/sys/ShowStorageGroupPlan.java | 1 + .../db/qp/physical/sys/ShowTimeSeriesPlan.java | 56 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 34 +- .../iotdb/db/qp/strategy/LogicalGenerator.java | 4 + .../iotdb/db/qp/strategy/PhysicalGenerator.java | 29 +- .../db/qp/{constant => utils}/DatetimeUtils.java | 12 +- .../db/query/aggregation/AggregateResult.java | 5 +- .../db/query/aggregation/impl/AvgAggrResult.java | 22 +- .../db/query/aggregation/impl/SumAggrResult.java | 20 +- .../iotdb/db/query/control/FileReaderManager.java | 16 +- .../iotdb/db/query/control/QueryFileManager.java | 2 + .../db/query/control/QueryResourceManager.java | 3 + .../iotdb/db/query/control/QueryTimeManager.java | 178 ++++ .../iotdb/db/query/control/TracingManager.java | 2 +- .../db/query/dataset/AlignByDeviceDataSet.java | 2 + .../db/query/dataset/NonAlignEngineDataSet.java | 18 +- .../dataset/RawQueryDataSetWithValueFilter.java | 68 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 119 ++- .../apache/iotdb/db/query/dataset/ShowDataSet.java | 78 ++ .../iotdb/db/query/dataset/ShowDevicesDataSet.java | 59 ++ .../db/query/dataset/ShowTimeseriesDataSet.java | 47 +- .../dataset/UDFInputDataSet.java} | 16 +- .../db/query/dataset/UDTFAlignByTimeDataSet.java | 6 +- .../apache/iotdb/db/query/dataset/UDTFDataSet.java | 2 +- .../dataset/groupby/GroupByEngineDataSet.java | 8 +- .../groupby/GroupByWithValueFilterDataSet.java | 3 +- .../db/query/executor/AggregationExecutor.java | 4 +- .../db/query/executor/RawDataQueryExecutor.java | 7 +- .../db/query/executor/fill/LastPointReader.java | 36 +- .../FixLengthIExternalSortFileDeserializer.java | 2 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 7 + .../chunk/metadata/MemChunkMetadataLoader.java | 4 + .../query/reader/series/SeriesAggregateReader.java | 1 + .../iotdb/db/query/reader/series/SeriesReader.java | 29 +- .../reader/universal/DescPriorityMergeReader.java | 4 +- .../reader/universal/PriorityMergeReader.java | 50 +- .../api/customizer/parameter/UDFParameters.java | 6 +- .../strategy/SlidingTimeWindowAccessStrategy.java | 2 +- .../db/query/udf/builtin/BuiltinFunction.java | 39 +- .../iotdb/db/query/udf/builtin/UDTFAbs.java} | 63 +- .../iotdb/db/query/udf/builtin/UDTFAcos.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFAsin.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFAtan.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFBottomK.java | 106 ++ .../iotdb/db/query/udf/builtin/UDTFCeil.java | 14 +- .../db/query/udf/builtin/UDTFCommonDerivative.java | 63 ++ .../udf/builtin/UDTFCommonValueDifference.java | 61 ++ .../iotdb/db/query/udf/builtin/UDTFContains.java} | 118 ++- .../apache/iotdb/db/query/udf/builtin/UDTFCos.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFDegrees.java | 14 +- .../db/query/udf/builtin/UDTFDerivative.java} | 45 +- .../apache/iotdb/db/query/udf/builtin/UDTFExp.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFFloor.java | 14 +- .../apache/iotdb/db/query/udf/builtin/UDTFLog.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFLog10.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFMatches.java} | 119 ++- .../iotdb/db/query/udf/builtin/UDTFMath.java} | 178 ++-- .../udf/builtin/UDTFNonNegativeDerivative.java | 63 ++ .../builtin/UDTFNonNegativeValueDifference.java | 61 ++ .../iotdb/db/query/udf/builtin/UDTFRadians.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFRound.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFSelectK.java | 156 +++ .../iotdb/db/query/udf/builtin/UDTFSign.java | 14 +- .../apache/iotdb/db/query/udf/builtin/UDTFSin.java | 14 +- .../iotdb/db/query/udf/builtin/UDTFSqrt.java | 14 +- .../apache/iotdb/db/query/udf/builtin/UDTFTan.java | 14 +- .../db/query/udf/builtin/UDTFTimeDifference.java} | 40 +- .../iotdb/db/query/udf/builtin/UDTFTopK.java | 104 ++ .../db/query/udf/builtin/UDTFValueDifference.java} | 45 +- .../iotdb/db/query/udf/builtin/UDTFValueTrend.java | 73 ++ .../iotdb/db/query/udf/core/access/RowImpl.java | 23 +- .../iotdb/db/query/udf/core/input/InputLayer.java | 122 ++- .../iotdb/db/query/udf/core/input/SafetyLine.java | 40 +- .../iotdb/db/query/udf/datastructure/Cache.java | 99 ++ .../primitive/ElasticSerializableIntList.java | 25 +- .../row/ElasticSerializableRowRecordList.java | 69 +- .../row/SerializableRowRecordList.java | 98 +- .../tv/ElasticSerializableTVList.java | 28 +- .../query/udf/service/UDFClassLoaderManager.java | 9 +- .../query/udf/service/UDFRegistrationService.java | 20 +- .../apache/iotdb/db/rescon/MemTableManager.java | 39 +- .../iotdb/db/rescon/PrimitiveArrayManager.java | 2 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 28 +- .../apache/iotdb/db/rescon/TVListAllocator.java | 14 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 5 +- .../apache/iotdb/db/service/RegisterManager.java | 17 +- .../org/apache/iotdb/db/service/ServiceType.java | 1 + .../org/apache/iotdb/db/service/TSServiceImpl.java | 1006 +++++++++---------- .../org/apache/iotdb/db/service/UpgradeSevice.java | 3 - .../db/sync/receiver/load/FileLoaderManager.java | 2 +- .../db/sync/receiver/transfer/SyncServiceImpl.java | 18 +- .../iotdb/db/sync/sender/transfer/SyncClient.java | 41 +- .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 4 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 11 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 54 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 480 +++------ .../db/tools/virtualsg/DeviceMappingViewer.java | 61 ++ .../watermark/GroupedLSBWatermarkEncoder.java | 6 + .../db/tools/watermark/WatermarkDetector.java | 2 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 12 +- .../org/apache/iotdb/db/utils/FilePathUtils.java | 90 +- .../java/org/apache/iotdb/db/utils/MergeUtils.java | 2 +- .../java/org/apache/iotdb/db/utils/MmapUtil.java | 19 +- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 14 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 5 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 3 + .../org/apache/iotdb/db/utils/UpgradeUtils.java | 120 ++- .../iotdb/db/utils/datastructure/BinaryTVList.java | 4 + .../db/utils/datastructure/BooleanTVList.java | 4 + .../iotdb/db/utils/datastructure/DoubleTVList.java | 4 + .../iotdb/db/utils/datastructure/FloatTVList.java | 4 + .../iotdb/db/utils/datastructure/IntTVList.java | 4 + .../iotdb/db/utils/datastructure/LongTVList.java | 4 + .../iotdb/db/utils/datastructure/TVList.java | 1 + .../iotdb/db/utils/datastructure/TimeSelector.java | 155 +++ .../writelog/manager/MultiFileLogNodeManager.java | 27 +- .../db/writelog/manager/WriteLogNodeManager.java | 7 +- .../db/writelog/node/ExclusiveWriteLogNode.java | 19 +- .../iotdb/db/writelog/node/WriteLogNode.java | 8 +- .../iotdb/db/writelog/recover/LogReplayer.java | 26 +- .../writelog/recover/TsFileRecoverPerformer.java | 29 +- .../IoTDBDefaultThreadExceptionHandlerTest.java | 1 + .../db/engine/cache/ChunkMetadataCacheTest.java | 2 +- .../db/engine/compaction/CompactionChunkTest.java | 215 ++++ .../engine/compaction/LevelCompactionLogTest.java | 2 + .../compaction/LevelCompactionMergeTest.java | 2 + .../compaction/LevelCompactionRecoverTest.java | 12 +- .../compaction/LevelCompactionSelectorTest.java | 2 + .../LevelCompactionTsFileManagementTest.java | 2 + .../NoCompactionTsFileManagementTest.java | 2 + .../db/engine/memtable/PrimitiveMemTableTest.java | 4 +- .../engine/merge/MaxFileMergeFileSelectorTest.java | 4 +- .../merge/MaxSeriesMergeFileSelectorTest.java | 8 +- .../apache/iotdb/db/engine/merge/MergeLogTest.java | 2 + .../iotdb/db/engine/merge/MergeOverLapTest.java | 3 +- .../iotdb/db/engine/merge/MergeTaskTest.java | 96 +- .../apache/iotdb/db/engine/merge/MergeTest.java | 3 + .../engine/modification/DeletionFileNodeTest.java | 27 +- .../storagegroup/StorageGroupProcessorTest.java | 42 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 34 +- .../engine/storagegroup/TsFileProcessorTest.java | 24 +- .../virtualSg/HashVirtualPartitionerTest.java | 64 ++ .../iotdb/db/integration/IOTDBGroupByIT.java | 1 - .../iotdb/db/integration/IoTDBClearCacheIT.java | 2 +- .../iotdb/db/integration/IoTDBCompleteIT.java | 5 + .../iotdb/db/integration/IoTDBCompressTypeIT.java | 101 ++ .../iotdb/db/integration/IoTDBDeletionIT.java | 39 + .../iotdb/db/integration/IoTDBDisableAlignIT.java | 6 +- .../iotdb/db/integration/IoTDBFilePathUtilsIT.java | 109 +++ .../apache/iotdb/db/integration/IoTDBFillIT.java | 6 + .../db/integration/IoTDBFlushQueryMergeIT.java | 2 +- .../integration/IoTDBGroupByFillWithRangeIT.java | 3 - .../iotdb/db/integration/IoTDBInsertNaNIT.java | 46 +- .../iotdb/db/integration/IoTDBKillQueryTest.java | 84 ++ .../apache/iotdb/db/integration/IoTDBLastIT.java | 1 - .../db/integration/IoTDBLevelCompactionIT.java | 4 - .../db/integration/IoTDBLoadExternalTsfileIT.java | 53 +- .../iotdb/db/integration/IoTDBMergeTest.java | 54 +- .../iotdb/db/integration/IoTDBMultiDeviceIT.java | 322 ++++++ .../iotdb/db/integration/IoTDBMultiSeriesIT.java | 26 +- .../db/integration/IoTDBNewTsFileCompactionIT.java | 1023 ++++++++++++++++++++ .../db/integration/IoTDBQueryTimeoutTest.java | 153 +++ .../db/integration/IoTDBRemovePartitionIT.java | 103 ++ .../iotdb/db/integration/IoTDBRestartIT.java | 16 + .../iotdb/db/integration/IoTDBSensorUpdateIT.java | 3 - .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 87 +- .../iotdb/db/integration/IoTDBUDFManagementIT.java | 31 +- .../db/integration/IoTDBUDTFBuiltinFunctionIT.java | 250 +++++ .../db/integration/IoTDBUDTFHybridQueryIT.java | 6 +- .../db/integration/IoTDBUDTFNonAlignQueryIT.java | 2 +- .../aggregation/IoTDBAggregationIT.java | 24 +- .../aggregation/IoTDBAggregationSmallDataIT.java | 4 +- .../db/integration/auth/IoTDBAuthorizationIT.java | 21 - .../iotdb/db/metadata/MManagerBasicTest.java | 53 +- .../org/apache/iotdb/db/metadata/MTreeTest.java | 56 +- .../iotdb/db/monitor/IoTDBStatMonitorTest.java | 172 ++++ .../java/org/apache/iotdb/db/qp/PlannerTest.java | 59 +- .../qp/{plan => logical}/IndexLogicalPlanTest.java | 2 +- .../qp/{plan => logical}/LogicalPlanSmallTest.java | 3 +- .../qp/{plan => physical}/ConcatOptimizerTest.java | 3 +- .../IndexSubMatchingPhysicalPlanTest.java | 3 +- .../IndexWholeMatchingPhysicalPlanTest.java | 3 +- .../db/qp/physical/InsertTabletMultiPlanTest.java | 99 ++ .../iotdb/db/qp/physical/InsertTabletPlanTest.java | 105 ++ .../db/qp/physical/PhysicalPlanSerializeTest.java | 305 ++++++ .../db/qp/{plan => physical}/PhysicalPlanTest.java | 5 +- .../qp/{plan => physical}/SerializationTest.java | 3 +- .../db/qp/sql/DatetimeQueryDataSetUtilsTest.java | 142 --- .../IoTDBsqlVisitorTest.java} | 4 +- .../db/qp/utils/DatetimeQueryDataSetUtilsTest.java | 190 ++++ .../iotdb/db/query/control/TracingManagerTest.java | 15 +- .../iotdb/db/query/dataset/ListDataSetTest.java | 2 +- .../db/query/reader/series/SeriesReaderTest.java | 22 +- .../query/reader/series/SeriesReaderTestUtil.java | 4 +- .../ElasticSerializableRowRecordListTest.java | 63 +- .../ElasticSerializableTVListTest.java | 2 + .../iotdb/db/query/udf/datastructure/LRUCache.java | 59 ++ .../db/query/udf/datastructure/LRUCacheTest.java | 115 +++ .../SerializableBinaryTVListTest.java | 2 + .../SerializableBooleanTVListTest.java | 2 + .../SerializableDoubleTVListTest.java | 2 + .../datastructure/SerializableFloatTVListTest.java | 2 + .../datastructure/SerializableIntTVListTest.java | 2 + .../udf/datastructure/SerializableListTest.java | 2 + .../datastructure/SerializableLongTVListTest.java | 2 + .../SerializableRowRecordListTest.java | 51 +- .../iotdb/db/query/udf/example/Accumulator.java | 8 +- .../apache/iotdb/db/query/udf/example/Adder.java | 8 +- .../apache/iotdb/db/query/udf/example/Counter.java | 8 +- .../org/apache/iotdb/db/query/udf/example/Max.java | 8 +- .../iotdb/db/query/udf/example/Multiplier.java | 8 +- .../SlidingSizeWindowConstructorTester0.java | 9 +- .../SlidingSizeWindowConstructorTester1.java | 9 +- .../SlidingTimeWindowConstructionTester.java | 9 +- .../db/query/udf/example/TerminateTester.java | 8 +- .../db/sync/receiver/load/FileLoaderTest.java | 43 +- .../recover/SyncReceiverLogAnalyzerTest.java | 6 +- .../org/apache/iotdb/db/tools/MLogParserTest.java | 35 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 13 +- .../apache/iotdb/db/utils/FilePathUtilsTest.java | 117 +++ .../apache/iotdb/db/utils/VersionUtilsTest.java | 55 -- .../db/utils/datastructure/PrecisionTest.java | 22 +- .../db/utils/datastructure/TimeSelectorTest.java | 217 +++++ .../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 25 +- .../apache/iotdb/db/writelog/PerformanceTest.java | 48 +- .../iotdb/db/writelog/WriteLogNodeManagerTest.java | 63 +- .../apache/iotdb/db/writelog/WriteLogNodeTest.java | 71 +- .../db/writelog/recover/DeviceStringTest.java | 8 +- .../iotdb/db/writelog/recover/LogReplayerTest.java | 43 +- .../recover/RecoverResourceFromReaderTest.java | 56 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 57 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 36 +- server/src/test/resources/logback.xml | 1 + service-rpc/pom.xml | 20 - .../org/apache/iotdb/rpc/AutoResizingBuffer.java | 15 +- .../iotdb/rpc/AutoScalingBufferReadTransport.java | 9 + .../iotdb/rpc/AutoScalingBufferWriteTransport.java | 5 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 5 +- .../org/apache/iotdb/rpc/RpcTransportFactory.java | 14 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 15 +- .../rpc/TCompressedElasticFramedTransport.java | 48 +- .../apache/iotdb/rpc/TElasticFramedTransport.java | 45 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + .../iotdb/rpc/TSnappyElasticFramedTransport.java | 2 +- .../rpc/TimeoutChangeableTFastFramedTransport.java | 13 +- .../TimeoutChangeableTSnappyFramedTransport.java | 2 + session/pom.xml | 47 +- .../main/java/org/apache/iotdb/session/Config.java | 16 +- .../java/org/apache/iotdb/session/Session.java | 271 +++++- .../apache/iotdb/session/SessionConnection.java | 37 +- .../org/apache/iotdb/session/SessionDataSet.java | 11 +- .../iotdb/session/pool/SessionDataSetWrapper.java | 1 + .../org/apache/iotdb/session/pool/SessionPool.java | 147 ++- .../iotdb/session/IoTDBSessionComplexIT.java | 3 +- .../iotdb/session/IoTDBSessionIteratorIT.java | 31 +- .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 161 ++- .../apache/iotdb/session/SessionCacheLeaderUT.java | 769 +++++++++++++++ .../java/org/apache/iotdb/session/SessionUT.java | 9 + .../apache/iotdb/session/pool/SessionPoolTest.java | 41 +- site/src/main/.vuepress/config.js | 2 + .../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +- .../e2e/base/docker-compose.yaml | 54 +- test/e2e/cases/README.md | 53 + .../e2e/cases/cli/README.md | 12 +- .../e2e/cases/cli/cleanup.sh | 15 +- .../e2e/cases/cli/docker-compose.yaml | 29 +- test/e2e/cases/cli/res/init.sql | 26 + .../e2e/cases/cli/run.sh | 45 +- thrift/pom.xml | 20 - thrift/rpc-changelist.md | 24 + thrift/src/main/thrift/cluster.thrift | 7 + thrift/src/main/thrift/rpc.thrift | 15 + tsfile/format-changelist.md | 4 +- tsfile/pom.xml | 47 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +- .../apache/iotdb/tsfile/compress/ICompressor.java | 83 +- .../iotdb/tsfile/compress/IUnCompressor.java | 50 + .../iotdb/tsfile/encoding/decoder/Decoder.java | 8 +- .../tsfile/encoding/decoder/FloatDecoder.java | 11 +- .../tsfile/encoding/decoder/IntRleDecoder.java | 11 +- .../tsfile/encoding/decoder/LongRleDecoder.java | 11 +- .../tsfile/encoding/decoder/PlainDecoder.java | 21 +- .../iotdb/tsfile/encoding/decoder/RleDecoder.java | 15 +- .../tsfile/encoding/encoder/FloatEncoder.java | 6 +- .../tsfile/encoding/encoder/IntRleEncoder.java | 8 +- .../tsfile/encoding/encoder/LongRleEncoder.java | 6 +- .../tsfile/encoding/encoder/PlainEncoder.java | 57 +- .../iotdb/tsfile/encoding/encoder/RleEncoder.java | 20 +- .../iotdb/tsfile/encoding/encoder/SDTEncoder.java | 72 +- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 13 +- .../write/UnSupportedDataTypeException.java | 4 +- .../org/apache/iotdb/tsfile/file/MetaMarker.java | 17 +- .../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 156 --- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 117 +++ .../iotdb/tsfile/file/header/ChunkHeader.java | 151 +-- .../iotdb/tsfile/file/header/PageHeader.java | 54 +- .../iotdb/tsfile/file/header/package-info.java | 30 - .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 36 +- .../file/metadata/MetadataIndexConstructor.java | 5 +- .../tsfile/file/metadata/MetadataIndexEntry.java | 4 +- .../tsfile/file/metadata/MetadataIndexNode.java | 12 +- .../tsfile/file/metadata/TimeseriesMetadata.java | 37 +- .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 76 +- .../file/metadata/enums/CompressionType.java | 64 +- .../file/metadata/enums/MetadataIndexNodeType.java | 57 +- .../tsfile/file/metadata/enums/TSDataType.java | 113 +-- .../tsfile/file/metadata/enums/TSEncoding.java | 94 +- .../file/metadata/statistics/BinaryStatistics.java | 13 +- .../metadata/statistics/BooleanStatistics.java | 48 +- .../file/metadata/statistics/DoubleStatistics.java | 23 +- .../file/metadata/statistics/FloatStatistics.java | 23 +- .../metadata/statistics/IntegerStatistics.java | 48 +- .../file/metadata/statistics/LongStatistics.java | 31 +- .../file/metadata/statistics/Statistics.java | 22 +- .../fileInputFactory/HDFSInputFactory.java | 1 + .../fileInputFactory/LocalFSInputFactory.java | 1 + .../fileOutputFactory/LocalFSOutputFactory.java | 1 + .../tsfile/fileSystem/fsFactory/HDFSFactory.java | 11 + .../fileSystem/fsFactory/LocalFSFactory.java | 11 + .../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 1 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 526 ++++++---- .../apache/iotdb/tsfile/read/common/BatchData.java | 2 +- .../org/apache/iotdb/tsfile/read/common/Chunk.java | 99 +- .../read/controller/CachedChunkLoaderImpl.java | 3 +- .../iotdb/tsfile/read/reader/LocalTsFileInput.java | 20 + .../iotdb/tsfile/read/reader/TsFileInput.java | 5 + .../tsfile/read/reader/chunk/ChunkReader.java | 105 +- .../iotdb/tsfile/read/reader/page/PageReader.java | 14 +- .../reader/series/AbstractFileSeriesReader.java | 1 + .../tsfile/utils/ReadWriteForEncodingUtils.java | 96 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 103 +- .../apache/iotdb/tsfile/utils/VersionUtils.java | 49 - .../v1/file/metadata/ChunkGroupMetaDataV1.java | 118 --- .../tsfile/v1/file/metadata/ChunkMetadataV1.java | 131 --- .../v1/file/metadata/TimeseriesMetadataForV1.java | 42 - .../v1/file/metadata/TsDeviceMetadataIndexV1.java | 78 -- .../v1/file/metadata/TsDeviceMetadataV1.java | 87 -- .../iotdb/tsfile/v1/file/metadata/TsDigestV1.java | 75 -- .../tsfile/v1/file/metadata/TsFileMetadataV1.java | 106 -- .../metadata/statistics/BinaryStatisticsV1.java | 84 -- .../metadata/statistics/BooleanStatisticsV1.java | 80 -- .../metadata/statistics/DoubleStatisticsV1.java | 79 -- .../metadata/statistics/FloatStatisticsV1.java | 79 -- .../metadata/statistics/IntegerStatisticsV1.java | 79 -- .../file/metadata/statistics/LongStatisticsV1.java | 80 -- .../v1/file/metadata/statistics/StatisticsV1.java | 225 ----- .../iotdb/tsfile/v1/file/utils/HeaderUtils.java | 141 --- .../tsfile/v1/read/TsFileSequenceReaderForV1.java | 409 -------- .../tsfile/v2/file/footer/ChunkGroupFooterV2.java | 86 ++ .../iotdb/tsfile/v2/file/header/ChunkHeaderV2.java | 108 +++ .../iotdb/tsfile/v2/file/header/PageHeaderV2.java | 51 + .../tsfile/v2/file/metadata/ChunkMetadataV2.java | 52 + .../v2/file/metadata/MetadataIndexEntryV2.java | 72 +- .../v2/file/metadata/MetadataIndexNodeV2.java | 47 + .../v2/file/metadata/TimeseriesMetadataV2.java | 42 + .../tsfile/v2/file/metadata/TsFileMetadataV2.java | 75 ++ .../v2/file/metadata/statistics/StatisticsV2.java | 104 ++ .../tsfile/v2/read/TsFileSequenceReaderForV2.java | 581 +++++++++++ .../tsfile/v2/read/reader/page/PageReaderV2.java | 85 ++ .../apache/iotdb/tsfile/write/TsFileWriter.java | 20 +- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 128 +-- .../iotdb/tsfile/write/chunk/IChunkWriter.java | 11 +- .../apache/iotdb/tsfile/write/page/PageWriter.java | 30 +- .../tsfile/write/schema/MeasurementSchema.java | 38 +- .../write/writer/ForceAppendTsFileWriter.java | 4 - .../tsfile/write/writer/LocalTsFileOutput.java | 5 + .../write/writer/RestorableTsFileIOWriter.java | 7 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 77 +- .../iotdb/tsfile/write/writer/TsFileOutput.java | 8 + .../org/apache/iotdb/tsfile/compress/GZIPTest.java | 88 ++ .../tsfile/encoding/decoder/IntRleDecoderTest.java | 83 +- .../encoding/decoder/LongRleDecoderTest.java | 34 +- .../regular/RegularDataEncoderIntegerTest.java | 2 - .../regular/RegularDataEncoderLongTest.java | 2 - .../iotdb/tsfile/file/header/PageHeaderTest.java | 2 +- .../metadata/statistics/DoubleStatisticsTest.java | 10 +- .../metadata/statistics/FloatStatisticsTest.java | 11 +- .../metadata/statistics/IntegerStatisticsTest.java | 10 +- .../metadata/statistics/LongStatisticsTest.java | 15 +- .../tsfile/file/metadata/utils/TestHelper.java | 9 - .../iotdb/tsfile/file/metadata/utils/Utils.java | 36 +- .../iotdb/tsfile/read/GetAllDevicesTest.java | 14 +- .../org/apache/iotdb/tsfile/read/ReadTest.java | 22 +- .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 5 - .../tsfile/read/TimeSeriesMetadataReadTest.java | 87 ++ .../tsfile/read/TsFileSequenceReaderTest.java | 66 +- .../read/query/executor/QueryExecutorTest.java | 18 +- .../iotdb/tsfile/read/reader/PageReaderTest.java | 59 +- .../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 13 +- .../apache/iotdb/tsfile/utils/FileGenerator.java | 22 +- .../utils/ReadWriteForEncodingUtilsTest.java | 54 ++ .../tsfile/write/DefaultDeviceTemplateTest.java | 110 +++ .../iotdb/tsfile/write/TsFileIOWriterTest.java | 26 +- .../iotdb/tsfile/write/TsFileWriterTest.java | 1 - .../iotdb/tsfile/write/writer/PageWriterTest.java | 54 +- .../write/writer/RestorableTsFileIOWriterTest.java | 30 +- zeppelin-interpreter/pom.xml | 51 +- .../apache/zeppelin/iotdb/IoTDBInterpreter.java | 212 ++-- .../zeppelin/iotdb/IoTDBInterpreterTest.java | 108 ++- 711 files changed, 23198 insertions(+), 11237 deletions(-) diff --cc cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConstant.java index 3ff7772,8d3a4da..2487249 --- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConstant.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConstant.java @@@ -40,5 -41,27 +41,27 @@@ public class ClusterConstant // constant class } - static final String CLUSTER_CONF = "CLUSTER_CONF"; + public static final String CLUSTER_CONF = "CLUSTER_CONF"; + + /** + * a failed election will restart in 2s~5s, this should be at least as long as a heartbeat + * interval, or a stale node may frequently issue elections and thus makes the leader step down + */ + public static long getElectionLeastTimeOutMs() { + return electionLeastTimeOutMs; + } + + public static long getElectionRandomTimeOutMs() { + return electionRandomTimeOutMs; + } + + @TestOnly + public static void setElectionLeastTimeOutMs(long electionLeastTimeOutMs) { + ClusterConstant.electionLeastTimeOutMs = electionLeastTimeOutMs; + } + + @TestOnly + public static void setElectionRandomTimeOutMs(long electionRandomTimeOutMs) { + ClusterConstant.electionRandomTimeOutMs = electionRandomTimeOutMs; + } } diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java index c7c7b77,74b9d3b..aa7ba59 --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java @@@ -263,11 -264,10 +264,11 @@@ public class MetaGroupMember extends Ra public MetaGroupMember() { } + public MetaGroupMember(TProtocolFactory factory, Node thisNode, Coordinator coordinator) throws QueryProcessException { - super("Meta", new AsyncClientPool("Meta", new AsyncMetaClient.FactoryAsync(factory)), + super("Meta", new AsyncClientPool(new AsyncMetaClient.FactoryAsync(factory)), new SyncClientPool(new SyncMetaClient.FactorySync(factory)), - new AsyncClientPool("Meta", new AsyncMetaHeartbeatClient.FactoryAsync(factory), false), + new AsyncClientPool(new AsyncMetaHeartbeatClient.FactoryAsync(factory)), new SyncClientPool(new SyncMetaHeartbeatClient.FactorySync(factory))); allNodes = new ArrayList<>(); initPeerMap(); diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java index 23b4f5b,fc83ba4..35742fa --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java @@@ -1276,28 -1251,13 +1251,12 @@@ public abstract class RaftMember if (ClusterConstant.EMPTY_NODE.equals(node) || node == null) { return null; } -- - AsyncClient client = null; - IOException lastException = null; - for (int i = 0; i < MAX_RETRY_TIMES_FOR_GET_CLIENT; i++) { - try { - client = pool.getClient(node); - if (!ClientUtils.isClientReady(client)) { - Thread.sleep(SYNC_CLIENT_TIMEOUT_MS); - } else { - return client; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return null; - } catch (IOException e) { - lastException = e; - } - } - if (logger.isDebugEnabled() && client == null && lastException != null) { - logger.debug("{} cannot connect to node {}", name, node, lastException); + try { + return pool.getClient(node, activatedOnly); + } catch (IOException e) { + logger.warn("{} cannot connect to node {}", name, node, e); + return null; } - return client; } /** diff --cc cluster/src/test/java/org/apache/iotdb/cluster/integration/BaseSingleNodeTest.java index 605aa77,a1e48da..55dbfd6 --- a/cluster/src/test/java/org/apache/iotdb/cluster/integration/BaseSingleNodeTest.java +++ b/cluster/src/test/java/org/apache/iotdb/cluster/integration/BaseSingleNodeTest.java @@@ -21,10 -21,9 +21,10 @@@ package org.apache.iotdb.cluster.integr import java.util.Collections; import java.util.List; - import org.apache.iotdb.cluster.common.EnvironmentUtils; + import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.cluster.config.ClusterDescriptor; import org.apache.iotdb.cluster.server.MetaClusterServer; +import org.apache.iotdb.cluster.utils.Constants; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.session.Session; import org.junit.After; diff --cc cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java index cd4df36,b04636b..fdc19ed --- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java +++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java @@@ -91,9 -90,9 +90,10 @@@ import org.apache.iotdb.cluster.server. import org.apache.iotdb.cluster.server.Response; import org.apache.iotdb.cluster.server.handlers.caller.GenericHandler; import org.apache.iotdb.cluster.server.heartbeat.DataHeartbeatServer; + import org.apache.iotdb.cluster.server.monitor.NodeStatusManager; import org.apache.iotdb.cluster.server.service.MetaAsyncService; import org.apache.iotdb.cluster.utils.ClusterUtils; +import org.apache.iotdb.cluster.utils.Constants; import org.apache.iotdb.cluster.utils.StatusUtils; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.authorizer.IAuthorizer; diff --cc docs/UserGuide/Server/Cluster Setup.md index 2fcaa5b,2a47ea7..936c1de --- a/docs/UserGuide/Server/Cluster Setup.md +++ b/docs/UserGuide/Server/Cluster Setup.md @@@ -72,13 -72,10 +72,10 @@@ When both exist, the specified configur > sed -i -e 's/31999/32000/g' cluster\target\cluster-0.11.0-SNAPSHOT\conf\iotdb-env.bat > sed -i -e 's/31999/32001/g' cluster\target\cluster-0.11.0-SNAPSHOT\conf\iotdb-env.bat > nohup cluster\target\cluster-0.11.0-SNAPSHOT\sbin\start-node.bat -> nohup cluster\target\cluster-0.11.0-SNAPSHOT1\sbin\start-node.bat -internal_meta_port 9005 -internal_data_port 40012 -cluster_rpc_port 55561 -> nohup cluster\target\cluster-0.11.0-SNAPSHOT2\sbin\start-node.bat -internal_meta_port 9007 -internal_data_port 40014 -cluster_rpc_port 55562 +> nohup cluster\target\cluster-0.11.0-SNAPSHOT1\sbin\start-node.bat -internal_meta_port 9005 -internal_data_port 40012 -cluster_rpc_port 6668 +> nohup cluster\target\cluster-0.11.0-SNAPSHOT2\sbin\start-node.bat -internal_meta_port 9007 -internal_data_port 40014 -cluster_rpc_port 6669 ``` - Note: The distributed version uses the 707 identifier to instruct the client to do the appropriate metadata cache so that the data can then be sent directly to the leader of the corresponding data group later. - Therefore, it is recommended to re-install `mvn install -pl jdbc -am -Dmaven.test.skip=true` and `mvn install -pl jdbc -am -Dmaven.test.skip=true` on current branch to update the latest client. - ## OverWrite the configurations of Stand-alone node Some configurations in the iotdb-engines.properties will be ignored diff --cc docs/zh/UserGuide/Server/Cluster Setup.md index 8ab0352,f5678e2..04c13f4 --- a/docs/zh/UserGuide/Server/Cluster Setup.md +++ b/docs/zh/UserGuide/Server/Cluster Setup.md @@@ -66,13 -66,10 +66,10 @@@ o > sed -i -e 's/31999/32000/g' cluster\target\cluster-0.11.0-SNAPSHOT\conf\iotdb-env.bat > sed -i -e 's/31999/32001/g' cluster\target\cluster-0.11.0-SNAPSHOT\conf\iotdb-env.bat > nohup cluster\target\cluster-0.11.0-SNAPSHOT\sbin\start-node.bat -> nohup cluster\target\cluster-0.11.0-SNAPSHOT1\sbin\start-node.bat -internal_meta_port 9005 -internal_data_port 40012 -cluster_rpc_port 55561 -> nohup cluster\target\cluster-0.11.0-SNAPSHOT2\sbin\start-node.bat -internal_meta_port 9007 -internal_data_port 40014 -cluster_rpc_port 55562 +> nohup cluster\target\cluster-0.11.0-SNAPSHOT1\sbin\start-node.bat -internal_meta_port 9005 -internal_data_port 40012 -cluster_rpc_port 6668 +> nohup cluster\target\cluster-0.11.0-SNAPSHOT2\sbin\start-node.bat -internal_meta_port 9007 -internal_data_port 40014 -cluster_rpc_port 6669 ``` - 注：分布式版使用了707标识符来指示客户端做相应的元数据缓存以便之后能够直接将数据发送给对应数据组的 leader。因此建议在当前分支重新 `mvn install -pl jdbc -am -Dmaven.test.skip=true` 和 - `mvn install -pl session -am -Dmaven.test.skip=true`以更新最新的客户端。 - ## 被覆盖的单机版选项 iotdb-engines.properties配置文件中的部分内容会不再生效： diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index a5fe156,988469a..b1c1c95 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@@ -834,11 -849,14 +849,20 @@@ public class IoTDBConfig private boolean debugState = false; /** + * whether enable the rpc service. This parameter has no a corresponding field + * in the iotdb-engine.properties + */ + private boolean enableRPCService = true; + ++ /** + * the size of ioTaskQueue + */ + private int ioTaskQueueSizeForFlushing = 10; + + /** + * the number of virtual storage groups per user-defined storage group + */ + private int virtualStorageGroupNum = 1; public IoTDBConfig() { // empty constructor @@@ -2236,11 -2278,11 +2284,20 @@@ this.mlogBufferSize = mlogBufferSize; } ++ + public boolean isEnableRPCService() { + return enableRPCService; + } + + public void setEnableRPCService(boolean enableRPCService) { + this.enableRPCService = enableRPCService; + } ++ + public int getIoTaskQueueSizeForFlushing() { + return ioTaskQueueSizeForFlushing; + } + + public void setIoTaskQueueSizeForFlushing(int ioTaskQueueSizeForFlushing) { + this.ioTaskQueueSizeForFlushing = ioTaskQueueSizeForFlushing; + } } diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 76dede8,38a0693..0abb3f7 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@@ -42,6 -45,7 +42,7 @@@ import org.slf4j.LoggerFactory public class IoTDBDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class); - private static CommandLine commandLine; ++ private IoTDBConfig conf = new IoTDBConfig(); protected IoTDBDescriptor() { @@@ -57,6 -60,40 +58,10 @@@ return conf; } - public void replaceProps(String[] params) { - Options options = new Options(); - final String RPC_PORT = "rpc_port"; - Option rpcPort = new Option(RPC_PORT, RPC_PORT, true, - "The jdbc service listens on the port"); - rpcPort.setRequired(false); - options.addOption(rpcPort); - - boolean ok = parseCommandLine(options, params); - if (!ok) { - logger.error("replaces properties failed, use default conf params"); - } else { - if (commandLine.hasOption(RPC_PORT)) { - conf.setRpcPort(Integer.parseInt(commandLine.getOptionValue(RPC_PORT))); - logger.debug("replace rpc port with={}", conf.getRpcPort()); - } - } - } - - private boolean parseCommandLine(Options options, String[] params) { - try { - CommandLineParser parser = new DefaultParser(); - commandLine = parser.parse(options, params); - } catch (ParseException e) { - logger.error("parse conf params failed, {}", e.toString()); - return false; - } - return true; - } - + /** + * get props url location + * @return url object if location exit, otherwise null. + */ public URL getPropsUrl() { // Check if a config-directory was specified first. String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
