This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch cluster_scalability in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 124190cfedf0726d8d6c442848b4ab99be91193d Merge: e5e4411 11edabd Author: lta <[email protected]> AuthorDate: Tue Feb 9 10:28:50 2021 +0800 merge master .github/workflows/client.yml | 98 +++ .github/workflows/main-linux.yml | 70 ++ .github/workflows/main-mac.yml | 46 ++ .github/workflows/{main-ci.yml => main-win.yml} | 69 +- .github/workflows/sonar_and_coverall.yml | 76 -- .travis.yml | 192 ----- Jenkinsfile | 4 +- README.md | 1 + README_ZH.md | 1 + .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 7 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 12 +- cli/src/main/java/org/apache/iotdb/cli/Cli.java | 18 +- .../java/org/apache/iotdb/cli/AbstractCliIT.java | 2 - client-cpp/README.md | 31 + client-cpp/pom.xml | 41 +- client-cpp/src/assembly/client-cpp.xml | 16 +- client-cpp/src/main/CMakeLists.txt | 23 +- client-cpp/src/main/Session.cpp | 173 ++++- client-cpp/src/main/Session.h | 19 +- client-cpp/src/test/CMakeLists.txt | 40 +- client-cpp/src/test/cpp/sessionIT.cpp | 133 +++- client-py/readme.md | 6 + client-py/src/SessionExample.py | 14 +- .../src/{SessionExample.py => SessionTest.py} | 79 ++- client-py/src/iotdb/Session.py | 143 +++- client-py/src/iotdb/utils/IoTDBRpcDataSet.py | 5 +- cluster/pom.xml | 14 + .../java/org/apache/iotdb/cluster/ClientMain.java | 2 +- .../java/org/apache/iotdb/cluster/ClusterMain.java | 4 +- .../iotdb/cluster/log/snapshot/FileSnapshot.java | 38 +- .../iotdb/cluster/partition/PartitionTable.java | 3 +- .../cluster/query/filter/SlotTsFileFilter.java | 16 +- .../org/apache/iotdb/cluster/common/TestUtils.java | 6 +- .../FilePartitionedSnapshotLogManagerTest.java | 1 + .../cluster/log/snapshot/PullSnapshotTaskTest.java | 19 +- .../apache/iotdb/cluster/query/BaseQueryTest.java | 2 + .../query/ClusterPhysicalGeneratorTest.java | 1 + .../cluster/query/ClusterPlanExecutorTest.java | 1 + .../cluster/server/member/DataGroupMemberTest.java | 21 +- .../cluster/server/member/MetaGroupMemberTest.java | 4 +- {client-cpp => code-coverage}/README.md | 12 +- .../copy-code-coverage-sources.sh | 22 +- code-coverage/pom.xml | 132 ++++ compile-tools/README.md | 106 ++- compile-tools/boost/pom.xml | 145 ---- compile-tools/boost/src/assembly/bundle.xml | 35 - compile-tools/pom.xml | 7 +- compile-tools/thrift/pom.xml | 51 +- docs/Community/Feedback.md | 2 + docs/Community/Materials.md | 132 +++- docs/Download/README.md | 18 +- docs/UserGuide/Client/Programming - Native API.md | 8 + .../Concept/Data Model and Terminology.md | 15 +- docs/UserGuide/Concept/Encoding.md | 2 +- docs/UserGuide/Concept/SDT.md | 21 +- .../DDL Data Definition Language.md | 44 +- docs/UserGuide/Operation Manual/SQL Reference.md | 19 +- docs/UserGuide/Server/Cluster Setup.md | 3 - docs/zh/Community/Feedback.md | 4 +- docs/zh/Community/Materials.md | 95 --- docs/zh/Download/README.md | 18 +- docs/zh/SystemDesign/DataQuery/SeriesReader.md | 2 +- docs/zh/SystemDesign/StorageEngine/Compaction.md | 38 +- .../UserGuide/Client/Programming - Native API.md | 10 +- .../Concept/Data Model and Terminology.md | 16 +- docs/zh/UserGuide/Concept/Encoding.md | 2 +- docs/zh/UserGuide/Concept/SDT.md | 17 +- docs/zh/UserGuide/Get Started/Publication.md | 4 +- .../DDL Data Definition Language.md | 46 +- .../zh/UserGuide/Operation Manual/SQL Reference.md | 19 +- docs/zh/UserGuide/Server/Cluster Setup.md | 3 - .../client-cpp-example}/README.md | 28 +- {client-cpp => example}/client-cpp-example/pom.xml | 12 +- .../client-cpp-example/src/CMakeLists.txt | 19 +- .../client-cpp-example/src/SessionExample.cpp | 0 .../apache/iotdb/kafka/KafkaConsumerThread.java | 1 + .../main/java/org/apache/iotdb/SessionExample.java | 20 +- example/udf/pom.xml | 48 +- .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 1 + hadoop/pom.xml | 47 +- .../apache/iotdb/hadoop/fileSystem/HDFSOutput.java | 1 + hive-connector/pom.xml | 53 +- jdbc/pom.xml | 47 +- .../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 5 +- .../main/java/org/apache/iotdb/jdbc/Activator.java | 2 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 80 ++- .../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 3 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 46 +- .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 2 +- .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 16 +- .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 17 +- pom.xml | 105 ++- server/pom.xml | 14 + .../resources/conf/iotdb-engine.properties | 4 +- server/src/assembly/resources/conf/iotdb-env.sh | 9 + .../src/assembly/resources/sbin/start-server.bat | 4 +- server/src/assembly/resources/sbin/start-server.sh | 2 - .../iotdb/db/concurrent/WrappedRunnable.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 7 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 11 +- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 7 + .../db/engine/cache/TimeSeriesMetadataCache.java | 1 + .../engine/compaction/utils/CompactionUtils.java | 6 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 43 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 11 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 36 +- .../iotdb/db/engine/memtable/WritableMemChunk.java | 18 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 15 +- .../db/engine/merge/task/MergeMultiChunkTask.java | 1 + .../iotdb/db/engine/merge/task/MergeTask.java | 5 + .../db/engine/modification/ModificationFile.java | 1 + .../engine/storagegroup/StorageGroupProcessor.java | 200 ++++-- .../metadata/IllegalParameterOfPathException.java} | 66 +- .../org/apache/iotdb/db/metadata/MManager.java | 32 +- .../java/org/apache/iotdb/db/metadata/MTree.java | 80 ++- .../org/apache/iotdb/db/metadata/PartialPath.java | 1 + .../org/apache/iotdb/db/metadata/mnode/MNode.java | 27 +- .../iotdb/db/metrics/source/MetricsSource.java | 10 + .../apache/iotdb/db/qp/executor/PlanExecutor.java | 10 +- .../db/qp/logical/sys/ShowDevicesOperator.java | 9 + .../iotdb/db/qp/physical/crud/LastQueryPlan.java | 1 + .../apache/iotdb/db/qp/physical/sys/CountPlan.java | 1 + .../db/qp/physical/sys/ShowChildPathsPlan.java | 1 + .../iotdb/db/qp/physical/sys/ShowDevicesPlan.java | 9 +- .../db/qp/physical/sys/ShowStorageGroupPlan.java | 1 + .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 5 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 +- .../iotdb/db/query/control/QueryFileManager.java | 20 +- .../db/query/dataset/AlignByDeviceDataSet.java | 2 + .../iotdb/db/query/dataset/ShowDevicesDataSet.java | 21 +- .../dataset/ShowDevicesResult.java} | 18 +- .../dataset/ShowResult.java} | 85 ++- .../db/query/dataset/ShowTimeSeriesResult.java | 22 +- .../db/query/dataset/ShowTimeseriesDataSet.java | 1 + .../iotdb/db/query/executor/LastQueryExecutor.java | 22 +- .../query/reader/series/SeriesAggregateReader.java | 1 + .../iotdb/db/query/reader/series/SeriesReader.java | 8 +- .../apache/iotdb/db/query/udf/builtin/UDTFAbs.java | 1 + .../iotdb/db/query/udf/builtin/UDTFBottomK.java | 1 + .../db/query/udf/builtin/UDTFCommonDerivative.java | 1 + .../udf/builtin/UDTFCommonValueDifference.java | 1 + .../iotdb/db/query/udf/builtin/UDTFTopK.java | 1 + .../apache/iotdb/db/rescon/MemTableManager.java | 39 +- .../iotdb/db/rescon/PrimitiveArrayManager.java | 2 +- .../apache/iotdb/db/rescon/TVListAllocator.java | 14 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 53 +- .../db/sync/receiver/transfer/SyncServiceImpl.java | 18 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1 + .../watermark/GroupedLSBWatermarkEncoder.java | 6 + .../org/apache/iotdb/db/utils/FilePathUtils.java | 77 ++- .../java/org/apache/iotdb/db/utils/MergeUtils.java | 2 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 1 - .../iotdb/db/utils/datastructure/BinaryTVList.java | 4 + .../db/utils/datastructure/BooleanTVList.java | 4 + .../iotdb/db/utils/datastructure/DoubleTVList.java | 4 + .../iotdb/db/utils/datastructure/FloatTVList.java | 4 + .../iotdb/db/utils/datastructure/IntTVList.java | 4 + .../iotdb/db/utils/datastructure/LongTVList.java | 4 + .../iotdb/db/utils/datastructure/TVList.java | 1 + .../iotdb/db/writelog/io/SingleFileLogReader.java | 5 +- .../db/writelog/node/ExclusiveWriteLogNode.java | 1 + .../IoTDBDefaultThreadExceptionHandlerTest.java | 1 + .../db/engine/compaction/CompactionChunkTest.java | 215 ++++++ .../engine/compaction/LevelCompactionLogTest.java | 2 + .../compaction/LevelCompactionMergeTest.java | 2 + .../compaction/LevelCompactionRecoverTest.java | 2 + .../compaction/LevelCompactionSelectorTest.java | 2 + .../LevelCompactionTsFileManagementTest.java | 2 + .../NoCompactionTsFileManagementTest.java | 2 + .../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +- .../iotdb/db/engine/merge/MergeOverLapTest.java | 2 + .../iotdb/db/engine/merge/MergeTaskTest.java | 97 ++- .../storagegroup/StorageGroupProcessorTest.java | 27 + .../iotdb/db/integration/IOTDBGroupByIT.java | 37 +- .../iotdb/db/integration/IoTDBAddSubDeviceIT.java | 42 +- .../iotdb/db/integration/IoTDBCheckConfigIT.java | 157 +++++ .../iotdb/db/integration/IoTDBCompressTypeIT.java | 101 +++ .../iotdb/db/integration/IoTDBDeletionIT.java | 1 - .../iotdb/db/integration/IoTDBEncodingIT.java | 276 ++++++++ .../iotdb/db/integration/IoTDBFilePathUtilsIT.java | 109 +++ .../iotdb/db/integration/IoTDBMergeTest.java | 51 +- .../iotdb/db/integration/IoTDBMetadataFetchIT.java | 37 + .../db/integration/IoTDBNewTsFileCompactionIT.java | 5 + .../db/integration/IoTDBRemovePartitionIT.java | 103 +++ .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 152 +++- .../iotdb/db/metadata/MManagerBasicTest.java | 53 +- .../org/apache/iotdb/db/metadata/MTreeTest.java | 33 +- .../iotdb/db/qp/physical/PhysicalPlanTest.java | 3 +- .../iotdb/db/query/dataset/ListDataSetTest.java | 21 + .../ElasticSerializableRowRecordListTest.java | 2 + .../ElasticSerializableTVListTest.java | 2 + .../SerializableBinaryTVListTest.java | 2 + .../SerializableBooleanTVListTest.java | 2 + .../SerializableDoubleTVListTest.java | 2 + .../datastructure/SerializableFloatTVListTest.java | 2 + .../datastructure/SerializableIntTVListTest.java | 2 + .../datastructure/SerializableLongTVListTest.java | 2 + .../SerializableRowRecordListTest.java | 2 + .../db/sync/receiver/load/FileLoaderTest.java | 3 - .../apache/iotdb/db/utils/FilePathUtilsTest.java | 117 ++++ service-rpc/pom.xml | 20 - .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 9 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + session/pom.xml | 47 +- .../main/java/org/apache/iotdb/session/Config.java | 2 +- .../java/org/apache/iotdb/session/Session.java | 111 ++- .../apache/iotdb/session/SessionConnection.java | 17 +- .../org/apache/iotdb/session/SessionDataSet.java | 8 +- .../iotdb/session/pool/SessionDataSetWrapper.java | 1 + .../org/apache/iotdb/session/pool/SessionPool.java | 88 ++- .../iotdb/session/IoTDBSessionIteratorIT.java | 1 - .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 31 +- .../apache/iotdb/session/SessionCacheLeaderUT.java | 769 +++++++++++++++++++++ .../java/org/apache/iotdb/session/SessionUT.java | 9 + .../apache/iotdb/session/pool/SessionPoolTest.java | 8 +- site/src/main/.vuepress/config.js | 10 +- thrift/pom.xml | 20 - thrift/src/main/thrift/rpc.thrift | 2 +- tsfile/pom.xml | 47 +- .../apache/iotdb/tsfile/compress/ICompressor.java | 83 ++- .../iotdb/tsfile/compress/IUnCompressor.java | 50 ++ .../iotdb/tsfile/encoding/encoder/SDTEncoder.java | 72 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 1 + .../tsfile/file/metadata/TimeseriesMetadata.java | 1 + .../fileInputFactory/HDFSInputFactory.java | 1 + .../fileInputFactory/LocalFSInputFactory.java | 1 + .../fileOutputFactory/LocalFSOutputFactory.java | 1 + .../tsfile/fileSystem/fsFactory/HDFSFactory.java | 11 + .../fileSystem/fsFactory/LocalFSFactory.java | 11 + .../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 1 + .../iotdb/tsfile/read/TsFileSequenceReader.java | 1 + .../reader/series/AbstractFileSeriesReader.java | 1 + .../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 38 + .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 107 ++- .../org/apache/iotdb/tsfile/compress/GZIPTest.java | 88 +++ .../encoding/decoder/LongRleDecoderTest.java | 17 +- .../regular/RegularDataEncoderIntegerTest.java | 2 - .../regular/RegularDataEncoderLongTest.java | 2 - .../read/query/executor/QueryExecutorTest.java | 17 +- .../iotdb/tsfile/read/reader/PageReaderTest.java | 5 - .../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 13 +- zeppelin-interpreter/pom.xml | 49 +- .../zeppelin/iotdb/IoTDBInterpreterTest.java | 13 + 243 files changed, 5712 insertions(+), 1903 deletions(-) diff --cc cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java index 65bb031,c2dba67..4653822 --- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java +++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/DataGroupMemberTest.java @@@ -125,8 -123,8 +125,9 @@@ public class DataGroupMemberTest extend private boolean hasInitialSnapshots; private boolean enableSyncLeader; private int prevReplicationNum; + private int raftId = 0; + @Override @Before public void setUp() throws Exception { prevReplicationNum = ClusterDescriptor.getInstance().getConfig().getReplicationNum(); @@@ -167,9 -169,10 +172,10 @@@ }; } + @Override - DataGroupMember getDataGroupMember(Node node) { - PartitionGroup nodes = partitionTable.getHeaderGroup(node); - return dataGroupMemberMap.computeIfAbsent(node, n -> getDataGroupMember(n, nodes)); + DataGroupMember getDataGroupMember(RaftNode raftNode) { + PartitionGroup nodes = partitionTable.getHeaderGroup(raftNode.getNode()); + return dataGroupMemberMap.computeIfAbsent(raftNode, n -> getDataGroupMember(n.getNode(), nodes)); } private DataGroupMember getDataGroupMember(Node node, PartitionGroup nodes) { @@@ -844,9 -833,10 +850,10 @@@ System.out.println("Start testGetPaths()"); String path = TestUtils.getTestSg(0); AtomicReference<GetAllPathsResult> pathResult = new AtomicReference<>(); - GenericHandler<GetAllPathsResult> handler = new GenericHandler<>(TestUtils.getNode(0), pathResult); + GenericHandler<GetAllPathsResult> handler = new GenericHandler<>(TestUtils.getNode(0), + pathResult); new DataAsyncService(dataGroupMember) - .getAllPaths(TestUtils.getNode(0), Collections.singletonList(path), false, handler); + .getAllPaths(TestUtils.getNode(0), raftId, Collections.singletonList(path), false, handler); List<String> result = pathResult.get().paths; assertEquals(20, result.size()); for (int i = 0; i < 10; i++) {
