This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch cluster_multi_raft in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1aea5a7d6f782f6d472ccf1dcfda5d24b37481c7 Merge: 7ee34da 923cf57 Author: lta <[email protected]> AuthorDate: Mon Dec 21 16:20:06 2020 +0800 merge master .github/workflows/sonar_and_coverall.yml | 2 +- .travis.yml | 2 +- Jenkinsfile | 29 +- LICENSE-binary | 2 - .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 12 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 110 +- cli/src/main/java/org/apache/iotdb/cli/Cli.java | 5 +- cli/src/main/java/org/apache/iotdb/cli/WinCli.java | 5 +- .../main/java/org/apache/iotdb/tool/ExportCsv.java | 4 +- .../java/org/apache/iotdb/cli/AbstractCliIT.java | 35 +- cluster/src/assembly/resources/sbin/add-node.bat | 6 +- cluster/src/assembly/resources/sbin/add-node.sh | 6 +- .../cluster/partition/slot/SlotPartitionTable.java | 36 +- compile-tools/boost/pom.xml | 4 + docs/SystemDesign/SchemaManager/SchemaManager.md | 20 +- docs/SystemDesign/TsFile/Format.md | 4 +- .../DML Data Manipulation Language.md | 100 +- docs/UserGuide/Operation Manual/SQL Reference.md | 7 +- docs/UserGuide/Server/Cluster Setup.md | 12 +- docs/UserGuide/Server/ServerFileList.md | 1 - .../UserGuide/System Tools/MLogParser Tool.md | 21 +- docs/UserGuide/System Tools/Sync Tool.md | 2 +- .../zh/SystemDesign/SchemaManager/SchemaManager.md | 20 +- docs/zh/SystemDesign/StorageEngine/Compaction.md | 10 +- docs/zh/SystemDesign/TsFile/Format.md | 4 +- docs/zh/UserGuide/Concept/SDT.md | 22 +- .../DML Data Manipulation Language.md | 103 +- .../zh/UserGuide/Operation Manual/SQL Reference.md | 7 +- docs/zh/UserGuide/Server/Cluster Setup.md | 16 +- docs/zh/UserGuide/Server/ServerFileList.md | 1 - .../zh/UserGuide/System Tools/MLogParser Tool.md | 21 +- docs/zh/UserGuide/System Tools/Sync Tool.md | 2 +- hive-connector/pom.xml | 6 +- pom.xml | 12 +- server/file-changelists/mlog-changelist.md | 4 + server/pom.xml | 5 - .../resources/conf/iotdb-engine.properties | 11 +- .../assembly/resources/tools/mlog/mLogParser.bat | 62 +- .../assembly/resources/tools/mlog/mLogParser.sh | 40 +- .../org/apache/iotdb/db/auth/AuthException.java | 5 - .../iotdb/db/auth/authorizer/BasicAuthorizer.java | 2 +- .../db/auth/authorizer/LocalFileAuthorizer.java | 2 +- .../iotdb/db/auth/authorizer/OpenIdAuthorizer.java | 10 +- .../apache/iotdb/db/auth/entity/PathPrivilege.java | 4 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 35 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 + .../db/engine/cache/TimeSeriesMetadataCache.java | 10 +- .../level/LevelCompactionTsFileManagement.java | 152 +-- .../no/NoCompactionTsFileManagement.java | 10 +- .../compaction/utils/CompactionLogAnalyzer.java | 19 +- .../engine/compaction/utils/CompactionLogger.java | 16 +- .../engine/compaction/utils/CompactionUtils.java | 84 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 1 + .../db/engine/storagegroup/TsFileResource.java | 14 +- .../{MLogWriter.java => MLogTxtWriter.java} | 72 +- .../org/apache/iotdb/db/metadata/MManager.java | 236 ++-- .../java/org/apache/iotdb/db/metadata/MTree.java | 127 ++- .../apache/iotdb/db/metadata/MetadataConstant.java | 10 +- .../iotdb/db/metadata/logfile/MLogReader.java | 74 ++ .../iotdb/db/metadata/logfile/MLogTxtReader.java | 79 ++ .../iotdb/db/metadata/logfile/MLogWriter.java | 454 ++++++++ .../org/apache/iotdb/db/metadata/mnode/MNode.java | 25 +- .../iotdb/db/metadata/mnode/MeasurementMNode.java | 37 +- .../iotdb/db/metadata/mnode/StorageGroupMNode.java | 22 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +- .../org/apache/iotdb/db/qp/logical/Operator.java | 5 +- .../iotdb/db/qp/logical/crud/QueryOperator.java | 18 + .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 28 +- .../iotdb/db/qp/physical/crud/GroupByTimePlan.java | 19 + ...tStorageGroupPlan.java => ChangeAliasPlan.java} | 74 +- ...rageGroupPlan.java => ChangeTagOffsetPlan.java} | 76 +- .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 4 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 66 +- .../db/qp/physical/sys/DeleteTimeSeriesPlan.java | 4 + .../{SetStorageGroupPlan.java => MNodePlan.java} | 88 +- .../db/qp/physical/sys/MeasurementMNodePlan.java | 143 +++ .../db/qp/physical/sys/SetStorageGroupPlan.java | 18 +- .../iotdb/db/qp/physical/sys/SetTTLPlan.java | 1 - ...geGroupPlan.java => StorageGroupMNodePlan.java} | 79 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 18 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 2 + .../iotdb/db/query/control/TracingManager.java | 2 +- .../db/query/dataset/ShowTimeseriesDataSet.java | 7 +- .../dataset/groupby/GroupByEngineDataSet.java | 70 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 18 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 5 +- .../db/sync/receiver/transfer/SyncServiceImpl.java | 37 +- .../org/apache/iotdb/db/tools/mlog/MLogParser.java | 194 ++++ .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1 + .../java/org/apache/iotdb/db/utils/QueryUtils.java | 1 - .../org/apache/iotdb/db/utils/SerializeUtils.java | 12 +- .../iotdb/db/utils/datastructure/TVList.java | 6 +- .../org/apache/iotdb/db/writelog/io/LogWriter.java | 56 +- .../db/writelog/node/ExclusiveWriteLogNode.java | 13 +- .../apache/iotdb/db/auth/AuthorityCheckerTest.java | 147 +++ .../{ => authorizer}/LocalFileAuthorizerTest.java | 295 +++-- .../db/auth/authorizer/OpenIdAuthorizerTest.java | 107 +- .../iotdb/db/auth/entity/PathPrivilegeTest.java | 42 + .../org/apache/iotdb/db/auth/entity/RoleTest.java} | 41 +- .../org/apache/iotdb/db/auth/entity/UserTest.java | 44 + .../auth/{ => role}/LocalFileRoleAccessorTest.java | 13 +- .../auth/{ => role}/LocalFileRoleManagerTest.java | 45 +- .../auth/{ => user}/LocalFileUserAccessorTest.java | 13 +- .../auth/{ => user}/LocalFileUserManagerTest.java | 69 +- .../engine/compaction/LevelCompactionLogTest.java | 82 ++ .../compaction/LevelCompactionMergeTest.java | 142 +++ .../compaction/LevelCompactionRecoverTest.java | 507 +++++++++ .../compaction/LevelCompactionSelectorTest.java | 73 ++ .../LevelCompactionTest.java} | 68 +- .../LevelCompactionTsFileManagementTest.java | 120 ++ .../NoCompactionTsFileManagementTest.java | 127 +++ .../apache/iotdb/db/engine/merge/MergeTest.java | 2 +- .../iotdb/db/integration/IOTDBGroupByIT.java | 101 ++ .../apache/iotdb/db/integration/IOTDBInsertIT.java | 4 +- .../iotdb/db/integration/IoTDBAddSubDeviceIT.java | 15 +- .../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 31 +- .../org/apache/iotdb/db/integration/IoTDBAsIT.java | 10 + .../db/integration/IoTDBAutoCreateSchemaIT.java | 19 +- .../iotdb/db/integration/IoTDBCompactionIT.java | 154 --- .../db/integration/IoTDBCreateSnapshotIT.java | 74 +- .../iotdb/db/integration/IoTDBDeletionIT.java | 18 + .../apache/iotdb/db/integration/IoTDBLastIT.java | 13 +- .../db/integration/IoTDBLevelCompactionIT.java | 1180 ++++++++++++++++++++ .../iotdb/db/integration/IoTDBRestartIT.java | 2 - .../iotdb/db/integration/IoTDBResultSetIT.java | 4 +- .../IoTDBAggregationByLevelIT.java | 9 +- .../IoTDBAggregationDeleteIT.java | 16 +- .../{ => aggregation}/IoTDBAggregationIT.java | 2 +- .../IoTDBAggregationLargeDataIT.java | 247 +--- .../IoTDBAggregationSmallDataIT.java | 179 +-- .../{ => auth}/IoTDBAuthorizationIT.java | 2 +- .../iotdb/db/mqtt/BrokerAuthenticatorTest.java | 16 + .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 9 +- .../db/query/aggregation/AggregateResultTest.java | 19 +- .../query/aggregation/DescAggregateResultTest.java | 126 +++ .../iotdb/db/query/control/TracingManagerTest.java | 111 ++ .../iotdb/db/query/dataset/ListDataSetTest.java | 126 +++ .../db/query/dataset/ShowTimeSeriesResultTest.java | 57 + .../iotdb/db/query/dataset/SingleDataSetTest.java | 133 +++ .../dataset/groupby/GroupByEngineDataSetTest.java | 476 ++++++++ .../{ => groupby}/GroupByFillDataSetTest.java | 2 +- .../{ => groupby}/GroupByLevelDataSetTest.java | 2 +- .../{ => groupby}/GroupByTimeDataSetTest.java | 2 +- .../query/executor/GroupByEngineDataSetTest.java | 177 --- .../org/apache/iotdb/db/tools/MLogParserTest.java | 150 +++ .../org/apache/iotdb/db/tools/WalCheckerTest.java | 5 +- .../iotdb/db/utils/CopyOnReadLinkedListTest.java | 67 ++ .../iotdb/db/utils/EncodingInferenceUtilsTest.java | 45 + .../org/apache/iotdb/db/utils/MemUtilsTest.java | 114 ++ .../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 64 ++ .../apache/iotdb/db/utils/SerializeUtilsTest.java | 312 ++++++ .../db/utils/datastructure/BinaryTVListTest.java | 58 + .../db/utils/datastructure/BooleanTVListTest.java | 62 + .../db/utils/datastructure/DoubleTVListTest.java | 72 ++ .../db/utils/datastructure/FloatTVListTest.java | 72 ++ .../db/utils/datastructure/IntTVListTest.java | 71 ++ .../db/utils/datastructure/LongTVListTest.java | 18 + .../iotdb/db/writelog/io/LogWriterReaderTest.java | 4 +- .../db/writelog/io/MultiFileLogReaderTest.java | 4 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 43 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 90 ++ .../iotdb/session/IoTDBSessionIteratorIT.java | 178 ++- .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 10 +- site/README-zh.md | 2 +- site/README.md | 2 +- .../theme/global-components/Contributor.vue | 14 + spark-iotdb-connector/pom.xml | 6 - tsfile/format-changelist.md | 2 +- .../org/apache/iotdb/tsfile/file/MetaMarker.java | 4 + .../iotdb/tsfile/file/footer/ChunkGroupFooter.java | 31 +- .../file/metadata/MetadataIndexConstructor.java | 9 +- .../tsfile/file/metadata/MetadataIndexNode.java | 7 +- .../file/metadata/enums/CompressionType.java | 118 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 20 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 10 +- .../tsfile/read/TsFileSequenceReaderTest.java | 2 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 8 + zeppelin-interpreter/IoTDB-Zeppelin-Demo.zpln | 377 +++++++ zeppelin-interpreter/README.md | 186 +++ zeppelin-interpreter/pom.xml | 126 +++ .../apache/zeppelin/iotdb/IoTDBInterpreter.java | 213 ++++ .../src/main/resources/interpreter-setting.json | 70 ++ .../zeppelin/iotdb/IoTDBInterpreterTest.java | 273 +++++ 184 files changed, 9383 insertions(+), 2169 deletions(-) diff --cc cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java index 355ac95,e9295f3..d06de9d --- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java @@@ -329,23 -310,22 +329,23 @@@ public class SlotPartitionTable impleme try { dataOutputStream.writeInt(totalSlotNumbers); dataOutputStream.writeInt(nodeSlotMap.size()); - for (Entry<Node, List<Integer>> entry : nodeSlotMap.entrySet()) { - SerializeUtils.serialize(entry.getKey(), dataOutputStream); - SerializeUtils.serializeIntList(entry.getValue(), dataOutputStream); + for (Entry<RaftNode, List<Integer>> entry : nodeSlotMap.entrySet()) { + SerializeUtils.serialize(entry.getKey().getNode(), dataOutputStream); + dataOutputStream.writeInt(entry.getKey().getRaftId()); - SerializeUtils.serialize(entry.getValue(), dataOutputStream); ++// SerializeUtils.serialize(entry.getValue(), dataOutputStream); } - dataOutputStream.writeInt(previousNodeMap.size()); - for (Entry<Node, Map<Integer, Node>> nodeMapEntry : previousNodeMap.entrySet()) { - dataOutputStream.writeInt(nodeMapEntry.getKey().getNodeIdentifier()); - - Map<Integer, Node> prevHolders = nodeMapEntry.getValue(); - dataOutputStream.writeInt(prevHolders.size()); - for (Entry<Integer, Node> integerNodeEntry : prevHolders.entrySet()) { - dataOutputStream.writeInt(integerNodeEntry.getKey()); - dataOutputStream.writeInt(integerNodeEntry.getValue().getNodeIdentifier()); - } - } +// dataOutputStream.writeInt(previousNodeMap.size()); +// for (Entry<Node, Map<Integer, Node>> nodeMapEntry : previousNodeMap.entrySet()) { +// dataOutputStream.writeInt(nodeMapEntry.getKey().getNodeIdentifier()); +// +// Map<Integer, Node> prevHolders = nodeMapEntry.getValue(); +// dataOutputStream.writeInt(prevHolders.size()); +// for (Entry<Integer, Node> integerNodeEntry : prevHolders.entrySet()) { +// dataOutputStream.writeInt(integerNodeEntry.getKey()); +// dataOutputStream.writeInt(integerNodeEntry.getValue().getNodeIdentifier()); +// } +// } dataOutputStream.writeLong(lastLogIndex); } catch (IOException ignored) { @@@ -357,46 -337,40 +357,46 @@@ @Override public void deserialize(ByteBuffer buffer) { -- logger.info("Initializing the partition table from buffer"); -- totalSlotNumbers = buffer.getInt(); -- int size = buffer.getInt(); -- Map<Integer, Node> idNodeMap = new HashMap<>(); -- for (int i = 0; i < size; i++) { -- Node node = new Node(); -- List<Integer> slots = new ArrayList<>(); -- SerializeUtils.deserialize(node, buffer); - int id = buffer.getInt(); - SerializeUtils.deserialize(slots, buffer); - RaftNode raftNode = new RaftNode(node, id); - nodeSlotMap.put(raftNode, slots); - SerializeUtils.deserializeIntList(slots, buffer); - nodeSlotMap.put(node, slots); -- idNodeMap.put(node.getNodeIdentifier(), node); -- for (Integer slot : slots) { - slotNodes[slot] = raftNode; - slotNodes[slot] = node; -- } -- } ++// logger.info("Initializing the partition table from buffer"); ++// totalSlotNumbers = buffer.getInt(); ++// int size = buffer.getInt(); ++// Map<Integer, Node> idNodeMap = new HashMap<>(); ++// for (int i = 0; i < size; i++) { ++// Node node = new Node(); ++// List<Integer> slots = new ArrayList<>(); ++// SerializeUtils.deserialize(node, buffer); ++// int id = buffer.getInt(); ++// SerializeUtils.deserialize(slots, buffer); ++// RaftNode raftNode = new RaftNode(node, id); ++// nodeSlotMap.put(raftNode, slots); ++// idNodeMap.put(node.getNodeIdentifier(), node); ++// for (Integer slot : slots) { ++// slotNodes[slot] = raftNode; ++// } ++// } + +// int prevNodeMapSize = buffer.getInt(); +// previousNodeMap = new HashMap<>(); +// for (int i = 0; i < prevNodeMapSize; i++) { +// int nodeId = buffer.getInt(); +// Node node = idNodeMap.get(nodeId); +// +// Map<Integer, Node> prevHolders = new HashMap<>(); +// int holderNum = buffer.getInt(); +// for (int i1 = 0; i1 < holderNum; i1++) { +// int slot = buffer.getInt(); +// Node holder = idNodeMap.get(buffer.getInt()); +// prevHolders.put(slot, holder); +// } +// previousNodeMap.put(node, prevHolders); +// } + lastLogIndex = buffer.getLong(); - int prevNodeMapSize = buffer.getInt(); - previousNodeMap = new HashMap<>(); - for (int i = 0; i < prevNodeMapSize; i++) { - int nodeId = buffer.getInt(); - Node node = idNodeMap.get(nodeId); - - Map<Integer, Node> prevHolders = new HashMap<>(); - int holderNum = buffer.getInt(); - for (int i1 = 0; i1 < holderNum; i1++) { - int slot = buffer.getInt(); - Node holder = idNodeMap.get(buffer.getInt()); - prevHolders.put(slot, holder); + for (RaftNode raftNode : nodeSlotMap.keySet()) { + if (!nodeRing.contains(raftNode.getNode())) { + nodeRing.add(raftNode.getNode()); } - previousNodeMap.put(node, prevHolders); } - lastLogIndex = buffer.getLong(); - - nodeRing.addAll(nodeSlotMap.keySet()); nodeRing.sort(Comparator.comparingInt(Node::getNodeIdentifier)); logger.info("All known nodes: {}", nodeRing);
