This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch cluster_scalability in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e7f8e286c27e22c23775f08478a8feae4aff9506 Merge: f8a1432 48d24e7 Author: lta <[email protected]> AuthorDate: Tue Jan 19 19:23:32 2021 +0800 merge master LICENSE-binary | 20 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 13 +- cluster/pom.xml | 6 +- cluster/src/assembly/cluster.xml | 4 +- cluster/src/assembly/resources/sbin/add-node.sh | 2 +- cluster/src/assembly/resources/sbin/start-node.sh | 2 +- .../iotdb/cluster/ClusterFileFlushPolicy.java | 2 +- .../iotdb/cluster/coordinator/Coordinator.java | 165 ++++-- .../cluster/log/applier/AsyncDataLogApplier.java | 13 +- .../iotdb/cluster/log/applier/BaseApplier.java | 2 +- .../iotdb/cluster/log/applier/DataLogApplier.java | 13 +- .../apache/iotdb/cluster/metadata/CMManager.java | 28 +- .../apache/iotdb/cluster/metadata/MetaPuller.java | 5 +- .../iotdb/cluster/query/ClusterPlanExecutor.java | 53 +- .../iotdb/cluster/query/ClusterPlanRouter.java | 74 +++ .../cluster/query/filter/SlotTsFileFilter.java | 4 +- .../utils/nodetool/function/NodeToolCmd.java | 22 +- .../DDL Data Definition Language.md | 2 + .../DML Data Manipulation Language.md | 37 +- docs/UserGuide/Operation Manual/Kill Query.md | 60 ++ docs/UserGuide/Operation Manual/SQL Reference.md | 15 + docs/UserGuide/System Tools/NodeTool.md | 9 +- docs/zh/SystemDesign/StorageEngine/Compaction.md | 2 +- .../DDL Data Definition Language.md | 2 + .../DML Data Manipulation Language.md | 42 -- docs/zh/UserGuide/Operation Manual/Kill Query.md | 61 +++ .../zh/UserGuide/Operation Manual/SQL Reference.md | 21 +- docs/zh/UserGuide/System Tools/NodeTool.md | 10 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 12 +- pom.xml | 12 +- server/pom.xml | 5 + .../resources/conf/iotdb-engine.properties | 5 + server/src/assembly/resources/conf/iotdb-env.bat | 1 - server/src/assembly/resources/conf/iotdb-env.sh | 1 - server/src/assembly/resources/conf/logback.xml | 68 +-- server/src/assembly/resources/sbin/start-server.sh | 2 +- .../assembly/resources/tools/start-WalChecker.sh | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 609 +++++++++++---------- .../engine/compaction/utils/CompactionUtils.java | 2 - .../iotdb/db/engine/flush/MemTableFlushTask.java | 36 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 44 +- .../apache/iotdb/db/engine/memtable/IMemTable.java | 17 +- .../db/engine/memtable/PrimitiveMemTable.java | 8 - .../iotdb/db/engine/merge/task/MergeFileTask.java | 6 +- .../db/engine/merge/task/MergeMultiChunkTask.java | 1 - .../iotdb/db/engine/merge/task/MergeTask.java | 28 +- .../iotdb/db/engine/modification/Deletion.java | 8 +- .../iotdb/db/engine/modification/Modification.java | 18 +- .../io/LocalTextModificationAccessor.java | 2 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 9 +- .../db/engine/storagegroup/StorageGroupInfo.java | 51 +- .../engine/storagegroup/StorageGroupProcessor.java | 344 +++++++----- .../db/engine/storagegroup/TsFileProcessor.java | 94 +++- .../db/engine/storagegroup/TsFileResource.java | 15 + .../virtualSg/HashVirtualPartitioner.java | 67 +++ .../storagegroup/virtualSg/VirtualPartitioner.java | 29 +- .../virtualSg/VirtualStorageGroupManager.java | 434 +++++++++++++++ .../org/apache/iotdb/db/metadata/MManager.java | 89 +-- .../org/apache/iotdb/db/monitor/StatMonitor.java | 14 +- .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 8 + .../apache/iotdb/db/qp/executor/PlanExecutor.java | 24 +- .../org/apache/iotdb/db/qp/logical/Operator.java | 2 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 7 +- .../db/qp/physical/crud/InsertMultiTabletPlan.java | 326 +++++++++++ .../db/qp/physical/crud/InsertTabletPlan.java | 6 + .../iotdb/db/qp/strategy/LogicalGenerator.java | 4 + .../iotdb/db/query/control/QueryTimeManager.java | 74 +-- .../db/query/dataset/NonAlignEngineDataSet.java | 22 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 37 +- .../db/query/executor/RawDataQueryExecutor.java | 11 +- .../db/query/executor/fill/LastPointReader.java | 36 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 4 + .../chunk/metadata/MemChunkMetadataLoader.java | 4 + .../iotdb/db/query/reader/series/SeriesReader.java | 33 +- .../reader/universal/DescPriorityMergeReader.java | 4 +- .../reader/universal/PriorityMergeReader.java | 50 +- .../iotdb/db/query/udf/core/input/InputLayer.java | 2 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 87 ++- .../apache/iotdb/db/tools/TsFileSketchTool.java | 18 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 34 +- .../db/tools/virtualsg/DeviceMappingViewer.java | 61 +++ .../org/apache/iotdb/db/utils/FilePathUtils.java | 9 + .../java/org/apache/iotdb/db/utils/MmapUtil.java | 25 +- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 5 +- .../writelog/manager/MultiFileLogNodeManager.java | 27 +- .../db/writelog/manager/WriteLogNodeManager.java | 7 +- .../db/writelog/node/ExclusiveWriteLogNode.java | 18 +- .../iotdb/db/writelog/node/WriteLogNode.java | 8 +- .../iotdb/db/writelog/recover/LogReplayer.java | 26 +- .../writelog/recover/TsFileRecoverPerformer.java | 29 +- .../db/engine/cache/ChunkMetadataCacheTest.java | 2 +- .../db/engine/memtable/PrimitiveMemTableTest.java | 4 +- .../apache/iotdb/db/engine/merge/MergeLogTest.java | 1 + .../iotdb/db/engine/merge/MergeOverLapTest.java | 1 - .../iotdb/db/engine/merge/MergeTaskTest.java | 3 +- .../apache/iotdb/db/engine/merge/MergeTest.java | 3 + .../engine/modification/DeletionFileNodeTest.java | 27 +- .../storagegroup/StorageGroupProcessorTest.java | 15 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 34 +- .../engine/storagegroup/TsFileProcessorTest.java | 10 +- .../virtualSg/HashVirtualPartitionerTest.java | 64 +++ .../iotdb/db/integration/IoTDBCompleteIT.java | 5 + .../iotdb/db/integration/IoTDBDeletionIT.java | 40 ++ .../db/integration/IoTDBLoadExternalTsfileIT.java | 33 +- .../iotdb/db/integration/IoTDBMultiDeviceIT.java | 322 +++++++++++ .../db/integration/IoTDBQueryTimeoutTest.java | 27 +- .../iotdb/db/integration/IoTDBRestartIT.java | 16 + .../db/integration/auth/IoTDBAuthorizationIT.java | 21 - .../iotdb/db/monitor/IoTDBStatMonitorTest.java | 21 +- .../db/qp/physical/InsertTabletMultiPlanTest.java | 99 ++++ .../iotdb/db/qp/physical/InsertTabletPlanTest.java | 105 ++++ .../iotdb/db/qp/physical/PhysicalPlanTest.java | 2 +- .../query/reader/series/SeriesReaderTestUtil.java | 4 +- .../db/sync/receiver/load/FileLoaderTest.java | 38 +- .../recover/SyncReceiverLogAnalyzerTest.java | 2 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 3 + .../apache/iotdb/db/utils/VersionUtilsTest.java | 55 -- .../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 25 +- .../apache/iotdb/db/writelog/PerformanceTest.java | 48 +- .../iotdb/db/writelog/WriteLogNodeManagerTest.java | 63 ++- .../apache/iotdb/db/writelog/WriteLogNodeTest.java | 71 ++- .../iotdb/db/writelog/recover/LogReplayerTest.java | 43 +- .../recover/RecoverResourceFromReaderTest.java | 56 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 55 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 36 +- .../org/apache/iotdb/rpc/AutoResizingBuffer.java | 15 +- .../iotdb/rpc/AutoScalingBufferReadTransport.java | 9 + .../iotdb/rpc/AutoScalingBufferWriteTransport.java | 5 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 7 + .../rpc/TCompressedElasticFramedTransport.java | 61 +-- .../apache/iotdb/rpc/TElasticFramedTransport.java | 14 +- .../rpc/TimeoutChangeableTFastFramedTransport.java | 2 + .../TimeoutChangeableTSnappyFramedTransport.java | 2 + .../main/java/org/apache/iotdb/session/Config.java | 1 - .../java/org/apache/iotdb/session/Session.java | 10 +- .../apache/iotdb/session/SessionConnection.java | 2 +- .../org/apache/iotdb/session/SessionDataSet.java | 3 +- site/src/main/.vuepress/config.js | 2 + thrift/rpc-changelist.md | 24 + .../iotdb/tsfile/file/metadata/TsFileMetadata.java | 31 -- .../iotdb/tsfile/read/TsFileSequenceReader.java | 14 - .../iotdb/tsfile/read/reader/LocalTsFileInput.java | 20 - .../apache/iotdb/tsfile/utils/VersionUtils.java | 49 -- .../apache/iotdb/tsfile/write/TsFileWriter.java | 5 - .../write/writer/ForceAppendTsFileWriter.java | 1 - .../write/writer/RestorableTsFileIOWriter.java | 6 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 23 +- .../tsfile/file/metadata/utils/TestHelper.java | 9 - .../iotdb/tsfile/write/TsFileIOWriterTest.java | 6 - .../iotdb/tsfile/write/TsFileWriterTest.java | 1 - .../write/writer/RestorableTsFileIOWriterTest.java | 15 +- 154 files changed, 3789 insertions(+), 1661 deletions(-) diff --cc cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java index dbfb9ff,de36ac4..c6bccfb --- a/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java @@@ -365,13 -368,13 +370,19 @@@ public class Coordinator if (entry.getValue().contains(thisNode)) { // the query should be handled by a group the local node is in, handle it with in the group long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_LOCAL_GROUP - .getOperationStartTime(); + .getOperationStartTime(); logger.debug("Execute {} in a local group of {}", entry.getKey(), ++<<<<<<< HEAD + entry.getValue().getHeader()); + result = metaGroupMember.getLocalDataMember(new RaftNode(entry.getValue().getHeader(), entry.getValue().getId())) + .executeNonQueryPlan(entry.getKey()); ++======= + entry.getValue().getHeader()); + result = metaGroupMember.getLocalDataMember(entry.getValue().getHeader()) + .executeNonQueryPlan(entry.getKey()); ++>>>>>>> master Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_LOCAL_GROUP - .calOperationCostTimeFromStart(startTime); + .calOperationCostTimeFromStart(startTime); } else { // forward the query to the group that should handle it long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY_IN_REMOTE_GROUP diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java index ceed787,e590489..84fad46 --- a/cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/applier/DataLogApplier.java @@@ -62,14 -62,12 +64,16 @@@ public class DataLogApplier extends Bas logger.debug("DataMember [{}] start applying Log {}", dataGroupMember.getName(), log); try { - if (log instanceof PhysicalPlanLog) { + if (log instanceof AddNodeLog) { + metaGroupMember.getDataClusterServer().preAddNodeForDataGroup((AddNodeLog) log, dataGroupMember); + } else if (log instanceof RemoveNodeLog) { + metaGroupMember.getDataClusterServer().preRemoveNodeForDataGroup((RemoveNodeLog) log, dataGroupMember); + } else if (log instanceof PhysicalPlanLog) { PhysicalPlanLog physicalPlanLog = (PhysicalPlanLog) log; PhysicalPlan plan = physicalPlanLog.getPlan(); - if (plan instanceof InsertPlan) { + if (plan instanceof InsertMultiTabletPlan) { + applyInsert((InsertMultiTabletPlan) plan); + } else if (plan instanceof InsertPlan) { applyInsert((InsertPlan) plan); } else { applyPhysicalPlan(plan, dataGroupMember); diff --cc cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java index 487b88d,8c63386..65f66dc --- a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java @@@ -116,9 -109,11 +117,11 @@@ public class ClusterPlanRouter } public Map<PhysicalPlan, PartitionGroup> splitAndRoutePlan(PhysicalPlan plan) - throws UnsupportedPlanException, MetadataException { + throws UnsupportedPlanException, MetadataException, UnknownLogTypeException { if (plan instanceof InsertTabletPlan) { return splitAndRoutePlan((InsertTabletPlan) plan); + } else if (plan instanceof InsertMultiTabletPlan) { + return splitAndRoutePlan((InsertMultiTabletPlan) plan); } else if (plan instanceof CountPlan) { return splitAndRoutePlan((CountPlan) plan); } else if (plan instanceof CreateTimeSeriesPlan) { diff --cc server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index dc553be,0df63f7..ab9335e --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@@ -366,8 -366,8 +371,8 @@@ public abstract class PhysicalPlan REVOKE_USER_PRIVILEGE, GRANT_ROLE_PRIVILEGE, GRANT_USER_PRIVILEGE, GRANT_USER_ROLE, MODIFY_PASSWORD, DELETE_USER, DELETE_STORAGE_GROUP, SHOW_TIMESERIES, DELETE_TIMESERIES, LOAD_CONFIGURATION, CREATE_MULTI_TIMESERIES, ALTER_TIMESERIES, FLUSH, CREATE_INDEX, DROP_INDEX, - CHANGE_TAG_OFFSET, CHANGE_ALIAS, MNODE, MEASUREMENT_MNODE, STORAGE_GROUP_MNODE, + CHANGE_TAG_OFFSET, CHANGE_ALIAS, MNODE, MEASUREMENT_MNODE, STORAGE_GROUP_MNODE, CLUSTER_LOG, - BATCH_INSERT_ONE_DEVICE + BATCH_INSERT_ONE_DEVICE, MULTI_BATCH_INSERT } public long getIndex() {
