This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch IOTDB-4619 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit db711b2051ded87b8447d4576b24b6666567e338 Merge: b42c96c6f8 cf6d85062d Author: JackieTien97 <[email protected]> AuthorDate: Mon Oct 17 16:20:54 2022 +0800 resolve conflicts .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 28 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 + .../main/java/org/apache/iotdb/tool/ImportCsv.java | 6 +- .../resources/conf/iotdb-confignode.properties | 16 + .../confignode/client/ConfigNodeRequestType.java | 2 +- .../client/sync/SyncConfigNodeClientPool.java | 4 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 141 ++++++ .../iotdb/confignode/conf/ConfigNodeConstant.java | 5 + .../confignode/conf/ConfigNodeDescriptor.java | 64 +++ .../confignode/conf/ConfigNodeRemoveCheck.java | 17 +- .../confignode/conf/SystemPropertiesUtils.java | 20 + .../consensus/request/ConfigPhysicalPlan.java | 4 + .../consensus/request/ConfigPhysicalPlanType.java | 1 + .../consensus/request/read/GetRoutingPlan.java | 15 +- .../consensus/request/read/GetTriggerJarPlan.java | 68 +++ .../consensus/response/DataNodeRegisterResp.java | 11 + .../consensus/response/TriggerJarResp.java | 36 +- .../statemachine/PartitionRegionStateMachine.java | 24 +- .../iotdb/confignode/manager/ConfigManager.java | 103 +++-- .../iotdb/confignode/manager/ConsensusManager.java | 38 +- .../apache/iotdb/confignode/manager/IManager.java | 15 +- .../iotdb/confignode/manager/ProcedureManager.java | 2 +- .../iotdb/confignode/manager/TriggerManager.java | 27 ++ .../iotdb/confignode/manager/node/NodeManager.java | 117 ++++- .../iotdb/confignode/persistence/NodeInfo.java | 12 +- .../iotdb/confignode/persistence/TriggerInfo.java | 24 + .../persistence/executor/ConfigPlanExecutor.java | 3 + .../persistence/metric/NodeInfoMetrics.java | 2 +- .../persistence/partition/PartitionInfo.java | 3 +- .../persistence/partition/RegionGroup.java | 2 +- .../partition/StorageGroupPartitionTable.java | 10 +- .../procedure/env/ConfigNodeProcedureEnv.java | 26 +- .../procedure/env/DataNodeRemoveHandler.java | 199 ++++---- .../impl/node/AddConfigNodeProcedure.java | 17 +- .../impl/node/RemoveConfigNodeProcedure.java | 32 +- .../impl/node/RemoveDataNodeProcedure.java | 7 +- .../impl/statemachine/RegionMigrateProcedure.java | 40 +- .../procedure/state/AddConfigNodeState.java | 2 +- .../procedure/state/RemoveConfigNodeState.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 44 +- .../confignode/service/ConfigNodeCommandLine.java | 20 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 32 +- .../request/ConfigPhysicalPlanSerDeTest.java | 17 +- .../thrift/ConfigNodeRPCServiceProcessorTest.java | 336 -------------- .../org/apache/iotdb/consensus/IStateMachine.java | 5 +- .../org/apache/iotdb/consensus/common/Peer.java | 26 +- .../iotdb/consensus/config/ConsensusConfig.java | 22 +- .../apache/iotdb/consensus/config/RatisConfig.java | 67 ++- .../multileader/MultiLeaderConsensus.java | 12 +- .../multileader/MultiLeaderServerImpl.java | 6 +- .../service/MultiLeaderRPCServiceProcessor.java | 4 +- .../ratis/ApplicationStateMachineProxy.java | 37 +- .../iotdb/consensus/ratis/RatisConsensus.java | 24 +- .../iotdb/consensus/ratis/SnapshotStorage.java | 53 +-- .../org/apache/iotdb/consensus/ratis/Utils.java | 36 +- .../consensus/standalone/StandAloneConsensus.java | 11 +- .../consensus/standalone/StandAloneServerImpl.java | 2 +- .../multileader/MultiLeaderConsensusTest.java | 7 +- .../iotdb/consensus/multileader/RecoveryTest.java | 5 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 7 +- .../apache/iotdb/consensus/ratis/SnapshotTest.java | 34 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 7 +- .../iotdb/consensus/standalone/RecoveryTest.java | 5 +- .../standalone/StandAloneConsensusTest.java | 30 +- docs/Download/README.md | 14 +- .../Maintenance-Tools/Maintenance-Command.md | 4 +- docs/UserGuide/Operate-Metadata/Storage-Group.md | 3 + docs/UserGuide/Query-Data/Aggregate-Query.md | 174 +++++++ docs/UserGuide/Reference/DataNode-Config-Manual.md | 9 + docs/UserGuide/UDF-Library/Data-Profiling.md | 10 +- docs/UserGuide/Write-Data/Write-Data.md | 4 +- docs/zh/Download/README.md | 14 +- .../Maintenance-Tools/Maintenance-Command.md | 8 +- .../zh/UserGuide/Operate-Metadata/Storage-Group.md | 2 + docs/zh/UserGuide/Query-Data/Aggregate-Query.md | 168 ++++++- .../UserGuide/Reference/DataNode-Config-Manual.md | 10 +- docs/zh/UserGuide/UDF-Library/Data-Profiling.md | 6 +- docs/zh/UserGuide/Write-Data/Write-Data.md | 2 + example/kafka/pom.xml | 2 +- .../util/TSFileConfigUtilCompletenessTest.java | 3 +- integration-test/checkstyle.xml | 8 +- integration-test/import-control.xml | 81 +++- .../java/org/apache/iotdb/it/env/AbstractEnv.java | 76 ++- .../java/org/apache/iotdb/it/env/MppConfig.java | 3 +- .../org/apache/iotdb/it/env/RemoteServerEnv.java | 14 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +- .../{ => it}/IoTDBClusterPartitionIT.java | 93 +++- .../iotdb/confignode/it/IoTDBClusterRestartIT.java | 88 ++++ .../confignode/{ => it}/IoTDBConfigNodeIT.java | 10 +- .../{ => it}/IoTDBConfigNodeSnapshotIT.java | 4 +- .../iotdb/confignode/it/IoTDBStorageGroupIT.java | 183 ++++++++ .../org/apache/iotdb/db/it/IoTDBFilterNullIT.java | 3 + .../it/aggregation/IoTDBAggregationByLevelIT.java | 3 + .../it/aggregation/IoTDBAggregationDeleteIT.java | 3 + .../aggregation/IoTDBAggregationScanOrderIT.java | 3 + .../aggregation/IoTDBAggregationSmallDataIT.java | 3 + .../db/it/aggregation/IoTDBTagAggregationIT.java | 514 +++++++++++++++++++++ .../aligned/IoTDBAggregationWithDeletion2IT.java | 1 + .../it/aligned/IoTDBAggregationWithDeletionIT.java | 1 + .../db/it/aligned/IoTDBAlignedLastQuery2IT.java | 1 + .../db/it/aligned/IoTDBAlignedLastQuery3IT.java | 1 + .../db/it/aligned/IoTDBAlignedLastQueryIT.java | 1 + .../db/it/aligned/IoTDBAlignedSeriesQuery2IT.java | 1 + .../db/it/aligned/IoTDBAlignedSeriesQuery3IT.java | 1 + .../db/it/aligned/IoTDBAlignedSeriesQueryIT.java | 1 + .../IoTDBAlignedSeriesQueryWithDeletionIT.java | 1 + .../db/it/aligned/IoTDBGroupByLevelQueryIT.java | 1 + ...GroupByQueryWithValueFilterWithDeletion2IT.java | 1 + ...BGroupByQueryWithValueFilterWithDeletionIT.java | 1 + ...upByQueryWithoutValueFilterWithDeletion2IT.java | 1 + ...oupByQueryWithoutValueFilterWithDeletionIT.java | 1 + .../it/aligned/IoTDBLastQueryWithDeletion2IT.java | 1 + .../it/aligned/IoTDBLastQueryWithDeletionIT.java | 1 + .../aligned/IoTDBLastQueryWithoutLastCache2IT.java | 1 + .../aligned/IoTDBLastQueryWithoutLastCacheIT.java | 1 + ...DBLastQueryWithoutLastCacheWithDeletion2IT.java | 1 + ...TDBLastQueryWithoutLastCacheWithDeletionIT.java | 1 + ...BRawQueryWithoutValueFilterWithDeletion2IT.java | 1 + ...DBRawQueryWithoutValueFilterWithDeletionIT.java | 1 + .../org/apache/iotdb/db/it/env/StandaloneEnv.java | 15 +- .../db/it/{aligned => utils}/AlignedWriteUtil.java | 2 +- .../db/it/withoutNull/IoTDBWithoutAllNullIT.java | 3 + .../db/it/withoutNull/IoTDBWithoutAnyNullIT.java | 3 + .../withoutNull/IoTDBWithoutNullAllFilterIT.java | 3 + .../withoutNull/IoTDBWithoutNullAnyFilterIT.java | 3 + .../{ => it}/IoTDBSessionAlignedInsertIT.java | 4 +- .../session/{ => it}/IoTDBSessionComplexIT.java | 4 +- .../{ => it}/IoTDBSessionDisableMemControlIT.java | 4 +- .../session/{ => it}/IoTDBSessionInsertNulIT.java | 4 +- .../session/{ => it}/IoTDBSessionSimpleIT.java | 4 +- .../{ => it}/IoTDBSessionSyntaxConventionIT.java | 4 +- .../{SessionTest.java => it/SessionIT.java} | 9 +- .../db/integration/IoTDBRepeatPatternNameIT.java | 67 +++ .../iotdb/db/integration/IoTDBSnapshotIT.java | 60 ++- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 15 +- library-udf/pom.xml | 12 + .../apache/iotdb/library/dprofile/UDTFSample.java | 70 ++- .../iotdb/library/dprofile/DProfileTests.java | 13 + .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 3 + .../commons/partition/DataPartitionTable.java | 3 +- .../commons/partition/SchemaPartitionTable.java | 8 + .../commons/partition/SeriesPartitionTable.java | 14 +- .../apache/iotdb/commons/path/MeasurementPath.java | 35 ++ .../apache/iotdb/commons/path/PatternTreeMap.java | 44 +- .../org/apache/iotdb/commons/utils/AuthUtils.java | 1 + .../apache/iotdb/commons/utils/NodeUrlUtils.java | 6 +- .../java/org/apache/iotdb/RewriteTsFileTool.java | 249 +++++++++- .../schemaregion/rocksdb/RSchemaRegion.java | 151 +++--- .../resources/conf/iotdb-datanode.properties | 18 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 29 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 126 ++++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 64 ++- .../db/consensus/DataRegionConsensusImpl.java | 16 + .../db/consensus/SchemaRegionConsensusImpl.java | 16 + .../org/apache/iotdb/db/engine/StorageEngine.java | 21 +- .../apache/iotdb/db/engine/StorageEngineV2.java | 13 +- .../RewriteCrossSpaceCompactionSelector.java | 7 +- .../impl/ReadChunkCompactionPerformer.java | 4 +- .../writer/CrossSpaceCompactionWriter.java | 4 +- .../writer/InnerSpaceCompactionWriter.java | 4 +- .../iotdb/db/engine/cq/ContinuousQueryService.java | 10 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 1 - .../db/engine/querycontext/QueryDataSource.java | 5 +- .../iotdb/db/engine/snapshot/SnapshotFileSet.java | 4 +- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 377 ++++++++++----- .../db/engine/snapshot/SnapshotLogAnalyzer.java | 61 +-- .../iotdb/db/engine/snapshot/SnapshotLogger.java | 47 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 16 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 54 ++- .../{StorageGroupInfo.java => DataRegionInfo.java} | 19 +- .../db/engine/storagegroup/TsFileProcessor.java | 73 ++- .../engine/storagegroup/TsFileProcessorInfo.java | 16 +- .../db/engine/storagegroup/TsFileResource.java | 3 +- .../dataregion/StorageGroupManager.java | 2 +- .../query/QueryTimeoutRuntimeException.java | 9 + .../localconfignode/LocalDataPartitionTable.java | 2 +- .../iotdb/db/metadata/LocalSchemaProcessor.java | 8 +- .../db/metadata/cache/DataNodeSchemaCache.java | 3 + .../iotdb/db/metadata/cache/SchemaCacheEntry.java | 11 +- .../apache/iotdb/db/metadata/idtable/IDTable.java | 8 +- .../db/metadata/idtable/IDTableHashmapImpl.java | 12 +- .../db/metadata/logfile/BufferedSerializer.java | 54 +++ .../db/metadata/logfile/FakeCRC32Deserializer.java | 91 ++++ .../db/metadata/logfile/FakeCRC32Serializer.java | 65 +++ .../logfile/IDeserializer.java} | 25 +- .../logfile/ISerializer.java} | 26 +- .../iotdb/db/metadata/logfile/MLogTxtWriter.java | 2 +- .../iotdb/db/metadata/logfile/SchemaLogReader.java | 247 ++++++++++ .../iotdb/db/metadata/logfile/SchemaLogWriter.java | 114 +++++ .../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 5 +- .../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 15 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 32 +- .../plan/schemaregion/ISchemaRegionPlan.java | 12 +- .../plan/schemaregion/SchemaRegionPlanType.java | 75 +++ .../plan/schemaregion/SchemaRegionPlanVisitor.java | 90 ++++ .../impl/ActivateTemplateInClusterPlanImpl.java | 79 ++++ .../impl/ActivateTemplatePlanImpl.java} | 28 +- .../impl/AutoCreateDeviceMNodePlanImpl.java} | 28 +- .../schemaregion/impl/ChangeAliasPlanImpl.java} | 39 +- .../impl/ChangeTagOffsetPlanImpl.java} | 39 +- .../impl/CreateAlignedTimeSeriesPlanImpl.java | 171 +++++++ .../impl/CreateTimeSeriesPlanImpl.java | 163 +++++++ .../impl/DeleteTimeSeriesPlanImpl.java} | 29 +- .../impl/PreDeleteTimeSeriesPlanImpl.java} | 28 +- .../impl/RollbackPreDeleteTimeSeriesPlanImpl.java} | 28 +- .../impl/SchemaRegionPlanDeserializer.java | 337 ++++++++++++++ .../schemaregion/impl/SchemaRegionPlanFactory.java | 140 ++++++ .../impl/SchemaRegionPlanSerializer.java | 399 ++++++++++++++++ .../impl/SchemaRegionPlanTxtSerializer.java | 238 ++++++++++ .../schemaregion/impl/SetTemplatePlanImpl.java} | 62 +-- .../schemaregion/impl/UnsetTemplatePlanImpl.java} | 62 +-- .../write/IActivateTemplateInClusterPlan.java | 60 +++ .../schemaregion/write/IActivateTemplatePlan.java} | 26 +- .../write/IAutoCreateDeviceMNodePlan.java} | 26 +- .../plan/schemaregion/write/IChangeAliasPlan.java} | 30 +- .../schemaregion/write/IChangeTagOffsetPlan.java} | 30 +- .../write/ICreateAlignedTimeSeriesPlan.java | 80 ++++ .../schemaregion/write/ICreateTimeSeriesPlan.java | 79 ++++ .../schemaregion/write/IDeleteTimeSeriesPlan.java} | 28 +- .../write/IPreDeleteTimeSeriesPlan.java} | 26 +- .../write/IRollbackPreDeleteTimeSeriesPlan.java} | 26 +- .../plan/schemaregion/write/ISetTemplatePlan.java} | 29 +- .../schemaregion/write/IUnsetTemplatePlan.java} | 29 +- .../db/metadata/schemaregion/ISchemaRegion.java | 38 +- .../schemaregion/SchemaRegionMemoryImpl.java | 445 ++++++++++++------ .../schemaregion/SchemaRegionSchemaFileImpl.java | 389 +++++++++++----- .../apache/iotdb/db/metadata/tag/TagManager.java | 15 + .../metadata/visitor/SchemaExecutionVisitor.java | 73 +-- .../timerangeiterator/AggrWindowIterator.java | 22 +- .../TimeRangeIteratorFactory.java | 2 +- .../db/mpp/common/header/ColumnHeaderConstant.java | 13 +- .../db/mpp/common/header/DatasetHeaderFactory.java | 12 + .../mpp/common/schematree/ClusterSchemaTree.java | 9 +- .../schematree/node/SchemaMeasurementNode.java | 14 + .../visitor/SchemaTreeMeasurementVisitor.java | 1 + .../operator/process/TagAggregationOperator.java | 233 ++++++++++ .../operator/schema/SchemaFetchScanOperator.java | 7 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 48 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 98 +++- .../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 13 +- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 2 + .../iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 2 + .../mpp/plan/analyze/StandaloneSchemaFetcher.java | 12 +- .../db/mpp/plan/execution/QueryExecution.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 23 + .../config/executor/ClusterConfigTaskExecutor.java | 93 +++- .../config/executor/IConfigTaskExecutor.java | 11 + .../executor/StandaloneConfigTaskExecutor.java | 43 +- .../config/metadata/CountStorageGroupTask.java | 2 +- .../GetRegionTask.java} | 51 +- .../GetSeriesSlotListTask.java} | 52 +-- .../GetTimeSlotListTask.java} | 51 +- .../execution/config/metadata/ShowRegionTask.java | 2 +- .../plan/execution/config/sys/AuthorizerTask.java | 4 +- .../execution/config/sys/sync/ShowPipeTask.java | 4 +- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 126 ++++- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 143 +++++- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 13 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 77 ++- .../db/mpp/plan/planner/SubPlanTypeExtractor.java | 9 + .../planner/distribution/ExchangeNodeAdder.java | 5 + .../plan/planner/distribution/SourceRewriter.java | 91 +++- .../plan/planner/plan/node/PlanGraphPrinter.java | 31 ++ .../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +- .../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 + .../node/metedata/read/SchemaFetchScanNode.java | 19 +- .../node/metedata/write/ActivateTemplateNode.java | 15 +- .../write/CreateAlignedTimeSeriesNode.java | 25 +- .../node/metedata/write/CreateTimeSeriesNode.java | 17 +- .../plan/node/process/GroupByLevelNode.java | 21 +- .../planner/plan/node/process/GroupByTagNode.java | 312 +++++++++++++ .../planner/plan/node/write/InsertRowNode.java | 7 + .../planner/plan/node/write/InsertTabletNode.java | 8 + .../plan/parameter/AggregationDescriptor.java | 4 +- ....java => CrossSeriesAggregationDescriptor.java} | 16 +- .../db/mpp/plan/statement/StatementVisitor.java | 16 + .../statement/component/GroupByTagComponent.java | 27 +- .../db/mpp/plan/statement/crud/QueryStatement.java | 22 + .../statement/internal/SchemaFetchStatement.java | 9 +- .../statement/metadata/GetRegionStatement.java | 100 ++++ .../metadata/GetSeriesSlotListStatement.java | 82 ++++ .../metadata/GetTimeSlotListStatement.java | 101 ++++ .../protocol/influxdb/input/InfluxLineParser.java | 4 +- .../protocol/influxdb/sql/InfluxDBSqlVisitor.java | 8 +- .../db/qp/physical/crud/GroupByTimeFillPlan.java | 6 +- .../sys/ActivateTemplateInClusterPlan.java | 19 +- .../db/qp/physical/sys/ActivateTemplatePlan.java | 8 +- .../qp/physical/sys/AutoCreateDeviceMNodePlan.java | 8 +- .../iotdb/db/qp/physical/sys/ChangeAliasPlan.java | 3 +- .../db/qp/physical/sys/ChangeTagOffsetPlan.java | 3 +- .../physical/sys/CreateAlignedTimeSeriesPlan.java | 38 +- .../qp/physical/sys/CreateContinuousQueryPlan.java | 4 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 3 +- .../db/qp/physical/sys/DeleteTimeSeriesPlan.java | 8 +- .../qp/physical/sys/PreDeleteTimeSeriesPlan.java | 3 +- .../sys/RollbackPreDeleteTimeSeriesPlan.java | 4 +- .../iotdb/db/qp/physical/sys/SetTemplatePlan.java | 3 +- .../db/qp/physical/sys/UnsetTemplatePlan.java | 3 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 32 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +- .../{DatetimeUtils.java => DateTimeUtils.java} | 21 +- .../apache/iotdb/db/qp/utils/WildcardsRemover.java | 8 +- .../query/dataset/groupby/GroupByTimeDataSet.java | 2 +- .../apache/iotdb/db/query/executor/fill/IFill.java | 2 +- .../iotdb/db/query/executor/fill/LinearFill.java | 6 +- .../iotdb/db/query/executor/fill/PreviousFill.java | 4 +- .../metadata/DiskAlignedChunkMetadataLoader.java | 2 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 2 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 46 +- .../java/org/apache/iotdb/db/service/DataNode.java | 124 ++++- .../db/service/DataNodeServerCommandLine.java | 36 +- .../iotdb/db/service/RegionMigrateService.java | 8 +- .../ResourcesInformationHolder.java} | 27 +- .../impl/DataNodeInternalRPCServiceImpl.java | 9 +- .../service/thrift/impl/DataNodeRegionManager.java | 4 +- .../java/org/apache/iotdb/db/sync/SyncService.java | 12 +- .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 6 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 6 +- .../apache/iotdb/db/tools/schema/MLogParser.java | 122 +---- .../db/tools/watermark/WatermarkDetector.java | 4 +- .../iotdb/db/trigger/executor/TriggerExecutor.java | 4 + .../trigger/service/TriggerManagementService.java | 161 ++++--- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 16 +- .../timerangeiterator/AggrWindowIterator.java | 18 +- .../TimeRangeIteratorFactory.java | 2 +- .../file/AbstractTsFileRecoverPerformer.java | 10 +- .../db/engine/snapshot/IoTDBSnapshotTest.java | 13 +- .../engine/storagegroup/TsFileProcessorTest.java | 4 +- .../engine/storagegroup/TsFileProcessorV2Test.java | 4 +- .../db/metadata/cache/DataNodeSchemaCacheTest.java | 22 +- .../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 16 +- .../db/metadata/path/MeasurementPathTest.java | 75 ++- .../iotdb/db/metadata/path/PatternTreeMapTest.java | 88 ++-- .../plan/SchemaRegionPlanCompatibilityTest.java | 318 +++++++++++++ .../mpp/execution/operator/OperatorMemoryTest.java | 7 +- .../schema/SchemaFetchScanOperatorTest.java | 3 +- .../plan/analyze/AggregationDescriptorTest.java | 19 +- .../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 2 +- .../mpp/plan/analyze/FakePartitionFetcherImpl.java | 0 .../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 9 + .../db/mpp/plan/parser/StatementGeneratorTest.java | 16 + .../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 93 +++- .../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 20 +- .../distribution/AggregationDistributionTest.java | 24 +- .../metadata/read/SchemaFetchScanNodeTest.java | 7 +- .../node/process/GroupByLevelNodeSerdeTest.java | 4 +- .../plan/node/process/GroupByTagNodeSerdeTest.java | 122 +++++ .../db/qp/utils/DatetimeQueryDataSetUtilsTest.java | 116 ++--- .../DataNodeInternalRPCServiceImplTest.java | 8 +- .../org/apache/iotdb/db/tools/MLogParserTest.java | 17 - .../src/main/thrift/confignode.thrift | 48 +- .../src/main/thrift/mutlileader.thrift | 2 + .../iotdb/tsfile/common/conf/TSFileConfig.java | 10 + .../file/metadata/AlignedTimeSeriesMetadata.java | 15 + .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 18 + .../tsfile/file/metadata/TimeseriesMetadata.java | 7 + .../iotdb/tsfile/read/filter/operator/Regexp.java | 48 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 2 +- 360 files changed, 10931 insertions(+), 2782 deletions(-) diff --cc confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 89783abc78,23cb095567..1789ac08ea --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@@ -180,11 -188,26 +188,31 @@@ public class ConfigNodeConfig private long partitionRegionRatisRpcLeaderElectionTimeoutMaxMs = 4000L; private long schemaRegionRatisRpcLeaderElectionTimeoutMaxMs = 4000L; + /** CQ related */ + private int cqSubmitThread = 2; + + private long cqMinEveryIntervalInMs = 1_000; + + /** RatisConsensus protocol, request timeout for ratis client */ + private long dataRegionRatisRequestTimeoutMs = 10000L; + + private long partitionRegionRatisRequestTimeoutMs = 10000L; + private long schemaRegionRatisRequestTimeoutMs = 10000L; + + /** RatisConsensus protocol, exponential back-off retry policy params */ + private int partitionRegionRatisMaxRetryAttempts = 10; + + private long partitionRegionRatisInitialSleepTimeMs = 100; + private long partitionRegionRatisMaxSleepTimeMs = 10000; + + private int dataRegionRatisMaxRetryAttempts = 10; + private long dataRegionRatisInitialSleepTimeMs = 100; + private long dataRegionRatisMaxSleepTimeMs = 10000; + + private int schemaRegionRatisMaxRetryAttempts = 10; + private long schemaRegionRatisInitialSleepTimeMs = 100; + private long schemaRegionRatisMaxSleepTimeMs = 10000; + public ConfigNodeConfig() { // empty constructor } @@@ -677,19 -716,100 +721,116 @@@ schemaRegionRatisRpcLeaderElectionTimeoutMaxMs; } + public int getCqSubmitThread() { + return cqSubmitThread; + } + + public void setCqSubmitThread(int cqSubmitThread) { + this.cqSubmitThread = cqSubmitThread; + } + + public long getCqMinEveryIntervalInMs() { + return cqMinEveryIntervalInMs; + } + + public void setCqMinEveryIntervalInMs(long cqMinEveryIntervalInMs) { + this.cqMinEveryIntervalInMs = cqMinEveryIntervalInMs; + } ++ + public long getDataRegionRatisRequestTimeoutMs() { + return dataRegionRatisRequestTimeoutMs; + } + + public void setDataRegionRatisRequestTimeoutMs(long dataRegionRatisRequestTimeoutMs) { + this.dataRegionRatisRequestTimeoutMs = dataRegionRatisRequestTimeoutMs; + } + + public long getPartitionRegionRatisRequestTimeoutMs() { + return partitionRegionRatisRequestTimeoutMs; + } + + public void setPartitionRegionRatisRequestTimeoutMs(long partitionRegionRatisRequestTimeoutMs) { + this.partitionRegionRatisRequestTimeoutMs = partitionRegionRatisRequestTimeoutMs; + } + + public long getSchemaRegionRatisRequestTimeoutMs() { + return schemaRegionRatisRequestTimeoutMs; + } + + public void setSchemaRegionRatisRequestTimeoutMs(long schemaRegionRatisRequestTimeoutMs) { + this.schemaRegionRatisRequestTimeoutMs = schemaRegionRatisRequestTimeoutMs; + } + + public int getPartitionRegionRatisMaxRetryAttempts() { + return partitionRegionRatisMaxRetryAttempts; + } + + public void setPartitionRegionRatisMaxRetryAttempts(int partitionRegionRatisMaxRetryAttempts) { + this.partitionRegionRatisMaxRetryAttempts = partitionRegionRatisMaxRetryAttempts; + } + + public long getPartitionRegionRatisInitialSleepTimeMs() { + return partitionRegionRatisInitialSleepTimeMs; + } + + public void setPartitionRegionRatisInitialSleepTimeMs( + long partitionRegionRatisInitialSleepTimeMs) { + this.partitionRegionRatisInitialSleepTimeMs = partitionRegionRatisInitialSleepTimeMs; + } + + public long getPartitionRegionRatisMaxSleepTimeMs() { + return partitionRegionRatisMaxSleepTimeMs; + } + + public void setPartitionRegionRatisMaxSleepTimeMs(long partitionRegionRatisMaxSleepTimeMs) { + this.partitionRegionRatisMaxSleepTimeMs = partitionRegionRatisMaxSleepTimeMs; + } + + public int getDataRegionRatisMaxRetryAttempts() { + return dataRegionRatisMaxRetryAttempts; + } + + public void setDataRegionRatisMaxRetryAttempts(int dataRegionRatisMaxRetryAttempts) { + this.dataRegionRatisMaxRetryAttempts = dataRegionRatisMaxRetryAttempts; + } + + public long getDataRegionRatisInitialSleepTimeMs() { + return dataRegionRatisInitialSleepTimeMs; + } + + public void setDataRegionRatisInitialSleepTimeMs(long dataRegionRatisInitialSleepTimeMs) { + this.dataRegionRatisInitialSleepTimeMs = dataRegionRatisInitialSleepTimeMs; + } + + public long getDataRegionRatisMaxSleepTimeMs() { + return dataRegionRatisMaxSleepTimeMs; + } + + public void setDataRegionRatisMaxSleepTimeMs(long dataRegionRatisMaxSleepTimeMs) { + this.dataRegionRatisMaxSleepTimeMs = dataRegionRatisMaxSleepTimeMs; + } + + public int getSchemaRegionRatisMaxRetryAttempts() { + return schemaRegionRatisMaxRetryAttempts; + } + + public void setSchemaRegionRatisMaxRetryAttempts(int schemaRegionRatisMaxRetryAttempts) { + this.schemaRegionRatisMaxRetryAttempts = schemaRegionRatisMaxRetryAttempts; + } + + public long getSchemaRegionRatisInitialSleepTimeMs() { + return schemaRegionRatisInitialSleepTimeMs; + } + + public void setSchemaRegionRatisInitialSleepTimeMs(long schemaRegionRatisInitialSleepTimeMs) { + this.schemaRegionRatisInitialSleepTimeMs = schemaRegionRatisInitialSleepTimeMs; + } + + public long getSchemaRegionRatisMaxSleepTimeMs() { + return schemaRegionRatisMaxSleepTimeMs; + } + + public void setSchemaRegionRatisMaxSleepTimeMs(long schemaRegionRatisMaxSleepTimeMs) { + this.schemaRegionRatisMaxSleepTimeMs = schemaRegionRatisMaxSleepTimeMs; + } } diff --cc confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index f1736bf482,12d3d7bbc6..05d7759908 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@@ -411,21 -409,72 +411,85 @@@ public class ConfigNodeDescriptor properties.getProperty( "schema_region_ratis_rpc_leader_election_timeout_max_ms", String.valueOf(conf.getSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs())))); + + conf.setPartitionRegionRatisRequestTimeoutMs( + Long.parseLong( + properties.getProperty( + "partition_region_ratis_request_timeout_ms", + String.valueOf(conf.getPartitionRegionRatisRequestTimeoutMs())))); + conf.setSchemaRegionRatisRequestTimeoutMs( + Long.parseLong( + properties.getProperty( + "schema_region_ratis_request_timeout_ms", + String.valueOf(conf.getSchemaRegionRatisRequestTimeoutMs())))); + conf.setDataRegionRatisRequestTimeoutMs( + Long.parseLong( + properties.getProperty( + "data_region_ratis_request_timeout_ms", + String.valueOf(conf.getDataRegionRatisRequestTimeoutMs())))); + + conf.setPartitionRegionRatisMaxRetryAttempts( + Integer.parseInt( + properties.getProperty( + "partition_region_ratis_max_retry_attempts", + String.valueOf(conf.getPartitionRegionRatisMaxRetryAttempts())))); + conf.setPartitionRegionRatisInitialSleepTimeMs( + Long.parseLong( + properties.getProperty( + "partition_region_ratis_initial_sleep_time_ms", + String.valueOf(conf.getPartitionRegionRatisInitialSleepTimeMs())))); + conf.setPartitionRegionRatisMaxSleepTimeMs( + Long.parseLong( + properties.getProperty( + "partition_region_ratis_max_sleep_time_ms", + String.valueOf(conf.getPartitionRegionRatisMaxSleepTimeMs())))); + + conf.setDataRegionRatisMaxRetryAttempts( + Integer.parseInt( + properties.getProperty( + "data_region_ratis_max_retry_attempts", + String.valueOf(conf.getDataRegionRatisMaxRetryAttempts())))); + conf.setDataRegionRatisInitialSleepTimeMs( + Long.parseLong( + properties.getProperty( + "data_region_ratis_initial_sleep_time_ms", + String.valueOf(conf.getDataRegionRatisInitialSleepTimeMs())))); + conf.setDataRegionRatisMaxSleepTimeMs( + Long.parseLong( + properties.getProperty( + "data_region_ratis_max_sleep_time_ms", + String.valueOf(conf.getDataRegionRatisMaxSleepTimeMs())))); + + conf.setSchemaRegionRatisMaxRetryAttempts( + Integer.parseInt( + properties.getProperty( + "schema_region_ratis_max_retry_attempts", + String.valueOf(conf.getSchemaRegionRatisMaxRetryAttempts())))); + conf.setSchemaRegionRatisInitialSleepTimeMs( + Long.parseLong( + properties.getProperty( + "schema_region_ratis_initial_sleep_time_ms", + String.valueOf(conf.getSchemaRegionRatisInitialSleepTimeMs())))); + conf.setSchemaRegionRatisMaxSleepTimeMs( + Long.parseLong( + properties.getProperty( + "schema_region_ratis_max_sleep_time_ms", + String.valueOf(conf.getSchemaRegionRatisMaxSleepTimeMs())))); } + private void loadCQConfig(Properties properties) { + conf.setCqSubmitThread( + Integer.parseInt( + properties.getProperty( + "continuous_query_submit_thread", String.valueOf(conf.getCqSubmitThread())))); + + conf.setCqMinEveryIntervalInMs( + Integer.parseInt( + properties.getProperty( + "continuous_query_min_every_interval_in_ms", + String.valueOf(conf.getCqMinEveryIntervalInMs())))); + } + /** * Check if the current ConfigNode is SeedConfigNode. * diff --cc confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 4508c3f91a,29da4384d9..be8fad06e7 --- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@@ -88,12 -88,8 +88,13 @@@ public enum ConfigPhysicalPlanType DeleteTriggerInTable, GetTriggerTable, UpdateTriggerStateInTable, + GetTriggerJar, GetRouting, GetSeriesSlotList, - GetTimeSlotList + GetTimeSlotList, + DROP_CQ, + ACTIVE_CQ, + ADD_CQ, + UPDATE_CQ_LAST_EXEC_TIME, + SHOW_CQ } diff --cc confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java index 6e657a4518,5c4881ca77..553b00239c --- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/DataNodeRegisterResp.java @@@ -36,9 -36,8 +37,10 @@@ public class DataNodeRegisterResp imple private Integer dataNodeId; private TGlobalConfig globalConfig; private TRatisConfig ratisConfig; + + private TCQConfig cqConfig; private byte[] templateInfo; + private List<ByteBuffer> allTriggerInformation; public DataNodeRegisterResp() { this.dataNodeId = null; @@@ -88,7 -91,7 +98,8 @@@ resp.setGlobalConfig(globalConfig); resp.setTemplateInfo(templateInfo); resp.setRatisConfig(ratisConfig); + resp.setCqConfig(cqConfig); + resp.setAllTriggerInformation(allTriggerInformation); } return resp; diff --cc confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java index be4936ad5e,bcaba4fec8..9dacd6bc5f --- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java @@@ -151,16 -158,19 +158,21 @@@ public class PartitionRegionStateMachin configManager.getProcedureManager().shiftExecutor(true); configManager.getLoadManager().startLoadBalancingService(); configManager.getNodeManager().startHeartbeatService(); + configManager.getNodeManager().startUnknownDataNodeDetector(); configManager.getPartitionManager().startRegionCleaner(); + configManager.getCQManager().startCQScheduler(); } else { LOGGER.info( - "Current node {} is not longer the leader, the new leader is {}", currentNode, newLeader); + "Current node [nodeId:{}, ip:port: {}] is not longer the leader, the new leader is [nodeId:{}]", + currentNodeId, + currentNodeTEndPoint, + newLeaderId); configManager.getProcedureManager().shiftExecutor(false); configManager.getLoadManager().stopLoadBalancingService(); configManager.getNodeManager().stopHeartbeatService(); + configManager.getNodeManager().stopUnknownDataNodeDetector(); configManager.getPartitionManager().stopRegionCleaner(); + configManager.getCQManager().stopCQScheduler(); } } diff --cc confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f416ec34f3,a398b6f64c..eb7973a871 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@@ -89,7 -86,7 +88,8 @@@ import org.apache.iotdb.confignode.pers import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; import org.apache.iotdb.confignode.persistence.sync.ClusterSyncInfo; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; @@@ -175,9 -172,8 +177,11 @@@ public class ConfigManager implements I /** Sync */ private final SyncManager syncManager; + /** CQ */ + private final CQManager cqManager; + + private final PartitionRegionStateMachine stateMachine; + public ConfigManager() throws IOException { // Build the persistence module NodeInfo nodeInfo = new NodeInfo(); @@@ -200,9 -195,8 +204,9 @@@ procedureInfo, udfInfo, triggerInfo, - syncInfo); + syncInfo, + cqInfo); - PartitionRegionStateMachine stateMachine = new PartitionRegionStateMachine(this, executor); + this.stateMachine = new PartitionRegionStateMachine(this, executor); // Build the manager module this.nodeManager = new NodeManager(this, nodeInfo); @@@ -214,11 -208,10 +218,11 @@@ this.triggerManager = new TriggerManager(this, triggerInfo); this.loadManager = new LoadManager(this); this.syncManager = new SyncManager(this, syncInfo); + this.cqManager = new CQManager(this); + } - // ConsensusManager must be initialized last, as it would load states from disk and reinitialize - // above managers - this.consensusManager = new ConsensusManager(this, stateMachine); + public void initConsensusManager() throws IOException { + this.consensusManager = new ConsensusManager(this, this.stateMachine); } public void close() throws IOException { diff --cc confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 75ba20d76f,6330eabdeb..3d7e055e41 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@@ -48,7 -48,7 +49,8 @@@ import org.apache.iotdb.confignode.mana import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateTriggerReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; @@@ -473,9 -469,7 +480,13 @@@ public interface IManager TGetSeriesSlotListResp getSeriesSlotList(GetSeriesSlotListPlan plan); + TSStatus createCQ(TCreateCQReq req); + + TSStatus dropCQ(TDropCQReq req); + + TShowCQResp showCQ(); ++ + TSStatus checkConfigNodeGlobalConfig(TConfigNodeRegisterReq req); + + TSStatus transfer(List<TDataNodeLocation> newUnknownDataList); } diff --cc confignode/src/main/java/org/apache/iotdb/confignode/manager/TriggerManager.java index e60915cc2a,acce3d6fc8..7fc4b3ba8b --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/TriggerManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/TriggerManager.java @@@ -41,7 -45,7 +45,8 @@@ import org.slf4j.LoggerFactory import java.io.IOException; import java.util.Collections; + import java.util.List; +import java.util.Optional; public class TriggerManager { private static final Logger LOGGER = LoggerFactory.getLogger(TriggerManager.class); diff --cc confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 6470d5f9a5,b3fcb65ef7..2ef6251d36 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@@ -59,8 -60,9 +60,10 @@@ import org.apache.iotdb.confignode.mana import org.apache.iotdb.confignode.persistence.NodeInfo; import org.apache.iotdb.confignode.persistence.metric.NodeInfoMetrics; import org.apache.iotdb.confignode.procedure.env.DataNodeRemoveHandler; +import org.apache.iotdb.confignode.rpc.thrift.TCQConfig; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; diff --cc confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 808d59e7e9,ee060dc306..0966636fc0 --- a/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@@ -70,8 -71,8 +71,9 @@@ import org.apache.iotdb.confignode.rpc. import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateTriggerReq; diff --cc confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index d4d7ce7769,ed8f2a26c8..ec3a4559bb --- a/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@@ -1006,70 -1002,18 +1008,82 @@@ public class ConfigPhysicalPlanSerDeTes } @Test + public void ActiveCQPlanTest() throws IOException { + ActiveCQPlan activeCQPlan0 = new ActiveCQPlan("testCq", "testCq_md5"); + ActiveCQPlan activeCQPlan1 = + (ActiveCQPlan) ConfigPhysicalPlan.Factory.create(activeCQPlan0.serializeToByteBuffer()); + + Assert.assertEquals(activeCQPlan0, activeCQPlan1); + } + + @Test + public void AddCQPlanTest() throws IOException { + long executionTime = System.currentTimeMillis(); + AddCQPlan addCQPlan0 = + new AddCQPlan( + new TCreateCQReq( + "testCq1", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select s1 into root.backup.d1.s1 from root.sg.d1", + "create cq testCq1 BEGIN select s1 into root.backup.d1.s1 from root.sg.d1 END", + "Asia"), + "testCq1_md5", + executionTime); + AddCQPlan addCQPlan1 = + (AddCQPlan) ConfigPhysicalPlan.Factory.create(addCQPlan0.serializeToByteBuffer()); + + Assert.assertEquals(addCQPlan0, addCQPlan1); + } + + @Test + public void DropCQPlanTest() throws IOException { + DropCQPlan dropCQPlan0 = new DropCQPlan("testCq1"); + DropCQPlan dropCQPlan1 = + (DropCQPlan) ConfigPhysicalPlan.Factory.create(dropCQPlan0.serializeToByteBuffer()); + Assert.assertEquals(dropCQPlan0, dropCQPlan1); + + dropCQPlan0 = new DropCQPlan("testCq1", "testCq1_md5"); + dropCQPlan1 = + (DropCQPlan) ConfigPhysicalPlan.Factory.create(dropCQPlan0.serializeToByteBuffer()); + Assert.assertEquals(dropCQPlan0, dropCQPlan1); + } + + @Test + public void ShowCQPlanTest() throws IOException { + ShowCQPlan showCQPlan0 = new ShowCQPlan(); + ShowCQPlan showCQPlan1 = + (ShowCQPlan) ConfigPhysicalPlan.Factory.create(showCQPlan0.serializeToByteBuffer()); + + Assert.assertEquals(showCQPlan0, showCQPlan1); + } + + @Test + public void UpdateCQLastExecTimePlanTest() throws IOException { + UpdateCQLastExecTimePlan updateCQLastExecTimePlan0 = + new UpdateCQLastExecTimePlan("testCq", System.currentTimeMillis(), "testCq_md5"); + UpdateCQLastExecTimePlan updateCQLastExecTimePlan1 = + (UpdateCQLastExecTimePlan) + ConfigPhysicalPlan.Factory.create(updateCQLastExecTimePlan0.serializeToByteBuffer()); + + Assert.assertEquals(updateCQLastExecTimePlan0, updateCQLastExecTimePlan1); + } + + public void GetTriggerJarPlanTest() throws IOException { + List<String> jarNames = new ArrayList<>(); + jarNames.add("test1"); + jarNames.add("test2"); + GetTriggerJarPlan getTriggerJarPlan0 = new GetTriggerJarPlan(jarNames); + + GetTriggerJarPlan getTriggerJarPlan1 = + (GetTriggerJarPlan) + ConfigPhysicalPlan.Factory.create(getTriggerJarPlan0.serializeToByteBuffer()); + Assert.assertEquals(getTriggerJarPlan0.getJarNames(), getTriggerJarPlan1.getJarNames()); + } + @Test public void GetRoutingPlanTest() throws IOException { GetRoutingPlan getRoutingPlan0 = diff --cc server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java index 8b231878b2,fb8c87da7b..93971e6829 --- a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java +++ b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java @@@ -39,8 -38,8 +38,9 @@@ import org.apache.iotdb.confignode.rpc. import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; + import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateTriggerReq; diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index a87242f0ab,b538cf9626..0e48194172 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@@ -1034,9 -1038,16 +1038,19 @@@ public class IoTDBConfig private long dataRatisConsensusLeaderElectionTimeoutMaxMs = 4000L; private long schemaRatisConsensusLeaderElectionTimeoutMaxMs = 4000L; + /** CQ related */ + private long cqMinEveryIntervalInMs = 1_000; + + private long dataRatisConsensusRequestTimeoutMs = 10000L; + private long schemaRatisConsensusRequestTimeoutMs = 10000L; + + private int dataRatisConsensusMaxRetryAttempts = 10; + private int schemaRatisConsensusMaxRetryAttempts = 10; + private long dataRatisConsensusInitialSleepTimeMs = 100L; + private long schemaRatisConsensusInitialSleepTimeMs = 100L; + private long dataRatisConsensusMaxSleepTimeMs = 10000L; + private long schemaRatisConsensusMaxSleepTimeMs = 10000L; + IoTDBConfig() {} public float getUdfMemoryBudgetInMB() { @@@ -3432,11 -3434,80 +3437,88 @@@ schemaRatisConsensusLeaderElectionTimeoutMaxMs; } + public long getCqMinEveryIntervalInMs() { + return cqMinEveryIntervalInMs; + } + + public void setCqMinEveryIntervalInMs(long cqMinEveryIntervalInMs) { + this.cqMinEveryIntervalInMs = cqMinEveryIntervalInMs; + } ++ + public double getUsableCompactionMemoryProportion() { + return 1.0d - chunkMetadataSizeProportion; + } + + public int getPatternMatchingThreshold() { + return patternMatchingThreshold; + } + + public void setPatternMatchingThreshold(int patternMatchingThreshold) { + this.patternMatchingThreshold = patternMatchingThreshold; + } + + public long getDataRatisConsensusRequestTimeoutMs() { + return dataRatisConsensusRequestTimeoutMs; + } + + public void setDataRatisConsensusRequestTimeoutMs(long dataRatisConsensusRequestTimeoutMs) { + this.dataRatisConsensusRequestTimeoutMs = dataRatisConsensusRequestTimeoutMs; + } + + public long getSchemaRatisConsensusRequestTimeoutMs() { + return schemaRatisConsensusRequestTimeoutMs; + } + + public void setSchemaRatisConsensusRequestTimeoutMs(long schemaRatisConsensusRequestTimeoutMs) { + this.schemaRatisConsensusRequestTimeoutMs = schemaRatisConsensusRequestTimeoutMs; + } + + public int getDataRatisConsensusMaxRetryAttempts() { + return dataRatisConsensusMaxRetryAttempts; + } + + public void setDataRatisConsensusMaxRetryAttempts(int dataRatisConsensusMaxRetryAttempts) { + this.dataRatisConsensusMaxRetryAttempts = dataRatisConsensusMaxRetryAttempts; + } + + public int getSchemaRatisConsensusMaxRetryAttempts() { + return schemaRatisConsensusMaxRetryAttempts; + } + + public void setSchemaRatisConsensusMaxRetryAttempts(int schemaRatisConsensusMaxRetryAttempts) { + this.schemaRatisConsensusMaxRetryAttempts = schemaRatisConsensusMaxRetryAttempts; + } + + public long getDataRatisConsensusInitialSleepTimeMs() { + return dataRatisConsensusInitialSleepTimeMs; + } + + public void setDataRatisConsensusInitialSleepTimeMs(long dataRatisConsensusInitialSleepTimeMs) { + this.dataRatisConsensusInitialSleepTimeMs = dataRatisConsensusInitialSleepTimeMs; + } + + public long getSchemaRatisConsensusInitialSleepTimeMs() { + return schemaRatisConsensusInitialSleepTimeMs; + } + + public void setSchemaRatisConsensusInitialSleepTimeMs( + long schemaRatisConsensusInitialSleepTimeMs) { + this.schemaRatisConsensusInitialSleepTimeMs = schemaRatisConsensusInitialSleepTimeMs; + } + + public long getDataRatisConsensusMaxSleepTimeMs() { + return dataRatisConsensusMaxSleepTimeMs; + } + + public void setDataRatisConsensusMaxSleepTimeMs(long dataRatisConsensusMaxSleepTimeMs) { + this.dataRatisConsensusMaxSleepTimeMs = dataRatisConsensusMaxSleepTimeMs; + } + + public long getSchemaRatisConsensusMaxSleepTimeMs() { + return schemaRatisConsensusMaxSleepTimeMs; + } + + public void setSchemaRatisConsensusMaxSleepTimeMs(long schemaRatisConsensusMaxSleepTimeMs) { + this.schemaRatisConsensusMaxSleepTimeMs = schemaRatisConsensusMaxSleepTimeMs; + } } diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 1b9b4e3c30,19731bc330..6913df7cb0 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@@ -1950,12 -1952,19 +1953,23 @@@ public class IoTDBDescriptor ratisConfig.getDataLeaderElectionTimeoutMax()); conf.setSchemaRatisConsensusLeaderElectionTimeoutMaxMs( ratisConfig.getSchemaLeaderElectionTimeoutMax()); + + conf.setDataRatisConsensusRequestTimeoutMs(ratisConfig.getDataRequestTimeout()); + conf.setSchemaRatisConsensusRequestTimeoutMs(ratisConfig.getSchemaRequestTimeout()); + + conf.setDataRatisConsensusMaxRetryAttempts(ratisConfig.getDataMaxRetryAttempts()); + conf.setDataRatisConsensusInitialSleepTimeMs(ratisConfig.getDataInitialSleepTime()); + conf.setDataRatisConsensusMaxSleepTimeMs(ratisConfig.getDataMaxSleepTime()); + + conf.setSchemaRatisConsensusMaxRetryAttempts(ratisConfig.getSchemaMaxRetryAttempts()); + conf.setSchemaRatisConsensusInitialSleepTimeMs(ratisConfig.getSchemaInitialSleepTime()); + conf.setSchemaRatisConsensusMaxSleepTimeMs(ratisConfig.getSchemaMaxSleepTime()); } + public void loadCQConfig(TCQConfig cqConfig) { + conf.setCqMinEveryIntervalInMs(cqConfig.getCqMinEveryIntervalInMs()); + } + public void reclaimConsensusMemory() { conf.setAllocateMemoryForStorageEngine( conf.getAllocateMemoryForStorageEngine() + conf.getAllocateMemoryForConsensus()); diff --cc thrift-confignode/src/main/thrift/confignode.thrift index fac5941545,d3d110d4bc..c5c5c5f0ac --- a/thrift-confignode/src/main/thrift/confignode.thrift +++ b/thrift-confignode/src/main/thrift/confignode.thrift @@@ -36,7 -36,7 +36,8 @@@ struct TDataNodeRegisterResp 4: optional TGlobalConfig globalConfig 5: optional binary templateInfo 6: optional TRatisConfig ratisConfig - 7: optional TCQConfig cqConfig + 7: optional list<binary> allTriggerInformation ++ 8: optional TCQConfig cqConfig } struct TGlobalConfig { @@@ -70,12 -70,18 +71,22 @@@ struct TRatisConfig 13: required i64 schemaLeaderElectionTimeoutMax 14: required i64 dataLeaderElectionTimeoutMax + + 15: required i64 schemaRequestTimeout + 16: required i64 dataRequestTimeout + + 17: required i32 schemaMaxRetryAttempts + 18: required i32 dataMaxRetryAttempts + 19: required i64 schemaInitialSleepTime + 20: required i64 dataInitialSleepTime + 21: required i64 schemaMaxSleepTime + 22: required i64 dataMaxSleepTime } +struct TCQConfig { + 1: required i64 cqMinEveryIntervalInMs +} + struct TDataNodeRemoveReq { 1: required list<common.TDataNodeLocation> dataNodeLocations }
