This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch fix_LZ4Uncompressor
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 824a2385f6e Fix the mistaken argument in LZ4Uncompressor
add 81f1cd725d2 Fix uncompress byte buffer in RepairDataFileScanUtil
add cdf002fb914 Pipe: Support tsfile mod & schema region snapshot & config
region snapshot transferring logic (#11998)
add 49e199ef0ea fix: consider more general type of nodes in optimization
above LimitNode (#12240)
add db9530568d3 Forcibly take snapshot when upgrade procedure WAL (#12252)
add ad75bb3a48c Merge branch 'master' into fix_LZ4Uncompressor
No new revisions were added by this update.
Summary of changes:
.github/workflows/pipe-it-2cluster.yml | 82 ++++++-
integration-test/pom.xml | 34 ++-
...1.java => MultiClusterIT2AutoCreateSchema.java} | 2 +-
...java => MultiClusterIT2ManualCreateSchema.java} | 2 +-
...erIT2.java => MultiClusterIT2Subscription.java} | 2 +-
.../scalar/IoTDBSubStringFunctionIT.java | 6 +-
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 16 +-
.../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 4 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 4 +-
.../autocreate/IoTDBPipeConnectorParallelIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 23 +-
.../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeNullValueIT.java | 4 +-
.../it/autocreate/IoTDBPipePatternFormatIT.java | 7 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 17 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 36 ++-
.../it/autocreate/IoTDBPipeSwitchStatusIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeSyntaxIT.java | 4 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 16 +-
.../iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java | 6 +-
.../pipe/it/manual/IoTDBPipeManualConflictIT.java | 6 +-
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 237 +++++++++++++++++++
.../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 6 +-
.../pipe/it/manual/IoTDBPipeMetaRestartIT.java | 14 +-
.../it/manual/IoTDBPipeMultiSchemaRegionIT.java | 6 +-
.../subscription/it/IoTDBSubscriptionBasicIT.java | 52 +++--
.../it/IoTDBSubscriptionConsumerGroupIT.java | 125 +++++-----
.../subscription/it/IoTDBSubscriptionTopicIT.java | 74 +++---
.../org/apache/iotdb/pipe/api/PipeExtractor.java | 2 +-
.../event/dml/insertion/TabletInsertionEvent.java | 12 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 4 +-
.../flink/sql/function/IoTDBCDCSourceFunction.java | 1 +
.../consensus/request/auth/AuthorPlan.java | 2 +-
.../statemachine/ConfigRegionStateMachine.java | 30 ++-
.../manager/consensus/ConsensusManager.java | 3 +-
.../runtime/PipeLeaderChangeHandler.java | 23 +-
.../runtime/PipeRuntimeCoordinator.java | 6 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 189 ++++++++++++++-
.../PipeConfigNodeCopiedFileDirStartupCleaner.java | 52 +++++
.../agent/receiver/IoTDBConfigNodeReceiver.java | 97 ++++++--
.../receiver/IoTDBConfigNodeReceiverAgent.java | 23 --
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 9 +-
.../agent/runtime/PipeConfigNodeRuntimeAgent.java | 6 +-
.../agent/task/PipeConfigNodeTaskAgent.java | 3 +-
.../config/IoTDBConfigRegionAirGapConnector.java | 81 ++++---
.../config/IoTDBConfigRegionConnector.java | 124 ++++------
.../request/PipeTransferConfigSnapshotSealReq.java | 55 ++++-
.../transfer/execution/PipeConfigNodeSubtask.java | 8 -
.../extractor/ConfigRegionListeningFilter.java | 2 +-
.../extractor/ConfigRegionListeningQueue.java | 40 +++-
.../extractor/IoTDBConfigRegionExtractor.java | 47 ++++
.../transfer/task/PipeConfigNodeTaskBuilder.java | 9 +-
.../transfer/task/PipeConfigNodeTaskStage.java | 4 +-
.../manager/schema/ClusterSchemaManager.java | 2 +-
.../iotdb/confignode/persistence/AuthorInfo.java | 4 +-
.../confignode/persistence/ProcedureInfo.java | 2 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +-
.../schema/CNPhysicalPlanGenerator.java | 81 +++++--
.../persistence/schema/CNSnapshotFileType.java | 35 ++-
.../schema/ConfignodeSnapshotParser.java | 32 +--
.../persistence/schema/TemplateTable.java | 2 +-
.../iotdb/confignode/service/ConfigNode.java | 9 +-
.../pipe/PipeConfigNodeThriftRequestTest.java | 15 +-
.../persistence/CNPhysicalPlanGeneratorTest.java | 142 +++++++-----
.../confignode/persistence/ClusterInfoTest.java | 2 +-
.../confignode/persistence/ProcedureInfoTest.java | 2 +-
.../UpgradeFromWALToConsensusLayerTest.java | 6 +-
.../org/apache/iotdb/consensus/IConsensus.java | 6 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 2 +-
.../apache/iotdb/consensus/ratis/DiskGuardian.java | 17 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 4 +-
.../iotdb/consensus/ratis/SnapshotStorage.java | 8 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 10 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 26 +--
.../dataregion/DataExecutionVisitor.java | 11 +-
.../schemaregion/SchemaRegionStateMachine.java | 47 +++-
.../agent/receiver/PipeDataNodeReceiverAgent.java | 21 +-
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 8 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 3 +-
.../client/IoTDBDataNodeAsyncClientManager.java | 1 +
.../request/PipeTransferSchemaSnapshotSealReq.java | 52 ++++-
...java => PipeTransferTsFilePieceWithModReq.java} | 22 +-
.../request/PipeTransferTsFileSealReq.java | 4 +-
....java => PipeTransferTsFileSealWithModReq.java} | 43 ++--
.../airgap/IoTDBDataNodeAirGapConnector.java | 11 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 102 +++++----
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 77 ++++---
.../connector/protocol/opcua/OpcUaConnector.java | 8 +-
.../async/IoTDBDataRegionAsyncConnector.java | 4 +
.../PipeTransferTabletBatchEventHandler.java | 13 +-
.../PipeTransferTabletInsertionEventHandler.java | 11 +-
.../PipeTransferTsFileInsertionEventHandler.java | 82 +++++--
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 17 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 172 +++++++-------
.../thrift/sync/IoTDBSchemaRegionConnector.java | 113 ++++-----
.../schema/PipeSchemaRegionSnapshotEvent.java | 157 ++++++++++++-
.../common/tsfile/PipeTsFileInsertionEvent.java | 60 ++++-
.../db/pipe/event/realtime/PipeRealtimeEvent.java | 2 +-
.../dataregion/IoTDBDataRegionExtractor.java | 12 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 39 +++-
.../realtime/PipeRealtimeDataRegionExtractor.java | 31 ++-
.../PipeRealtimeDataRegionLogExtractor.java | 2 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 7 +-
.../realtime/epoch/TsFileEpochManager.java | 2 +-
.../listener/PipeInsertionDataNodeListener.java | 4 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 58 +++--
.../schemaregion/SchemaRegionListeningFilter.java | 6 +-
.../schemaregion/SchemaRegionListeningQueue.java | 18 +-
.../processor/aggregate/AggregateProcessor.java | 8 +-
.../window/datastructure/TimeSeriesWindow.java | 3 -
.../pipe/receiver/PipePlanToStatementVisitor.java | 5 -
.../receiver/PipeStatementExceptionVisitor.java | 27 ++-
.../receiver/PipeStatementTSStatusVisitor.java | 6 +-
.../pipe/receiver/airgap/IoTDBAirGapReceiver.java | 12 +-
.../receiver/thrift/IoTDBDataNodeReceiver.java | 117 +++++-----
...NodeHardlinkOrCopiedFileDirStartupCleaner.java} | 26 ++-
.../resource/tsfile/PipeTsFileResourceManager.java | 70 +++---
.../db/pipe/task/builder/PipeDataNodeBuilder.java | 28 ++-
.../db/queryengine/execution/driver/Driver.java | 2 +-
.../fragment/FragmentInstanceExecution.java | 2 +-
.../execution/load/LoadTsFileManager.java | 2 +-
.../analyze/cache/partition/PartitionCache.java | 12 +-
.../config/executor/ClusterConfigTaskExecutor.java | 2 +-
.../plan/optimization/LimitOffsetPushDown.java | 48 ++--
.../db/queryengine/plan/parser/ASTVisitor.java | 59 ++---
.../plan/planner/plan/node/PlanNodeType.java | 4 +-
.../node/metedata/write/DeleteTimeSeriesNode.java | 6 +-
.../write/InternalCreateTimeSeriesNode.java | 6 +-
.../queryengine/plan/statement/StatementType.java | 12 +-
.../metadata/AlterTimeSeriesStatement.java | 4 +-
.../metadata/CreateAlignedTimeSeriesStatement.java | 2 +-
.../metadata/CreateMultiTimeSeriesStatement.java | 2 +-
.../metadata/CreateTimeSeriesStatement.java | 2 +-
.../metadata/DeleteTimeSeriesStatement.java | 2 +-
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 2 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 2 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 20 +-
.../schemaregion/mtree/traverser/Traverser.java | 13 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 14 +-
.../db/storageengine/dataregion/DataRegion.java | 71 +++---
.../compaction/repair/RepairDataFileScanUtil.java | 6 +-
.../dataregion/read/control/FileReaderManager.java | 4 +-
.../dataregion/snapshot/SnapshotTaker.java | 2 +-
.../wal/allocation/FirstCreateStrategy.java | 2 +-
.../dataregion/wal/recover/WALNodeRecoverTask.java | 2 +-
.../quotas/DataNodeThrottleQuotaManager.java | 14 +-
.../db/subscription/broker/SubscriptionBroker.java | 14 +-
.../db/tools/schema/SRStatementGenerator.java | 51 +++--
.../tools/schema/SchemaRegionSnapshotParser.java | 67 +++---
.../connector/PipeDataNodeThriftRequestTest.java | 36 ++-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 2 +-
.../PipeDataNodeSnapshotResourceManagerTest.java | 25 +-
.../resource/PipeTsFileResourceManagerTest.java | 2 +-
.../PipeWALHardlinkResourceManagerTest.java | 2 +-
.../dataregion/compaction/TsFileIdentifierUT.java | 2 +-
.../repair/RepairDataFileScanUtilTest.java | 29 +++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 10 +-
.../iotdb/db/tools/TsFileAndModSettleToolTest.java | 2 +-
.../apache/iotdb/db/utils/sort/SortUtilTest.java | 2 +-
.../apache/iotdb/metrics/utils/SystemMetric.java | 2 +-
.../commons/auth/role/LocalFileRoleAccessor.java | 6 +-
.../commons/auth/user/LocalFileUserAccessor.java | 8 +-
.../sync/SyncThriftClientWithErrorHandler.java | 1 +
.../iotdb/commons/consensus/ConsensusGroupId.java | 2 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 5 +-
.../config/constant/PipeExtractorConstant.java | 5 +
.../pipe/connector/PipeReceiverStatusHandler.java | 52 ++++-
.../pipe/connector/client/IoTDBClientManager.java | 8 +
.../connector/client/IoTDBSyncClientManager.java | 1 +
.../payload/thrift/request/PipeRequestType.java | 10 +-
...SealReq.java => PipeTransferFileSealReqV1.java} | 8 +-
.../thrift/request/PipeTransferFileSealReqV2.java | 165 +++++++++++++
.../connector/protocol/IoTDBAirGapConnector.java | 66 +++++-
.../connector/protocol/IoTDBSslSyncConnector.java | 77 +++++++
.../options/PipeInclusionOptions.java | 19 +-
.../listening/AbstractPipeListeningQueue.java | 78 ++-----
.../commons/pipe/event/PipeSnapshotEvent.java | 69 ++----
.../commons/pipe/extractor/IoTDBExtractor.java | 2 +-
.../extractor/IoTDBNonDataRegionExtractor.java | 62 ++++-
.../connector/donothing/DoNothingConnector.java | 16 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 254 ++++++++++++++++-----
.../commons/pipe/receiver/IoTDBReceiverAgent.java | 19 ++
.../pipe/resource/PipeSnapshotResourceManager.java | 62 ++---
.../commons/pipe/task/stage/PipeTaskStage.java | 4 +-
.../service/AbstractThriftServiceThread.java | 4 +-
.../commons/subscription/meta/topic/TopicMeta.java | 4 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 61 +++--
.../commons/utils/ThriftCommonsSerDeUtils.java | 1 +
192 files changed, 3560 insertions(+), 1653 deletions(-)
copy
integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT1.java
=> MultiClusterIT2AutoCreateSchema.java} (94%)
copy
integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT1.java
=> MultiClusterIT2ManualCreateSchema.java} (94%)
rename
integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2.java
=> MultiClusterIT2Subscription.java} (94%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/resource/PipeConfigNodeCopiedFileDirStartupCleaner.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/{PipeTransferSchemaSnapshotPieceReq.java
=> PipeTransferTsFilePieceWithModReq.java} (74%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/{PipeTransferSchemaSnapshotPieceReq.java
=> PipeTransferTsFileSealWithModReq.java} (53%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/{PipeHardlinkFileDirStartupCleaner.java
=> PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java} (66%)
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/thrift/request/{PipeTransferFileSealReq.java
=> PipeTransferFileSealReqV1.java} (91%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/thrift/request/PipeTransferFileSealReqV2.java
