This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch batch_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 18cac0053f4 fix confict
add ba853b3147b Pipe: Fixed the bug that aggregate processor cannot clear
the last received time after drop (#12383)
add 9051aa57908 Update iotdb-doap.rdf for v1.3.1 release (#12390)
add b947dd83b6c Remove useless try-catch for RegionPriorityMap (#12389)
add 7f4724dd3d7 Enhance concurrent control of data partition policy table
(#12388)
add ac648a3a2a0 Deduplication IoTConsensus Configuration And Remove
Unuseless File Delete (#12385)
add 3bfc0ea84a5 Provide more precise info for exception during conf
loading (#12382)
add e825af4aade feature add iotdb backup (#12365)
add b7e9d6ec3d5 Some region migration related work (#12376)
add 6b2e4b15925 Fix effective ratio calculation method (#12394)
add 1ce494b5f17 Fixed backup data backend running issues (#12396)
add 74eb4e2e392 Pipe: Fixed the bug that pure data transfer is forbidden
when SchemaRegion is in Simple consensus (#12397)
add 638be8ae212 Pipe: Added batch to schema snapshot execution in template
activation and timeseries creation & Fixed the bug that the
"CreateMultiTimeSeries" group with alias is not idempotent (#12380)
add bd22fe65461 Pipe: distinguish between old and new process subtasks
before and after stuck restart to avoid unexpected hashmap override and removal
(#12400)
add c36f7144be7 Fixed concurrency issues with RegionMigration thread pool
(#12403)
add 4d7362659d6 Fixed insertTablet http api lost data event return
SUCCESS_STATUS (#12387)
add ed6f56b25ab Fix partition cache after region migration (#12404)
add e1e0055c940 Fix IoTDBTimezoneIT
add c911c406bd1 Pipe: Fixed FileNotFoundException caused by processing
files deleted by compaction & Changed the "isEmpty()" judgement to avoid
"getDevice()" attempts to read from a deleted file when using fileTimeIndex
(#12405)
add 719834aea89 Pipe: refine procedure timeout message (#12402)
add f5cf782df5b Pipe: Two stage aggregate framework &
count-point-processor plugin (#12328)
add 03064328b43 Change log level and opt error msg for source path of view
is deleted (#12409)
add e715db52084 Pipe: remove unnecessary fsync operations for file pieces'
transferring (#12410)
add c232ee02239 fix pipe snapshot parser
add 8064ac58aa3 Pipe: Properly handle OUT_OF_TTL status code on receiver
side when data syncing (#12414)
add a8411f66837 Fix Concurrency Instability When IoTConsensus
LogDispatcher Exits (#12411)
add 544a3bccbb9 update Export/ImportDataTestIT (#12399)
add 1d30805f266 Pipe: Added check for wal disable and ratis consensus when
enabling data region processing (#12407)
add 447bf54a356 revert -XX:GuaranteedSafepointInterval=0 option (#12417)
add 95384d4a9eb Randomly choosing coordinator of add region (#12416)
add d23729f876a [RatisConsensus] update ratis snapshot version (#12419)
add 98b2272dbc1 Pipe: Reduce exception messages to avoid excess rpc
payload and show pipe response (#12415)
add 52bc1889d53 Copy tsfile, its resources, and mods to the same directory
when using multiple data directories (#12420)
add d866bc5dbaf fix snapshot parser null point.
add 1b981972bd7 Pipe: Support InsertRows in
PipeInsertNodeTabletInsertionEvent (#12286)
add 5a3134c277b Merge branch 'master' of github.com:apache/iotdb into
batch_wal
add 2aa0e46a1bf adapt pipe
No new revisions were added by this update.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java | 2 +-
.../apache/iotdb/tools/it/ExportDataTestIT.java | 6 +-
.../apache/iotdb/tools/it/ImportDataTestIT.java | 4 +-
.../cli/src/assembly/resources/tools/backup.bat | 125 +++
.../cli/src/assembly/resources/tools/backup.sh | 134 +++
.../org/apache/iotdb/tool/AbstractDataTool.java | 1 +
.../java/org/apache/iotdb/tool/ExportData.java | 21 +-
.../java/org/apache/iotdb/tool/ImportData.java | 73 +-
.../org/apache/iotdb/tool/IoTDBDataBackTool.java | 985 +++++++++++++++++++++
.../confignode/conf/ConfigNodeDescriptor.java | 1 +
.../confignode/conf/ConfigNodeRemoveCheck.java | 5 +-
.../confignode/conf/SystemPropertiesUtils.java | 3 +-
.../consensus/request/ConfigPhysicalPlan.java | 1 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/ConfigPhysicalPlanVisitor.java | 6 +
.../consensus/request/auth/AuthorPlan.java | 3 +
.../response/pipe/task/PipeTableResp.java | 8 +-
.../confignode/manager/PermissionManager.java | 3 +-
.../iotdb/confignode/manager/ProcedureManager.java | 93 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/PartitionBalancer.java | 10 +-
.../manager/load/balancer/RouteBalancer.java | 32 +-
.../manager/partition/PartitionManager.java | 14 +-
.../receiver/IoTDBConfigNodeReceiverAgent.java | 6 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 5 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 53 +-
.../pipe/event/PipeConfigRegionWritePlanEvent.java | 46 +-
.../extractor/ConfigRegionListeningFilter.java | 5 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 13 +-
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 1 +
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 9 +
.../iotdb/confignode/persistence/AuthorInfo.java | 3 +
.../persistence/executor/ConfigPlanExecutor.java | 1 +
.../partition/DatabasePartitionTable.java | 11 +-
.../persistence/partition/PartitionInfo.java | 11 +-
.../schema/CNPhysicalPlanGenerator.java | 9 +-
.../procedure/env/RegionMaintainHandler.java | 122 ++-
.../impl/pipe/task/AlterPipeProcedureV2.java | 2 +-
.../impl/region/AddRegionPeerProcedure.java | 28 +-
.../impl/region/RemoveRegionPeerProcedure.java | 2 +-
.../confignode/persistence/AuthorInfoTest.java | 21 +
.../persistence/CNPhysicalPlanGeneratorTest.java | 3 +
.../consensus/iot/IoTConsensusServerImpl.java | 47 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 8 +
.../apache/iotdb/consensus/iot/ReplicateTest.java | 36 +-
.../src/assembly/resources/conf/datanode-env.bat | 3 -
.../src/assembly/resources/conf/datanode-env.sh | 3 -
.../assembly/resources/conf/logback-datanode.xml | 15 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +-
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 1 +
.../metadata/view/BrokenViewException.java | 13 +-
.../PipeDataRegionProcessorConstructor.java | 3 +
.../agent/receiver/PipeDataNodeReceiverAgent.java | 5 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 9 +
.../request/PipeTransferTabletInsertNodeReq.java | 46 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 9 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 5 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 14 +-
.../connector/protocol/opcua/OpcUaConnector.java | 4 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 10 +-
.../protocol/websocket/WebSocketConnector.java | 17 +-
.../websocket/WebSocketConnectorServer.java | 5 +-
.../db/pipe/event/common/row/PipeRowCollector.java | 4 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 64 +-
.../schema/PipeSchemaRegionWritePlanEvent.java | 44 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 122 ++-
.../common/tablet/PipeRawTabletInsertionEvent.java | 5 +
.../tablet/TabletInsertionDataContainer.java | 15 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 34 +
.../common/tsfile/TsFileInsertionPointCounter.java | 169 ++++
.../common/watermark/PipeWatermarkEvent.java} | 21 +-
.../dataregion/DataRegionWatermarkInjector.java | 76 ++
.../dataregion/IoTDBDataRegionExtractor.java | 82 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 24 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 6 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 13 +-
.../processor/aggregate/AggregateProcessor.java | 3 +-
.../pipe/processor/twostage/combiner/Combiner.java | 107 +++
.../twostage/combiner/PipeCombineHandler.java | 198 +++++
.../combiner/PipeCombineHandlerManager.java | 163 ++++
.../twostage/exchange/payload/CombineRequest.java | 139 +++
.../payload/FetchCombineResultRequest.java | 119 +++
.../payload/FetchCombineResultResponse.java | 99 +++
.../twostage/exchange/payload/RequestType.java | 16 +-
.../receiver/TwoStageAggregateReceiver.java | 82 ++
.../exchange/sender/TwoStageAggregateSender.java | 235 +++++
.../processor/twostage/operator/CountOperator.java | 32 +-
.../processor/twostage/operator/Operator.java} | 12 +-
.../twostage/plugin/TwoStageCountProcessor.java | 391 ++++++++
.../processor/twostage/state/CountState.java} | 26 +-
.../db/pipe/processor/twostage/state/State.java} | 11 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 72 +-
.../thrift/IoTDBDataNodeReceiverAgent.java | 3 +
.../visitor/PipePlanToStatementVisitor.java | 105 ++-
.../visitor/PipeStatementTSStatusVisitor.java | 57 +-
.../visitor/PipeStatementToBatchVisitor.java | 185 ++++
.../pipe/task/connection/PipeEventCollector.java | 3 +-
.../subtask/processor/PipeProcessorSubtask.java | 57 +-
.../rest/utils/InsertTabletSortDataUtils.java | 16 +-
.../execution/executor/RegionWriteExecutor.java | 12 +-
.../analyze/cache/partition/PartitionCache.java | 1 +
.../visitor/CompleteMeasurementSchemaVisitor.java | 6 +-
.../planner/plan/node/write/InsertRowsNode.java | 9 +-
.../iotdb/db/service/RegionMigrateService.java | 10 +-
.../dataregion/memtable/TsFileProcessor.java | 24 +-
.../dataregion/snapshot/SnapshotLoader.java | 12 +-
.../dataregion/tsfile/TsFileResource.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 7 +-
.../broker/SubscriptionPrefetchingQueue.java | 28 +-
.../db/tools/schema/SRStatementGenerator.java | 104 ++-
.../db/auth/user/LocalFileUserManagerTest.java | 8 +
.../extractor/IoTDBDataRegionExtractorTest.java | 32 +-
.../dataregion/wal/WALManagerTest.java | 5 +
.../dataregion/wal/node/WALNodeTest.java | 5 +
.../wal/node/WalDeleteOutdatedNewTest.java | 5 +
.../db/utils/SchemaRegionSnapshotParserTest.java | 86 +-
.../src/assembly/resources/sbin/start-all.sh | 2 +-
.../src/assembly/resources/sbin/stop-all.sh | 2 +-
.../commons/auth/authorizer/BasicAuthorizer.java | 12 +-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 2 +
.../iotdb/commons/auth/user/BasicUserManager.java | 16 +-
.../iotdb/commons/auth/user/IUserManager.java | 3 +-
.../commons/concurrent/IoTDBThreadPoolFactory.java | 12 -
.../apache/iotdb/commons/conf/CommonConfig.java | 41 +
.../iotdb/commons/conf/CommonDescriptor.java | 22 +
.../commons/consensus/index/ProgressIndex.java | 7 +-
.../commons/consensus/index/ProgressIndexType.java | 7 +-
.../consensus/index/impl/HybridProgressIndex.java | 4 +
.../consensus/index/impl/MetaProgressIndex.java | 2 +-
.../consensus/index/impl/StateProgressIndex.java | 234 +++++
.../iotdb/commons/pipe/config/PipeConfig.java | 30 +
.../config/constant/PipeExtractorConstant.java | 4 +
.../config/constant/PipeProcessorConstant.java | 2 +
.../request/IoTDBConnectorRequestVersion.java | 1 +
.../commons/pipe/event/PipeSnapshotEvent.java | 19 +-
.../commons/pipe/event/PipeWritePlanEvent.java | 35 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 2 +
.../TwoStageCountProcessor.java} | 8 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 20 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 29 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 28 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 18 +
iotdb-doap.rdf | 26 +-
pom.xml | 2 +-
146 files changed, 5086 insertions(+), 828 deletions(-)
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
create mode 100755 iotdb-client/cli/src/assembly/resources/tools/backup.sh
create mode 100644
iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionPointCounter.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/schedule/constant/CompactionType.java
=> pipe/event/common/watermark/PipeWatermarkEvent.java} (68%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultResponse.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeResponseType.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/RequestType.java
(77%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/receiver/TwoStageAggregateReceiver.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/sender/TwoStageAggregateSender.java
copy
integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/Cluster1Env.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/CountOperator.java
(52%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/PipeTask.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/Operator.java}
(80%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/template/alter/TemplateAlterInfo.java
=> pipe/processor/twostage/state/CountState.java} (69%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/IProcedureFactory.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/state/State.java}
(79%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementToBatchVisitor.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/StateProgressIndex.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/{downsampling/SwingingDoorTrendingSamplingProcessor.java
=> twostage/TwoStageCountProcessor.java} (79%)
