This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/leader-write
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from e9963110be4 Merge branch 'master' into feature/leader-write
add ba853b3147b Pipe: Fixed the bug that aggregate processor cannot clear
the last received time after drop (#12383)
add 9051aa57908 Update iotdb-doap.rdf for v1.3.1 release (#12390)
add b947dd83b6c Remove useless try-catch for RegionPriorityMap (#12389)
add 7f4724dd3d7 Enhance concurrent control of data partition policy table
(#12388)
add ac648a3a2a0 Deduplication IoTConsensus Configuration And Remove
Unuseless File Delete (#12385)
add 3bfc0ea84a5 Provide more precise info for exception during conf
loading (#12382)
add e825af4aade feature add iotdb backup (#12365)
add b7e9d6ec3d5 Some region migration related work (#12376)
add 6b2e4b15925 Fix effective ratio calculation method (#12394)
add 1ce494b5f17 Fixed backup data backend running issues (#12396)
add 74eb4e2e392 Pipe: Fixed the bug that pure data transfer is forbidden
when SchemaRegion is in Simple consensus (#12397)
add 638be8ae212 Pipe: Added batch to schema snapshot execution in template
activation and timeseries creation & Fixed the bug that the
"CreateMultiTimeSeries" group with alias is not idempotent (#12380)
add bd22fe65461 Pipe: distinguish between old and new process subtasks
before and after stuck restart to avoid unexpected hashmap override and removal
(#12400)
add c36f7144be7 Fixed concurrency issues with RegionMigration thread pool
(#12403)
add 4d7362659d6 Fixed insertTablet http api lost data event return
SUCCESS_STATUS (#12387)
add ed6f56b25ab Fix partition cache after region migration (#12404)
add e1e0055c940 Fix IoTDBTimezoneIT
add c911c406bd1 Pipe: Fixed FileNotFoundException caused by processing
files deleted by compaction & Changed the "isEmpty()" judgement to avoid
"getDevice()" attempts to read from a deleted file when using fileTimeIndex
(#12405)
add 719834aea89 Pipe: refine procedure timeout message (#12402)
add f5cf782df5b Pipe: Two stage aggregate framework &
count-point-processor plugin (#12328)
add 03064328b43 Change log level and opt error msg for source path of view
is deleted (#12409)
add e715db52084 Pipe: remove unnecessary fsync operations for file pieces'
transferring (#12410)
add c232ee02239 fix pipe snapshot parser
add 8064ac58aa3 Pipe: Properly handle OUT_OF_TTL status code on receiver
side when data syncing (#12414)
new bdf89f8ab05 Merge branch 'master' into feature/leader-write
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java | 2 +-
.../cli/src/assembly/resources/tools/backup.bat | 125 +++
.../cli/src/assembly/resources/tools/backup.sh | 134 +++
.../org/apache/iotdb/tool/IoTDBDataBackTool.java | 985 +++++++++++++++++++++
.../confignode/conf/ConfigNodeDescriptor.java | 1 +
.../confignode/conf/ConfigNodeRemoveCheck.java | 5 +-
.../confignode/conf/SystemPropertiesUtils.java | 3 +-
.../consensus/request/ConfigPhysicalPlan.java | 1 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 90 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/PartitionBalancer.java | 10 +-
.../manager/load/balancer/RouteBalancer.java | 32 +-
.../manager/partition/PartitionManager.java | 14 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 13 +-
.../iotdb/confignode/persistence/AuthorInfo.java | 3 +
.../partition/DatabasePartitionTable.java | 11 +-
.../persistence/partition/PartitionInfo.java | 11 +-
.../procedure/env/RegionMaintainHandler.java | 119 ++-
.../impl/region/AddRegionPeerProcedure.java | 28 +-
.../impl/region/RemoveRegionPeerProcedure.java | 2 +-
.../confignode/persistence/AuthorInfoTest.java | 21 +
.../consensus/iot/IoTConsensusServerImpl.java | 47 +-
.../assembly/resources/conf/logback-datanode.xml | 15 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +-
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 1 +
.../metadata/view/BrokenViewException.java | 13 +-
.../PipeDataRegionProcessorConstructor.java | 3 +
.../agent/receiver/PipeDataNodeReceiverAgent.java | 5 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 9 +
.../schema/PipeSchemaRegionSnapshotEvent.java | 55 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 5 +
.../common/tablet/PipeRawTabletInsertionEvent.java | 5 +
.../common/tsfile/PipeTsFileInsertionEvent.java | 25 +
.../common/tsfile/TsFileInsertionPointCounter.java | 169 ++++
.../common/watermark/PipeWatermarkEvent.java} | 21 +-
.../dataregion/DataRegionWatermarkInjector.java | 76 ++
.../dataregion/IoTDBDataRegionExtractor.java | 39 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 24 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 13 +-
.../processor/aggregate/AggregateProcessor.java | 1 +
.../pipe/processor/twostage/combiner/Combiner.java | 107 +++
.../twostage/combiner/PipeCombineHandler.java | 198 +++++
.../combiner/PipeCombineHandlerManager.java | 163 ++++
.../twostage/exchange/payload/CombineRequest.java | 139 +++
.../payload/FetchCombineResultRequest.java | 119 +++
.../payload/FetchCombineResultResponse.java | 99 +++
.../twostage/exchange/payload/RequestType.java | 16 +-
.../receiver/TwoStageAggregateReceiver.java | 82 ++
.../exchange/sender/TwoStageAggregateSender.java | 235 +++++
.../processor/twostage/operator/CountOperator.java | 32 +-
.../processor/twostage/operator/Operator.java} | 12 +-
.../twostage/plugin/TwoStageCountProcessor.java | 388 ++++++++
.../processor/twostage/state/CountState.java} | 26 +-
.../db/pipe/processor/twostage/state/State.java} | 11 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 59 +-
.../thrift/IoTDBDataNodeReceiverAgent.java | 3 +
.../visitor/PipeStatementTSStatusVisitor.java | 57 +-
.../visitor/PipeStatementToBatchVisitor.java | 185 ++++
.../subtask/processor/PipeProcessorSubtask.java | 21 +-
.../rest/utils/InsertTabletSortDataUtils.java | 16 +-
.../execution/executor/RegionWriteExecutor.java | 12 +-
.../analyze/cache/partition/PartitionCache.java | 1 +
.../visitor/CompleteMeasurementSchemaVisitor.java | 6 +-
.../iotdb/db/service/RegionMigrateService.java | 10 +-
.../dataregion/tsfile/TsFileResource.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 7 +-
.../db/tools/schema/SRStatementGenerator.java | 91 +-
.../db/auth/user/LocalFileUserManagerTest.java | 8 +
.../db/utils/SchemaRegionSnapshotParserTest.java | 86 +-
.../commons/auth/authorizer/BasicAuthorizer.java | 12 +-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 2 +
.../iotdb/commons/auth/user/BasicUserManager.java | 16 +-
.../iotdb/commons/auth/user/IUserManager.java | 3 +-
.../commons/concurrent/IoTDBThreadPoolFactory.java | 12 -
.../apache/iotdb/commons/conf/CommonConfig.java | 41 +
.../iotdb/commons/conf/CommonDescriptor.java | 22 +
.../commons/consensus/index/ProgressIndex.java | 7 +-
.../commons/consensus/index/ProgressIndexType.java | 7 +-
.../consensus/index/impl/HybridProgressIndex.java | 4 +
.../consensus/index/impl/MetaProgressIndex.java | 2 +-
.../consensus/index/impl/StateProgressIndex.java | 234 +++++
.../iotdb/commons/pipe/config/PipeConfig.java | 30 +
.../config/constant/PipeExtractorConstant.java | 4 +
.../config/constant/PipeProcessorConstant.java | 2 +
.../request/IoTDBConnectorRequestVersion.java | 1 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 2 +
.../TwoStageCountProcessor.java} | 8 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 20 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 18 +
iotdb-doap.rdf | 26 +-
92 files changed, 4370 insertions(+), 452 deletions(-)
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
create mode 100755 iotdb-client/cli/src/assembly/resources/tools/backup.sh
create mode 100644
iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionPointCounter.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/schedule/constant/CompactionType.java
=> pipe/event/common/watermark/PipeWatermarkEvent.java} (68%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultResponse.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeResponseType.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/RequestType.java
(77%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/receiver/TwoStageAggregateReceiver.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/sender/TwoStageAggregateSender.java
copy
integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/Cluster1Env.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/CountOperator.java
(52%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/PipeTask.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/Operator.java}
(80%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/template/alter/TemplateAlterInfo.java
=> pipe/processor/twostage/state/CountState.java} (69%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/IProcedureFactory.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/state/State.java}
(79%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementToBatchVisitor.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/StateProgressIndex.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/{downsampling/SwingingDoorTrendingSamplingProcessor.java
=> twostage/TwoStageCountProcessor.java} (79%)
