This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 4059f88df97 merge
add ba853b3147b Pipe: Fixed the bug that aggregate processor cannot clear
the last received time after drop (#12383)
add 9051aa57908 Update iotdb-doap.rdf for v1.3.1 release (#12390)
add b947dd83b6c Remove useless try-catch for RegionPriorityMap (#12389)
add 7f4724dd3d7 Enhance concurrent control of data partition policy table
(#12388)
add ac648a3a2a0 Deduplication IoTConsensus Configuration And Remove
Unuseless File Delete (#12385)
add 3bfc0ea84a5 Provide more precise info for exception during conf
loading (#12382)
add e825af4aade feature add iotdb backup (#12365)
add b7e9d6ec3d5 Some region migration related work (#12376)
add 6b2e4b15925 Fix effective ratio calculation method (#12394)
add 1ce494b5f17 Fixed backup data backend running issues (#12396)
add 74eb4e2e392 Pipe: Fixed the bug that pure data transfer is forbidden
when SchemaRegion is in Simple consensus (#12397)
add 638be8ae212 Pipe: Added batch to schema snapshot execution in template
activation and timeseries creation & Fixed the bug that the
"CreateMultiTimeSeries" group with alias is not idempotent (#12380)
add bd22fe65461 Pipe: distinguish between old and new process subtasks
before and after stuck restart to avoid unexpected hashmap override and removal
(#12400)
add c36f7144be7 Fixed concurrency issues with RegionMigration thread pool
(#12403)
add 4d7362659d6 Fixed insertTablet http api lost data event return
SUCCESS_STATUS (#12387)
add ed6f56b25ab Fix partition cache after region migration (#12404)
add e1e0055c940 Fix IoTDBTimezoneIT
add c911c406bd1 Pipe: Fixed FileNotFoundException caused by processing
files deleted by compaction & Changed the "isEmpty()" judgement to avoid
"getDevice()" attempts to read from a deleted file when using fileTimeIndex
(#12405)
add 719834aea89 Pipe: refine procedure timeout message (#12402)
add f5cf782df5b Pipe: Two stage aggregate framework &
count-point-processor plugin (#12328)
add 03064328b43 Change log level and opt error msg for source path of view
is deleted (#12409)
new c9e7b534ebc Merge remote-tracking branch 'origin/master' into
ty/TableModelGrammar
new edd9f243828 partial
add e715db52084 Pipe: remove unnecessary fsync operations for file pieces'
transferring (#12410)
add c232ee02239 fix pipe snapshot parser
add 8064ac58aa3 Pipe: Properly handle OUT_OF_TTL status code on receiver
side when data syncing (#12414)
add a8411f66837 Fix Concurrency Instability When IoTConsensus
LogDispatcher Exits (#12411)
add 544a3bccbb9 update Export/ImportDataTestIT (#12399)
add 1d30805f266 Pipe: Added check for wal disable and ratis consensus when
enabling data region processing (#12407)
add 447bf54a356 revert -XX:GuaranteedSafepointInterval=0 option (#12417)
add 95384d4a9eb Randomly choosing coordinator of add region (#12416)
add d23729f876a [RatisConsensus] update ratis snapshot version (#12419)
add 98b2272dbc1 Pipe: Reduce exception messages to avoid excess rpc
payload and show pipe response (#12415)
add 52bc1889d53 Copy tsfile, its resources, and mods to the same directory
when using multiple data directories (#12420)
add d866bc5dbaf fix snapshot parser null point.
add 1b981972bd7 Pipe: Support InsertRows in
PipeInsertNodeTabletInsertionEvent (#12286)
add 3d87ecc1346 Refactor MultiTsFileDeviceIterator.MeasurementIterator
(#12221)
new 37f5135e55b Merge remote-tracking branch 'origin/master' into
ty/TableModelGrammar
new 9e3e9d6cda8 support SortNode, MergeSortNode and TopKNode
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java | 2 +-
.../apache/iotdb/tools/it/ExportDataTestIT.java | 6 +-
.../apache/iotdb/tools/it/ImportDataTestIT.java | 4 +-
.../cli/src/assembly/resources/tools/backup.bat | 125 +++
.../cli/src/assembly/resources/tools/backup.sh | 134 +++
.../org/apache/iotdb/tool/AbstractDataTool.java | 1 +
.../java/org/apache/iotdb/tool/ExportData.java | 21 +-
.../java/org/apache/iotdb/tool/ImportData.java | 73 +-
.../org/apache/iotdb/tool/IoTDBDataBackTool.java | 985 +++++++++++++++++++++
.../confignode/conf/ConfigNodeDescriptor.java | 1 +
.../confignode/conf/ConfigNodeRemoveCheck.java | 5 +-
.../confignode/conf/SystemPropertiesUtils.java | 3 +-
.../consensus/request/ConfigPhysicalPlan.java | 1 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/ConfigPhysicalPlanVisitor.java | 6 +
.../consensus/request/auth/AuthorPlan.java | 3 +
.../response/pipe/task/PipeTableResp.java | 8 +-
.../confignode/manager/PermissionManager.java | 3 +-
.../iotdb/confignode/manager/ProcedureManager.java | 93 +-
.../iotdb/confignode/manager/load/LoadManager.java | 8 +-
.../manager/load/balancer/PartitionBalancer.java | 10 +-
.../manager/load/balancer/RouteBalancer.java | 32 +-
.../manager/partition/PartitionManager.java | 14 +-
.../receiver/IoTDBConfigNodeReceiverAgent.java | 6 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 5 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 53 +-
.../pipe/event/PipeConfigRegionWritePlanEvent.java | 46 +-
.../extractor/ConfigRegionListeningFilter.java | 5 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 13 +-
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 1 +
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 9 +
.../iotdb/confignode/persistence/AuthorInfo.java | 3 +
.../persistence/executor/ConfigPlanExecutor.java | 1 +
.../partition/DatabasePartitionTable.java | 11 +-
.../persistence/partition/PartitionInfo.java | 11 +-
.../schema/CNPhysicalPlanGenerator.java | 9 +-
.../procedure/env/RegionMaintainHandler.java | 122 ++-
.../impl/pipe/task/AlterPipeProcedureV2.java | 2 +-
.../impl/region/AddRegionPeerProcedure.java | 28 +-
.../impl/region/RemoveRegionPeerProcedure.java | 2 +-
.../confignode/persistence/AuthorInfoTest.java | 21 +
.../persistence/CNPhysicalPlanGeneratorTest.java | 3 +
.../consensus/iot/IoTConsensusServerImpl.java | 47 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 8 +
.../apache/iotdb/consensus/iot/ReplicateTest.java | 36 +-
.../src/assembly/resources/conf/datanode-env.bat | 3 -
.../src/assembly/resources/conf/datanode-env.sh | 3 -
.../assembly/resources/conf/logback-datanode.xml | 15 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +-
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 1 +
.../metadata/view/BrokenViewException.java | 13 +-
.../PipeDataRegionProcessorConstructor.java | 3 +
.../agent/receiver/PipeDataNodeReceiverAgent.java | 5 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 9 +
.../request/PipeTransferTabletInsertNodeReq.java | 46 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 9 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 5 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 14 +-
.../connector/protocol/opcua/OpcUaConnector.java | 4 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 10 +-
.../protocol/websocket/WebSocketConnector.java | 17 +-
.../websocket/WebSocketConnectorServer.java | 5 +-
.../db/pipe/event/common/row/PipeRowCollector.java | 4 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 64 +-
.../schema/PipeSchemaRegionWritePlanEvent.java | 44 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 122 ++-
.../common/tablet/PipeRawTabletInsertionEvent.java | 5 +
.../tablet/TabletInsertionDataContainer.java | 15 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 34 +
.../common/tsfile/TsFileInsertionPointCounter.java | 169 ++++
.../common/watermark/PipeWatermarkEvent.java} | 21 +-
.../dataregion/DataRegionWatermarkInjector.java | 76 ++
.../dataregion/IoTDBDataRegionExtractor.java | 82 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 24 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 6 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 13 +-
.../processor/aggregate/AggregateProcessor.java | 3 +-
.../pipe/processor/twostage/combiner/Combiner.java | 107 +++
.../twostage/combiner/PipeCombineHandler.java | 198 +++++
.../combiner/PipeCombineHandlerManager.java | 163 ++++
.../twostage/exchange/payload/CombineRequest.java | 139 +++
.../payload/FetchCombineResultRequest.java | 119 +++
.../payload/FetchCombineResultResponse.java | 99 +++
.../twostage/exchange/payload/RequestType.java | 16 +-
.../receiver/TwoStageAggregateReceiver.java | 82 ++
.../exchange/sender/TwoStageAggregateSender.java | 235 +++++
.../processor/twostage/operator/CountOperator.java | 32 +-
.../processor/twostage/operator/Operator.java} | 12 +-
.../twostage/plugin/TwoStageCountProcessor.java | 391 ++++++++
.../processor/twostage/state/CountState.java} | 26 +-
.../db/pipe/processor/twostage/state/State.java} | 11 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 72 +-
.../thrift/IoTDBDataNodeReceiverAgent.java | 3 +
.../visitor/PipePlanToStatementVisitor.java | 105 ++-
.../visitor/PipeStatementTSStatusVisitor.java | 57 +-
.../visitor/PipeStatementToBatchVisitor.java | 185 ++++
.../pipe/task/connection/PipeEventCollector.java | 3 +-
.../subtask/processor/PipeProcessorSubtask.java | 57 +-
.../rest/utils/InsertTabletSortDataUtils.java | 16 +-
.../execution/executor/RegionWriteExecutor.java | 12 +-
.../process/join/merge/MergeSortComparator.java | 20 +
.../analyze/cache/partition/PartitionCache.java | 1 +
.../visitor/CompleteMeasurementSchemaVisitor.java | 6 +-
.../plan/planner/TableOperatorGenerator.java | 156 +++-
.../planner/plan/node/write/InsertRowsNode.java | 9 +-
.../plan/relational/planner/node/LimitNode.java | 8 +-
.../relational/planner/node/MergeSortNode.java | 33 +-
.../plan/relational/planner/node/OffsetNode.java | 8 +-
.../plan/relational/planner/node/SortNode.java | 12 +-
.../plan/relational/planner/node/TopKNode.java | 43 +-
.../iotdb/db/service/RegionMigrateService.java | 10 +-
.../impl/ReadChunkCompactionPerformer.java | 12 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 146 +--
.../dataregion/snapshot/SnapshotLoader.java | 12 +-
.../dataregion/tsfile/TsFileResource.java | 2 +-
.../storageengine/dataregion/wal/node/WALNode.java | 7 +-
.../broker/SubscriptionPrefetchingQueue.java | 28 +-
.../db/tools/schema/SRStatementGenerator.java | 104 ++-
.../db/auth/user/LocalFileUserManagerTest.java | 8 +
.../extractor/IoTDBDataRegionExtractorTest.java | 32 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 70 ++
.../dataregion/wal/WALManagerTest.java | 5 +
.../dataregion/wal/node/WALNodeTest.java | 5 +
.../wal/node/WalDeleteOutdatedNewTest.java | 5 +
.../db/utils/SchemaRegionSnapshotParserTest.java | 86 +-
.../src/assembly/resources/sbin/start-all.sh | 2 +-
.../src/assembly/resources/sbin/stop-all.sh | 2 +-
.../commons/auth/authorizer/BasicAuthorizer.java | 12 +-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 2 +
.../iotdb/commons/auth/user/BasicUserManager.java | 16 +-
.../iotdb/commons/auth/user/IUserManager.java | 3 +-
.../commons/concurrent/IoTDBThreadPoolFactory.java | 12 -
.../apache/iotdb/commons/conf/CommonConfig.java | 41 +
.../iotdb/commons/conf/CommonDescriptor.java | 22 +
.../commons/consensus/index/ProgressIndex.java | 7 +-
.../commons/consensus/index/ProgressIndexType.java | 7 +-
.../consensus/index/impl/HybridProgressIndex.java | 4 +
.../consensus/index/impl/MetaProgressIndex.java | 2 +-
.../consensus/index/impl/StateProgressIndex.java | 234 +++++
.../iotdb/commons/pipe/config/PipeConfig.java | 30 +
.../config/constant/PipeExtractorConstant.java | 4 +
.../config/constant/PipeProcessorConstant.java | 2 +
.../request/IoTDBConnectorRequestVersion.java | 1 +
.../commons/pipe/event/PipeSnapshotEvent.java | 19 +-
.../commons/pipe/event/PipeWritePlanEvent.java | 35 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 2 +
.../TwoStageCountProcessor.java} | 8 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 20 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 29 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 28 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 18 +
iotdb-doap.rdf | 26 +-
pom.xml | 2 +-
155 files changed, 5488 insertions(+), 910 deletions(-)
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
create mode 100755 iotdb-client/cli/src/assembly/resources/tools/backup.sh
create mode 100644
iotdb-client/cli/src/main/java/org/apache/iotdb/tool/IoTDBDataBackTool.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionPointCounter.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/schedule/constant/CompactionType.java
=> pipe/event/common/watermark/PipeWatermarkEvent.java} (68%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandler.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/PipeCombineHandlerManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultResponse.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeResponseType.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/RequestType.java
(77%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/receiver/TwoStageAggregateReceiver.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/sender/TwoStageAggregateSender.java
copy
integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/Cluster1Env.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/CountOperator.java
(52%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/PipeTask.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/operator/Operator.java}
(80%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/template/alter/TemplateAlterInfo.java
=> pipe/processor/twostage/state/CountState.java} (69%)
copy
iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/IProcedureFactory.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/state/State.java}
(79%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementToBatchVisitor.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/StateProgressIndex.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/{downsampling/SwingingDoorTrendingSamplingProcessor.java
=> twostage/TwoStageCountProcessor.java} (79%)
