This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/leader-write
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 3968c40419a Merge branch 'master' into feature/leader-write
add 9a29af7ce1d [IOTDB-6325] Support RegionScan for active metadata
queries [BE Part One]
add 40934ddc572 Pipe: Add completion signal to historical events & allow
all data regions' completion signal to drop the pipe (#12490)
add 0ebac6b7e11 Modify strategy of wait task time limitation & Fix proc id
(#12552)
add b2c5680f3d3 [RatisConsensus] add ut for force snapshot (#12560)
add 911d7b673a6 Pipe: Fixed the bug that schema region listening queue is
not cleared when the schema region is deleted / migrated (#12575)
add 462c9004a55 Introducing a memory control mechanism during the query
planning stage
add 7ef1eb722e9 Pipe: add parameter realtime.loose-range to support
subscription loose range semantics (#12580)
add e49a07d1129 Pipe: fix PipeEnrichedStatement can't contain redirect
info (#12579)
add f60694b4a46 Add TsFile develop branch for IoTDB in README (#12581)
add fad655319e1 Avoid infinite and unecessary loop when CN RPC processors
are killed/interrupted by OS (#12584)
add 1517e81e878 Fix python ci security warning and update c++ ci commands
(#12583)
add aa2687c1fe7 Pipe: Fix BatchActivateTemplateStatement is not handled
correctly when some of the timeseries already exists (#12587)
add 6a0da36bc56 Merge multi-language client CI workflows to one ymls
(#12586)
add 21a23967d7c Fix error compare method in IT framework
add 6c65dfe0de1 Merge branch 'master' into feature/leader-write
add 28bb9b8cd6c Pipe: Support pipe's endpoint & global connector transfer
rate limit & Fix RPC compression not enabled in stream batch mode (#12543)
add a9c27bb249d Fix potential NPE for empty InsertRowsOfOneDevice #12593
add b161163858c merge
No new revisions were added by this update.
Summary of changes:
.github/workflows/client-go.yml | 56 ----
.github/workflows/client-python.yml | 58 ----
.../{client-cpp.yml => multi-language-client.yml} | 92 +++---
.mvn/wrapper/maven-wrapper.properties | 4 +-
README.md | 2 +
README_ZH.md | 2 +
.../iotdb/itbase/runtime/RequestDelegate.java | 26 +-
...torParallelIT.java => IoTDBPipeAutoDropIT.java} | 49 ++--
.../iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java | 4 +-
iotdb-client/client-py/requirements_dev.txt | 1 -
.../heartbeat/DataNodeHeartbeatHandler.java | 3 +-
.../iotdb/confignode/manager/ClusterManager.java | 2 +
.../manager/consensus/ConsensusManager.java | 1 +
.../protocol/IoTDBConfigRegionAirGapConnector.java | 51 ++--
.../protocol/IoTDBConfigRegionConnector.java | 52 ++--
.../runtime/PipeRuntimeCoordinator.java | 18 +-
.../runtime/heartbeat/PipeHeartbeat.java | 61 ++++
.../{ => heartbeat}/PipeHeartbeatParser.java | 51 ++--
.../{ => heartbeat}/PipeHeartbeatScheduler.java | 20 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 89 +++---
.../procedure/env/RegionMaintainHandler.java | 40 ++-
.../impl/region/AddRegionPeerProcedure.java | 4 +-
.../iotdb/consensus/ratis/DiskGuardianTest.java | 28 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 15 +
.../apache/iotdb/consensus/ratis/TestUtils.java | 21 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../schemaregion/SchemaExecutionVisitor.java | 25 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 107 ++++++-
.../payload/evolvable/builder/PipeEventBatch.java | 18 ++
.../builder/PipeTransferBatchReqBuilder.java | 8 +-
.../airgap/IoTDBDataNodeAirGapConnector.java | 20 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 55 ++--
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 32 +-
.../async/IoTDBDataRegionAsyncConnector.java | 11 +-
.../PipeTransferTabletBatchEventHandler.java | 19 +-
.../PipeTransferTabletInsertNodeEventHandler.java | 8 +-
.../PipeTransferTabletInsertionEventHandler.java | 5 +
.../PipeTransferTsFileInsertionEventHandler.java | 35 ++-
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 17 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 136 +++++----
.../thrift/sync/IoTDBSchemaRegionConnector.java | 36 ++-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 13 +
.../common/terminate/PipeTerminateEvent.java} | 53 ++--
.../event/realtime/PipeRealtimeEventFactory.java | 1 +
.../PipeHistoricalDataRegionTsFileExtractor.java | 56 ++--
.../realtime/PipeRealtimeDataRegionExtractor.java | 22 ++
.../visitor/PipeStatementExceptionVisitor.java | 39 ++-
.../visitor/PipeStatementTSStatusVisitor.java | 10 +
.../iotdb/db/pipe/task/PipeDataNodeTask.java | 20 +-
.../thrift/impl/DataNodeRegionManager.java | 2 +
.../db/queryengine/common/MPPQueryContext.java | 60 ++++
.../exception/MemoryNotEnoughException.java | 7 +-
.../execution/MemoryEstimationHelper.java | 6 +-
.../queryengine/execution/driver/DataDriver.java | 25 +-
.../execution/driver/DataDriverContext.java | 4 +-
.../fragment/FragmentInstanceContext.java | 150 ++++++++--
.../execution/fragment/QueryContext.java | 76 +++--
.../execution/load/LoadTsFileRateLimiter.java | 32 +-
.../source/AbstractDataSourceOperator.java | 5 +-
.../operator/source/AlignedSeriesScanOperator.java | 5 +-
.../operator/source/DataSourceOperator.java | 4 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 3 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 154 ++++++----
.../plan/analyze/ConcatPathRewriter.java | 33 ++-
.../plan/analyze/ExpressionAnalyzer.java | 42 ++-
.../queryengine/plan/analyze/ExpressionUtils.java | 66 +++--
.../queryengine/plan/analyze/TemplatedAnalyze.java | 12 +-
.../queryengine/plan/execution/QueryExecution.java | 3 +
.../db/queryengine/plan/expression/Expression.java | 3 +-
.../plan/expression/binary/BinaryExpression.java | 12 +
.../plan/expression/leaf/ConstantOperand.java | 9 +
.../plan/expression/leaf/NullOperand.java | 10 +
.../plan/expression/leaf/TimeSeriesOperand.java | 10 +
.../plan/expression/leaf/TimestampOperand.java | 10 +
.../plan/expression/multi/FunctionExpression.java | 26 ++
.../expression/other/CaseWhenThenExpression.java | 16 +
.../expression/other/GroupByTimeExpression.java | 9 +
.../plan/expression/ternary/BetweenExpression.java | 14 +
.../plan/expression/unary/InExpression.java | 12 +-
.../plan/expression/unary/IsNullExpression.java | 10 +
.../plan/expression/unary/LikeExpression.java | 12 +
.../plan/expression/unary/LogicNotExpression.java | 11 +
.../plan/expression/unary/NegationExpression.java | 11 +
.../plan/expression/unary/RegularExpression.java | 12 +
.../cartesian/BindSchemaForExpressionVisitor.java | 50 +++-
.../cartesian/BindSchemaForPredicateVisitor.java | 43 ++-
.../visitor/cartesian/CartesianProductVisitor.java | 23 +-
...catDeviceAndBindSchemaForExpressionVisitor.java | 27 +-
...ncatDeviceAndBindSchemaForPredicateVisitor.java | 30 +-
.../ConcatExpressionWithSuffixPathsVisitor.java | 28 +-
.../visitor/cartesian/QueryContextProvider.java | 7 +-
.../plan/optimization/AggregationPushDown.java | 91 +++++-
.../plan/parser/StatementGenerator.java | 6 +-
.../plan/planner/LocalExecutionPlanner.java | 22 +-
.../plan/planner/LogicalPlanBuilder.java | 76 +++--
.../queryengine/plan/planner/TreeModelPlanner.java | 11 +-
.../distribution/DistributionPlanContext.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 9 +
.../plan/node/source/AlignedLastQueryScanNode.java | 13 +
.../source/AlignedSeriesAggregationScanNode.java | 11 +
.../plan/node/source/AlignedSeriesScanNode.java | 12 +
.../plan/node/source/LastQueryScanNode.java | 13 +
.../node/source/SeriesAggregationScanNode.java | 12 +
.../planner/plan/node/source/SeriesScanNode.java | 12 +
.../planner/plan/node/source/SeriesSourceNode.java | 4 +-
.../schemaregion/utils/ResourceByPathUtils.java | 83 +-----
.../db/storageengine/dataregion/DataRegion.java | 208 +++++++++++--
.../dataregion/IDataRegionForQuery.java | 20 +-
.../dataregion/VirtualDataRegion.java | 27 ++
.../dataregion/memtable/AbstractMemTable.java | 311 ++++++++++++++++++++
.../memtable/AlignedWritableMemChunk.java | 4 +
.../dataregion/memtable/IMemTable.java | 19 ++
.../dataregion/memtable/TsFileProcessor.java | 322 +++++++++++++++++++++
.../dataregion/read/IQueryDataSource.java} | 12 +-
.../dataregion/read/QueryDataSource.java | 11 +-
.../read/QueryDataSourceForRegionScan.java | 64 ++++
.../dataregion/read/QueryDataSourceType.java | 10 +-
.../dataregion/read/filescan/IChunkHandle.java | 34 ++-
.../dataregion/read/filescan/IFileScanHandle.java | 99 +++++++
.../filescan/impl/ClosedFileScanHandleImpl.java | 211 ++++++++++++++
.../filescan/impl/DiskAlignedChunkHandleImpl.java | 81 ++++++
.../read/filescan/impl/DiskChunkHandleImpl.java | 137 +++++++++
.../filescan/impl/MemAlignedChunkHandleImpl.java | 72 +++++
.../read/filescan/impl/MemChunkHandleImpl.java} | 35 ++-
.../filescan/impl/UnclosedFileScanHandleImpl.java | 180 ++++++++++++
.../filescan/model/AbstractChunkOffset.java} | 33 ++-
.../model/AbstractDeviceChunkMetaData.java} | 20 +-
.../read/filescan/model/AlignedChunkOffset.java | 52 ++++
.../model/AlignedDeviceChunkMetaData.java} | 28 +-
.../filescan/model/ChunkOffset.java} | 32 +-
.../filescan/model/DeviceChunkMetaData.java} | 29 +-
.../filescan/model/DeviceStartEndTime.java} | 26 +-
.../dataregion/utils/SharedTimeDataBuffer.java | 91 ++++++
.../utils/TsFileDeviceStartEndTimeIterator.java | 52 ++++
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 +
.../apache/iotdb/db/utils/ModificationUtils.java | 97 +++++++
.../db/utils/datastructure/AlignedTVList.java | 7 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 3 +-
.../db/utils/datastructure/BooleanTVList.java | 3 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 3 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 3 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 3 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 3 +-
.../iotdb/db/utils/datastructure/TVList.java | 18 +-
.../iotdb/db/metadata/path/PatternTreeMapTest.java | 55 ++++
.../exception/MemoryNotEnoughExceptionTest.java | 37 ---
.../plan/analyze/ExpressionAnalyzerTest.java | 8 +-
.../resources/conf/iotdb-common.properties | 10 +
.../async/AsyncPipeDataTransferServiceClient.java | 4 +
.../apache/iotdb/commons/conf/CommonConfig.java | 10 +
.../iotdb/commons/conf/CommonDescriptor.java | 6 +
.../apache/iotdb/commons/path/PatternTreeMap.java | 41 +++
.../iotdb/commons/pipe/config/PipeConfig.java | 8 +
.../config/constant/PipeConnectorConstant.java | 4 +
.../config/constant/PipeExtractorConstant.java | 8 +
.../pipe/connector/client/IoTDBSyncClient.java | 7 +
.../pipe/connector/limiter/GlobalRateLimiter.java} | 55 ++--
.../connector/limiter/PipeEndPointRateLimiter.java | 59 ++++
.../connector/protocol/IoTDBAirGapConnector.java | 48 ++-
.../pipe/connector/protocol/IoTDBConnector.java | 45 ++-
.../connector/protocol/IoTDBSslSyncConnector.java | 24 +-
.../listening/AbstractPipeListeningQueue.java | 11 +-
.../AbstractSerializableListeningQueue.java | 25 +-
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 38 ++-
.../commons/pipe/task/meta/PipeTemporaryMeta.java} | 35 +--
.../src/main/thrift/datanode.thrift | 2 +
pom.xml | 2 +-
168 files changed, 4619 insertions(+), 1263 deletions(-)
delete mode 100644 .github/workflows/client-go.yml
delete mode 100644 .github/workflows/client-python.yml
rename .github/workflows/{client-cpp.yml => multi-language-client.yml} (58%)
copy
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/{IoTDBPipeConnectorParallelIT.java
=> IoTDBPipeAutoDropIT.java} (66%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeat.java
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/{
=> heartbeat}/PipeHeartbeatParser.java (88%)
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/{
=> heartbeat}/PipeHeartbeatScheduler.java (90%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/ProgressReportEvent.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java}
(53%)
copy
iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeConfiguration.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/QueryContextProvider.java
(79%)
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/PipeTask.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/IQueryDataSource.java}
(85%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSourceForRegionScan.java
copy
iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeRuntimeConfiguration.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSourceType.java
(83%)
copy
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/access/RowIterator.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/IChunkHandle.java
(55%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/IFileScanHandle.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/MemAlignedChunkHandleImpl.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/processor/twostage/state/CountState.java
=> storageengine/dataregion/read/filescan/impl/MemChunkHandleImpl.java} (52%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/selector/utils/DeviceInfo.java
=> read/filescan/model/AbstractChunkOffset.java} (54%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/selector/utils/DeviceInfo.java
=> read/filescan/model/AbstractDeviceChunkMetaData.java} (70%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/AlignedChunkOffset.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{PartitionLastFlushTime.java
=> read/filescan/model/AlignedDeviceChunkMetaData.java} (57%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/selector/utils/DeviceInfo.java
=> read/filescan/model/ChunkOffset.java} (50%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/selector/utils/DeviceInfo.java
=> read/filescan/model/DeviceChunkMetaData.java} (56%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/selector/utils/DeviceInfo.java
=> read/filescan/model/DeviceStartEndTime.java} (64%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/exception/MemoryNotEnoughExceptionTest.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java}
(51%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/common/NodeRef.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/meta/PipeTemporaryMeta.java}
(55%)
