This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/AggregationPushDown
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 1c1f369fdd8 fix review
add f1b42deb387 Fix PartialPath#intersectWithPrefixPattern return wrong
result (#12465)
add 57a6ff4b44f add alter view statement in snapshot parser
add c63a940a74c remove connector it (#12471)
add 6f29fcf26d8 Pipe: Fixed connector subtask manager to allow restarted
DataRegions to reuse connectors from other not restarted DataRegions (#12470)
add 46319bd403b Pipe IT: Return directly when SQL retry failed in IT to
avoid potential fake failure interference (#12468)
add 2db087dc0fd Subscription: randomly generate consumer group id and
consumer id if not exist & validate topic config ahead on DN (#12467)
add f7f34de5d3f Load: Avoid NPE when load task ends after clean task's
execution (#12474)
add 139720d7f9a Limit direct buffer memory cost when create consensus
region (#12431)
add e383b0faf5b Fix region migration code (#12472)
add 1a16b6d333a Normalize LoadCache interfaces (#12441)
add 07578e14d70 Fixed backup path detection and optimized output log file
names (#12454)
add 093512b84fb Remove a line of log (#12480)
add a6676d3d529 Fix insertion cross space compaction recover null pointer
exception (#12439)
add 318348e2202 Enhance estimate of FI memory usage
add 1a714fb695d Merge branch 'master' into lmh/AggregationPushDown
No new revisions were added by this update.
Summary of changes:
distribution/src/assembly/all.xml | 5 +
distribution/src/assembly/cli.xml | 5 +
integration-test/pom.xml | 61 +--
.../org/apache/iotdb/flink/it/AbstractTest.java | 46 --
.../java/org/apache/iotdb/flink/it/SinkTest.java | 163 ------
.../java/org/apache/iotdb/flink/it/SourceTest.java | 192 -------
.../test/java/org/apache/iotdb/flink/it/Utils.java | 94 ----
.../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 32 +-
.../pipe/it/manual/IoTDBPipeMetaRestartIT.java | 28 +-
.../org/apache/iotdb/spark/it/AbstractTest.java | 55 --
.../java/org/apache/iotdb/spark/it/ReadTest.java | 145 ------
.../test/java/org/apache/iotdb/spark/it/Utils.java | 95 ----
.../java/org/apache/iotdb/spark/it/WriteTest.java | 116 -----
.../it/dual/IoTDBSubscriptionTopicIT.java | 47 ++
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 390 --------------
.../src/assembly/resources/conf/logback-cli.xml | 19 +-
.../cli/src/assembly/resources/tools/backup.bat | 127 +++++
.../cli/src/assembly/resources/tools/backup.sh | 134 +++++
.../org/apache/iotdb/tool/IoTDBDataBackTool.java | 223 ++++++--
.../rpc/subscription/config/ConsumerConfig.java | 8 +
.../response/PipeSubscribeHandshakeResp.java | 26 +-
.../session/subscription/SubscriptionConsumer.java | 30 +-
.../subscription/SubscriptionEndpointsSyncer.java | 9 +-
.../session/subscription/SubscriptionProvider.java | 25 +-
.../SubscriptionSessionConnection.java | 6 +-
.../iotdb/confignode/manager/load/LoadManager.java | 46 +-
.../manager/load/balancer/RouteBalancer.java | 5 +-
.../router/leader/AbstractLeaderBalancer.java | 48 +-
.../router/leader/GreedyLeaderBalancer.java | 14 +-
.../router/leader/MinCostFlowLeaderBalancer.java | 137 ++---
.../confignode/manager/load/cache/LoadCache.java | 181 +++++--
.../consensus/ConsensusGroupHeartbeatSample.java | 7 +
.../cache/consensus/ConsensusGroupStatistics.java | 7 +
.../manager/load/cache/node/NodeStatistics.java | 4 +-
.../manager/load/cache/region/RegionCache.java | 3 +-
.../load/cache/region/RegionGroupCache.java | 30 +-
.../load/cache/region/RegionHeartbeatSample.java | 7 +
.../manager/load/service/EventService.java | 133 +++--
.../iotdb/confignode/manager/node/NodeManager.java | 7 +-
.../manager/schema/ClusterSchemaManager.java | 36 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 38 +-
.../procedure/env/RegionMaintainHandler.java | 14 +-
.../impl/node/AddConfigNodeProcedure.java | 2 +-
.../impl/region/AddRegionPeerProcedure.java | 48 +-
.../impl/region/CreateRegionGroupsProcedure.java | 9 +-
.../impl/region/RemoveRegionPeerProcedure.java | 6 +-
.../confignode/manager/load/FakeSubscriber.java | 92 ++++
.../confignode/manager/load/LoadManagerTest.java | 295 +++++++++++
.../router/leader/CFDLeaderBalancerTest.java | 34 +-
.../router/leader/GreedyLeaderBalancerTest.java | 30 +-
.../leader/LeaderBalancerComparisonTest.java | 39 +-
.../manager/load/cache/RegionGroupCacheTest.java | 23 +-
.../src/assembly/resources/sbin/start-datanode.bat | 1 +
.../src/assembly/resources/sbin/start-datanode.sh | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 23 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +
.../connector/PipeConnectorSubtaskManager.java | 29 +-
.../execution/MemoryEstimationHelper.java | 87 ++++
.../queryengine/execution/exchange/sink/ISink.java | 3 +-
.../execution/exchange/sink/LocalSinkChannel.java | 11 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 14 +
.../execution/exchange/sink/SinkChannel.java | 14 +
.../execution/exchange/source/ISourceHandle.java | 3 +-
.../exchange/source/LocalSourceHandle.java | 13 +-
.../execution/exchange/source/SourceHandle.java | 13 +
.../execution/load/LoadTsFileManager.java | 6 +-
.../execution/operator/ExplainAnalyzeOperator.java | 21 +-
.../queryengine/execution/operator/Operator.java | 3 +-
.../execution/operator/OperatorContext.java | 15 +-
.../process/AggregationMergeSortOperator.java | 17 +
.../operator/process/AggregationOperator.java | 16 +
.../operator/process/ColumnInjectOperator.java | 12 +
.../operator/process/DeviceViewIntoOperator.java | 13 +
.../operator/process/DeviceViewOperator.java | 14 +
.../execution/operator/process/FillOperator.java | 11 +
.../operator/process/FilterAndProjectOperator.java | 13 +
.../execution/operator/process/IntoOperator.java | 20 +
.../execution/operator/process/LimitOperator.java | 11 +
.../operator/process/LinearFillOperator.java | 12 +
.../operator/process/MergeSortOperator.java | 17 +
.../execution/operator/process/OffsetOperator.java | 11 +
.../operator/process/ProjectOperator.java | 11 +
.../process/RawDataAggregationOperator.java | 13 +
.../operator/process/SingleDeviceViewOperator.java | 11 +
.../process/SlidingWindowAggregationOperator.java | 12 +
.../execution/operator/process/SortOperator.java | 14 +
.../operator/process/TagAggregationOperator.java | 19 +
.../execution/operator/process/TopKOperator.java | 16 +
.../operator/process/TransformOperator.java | 14 +
.../process/join/FullOuterTimeJoinOperator.java | 20 +
.../process/join/HorizontallyConcatOperator.java | 17 +
.../process/join/InnerTimeJoinOperator.java | 17 +
.../process/join/LeftOuterTimeJoinOperator.java | 14 +
.../last/AlignedUpdateLastCacheOperator.java | 16 +
.../AlignedUpdateViewPathLastCacheOperator.java | 6 +
.../process/last/LastQueryCollectOperator.java | 16 +
.../process/last/LastQueryMergeOperator.java | 18 +
.../operator/process/last/LastQueryOperator.java | 17 +
.../process/last/LastQuerySortOperator.java | 17 +
.../process/last/LastQueryTransformOperator.java | 15 +
.../process/last/UpdateLastCacheOperator.java | 16 +
.../last/UpdateViewPathLastCacheOperator.java | 6 +
.../schema/CountGroupByLevelMergeOperator.java | 14 +
.../schema/CountGroupByLevelScanOperator.java | 18 +-
.../operator/schema/CountMergeOperator.java | 12 +
.../schema/NodeManageMemoryMergeOperator.java | 12 +
.../operator/schema/NodePathsConvertOperator.java | 12 +
.../operator/schema/NodePathsCountOperator.java | 12 +
.../operator/schema/SchemaCountOperator.java | 12 +
.../operator/schema/SchemaFetchMergeOperator.java | 15 +
.../operator/schema/SchemaFetchScanOperator.java | 12 +
.../operator/schema/SchemaQueryMergeOperator.java | 13 +
.../schema/SchemaQueryOrderByHeatOperator.java | 14 +
.../operator/schema/SchemaQueryScanOperator.java | 14 +
.../operator/sink/IdentitySinkOperator.java | 16 +
.../operator/sink/ShuffleHelperOperator.java | 16 +
.../AlignedSeriesAggregationScanOperator.java | 16 +
.../operator/source/AlignedSeriesScanOperator.java | 13 +
.../operator/source/ExchangeOperator.java | 13 +
.../operator/source/LastCacheScanOperator.java | 12 +
.../source/SeriesAggregationScanOperator.java | 16 +
.../operator/source/SeriesScanOperator.java | 13 +
.../execution/operator/source/SeriesScanUtil.java | 19 +-
.../operator/source/ShowQueriesOperator.java | 12 +
.../analyze/cache/partition/PartitionCache.java | 1 -
.../config/executor/ClusterConfigTaskExecutor.java | 31 +-
.../plan/execution/memory/MemorySourceHandle.java | 9 +
.../plan/planner/LocalExecutionPlanContext.java | 52 ++
.../plan/planner/LocalExecutionPlanner.java | 27 +-
.../plan/planner/OperatorTreeGenerator.java | 44 +-
...ConsumeAllChildrenPipelineMemoryEstimator.java} | 20 +-
...umeChildrenOneByOnePipelineMemoryEstimator.java | 69 +++
.../planner/memory/PipelineMemoryEstimator.java | 79 +++
.../memory/PipelineMemoryEstimatorFactory.java | 51 ++
.../plan/planner/plan/node/PlanNodeId.java | 12 +-
.../node/process/join/FullOuterTimeJoinNode.java | 5 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 17 +
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 18 +
.../db/storageengine/dataregion/DataRegion.java | 30 ++
.../task/InsertionCrossSpaceCompactionTask.java | 13 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 34 ++
.../receiver/SubscriptionReceiverV1.java | 39 +-
.../SubscriptionConnectorSubtaskManager.java | 37 +-
.../db/tools/schema/SRStatementGenerator.java | 66 +--
.../java/org/apache/iotdb/db/utils/MemUtils.java | 20 +
.../queryengine/execution/exchange/StubSink.java | 8 +
.../execution/operator/FillOperatorTest.java | 15 +
.../operator/LastQueryMergeOperatorTest.java | 20 +
.../execution/operator/LinearFillOperatorTest.java | 35 ++
.../process/join/InnerTimeJoinOperatorTest.java | 60 +++
.../join/LeftOuterTimeJoinOperatorTest.java | 30 ++
.../plan/planner/PipelineBuilderTest.java | 572 +++++++++++++++++++++
.../InsertionCrossSpaceCompactionRecoverTest.java | 82 +++
.../org/apache/iotdb/commons/path/PartialPath.java | 14 +-
.../apache/iotdb/commons/path/PartialPathTest.java | 8 +
155 files changed, 4062 insertions(+), 1871 deletions(-)
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/flink/it/AbstractTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/flink/it/SinkTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/flink/it/SourceTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/flink/it/Utils.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/spark/it/AbstractTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/spark/it/ReadTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/spark/it/Utils.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/spark/it/WriteTest.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/zeppelin/it/IoTDBInterpreterIT.java
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.sh
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/FakeSubscriber.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/schemaregion/mtree/impl/mem/mnode/estimator/MNodeSizeEstimator.java
=>
queryengine/plan/planner/memory/ConsumeAllChildrenPipelineMemoryEstimator.java}
(58%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/ConsumeChildrenOneByOnePipelineMemoryEstimator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/PipelineMemoryEstimator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/PipelineMemoryEstimatorFactory.java
