This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch wal-compress-formal-branch
in repository https://gitbox.apache.org/repos/asf/iotdb.git
discard 834e61d3f06 edit configuration item
discard 6249a4ed28c support multi version WAL
discard a974438e86e support compression type in WAL Compress Header
discard defe11908b5 recover some code
discard 4811fbd2810 remove useless code
discard b15d61b2083 fix compilation problem
discard a08d43180b5 fix bug
discard c7ed021ae61 enable wal compression
add 07578e14d70 Fixed backup path detection and optimized output log file
names (#12454)
add 093512b84fb Remove a line of log (#12480)
add a6676d3d529 Fix insertion cross space compaction recover null pointer
exception (#12439)
add 318348e2202 Enhance estimate of FI memory usage
add b8fd82182d2 Implement new plan optimizer: AggregationPushDown
add 411203c52d2 Remove restrictions on the number and length of tags and
attributes
add 3cca91d2e51 fixed start cli output log issue (#12483)
add 4cb0a3965b4 Pipe: Allowed the transfer of logical view alter statement
in schema snapshot (#12488)
add 7036e9ff935 Pipe: dynamically adjust connection timeout to handle
SocketTimeoutException & Avoid resource cleaning when load task is in process
(#12485)
add 3782215a21d Fix redundant lock taking during create multi timeseries
add 7bbd83d4042 [IOTDB-6326] Normalize the table headers of Auth related
statement
add e737aeb6693 Fix tsfile flush error when drop database or remove peer
(#12493)
add a7aaeb4c75d modify direct buffer memory proportion (#12496)
add d675ba14ab5 Pipe: Globally adjust timeout when syncing huge tsfiles &
Speed up file transfer after sync task failed (#12491)
add 5c434dc6a31 Pipe: Changed the hybrid switching status to avoid first
data is not synced realtime in hybrid mode (#12495)
add ddd10057fd7 Load: Add LoadTsfilePointRateLimiter to control load
tsfile throughput rate (#12356)
add dc323ffed6e Fix insert rows mem control error
add c486e3fd1a9 fixed backup bug (#12504)
add 5d916c08e94 Fix timeout doesn't take effect while using builder to
build session
add 679fc98021f Pipe: fix infinite loop with lock when retrying syncing
tsfiles in async connector (which may cause selector & connector worker
deadlock) (#12501)
add c710efbb51d Adjust some access modifier of confignode (#12505)
add 2c59762ad5c enable wal compression
add b8057212491 fix bug
add dcb13a5b893 fix compilation problem
add 175a5f78491 remove useless code
add 648c18ee43e recover some code
add 3d59603d767 support compression type in WAL Compress Header
add de9e810029b support multi version WAL
add 70d84004231 edit configuration item
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (834e61d3f06)
\
N -- N -- N refs/heads/wal-compress-formal-branch (70d84004231)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
distribution/src/assembly/all.xml | 7 +
distribution/src/assembly/cli.xml | 7 +
.../it/env/cluster/config/MppCommonConfig.java | 12 -
.../env/cluster/config/MppSharedCommonConfig.java | 14 -
.../it/env/remote/config/RemoteCommonConfig.java | 10 -
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 -
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 5 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 4 +-
.../iotdb/db/it/auth/IoTDBClusterAuthorityIT.java | 4 +-
.../apache/iotdb/db/it/schema/IoTDBTagLimitIT.java | 46 --
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 28 +-
.../conf/{logback-cli.xml => logback-backup.xml} | 19 +-
.../cli/src/assembly/resources/tools/backup.bat | 127 +++++
.../cli/src/assembly/resources/tools/backup.sh | 134 +++++
.../org/apache/iotdb/tool/IoTDBDataBackTool.java | 329 +++++++++---
.../java/org/apache/iotdb/session/Session.java | 1 +
.../org/apache/iotdb/session/pool/SessionPool.java | 10 +
.../java/org/apache/iotdb/session/SessionTest.java | 6 +
.../iotdb/confignode/manager/ConfigManager.java | 35 +-
.../iotdb/confignode/manager/load/LoadManager.java | 12 +-
.../manager/load/service/HeartbeatService.java | 16 +-
.../iotdb/confignode/manager/node/NodeManager.java | 6 +-
.../iotdb/confignode/persistence/AuthorInfo.java | 10 +-
.../confignode/persistence/node/NodeInfo.java | 7 +-
.../iotdb/confignode/service/ConfigNode.java | 28 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 4 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 22 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 24 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 +
.../client/IoTDBDataNodeAsyncClientManager.java | 4 +
.../async/IoTDBDataRegionAsyncConnector.java | 118 +++--
.../PipeTransferTsFileInsertionEventHandler.java | 33 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 2 +
.../schema/PipeSchemaRegionSnapshotEvent.java | 2 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 67 +--
.../schemaregion/IoTDBSchemaRegionExtractor.java | 8 +-
.../exchange/sender/TwoStageAggregateSender.java | 2 +-
.../protocol/airgap/IoTDBAirGapReceiver.java | 2 +-
.../common/header/ColumnHeaderConstant.java | 12 +
.../execution/MemoryEstimationHelper.java | 87 ++++
.../queryengine/execution/exchange/sink/ISink.java | 3 +-
.../execution/exchange/sink/LocalSinkChannel.java | 11 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 14 +
.../execution/exchange/sink/SinkChannel.java | 14 +
.../execution/exchange/source/ISourceHandle.java | 3 +-
.../exchange/source/LocalSourceHandle.java | 13 +-
.../execution/exchange/source/SourceHandle.java | 13 +
.../execution/load/AlignedChunkData.java | 3 +
.../execution/load/LoadTsFileManager.java | 65 ++-
.../execution/load/LoadTsFileRateLimiter.java | 79 +++
.../execution/load/NonAlignedChunkData.java | 3 +
.../execution/operator/ExplainAnalyzeOperator.java | 21 +-
.../queryengine/execution/operator/Operator.java | 3 +-
.../execution/operator/OperatorContext.java | 15 +-
.../process/AggregationMergeSortOperator.java | 17 +
.../operator/process/AggregationOperator.java | 16 +
.../operator/process/ColumnInjectOperator.java | 12 +
.../operator/process/DeviceViewIntoOperator.java | 13 +
.../operator/process/DeviceViewOperator.java | 14 +
.../execution/operator/process/FillOperator.java | 11 +
.../operator/process/FilterAndProjectOperator.java | 13 +
.../execution/operator/process/IntoOperator.java | 20 +
.../execution/operator/process/LimitOperator.java | 11 +
.../operator/process/LinearFillOperator.java | 12 +
.../operator/process/MergeSortOperator.java | 17 +
.../execution/operator/process/OffsetOperator.java | 11 +
.../operator/process/ProjectOperator.java | 11 +
.../process/RawDataAggregationOperator.java | 13 +
.../operator/process/SingleDeviceViewOperator.java | 11 +
.../process/SlidingWindowAggregationOperator.java | 12 +
.../execution/operator/process/SortOperator.java | 14 +
.../operator/process/TagAggregationOperator.java | 19 +
.../execution/operator/process/TopKOperator.java | 16 +
.../operator/process/TransformOperator.java | 14 +
.../process/join/FullOuterTimeJoinOperator.java | 20 +
.../process/join/HorizontallyConcatOperator.java | 17 +
.../process/join/InnerTimeJoinOperator.java | 17 +
.../process/join/LeftOuterTimeJoinOperator.java | 14 +
.../last/AlignedUpdateLastCacheOperator.java | 16 +
.../AlignedUpdateViewPathLastCacheOperator.java | 6 +
.../process/last/LastQueryCollectOperator.java | 16 +
.../process/last/LastQueryMergeOperator.java | 18 +
.../operator/process/last/LastQueryOperator.java | 17 +
.../process/last/LastQuerySortOperator.java | 17 +
.../process/last/LastQueryTransformOperator.java | 15 +
.../process/last/UpdateLastCacheOperator.java | 16 +
.../last/UpdateViewPathLastCacheOperator.java | 6 +
.../schema/CountGroupByLevelMergeOperator.java | 14 +
.../schema/CountGroupByLevelScanOperator.java | 18 +-
.../operator/schema/CountMergeOperator.java | 12 +
.../schema/NodeManageMemoryMergeOperator.java | 12 +
.../operator/schema/NodePathsConvertOperator.java | 12 +
.../operator/schema/NodePathsCountOperator.java | 12 +
.../operator/schema/SchemaCountOperator.java | 12 +
.../operator/schema/SchemaFetchMergeOperator.java | 15 +
.../operator/schema/SchemaFetchScanOperator.java | 12 +
.../operator/schema/SchemaQueryMergeOperator.java | 13 +
.../schema/SchemaQueryOrderByHeatOperator.java | 14 +
.../operator/schema/SchemaQueryScanOperator.java | 14 +
.../operator/sink/IdentitySinkOperator.java | 16 +
.../operator/sink/ShuffleHelperOperator.java | 16 +
.../AlignedSeriesAggregationScanOperator.java | 16 +
.../operator/source/AlignedSeriesScanOperator.java | 13 +
.../operator/source/ExchangeOperator.java | 13 +
.../operator/source/LastCacheScanOperator.java | 12 +
.../source/SeriesAggregationScanOperator.java | 16 +
.../operator/source/SeriesScanOperator.java | 13 +
.../execution/operator/source/SeriesScanUtil.java | 19 +-
.../operator/source/ShowQueriesOperator.java | 12 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 38 +-
.../analyze/cache/partition/PartitionCache.java | 1 -
.../plan/execution/memory/MemorySourceHandle.java | 9 +
.../plan/optimization/AggregationPushDown.java | 471 +++++++++++++++++
.../plan/optimization/ColumnInjectionPushDown.java | 30 +-
.../plan/planner/LocalExecutionPlanContext.java | 52 ++
.../plan/planner/LocalExecutionPlanner.java | 27 +-
.../plan/planner/LogicalPlanBuilder.java | 363 +------------
.../plan/planner/LogicalPlanVisitor.java | 201 ++------
.../queryengine/plan/planner/LogicalPlanner.java | 6 +-
.../plan/planner/OperatorTreeGenerator.java | 326 +++++++-----
.../planner/distribution/ExchangeNodeAdder.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 5 +-
...ConsumeAllChildrenPipelineMemoryEstimator.java} | 20 +-
...umeChildrenOneByOnePipelineMemoryEstimator.java | 69 +++
.../planner/memory/PipelineMemoryEstimator.java | 79 +++
.../memory/PipelineMemoryEstimatorFactory.java | 51 ++
.../plan/planner/plan/node/PlanGraphPrinter.java | 22 +
.../plan/planner/plan/node/PlanNodeId.java | 12 +-
.../plan/planner/plan/node/PlanNodeType.java | 5 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../planner/plan/node/process/ProjectNode.java | 6 +-
...gationNode.java => RawDataAggregationNode.java} | 114 +---
.../node/process/join/FullOuterTimeJoinNode.java | 5 +
.../scheduler/load/LoadTsFileDispatcherImpl.java | 195 ++++---
.../schemaengine/schemaregion/tag/TagLogFile.java | 14 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 +-
.../db/storageengine/dataregion/DataRegion.java | 48 +-
.../task/InsertionCrossSpaceCompactionTask.java | 13 +-
.../compaction/schedule/CompactionTaskManager.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 3 +
.../dataregion/memtable/TsFileProcessor.java | 167 +++++-
.../dataregion/memtable/WritableMemChunkGroup.java | 3 +
.../db/storageengine/rescon/memory/SystemInfo.java | 2 +-
.../queryengine/execution/exchange/StubSink.java | 8 +
.../execution/operator/FillOperatorTest.java | 15 +
.../operator/LastQueryMergeOperatorTest.java | 20 +
.../execution/operator/LinearFillOperatorTest.java | 35 ++
.../process/join/InnerTimeJoinOperatorTest.java | 60 +++
.../join/LeftOuterTimeJoinOperatorTest.java | 30 ++
.../plan/optimization/AggregationPushDownTest.java | 265 ++++++++++
.../optimization/ColumnInjectionPushDownTest.java | 214 ++++----
.../plan/optimization/LimitOffsetPushDownTest.java | 6 +-
.../plan/optimization/OptimizationTestUtil.java | 36 +-
.../plan/optimization/PredicatePushDownTest.java | 6 +-
.../plan/optimization/TestPlanBuilder.java | 28 +-
.../plan/planner/PipelineBuilderTest.java | 572 +++++++++++++++++++++
.../distribution/AggregationAlignByDeviceTest.java | 26 +-
.../distribution/AggregationDistributionTest.java | 4 +-
.../AlignByDeviceOrderByLimitOffsetTest.java | 18 +-
.../planner/distribution/AlignedByDeviceTest.java | 215 ++++++--
.../logical/DataQueryLogicalPlannerTest.java | 394 ++++++--------
.../logical/SchemaQueryLogicalPlannerTest.java | 11 +-
.../InsertionCrossSpaceCompactionRecoverTest.java | 82 +++
.../dataregion/memtable/TsFileProcessorTest.java | 200 +++++++
.../resources/conf/iotdb-common.properties | 10 -
.../async/AsyncPipeDataTransferServiceClient.java | 13 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 53 +-
.../iotdb/commons/conf/CommonDescriptor.java | 16 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 4 -
.../iotdb/commons/pipe/config/PipeConfig.java | 11 +-
.../pipe/connector/client/IoTDBClientManager.java | 45 ++
.../connector/client/IoTDBSyncClientManager.java | 4 +-
.../connector/protocol/IoTDBAirGapConnector.java | 2 +-
.../commons/pipe/progress/PipeEventCommitter.java | 17 +
174 files changed, 5462 insertions(+), 1655 deletions(-)
copy iotdb-client/cli/src/assembly/resources/conf/{logback-cli.xml =>
logback-backup.xml} (61%)
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.bat
create mode 100644 iotdb-client/cli/src/assembly/resources/tools/backup.sh
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDown.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/schemaregion/mtree/impl/mem/mnode/estimator/MNodeSizeEstimator.java
=>
queryengine/plan/planner/memory/ConsumeAllChildrenPipelineMemoryEstimator.java}
(58%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/ConsumeChildrenOneByOnePipelineMemoryEstimator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/PipelineMemoryEstimator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/PipelineMemoryEstimatorFactory.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/{AggregationNode.java
=> RawDataAggregationNode.java} (72%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/AggregationPushDownTest.java
