This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a change to branch wal_compression
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from fff943deef8 enhance version judgement
add 5d552729ae8 Optimize node startup process & Normalize
system.properties management (#12752)
add b18412d62d5 [IOTDB-6340] Clear Coordinator.queryExecutionMap while
encountering MemoryNotEnough Exception
add 3d436b93802 [IOTDB-6342] Clear
FragmentInstanceManager.dataNodeQueryContextMap while encountering
MemoryNotEnough Exception (#12762)
add 612ecd4f75e Refactor node registration check (#12766)
add f22e3791694 Fix overlap between tsfile is not correctly marked (#12748)
add 6cbdf31bf6f Subscription: support topic loose range for path and time
(#12760)
add 36dd4890577 Add clean logic for FragmentInstance in case that callback
is not added. (#12768)
add 4e50e847378 SystemPropertiesHandler use Files::move instead of
File::renameTo (#12770)
add f196cf20900 Pipe: Filter devices by pattern before reading device
metadata from TsFile (#12765)
add 48028a15f3f Pipe: Adjusted some loggers of metrics to avoid
unnecessary warns & Include "lastEvent" into pipe's event count metrics &
Removed the "userConflict" judgment to data sync failure caused by
METADATA_ERROR (#12758)
add 881816265f6 Synchronize takeSnapshotAsync on RaftGroupID (#12767)
add 60b12d87351 SystemPropertiesHandler close reader and writer #12772
add 31cd61f771d Pipe Consensus: Fix events in connector are not in order
because of parameter `'realtime-first' = 'true'` (#12773)
add df68083d95a Pipe: Avoid event being continuously rate-limited even its
pipe is dropped (#12753)
add fe52d5de3e6 Load: Add some load metrics of time cost, write point and
disk throughput (#12735)
add 9c3d3ea8036 Update tsfile version (#12774)
add e58d427f2f6 Load: Avoid potential NPE in warn log when
LoadRuntimeOutOfMemoryException occurs & load-tsfile: Polish logs (#12777)
add 9e0b62181d0 Support editing cluster name (#12776)
add 500d4a911dd remove wal files when drop database (#12778)
add 4900407d8b1 load-tsfile: Add retry logic for load memory controls
(#12782)
add 3dd6c8c3213 update tsfile version
add bce9631e108 Add memory control for MergeReader
add 7ee30fb631a Classification properties (#12764)
add b1e9d96f49e bind cluster name to metrics (#12786)
add 4dcec3e7e95 Merge branch 'master' into wal_compression
No new revisions were added by this update.
Summary of changes:
distribution/src/assembly/all.xml | 9 -
distribution/src/assembly/confignode.xml | 9 -
distribution/src/assembly/datanode.xml | 9 -
.../iotdb/it/env/cluster/ClusterConstant.java | 3 -
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../it/env/cluster/node/ConfigNodeWrapper.java | 4 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 2 +-
.../iotdb/db/it/IoTDBSetConfigurationIT.java | 91 +-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 36 +
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 15 +-
.../it/IoTDBSubscriptionITConstant.java | 2 +-
.../it/dual/IoTDBSubscriptionTopicIT.java | 135 ++
.../java/org/apache/iotdb/tool/ImportTsFile.java | 31 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 34 +-
.../rpc/subscription/config/TopicConstant.java | 6 +
.../handlers/ConfigNodeAsyncRequestContext.java | 6 +
.../confignode/conf/ConfigNodeDescriptor.java | 9 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 3 +-
.../ConfigNodeSystemPropertiesHandler.java} | 44 +-
.../confignode/conf/SystemPropertiesUtils.java | 163 +-
.../response/datanode/DataNodeRegisterResp.java | 6 +-
.../iotdb/confignode/manager/ConfigManager.java | 26 +-
.../apache/iotdb/confignode/manager/IManager.java | 2 +
.../manager/node/ClusterNodeStartUtils.java | 133 +-
.../iotdb/confignode/manager/node/NodeManager.java | 26 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 7 +-
.../protocol/IoTDBConfigRegionConnector.java | 7 +-
.../metric/PipeConfigNodeRemainingTimeMetrics.java | 10 +-
.../metric/PipeConfigRegionConnectorMetrics.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 5 +
.../iotdb/confignode/persistence/NodeInfoTest.java | 3 +-
.../pipe/consensuspipe/ConsensusPipeManager.java | 2 +
.../iotdb/consensus/ratis/RatisConsensus.java | 22 +-
.../DataNodeSystemPropertiesHandler.java} | 40 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 181 +-
.../db/consensus/DataRegionConsensusImpl.java | 9 +-
.../schemaregion/SchemaExecutionVisitor.java | 4 +-
.../schemaregion/SchemaRegionStateMachine.java | 16 +-
.../{PipeAgent.java => PipeDataNodeAgent.java} | 20 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 6 +-
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 19 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 21 +-
.../payload/evolvable/builder/PipeEventBatch.java | 18 +-
.../airgap/IoTDBDataNodeAirGapConnector.java | 1 +
.../airgap/IoTDBDataRegionAirGapConnector.java | 16 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 6 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 4 +-
.../PipeTransferTabletBatchEventHandler.java | 9 +-
.../PipeTransferTabletInsertionEventHandler.java | 22 +-
.../PipeTransferTsFileInsertionEventHandler.java | 8 +-
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 1 +
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 20 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 6 +-
.../protocol/writeback/WriteBackConnector.java | 4 +-
.../consensus/ConsensusPipeDataNodeDispatcher.java | 4 +-
.../ConsensusPipeDataNodeRuntimeAgentGuardian.java | 4 +-
.../consensus/ProgressIndexDataNodeManager.java | 6 +-
.../event/common/terminate/PipeTerminateEvent.java | 4 +-
.../tsfile/TsFileInsertionDataContainer.java | 74 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 8 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 8 +-
.../PipeRealtimeDataRegionLogExtractor.java | 11 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 8 +-
.../listener/PipeInsertionDataNodeListener.java | 4 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 13 +-
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 12 +
.../PipeDataNodeRemainingEventAndTimeOperator.java | 16 +
.../metric/PipeDataRegionConnectorMetrics.java | 4 +-
.../metric/PipeDataRegionExtractorMetrics.java | 8 +-
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 48 +-
.../metric/PipeSchemaRegionConnectorMetrics.java | 2 +-
.../processor/aggregate/AggregateProcessor.java | 4 +-
.../downsampling/PartialPathLastObjectCache.java | 16 +-
.../twostage/combiner/PipeCombineHandler.java | 4 +-
.../combiner/PipeCombineHandlerManager.java | 4 +-
.../protocol/airgap/IoTDBAirGapReceiver.java | 6 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 4 +-
.../visitor/PipeStatementTSStatusVisitor.java | 12 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 4 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 4 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 4 +-
.../pipe/task/connection/PipeEventCollector.java | 4 +-
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 6 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 6 +-
.../subtask/connector/PipeConnectorSubtask.java | 22 +-
.../connector/PipeConnectorSubtaskManager.java | 8 +-
.../subtask/processor/PipeProcessorSubtask.java | 15 +-
.../processor/PipeProcessorSubtaskWorker.java | 21 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 9 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 66 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 27 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +-
.../thrift/impl/DataNodeRegionManager.java | 4 +-
.../db/queryengine/common/MPPQueryContext.java | 51 +-
.../execution/exchange/MPPDataExchangeManager.java | 15 +-
.../fragment/FragmentInstanceContext.java | 31 +-
.../fragment/FragmentInstanceExecution.java | 4 +-
.../fragment/FragmentInstanceManager.java | 10 +
.../execution/load/LoadTsFileManager.java | 9 +-
.../execution/load/LoadTsFileRateLimiter.java | 45 +-
.../queryengine/execution/memory/MemoryPool.java | 9 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/SeriesScanOperator.java | 3 +-
.../execution/operator/source/SeriesScanUtil.java | 3 +-
.../load/LoadTsFileAnalyzeSchemaMemoryBlock.java | 2 +-
.../metric/load/LoadTsFileCostMetricsSet.java | 131 ++
.../metric/{ => load}/LoadTsFileMemMetricSet.java | 2 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 8 +
.../plan/analyze/LoadTsfileAnalyzer.java | 4 +-
.../queryengine/plan/execution/QueryExecution.java | 3 +-
.../config/executor/ClusterConfigTaskExecutor.java | 17 +-
.../plan/optimization/AggregationPushDown.java | 25 +-
.../plan/planner/LocalExecutionPlanner.java | 13 +-
...stimator.java => MemoryReservationManager.java} | 34 +-
.../NotThreadSafeMemoryReservationManager.java | 94 +
...ava => ThreadSafeMemoryReservationManager.java} | 33 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 6 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 46 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 63 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 6 +-
.../CompactionValidationFailedException.java | 14 +-
.../execute/task/AbstractCompactionTask.java | 17 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 18 +-
.../dataregion/memtable/TsFileProcessor.java | 8 +-
.../read/reader/common/PriorityMergeReader.java | 31 +-
.../storageengine/dataregion/wal/WALManager.java | 7 +-
.../file/AbstractTsFileRecoverPerformer.java | 4 +-
.../file/UnsealedTsFileRecoverPerformer.java | 4 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 43 +-
.../broker/SubscriptionPrefetchingQueue.java | 6 +-
.../event/SubscriptionEventBinaryCache.java | 8 +-
.../SubscriptionConnectorSubtaskManager.java | 4 +-
.../extractor/SchemaRegionListeningQueueTest.java | 22 +-
.../resource/PipeTsFileResourceManagerTest.java | 8 +-
.../execution/operator/OperatorMemoryTest.java | 19 +-
.../repair/RepairUnsortedFileCompactionTest.java | 25 +
.../apache/iotdb/metrics/config/MetricConfig.java | 8 +-
iotdb-core/node-commons/pom.xml | 73 +-
.../resources/conf/generate_properties.bat | 65 -
.../assembly/resources/conf/generate_properties.sh | 59 -
.../resources/conf/iotdb-system.properties | 1859 +-------------------
...properties => iotdb-system.properties.template} | 1052 ++++++-----
.../apache/iotdb/commons/conf/CommonConfig.java | 11 +
.../iotdb/commons/conf/CommonDescriptor.java | 5 +
.../iotdb/commons/conf/ConfigurationFileUtils.java | 71 +-
.../iotdb/commons/file/SystemFileFactory.java | 4 -
.../commons/file/SystemPropertiesHandler.java | 200 +++
.../commons/pipe/agent/task/PipeTaskAgent.java | 9 +
.../iotdb/commons/pipe/config/PipeConfig.java | 7 +
.../pipe/connector/limiter/GlobalRateLimiter.java | 43 +-
.../connector/limiter/PipeEndPointRateLimiter.java | 39 +-
.../connector/protocol/IoTDBAirGapConnector.java | 38 +-
.../pipe/connector/protocol/IoTDBConnector.java | 15 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 14 +-
.../queue/ConcurrentIterableLinkedQueue.java | 27 +-
.../commons/pipe/task/meta/PipeTemporaryMeta.java | 8 +-
.../iotdb/commons/service/metric/enums/Metric.java | 6 +-
.../commons/subscription/meta/topic/TopicMeta.java | 2 +
.../src/main/thrift/confignode.thrift | 6 +-
pom.xml | 2 +-
168 files changed, 2625 insertions(+), 3765 deletions(-)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/{manager/pipe/resource/PipeConfigNodeCopiedFileDirStartupCleaner.java
=> conf/ConfigNodeSystemPropertiesHandler.java} (50%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/consensus/ConsensusPipeDataNodeRuntimeAgentGuardian.java
=> conf/DataNodeSystemPropertiesHandler.java} (50%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/{PipeAgent.java
=> PipeDataNodeAgent.java} (81%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/load/LoadTsFileCostMetricsSet.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/{ =>
load}/LoadTsFileMemMetricSet.java (98%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/{ConsumeAllChildrenPipelineMemoryEstimator.java
=> MemoryReservationManager.java} (50%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/NotThreadSafeMemoryReservationManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/{ConsumeAllChildrenPipelineMemoryEstimator.java
=> ThreadSafeMemoryReservationManager.java} (53%)
delete mode 100644
iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat
delete mode 100644
iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh
copy
iotdb-core/node-commons/src/assembly/resources/conf/{iotdb-system.properties =>
iotdb-system.properties.template} (82%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java
