This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch speed_up_recover in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7301c67293c2a80c27ee61118003d212474855bc Merge: c112a4bc63d ed9204e56b0 Author: HTHou <[email protected]> AuthorDate: Fri Jul 26 11:22:28 2024 +0800 Merge branch 'master' of github.com:apache/iotdb into speed_up_recover .github/workflows/multi-language-client.yml | 11 +- distribution/src/assembly/all.xml | 26 +- distribution/src/assembly/cli.xml | 6 +- distribution/src/assembly/client-cpp.xml | 2 +- distribution/src/assembly/common-files.xml | 12 +- distribution/src/assembly/confignode.xml | 10 +- distribution/src/assembly/datanode.xml | 16 +- distribution/src/assembly/library-udf.xml | 8 +- .../apache/iotdb/opcua/ClientExampleRunner.java | 2 + .../apache/iotdb/SubscriptionSessionExample.java | 52 +- integration-test/src/assembly/mpp-test.xml | 20 +- .../it/triple/AbstractSubscriptionTripleIT.java | 78 +++ .../it/triple/IoTDBSubscriptionSharingIT.java | 560 +++++++++++++++++++++ .../java/org/apache/iotdb/tool/ExportData.java | 134 ++--- .../java/org/apache/iotdb/tool/ImportData.java | 2 +- .../java/org/apache/iotdb/tool/ImportTsFile.java | 5 + .../org/apache/iotdb/tool/IoTDBDataBackTool.java | 19 +- .../org/apache/iotdb/tool/WriteDataFileTest.java | 14 + iotdb-client/client-py/.gitignore | 5 +- iotdb-client/client-py/README.md | 97 +--- iotdb-client/client-py/SessionExample.py | 28 -- iotdb-client/client-py/iotdb/Session.py | 67 +++ .../client-py/iotdb/template/InternalNode.py | 41 -- .../client-py/iotdb/template/MeasurementNode.py | 10 + iotdb-client/client-py/iotdb/template/Template.py | 8 + .../client-py/iotdb/template/TemplateNode.py | 8 + iotdb-client/client-py/pom.xml | 8 +- iotdb-client/client-py/release.sh | 3 +- iotdb-client/client-py/requirements_dev.txt | 5 +- .../client-py/{ => resources}/pyproject.toml | 42 +- iotdb-client/client-py/resources/setup.py | 64 --- .../integration/__init__.py} | 28 -- .../tablet_performance_comparison.py | 0 .../{ => integration}/test_aligned_timeseries.py | 0 .../tests/{ => integration}/test_dataframe.py | 0 .../tests/{ => integration}/test_delete_data.py | 0 .../tests/{ => integration}/test_one_device.py | 0 .../tests/{ => integration}/test_session.py | 0 .../tests/{ => integration}/test_session_pool.py | 0 .../tests/{ => integration}/test_tablet.py | 0 .../client-py/tests/{ => integration}/test_todf.py | 0 iotdb-client/client-py/tests/test_template.py | 110 ---- .../unit/__init__.py} | 11 - .../tests/{ => unit}/test_numpy_tablet.py | 0 .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +- .../iotdb/rpc/subscription/config/TopicConfig.java | 47 +- .../exception/SubscriptionConnectionException.java | 5 + .../SubscriptionIdentifierSemanticException.java | 5 + .../SubscriptionIncompatibleHandlerException.java | 5 + .../SubscriptionParameterNotValidException.java | 5 + .../SubscriptionRuntimeCriticalException.java | 5 + .../exception/SubscriptionRuntimeException.java | 5 + .../SubscriptionRuntimeNonCriticalException.java | 5 + .../org/apache/iotdb/session/NodesSupplier.java | 1 + .../java/org/apache/iotdb/session/Session.java | 1 + .../apache/iotdb/session/SessionConnection.java | 1 + .../org/apache/iotdb/session/pool/SessionPool.java | 1 + .../consumer/SubscriptionConsumer.java | 43 +- iotdb-core/confignode/src/assembly/confignode.xml | 4 +- .../confignode/conf/ConfigNodeDescriptor.java | 2 + .../manager/load/balancer/RouteBalancer.java | 3 +- .../router/leader/AbstractLeaderBalancer.java | 20 + .../confignode/manager/load/cache/LoadCache.java | 18 + .../manager/load/service/EventService.java | 1 + .../ConsensusGroupStatisticsChangeEvent.java | 8 + .../manager/pipe/metric/PipeConfigNodeMetrics.java | 2 + .../pipe/metric/PipeConfigNodeReceiverMetrics.java | 169 +++++++ .../metric/PipeConfigNodeRemainingTimeMetrics.java | 1 + .../metric/PipeConfigRegionConnectorMetrics.java | 1 + .../pipe/metric/PipeTemporaryMetaMetrics.java | 1 + .../receiver/protocol/IoTDBConfigNodeReceiver.java | 44 +- .../iotdb/confignode/persistence/ClusterInfo.java | 6 + .../confignode/persistence/ProcedureInfo.java | 5 + .../iotdb/confignode/persistence/TTLInfo.java | 6 + .../confignode/persistence/pipe/PipeInfo.java | 1 + .../persistence/pipe/PipePluginInfo.java | 27 +- .../schema/CNPhysicalPlanGenerator.java | 1 + .../impl/region/AddRegionPeerProcedure.java | 6 + .../impl/region/RemoveRegionPeerProcedure.java | 6 + .../subscription/topic/CreateTopicProcedure.java | 6 +- .../impl/testonly/NeverFinishProcedure.java | 6 + .../exception/ConsensusGroupNotExistException.java | 5 + .../apache/iotdb/consensus/iot/IoTConsensus.java | 6 +- .../consensus/iot/IoTConsensusServerImpl.java | 7 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 2 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 5 +- .../consensus/pipe/PipeConsensusServerImpl.java | 5 + .../apache/iotdb/consensus/ratis/utils/Utils.java | 2 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 11 +- iotdb-core/datanode/pom.xml | 2 +- iotdb-core/datanode/src/assembly/server.xml | 10 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 + .../batch/PipeTabletEventTsFileBatch.java | 7 +- .../request/PipeTransferTabletRawReq.java | 5 + .../protocol/opcua/OpcUaServerBuilder.java | 139 ++--- .../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +- .../async/IoTDBDataRegionAsyncConnector.java | 1 + .../event/realtime/PipeRealtimeEventFactory.java | 15 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 16 +- .../PipeRealtimeDataRegionHeartbeatExtractor.java | 3 + .../PipeRealtimeDataRegionHybridExtractor.java | 8 +- .../PipeRealtimeDataRegionLogExtractor.java | 8 +- .../PipeRealtimeDataRegionTsFileExtractor.java | 8 +- .../realtime/assigner/PipeDataRegionAssigner.java | 24 +- .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 2 + .../pipe/metric/PipeDataNodeReceiverMetrics.java | 333 ++++++++++++ .../PipeDataNodeRemainingEventAndTimeMetrics.java | 1 + .../metric/PipeDataRegionConnectorMetrics.java | 1 + .../metric/PipeDataRegionExtractorMetrics.java | 1 + .../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 1 + .../metric/PipeSchemaRegionConnectorMetrics.java | 1 + .../metric/PipeSchemaRegionExtractorMetrics.java | 1 + .../metric/PipeSchemaRegionListenerMetrics.java | 1 + .../pipe/metric/PipeWALInsertNodeCacheMetrics.java | 1 + .../pipeconsensus/PipeConsensusReceiver.java | 31 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 112 +++-- .../pipe/resource/wal/PipeWALResourceManager.java | 1 + .../pipe/task/connection/PipeEventCollector.java | 26 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 1 + .../impl/DataNodeInternalRPCServiceImpl.java | 1 + .../execution/executor/RegionReadExecutor.java | 10 + .../operator/process/FilterAndProjectOperator.java | 3 +- .../analyze/cache/partition/PartitionCache.java | 5 +- .../analyze/cache/schema/SchemaCacheEntry.java | 1 + .../cache/schema/TimeSeriesSchemaCache.java | 3 +- .../dualkeycache/impl/FIFOCacheEntryManager.java | 2 + .../dualkeycache/impl/LRUCacheEntryManager.java | 2 + .../config/executor/ClusterConfigTaskExecutor.java | 30 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 65 ++- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 3 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 3 +- .../impl/pbtree/mnode/info/CacheMNodeInfo.java | 1 + .../mtree/impl/pbtree/schemafile/SchemaFile.java | 2 + .../mtree/loader/MNodeFactoryLoader.java | 4 + .../metrics/IoTDBInternalLocalReporter.java | 100 +++- .../iotdb/db/storageengine/StorageEngine.java | 5 + .../db/storageengine/dataregion/DataRegion.java | 1 + .../CompactionFileCountExceededException.java | 1 + .../CompactionLastTimeCheckFailedException.java | 1 + .../CompactionMemoryNotEnoughException.java | 1 + .../CompactionValidationFailedException.java | 1 + .../performer/impl/FastCompactionPerformer.java | 9 +- .../execute/task/SettleCompactionTask.java | 10 +- .../execute/utils/MultiTsFileDeviceIterator.java | 3 +- .../compaction/io/CompactionTsFileReader.java | 1 + .../repair/RepairTimePartitionScanTask.java | 1 + .../repair/UnsortedFileRepairTaskScheduler.java | 1 + .../schedule/CompactionScheduleTaskManager.java | 28 +- .../schedule/CompactionScheduleTaskWorker.java | 1 + .../compaction/schedule/CompactionTaskManager.java | 2 + .../compaction/schedule/TTLScheduleTask.java | 1 + .../dataregion/compaction/tool/PrintUtil.java | 2 +- .../compaction/tool/TimePartitionProcessTask.java | 8 +- .../tool/TimePartitionProcessWorker.java | 9 +- .../dataregion/modification/ModificationFile.java | 2 +- .../io/LocalTextModificationAccessor.java | 1 + .../dataregion/snapshot/SnapshotTaker.java | 5 +- .../storageengine/dataregion/wal/WALManager.java | 2 +- .../dataregion/wal/buffer/WALBuffer.java | 5 +- .../storageengine/dataregion/wal/io/LogWriter.java | 9 +- .../dataregion/wal/io/WALFileVersion.java | 54 +- .../dataregion/wal/io/WALInputStream.java | 24 +- .../dataregion/wal/io/WALMetaData.java | 18 +- .../storageengine/dataregion/wal/io/WALWriter.java | 14 +- .../storageengine/dataregion/wal/node/WALNode.java | 180 +++---- .../dataregion/wal/recover/WALRepairWriter.java | 30 +- .../file/UnsealedTsFileRecoverPerformer.java | 4 +- .../rescon/disk/DirectoryChecker.java | 2 + .../db/storageengine/rescon/memory/SystemInfo.java | 20 +- .../broker/SubscriptionPrefetchingQueue.java | 56 +-- .../broker/SubscriptionPrefetchingTabletQueue.java | 58 +-- .../broker/SubscriptionPrefetchingTsFileQueue.java | 68 +-- .../batch/SubscriptionPipeTabletEventBatch.java | 87 +++- .../batch/SubscriptionPipeTsFileEventBatch.java | 68 ++- .../db/tools/schema/PBTreeFileSketchTool.java | 11 +- .../iotdb/db/utils/constant/TestConstant.java | 8 +- .../schemaRegion/SchemaRegionAliasAndTagTest.java | 1 + .../settle/SettleCompactionRecoverTest.java | 4 +- .../wal/compression/WALCompressionTest.java | 13 +- .../dataregion/wal/io/WALFileTest.java | 12 +- .../wal/recover/WALRepairWriterTest.java | 7 +- .../wal/utils/WALInsertNodeCacheTest.java | 4 +- .../apache/iotdb/commons/conf/CommonConfig.java | 2 +- .../commons/consensus/index/ProgressIndex.java | 5 + .../PipeRuntimeConnectorCriticalException.java | 5 + .../pipe/PipeRuntimeNonCriticalException.java | 5 + .../PipeRuntimeOutOfMemoryCriticalException.java | 5 + .../connector/limiter/PipeEndPointRateLimiter.java | 1 + .../connector/protocol/IoTDBAirGapConnector.java | 1 + .../pipe/plugin/meta/PipePluginMetaKeeper.java | 2 +- .../service/PipePluginExecutableManager.java | 18 +- .../commons/pipe/progress/PipeEventCommitter.java | 1 + .../task/subtask/PipeAbstractConnectorSubtask.java | 1 + .../pipe/task/subtask/PipeReportableSubtask.java | 2 +- .../commons/service/metric/GcTimeAlerter.java | 2 + .../iotdb/commons/service/metric/enums/Metric.java | 2 + .../commons/subscription/meta/topic/TopicMeta.java | 4 +- .../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 1 + .../src/main/thrift/datanode.thrift | 1 + .../org/apache/iotdb/library/dlearn/UDTFAR.java | 1 + .../apache/iotdb/library/dmatch/UDTFPtnSym.java | 1 + .../apache/iotdb/library/dprofile/UDTFSample.java | 1 + pom.xml | 4 +- 204 files changed, 2707 insertions(+), 1263 deletions(-)
