This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch preallocate_array_list in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a357ee7c2cb09ec905ababeb797163b3c2883a6a Merge: 3318788728c 3215a4f4782 Author: Tian Jiang <[email protected]> AuthorDate: Tue Apr 22 14:18:07 2025 +0800 Merge branch 'master' into preallocate_array_list .asf.yaml | 3 + .github/workflows/multi-language-client.yml | 6 +- README.md | 2 +- README_ZH.md | 2 +- RELEASE_NOTES.md | 100 ++++ code-coverage/pom.xml | 2 +- distribution/pom.xml | 12 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- .../TableModelSubscriptionSessionExample.java | 1 + example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- .../iotdb/udf/table/ExcludeColumnExample.java | 2 +- .../org/apache/iotdb/udf/table/RepeatExample.java | 2 +- .../org/apache/iotdb/udf/table/SplitExample.java | 2 +- integration-test/pom.xml | 32 +- .../partition/IoTDBPartitionTableAutoCleanIT.java | 102 +++- .../iotdb/db/it/aggregation/IoTDBCountIfIT.java | 13 + .../tablemodel/manual/basic/IoTDBPipeAlterIT.java | 6 +- .../manual/enhanced/IoTDBPipeMetaIT.java | 52 ++ .../relational/it/db/it/IoTDBWindowTVFIT.java | 123 +++- .../db/it/udf/IoTDBUserDefinedTableFunctionIT.java | 80 ++- .../query/old/query/IoTDBGreatestLeastTableIT.java | 6 +- .../it/query/recent/IoTDBComplexQueryIT.java | 84 +++ .../it/query/recent/IoTDBDistinctTagIT.java | 257 ++++++++ .../it/query/recent/IoTDBTableAggregationIT.java | 16 + .../relational/it/rest/it/IoTDBRestServiceIT.java | 14 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 34 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 73 +++ ...TDBConsumer2With1TopicShareProcessTsfileIT.java | 23 +- .../pattern/IoTDBTSPatternPullConsumeTsfileIT.java | 2 - ...TDBConsumer2With1TopicShareProcessTsfileIT.java | 8 +- .../IoTDBTSPatternTsfilePushConsumerIT.java | 2 +- iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- .../api/customizer/parameter/PipeParameters.java | 18 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- iotdb-client/cli/pom.xml | 22 +- .../cli/src/assembly/resources/sbin/start-cli.bat | 3 + .../cli/src/assembly/resources/tools/backup.bat | 7 +- .../org/apache/iotdb/cli/utils/JlineUtils.java | 4 + .../org/apache/iotdb/tool/common/Constants.java | 3 + .../org/apache/iotdb/tool/schema/ExportSchema.java | 16 +- .../iotdb/tool/schema/ExportSchemaTable.java | 117 ++-- iotdb-client/client-cpp/pom.xml | 28 +- .../client-cpp/src/main/AbstractSessionBuilder.h | 3 + iotdb-client/client-cpp/src/main/NodesSupplier.cpp | 221 +++++++ iotdb-client/client-cpp/src/main/NodesSupplier.h | 137 +++++ iotdb-client/client-cpp/src/main/Session.cpp | 169 +++++- iotdb-client/client-cpp/src/main/Session.h | 71 ++- .../client-cpp/src/main/SessionConnection.cpp | 256 ++++++++ .../client-cpp/src/main/SessionConnection.h | 81 +++ .../client-cpp/src/main/ThriftConnection.cpp | 158 +++++ .../client-cpp/src/main/ThriftConnection.h | 68 +++ iotdb-client/client-py/pom.xml | 19 +- iotdb-client/isession/pom.xml | 8 +- .../org/apache/iotdb/isession/SessionDataSet.java | 18 + iotdb-client/jdbc/pom.xml | 8 +- iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 17 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + .../rpc/subscription/annotation/TableModel.java} | 28 +- .../subscription/payload/poll/ErrorPayload.java | 4 + .../subscription/payload/poll/FileSealPayload.java | 27 +- .../subscription/payload/poll/TabletsPayload.java | 41 +- iotdb-client/session/pom.xml | 10 +- .../base/AbstractSubscriptionConsumer.java | 45 +- .../payload/SubscriptionFileHandler.java | 7 - .../subscription/payload/SubscriptionMessage.java | 29 +- .../payload/SubscriptionMessageHandler.java | 7 +- .../payload/SubscriptionSessionDataSet.java | 12 +- .../SubscriptionSessionDataSetsHandler.java | 56 +- .../payload/SubscriptionTsFileHandler.java | 14 +- iotdb-core/ainode/pom.xml | 10 +- iotdb-core/antlr/pom.xml | 2 +- iotdb-core/confignode/pom.xml | 26 +- .../src/assembly/resources/conf/confignode-env.bat | 14 +- .../assembly/resources/sbin/start-confignode.bat | 2 +- .../confignode/persistence/schema/ConfigMTree.java | 23 +- iotdb-core/consensus/pom.xml | 14 +- .../consensus/config/PipeConsensusConfig.java | 28 - .../apache/iotdb/consensus/iot/IoTConsensus.java | 4 +- .../consensus/iot/IoTConsensusServerImpl.java | 44 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 6 +- iotdb-core/datanode/pom.xml | 40 +- .../src/assembly/resources/conf/datanode-env.bat | 14 +- .../src/assembly/resources/sbin/start-datanode.bat | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 29 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 49 +- .../db/consensus/DataRegionConsensusImpl.java | 4 - .../db/exception/runtime/IntoProcessException.java | 17 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 16 +- .../agent/task/builder/PipeDataNodeBuilder.java | 4 +- .../PipeRealtimePriorityBlockingQueue.java | 9 + .../batch/PipeTabletEventTsFileBatch.java | 18 +- .../request/PipeTransferSchemaSnapshotSealReq.java | 5 +- .../connector/protocol/opcda/OpcDaConnector.java | 8 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 5 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 5 +- .../PipeConsensusTsFileInsertionEventHandler.java | 27 +- .../PipeConsensusTransferBatchReqBuilder.java | 1 + .../thrift/sync/IoTDBSchemaRegionConnector.java | 8 +- ...ilder.java => PipeTableModelTsFileBuilder.java} | 154 ++++- .../builder/PipeTableModelTsFileBuilderV2.java | 259 ++++++++ .../util/builder/PipeTreeModelTsFileBuilder.java | 68 ++- .../util/builder/PipeTreeModelTsFileBuilderV2.java | 198 +++++++ .../connector/util/builder/PipeTsFileBuilder.java | 24 +- .../pipe/consensus/deletion/DeletionResource.java | 6 + .../deletion/persist/PageCacheDeletionBuffer.java | 22 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 4 +- .../schemaregion/SchemaRegionListeningFilter.java | 15 + .../pipeconsensus/PipeConsensusReceiver.java | 230 ++++++-- .../pipeconsensus/PipeConsensusReceiverAgent.java | 36 +- .../resource/memory/InsertNodeMemoryEstimator.java | 4 + .../db/protocol/client/DataNodeInternalClient.java | 8 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 45 +- .../table/v1/handler/ExecuteStatementHandler.java | 25 +- .../table/v1/handler/RequestValidationHandler.java | 8 - .../rest/table/v1/impl/RestApiServiceImpl.java | 11 +- .../rest/v1/handler/ExecuteStatementHandler.java | 2 + .../rest/v1/handler/QueryDataSetHandler.java | 2 + .../rest/v2/handler/ExecuteStatementHandler.java | 2 + .../rest/v2/handler/QueryDataSetHandler.java | 2 + .../execution/executor/RegionReadExecutor.java | 17 +- .../execution/executor/RegionWriteExecutor.java | 94 ++- .../operator/process/AbstractIntoOperator.java | 6 +- .../operator/process/FilterAndProjectOperator.java | 15 + .../execution/operator/process/LimitOperator.java | 6 + .../execution/operator/process/OffsetOperator.java | 6 + .../process/join/LeftOuterTimeJoinOperator.java | 15 +- .../join/TableLeftOuterTimeJoinOperator.java | 55 ++ .../relational/AbstractAggTableScanOperator.java | 11 +- .../AbstractDefaultAggTableScanOperator.java | 2 +- .../relational/AbstractTableScanOperator.java | 56 +- .../relational/DeviceIteratorScanOperator.java | 233 ++++++++ .../relational/LastQueryAggTableScanOperator.java | 12 +- .../MeasurementToTableViewAdaptorUtils.java | 103 ++++ .../TreeToTableViewAdaptorOperator.java} | 90 +-- .../relational/ColumnTransformerBuilder.java | 2 +- .../iotdb/db/queryengine/plan/ClusterTopology.java | 4 + .../cache/schema/dualkeycache/IDualKeyCache.java | 28 +- .../dualkeycache/impl/CacheEntryGroupImpl.java | 57 +- .../cache/schema/dualkeycache/impl/CacheStats.java | 47 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 526 ++++++----------- .../schema/dualkeycache/impl/ICacheEntryGroup.java | 13 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 67 ++- .../execution/config/TableConfigTaskVisitor.java | 50 +- .../plan/planner/LocalExecutionPlanner.java | 13 + .../plan/planner/TableOperatorGenerator.java | 595 ++++++++++++++++--- .../planner/plan/node/write/InsertTabletNode.java | 6 +- .../node/write/RelationalInsertTabletNode.java | 14 +- .../plan/parameter/AggregationDescriptor.java | 2 +- .../plan/relational/planner/SymbolAllocator.java | 4 + .../distribute/TableDistributedPlanGenerator.java | 86 ++- .../rule/SingleDistinctAggregationToGroupBy.java | 3 +- .../planner/node/AggregationTableScanNode.java | 11 +- .../TransformAggregationToStreamable.java | 33 +- .../plan/relational/sql/ast/Expression.java | 3 + .../queryengine/plan/relational/sql/ast/Row.java | 33 ++ .../relational/sql/ast/TableExpressionType.java | 3 +- .../plan/relational/sql/parser/AstBuilder.java | 90 ++- .../multi/BooleanGreatestColumnTransformer.java | 2 +- .../multi/BooleanLeastColumnTransformer.java | 2 +- .../iotdb/db/service/metrics/FileMetrics.java | 10 + .../iotdb/db/service/metrics/WritingMetrics.java | 4 + .../db/service/metrics/file/ModsFileMetrics.java | 4 +- .../db/storageengine/dataregion/DataRegion.java | 25 +- .../execute/task/AbstractCompactionTask.java | 3 + .../execute/utils/CompactionPathUtils.java | 20 +- .../execute/utils/MultiTsFileDeviceIterator.java | 10 +- .../fast/FastAlignedSeriesCompactionExecutor.java | 8 +- .../FastNonAlignedSeriesCompactionExecutor.java | 5 +- .../executor/fast/SeriesCompactionExecutor.java | 7 +- .../compaction/repair/RepairDataFileScanUtil.java | 6 +- .../compaction/schedule/CompactionScheduler.java | 16 +- .../selector/impl/SettleSelectorImpl.java | 106 ++-- .../dataregion/flush/MemTableFlushTask.java | 25 +- .../memtable/AbstractWritableMemChunk.java | 4 +- .../memtable/AlignedReadOnlyMemChunk.java | 6 +- .../memtable/AlignedWritableMemChunk.java | 160 ++--- .../dataregion/memtable/IWritableMemChunk.java | 3 +- .../dataregion/memtable/ReadOnlyMemChunk.java | 19 +- .../dataregion/memtable/TsFileProcessor.java | 97 +-- .../dataregion/memtable/WritableMemChunk.java | 108 +--- .../dataregion/modification/TreeDeletionEntry.java | 6 +- .../dataregion/tsfile/TsFileResource.java | 1 - .../wal/utils/MemoryControlledWALEntryQueue.java | 14 + .../load/metrics/LoadTsFileCostMetricsSet.java | 14 +- .../agent/SubscriptionBrokerAgent.java | 9 + .../broker/SubscriptionBlockingPendingQueue.java | 4 + .../db/subscription/broker/SubscriptionBroker.java | 10 + .../broker/SubscriptionPrefetchingQueue.java | 49 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 10 + .../broker/SubscriptionPrefetchingTsFileQueue.java | 13 + .../TsFileDeduplicationBlockingPendingQueue.java | 10 + .../db/subscription/event/SubscriptionEvent.java | 4 +- .../batch/SubscriptionPipeTabletEventBatch.java | 27 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 5 +- .../response/SubscriptionEventTabletResponse.java | 20 +- .../response/SubscriptionEventTsFileResponse.java | 9 +- .../receiver/SubscriptionReceiverV1.java | 29 +- .../db/utils/datastructure/AlignedTVList.java | 162 ++++- .../db/utils/datastructure/BatchEncodeInfo.java | 60 ++ .../db/utils/datastructure/MemPointIterator.java | 3 + .../datastructure/MemPointIteratorFactory.java | 194 ++++-- .../MergeSortMultiAlignedTVListIterator.java | 94 ++- .../MergeSortMultiTVListIterator.java | 76 ++- .../datastructure/MultiAlignedTVListIterator.java | 14 +- .../utils/datastructure/MultiTVListIterator.java | 24 +- .../OrderedMultiAlignedTVListIterator.java | 21 +- .../datastructure/OrderedMultiTVListIterator.java | 36 +- .../iotdb/db/utils/datastructure/TVList.java | 144 ++++- .../cache/dualkeycache/DualKeyCacheTest.java | 171 ------ .../db/pipe/consensus/DeletionResourceTest.java | 38 +- .../operator/DeviceIteratorScanOperatorTest.java | 228 ++++++++ ...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 650 +++++++++++++++++++++ .../TreeToTableViewAdaptorOperatorTest.java | 237 ++++++++ .../compaction/AbstractCompactionTest.java | 4 +- .../compaction/ReadChunkInnerCompactionTest.java | 70 +++ ...eCrossSpaceCompactionWithFastPerformerTest.java | 14 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 12 - .../InnerSeqCompactionWithFastPerformerTest.java | 7 +- ...nerSeqCompactionWithReadChunkPerformerTest.java | 7 +- .../settle/SettleCompactionSelectorTest.java | 20 +- .../reader/chunk/MemAlignedChunkLoaderTest.java | 5 +- .../read/reader/chunk/MemChunkLoaderTest.java | 19 +- iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 24 +- .../conf/iotdb-system.properties.template | 19 +- .../src/assembly/resources/sbin/health_check.bat | 33 +- .../container/PipeConsensusClientMgrContainer.java | 20 +- .../iotdb/commons/concurrent/ThreadName.java | 8 +- .../apache/iotdb/commons/path/MeasurementPath.java | 4 + .../apache/iotdb/commons/path/PathPatternTree.java | 20 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 2 +- .../task/connection/BlockingPendingQueue.java | 4 + .../options/PipeInclusionOptions.java | 117 ++-- .../pipe/datastructure/pattern/TablePattern.java | 2 + .../commons/pipe/extractor/IoTDBExtractor.java | 15 +- .../apache/iotdb/commons/schema/ttl/TTLCache.java | 23 +- .../service/AbstractThriftServiceThread.java | 20 +- .../iotdb/commons/service/ThriftServiceThread.java | 2 - .../relational/TableBuiltinTableFunction.java | 8 + .../relational/tvf/CumulateTableFunction.java | 150 +++++ .../relational/tvf/TumbleTableFunction.java | 130 +++++ .../org/apache/iotdb/commons/utils/FileUtils.java | 114 +++- iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../db/relational/grammar/sql/RelationalSql.g4 | 9 +- iotdb-doap.rdf | 16 + iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- iotdb-protocol/thrift-consensus/pom.xml | 4 +- .../src/main/thrift/pipeconsensus.thrift | 3 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- library-udf/pom.xml | 4 +- pom.xml | 4 +- 273 files changed, 8480 insertions(+), 2312 deletions(-)
