This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a change to branch pipe-memory-estimation-short-term
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 82785393f20 Pipe: localize memory check messages
add fcd52d7ff06 Support IoTDBLocal in UDF for table model (#18035)
add 27af5a6810d Support strict tree table pipe isolation (#17948)
add 837711dcae7 Pipe: Implemented schema batch for meta transfer (#17665)
add bd952e15645 Fix encrypted password pull consumer commit mode (#18044)
add d737f4b301b Stabilize ZSTD compressor level pipe IT (#17917)
add 3bc7b9032f1 Pipe: Fix async sink handshake backoff (#18054)
add b4c4e5483f4 Fix stale deletion resource manager reuse (#18057)
add a76341d9408 Enable show queries can be executed when the available
memory for Operators is insufficient (#18052)
add 6fcbaddc0d6 Add copy fallback for compaction hard links (#18045)
add aabf552629b Avoid copying measurements when updating last cache
(#18059)
add 186ce3672a7 Throttle empty subscription poll logs (#18063)
add b35a575199a Fix deletion visibility in region migration IT (#18065)
add 2fedd8a395d Add degraded status to show pipes (#18047)
add d00ec094a0e Support multiple regions in MIGRATE REGION and multiple
DataNodes in REMOVE DATANODE (#18046)
add 3b7df259b8d Use procedures for region create/delete to fix the
delete-region ghost task (#18033)
add 522c31c7750 Display data partition repair progress by providing a new
SQL statement (#17637)
add 127eff05577 Reduce log level of TsFileGenerator (#18068)
add e9fe467592f Support Thrift client mutual TLS (#18026)
add 4b1d5e50e0b Fix ConfigNode NPE when extend/reconstruct region targets
a non-DataNode id (#18075)
add 383458fb72c Fix JDBC reusing stale statement id after reconnect
(#18078)
add 1c8ed311c95 Support target database in write-back sink (#18058)
add 9554e5d8ef0 Pipe: add ConfigNode memory control for log reducer
(#18069)
add fa194d32bb4 [Pipe] Improve pipe control-flow lock responsiveness
(#18076)
add 220e7a319c2 Fix two flaky multi-DataNode/multi-region ITs from #18046
(remove-DataNode selection + migrate precondition) (#18082)
add 660b0ad1c94 Update version to 2.0.11-SNAPSHOT (#18093)
add 0a3e21231ed Fix region-group cleanup: recover-safe submission + retry
that actually re-runs the delete (#18097)
add 8439807f722 Fix C++ client DATE 1000-01-01 query crash (#18096)
add 5ce95b1e05c Pipe: Remove Ratis data region source restriction (#18089)
add 19a9b15c410 Avoid login lock for pipe auth retry (#18048)
add ef5f4ca6441 Fix stale pipe runtime error messages (#18061)
add 2e099a68eb5 Hot-reload procedure completed-cleaner & disk space
warning threshold (+ fix evict-TTL unit) (#18091)
add b9a548c7c5b Merge remote-tracking branch 'origin/master' into
pipe-memory-estimation-short-term
No new revisions were added by this update.
Summary of changes:
CLAUDE.md | 2 +-
code-coverage/pom.xml | 2 +-
distribution/pom.xml | 16 +-
docker/ReadMe.md | 2 +-
example/jdbc/pom.xml | 2 +-
example/mqtt-customize/pom.xml | 2 +-
example/mqtt/pom.xml | 2 +-
example/pipe-count-point-processor/pom.xml | 2 +-
example/pipe-opc-ua-sink/pom.xml | 2 +-
example/pom.xml | 2 +-
example/rest-java-example/pom.xml | 2 +-
example/schema/pom.xml | 2 +-
example/session/pom.xml | 2 +-
example/subscription/pom.xml | 2 +-
example/trigger/pom.xml | 2 +-
example/udf/pom.xml | 2 +-
external-service-impl/mqtt/pom.xml | 12 +-
external-service-impl/pom.xml | 4 +-
external-service-impl/rest-openapi/pom.xml | 2 +-
external-service-impl/rest/pom.xml | 18 +-
integration-test/pom.xml | 42 +-
.../iotdblocal/CreateTableProbeFunction.java | 12 +-
.../DeviceIdListInProcessTableFunction.java | 72 +--
.../iotdblocal/DeviceIdListTableFunction.java | 95 ++--
.../DeviceNameEnrichBeforeStartTableFunction.java | 144 ++++++
.../DeviceNameEnrichInProcessTableFunction.java | 120 +++--
.../relational/iotdblocal/DeviceNameFunction.java | 68 +--
.../iotdblocal/DeviceSummaryFunction.java | 98 ++++
.../iotdblocal/DeviceSummaryNoCloseFunction.java | 93 ++++
.../iotdblocal/DropTableProbeFunction.java | 11 +-
.../relational/iotdblocal/InsertProbeFunction.java | 15 +-
.../iotdblocal/IoTDBLocalLogAggregateFunction.java | 93 ++--
.../IoTDBLocalLogFunction.java} | 49 +-
.../relational/iotdblocal/IoTDBLocalLogHelper.java | 93 ++++
.../IoTDBLocalLogTableFunction.java} | 63 ++-
...QueryUdafAtBeforeDestroyAggregateFunction.java} | 37 +-
...alQueryUdafAtBeforeStartAggregateFunction.java} | 51 +-
.../LocalQueryUdafInAddInputAggregateFunction.java | 81 ++-
...alQueryUdafInCombineStateAggregateFunction.java | 81 ++-
...alQueryUdafInOutputFinalAggregateFunction.java} | 40 +-
.../MutatingQueryProbeFunction.java} | 29 +-
.../SecretTableQueryFunction.java} | 30 +-
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 29 ++
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/it/utils/TsFileGenerator.java | 6 +-
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
...ataPartitionTableIntegrityCheckProcedureIT.java | 36 ++
.../commit/IoTDBMigrateMultiRegionForIoTV1IT.java | 213 ++++++++
.../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 59 +++
.../commit/IoTDBRegionReconstructForIoTV1IT.java | 55 ++
...TDBRegionMigrateWithDeletionMultiDataDirIT.java | 58 +--
...gionMigrateWithDeletionMultiDataDirTableIT.java | 58 +--
.../daily/iotv1/RegionMigrateFileAssertions.java | 167 ++++++
.../IoTDBRemoveDataNodeNormalIT.java | 24 +-
.../removedatanode/IoTDBRemoveDataNodeUtils.java | 96 ++++
.../manual/basic/IoTDBPipeDataSinkIT.java | 21 +-
.../manual/basic/IoTDBPipeIsolationIT.java | 215 +++++++-
.../manual/basic/IoTDBPipeLifeCycleIT.java | 10 +
.../tablemodel/manual/basic/IoTDBPipeMemoryIT.java | 1 +
.../manual/basic/IoTDBPipePermissionIT.java | 1 +
.../manual/basic/IoTDBPipeProtocolIT.java | 4 +
.../tablemodel/manual/basic/IoTDBPipeSourceIT.java | 27 +-
.../manual/basic/IoTDBPipeSwitchStatusIT.java | 4 +
.../IoTDBPipeTsFileDecompositionWithModsIT.java | 7 +-
.../manual/basic/IoTDBPipeWithLoadIT.java | 6 +
.../manual/basic/IoTDBTablePatternFormatIT.java | 7 +
.../manual/enhanced/IoTDBPipeAutoConflictIT.java | 4 +
.../manual/enhanced/IoTDBPipeAutoDropIT.java | 1 +
.../manual/enhanced/IoTDBPipeClusterIT.java | 13 +
.../manual/enhanced/IoTDBPipeDoubleLivingIT.java | 64 ++-
.../manual/enhanced/IoTDBPipeIdempotentIT.java | 1 +
.../manual/enhanced/IoTDBPipeMetaIT.java | 3 +
.../manual/enhanced/IoTDBPipeNullValueIT.java | 1 +
.../enhanced/IoTDBPipeSinkCompressionIT.java | 16 +-
.../manual/enhanced/IoTDBPipeSinkParallelIT.java | 2 +
.../auto/enhanced/IoTDBPipeSinkCompressionIT.java | 180 +++----
...RegionIT.java => IoTDBPipeWriteBackSinkIT.java} | 60 +--
.../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 1 +
.../relational/it/db/it/udf/IoTDBLocalIT.java | 451 +++++++++++++++++
.../relational/it/db/it/udf/IoTDBLocalLogIT.java | 159 ++++++
.../relational/it/db/it/udf/SQLFunctionUtils.java | 1 -
.../relational/it/schema/IoTDBDatabaseIT.java | 5 +-
...lientSSLIT.java => IoTDBClientMutualSSLIT.java} | 115 +++--
.../IoTDBEncryptedPasswordPullConsumerIT.java | 1 +
iotdb-api/external-api/pom.xml | 2 +-
iotdb-api/external-service-api/pom.xml | 2 +-
iotdb-api/pipe-api/pom.xml | 2 +-
iotdb-api/pom.xml | 2 +-
iotdb-api/trigger-api/pom.xml | 2 +-
iotdb-api/udf-api/pom.xml | 2 +-
.../java/org/apache/iotdb/udf/api/IoTDBLocal.java | 70 +++
.../udf/api/{State.java => UDFResultSet.java} | 28 +-
.../udf/api/relational/AggregateFunction.java | 33 ++
.../iotdb/udf/api/relational/ScalarFunction.java | 19 +
.../processor/TableFunctionDataProcessor.java | 34 ++
.../processor/TableFunctionLeafProcessor.java | 17 +
iotdb-client/cli/pom.xml | 26 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 44 ++
.../src/main/java/org/apache/iotdb/cli/Cli.java | 23 +-
.../org/apache/iotdb/tool/common/Constants.java | 8 +
.../org/apache/iotdb/tool/common/OptionsUtil.java | 20 +
.../apache/iotdb/tool/data/AbstractDataTool.java | 21 +
.../iotdb/tool/schema/AbstractSchemaTool.java | 18 +
iotdb-client/client-cpp/README.md | 4 +-
iotdb-client/client-cpp/README_zh.md | 4 +-
iotdb-client/client-cpp/pom.xml | 4 +-
iotdb-client/client-cpp/src/include/Common.h | 2 +-
iotdb-client/client-cpp/src/session/Date.cpp | 3 -
iotdb-client/client-cpp/src/session/SessionC.cpp | 4 +-
iotdb-client/client-cpp/test/cpp/sessionCIT.cpp | 30 ++
iotdb-client/client-cpp/test/cpp/sessionIT.cpp | 28 ++
iotdb-client/client-py/README.md | 36 +-
iotdb-client/client-py/iotdb/Session.py | 28 +-
iotdb-client/client-py/iotdb/SessionPool.py | 8 +
iotdb-client/client-py/iotdb/dbapi/Connection.py | 35 +-
iotdb-client/client-py/iotdb/table_session.py | 6 +
iotdb-client/client-py/iotdb/table_session_pool.py | 4 +
iotdb-client/client-py/pom.xml | 8 +-
iotdb-client/client-py/session_ssl_example.py | 16 +-
.../client-py/tests/unit/test_session_ssl.py | 152 ++++++
iotdb-client/isession/pom.xml | 8 +-
iotdb-client/jdbc/pom.xml | 8 +-
.../main/java/org/apache/iotdb/jdbc/Config.java | 4 +
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 4 +-
.../apache/iotdb/jdbc/IoTDBConnectionParams.java | 18 +
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 35 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 8 +
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 28 ++
iotdb-client/pom.xml | 2 +-
iotdb-client/service-rpc/pom.xml | 6 +-
.../apache/iotdb/rpc/BaseRpcTransportFactory.java | 24 +-
.../java/org/apache/iotdb/rpc/RpcSslUtils.java | 75 ++-
.../java/org/apache/iotdb/rpc/RpcUtilsTest.java | 14 +
iotdb-client/session/pom.xml | 10 +-
.../iotdb/session/AbstractSessionBuilder.java | 2 +
.../org/apache/iotdb/session/NodesSupplier.java | 12 +
.../java/org/apache/iotdb/session/Session.java | 16 +
.../apache/iotdb/session/SessionConnection.java | 18 +-
.../apache/iotdb/session/TableSessionBuilder.java | 24 +
.../org/apache/iotdb/session/ThriftConnection.java | 6 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 22 +
.../session/pool/TableSessionPoolBuilder.java | 24 +
iotdb-client/subscription/pom.xml | 14 +-
.../subscription/i18n/SubscriptionMessages.java | 6 +
.../subscription/i18n/SubscriptionMessages.java | 4 +
.../base/AbstractSubscriptionPullConsumer.java | 20 +-
.../base/AbstractSubscriptionPushConsumer.java | 20 +-
.../consumer/base/EmptyPollLogThrottler.java | 62 +++
.../consumer/base/EmptyPollLogThrottlerTest.java | 63 +++
iotdb-core/ainode/pom.xml | 12 +-
iotdb-core/antlr/pom.xml | 2 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 3 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 11 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 6 +-
iotdb-core/calc-commons/pom.xml | 14 +-
.../function/TableFunctionLeafOperator.java | 16 +-
.../process/function/TableFunctionOperator.java | 28 +-
.../relational/aggregation/AccumulatorFactory.java | 81 ++-
.../UserDefinedAggregateFunctionAccumulator.java | 65 ++-
.../GroupedUserDefinedAggregateAccumulator.java | 44 +-
.../relational/ColumnTransformerBuilder.java | 71 ++-
.../calc/plan/planner/TableOperatorGenerator.java | 111 +++-
.../planner/memory/MemoryReservationManager.java | 6 +
.../udf/UserDefineScalarFunctionTransformer.java | 46 +-
iotdb-core/confignode/pom.xml | 30 +-
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 15 +
.../iotdb/confignode/i18n/ManagerMessages.java | 5 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 10 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 15 +
.../iotdb/confignode/i18n/ManagerMessages.java | 5 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 10 +
.../client/async/CnToDnAsyncRequestType.java | 1 -
.../CnToDnInternalServiceAsyncRequestManager.java | 5 -
.../heartbeat/DataNodeHeartbeatHandler.java | 3 +-
.../client/sync/CnToDnSyncRequestType.java | 1 +
.../client/sync/SyncDataNodeClientPool.java | 3 +
.../confignode/conf/ConfigNodeDescriptor.java | 41 ++
.../confignode/conf/ConfigNodeMemoryConfig.java | 89 ++++
.../request/write/pipe/task/AlterPipePlanV2.java | 74 +++
.../request/write/pipe/task/DropPipePlanV2.java | 42 +-
.../pipe/task/OperateMultiplePipesPlanV2.java | 21 +-
.../write/pipe/task/SetPipeStatusPlanV2.java | 52 +-
...eStatusWithStoppedByRuntimeExceptionPlanV2.java | 36 +-
.../response/pipe/task/PipeTableResp.java | 35 +-
.../iotdb/confignode/manager/ConfigManager.java | 29 +-
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ProcedureManager.java | 284 ++++++++---
.../manager/partition/PartitionManager.java | 346 ++++++-------
.../agent/runtime/PipeConfigNodeRuntimeAgent.java | 20 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 17 +-
.../runtime/PipeRuntimeCoordinator.java | 6 +-
.../runtime/heartbeat/PipeHeartbeat.java | 22 +-
.../runtime/heartbeat/PipeHeartbeatParser.java | 91 ++--
.../runtime/heartbeat/PipeHeartbeatScheduler.java | 6 +-
.../pipe/coordinator/task/PipeTaskCoordinator.java | 149 +++++-
.../resource/PipeConfigNodeResourceManager.java | 14 +
.../persistence/partition/PartitionInfo.java | 27 +-
.../confignode/persistence/pipe/PipeInfo.java | 29 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 387 ++++++++++++--
.../procedure/CompletedProcedureRecycler.java | 14 +-
.../confignode/procedure/ProcedureExecutor.java | 34 +-
.../procedure/env/RegionMaintainHandler.java | 12 +-
.../DataPartitionTableIntegrityCheckProcedure.java | 141 +++++-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 77 ++-
.../impl/pipe/task/AlterPipeProcedureV2.java | 64 ++-
.../impl/pipe/task/CreatePipeProcedureV2.java | 60 ++-
.../impl/pipe/task/DropPipeProcedureV2.java | 110 +++-
.../impl/pipe/task/StartPipeProcedureV2.java | 93 +++-
.../impl/pipe/task/StopPipeProcedureV2.java | 56 ++-
.../impl/region/CreateRegionGroupsProcedure.java | 44 +-
.../impl/region/RemoveRegionGroupProcedure.java | 386 ++++++++++++++
.../impl/schema/DeleteDatabaseProcedure.java | 111 +---
...bstractOperateSubscriptionAndPipeProcedure.java | 39 +-
.../subscription/CreateSubscriptionProcedure.java | 18 +-
.../subscription/DropSubscriptionProcedure.java | 17 +-
...ationState.java => RemoveRegionGroupState.java} | 4 +-
.../state/schema/DeleteDatabaseState.java | 5 +-
.../procedure/store/ProcedureFactory.java | 6 +
.../confignode/procedure/store/ProcedureType.java | 1 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../conf/ConfigNodeMemoryConfigTest.java | 61 +++
.../request/ConfigPhysicalPlanSerDeTest.java | 131 +++++
.../consensus/response/pipe/PipeTableRespTest.java | 125 +++++
.../runtime/heartbeat/PipeHeartbeatParserTest.java | 229 ++++++++-
.../confignode/persistence/PartitionInfoTest.java | 27 +
.../pipe/PipeTaskInfoAutoRestartTest.java | 165 +++++-
.../procedure/CompletedProcedureRecyclerTest.java | 76 +++
.../procedure/TestProcedureExecutor.java | 14 +
...aPartitionTableIntegrityCheckProcedureTest.java | 13 +
.../pipe/AbstractOperatePipeProcedureV2Test.java | 168 +++++++
.../impl/pipe/task/AlterPipeProcedureV2Test.java | 1 +
.../impl/pipe/task/AlterPipeProcedureV3Test.java | 1 +
.../impl/pipe/task/CreatePipeProcedureV2Test.java | 20 +
.../impl/pipe/task/DropPipeProcedureV2Test.java | 22 +
.../impl/pipe/task/StartPipeProcedureV2Test.java | 20 +
.../impl/pipe/task/StopPipeProcedureV2Test.java | 22 +-
.../region/RemoveRegionGroupProcedureTest.java | 100 ++++
iotdb-core/consensus/pom.xml | 14 +-
iotdb-core/datanode/pom.xml | 44 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 22 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 18 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 +
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 20 +
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 34 +-
.../subtask/processor/PipeProcessorSubtask.java | 2 +
.../agent/task/subtask/sink/PipeSinkSubtask.java | 45 ++
.../deletion/DeletionResourceManager.java | 3 +
.../event/common/terminate/PipeTerminateEvent.java | 2 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 33 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 10 +-
.../metric/schema/PipeSchemaRegionSinkMetrics.java | 58 +++
.../resource/tsfile/PipeTsFileResourceManager.java | 5 +
.../batch/PipeSchemaRegionWritePlanEventBatch.java | 557 +++++++++++++++++++++
.../evolvable/request/PipeTransferPlanNodeReq.java | 28 +-
.../airgap/IoTDBSchemaRegionAirGapSink.java | 182 ++++++-
.../sink/protocol/legacy/IoTDBLegacyPipeSink.java | 7 +-
.../thrift/async/IoTDBDataRegionAsyncSink.java | 45 +-
.../thrift/sync/IoTDBSchemaRegionSink.java | 187 ++++++-
.../sink/protocol/writeback/WriteBackSink.java | 383 ++++++++++++--
.../source/dataregion/IoTDBDataRegionSource.java | 88 ----
...istoricalDataRegionTsFileAndDeletionSource.java | 12 +-
.../PipeRealtimeDataRegionHybridSource.java | 98 +++-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 12 +-
.../iotdb/db/protocol/session/SessionManager.java | 25 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 156 +++---
.../thrift/impl/PushMultiPipeMetaHelper.java | 86 ++++
.../common/header/DatasetHeaderFactory.java | 5 +
.../fragment/FragmentInstanceContext.java | 14 +
.../fragment/FragmentInstanceManager.java | 2 +
.../execution/operator/OperatorContext.java | 5 +
.../source/relational/CteScanOperator.java | 2 +-
.../InformationSchemaContentSupplierFactory.java | 5 +
.../execution/schedule/task/DriverTask.java | 21 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 42 ++
.../execution/config/TreeConfigTaskVisitor.java | 9 +
.../config/executor/ClusterConfigTaskExecutor.java | 437 +++++++++++-----
.../config/executor/IConfigTaskExecutor.java | 2 +
.../config/metadata/region/MigrateRegionTask.java | 2 +-
.../ShowRepairDataPartitionTableProgressTask.java} | 54 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 5 +
.../db/queryengine/plan/parser/ASTVisitor.java | 15 +-
.../planner/DataNodeTableOperatorGenerator.java | 25 +-
.../plan/planner/LocalExecutionPlanContext.java | 4 +
.../plan/planner/LocalExecutionPlanner.java | 96 ++--
.../memory/FakedMemoryReservationManager.java | 3 +
.../NotThreadSafeMemoryReservationManager.java | 111 +++-
.../memory/ThreadSafeMemoryReservationManager.java | 5 +
.../plan/planner/plan/node/write/InsertNode.java | 12 +-
.../planner/plan/node/write/InsertRowNode.java | 7 +-
.../planner/plan/node/write/InsertTabletNode.java | 7 +-
.../plan/node/write/RelationalInsertRowNode.java | 8 +-
.../node/write/RelationalInsertTabletNode.java | 9 +-
.../fetcher/cache/TableDeviceCacheEntry.java | 20 +-
.../fetcher/cache/TableDeviceLastCache.java | 44 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 26 +-
.../fetcher/cache/TreeDeviceNormalSchema.java | 13 +-
.../security/TreeAccessCheckVisitor.java | 11 +
.../plan/relational/sql/ast/MigrateRegion.java | 22 +-
.../plan/relational/sql/parser/AstBuilder.java | 8 +-
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 7 +
.../metadata/region/MigrateRegionStatement.java | 14 +-
...RepairDataPartitionTableProgressStatement.java} | 11 +-
.../db/queryengine/udf/InternalQueryExecutor.java | 133 +++++
.../db/queryengine/udf/InternalQueryResult.java | 65 +++
.../iotdb/db/queryengine/udf/IoTDBLocalImpl.java | 147 ++++++
.../iotdb/db/queryengine/udf/UDFResultSetImpl.java | 154 ++++++
.../iotdb/db/service/ExternalRPCService.java | 84 +++-
.../iotdb/db/service/RegionMigrateService.java | 12 +-
.../RepairUnsortedFileCompactionPerformer.java | 9 +-
.../task/InsertionCrossSpaceCompactionTask.java | 8 +-
.../modification/v1/ModificationFileV1.java | 3 +-
.../dataregion/tsfile/TsFileResource.java | 13 +-
.../apache/iotdb/db/utils/cte/MemoryReader.java | 18 +-
.../cache/TreeDeviceSchemaCacheManagerTest.java | 39 ++
.../db/pipe/consensus/DeletionResourceTest.java | 10 +
.../schema/PipeSchemaRegionSinkMetricsTest.java | 154 ++++++
.../pipe/sink/PipeDataNodeThriftRequestTest.java | 38 ++
.../db/pipe/sink/PipeSchemaRegionSinkTest.java | 386 ++++++++++++++
.../PipeSchemaRegionWritePlanEventBatchTest.java | 464 +++++++++++++++++
.../apache/iotdb/db/pipe/sink/PipeSinkTest.java | 213 ++++++++
.../db/pipe/source/PipeRealtimeExtractTest.java | 151 +++++-
...nternalRPCServiceImplPushMultiPipeMetaTest.java | 246 +++++++++
.../fragment/QueryModificationLoaderTest.java | 3 +
.../execution/operator/CteScanOperatorTest.java | 2 +-
.../config/sys/pipe/ShowPipeTaskTest.java | 72 +++
.../parser/MigrateRegionMultiRegionParseTest.java | 63 +++
.../parser/RemoveDataNodeMultiNodeParseTest.java | 59 +++
.../plan/parser/StatementGeneratorTest.java | 21 +
.../LocalExecutionPlannerOperatorsMemoryTest.java | 164 ++++++
.../write/InsertNodeIsMeasurementFailedTest.java | 22 +
... => MigrateRegionMultiRegionStatementTest.java} | 42 +-
...a => RemoveDataNodeMultiNodeStatementTest.java} | 36 +-
iotdb-core/metrics/core/pom.xml | 4 +-
iotdb-core/metrics/interface/pom.xml | 8 +-
iotdb-core/metrics/pom.xml | 2 +-
iotdb-core/node-commons/pom.xml | 30 +-
.../conf/iotdb-system.properties.template | 18 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 11 +
.../iotdb/commons/conf/CommonDescriptor.java | 27 +
... => RepairDataPartitionTableProgressState.java} | 12 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 118 +++--
.../agent/task/execution/PipeSubtaskExecutor.java | 12 +-
.../pipe/agent/task/meta/PipeMetaKeeper.java | 177 ++++++-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 9 +
.../pipe/agent/task/meta/PipeStaticMeta.java | 4 +
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 4 +
.../pipe/agent/task/meta/PipeTemporaryMeta.java | 29 +-
.../agent/task/meta/PipeTemporaryMetaInAgent.java | 25 +-
.../task/meta/PipeTemporaryMetaInCoordinator.java | 29 +-
.../task/subtask/PipeAbstractSinkSubtask.java | 53 +-
.../agent/task/subtask/PipeReportableSubtask.java | 40 +-
.../pipe/agent/task/subtask/PipeSubtask.java | 11 +
.../pipe/config/constant/SystemConstant.java | 10 +
.../pipe/datastructure/pattern/TablePattern.java | 17 +-
.../pipe/datastructure/pattern/TreePattern.java | 17 +-
.../datastructure/visibility/VisibilityUtils.java | 84 ++--
.../resource/log/PipePeriodicalLogReducer.java | 26 +-
.../pipe/sink/client/IoTDBSyncClientManager.java | 6 +-
.../pipe/sink/limiter/PipeEndPointRateLimiter.java | 2 +-
.../commons/pipe/sink/protocol/IoTDBSink.java | 8 +
.../protocol/PipeSinkWithSchedulingDelay.java} | 12 +-
.../iotdb/commons/pipe/source/IoTDBSource.java | 20 -
.../queryengine/utils/cte/CteDataStore.java | 14 +
.../schema/column/ColumnHeaderConstant.java | 16 +-
.../commons/schema/table/InformationSchema.java | 3 +
.../iotdb/commons/service/metric/enums/Metric.java | 2 +
.../commons/subscription/meta/topic/TopicMeta.java | 4 +
.../org/apache/iotdb/commons/utils/FileUtils.java | 17 +
...monDescriptorDiskSpaceWarningThresholdTest.java | 78 +++
.../agent/task/meta/PipeTemporaryMetaTest.java | 92 ++++
.../visibility/VisibilityUtilsTest.java | 97 ++++
.../resource/log/PipePeriodicalLogReducerTest.java | 65 +++
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 26 +
.../commons/pipe/task/PipeMetaKeeperTest.java | 189 +++++++
.../commons/pipe/task/PipeSleepIntervalTest.java | 86 +++-
iotdb-core/pom.xml | 2 +-
iotdb-core/relational-grammar/pom.xml | 2 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 4 +-
iotdb-protocol/pom.xml | 2 +-
iotdb-protocol/thrift-ainode/pom.xml | 4 +-
iotdb-protocol/thrift-commons/pom.xml | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 1 +
iotdb-protocol/thrift-confignode/pom.xml | 4 +-
.../src/main/thrift/confignode.thrift | 12 +-
iotdb-protocol/thrift-consensus/pom.xml | 4 +-
iotdb-protocol/thrift-datanode/pom.xml | 4 +-
.../src/main/thrift/datanode.thrift | 15 +
library-udf/pom.xml | 4 +-
pom.xml | 2 +-
393 files changed, 15130 insertions(+), 2483 deletions(-)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/TsBlockSerdeFactory.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/CreateTableProbeFunction.java
(72%)
copy
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/function/Split.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceIdListInProcessTableFunction.java
(63%)
copy example/udf/src/main/java/org/apache/iotdb/udf/table/SplitExample.java =>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceIdListTableFunction.java
(60%)
create mode 100644
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceNameEnrichBeforeStartTableFunction.java
copy
example/udf/src/main/java/org/apache/iotdb/udf/table/ExcludeColumnExample.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceNameEnrichInProcessTableFunction.java
(50%)
copy example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceNameFunction.java
(51%)
create mode 100644
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceSummaryFunction.java
create mode 100644
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DeviceSummaryNoCloseFunction.java
copy
iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/NodeType.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/DropTableProbeFunction.java
(74%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/ConfigRegionId.java
=>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/InsertProbeFunction.java
(71%)
copy
example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java =>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/IoTDBLocalLogAggregateFunction.java
(52%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{DatePlus.java
=> iotdblocal/IoTDBLocalLogFunction.java} (56%)
create mode 100644
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/IoTDBLocalLogHelper.java
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{MySplit.java
=> iotdblocal/IoTDBLocalLogTableFunction.java} (69%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{MyCount.java
=> iotdblocal/LocalQueryUdafAtBeforeDestroyAggregateFunction.java} (70%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{MyCount.java
=> iotdblocal/LocalQueryUdafAtBeforeStartAggregateFunction.java} (61%)
copy
example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java =>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/LocalQueryUdafInAddInputAggregateFunction.java
(57%)
copy
example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java =>
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/iotdblocal/LocalQueryUdafInCombineStateAggregateFunction.java
(55%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{MyCount.java
=> iotdblocal/LocalQueryUdafInOutputFinalAggregateFunction.java} (68%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{ContainNull.java
=> iotdblocal/MutatingQueryProbeFunction.java} (67%)
copy
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{ContainNull.java
=> iotdblocal/SecretTableQueryFunction.java} (64%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/commit/IoTDBMigrateMultiRegionForIoTV1IT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/RegionMigrateFileAssertions.java
copy
integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/{IoTDBPipeMultiSchemaRegionIT.java
=> IoTDBPipeWriteBackSinkIT.java} (60%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBLocalIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBLocalLogIT.java
copy
integration-test/src/test/java/org/apache/iotdb/session/it/{IoTDBClientSSLIT.java
=> IoTDBClientMutualSSLIT.java} (62%)
create mode 100644
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/IoTDBLocal.java
copy iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/{State.java =>
UDFResultSet.java} (54%)
create mode 100644 iotdb-client/client-py/tests/unit/test_session_ssl.py
create mode 100644
iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/EmptyPollLogThrottler.java
create mode 100644
iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/EmptyPollLogThrottlerTest.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeMemoryConfig.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/RemoveRegionGroupProcedure.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/{NotifyRegionMigrationState.java
=> RemoveRegionGroupState.java} (93%)
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/ConfigNodeMemoryConfigTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecyclerTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2Test.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemoveRegionGroupProcedureTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeSchemaRegionWritePlanEventBatch.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/PushMultiPipeMetaHelper.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/{session/ShowCurrentUserTask.java
=> sys/ShowRepairDataPartitionTableProgressTask.java} (60%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{RepairDataPartitionTable.java
=> ShowRepairDataPartitionTableProgressStatement.java} (81%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/udf/InternalQueryExecutor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/udf/InternalQueryResult.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/udf/IoTDBLocalImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/udf/UDFResultSetImpl.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/schema/PipeSchemaRegionSinkMetricsTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSchemaRegionSinkTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSchemaRegionWritePlanEventBatchTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplPushMultiPipeMetaTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTaskTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/MigrateRegionMultiRegionParseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/RemoveDataNodeMultiNodeParseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerOperatorsMemoryTest.java
copy
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/{CountDBStatementTest.java
=> MigrateRegionMultiRegionStatementTest.java} (53%)
copy
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/{CountDBStatementTest.java
=> RemoveDataNodeMultiNodeStatementTest.java} (61%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/{ReadConsistencyLevel.java
=> RepairDataPartitionTableProgressState.java} (76%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/NodeDistributionType.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/PipeSinkWithSchedulingDelay.java}
(83%)
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonDescriptorDiskSpaceWarningThresholdTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTemporaryMetaTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtilsTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/resource/log/PipePeriodicalLogReducerTest.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaKeeperTest.java
