This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a change to branch fix-audit-logger
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from a60c0255f73 fix
add 3ddd7b47dc4 with-pro
add 1b2b32bc2a3 perf: Optimize TypeManager allocation in StatementAnalyzer
(#16870)
add 1746cdb28b1 Add system table named connections to resolve the idle
session can be found (#16846)
add 2db7b21bce1 Switch to at.yawk.lz4:lz4-java:1.10.0 (#16871)
add 0bb9e96ece7 Bump at.yawk.lz4:lz4-java from 1.10.0 to 1.10.1 (#16874)
add c560247f8b5 Pipe: Fixed the NPE caused by new regions + history only
logic (#16879)
add a0dbf9b18e0 Pipe: Fixed the bug that reused plugins may not get loader
and visibility (#16877)
add 7436c88304e Implement PreparedStmt on the Server side (#16764) (#16880)
add bc4f8e9bd81 Load: Fix excessive GC caused by loading too many TsFiles
at once (#16853)
add 13b0582dfb1 Pipe: Modify the TableRawReq deserialization method to
support directconversion to TableStatement. (#16844)
add a899c48a09c Optimize memtable region scan (#16883)
add d7898c44343 [AINode] Refactoring of Model Storage, Loading, and
Inference Pipeline (#16819)
add 4deec8fa5fa Merge remote-tracking branch 'upstream/master' into
fix-audit-logger
add 8a6b8d523ad sptls
add b2b84f3c235 fix-temp
add fd352cecc45 pattern-priv
No new revisions were added by this update.
Summary of changes:
.github/workflows/cluster-it-1c1d1a.yml | 3 -
LICENSE-binary | 4 +-
dependencies.json | 2 +-
.../iotdb/it/env/cluster/node/AINodeWrapper.java | 2 +-
.../iotdb/itbase/runtime/ClusterTestStatement.java | 7 +
.../iotdb/ainode/it/AINodeCallInferenceIT.java | 117 ++
.../ainode/it/AINodeConcurrentForecastIT.java | 113 ++
.../ainode/it/AINodeConcurrentInferenceIT.java | 187 ---
.../apache/iotdb/ainode/it/AINodeForecastIT.java | 98 ++
.../iotdb/ainode/it/AINodeInferenceSQLIT.java | 344 ------
.../ainode/it/AINodeInstanceManagementIT.java | 79 +-
.../iotdb/ainode/it/AINodeModelManageIT.java | 53 +-
.../apache/iotdb/ainode/utils/AINodeTestUtils.java | 119 +-
.../it/db/it/IoTDBPreparedStatementIT.java | 385 ++++++
.../relational/it/schema/IoTDBDatabaseIT.java | 14 +-
.../iotdb/session/it/IoTDBConnectionsIT.java | 364 ++++++
.../src/test/resources/ainode-example/config.yaml | 5 -
.../src/test/resources/ainode-example/model.pt | Bin 1906 -> 0 bytes
iotdb-client/session/pom.xml | 14 +
iotdb-core/ainode/ainode.spec | 138 ++-
iotdb-core/ainode/iotdb/ainode/core/config.py | 33 +-
iotdb-core/ainode/iotdb/ainode/core/constant.py | 173 +--
iotdb-core/ainode/iotdb/ainode/core/exception.py | 2 +-
.../ainode/core/inference/inference_request.py | 22 +-
.../core/inference/inference_request_pool.py | 116 +-
.../inference/{strategy => pipeline}/__init__.py | 0
.../core/inference/pipeline/basic_pipeline.py | 87 ++
.../core/inference/pipeline/pipeline_loader.py | 56 +
.../iotdb/ainode/core/inference/pool_controller.py | 83 +-
.../pool_scheduler/basic_pool_scheduler.py | 2 +-
.../strategy/abstract_inference_pipeline.py | 60 -
.../ainode/iotdb/ainode/core/inference/utils.py | 45 +-
.../iotdb/ainode/core/manager/inference_manager.py | 172 +--
.../iotdb/ainode/core/manager/model_manager.py | 162 +--
.../ainode/iotdb/ainode/core/manager/utils.py | 7 +-
.../ainode/core/model/built_in_model_factory.py | 1238 --------------------
.../thread_name.py => model/model_constants.py} | 30 +-
.../ainode/iotdb/ainode/core/model/model_enums.py | 70 --
.../iotdb/ainode/core/model/model_factory.py | 291 -----
.../ainode/iotdb/ainode/core/model/model_info.py | 130 +-
.../ainode/iotdb/ainode/core/model/model_loader.py | 156 +++
.../iotdb/ainode/core/model/model_storage.py | 808 +++++++------
.../core/model/{timerxl => sktime}/__init__.py | 0
.../ainode/core/model/sktime/arima/config.json | 25 +
.../core/model/sktime/configuration_sktime.py | 409 +++++++
.../model/sktime/exponential_smoothing/config.json | 11 +
.../core/model/sktime/gaussian_hmm/config.json | 22 +
.../ainode/core/model/sktime/gmm_hmm/config.json | 24 +
.../ainode/core/model/sktime/modeling_sktime.py | 180 +++
.../core/model/sktime/naive_forecaster/config.json | 9 +
.../ainode/core/model/sktime/pipeline_sktime.py | 68 ++
.../core/model/sktime/stl_forecaster/config.json | 22 +
.../ainode/core/model/sktime/stray/config.json | 11 +
.../ainode/core/model/sundial/modeling_sundial.py | 21 +-
.../sundial/pipeline_sundial.py} | 42 +-
.../iotdb/ainode/core/model/timer_xl}/__init__.py | 0
.../{timerxl => timer_xl}/configuration_timer.py | 0
.../model/{timerxl => timer_xl}/modeling_timer.py | 15 +-
.../timer_xl/pipeline_timer.py} | 36 +-
.../{timerxl => timer_xl}/ts_generation_mixin.py | 0
.../ainode/iotdb/ainode/core/model/uri_utils.py | 137 ---
iotdb-core/ainode/iotdb/ainode/core/model/utils.py | 98 ++
iotdb-core/ainode/iotdb/ainode/core/rpc/client.py | 39 -
iotdb-core/ainode/iotdb/ainode/core/rpc/handler.py | 47 +-
iotdb-core/ainode/pyproject.toml | 9 +-
.../async/AsyncAINodeHeartbeatClientPool.java | 19 +-
.../async/AsyncDataNodeHeartbeatClientPool.java | 1 -
.../consensus/request/ConfigPhysicalPlan.java | 24 -
.../request/read/model/GetModelInfoPlan.java | 64 -
.../request/read/model/ShowModelPlan.java | 70 --
.../request/write/model/CreateModelPlan.java | 79 --
.../request/write/model/DropModelInNodePlan.java | 70 --
.../request/write/model/DropModelPlan.java | 79 --
.../request/write/model/UpdateModelInfoPlan.java | 122 --
.../consensus/response/model/GetModelInfoResp.java | 63 -
.../consensus/response/model/ModelTableResp.java | 62 -
.../iotdb/confignode/manager/ConfigManager.java | 179 ---
.../apache/iotdb/confignode/manager/IManager.java | 42 -
.../iotdb/confignode/manager/ModelManager.java | 245 ----
.../iotdb/confignode/manager/ProcedureManager.java | 20 -
.../PipeConfigTreePrivilegeParseVisitor.java | 8 +-
.../iotdb/confignode/persistence/ModelInfo.java | 378 ------
.../persistence/executor/ConfigPlanExecutor.java | 25 -
.../persistence/pipe/PipePluginInfo.java | 78 +-
.../procedure/impl/model/CreateModelProcedure.java | 250 ----
.../procedure/impl/model/DropModelProcedure.java | 200 ----
.../procedure/impl/node/RemoveAINodeProcedure.java | 17 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 1 +
.../procedure/state/RemoveAINodeState.java | 1 -
.../procedure/store/ProcedureFactory.java | 12 -
.../confignode/procedure/store/ProcedureType.java | 2 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 25 -
.../PipeConfigTreePrivilegeParseVisitorTest.java | 97 +-
iotdb-core/datanode/pom.xml | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 55 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +
.../request/PipeTransferTabletBatchReqV2.java | 7 +-
.../request/PipeTransferTabletRawReq.java | 110 +-
.../request/PipeTransferTabletRawReqV2.java | 50 +-
.../pipe/sink/util/TabletStatementConverter.java | 476 ++++++++
.../sink/util/sorter/InsertEventDataAdapter.java | 127 ++
.../util/sorter/InsertTabletStatementAdapter.java | 118 ++
...EventSorter.java => PipeInsertEventSorter.java} | 94 +-
.../sorter/PipeTableModelTabletEventSorter.java | 67 +-
.../sorter/PipeTreeModelTabletEventSorter.java | 48 +-
.../db/pipe/sink/util/sorter/TabletAdapter.java | 113 ++
.../db/protocol/client/AINodeClientFactory.java | 133 ---
.../iotdb/db/protocol/client/ConfigNodeClient.java | 30 +-
.../protocol/client/DataNodeClientPoolFactory.java | 42 +-
.../db/protocol/client/ainode/AINodeClient.java | 401 -------
.../client/ainode/AINodeClientManager.java | 75 --
.../iotdb/db/protocol/client/an/AINodeClient.java | 321 +++++
.../client/an/AINodeClientManager.java} | 31 +-
.../iotdb/db/protocol/session/ClientSession.java | 23 +
.../iotdb/db/protocol/session/IClientSession.java | 47 +
.../db/protocol/session/InternalClientSession.java | 24 +
.../db/protocol/session/MqttClientSession.java | 24 +
.../db/protocol/session/PreparedStatementInfo.java | 99 ++
.../db/protocol/session/RestClientSession.java | 25 +
.../iotdb/db/protocol/session/SessionManager.java | 55 +-
.../thrift/handler/BaseServerContextHandler.java | 1 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 320 ++++-
.../db/queryengine/common/ConnectionInfo.java | 64 +
.../operator/process/ai/InferenceOperator.java | 82 +-
.../InformationSchemaContentSupplierFactory.java | 152 +--
.../iotdb/db/queryengine/plan/Coordinator.java | 106 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 134 +--
.../db/queryengine/plan/analyze/IModelFetcher.java | 4 -
.../db/queryengine/plan/analyze/ModelFetcher.java | 51 +-
.../execution/config/TableConfigTaskVisitor.java | 25 +-
.../config/executor/ClusterConfigTaskExecutor.java | 43 +-
.../execution/config/session/DeallocateTask.java | 72 ++
.../plan/execution/config/session/PrepareTask.java | 85 ++
.../session/PreparedStatementMemoryManager.java | 157 +++
.../plan/node/process/AI/InferenceNode.java | 3 +-
.../parameter/model/ModelInferenceDescriptor.java | 61 +-
.../relational/analyzer/StatementAnalyzer.java | 91 +-
.../analyzer/StatementAnalyzerFactory.java | 17 +-
.../function/tvf/ForecastTableFunction.java | 39 +-
.../plan/relational/metadata/Metadata.java | 6 -
.../relational/metadata/TableMetadataImpl.java | 5 -
.../plan/relational/planner/TableModelPlanner.java | 25 +-
.../DataNodeLocationSupplierFactory.java | 2 +-
.../plan/relational/sql/AstMemoryEstimator.java | 67 ++
.../plan/relational/sql/ParameterExtractor.java | 121 ++
.../plan/relational/sql/ast/AstVisitor.java | 16 +
.../sql/ast/{Use.java => Deallocate.java} | 34 +-
.../sql/ast/{RangeQuantifier.java => Execute.java} | 65 +-
.../{TableSubquery.java => ExecuteImmediate.java} | 60 +-
.../plan/relational/sql/ast/LoadTsFile.java | 61 +-
.../sql/ast/{Table.java => Prepare.java} | 52 +-
.../plan/relational/sql/ast/Statement.java | 25 +
.../plan/relational/sql/parser/AstBuilder.java | 38 +
.../db/queryengine/plan/statement/Statement.java | 23 +
.../plan/statement/crud/InsertBaseStatement.java | 10 +
.../plan/statement/crud/InsertTabletStatement.java | 197 ++++
.../plan/statement/crud/LoadTsFileStatement.java | 48 +
.../db/queryengine/plan/udf/UDTFForecast.java | 25 +-
.../db/storageengine/dataregion/DataRegion.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 86 +-
.../memtable/AlignedWritableMemChunk.java | 132 ++-
.../dataregion/memtable/IMemTable.java | 6 +-
.../dataregion/memtable/TsFileProcessor.java | 18 +-
.../dataregion/memtable/WritableMemChunk.java | 52 +-
.../pipe/sink/PipeDataNodeThriftRequestTest.java | 4 +-
.../db/pipe/sink/PipeStatementEventSorterTest.java | 313 +++++
.../sink/util/TabletStatementConverterTest.java | 607 ++++++++++
.../plan/relational/analyzer/AnalyzerTest.java | 7 +-
.../plan/relational/analyzer/AuthTest.java | 10 +-
.../analyzer/RowPatternRecognitionTest.java | 4 +-
.../plan/relational/analyzer/TSBSMetadata.java | 6 -
.../relational/analyzer/TableFunctionTest.java | 3 -
.../plan/relational/analyzer/TestMetadata.java | 19 -
.../plan/relational/planner/PlanTester.java | 4 +-
.../memtable/WritableMemChunkRegionScanTest.java | 359 ++++++
iotdb-core/node-commons/pom.xml | 5 +
.../iotdb/commons/client/ClientPoolFactory.java | 28 +
.../async/AsyncAINodeInternalServiceClient.java} | 25 +-
.../iotdb/commons/model/ModelInformation.java | 43 +-
.../org/apache/iotdb/commons/model/ModelTable.java | 4 +-
.../service/PipePluginExecutableManager.java | 9 +
.../schema/column/ColumnHeaderConstant.java | 8 +
.../commons/schema/table/InformationSchema.java | 35 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 23 +
.../thrift-ainode/src/main/thrift/ainode.thrift | 8 +-
.../src/main/thrift/confignode.thrift | 63 -
.../thrift-datanode/src/main/thrift/client.thrift | 1 +
pom.xml | 16 +-
188 files changed, 8748 insertions(+), 7413 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeCallInferenceIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeConcurrentForecastIT.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeConcurrentInferenceIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeForecastIT.java
delete mode 100644
integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeInferenceSQLIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPreparedStatementIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java
delete mode 100644
integration-test/src/test/resources/ainode-example/config.yaml
delete mode 100644 integration-test/src/test/resources/ainode-example/model.pt
rename iotdb-core/ainode/iotdb/ainode/core/inference/{strategy =>
pipeline}/__init__.py (100%)
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/inference/pipeline/basic_pipeline.py
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/inference/pipeline/pipeline_loader.py
delete mode 100644
iotdb-core/ainode/iotdb/ainode/core/inference/strategy/abstract_inference_pipeline.py
delete mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/built_in_model_factory.py
copy iotdb-core/ainode/iotdb/ainode/core/{util/thread_name.py =>
model/model_constants.py} (54%)
delete mode 100644 iotdb-core/ainode/iotdb/ainode/core/model/model_enums.py
delete mode 100644 iotdb-core/ainode/iotdb/ainode/core/model/model_factory.py
create mode 100644 iotdb-core/ainode/iotdb/ainode/core/model/model_loader.py
rename iotdb-core/ainode/iotdb/ainode/core/model/{timerxl =>
sktime}/__init__.py (100%)
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/arima/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/configuration_sktime.py
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/exponential_smoothing/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/gaussian_hmm/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/gmm_hmm/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/modeling_sktime.py
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/naive_forecaster/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/pipeline_sktime.py
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/stl_forecaster/config.json
create mode 100644
iotdb-core/ainode/iotdb/ainode/core/model/sktime/stray/config.json
rename
iotdb-core/ainode/iotdb/ainode/core/{inference/strategy/timer_sundial_inference_pipeline.py
=> model/sundial/pipeline_sundial.py} (56%)
copy {iotdb-client/client-py/iotdb =>
iotdb-core/ainode/iotdb/ainode/core/model/timer_xl}/__init__.py (100%)
rename iotdb-core/ainode/iotdb/ainode/core/model/{timerxl =>
timer_xl}/configuration_timer.py (100%)
rename iotdb-core/ainode/iotdb/ainode/core/model/{timerxl =>
timer_xl}/modeling_timer.py (98%)
rename
iotdb-core/ainode/iotdb/ainode/core/{inference/strategy/timerxl_inference_pipeline.py
=> model/timer_xl/pipeline_timer.py} (52%)
rename iotdb-core/ainode/iotdb/ainode/core/model/{timerxl =>
timer_xl}/ts_generation_mixin.py (100%)
delete mode 100644 iotdb-core/ainode/iotdb/ainode/core/model/uri_utils.py
create mode 100644 iotdb-core/ainode/iotdb/ainode/core/model/utils.py
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelInNodePlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/GetModelInfoResp.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/ModelTableResp.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ModelInfo.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateModelProcedure.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/DropModelProcedure.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/InsertEventDataAdapter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/InsertTabletStatementAdapter.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/{PipeTabletEventSorter.java
=> PipeInsertEventSorter.java} (65%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/TabletAdapter.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/AINodeClientFactory.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ainode/AINodeClient.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ainode/AINodeClientManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/an/AINodeClient.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/schemaregion/write/req/impl/AutoCreateDeviceMNodePlanImpl.java
=> protocol/client/an/AINodeClientManager.java} (52%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedStatementInfo.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/ConnectionInfo.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/DeallocateTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/PrepareTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/PreparedStatementMemoryManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/AstMemoryEstimator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ParameterExtractor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{Use.java
=> Deallocate.java} (65%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{RangeQuantifier.java
=> Execute.java} (53%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{TableSubquery.java
=> ExecuteImmediate.java} (52%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{Table.java
=> Prepare.java} (57%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementEventSorterTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
rename
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/protocol/client/ainode/AsyncAINodeServiceClient.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/client/async/AsyncAINodeInternalServiceClient.java}
(83%)
