This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/joinOperator in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit af66cb885f02e4592e73f4d4711d4a23c4ea9e27 Merge: 9f1d925a566 2ec74a41a63 Author: Beyyes <[email protected]> AuthorDate: Wed Sep 11 11:26:40 2024 +0800 Merge branch 'master' into beyyes/joinOperator .asf.yaml | 2 - .github/workflows/multi-language-client.yml | 17 +- data.zip | Bin 0 -> 101329 bytes .../org/apache/iotdb/it/env/cluster/env/AIEnv.java | 2 +- .../java/org/apache/iotdb/ainode/it/AIEnvIT.java | 80 -- .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 228 ++++ .../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 40 +- .../apache/iotdb/db/it/IoTDBRecoverUnclosedIT.java | 29 +- .../db/it/aggregation/IoTDBAggregationIT.java | 237 ++-- .../aggregation/IoTDBAggregationLargeDataIT.java | 133 +- .../aggregation/IoTDBAggregationSmallDataIT.java | 113 +- .../db/it/orderBy/IoTDBOrderByForDebugIT.java | 1406 -------------------- .../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 8 +- .../db/it/schema/IoTDBDeleteTimeSeriesIT.java | 52 +- .../iotdb/db/it/udaf/IoTDBUDAFNormalQueryIT.java | 44 +- .../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 6 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 13 +- .../it/query/recent/IoTDBNullIdQueryIT.java | 20 + .../relational/it/schema/IoTDBDatabaseIT.java | 119 +- .../iotdb/relational/it/schema/IoTDBDeviceIT.java | 11 +- .../src/test/resources/ainode-example/config.yaml | 5 + .../src/test/resources/ainode-example/model.pt | Bin 0 -> 1906 bytes .../configuration/PipeRuntimeEnvironment.java | 2 + iotdb-client/cli/pom.xml | 10 + .../cli/src/assembly/resources/tools/backup.bat | 2 +- .../cli/src/assembly/resources/tools/backup.sh | 2 +- .../src/assembly/resources/tools/export-data.bat | 2 +- .../src/assembly/resources/tools/export-data.sh | 2 +- .../src/assembly/resources/tools/export-schema.bat | 2 +- .../src/assembly/resources/tools/export-schema.sh | 2 +- .../src/assembly/resources/tools/export-tsfile.bat | 2 +- .../src/assembly/resources/tools/export-tsfile.sh | 2 +- .../src/assembly/resources/tools/import-data.bat | 2 +- .../src/assembly/resources/tools/import-data.sh | 2 +- .../src/assembly/resources/tools/import-schema.bat | 2 +- .../src/assembly/resources/tools/import-schema.sh | 2 +- .../src/assembly/resources/tools/load-tsfile.bat | 2 +- .../src/assembly/resources/tools/load-tsfile.sh | 2 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 4 +- .../iotdb/tool/{ => backup}/IoTDBDataBackTool.java | 3 +- .../iotdb/tool/{ => data}/AbstractDataTool.java | 2 +- .../apache/iotdb/tool/{ => data}/ExportData.java | 2 +- .../apache/iotdb/tool/{ => data}/ImportData.java | 2 +- .../tool/{ => schema}/AbstractSchemaTool.java | 2 +- .../iotdb/tool/{ => schema}/ExportSchema.java | 2 +- .../iotdb/tool/{ => schema}/ImportSchema.java | 2 +- .../tool/{ => tsfile}/AbstractTsFileTool.java | 2 +- .../iotdb/tool/{ => tsfile}/ExportTsFile.java | 2 +- .../iotdb/tool/{ => tsfile}/ImportTsFile.java | 302 +---- .../apache/iotdb/tool/tsfile/ImportTsFileBase.java | 243 ++++ .../iotdb/tool/tsfile/ImportTsFileLocally.java | 53 + .../iotdb/tool/tsfile/ImportTsFileRemotely.java | 338 +++++ .../iotdb/tool/tsfile/ImportTsFileScanTool.java | 95 ++ .../org/apache/iotdb/tool/WriteDataFileTest.java | 2 + iotdb-client/client-py/TableModelSessionExample.py | 154 +++ .../client-py/TableModelSessionPoolExample.py | 160 +++ iotdb-client/client-py/iotdb/Session.py | 113 +- iotdb-client/client-py/iotdb/SessionPool.py | 11 + iotdb-client/client-py/iotdb/utils/Field.py | 15 +- .../client-py/iotdb/utils/IoTDBConstants.py | 6 + .../client-py/iotdb/utils/IoTDBRpcDataSet.py | 16 +- iotdb-client/client-py/iotdb/utils/NumpyTablet.py | 58 +- .../client-py/iotdb/utils/SessionDataSet.py | 4 +- iotdb-client/client-py/iotdb/utils/Tablet.py | 67 +- iotdb-client/client-py/pom.xml | 2 +- iotdb-client/client-py/release.sh | 1 + .../tests/integration/test_relational_session.py | 143 ++ .../org/apache/iotdb/isession/SessionDataSet.java | 34 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 42 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 6 +- .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 6 +- .../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 9 + .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 443 +++--- .../apache/iotdb/session/SessionConnection.java | 15 +- .../org/apache/iotdb/session/ThriftConnection.java | 3 +- .../apache/iotdb/session/pool/SessionPoolTest.java | 10 +- iotdb-core/ainode/iotdb/ainode/client.py | 225 +--- iotdb-core/ainode/iotdb/ainode/config.py | 240 ++-- iotdb-core/ainode/iotdb/ainode/constant.py | 1 - iotdb-core/ainode/iotdb/ainode/exception.py | 23 +- iotdb-core/ainode/iotdb/ainode/factory.py | 272 ---- iotdb-core/ainode/iotdb/ainode/handler.py | 105 +- iotdb-core/ainode/iotdb/ainode/log.py | 45 +- .../ainode/{encryption.py => manager/__init__.py} | 0 .../ainode/iotdb/ainode/manager/cluster_manager.py | 41 + .../{inference.py => manager/inference_manager.py} | 176 ++- .../ainode/iotdb/ainode/manager/model_manager.py | 84 ++ .../ainode/{encryption.py => model/__init__.py} | 0 .../built_in_model_factory.py} | 315 ++++- .../ainode/iotdb/ainode/model/model_factory.py | 237 ++++ .../ainode/iotdb/ainode/model/model_storage.py | 113 ++ iotdb-core/ainode/iotdb/ainode/parser.py | 191 --- iotdb-core/ainode/iotdb/ainode/script.py | 177 ++- iotdb-core/ainode/iotdb/ainode/service.py | 100 +- iotdb-core/ainode/iotdb/ainode/storage.py | 318 ----- iotdb-core/ainode/iotdb/ainode/util.py | 79 -- .../ainode/{encryption.py => util/__init__.py} | 0 .../ainode/{encryption.py => util/decorator.py} | 9 + iotdb-core/ainode/iotdb/ainode/util/lock.py | 84 ++ iotdb-core/ainode/iotdb/ainode/{ => util}/serde.py | 0 .../ainode/iotdb/ainode/util/status.py | 31 +- .../iotdb/confignode/manager/ConfigManager.java | 36 +- .../manager/node/ClusterNodeStartUtils.java | 26 + .../pipe/extractor/ConfigRegionListeningQueue.java | 19 +- .../manager/schema/ClusterSchemaManager.java | 5 +- .../persistence/schema/ClusterSchemaInfo.java | 223 ++-- .../consensus/iot/client/DispatchLogHandler.java | 41 +- .../assembly/resources/sbin/remove-datanode.bat | 12 +- .../dataregion/DataExecutionVisitor.java | 8 +- .../dataregion/DataRegionStateMachine.java | 2 +- .../client/IoTDBDataNodeAsyncClientManager.java | 49 +- .../protocol/legacy/IoTDBLegacyPipeConnector.java | 12 +- .../connector/protocol/opcua/OpcUaConnector.java | 57 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 18 +- .../protocol/opcua/OpcUaServerBuilder.java | 43 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 9 +- .../async/IoTDBDataRegionAsyncConnector.java | 8 + .../tablet/PipeInsertNodeTabletInsertionEvent.java | 6 +- .../tablet/TabletInsertionDataContainer.java | 4 +- .../event/realtime/PipeRealtimeEventFactory.java | 2 +- .../schemaregion/SchemaRegionListeningQueue.java | 4 +- .../processor/aggregate/AggregateProcessor.java | 5 +- .../visitor/PipePlanToStatementVisitor.java | 2 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 3 +- .../thrift/impl/DataNodeRegionManager.java | 22 +- .../queryengine/common/header/DatasetHeader.java | 37 +- .../plan/analyze/ClusterPartitionFetcher.java | 12 +- .../cache/schema/DataNodeDevicePathCache.java | 12 +- .../execution/config/TableConfigTaskVisitor.java | 4 +- .../config/executor/ClusterConfigTaskExecutor.java | 23 +- .../config/metadata/relational/CreateDBTask.java | 4 +- .../config/metadata/relational/ShowTablesTask.java | 2 +- .../TableModelStatementMemorySourceVisitor.java | 5 +- .../plan/planner/TableOperatorGenerator.java | 13 + .../planner/distribution/DistributionPlanner.java | 2 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 58 + .../plan/planner/plan/node/PlanNodeType.java | 8 + .../plan/planner/plan/node/PlanVisitor.java | 12 + .../plan/node/pipe/PipeEnrichedInsertNode.java | 8 +- .../ContinuousSameSearchIndexSeparatorNode.java | 83 +- .../plan/planner/plan/node/write/InsertNode.java | 33 +- .../planner/plan/node/write/InsertRowNode.java | 21 +- .../planner/plan/node/write/InsertRowsNode.java | 2 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 12 +- .../planner/plan/node/write/InsertTabletNode.java | 24 +- .../plan/node/write/RelationalInsertRowNode.java | 20 +- .../plan/node/write/RelationalInsertRowsNode.java | 2 +- .../node/write/RelationalInsertTabletNode.java | 13 +- .../plan/relational/analyzer/Analysis.java | 38 + .../relational/analyzer/ExpressionAnalyzer.java | 27 + .../PredicatePushIntoMetadataChecker.java | 11 +- .../schema/CheckSchemaPredicateVisitor.java | 6 +- .../ConvertSchemaPredicateToFilterVisitor.java | 105 +- .../schema/ExtractPredicateColumnNameVisitor.java | 27 +- .../relational/metadata/FunctionNullability.java | 10 + .../plan/relational/metadata/Metadata.java | 40 +- .../metadata/TableBuiltinAggregationFunction.java | 116 ++ .../relational/metadata/TableMetadataImpl.java | 8 +- .../metadata/fetcher/SchemaPredicateUtil.java | 70 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 26 +- .../plan/relational/planner/NodeAndMappings.java | 38 + .../plan/relational/planner/QueryPlanner.java | 357 +++++ .../plan/relational/planner/SymbolAllocator.java | 10 + .../plan/relational/planner/SymbolsExtractor.java | 27 +- .../relational/planner/TableLogicalPlanner.java | 24 +- .../plan/relational/planner/TableModelPlanner.java | 6 +- .../distribute/TableDistributedPlanGenerator.java | 194 ++- .../distribute/TableDistributedPlanner.java | 18 +- .../TableModelTypeProviderExtractor.java | 12 + .../iterative/rule/PruneAggregationColumns.java | 47 + .../rule/PruneAggregationSourceColumns.java | 54 + .../iterative/rule/PruneTableScanColumns.java | 5 +- .../relational/planner/node/AggregationNode.java | 542 ++++++++ .../planner/node/AggregationTableScanNode.java | 261 ++++ .../plan/relational/planner/node/Patterns.java | 9 +- .../relational/planner/node/TableScanNode.java | 45 +- .../optimizations/LogicalOptimizeFactory.java | 17 + .../PushAggregationIntoTableScan.java | 226 ++++ .../PushLimitOffsetIntoTableScan.java | 7 + .../optimizations/PushPredicateIntoTableScan.java | 76 ++ .../planner/optimizations/SymbolMapper.java | 248 ++++ .../TransformAggregationToStreamable.java | 160 +++ .../optimizations/TransformSortToStreamSort.java | 26 +- .../optimizations/UnaliasSymbolReferences.java | 470 +++++++ .../relational/planner/optimizations/Util.java | 189 +++ .../plan/statement/crud/InsertTabletStatement.java | 2 +- .../apache/iotdb/db/schemaengine/SchemaEngine.java | 28 +- .../metric/SchemaRegionCachedMetric.java | 2 +- .../rescon/MemSchemaRegionStatistics.java | 31 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 10 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 2 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 301 +++-- .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- ...ShutdownHook.java => DataNodeShutdownHook.java} | 14 +- .../db/storageengine/dataregion/DataRegion.java | 10 +- .../execute/task/SettleCompactionTask.java | 8 + ...BatchedFastAlignedSeriesCompactionExecutor.java | 41 +- ...edReadChunkAlignedSeriesCompactionExecutor.java | 42 +- .../utils/AlignedSeriesBatchCompactionUtils.java | 161 ++- .../ReadChunkAlignedSeriesCompactionExecutor.java | 94 +- .../selector/utils/TsFileResourceCandidate.java | 3 +- .../dataregion/snapshot/SnapshotLoader.java | 7 +- .../dataregion/wal/buffer/WALEntry.java | 3 + .../dataregion/wal/io/WALInputStream.java | 1 - .../file/UnsealedTsFileRecoverPerformer.java | 3 + .../event/batch/SubscriptionPipeEventBatches.java | 4 +- .../db/trigger/executor/TriggerFireVisitor.java | 8 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 + .../schemaRegion/AbstractSchemaRegionTest.java | 5 +- .../SchemaRegionSimpleRecoverTest.java | 3 +- ...nuousSameSearchIndexSeparatorNodeSerDeTest.java | 56 + .../node/write/InsertRowsNodeSerdeTest.java | 228 ++-- .../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 2 +- .../node/write/InsertTabletNodeSerdeTest.java | 86 +- .../planner/node/write/WritePlanNodeSplitTest.java | 28 +- .../plan/relational/analyzer/AggregationTest.java | 584 ++++++++ .../plan/relational/analyzer/AnalyzerTest.java | 15 + .../plan/relational/analyzer/JoinTest.java | 68 +- .../relational/analyzer/MockTSBSDataPartition.java | 173 +++ .../plan/relational/analyzer/SubQueryTest.java | 54 +- .../{TestMatadata.java => TSBSMetadata.java} | 303 +++-- .../plan/relational/analyzer/TSBSTest.java | 1342 +++++++++++++++++++ .../plan/relational/analyzer/TestMatadata.java | 8 +- .../plan/relational/planner/ExampleTest.java | 13 +- .../plan/relational/planner/PlanTester.java | 34 +- .../planner/assertions/AggregationFunction.java | 62 + .../assertions/AggregationFunctionMatcher.java | 72 + .../assertions/AggregationFunctionProvider.java | 87 ++ .../planner/assertions/AggregationMatcher.java | 145 ++ .../planner/assertions/AggregationStepMatcher.java | 57 + .../assertions/AggregationTableScanMatcher.java | 186 +++ .../planner/assertions/ColumnReference.java | 10 +- .../planner/assertions/ExpectedValueProvider.java | 30 + .../planner/assertions/PlanMatchPattern.java | 179 ++- .../planner/assertions/PredicateMatcher.java | 46 + .../planner/assertions/TableScanMatcher.java | 10 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 76 ++ ...nkCompactionPerformerWithAlignedSeriesTest.java | 30 + .../settle/SettleCompactionTaskTest.java | 31 + .../compaction/utils/BatchCompactionUtilsTest.java | 12 +- .../wal/node/ConsensusReqReaderTest.java | 2 +- .../iotdb/commons/concurrent/ThreadName.java | 4 +- .../org/apache/iotdb/commons/path/PartialPath.java | 19 + .../plugin/env/PipeTaskRuntimeEnvironment.java | 4 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 7 +- .../iotdb/commons/pipe/progress/CommitterKey.java | 95 ++ .../pipe/progress/PipeEventCommitManager.java | 97 +- .../commons/pipe/progress/PipeEventCommitter.java | 34 +- .../org/apache/iotdb/commons/utils/PathUtils.java | 4 +- .../thrift-datanode/src/main/thrift/client.thrift | 1 + pom.xml | 1 + 251 files changed, 12122 insertions(+), 5384 deletions(-)
