This is an automated email from the ASF dual-hosted git repository. lancelly pushed a commit to branch support_correlated_scalar_subquery in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 113c0051f729c2e20aac731cbfb5a2eda54cbde8 Merge: 27e721b51e1 379993ec38a Author: lancelly <[email protected]> AuthorDate: Mon Feb 17 16:25:04 2025 +0800 merge master .github/workflows/cluster-it-1c1d.yml | 9 +- .github/workflows/cluster-it-1c1d1a.yml | 9 +- .github/workflows/cluster-it-1c3d.yml | 9 +- .github/workflows/compile-check.yml | 11 +- .github/workflows/dependency-check.yml | 1 - .github/workflows/multi-language-client.yml | 1 - .../{pipe-it-2cluster.yml => pipe-it.yml} | 314 +++++- .github/workflows/sonar-codecov.yml | 1 - .github/workflows/table-cluster-it-1c1d.yml | 9 +- .github/workflows/table-cluster-it-1c3d.yml | 9 +- .github/workflows/todos-check.yml | 19 +- .github/workflows/unit-test.yml | 1 - code-coverage/pom.xml | 2 +- distribution/pom.xml | 12 +- docker/src/main/DockerCompose/entrypoint.sh | 5 +- .../main/DockerCompose/replace-conf-from-env.sh | 33 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 74 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 12 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 10 + ...va => MultiClusterIT2DualTableManualBasic.java} | 2 +- ...=> MultiClusterIT2DualTableManualEnhanced.java} | 2 +- ....java => MultiClusterIT2DualTreeAutoBasic.java} | 2 +- ...va => MultiClusterIT2DualTreeAutoEnhanced.java} | 2 +- ...del.java => MultiClusterIT2DualTreeManual.java} | 2 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 + .../iotdb/itbase/runtime/ClusterTestStatement.java | 5 +- .../apache/iotdb/cli/it/StartClientScriptIT.java | 4 +- .../IoTDBRemoveConfigNodeITFramework.java | 206 ++++ .../IoTDBRemoveConfigNodeNormalIT.java | 28 +- .../IoTDBRemoveDataNodeITFramework.java | 2 +- .../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 1 + .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 ++ .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 38 + .../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 183 ++++ .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 2 +- .../apache/iotdb/db/it/query/IoTDBQueryDemoIT.java | 25 + .../org/apache/iotdb/db/it/utils/TestUtils.java | 227 ++++- .../it/{ => dual}/tablemodel/TableModelUtils.java | 2 +- .../AbstractPipeTableModelDualManualIT.java} | 4 +- .../tablemodel/manual/basic}/IoTDBPipeAlterIT.java | 10 +- .../manual/basic}/IoTDBPipeDataSinkIT.java | 111 ++- .../manual/basic}/IoTDBPipeExtractorIT.java | 10 +- .../manual/basic}/IoTDBPipeIsolationIT.java | 10 +- .../manual/basic}/IoTDBPipeLifeCycleIT.java | 157 ++- .../manual/basic}/IoTDBPipeProtocolIT.java | 10 +- .../manual/basic}/IoTDBPipeSwitchStatusIT.java | 10 +- .../manual/basic}/IoTDBPipeSyntaxIT.java | 9 +- .../manual/basic}/IoTDBPipeWithLoadIT.java | 10 +- .../manual/basic}/IoTDBTablePatternFormatIT.java | 10 +- .../manual/enhanced}/IoTDBPipeAutoConflictIT.java | 10 +- .../manual/enhanced}/IoTDBPipeAutoDropIT.java | 10 +- .../manual/enhanced}/IoTDBPipeClusterIT.java | 10 +- .../enhanced}/IoTDBPipeConnectorCompressionIT.java | 10 +- .../enhanced}/IoTDBPipeConnectorParallelIT.java | 10 +- .../manual/enhanced}/IoTDBPipeDoubleLivingIT.java | 13 +- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 197 ++++ .../manual/enhanced/IoTDBPipeMetaIT.java} | 87 +- .../manual/enhanced}/IoTDBPipeNullValueIT.java | 10 +- .../IoTDBPipeTypeConversionISessionIT.java | 18 +- .../enhanced}/IoTDBPipeTypeConversionIT.java | 18 +- .../auto/AbstractPipeDualTreeModelAutoIT.java} | 4 +- .../treemodel/auto/basic}/IoTDBPipeAlterIT.java | 9 +- .../auto/basic}/IoTDBPipeConnectorParallelIT.java | 9 +- .../treemodel/auto/basic}/IoTDBPipeDataSinkIT.java | 80 +- .../auto/basic}/IoTDBPipeExtractorIT.java | 9 +- .../auto/basic}/IoTDBPipeLifeCycleIT.java | 22 +- .../auto/basic}/IoTDBPipeProcessorIT.java | 9 +- .../treemodel/auto/basic}/IoTDBPipeProtocolIT.java | 9 +- .../auto/basic}/IoTDBPipeSwitchStatusIT.java | 9 +- .../treemodel/auto/basic}/IoTDBPipeSyntaxIT.java | 9 +- .../auto/basic}/IoTDBTreePatternFormatIT.java | 9 +- .../auto/enhanced}/IoTDBPipeAutoConflictIT.java | 9 +- .../auto/enhanced}/IoTDBPipeAutoDropIT.java | 9 +- .../auto/enhanced}/IoTDBPipeClusterIT.java | 11 +- .../IoTDBPipeConditionalOperationsIT.java | 9 +- .../enhanced}/IoTDBPipeConnectorCompressionIT.java | 9 +- .../auto/enhanced}/IoTDBPipeIdempotentIT.java | 100 +- .../auto/enhanced}/IoTDBPipeNullValueIT.java | 9 +- .../auto/enhanced}/IoTDBPipeWithLoadIT.java | 9 +- .../auto/enhanced}/PipeNowFunctionIT.java | 9 +- .../manual/AbstractPipeDualTreeModelManualIT.java} | 4 +- .../treemodel}/manual/IoTDBPipeInclusionIT.java | 8 +- .../manual/IoTDBPipeManualConflictIT.java | 8 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 8 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 8 +- .../treemodel}/manual/IoTDBPipeMetaRestartIT.java | 8 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 8 +- .../treemodel}/manual/IoTDBPipePermissionIT.java | 8 +- .../treemodel}/manual/IoTDBPipeReqAutoSliceIT.java | 8 +- .../manual/IoTDBPipeTypeConversionISessionIT.java | 8 +- .../manual/IoTDBPipeTypeConversionIT.java | 8 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 2 +- .../it/db/it/IoTDBAuthenticationTableIT.java | 1001 +++++++++++++++++++ .../relational/it/db/it/IoTDBDeletionTableIT.java | 44 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 17 +- .../it/db/it/IoTDBLoadConfigurationTableIT.java | 85 ++ .../it/db/it/IoTDBSetSystemStatusTableIT.java | 104 ++ .../query/old/query/IoTDBGreatestLeastTableIT.java | 305 ++++++ .../it/query/recent/IoTDBMaintainAuthIT.java | 289 ++++++ .../it/query/recent/IoTDBNullIdQueryIT.java | 33 + .../it/query/recent/IoTDBNullValueIT.java | 75 ++ .../it/query/recent/IoTDBQueryAuthIT.java | 320 ++++++ .../IoTDBCorrelatedScalarSubqueryIT.java | 1 + .../relational/it/schema/IoTDBDatabaseIT.java | 151 ++- .../iotdb/relational/it/schema/IoTDBTableIT.java | 57 ++ .../iotdb/session/it/IoTDBSessionInsertNullIT.java | 128 +++ .../it/local/IoTDBSubscriptionBasicIT.java | 67 ++ iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- iotdb-client/cli/pom.xml | 22 +- .../src/main/java/org/apache/iotdb/cli/Cli.java | 4 +- .../org/apache/iotdb/tool/common/Constants.java | 319 ++++++ .../iotdb/tool/common/ImportTsFileOperation.java | 59 ++ .../org/apache/iotdb/tool/common/OptionsUtil.java | 915 ++++++++++++++++++ .../apache/iotdb/tool/data/AbstractDataTool.java | 1019 +++++--------------- .../apache/iotdb/tool/data/AbstractExportData.java | 70 ++ .../AbstractImportData.java} | 216 +++-- .../apache/iotdb/tool/data/AsyncImportData.java | 160 --- .../org/apache/iotdb/tool/data/ExportData.java | 775 +++------------ .../apache/iotdb/tool/data/ExportDataTable.java | 341 +++++++ .../org/apache/iotdb/tool/data/ExportDataTree.java | 420 ++++++++ .../org/apache/iotdb/tool/data/ImportData.java | 795 ++++++--------- .../apache/iotdb/tool/data/ImportDataScanTool.java | 4 - .../apache/iotdb/tool/data/ImportDataTable.java | 392 ++++++++ .../org/apache/iotdb/tool/data/ImportDataTree.java | 476 +++++++++ .../org/apache/iotdb/tool/tsfile/ImportTsFile.java | 69 +- .../apache/iotdb/tool/tsfile/ImportTsFileBase.java | 9 +- .../iotdb/tool/tsfile/ImportTsFileLocally.java | 8 +- .../iotdb/tool/tsfile/ImportTsFileRemotely.java | 8 + .../iotdb/tool/ImportTsFileOperationTest.java | 49 + iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-py/iotdb/Session.py | 44 +- iotdb-client/client-py/iotdb/SessionPool.py | 8 + iotdb-client/client-py/iotdb/table_session.py | 6 + iotdb-client/client-py/iotdb/table_session_pool.py | 4 + iotdb-client/client-py/pom.xml | 6 +- iotdb-client/client-py/session_ssl_example.py | 85 ++ iotdb-client/isession/pom.xml | 8 +- iotdb-client/jdbc/pom.xml | 8 +- .../main/java/org/apache/iotdb/jdbc/Constant.java | 3 + .../org/apache/iotdb/jdbc/IoTDBConnection.java | 4 + .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 4 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 7 + iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 14 + .../java/org/apache/iotdb/rpc/RpcUtilsTest.java | 10 + iotdb-client/session/pom.xml | 10 +- .../java/org/apache/iotdb/session/Session.java | 25 +- .../apache/iotdb/session/SessionConnection.java | 12 +- .../apache/iotdb/session/TableSessionBuilder.java | 4 +- .../org/apache/iotdb/session/pool/SessionPool.java | 5 +- .../iotdb/session/pool/TableSessionWrapper.java | 15 + .../base/AbstractSubscriptionConsumer.java | 4 +- .../base/AbstractSubscriptionProviders.java | 48 +- iotdb-core/ainode/pom.xml | 10 +- iotdb-core/ainode/pyproject.toml | 14 +- iotdb-core/ainode/resources/conf/ainode-env.sh | 2 +- iotdb-core/antlr/pom.xml | 2 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 6 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 21 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 24 + iotdb-core/confignode/pom.xml | 26 +- .../assembly/resources/sbin/remove-confignode.bat | 135 --- .../assembly/resources/sbin/remove-confignode.sh | 69 -- .../confignode/conf/ConfigNodeRemoveCheck.java | 5 +- .../request/ConfigPhysicalPlanVisitor.java | 221 ++++- .../iotdb/confignode/manager/ConfigManager.java | 3 +- .../confignode/manager/PermissionManager.java | 7 +- .../iotdb/confignode/manager/ProcedureManager.java | 36 +- .../client/IoTDBConfigNodeSyncClientManager.java | 6 +- .../protocol/IoTDBConfigRegionAirGapConnector.java | 3 + .../protocol/IoTDBConfigRegionConnector.java | 6 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 14 +- .../extractor/ConfigRegionListeningFilter.java | 84 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 23 +- ...ConfigPhysicalPlanTablePatternParseVisitor.java | 62 ++ ...peConfigPhysicalPlanTableScopeParseVisitor.java | 86 ++ ...eConfigPhysicalPlanTreePatternParseVisitor.java | 30 +- ...ipeConfigPhysicalPlanTreeScopeParseVisitor.java | 80 ++ .../receiver/protocol/IoTDBConfigNodeReceiver.java | 162 +++- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 207 +++- .../iotdb/confignode/persistence/AuthorInfo.java | 70 +- .../persistence/schema/ClusterSchemaInfo.java | 6 +- ...ipeConfigPhysicalPlanScopeParseVisitorTest.java | 116 +++ ...igPhysicalPlanTablePatternParseVisitorTest.java | 25 + iotdb-core/consensus/pom.xml | 14 +- .../org/apache/iotdb/consensus/common/Peer.java | 13 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 30 +- .../consensus/iot/IoTConsensusServerImpl.java | 226 +---- .../consensus/iot/logdispatcher/LogDispatcher.java | 9 +- .../service/IoTConsensusRPCServiceProcessor.java | 2 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 90 +- .../consensus/pipe/PipeConsensusPeerManager.java | 89 +- .../consensus/pipe/PipeConsensusServerImpl.java | 33 +- .../iotdb/consensus/ratis/RatisConsensus.java | 11 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 63 +- iotdb-core/datanode/pom.xml | 40 +- .../assembly/resources/sbin/remove-datanode.bat | 148 --- .../src/assembly/resources/sbin/remove-datanode.sh | 91 -- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 3 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 + .../PipeRealtimePriorityBlockingQueue.java | 2 + .../subtask/processor/PipeProcessorSubtask.java | 5 +- .../processor/PipeProcessorSubtaskWorker.java | 13 +- .../client/IoTDBDataNodeAsyncClientManager.java | 9 +- .../client/IoTDBDataNodeSyncClientManager.java | 6 +- .../airgap/IoTDBDataNodeAirGapConnector.java | 3 + .../async/IoTDBDataRegionAsyncConnector.java | 3 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 6 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 49 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 18 +- .../schemaregion/SchemaRegionListeningFilter.java | 8 +- .../iotdb/db/pipe/metric/PipeResourceMetrics.java | 6 + .../pipeconsensus/PipeConsensusReceiver.java | 55 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 44 +- .../visitor/PipePlanToStatementVisitor.java | 18 +- .../resource/tsfile/PipeTsFileResourceManager.java | 15 + .../iotdb/db/protocol/client/ConfigNodeClient.java | 3 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 27 +- .../execution/exchange/SharedTsBlockQueue.java | 8 +- .../execution/exchange/sink/SinkChannel.java | 11 +- .../exchange/source/LocalSourceHandle.java | 4 +- .../execution/executor/RegionWriteExecutor.java | 33 +- .../fragment/FragmentInstanceContext.java | 49 +- .../execution/operator/AbstractOperator.java | 2 +- .../operator/process/AbstractSortOperator.java | 2 +- .../execution/operator/process/SortOperator.java | 2 +- .../operator/process/TableStreamSortOperator.java | 2 +- .../execution/operator/source/FileLoaderUtils.java | 4 +- .../InformationSchemaContentSupplierFactory.java | 341 +++++-- .../grouped/HashAggregationOperator.java | 4 +- .../grouped/StreamingAggregationOperator.java | 9 +- .../grouped/StreamingHashAggregationOperator.java | 4 +- .../relational/ColumnTransformerBuilder.java | 15 + .../execution/schedule/DriverScheduler.java | 8 - .../iotdb/db/queryengine/plan/Coordinator.java | 18 +- .../db/queryengine/plan/analyze/AnalyzeUtils.java | 9 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +- .../analyze/cache/partition/PartitionCache.java | 5 + .../schema/dualkeycache/IDualKeyCacheStats.java | 6 + .../cache/schema/dualkeycache/impl/CacheStats.java | 24 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 122 ++- .../plan/analyze/load/LoadTsFileAnalyzer.java | 9 +- .../plan/analyze/schema/SchemaValidator.java | 13 +- .../plan/execution/IQueryExecution.java | 2 + .../queryengine/plan/execution/QueryExecution.java | 5 + .../plan/execution/config/ConfigExecution.java | 9 +- .../execution/config/TableConfigTaskVisitor.java | 91 ++ .../execution/config/TreeConfigTaskVisitor.java | 31 + .../config/executor/ClusterConfigTaskExecutor.java | 114 ++- .../config/executor/IConfigTaskExecutor.java | 23 +- ...teRegionTask.java => RemoveConfigNodeTask.java} | 14 +- .../config/metadata/ShowPipePluginsTask.java | 4 +- .../config/metadata/region/ExtendRegionTask.java | 20 +- .../config/metadata/region/MigrateRegionTask.java | 21 +- .../metadata/region/ReconstructRegionTask.java | 25 +- .../config/metadata/region/RemoveRegionTask.java | 20 +- .../SetSqlDialectTask.java} | 15 +- .../db/queryengine/plan/parser/ASTVisitor.java | 43 +- .../plan/planner/TableOperatorGenerator.java | 9 +- .../plan/node/load/LoadTsFilePieceNode.java | 6 +- .../plan/node/write/RelationalDeleteDataNode.java | 4 +- .../relational/analyzer/StatementAnalyzer.java | 11 +- .../relational/metadata/TableMetadataImpl.java | 20 + .../fetcher/cache/TableDeviceSchemaCache.java | 19 +- .../cache/TableDeviceSchemaCacheMetrics.java | 34 + .../DataNodeLocationSupplierFactory.java | 22 +- .../plan/relational/security/AccessControl.java | 8 + .../relational/security/AccessControlImpl.java | 62 +- .../relational/security/AllowAllAccessControl.java | 5 + .../plan/relational/sql/ast/AstVisitor.java | 32 + .../plan/relational/sql/ast/ExtendRegion.java | 77 ++ .../plan/relational/sql/ast/InsertRows.java | 2 +- .../plan/relational/sql/ast/LoadConfiguration.java | 16 +- .../plan/relational/sql/ast/LoadTsFile.java | 7 + .../plan/relational/sql/ast/MigrateRegion.java | 90 ++ .../plan/relational/sql/ast/ReconstructRegion.java | 76 ++ .../plan/relational/sql/ast/RemoveConfigNode.java | 72 ++ .../plan/relational/sql/ast/RemoveRegion.java | 77 ++ .../plan/relational/sql/ast/SetSqlDialect.java | 79 ++ .../plan/relational/sql/ast/SetSystemStatus.java | 16 +- .../plan/relational/sql/parser/AstBuilder.java | 170 +++- .../plan/scheduler/load/LoadTsFileScheduler.java | 47 +- .../plan/statement/StatementVisitor.java | 21 + .../metadata/RemoveConfigNodeStatement.java | 71 ++ .../plan/statement/sys/SetSqlDialectStatement.java | 76 ++ .../sys/ShowCurrentSqlDialectStatement.java | 45 + .../statement/sys/ShowCurrentUserStatement.java | 45 + .../AbstractGreatestLeastColumnTransformer.java | 109 +++ .../multi/BinaryGreatestColumnTransformer.java | 53 + .../column/multi/BinaryLeastColumnTransformer.java | 53 + .../multi/BooleanGreatestColumnTransformer.java | 52 + .../multi/BooleanLeastColumnTransformer.java | 49 + .../multi/DoubleGreatestColumnTransformer.java | 52 + .../column/multi/DoubleLeastColumnTransformer.java | 52 + .../multi/FloatGreatestColumnTransformer.java | 52 + .../column/multi/FloatLeastColumnTransformer.java | 52 + .../multi/Int32GreatestColumnTransformer.java | 52 + .../column/multi/Int32LeastColumnTransformer.java | 52 + .../multi/Int64GreatestColumnTransformer.java | 52 + .../column/multi/Int64LeastColumnTransformer.java | 52 + .../schemaregion/utils/ResourceByPathUtils.java | 31 +- .../iotdb/db/storageengine/StorageEngine.java | 23 +- .../impl/ReadChunkCompactionPerformer.java | 11 +- .../execute/utils/MultiTsFileDeviceIterator.java | 14 +- ...BatchedFastAlignedSeriesCompactionExecutor.java | 26 +- ...edReadChunkAlignedSeriesCompactionExecutor.java | 40 +- .../utils/AlignedSeriesBatchCompactionUtils.java | 28 +- .../fast/FastAlignedSeriesCompactionExecutor.java | 23 +- .../executor/fast/SeriesCompactionExecutor.java | 10 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 31 +- .../utils/writer/FastCrossCompactionWriter.java | 10 +- .../utils/writer/FastInnerCompactionWriter.java | 10 +- .../compaction/io/CompactionDiskTSMIterator.java} | 29 +- .../compaction/io/CompactionTsFileInput.java | 122 ++- .../compaction/io/CompactionTsFileReader.java | 60 +- .../compaction/io/CompactionTsFileWriter.java | 9 + .../dataregion/flush/MemTableFlushTask.java | 2 +- .../memtable/AlignedReadOnlyMemChunk.java | 10 +- .../memtable/AlignedWritableMemChunk.java | 13 +- .../dataregion/memtable/TsFileProcessor.java | 7 +- .../dataregion/memtable/WritableMemChunkGroup.java | 2 +- .../read/reader/chunk/DiskAlignedChunkLoader.java | 5 +- .../read/reader/chunk/MemAlignedChunkReader.java | 4 +- .../read/reader/chunk/MemAlignedPageReader.java | 6 +- .../metadata/DiskAlignedChunkMetadataLoader.java | 4 +- .../db/storageengine/load/LoadTsFileManager.java | 5 +- .../load/active/ActiveLoadTsFileLoader.java | 3 +- .../load/config/LoadTsFileConfigurator.java | 20 + ...leStatementDataTypeConvertExecutionVisitor.java | 4 + ...eeStatementDataTypeConvertExecutionVisitor.java | 4 + .../broker/SubscriptionPrefetchingQueue.java | 11 + .../response/SubscriptionEventTabletResponse.java | 5 - .../utils/TsFileRewriteOverPrecisedI64Scan.java | 223 +++++ .../utils/TsFileRewriteSmallRangeI64Scan.java | 230 +++++ .../iotdb/db/tools/utils/TsFileSequenceScan.java | 34 +- .../iotdb/db/tools/utils/TsFileStatisticScan.java | 266 +++++ .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 6 +- .../apache/iotdb/db/utils/ModificationUtils.java | 9 +- .../iotdb/db/utils/TimestampPrecisionUtils.java | 4 +- .../iotdb/db/utils/constant/SqlConstant.java | 2 +- .../db/queryengine/execution/exchange/Utils.java | 3 + .../operator/MergeTreeSortOperatorTest.java | 6 + .../execution/operator/OperatorMemoryTest.java | 2 + .../plan/parser/StatementGeneratorTest.java | 3 + .../plan/relational/analyzer/ShowQueriesTest.java | 7 +- ...InconsistentCompressionTypeAndEncodingTest.java | 6 +- .../FastInnerCompactionPerformerTest.java | 6 +- .../repair/RepairUnsortedFileCompactionTest.java | 8 +- .../CompactionTableModelTestFileWriter.java | 2 +- .../compaction/utils/BatchCompactionUtilsTest.java | 39 +- .../db/utils/TimestampPrecisionUtilsTest.java | 10 + iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 24 +- .../conf/iotdb-system.properties.template | 10 +- .../commons/auth/authorizer/BasicAuthorizer.java | 21 + .../iotdb/commons/auth/authorizer/IAuthorizer.java | 4 + .../org/apache/iotdb/commons/auth/entity/Role.java | 8 + .../apache/iotdb/commons/conf/CommonConfig.java | 29 +- .../iotdb/commons/conf/CommonDescriptor.java | 30 +- .../iotdb/commons/consensus/ConsensusGroupId.java | 7 +- .../task/connection/BlockingPendingQueue.java | 14 + .../iotdb/commons/pipe/config/PipeConfig.java | 10 + .../config/constant/PipeConnectorConstant.java | 5 + .../pipe/connector/client/IoTDBClientManager.java | 6 +- .../connector/client/IoTDBSyncClientManager.java | 9 +- .../common/PipeTransferHandshakeConstant.java | 1 + .../pipe/connector/protocol/IoTDBConnector.java | 8 + .../connector/protocol/IoTDBSslSyncConnector.java | 6 +- .../options/PipeInclusionOptions.java | 15 +- .../commons/pipe/receiver/IoTDBFileReceiver.java | 11 +- .../schema/column/ColumnHeaderConstant.java | 30 + .../commons/schema/table/InformationSchema.java | 109 +++ .../iotdb/commons/service/metric/enums/Metric.java | 2 + .../relational/TableBuiltinScalarFunction.java | 2 + iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../db/relational/grammar/sql/RelationalSql.g4 | 38 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 4 + iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- library-udf/pom.xml | 4 +- pom.xml | 19 +- 407 files changed, 14855 insertions(+), 4604 deletions(-) diff --cc integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java index 4e02443d61f,00000000000..7457a0d9f50 mode 100644,000000..100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java @@@ -1,149 -1,0 +1,150 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.correlated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCorrelatedScalarSubqueryIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testCorrelatedScalarSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: Aggregation with correlated filter in scalar subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT max(%s) from table3 t3 where t1.%s = t3.%s)"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: Non-Aggregation with correlated filter in scalar subquery + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT distinct %s from table3 t3 where t1.%s = t3.%s and %s > 30)"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement, + measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testNonComparisonFilterInCorrelatedScalarSubquery() { + // Legality check: Correlated subquery with Non-equality comparison is not support for now. + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 > t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 >= t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 < t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 <= t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select max(s1) from table3 t3 where t1.s1 != t3.s1)", + "For now, FullOuterJoin and LeftJoin only support EquiJoinClauses", + DATABASE_NAME); + } + + @Test + public void testCorrelatedScalarSubqueryLegalityCheck() { + // Legality check: Correlated subqueries can only access columns from the immediately outer + // scope and cannot access columns from the further outer queries. + tableAssertTestFail( + "select s1 from table1 t1 where s1 > (select s1 from table3 t3 where t1.s1 = t3.s1 and s1 > (select s1 from table2 t2 where t2.s1 = t1.s1 limit 1) limit 1)", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + + // Legality check: Correlated subqueries with limit clause and limit count greater than 1 is not + // supported for now + tableAssertTestFail( + "select s1 from table3 t3 where 30 = t3.s1 and s1 > (select max(s1) from table2 t2 where t2.s1 = t3.s1 limit 2)", + "701: Given correlated subquery is not supported", + DATABASE_NAME); + } + + // todo: find out why this fails occasionally ++ @Test + public void testMultipleRowsReturnedByScalarSubquery() { + // Legality check: Scalar subquery should only return one row + tableAssertTestFail( + "select s1 from table1 t1 where s1 >= (select s1 from table3 t3 where t3.s1 = t1.s1)", + "701: Scalar sub-query has returned multiple rows", + DATABASE_NAME); + } +} diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 2c01df007a2,badd487b5fe..3d072ed0c40 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@@ -980,10 -978,20 +981,24 @@@ public class ColumnTransformerBuilde } return new FormatColumnTransformer( STRING, columnTransformers, context.sessionInfo.getZoneId()); + } else if (FAIL_FUNCTION_NAME.equalsIgnoreCase(functionName)) { + checkArgument(children.size() == 1 && children.get(0) instanceof StringLiteral); + return new FailFunctionColumnTransformer( + STRING, ((StringLiteral) children.get(0)).getValue()); + } else if (TableBuiltinScalarFunction.GREATEST + .getFunctionName() + .equalsIgnoreCase(functionName)) { + List<ColumnTransformer> columnTransformers = + children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); + Type returnType = columnTransformers.get(0).getType(); + return AbstractGreatestLeastColumnTransformer.getGreatestColumnTransformer( + returnType, columnTransformers); + } else if (TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { + List<ColumnTransformer> columnTransformers = + children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); + Type returnType = columnTransformers.get(0).getType(); + return AbstractGreatestLeastColumnTransformer.getLeastColumnTransformer( + returnType, columnTransformers); } else { // user defined function if (TableUDFUtils.isScalarFunction(functionName)) { diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 10b980ec053,8364ea489b3..faff5c7ec01 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@@ -542,8 -541,15 +542,17 @@@ public class TableMetadataImpl implemen + " must have at least two arguments, and first argument must be char type."); } return STRING; + } else if (FAIL_FUNCTION_NAME.equalsIgnoreCase(functionName)) { + return UNKNOWN; + } else if (TableBuiltinScalarFunction.GREATEST.getFunctionName().equalsIgnoreCase(functionName) + || TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { + if (argumentTypes.size() < 2 || !areAllTypesSameAndComparable(argumentTypes)) { + throw new SemanticException( + "Scalar function " + + functionName.toLowerCase(Locale.ENGLISH) + + " must have at least two arguments, and all type must be the same."); + } + return argumentTypes.get(0); } // builtin aggregation function
