This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5b8c42268bde6b43110f1ecd7c3cb38ef11cedc Merge: c00011d0285 00b309bb55c Author: JackieTien97 <[email protected]> AuthorDate: Fri Mar 1 14:29:20 2024 +0800 fix conflicts .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 2 +- .../db/it/aligned/IoTDBPredicatePushDown2IT.java | 55 ++ .../db/it/aligned/IoTDBPredicatePushDown3IT.java | 55 ++ .../db/it/aligned/IoTDBPredicatePushDown4IT.java | 55 ++ .../db/it/aligned/IoTDBPredicatePushDownIT.java | 601 +++++++++++++++++++++ .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 +- .../confignode/client/DataNodeRequestType.java | 3 +- .../client/async/AsyncDataNodeClientPool.java | 9 +- .../client/async/handlers/AsyncClientHandler.java | 3 +- .../statemachine/ConfigRegionStateMachine.java | 6 +- .../iotdb/confignode/manager/ClusterManager.java | 10 - .../iotdb/confignode/manager/ConfigManager.java | 13 +- .../apache/iotdb/confignode/manager/IManager.java | 5 +- .../manager/consensus/ConsensusManager.java | 58 +- .../iotdb/confignode/manager/cq/CQManager.java | 11 - .../iotdb/confignode/manager/node/NodeManager.java | 18 +- .../pipe/coordinator/runtime/PipeMetaSyncer.java | 10 - .../thrift/ConfigNodeRPCServiceProcessor.java | 9 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 5 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +- .../protocol/writeback/WriteBackConnector.java | 2 +- .../legacy/IoTDBLegacyPipeReceiverAgent.java | 2 +- .../receiver/legacy/loader/DeletionLoader.java | 2 +- .../pipe/receiver/legacy/loader/TsFileLoader.java | 2 +- .../receiver/thrift/IoTDBThriftReceiverV1.java | 2 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 47 +- .../protocol/rest/filter/AuthorizationFilter.java | 2 +- .../iotdb/db/protocol/session/SessionManager.java | 11 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 16 +- .../db/queryengine/common/MPPQueryContext.java | 5 + .../iotdb/db/queryengine/common/SessionInfo.java | 16 +- .../fragment/FragmentInstanceContext.java | 2 +- .../operator/process/ProjectOperator.java | 101 ++++ .../db/queryengine/plan/analyze/Analysis.java | 15 + .../queryengine/plan/analyze/AnalyzeVisitor.java | 87 +-- .../plan/analyze/ExpressionTypeAnalyzer.java | 10 + .../queryengine/plan/analyze/ExpressionUtils.java | 11 + .../queryengine/plan/analyze/PredicateUtils.java | 155 +++--- .../queryengine/plan/analyze/TemplatedAnalyze.java | 20 +- .../db/queryengine/plan/analyze/TemplatedInfo.java | 168 +++--- .../queryengine/plan/execution/QueryExecution.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 18 +- .../config/executor/ClusterConfigTaskExecutor.java | 34 +- .../config/executor/IConfigTaskExecutor.java | 4 +- ...epairDataTask.java => StartRepairDataTask.java} | 12 +- ...RepairDataTask.java => StopRepairDataTask.java} | 12 +- .../memory/StatementMemorySourceVisitor.java | 8 +- .../plan/expression/ExpressionFactory.java | 35 ++ .../plan/expression/leaf/ConstantOperand.java | 15 +- .../plan/expression/ternary/BetweenExpression.java | 6 +- .../BindTypeForTimeSeriesOperandVisitor.java | 13 - .../visitor/ExpressionNormalizeVisitor.java | 13 +- .../visitor/GetMeasurementExpressionVisitor.java | 18 - .../visitor/LowercaseNormalizeVisitor.java | 13 +- .../expression/visitor/ReconstructVisitor.java | 33 ++ .../ReplaceRawPathWithGroupedPathVisitor.java | 23 - .../visitor/ReplaceSubTreeWithViewVisitor.java | 16 - .../visitor/logical/LogicalAndVisitor.java | 70 +++ .../visitor/logical/LogicalOrVisitor.java | 70 +++ .../PredicateCanPushDownToSourceChecker.java} | 21 +- .../visitor/logical/TimeFilterExistChecker.java} | 17 +- .../predicate/ConvertPredicateToFilterVisitor.java | 406 ++++++++++++++ .../ConvertPredicateToTimeFilterVisitor.java | 30 +- .../predicate/PredicatePushIntoScanChecker.java | 171 ++++++ .../visitor/predicate/PredicateSimplifier.java | 279 ++++++++++ .../plan/optimization/PredicatePushDown.java | 468 ++++++++++++++++ .../db/queryengine/plan/parser/ASTVisitor.java | 35 +- .../plan/parser/StatementGenerator.java | 13 +- .../plan/planner/LocalExecutionPlanContext.java | 14 + .../plan/planner/LogicalPlanBuilder.java | 61 +-- .../plan/planner/LogicalPlanVisitor.java | 15 +- .../queryengine/plan/planner/LogicalPlanner.java | 7 +- .../plan/planner/OperatorTreeGenerator.java | 386 ++++++------- .../plan/planner/SubPlanTypeExtractor.java | 11 + .../plan/planner/TemplatedLogicalPlan.java | 7 +- .../plan/planner/TemplatedLogicalPlanBuilder.java | 4 +- .../planner/distribution/ExchangeNodeAdder.java | 6 + .../plan/planner/distribution/SourceRewriter.java | 27 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 49 +- .../plan/planner/plan/node/PlanNodeType.java | 2 + .../plan/planner/plan/node/PlanVisitor.java | 9 +- .../plan/planner/plan/node/process/FilterNode.java | 16 +- .../planner/plan/node/process/ProjectNode.java | 25 +- .../planner/plan/node/process/TransformNode.java | 25 +- .../plan/node/process/join/InnerTimeJoinNode.java | 24 +- .../node/process/join/LeftOuterTimeJoinNode.java | 4 +- .../plan/node/source/AlignedSeriesScanNode.java | 128 +---- .../planner/plan/node/source/SeriesScanNode.java | 119 +--- ...riesScanNode.java => SeriesScanSourceNode.java} | 163 ++---- .../queryengine/plan/statement/StatementType.java | 3 +- .../plan/statement/StatementVisitor.java | 11 +- .../plan/statement/component/SelectComponent.java | 11 +- .../metadata/CreateContinuousQueryStatement.java | 4 - .../plan/statement/sys/ShowQueriesStatement.java | 11 - ...tatement.java => StartRepairDataStatement.java} | 6 +- ...Statement.java => StopRepairDataStatement.java} | 6 +- .../java/org/apache/iotdb/db/service/DataNode.java | 4 + .../metrics/IoTDBInternalLocalReporter.java | 2 +- .../iotdb/db/storageengine/StorageEngine.java | 48 +- .../db/storageengine/dataregion/DataRegion.java | 98 ++-- .../CompactionValidationFailedException.java | 15 + .../execute/task/AbstractCompactionTask.java | 62 ++- .../task/InsertionCrossSpaceCompactionTask.java | 4 +- .../compaction/repair/RepairDataFileScanUtil.java | 271 ++++++++++ .../dataregion/compaction/repair/RepairLogger.java | 146 +++-- .../RepairProgress.java} | 23 +- .../repair/RepairTaskRecoverLogParser.java | 30 +- .../compaction/repair/RepairTaskStatus.java} | 13 +- .../compaction/repair/RepairTimePartition.java | 4 + .../repair/RepairTimePartitionScanTask.java | 202 +++++++ .../repair/UnsortedFileRepairTaskScheduler.java | 316 ++++------- .../schedule/CompactionScheduleTaskManager.java | 317 +++++++++++ .../schedule/CompactionScheduleTaskWorker.java | 69 +++ .../compaction/schedule/CompactionScheduler.java | 101 ++-- .../compaction/schedule/CompactionTaskManager.java | 6 +- .../compaction/schedule/CompactionTaskQueue.java | 101 ++++ .../compaction/schedule/CompactionWorker.java | 60 +- .../dataregion/utils/TsFileResourceUtils.java | 2 +- .../db/storageengine/rescon/memory/SystemInfo.java | 88 ++- .../datastructure/FixedPriorityBlockingQueue.java | 12 +- .../predicate/PredicatePushIntoScanTest.java | 140 +++++ .../predicate/PredicateRemoveNotTest.java | 2 +- .../expression/predicate/PredicateUtilsTest.java | 77 +++ .../db/queryengine/plan/analyze/AnalyzeTest.java | 10 +- .../optimization/ColumnInjectionPushDownTest.java | 51 +- .../plan/optimization/LimitOffsetPushDownTest.java | 38 +- .../plan/optimization/OptimizationTestUtil.java | 71 ++- .../plan/optimization/PredicatePushDownTest.java | 449 +++++++++++++++ .../plan/optimization/TestPlanBuilder.java | 94 +++- .../plan/parser/StatementGeneratorTest.java | 6 +- .../plan/planner/FragmentInstanceSerdeTest.java | 3 +- .../queryengine/plan/planner/QueryPlannerTest.java | 106 ---- .../distribution/AggregationAlignByDeviceTest.java | 34 +- .../AlignByDeviceOrderByLimitOffsetTest.java | 242 ++++----- .../planner/distribution/AlignedByDeviceTest.java | 375 +++++++++++-- .../plan/planner/distribution/Util.java | 2 +- .../plan/planner/distribution/Util2.java | 2 +- .../DataQueryLogicalPlannerTest.java} | 217 +++++--- .../logical/LogicalPlannerTestUtil.java} | 53 +- .../SchemaQueryLogicalPlannerTest.java} | 116 +--- .../planner/node/process/FilterNodeSerdeTest.java | 2 - .../node/process/TimeJoinNodeSerdeTest.java | 19 +- .../plan/relational/analyzer/AnalyzerTest.java | 2 +- .../builder/EvaluationDAGBuilderTest.java | 3 +- .../compaction/CompactionOverlapCheckTest.java | 14 + .../compaction/CompactionWorkerTest.java | 103 +++- .../FastCrossCompactionPerformerTest.java | 22 +- .../cross/CrossSpaceCompactionSelectorTest.java | 38 +- .../cross/InsertionCrossSpaceCompactionTest.java | 13 +- .../compaction/inner/InnerCompactionLogTest.java | 2 +- .../inner/InnerCompactionMoreDataTest.java | 3 +- .../SizeTieredCompactionSelectorTest.java | 38 -- .../compaction/repair/AbstractRepairDataTest.java | 66 +++ .../repair/RepairDataFileScanUtilTest.java | 138 +++++ .../compaction/repair/RepairLoggerTest.java | 73 ++- .../repair/RepairUnsortedFileCompactionTest.java | 84 ++- .../compaction/utils/CompactionTaskQueueTest.java | 201 +++++++ .../resources/conf/iotdb-common.properties | 4 + .../iotdb/commons/concurrent/ThreadName.java | 1 + .../threadpool/WrappedThreadPoolExecutor.java | 2 +- .../org/apache/iotdb/commons/path/AlignedPath.java | 8 + .../apache/iotdb/commons/service/ServiceType.java | 2 + .../tsfile/read/filter/basic/ValueFilter.java | 4 + .../tsfile/read/filter/factory/ValueFilterApi.java | 8 + .../src/main/thrift/confignode.thrift | 5 +- .../src/main/thrift/datanode.thrift | 4 +- 168 files changed, 7234 insertions(+), 2388 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/SessionInfo.java index 4ec877d1886,bc3c471e166..fa401c33645 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/SessionInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/SessionInfo.java @@@ -28,38 -25,27 +28,40 @@@ import javax.annotation.Nullable import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; + import java.time.ZoneId; + import java.util.Objects; +import java.util.Optional; public class SessionInfo { private final long sessionId; private final String userName; - private final String zoneId; + private final ZoneId zoneId; + @Nullable private final String databaseName; + private ClientVersion version = ClientVersion.V_1_0; - public SessionInfo(long sessionId, String userName, String zoneId) { + public SessionInfo(long sessionId, String userName, ZoneId zoneId) { this.sessionId = sessionId; this.userName = userName; this.zoneId = zoneId; + this.databaseName = null; } - public SessionInfo(long sessionId, String userName, String zoneId, ClientVersion version) { + public SessionInfo(long sessionId, String userName, ZoneId zoneId, ClientVersion version) { this.sessionId = sessionId; this.userName = userName; this.zoneId = zoneId; this.version = version; + this.databaseName = null; + } + + public SessionInfo( - long sessionId, String userName, String zoneId, @Nullable String databaseName) { ++ long sessionId, String userName, ZoneId zoneId, @Nullable String databaseName) { + this.sessionId = sessionId; + this.userName = userName; + this.zoneId = zoneId; + this.databaseName = databaseName; } public long getSessionId() { diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index d12e478efa9,00000000000..cd9b62518d6 mode 100644,000000..100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@@ -1,145 -1,0 +1,145 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.relational.function.BoundSignature; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionId; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionKind; +import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnHandle; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableHandle; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.relational.sql.tree.Statement; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector.NOOP; +import static org.apache.iotdb.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.iotdb.tsfile.read.common.type.IntType.INT32; +import static org.apache.iotdb.tsfile.read.common.type.LongType.INT64; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.eq; + +public class AnalyzerTest { + + private final SqlParser sqlParser = new SqlParser(); + + private final NopAccessControl nopAccessControl = new NopAccessControl(); + + @Test + public void testRawDataQuery() throws OperatorNotFoundException { + String sql = "SELECT s1, (s1 + 1) as t from table1 where time > 100 and s2 > 10"; + Metadata metadata = Mockito.mock(Metadata.class); + Mockito.when(metadata.tableExists(Mockito.any())).thenReturn(true); + + TableHandle tableHandle = Mockito.mock(TableHandle.class); + Mockito.when( + metadata.getTableHandle(Mockito.any(), eq(new QualifiedObjectName("testdb", "table1")))) + .thenReturn(Optional.of(tableHandle)); + + Map<String, ColumnHandle> map = new HashMap<>(); + TableSchema tableSchema = Mockito.mock(TableSchema.class); + Mockito.when(tableSchema.getTableName()).thenReturn("table1"); + ColumnSchema column1 = + ColumnSchema.builder().setName("time").setType(INT64).setHidden(false).build(); + ColumnHandle column1Handle = Mockito.mock(ColumnHandle.class); + map.put("time", column1Handle); + ColumnSchema column2 = + ColumnSchema.builder().setName("s1").setType(INT32).setHidden(false).build(); + ColumnHandle column2Handle = Mockito.mock(ColumnHandle.class); + map.put("s1", column2Handle); + ColumnSchema column3 = + ColumnSchema.builder().setName("s2").setType(INT64).setHidden(false).build(); + ColumnHandle column3Handle = Mockito.mock(ColumnHandle.class); + map.put("s2", column3Handle); + List<ColumnSchema> columnSchemaList = Arrays.asList(column1, column2, column3); + Mockito.when(tableSchema.getColumns()).thenReturn(columnSchemaList); + + Mockito.when(metadata.getTableSchema(Mockito.any(), eq(tableHandle))).thenReturn(tableSchema); + Mockito.when(metadata.getColumnHandles(Mockito.any(), eq(tableHandle))).thenReturn(map); + + ResolvedFunction lLessThanI = + new ResolvedFunction( + new BoundSignature("l<i", BOOLEAN, Arrays.asList(INT64, INT32)), + new FunctionId("l<i"), + FunctionKind.SCALAR, + true); + + ResolvedFunction iAddi = + new ResolvedFunction( + new BoundSignature("l+i", INT64, Arrays.asList(INT32, INT32)), + new FunctionId("l+i"), + FunctionKind.SCALAR, + true); + + Mockito.when( + metadata.resolveOperator(eq(OperatorType.LESS_THAN), eq(Arrays.asList(INT64, INT32)))) + .thenReturn(lLessThanI); + Mockito.when(metadata.resolveOperator(eq(OperatorType.ADD), eq(Arrays.asList(INT32, INT32)))) + .thenReturn(iAddi); + + Analysis actualAnalysis = analyzeSQL(sql, metadata); + assertNotNull(actualAnalysis); + System.out.println(actualAnalysis.getTypes()); + } + + private Analysis analyzeSQL(String sql, Metadata metadata) { + try { + Statement statement = sqlParser.createStatement(sql); - SessionInfo session = new SessionInfo(0, "test", ZoneId.systemDefault().getId(), "testdb"); ++ SessionInfo session = new SessionInfo(0, "test", ZoneId.systemDefault(), "testdb"); + StatementAnalyzerFactory statementAnalyzerFactory = + new StatementAnalyzerFactory(metadata, sqlParser, nopAccessControl); + + Analyzer analyzer = + new Analyzer( + session, + statementAnalyzerFactory, + Collections.emptyList(), + Collections.emptyMap(), + NOOP); + return analyzer.analyze(statement); + } catch (Exception e) { + e.printStackTrace(); + fail(sql + ", " + e.getMessage()); + } + fail(); + return null; + } + + private static class NopAccessControl implements AccessControl {} +}
