This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch TableModelIngestion in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a79a80244a813e4ff87c98d92e4033580a214ddd Merge: e6ce64880b9 0bef1d93f86 Author: DESKTOP-L0L5GPJ\jt <[email protected]> AuthorDate: Fri Jun 28 12:13:00 2024 +0800 Merge branch 'ty/TableModelGrammar' into TableModelIngestion # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java # iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java .../load/subscriber/NodeStatisticsChangeEvent.java | 31 -- .../db/queryengine/common/MPPQueryContext.java | 14 - .../db/queryengine/common/header/ColumnHeader.java | 12 +- .../fragment/FragmentInstanceContext.java | 9 + .../operator/process/DeviceViewOperator.java | 9 +- .../schema/source/SchemaSourceFactory.java | 20 + .../schema/source/TableDeviceFetchSource.java | 121 ++++++ .../schema/source/TableDeviceQuerySource.java | 242 ++++++++++++ .../source/AbstractSeriesScanOperator.java | 10 +- .../operator/source/AlignedSeriesScanOperator.java | 15 +- .../source/relational/TableScanOperator.java | 107 +----- .../relational/ColumnTransformerBuilder.java | 55 +-- .../db/queryengine/plan/analyze/Analysis.java | 70 ++-- .../queryengine/plan/analyze/AnalyzeVisitor.java | 159 ++++---- .../queryengine/plan/analyze/TemplatedAnalyze.java | 9 +- .../TableModelStatementMemorySourceVisitor.java | 7 +- .../plan/optimization/AggregationPushDown.java | 14 +- .../plan/planner/LogicalPlanBuilder.java | 28 +- .../plan/planner/LogicalPlanVisitor.java | 21 +- .../plan/planner/OperatorTreeGenerator.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 102 +++-- .../plan/planner/TemplatedLogicalPlan.java | 15 +- .../plan/planner/distribution/SourceRewriter.java | 29 +- .../plan/planner/plan/FragmentInstance.java | 13 + .../plan/planner/plan/node/PlanGraphPrinter.java | 43 ++- .../plan/planner/plan/node/PlanNodeType.java | 11 +- .../plan/planner/plan/node/PlanVisitor.java | 33 +- .../node/metedata/read/TableDeviceFetchNode.java | 173 +++++++++ .../node/metedata/read/TableDeviceQueryNode.java | 219 +++++++++++ .../node/metedata/read/TableDeviceSourceNode.java | 100 +++++ .../planner/plan/node/process/DeviceViewNode.java | 50 +-- .../plan/node/process/SingleDeviceViewNode.java | 27 +- .../plan/relational/analyzer/Analysis.java | 44 +-- .../relational/analyzer/ExpressionAnalyzer.java | 43 +-- .../relational/analyzer/StatementAnalyzer.java | 16 +- .../predicate/ConvertPredicateToFilterVisitor.java | 8 + .../schema/CheckSchemaPredicateVisitor.java | 136 +++++++ .../ConvertSchemaPredicateToFilterVisitor.java | 173 +++++++++ .../plan/relational/metadata/ColumnSchema.java | 12 + .../plan/relational/metadata/Metadata.java | 23 ++ .../relational/metadata/QualifiedObjectName.java | 2 + .../relational/metadata/TableMetadataImpl.java | 105 ++++-- .../fetcher/DeviceInCacheFilterVisitor.java | 66 ++++ .../metadata/fetcher/SchemaPredicateUtil.java | 176 +++++++++ .../metadata/fetcher/TableDeviceSchemaFetcher.java | 380 +++++++++++++++++++ .../fetcher/TableDeviceSchemaValidator.java | 262 +++++++++++++ .../fetcher/TableHeaderSchemaValidator.java | 175 +++++++++ .../plan/relational/planner/LogicalPlanner.java | 133 +++++-- .../plan/relational/planner/OrderingScheme.java | 2 +- .../plan/relational/planner/PlanBuilder.java | 131 ++++--- .../plan/relational/planner/QueryPlanner.java | 65 +--- .../plan/relational/planner/RelationPlanner.java | 13 +- .../relational/planner/RelationalModelPlanner.java | 8 +- .../plan/relational/planner/SymbolAllocator.java | 24 ++ .../plan/relational/planner/TranslationMap.java | 334 ++++++++++++++++ .../planner/distribute/ExchangeNodeGenerator.java | 137 ++++++- .../distribute/TableDistributionPlanner.java | 2 +- .../distribute/TableModelQueryFragmentPlanner.java | 4 - .../TableModelTypeProviderExtractor.java | 37 +- .../relational/planner/ir/ExpressionRewriter.java | 23 ++ .../planner/ir/ExpressionTranslateVisitor.java | 149 -------- .../planner/ir/ExpressionTreeRewriter.java | 64 ++++ .../ir/GlobalTimePredicateExtractVisitor.java | 104 +++-- .../plan/relational/planner/ir/IrVisitor.java | 5 + .../relational/planner/node/TableScanNode.java | 112 ++++-- .../planner/optimizations/FilterScanCombine.java | 227 ----------- .../planner/optimizations/IndexScan.java | 256 ------------- .../planner/optimizations/PruneUnUsedColumns.java | 10 +- .../optimizations/PushPredicateIntoTableScan.java | 337 +++++++++++++++++ .../optimizations/RelationalPlanOptimizer.java | 2 - .../RemoveRedundantIdentityProjections.java | 5 +- .../planner/optimizations/SimplifyExpressions.java | 6 +- .../plan/relational/sql/ast/AstVisitor.java | 8 + .../plan/relational/sql/ast/FetchDevice.java | 111 ++++++ .../plan/relational/sql/ast/LikePredicate.java | 2 +- .../plan/relational/sql/ast/QualifiedName.java | 2 +- .../plan/relational/sql/ast/ShowDevice.java | 159 ++++++++ .../plan/relational/type/InternalTypeManager.java | 16 + .../dag/column/CaseWhenThenColumnTransformer.java | 13 +- .../dag/column/ColumnTransformer.java | 8 +- .../binary/CompareBinaryColumnTransformer.java | 4 +- .../binary/CompareEqualToColumnTransformer.java | 12 +- .../binary/CompareNonEqualColumnTransformer.java | 12 +- .../column/ternary/BetweenColumnTransformer.java | 5 +- .../ternary/CompareTernaryColumnTransformer.java | 9 +- .../dag/column/unary/InColumnTransformer.java | 8 + .../dag/column/unary/RegularColumnTransformer.java | 5 +- .../scalar/CastFunctionColumnTransformer.java | 38 ++ .../schemaengine/schemaregion/ISchemaRegion.java | 2 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 4 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 50 ++- .../schemaRegion/SchemaRegionTestUtil.java | 12 +- .../execution/operator/DeviceViewOperatorTest.java | 8 +- .../execution/operator/MergeSortOperatorTest.java | 12 +- .../execution/operator/OperatorMemoryTest.java | 29 +- .../execution/operator/TopKOperatorTest.java | 12 +- .../db/queryengine/plan/analyze/AnalyzeTest.java | 56 +-- .../plan/optimization/TestPlanBuilder.java | 19 +- .../plan/planner/PipelineBuilderTest.java | 3 +- .../logical/DataQueryLogicalPlannerTest.java | 43 ++- .../plan/planner/node/PlanGraphPrinterTest.java | 3 +- .../node/process/DeviceViewNodeSerdeTest.java | 7 +- .../process/SingleDeviceViewNodeSerdeTest.java | 3 +- .../plan/relational/analyzer/AnalyzerTest.java | 418 +++++++++++++-------- .../plan/relational/analyzer/TestMatadata.java | 124 ++++-- .../iotdb/commons/partition/DataPartition.java | 40 ++ .../apache/iotdb/commons/path/PathPatternTree.java | 4 +- .../commons/schema/filter/SchemaFilterVisitor.java | 2 +- .../org/apache/iotdb/commons/utils/PathUtils.java | 4 +- 110 files changed, 5143 insertions(+), 1711 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 944ad8ccb33,988224da729..a593aee3035 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@@ -90,12 -89,6 +89,8 @@@ public class MPPQueryContext private final LocalExecutionPlanner LOCAL_EXECUTION_PLANNER = LocalExecutionPlanner.getInstance(); - // splits predicate expression in table model into three parts, - // index 0 represents metadataExpressions, index 1 represents expressionsCanPushDownToOperator, - // index 2 represents expressionsCannotPushDownToOperator - private List<List<Expression>> tableModelPredicateExpressions; + private boolean isTableQuery = false; + public MPPQueryContext(QueryId queryId) { this.queryId = queryId; this.endPointBlackList = new LinkedList<>(); diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index b82f7d80242,929775f88e1..8fe4c3f01de --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@@ -109,7 -111,7 +111,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; + import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index e504de0cc9e,cfd760bb7ba..bc003474a5f --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@@ -113,9 -116,7 +116,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateTableDeviceNode; - import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; @SuppressWarnings("java:S6539") // suppress "Monster class" warning diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 6bb868ec0ea,25a8e6edc7f..891b6e86c4b --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@@ -170,24 -166,9 +169,15 @@@ public class Analysis implements IAnaly // indicate is there a value filter private boolean hasValueFilter = false; + private TSStatus failStatus; + - public Expression getGlobalTableModelTimePredicate() { - return this.globalTableModelTimePredicate; - } - - public void setGlobalTableModelTimePredicate(Expression globalTableModelTimePredicate) { - this.globalTableModelTimePredicate = globalTableModelTimePredicate; - } + // if emptyDataSource, there is no need to execute the query in BE + private boolean emptyDataSource = false; + public DataPartition getDataPartition() { + return dataPartition; + } + - public void setDataPartition(DataPartition dataPartition) { - this.dataPartition = dataPartition; - } - public Analysis(@Nullable Statement root, Map<NodeRef<Parameter>, Expression> parameters) { this.root = root; this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); @@@ -626,19 -605,6 +624,11 @@@ return finishQueryAfterAnalyze; } - private boolean hasDataSource() { - return (dataPartition != null && !dataPartition.isEmpty()); - // || (schemaPartition != null && !schemaPartition.isEmpty()) - // || statement instanceof ShowQueriesStatement - // || (statement instanceof QueryStatement - // && ((QueryStatement) statement).isAggregationQuery()); - } - + @Override + public void setDataPartitionInfo(DataPartition dataPartition) { + this.dataPartition = dataPartition; + } + @Override public TsBlock constructResultForMemorySource(MPPQueryContext context) { requireNonNull(getStatement(), "root statement is analysis is null"); @@@ -688,9 -654,21 +678,21 @@@ return dataPartition; } + public void setDataPartition(DataPartition dataPartition) { + this.dataPartition = dataPartition; + } + + public void upsertDataPartition(DataPartition targetDataPartition) { + if (this.dataPartition == null) { + this.dataPartition = targetDataPartition; + } else { + this.dataPartition.upsertDataPartition(targetDataPartition); + } + } + @Override public void setRedirectNodeList(List<TEndPoint> redirectNodeList) { - throw new UnsupportedOperationException(); + this.redirectNodeList = redirectNodeList; } @Override diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java index 8a0ed2f1716,36afc7d1461..5ada86f4f0e --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/ColumnSchema.java @@@ -20,15 -20,17 +20,19 @@@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.tsfile.read.common.type.BinaryType; + import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.BooleanType; + import org.apache.tsfile.read.common.type.DateType; import org.apache.tsfile.read.common.type.DoubleType; import org.apache.tsfile.read.common.type.FloatType; + import org.apache.tsfile.read.common.type.StringType; + import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import org.apache.tsfile.read.common.type.TypeFactory; import org.apache.tsfile.read.common.type.UnknownType; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java index f1dcea2f310,76d76fd000a..d5eb7b9dade --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java @@@ -23,8 -25,9 +25,10 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; + import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; + import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceFetchNode; + import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType; @@@ -39,10 -41,12 +42,13 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.SimplifyExpressions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; + import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; + import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedStatement; + import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java index aeae8d85deb,dcc1e68368e..68eeeabee8a --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java @@@ -16,10 -16,8 +16,9 @@@ package org.apache.iotdb.db.queryengine import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; - import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index 3c6c642ca44,0369e96d111..407efb213e7 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@@ -68,17 -48,9 +69,18 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException; +import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature; +import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; + import org.junit.Ignore; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.read.common.type.Type; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.time.ZoneId; @@@ -95,9 -67,10 +97,11 @@@ import static org.apache.tsfile.read.co import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; + import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.eq; @@@ -505,107 -598,57 +629,151 @@@ public class AnalyzerTest assertEquals(2, ((LogicalExpression) tableScanNode.getPushDownPredicate()).getTerms().size()); } + @Test + public void limitOffsetTest() { + sql = "SELECT tag1, attr1, s1 FROM table1 offset 3 limit 5"; + context = new MPPQueryContext(sql, queryId, sessionInfo, null, null); + actualAnalysis = analyzeSQL(sql, metadata); + logicalQueryPlan = + new LogicalPlanner(context, metadata, sessionInfo, WarningCollector.NOOP) + .plan(actualAnalysis); + rootNode = logicalQueryPlan.getRootNode(); + assertTrue(rootNode.getChildren().get(0) instanceof LimitNode); + LimitNode limitNode = (LimitNode) rootNode.getChildren().get(0); + assertEquals(5, limitNode.getCount()); + OffsetNode offsetNode = (OffsetNode) limitNode.getChild(); + assertEquals(3, offsetNode.getCount()); + + sql = + "SELECT *, s1/2, s2+1 FROM table1 WHERE tag1 in ('A', 'B') and tag2 = 'C' " + + "and s2 iS NUll and S1 = 6 and s3 < 8.0 and tAG1 LIKE '%m' offset 3 limit 5"; + context = new MPPQueryContext(sql, queryId, sessionInfo, null, null); + actualAnalysis = analyzeSQL(sql, metadata); + logicalQueryPlan = + new LogicalPlanner(context, metadata, sessionInfo, WarningCollector.NOOP) + .plan(actualAnalysis); + rootNode = logicalQueryPlan.getRootNode(); + assertTrue(rootNode.getChildren().get(0) instanceof ProjectNode); + assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof LimitNode); + limitNode = (LimitNode) rootNode.getChildren().get(0).getChildren().get(0); + assertEquals(5, limitNode.getCount()); + offsetNode = (OffsetNode) limitNode.getChild(); + assertEquals(3, offsetNode.getCount()); + } + + @Test + public void sortTest() { + // when TableScan locates multi regions, use default MergeSortNode + sql = "SELECT * FROM table1 "; + context = new MPPQueryContext(sql, queryId, sessionInfo, null, null); + actualAnalysis = analyzeSQL(sql, metadata); + logicalQueryPlan = + new LogicalPlanner(context, metadata, sessionInfo, WarningCollector.NOOP) + .plan(actualAnalysis); + rootNode = logicalQueryPlan.getRootNode(); + } + + @Test + public void analyzeTablet() { + + TableSchema tableSchema = StatementTestUtils.genTableSchema(); + Metadata mockMetadata = new TestMatadata() { + @Override + public TableSchema validateTableHeaderSchema(String database, TableSchema schema, + MPPQueryContext context) { + assertEquals(tableSchema, schema); + return tableSchema; + } + + @Override + public void validateDeviceSchema(ITableDeviceSchemaValidation schemaValidation, + MPPQueryContext context) { + assertEquals(sessionInfo.getDatabaseName().get(), schemaValidation.getDatabase()); + assertEquals(StatementTestUtils.tableName(), schemaValidation.getTableName()); + Object[] columns = StatementTestUtils.genColumns(); + for (int i = 0; i < schemaValidation.getDeviceIdList().size(); i++) { + Object[] objects = schemaValidation.getDeviceIdList().get(i); + assertEquals(objects[0].toString(), StatementTestUtils.tableName()); + assertEquals(objects[1].toString(), ((String[]) columns[0])[i]); + } + List<String> attributeColumnNameList = schemaValidation.getAttributeColumnNameList(); + assertEquals(Collections.singletonList("attr1"), attributeColumnNameList); + assertEquals(1, schemaValidation.getAttributeValueList().size()); + assertArrayEquals((Object[]) columns[1], schemaValidation.getAttributeValueList().get(0)); + } + + @Override + public DataPartition getOrCreateDataPartition( + List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) { + int seriesSlotNum = 1000; + String partitionExecutorName = "org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor"; + SeriesPartitionExecutor seriesPartitionExecutor = SeriesPartitionExecutor.getSeriesPartitionExecutor( + partitionExecutorName, seriesSlotNum); + + Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>> + dataPartitionMap = new HashMap<>(); + assertEquals(3, dataPartitionQueryParams.size()); + + for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) { + String databaseName = dataPartitionQueryParam.getDatabaseName(); + assertEquals(sessionInfo.getDatabaseName().get(), databaseName); + + String tableName = dataPartitionQueryParam.getDeviceID().getTableName(); + assertEquals(StatementTestUtils.tableName(), tableName); + + TSeriesPartitionSlot partitionSlot = seriesPartitionExecutor.getSeriesPartitionSlot( + dataPartitionQueryParam.getDeviceID()); + for (TTimePartitionSlot tTimePartitionSlot : dataPartitionQueryParam.getTimePartitionSlotList()) { + dataPartitionMap.computeIfAbsent(databaseName, d -> new HashMap<>()) + .computeIfAbsent(partitionSlot, slot -> new HashMap<>()) + .computeIfAbsent(tTimePartitionSlot, slot -> new ArrayList<>()) + .add(new TRegionReplicaSet(new TConsensusGroupId( + TConsensusGroupType.DataRegion, partitionSlot.slotId), Collections.singletonList( + new TDataNodeLocation(partitionSlot.slotId, null, null, null, null, null)))); + } + } + return new DataPartition(dataPartitionMap, partitionExecutorName, seriesSlotNum); + } + }; + + InsertTabletStatement insertTabletStatement = StatementTestUtils.genInsertTabletStatement(true); + context = new MPPQueryContext("", queryId, sessionInfo, null, null); + actualAnalysis = analyzeStatement(insertTabletStatement.toRelationalStatement(context), + mockMetadata, new SqlParser(), sessionInfo); + logicalQueryPlan = + new LogicalPlanner( + context, mockMetadata, sessionInfo, getFakePartitionFetcher(), WarningCollector.NOOP) + .plan(actualAnalysis); + + RelationalInsertTabletNode insertTabletNode = (RelationalInsertTabletNode) logicalQueryPlan.getRootNode(); + + assertEquals(insertTabletNode.getTableName(), StatementTestUtils.tableName()); + assertEquals(3, insertTabletNode.getRowCount()); + Object[] columns = StatementTestUtils.genColumns(); + for (int i = 0; i < insertTabletNode.getRowCount(); i++) { + assertEquals(Factory.DEFAULT_FACTORY.create(new String[]{StatementTestUtils.tableName(), + ((String[]) columns[0])[i]}), insertTabletNode.getDeviceID(i)); + } + assertEquals(columns, insertTabletNode.getColumns()); + assertArrayEquals(StatementTestUtils.genTimestamps(), insertTabletNode.getTimes()); + + distributionPlanner = new TableDistributionPlanner(actualAnalysis, logicalQueryPlan, context); + distributedQueryPlan = distributionPlanner.plan(); + assertEquals(3, distributedQueryPlan.getInstances().size()); + } + public static Analysis analyzeSQL(String sql, Metadata metadata) { + SqlParser sqlParser = new SqlParser(); + Statement statement = sqlParser.createStatement(sql, ZoneId.systemDefault()); + SessionInfo session = + new SessionInfo( + 0, "test", ZoneId.systemDefault(), "testdb", IClientSession.SqlDialect.TABLE); + return analyzeStatement(statement, metadata, sqlParser, session); + } + + public static Analysis analyzeStatement(Statement statement, Metadata metadata, + SqlParser sqlParser, SessionInfo session) { try { - SqlParser sqlParser = new SqlParser(); - Statement statement = sqlParser.createStatement(sql, ZoneId.systemDefault()); - SessionInfo session = - new SessionInfo( - 0, "test", ZoneId.systemDefault(), "testdb", IClientSession.SqlDialect.TABLE); StatementAnalyzerFactory statementAnalyzerFactory = new StatementAnalyzerFactory(metadata, sqlParser, nopAccessControl); @@@ -625,83 -668,5 +793,7 @@@ return null; } - private static final DataPartition DATA_PARTITION = MockTablePartition.constructDataPartition(); - private static final SchemaPartition SCHEMA_PARTITION = - MockTablePartition.constructSchemaPartition(); - - private static IPartitionFetcher getFakePartitionFetcher() { - - return new IPartitionFetcher() { - - @Override - public SchemaPartition getSchemaPartition(PathPatternTree patternTree) { - return SCHEMA_PARTITION; - } - - @Override - public SchemaPartition getOrCreateSchemaPartition( - PathPatternTree patternTree, String userName) { - return SCHEMA_PARTITION; - } - - @Override - public DataPartition getDataPartition( - Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) { - return DATA_PARTITION; - } - - @Override - public DataPartition getDataPartitionWithUnclosedTimeRange( - Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) { - return DATA_PARTITION; - } - - @Override - public DataPartition getOrCreateDataPartition( - Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) { - return DATA_PARTITION; - } - - @Override - public DataPartition getOrCreateDataPartition( - List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) { - return DATA_PARTITION; - } - - @Override - public SchemaNodeManagementPartition getSchemaNodeManagementPartitionWithLevel( - PathPatternTree patternTree, PathPatternTree scope, Integer level) { - return null; - } - - @Override - public boolean updateRegionCache(TRegionRouteReq req) { - return false; - } - - @Override - public void invalidAllCache() { - } - - @Override - public SchemaPartition getOrCreateSchemaPartition( - String database, List<IDeviceID> deviceIDList, String userName) { - return null; - } - - @Override - public SchemaPartition getSchemaPartition(String database, List<IDeviceID> deviceIDList) { - return null; - } - - @Override - public SchemaPartition getSchemaPartition(String database) { - return null; - } - }; - } - - private static class NopAccessControl implements AccessControl {} + private static class NopAccessControl implements AccessControl { + + } }
