This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch rc-133-having in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9069eb40e0ff2e56f77fa3178e349b6b1a6b6676 Author: Beyyes <[email protected]> AuthorDate: Tue Oct 29 10:04:22 2024 +0800 Fix the case that there is nonexist measurement in having clause --- .../db/it/alignbydevice/IoTDBAlignByDeviceIT.java | 45 +++++++++++++ .../queryengine/plan/analyze/AnalyzeVisitor.java | 9 ++- .../plan/analyze/ExpressionAnalyzer.java | 13 ++++ .../visitor/ColumnTransformerVisitor.java | 15 ++++- .../visitor/ExistUnknownTypeInExpression.java | 60 +++++++++++++++++ .../ConcatDeviceAndBindSchemaForHavingVisitor.java | 77 ++++++++++++++++++++++ 6 files changed, 216 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java index cc5fdeb5b27..1bb3e76061d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceIT.java @@ -100,6 +100,7 @@ public class IoTDBAlignByDeviceIT { "insert into root.vehicle.d1(timestamp,s0) values(1,999)", "insert into root.vehicle.d1(timestamp,s0) values(1000,888)", "insert into root.other.d1(timestamp,s0) values(2, 3.14)", + "insert into root.other.d2(timestamp,s6) values(6, 6.66)", }; @BeforeClass @@ -1184,4 +1185,48 @@ public class IoTDBAlignByDeviceIT { + e.getMessage()); } } + + @Test + public void nonExistMeasurementInHavingTest() { + String[] retArray = + new String[] { + "1,root.other.d1,3.14,null,", "5,root.other.d2,null,6.66,", + }; + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + try (ResultSet resultSet = + statement.executeQuery( + "select last_value(s0),last_value(s6) from root.other.** group by ([1,10),2ms) having last_value(s0) is not null or last_value(s6) is not null align by device")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + List<Integer> actualIndexToExpectedIndexList = + checkHeader( + resultSetMetaData, + "Time,Device,last_value(s0),last_value(s6)", + new int[] { + Types.TIMESTAMP, Types.VARCHAR, Types.FLOAT, Types.DOUBLE, + }); + + int cnt = 0; + while (resultSet.next()) { + String[] expectedStrings = retArray[cnt].split(","); + StringBuilder expectedBuilder = new StringBuilder(); + StringBuilder actualBuilder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + actualBuilder.append(resultSet.getString(i)).append(","); + expectedBuilder + .append(expectedStrings[actualIndexToExpectedIndexList.get(i - 1)]) + .append(","); + } + Assert.assertEquals(expectedBuilder.toString(), actualBuilder.toString()); + cnt++; + } + Assert.assertEquals(retArray.length, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 012f48c271e..5178a848805 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -82,6 +82,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.binary.CompareBinaryExpre import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; +import org.apache.iotdb.db.queryengine.plan.expression.visitor.ExistUnknownTypeInExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.MeasurementGroup; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.FillDescriptor; @@ -200,6 +201,7 @@ import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PART import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.bindSchemaForExpression; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression; +import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForHaving; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.getMeasurementExpression; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.normalizeExpression; import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.searchAggregationExpressions; @@ -1056,8 +1058,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> for (PartialPath device : deviceSet) { List<Expression> expressionsInHaving = - concatDeviceAndBindSchemaForExpression( - havingExpression, device, schemaTree, queryContext); + concatDeviceAndBindSchemaForHaving(havingExpression, device, schemaTree, queryContext); conJunctions.addAll( expressionsInHaving.stream() @@ -1070,6 +1071,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> for (Expression aggregationExpression : searchAggregationExpressions(expression)) { Expression normalizedAggregationExpression = normalizeExpression(aggregationExpression); + if (!new ExistUnknownTypeInExpression().process(aggregationExpression, null).isEmpty()) { + continue; + } + analyzeExpressionType(analysis, aggregationExpression); analyzeExpressionType(analysis, normalizedAggregationExpression); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java index fa29e4a20f8..96949794611 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java @@ -52,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.visitor.ReplaceSubTreeWit import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.BindSchemaForExpressionVisitor; import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.BindSchemaForPredicateVisitor; import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.ConcatDeviceAndBindSchemaForExpressionVisitor; +import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.ConcatDeviceAndBindSchemaForHavingVisitor; import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.ConcatDeviceAndBindSchemaForPredicateVisitor; import org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.ConcatExpressionWithSuffixPathsVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn; @@ -489,6 +490,18 @@ public class ExpressionAnalyzer { devicePath, schemaTree, isWhere, queryContext)); } + public static List<Expression> concatDeviceAndBindSchemaForHaving( + final Expression predicate, + final PartialPath devicePath, + final ISchemaTree schemaTree, + final MPPQueryContext queryContext) { + return new ConcatDeviceAndBindSchemaForHavingVisitor() + .process( + predicate, + new ConcatDeviceAndBindSchemaForHavingVisitor.Context( + devicePath, schemaTree, queryContext)); + } + /** * Search for subexpressions that can be queried natively, including all time series. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ColumnTransformerVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ColumnTransformerVisitor.java index 09da66f3d9c..6c871be905f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ColumnTransformerVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ColumnTransformerVisitor.java @@ -81,6 +81,7 @@ import java.util.Map; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionType.BETWEEN; +import static org.apache.tsfile.enums.TSDataType.UNKNOWN; /** Responsible for constructing {@link ColumnTransformer} through Expression. */ public class ColumnTransformerVisitor @@ -581,7 +582,19 @@ public class ColumnTransformerVisitor if (typeProvider != null) { return typeProvider.getType(expression.getOutputSymbol()); } - return expressionTypes.get(NodeRef.of(expression)); + if (expressionTypes.get(NodeRef.of(expression)) != UNKNOWN) { + return expressionTypes.get(NodeRef.of(expression)); + } else { + for (Map.Entry<NodeRef<Expression>, TSDataType> entry : expressionTypes.entrySet()) { + if (entry.getKey().getNode().equals(expression) && entry.getValue() != UNKNOWN) { + return entry.getValue(); + } + } + throw new IllegalStateException( + String.format( + "Unknown expression type: %s, perhaps it has non existent measurement.", + expression.getOutputSymbol())); + } } public TypeProvider getTypeProvider() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java new file mode 100644 index 00000000000..d786e48dd00 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/ExistUnknownTypeInExpression.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.expression.visitor; + +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.LeafOperand; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; +import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.Collections; +import java.util.List; + +public class ExistUnknownTypeInExpression extends CollectVisitor { + + @Override + public List<Expression> visitLeafOperand(LeafOperand leafOperand, Void context) { + return Collections.emptyList(); + } + + @Override + public List<Expression> visitFunctionExpression( + FunctionExpression functionExpression, Void context) { + List<List<Expression>> ret = getResultsFromChild(functionExpression, null); + for (List<Expression> row : ret) { + if (!row.isEmpty()) { + return row; + } + } + return Collections.emptyList(); + } + + @Override + public List<Expression> visitTimeSeriesOperand( + TimeSeriesOperand timeSeriesOperand, Void context) { + if (timeSeriesOperand.getPath().getSeriesType() == TSDataType.UNKNOWN) { + return Collections.singletonList(timeSeriesOperand); + } + + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/ConcatDeviceAndBindSchemaForHavingVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/ConcatDeviceAndBindSchemaForHavingVisitor.java new file mode 100644 index 00000000000..79fdd754f27 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/cartesian/ConcatDeviceAndBindSchemaForHavingVisitor.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian; + +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionUtils; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.iotdb.db.queryengine.plan.expression.visitor.cartesian.BindSchemaForExpressionVisitor.transformViewPath; + +public class ConcatDeviceAndBindSchemaForHavingVisitor + extends ConcatDeviceAndBindSchemaForExpressionVisitor { + @Override + public List<Expression> visitTimeSeriesOperand( + TimeSeriesOperand timeSeriesOperand, Context context) { + PartialPath measurement = timeSeriesOperand.getPath(); + PartialPath concatPath = context.getDevicePath().concatPath(measurement); + + List<MeasurementPath> actualPaths = + context.getSchemaTree().searchMeasurementPaths(concatPath).left; + if (actualPaths.isEmpty()) { + return Collections.singletonList( + new TimeSeriesOperand(new MeasurementPath(concatPath, TSDataType.UNKNOWN))); + } + + List<MeasurementPath> nonViewActualPaths = new ArrayList<>(); + List<MeasurementPath> viewPaths = new ArrayList<>(); + for (MeasurementPath measurementPath : actualPaths) { + if (measurementPath.getMeasurementSchema().isLogicalView()) { + viewPaths.add(measurementPath); + } else { + nonViewActualPaths.add(measurementPath); + } + } + List<Expression> reconstructTimeSeriesOperands = + ExpressionUtils.reconstructTimeSeriesOperandsWithMemoryCheck( + timeSeriesOperand, nonViewActualPaths, context.getQueryContext()); + // handle logical views + for (MeasurementPath measurementPath : viewPaths) { + Expression replacedExpression = transformViewPath(measurementPath, context.getSchemaTree()); + if (!(replacedExpression instanceof TimeSeriesOperand)) { + throw new SemanticException( + "Only writable view timeseries are supported in ALIGN BY DEVICE queries."); + } + + replacedExpression.setViewPath(measurementPath); + reconstructTimeSeriesOperands.add(replacedExpression); + } + return reconstructTimeSeriesOperands; + } +}
