This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7c7e21bbd7 [IOTDB-3600] [IOTDB-3605] [IOTDB-3625] Fix semantic check
issue about UDF&expression in MPP (#6471)
7c7e21bbd7 is described below
commit 7c7e21bbd74639146f197774ae7f3c6e6ddc8b42
Author: liuminghui233 <[email protected]>
AuthorDate: Mon Jul 4 15:04:27 2022 +0800
[IOTDB-3600] [IOTDB-3605] [IOTDB-3625] Fix semantic check issue about
UDF&expression in MPP (#6471)
---
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 26 +---
.../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 145 +++++++++++++++++----
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 69 ++++++----
.../plan/expression/multi/FunctionExpression.java | 2 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 3 +-
5 files changed, 168 insertions(+), 77 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java
index 4f1f57386c..9110a2c7e4 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBNestedQueryIT.java
@@ -128,9 +128,7 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
@Test
- @Ignore
public void testNestedArithmeticExpressions() {
String sqlStr =
"select d1.s1, d2.s2, d1.s1 + d1.s2 - (d2.s1 + d2.s2), d1.s2 * (d2.s1
/ d1.s1), d1.s2 + d1.s2 * d2.s1 - d2.s1, d1.s1 - (d1.s1 - (-d1.s1)), (-d2.s1) *
(-d2.s2) / (-d1.s2) from root.vehicle";
@@ -161,8 +159,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testNestedRowByRowUDFExpressions() {
String sqlStr =
@@ -194,8 +190,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testUDFTerminateMethodInNestedExpressions() {
String sqlStr =
@@ -223,8 +217,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testUDFWithMultiInputsInNestedExpressions() {
String sqlStr =
@@ -372,8 +364,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testInvalidNestedBuiltInAggregation() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -392,9 +382,7 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: Exception: Constant is not supported
@Test
- @Ignore
public void testRawDataQueryWithConstants() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
@@ -424,8 +412,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testDuplicatedRawDataQueryWithConstants() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -447,8 +433,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testCommutativeLaws() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -473,9 +457,7 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: Exception: Constant is not supported
@Test
- @Ignore
public void testAssociativeLaws() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
@@ -502,9 +484,7 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: Exception: Constant is not supported
@Test
- @Ignore
public void testDistributiveLaw() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
@@ -529,9 +509,7 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: Exception: Constant is not supported
@Test
- @Ignore
public void testOrderOfArithmeticOperations() {
// Priority from high to low:
// 1. exponentiation and root extraction (not supported yet)
@@ -642,8 +620,6 @@ public class IoTDBNestedQueryIT {
}
}
- // todo: remove ignore after iotdb-3349 is merged
- @Ignore
@Test
public void testRegularLikeInExpressions() {
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -664,7 +640,7 @@ public class IoTDBNestedQueryIT {
}
@Test
- @Ignore
+ @Ignore // not supported in 0.13
public void testTimeExpressions() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java
index 525b641926..e915f455df 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java
@@ -32,9 +32,12 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -56,6 +59,8 @@ public class IoTDBUDTFHybridQueryIT {
statement.execute("SET STORAGE GROUP TO root.vehicle");
statement.execute("CREATE TIMESERIES root.vehicle.d1.s1 with
datatype=INT32,encoding=PLAIN");
statement.execute("CREATE TIMESERIES root.vehicle.d1.s2 with
datatype=INT32,encoding=PLAIN");
+ statement.execute("CREATE TIMESERIES root.vehicle.d2.s1 with
datatype=INT32,encoding=PLAIN");
+ statement.execute("CREATE TIMESERIES root.vehicle.d2.s2 with
datatype=INT32,encoding=PLAIN");
} catch (SQLException throwable) {
fail(throwable.getMessage());
}
@@ -64,10 +69,21 @@ public class IoTDBUDTFHybridQueryIT {
private static void generateData() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- for (int i = 0; i < 10; ++i) {
+ for (int i = 0; i < 5; ++i) {
statement.execute(
(String.format(
"insert into root.vehicle.d1(timestamp,s1,s2)
values(%d,%d,%d)", i, i, i)));
+ statement.execute(
+ (String.format(
+ "insert into root.vehicle.d2(timestamp,s1,s2)
values(%d,%d,%d)", i, i, i)));
+ }
+ for (int i = 2; i < 4; ++i) {
+ statement.execute(
+ (String.format("insert into root.vehicle.d1(timestamp,s3)
values(%d,%d)", i, i)));
+ }
+ for (int i = 7; i < 10; ++i) {
+ statement.execute(
+ (String.format("insert into root.vehicle.d1(timestamp,s3)
values(%d,%d)", i, i)));
}
} catch (SQLException throwable) {
fail(throwable.getMessage());
@@ -89,7 +105,6 @@ public class IoTDBUDTFHybridQueryIT {
EnvFactory.getEnv().cleanAfterClass();
}
- @Ignore
@Test
public void testUserDefinedBuiltInHybridAggregationQuery() {
String sql =
@@ -101,33 +116,85 @@ public class IoTDBUDTFHybridQueryIT {
Statement statement = connection.createStatement()) {
statement.executeQuery(sql);
fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("not supported"));
+ }
+ }
+
+ @Test
+ public void testUserDefinedBuiltInHybridAggregationQuery2() {
+ String[] retArray = new String[]
{"0,2.0,0.9092974268256817,3.0,-10.0,12.0"};
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "select avg(s1), sin(avg(s2)), avg(s1) + 1, -sum(s2), avg(s1) +
sum(s2) from root.vehicle.d1")) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("avg(root.vehicle.d1.s1)")
+ + ","
+ + resultSet.getString("sin(avg(root.vehicle.d1.s2))")
+ + ","
+ + resultSet.getString("avg(root.vehicle.d1.s1) + 1")
+ + ","
+ + resultSet.getString("-sum(root.vehicle.d1.s2)")
+ + ","
+ + resultSet.getString("avg(root.vehicle.d1.s1) +
sum(root.vehicle.d1.s2)");
+ assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray.length, cnt);
+ }
} catch (SQLException throwable) {
- assertTrue(
- throwable
- .getMessage()
- .contains("User-defined and built-in hybrid aggregation is not
supported together."));
+ throwable.printStackTrace();
+ fail(throwable.getMessage());
}
}
- @Ignore
@Test
+ @Ignore // TODO fill function incompatible
public void testUserDefinedFunctionFillFunctionHybridQuery() {
- String sql =
- String.format(
- "select temperature, counter(temperature, '%s'='%s') from
root.sgcc.wf03.wt01 where time = 2017-11-01T16:37:50.000 fill(float [linear,
1m, 1m])",
- UDFTestConstant.ACCESS_STRATEGY_KEY,
UDFTestConstant.ACCESS_STRATEGY_ROW_BY_ROW);
+ String[] retArray =
+ new String[] {
+ "0,0.0,1.0,3.14",
+ "1,0.8414709848078965,2.0,3.14",
+ "2,0.9092974268256817,3.0,2.0",
+ "3,0.1411200080598672,4.0,3.0",
+ "4,-0.7568024953079282,5.0,3.14",
+ "7,3.14,3.14,7.0",
+ "8,3.14,3.14,8.0",
+ "9,3.14,3.14,9.0"
+ };
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- statement.executeQuery(sql);
- fail();
+ try (ResultSet resultSet =
+ statement.executeQuery("select sin(s1), s1 + 1, s3 from
root.vehicle.d1 fill(3.14)")) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("sin(root.vehicle.d1.s1)")
+ + ","
+ + resultSet.getString("root.vehicle.d1.s1 + 1")
+ + ","
+ + resultSet.getString("root.vehicle.d1.s3");
+ assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray.length, cnt);
+ }
} catch (SQLException throwable) {
- assertTrue(
- throwable.getMessage().contains("Fill functions are not supported in
UDF queries."));
+ throwable.printStackTrace();
+ fail(throwable.getMessage());
}
}
- @Ignore
@Test
public void testLastUserDefinedFunctionQuery() {
String sql =
@@ -145,23 +212,45 @@ public class IoTDBUDTFHybridQueryIT {
}
}
- @Ignore
@Test
+ @Ignore // TODO remove after IOTDB-3674
public void testUserDefinedFunctionAlignByDeviceQuery() {
- String sql =
- String.format(
- "select adder(temperature), counter(temperature, '%s'='%s') from
root.sgcc.wf03.wt01 align by device",
- UDFTestConstant.ACCESS_STRATEGY_KEY,
UDFTestConstant.ACCESS_STRATEGY_ROW_BY_ROW);
+ String[] retArray =
+ new String[] {
+ "0,root.vehicle.d1,1.0,0.0",
+ "1,root.vehicle.d1,2.0,0.8414709848078965",
+ "2,root.vehicle.d1,3.0,0.9092974268256817",
+ "3,root.vehicle.d1,4.0,0.1411200080598672",
+ "4,root.vehicle.d1,5.0,0.1411200080598672",
+ "0,root.vehicle.d2,1.0,0.0",
+ "1,root.vehicle.d2,2.0,0.8414709848078965",
+ "2,root.vehicle.d2,3.0,0.9092974268256817",
+ "3,root.vehicle.d2,4.0,0.1411200080598672",
+ "4,root.vehicle.d2,5.0,0.1411200080598672"
+ };
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- statement.executeQuery(sql);
- fail();
- } catch (SQLException throwable) {
- assertTrue(
- throwable
- .getMessage()
- .contains("ALIGN BY DEVICE clause is not supported in UDF
queries."));
+ try (ResultSet resultSet =
+ statement.executeQuery("select s1 + 1, sin(s2) from root.vehicle.*
align by device")) {
+ int cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("Device")
+ + ","
+ + resultSet.getString("s1 + 1")
+ + ","
+ + resultSet.getString("sin(s2)");
+ assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray.length, cnt);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
}
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index 100e8f2390..3569ba75f1 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -118,14 +118,15 @@ public class ExpressionAnalyzer {
* @param expression expression to be checked
* @return true if this expression is valid
*/
- public static ResultColumn.ColumnType identifyOutputColumnType(Expression
expression) {
+ public static ResultColumn.ColumnType identifyOutputColumnType(
+ Expression expression, boolean isRoot) {
if (expression instanceof TernaryExpression) {
ResultColumn.ColumnType firstType =
- identifyOutputColumnType(((TernaryExpression)
expression).getFirstExpression());
+ identifyOutputColumnType(((TernaryExpression)
expression).getFirstExpression(), false);
ResultColumn.ColumnType secondType =
- identifyOutputColumnType(((TernaryExpression)
expression).getSecondExpression());
+ identifyOutputColumnType(((TernaryExpression)
expression).getSecondExpression(), false);
ResultColumn.ColumnType thirdType =
- identifyOutputColumnType(((TernaryExpression)
expression).getThirdExpression());
+ identifyOutputColumnType(((TernaryExpression)
expression).getThirdExpression(), false);
boolean rawFlag = false, aggregationFlag = false;
if (firstType == ResultColumn.ColumnType.RAW
|| secondType == ResultColumn.ColumnType.RAW
@@ -146,9 +147,9 @@ public class ExpressionAnalyzer {
return thirdType;
} else if (expression instanceof BinaryExpression) {
ResultColumn.ColumnType leftType =
- identifyOutputColumnType(((BinaryExpression)
expression).getLeftExpression());
+ identifyOutputColumnType(((BinaryExpression)
expression).getLeftExpression(), false);
ResultColumn.ColumnType rightType =
- identifyOutputColumnType(((BinaryExpression)
expression).getRightExpression());
+ identifyOutputColumnType(((BinaryExpression)
expression).getRightExpression(), false);
if ((leftType == ResultColumn.ColumnType.RAW
&& rightType == ResultColumn.ColumnType.AGGREGATION)
|| (leftType == ResultColumn.ColumnType.AGGREGATION
@@ -156,7 +157,8 @@ public class ExpressionAnalyzer {
throw new SemanticException(
"Raw data and aggregation result hybrid calculation is not
supported.");
}
- if (leftType == ResultColumn.ColumnType.CONSTANT
+ if (isRoot
+ && leftType == ResultColumn.ColumnType.CONSTANT
&& rightType == ResultColumn.ColumnType.CONSTANT) {
throw new SemanticException("Constant column is not supported.");
}
@@ -165,18 +167,48 @@ public class ExpressionAnalyzer {
}
return rightType;
} else if (expression instanceof UnaryExpression) {
- return identifyOutputColumnType(((UnaryExpression)
expression).getExpression());
+ return identifyOutputColumnType(((UnaryExpression)
expression).getExpression(), false);
} else if (expression instanceof FunctionExpression) {
- if (!expression.isBuiltInAggregationFunctionExpression()) {
- return ResultColumn.ColumnType.RAW;
- }
- for (Expression childExpression : expression.getExpressions()) {
- if (identifyOutputColumnType(childExpression) ==
ResultColumn.ColumnType.AGGREGATION) {
+ List<Expression> inputExpressions = expression.getExpressions();
+ if (expression.isBuiltInAggregationFunctionExpression()) {
+ for (Expression inputExpression : inputExpressions) {
+ if (identifyOutputColumnType(inputExpression, false)
+ == ResultColumn.ColumnType.AGGREGATION) {
+ throw new SemanticException(
+ "Aggregation results cannot be as input of the aggregation
function.");
+ }
+ }
+ return ResultColumn.ColumnType.AGGREGATION;
+ } else {
+ ResultColumn.ColumnType checkedType = null;
+ int lastCheckedIndex = 0;
+ for (int i = 0; i < inputExpressions.size(); i++) {
+ ResultColumn.ColumnType columnType =
+ identifyOutputColumnType(inputExpressions.get(i), false);
+ if (columnType != ResultColumn.ColumnType.CONSTANT) {
+ checkedType = columnType;
+ lastCheckedIndex = i;
+ break;
+ }
+ }
+ if (checkedType == null) {
throw new SemanticException(
- "Aggregation results cannot be as input of the aggregation
function.");
+ String.format(
+ "Input of '%s' is illegal.",
+ ((FunctionExpression) expression).getFunctionName()));
+ }
+ for (int i = lastCheckedIndex; i < inputExpressions.size(); i++) {
+ ResultColumn.ColumnType columnType =
+ identifyOutputColumnType(inputExpressions.get(i), false);
+ if (columnType != ResultColumn.ColumnType.CONSTANT && columnType !=
checkedType) {
+ throw new SemanticException(
+ String.format(
+ "Raw data and aggregation result hybrid input of '%s' is
not supported.",
+ ((FunctionExpression) expression).getFunctionName()));
+ }
}
+ return checkedType;
}
- return ResultColumn.ColumnType.AGGREGATION;
} else if (expression instanceof TimeSeriesOperand || expression
instanceof TimestampOperand) {
return ResultColumn.ColumnType.RAW;
} else if (expression instanceof ConstantOperand) {
@@ -278,13 +310,6 @@ public class ExpressionAnalyzer {
((BinaryExpression) predicate).getLeftExpression(), prefixPaths,
patternTree);
constructPatternTreeFromExpression(
((BinaryExpression) predicate).getRightExpression(), prefixPaths,
patternTree);
- } else if (predicate instanceof TernaryExpression) {
- constructPatternTreeFromExpression(
- ((TernaryExpression) predicate).getFirstExpression(), prefixPaths,
patternTree);
- constructPatternTreeFromExpression(
- ((TernaryExpression) predicate).getSecondExpression(), prefixPaths,
patternTree);
- constructPatternTreeFromExpression(
- ((TernaryExpression) predicate).getThirdExpression(), prefixPaths,
patternTree);
} else if (predicate instanceof UnaryExpression) {
constructPatternTreeFromExpression(
((UnaryExpression) predicate).getExpression(), prefixPaths,
patternTree);
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
index 88d38aebd3..870b91d47f 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
@@ -259,7 +259,7 @@ public class FunctionExpression extends Expression {
expression.inferTypes(typeProvider);
}
- if (isTimeSeriesGeneratingFunctionExpression()) {
+ if (!isBuiltInAggregationFunctionExpression()) {
typeProvider.setType(
expressionString,
new UDTFTypeInferrer(functionName)
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 04d50329f9..affb77b9ae 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -747,7 +747,8 @@ public class ASTVisitor extends
IoTDBSqlParserBaseVisitor<Statement> {
if (resultColumnContext.AS() != null) {
alias = parseAlias(resultColumnContext.alias());
}
- ResultColumn.ColumnType columnType =
ExpressionAnalyzer.identifyOutputColumnType(expression);
+ ResultColumn.ColumnType columnType =
+ ExpressionAnalyzer.identifyOutputColumnType(expression, true);
return new ResultColumn(expression, alias, columnType);
}
