This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch 2.0.1-rc04-patch
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/2.0.1-rc04-patch by this push:
new a3483f5be6 [fix](Nereids) fix ctas bugs
a3483f5be6 is described below
commit a3483f5be62b6385f5dfd4c813058d14725f3e3a
Author: morrySnow <[email protected]>
AuthorDate: Tue Sep 12 21:37:12 2023 +0800
[fix](Nereids) fix ctas bugs
1. ctas should support without distribution desc
2. ctas should support column name list
3. ctas should throw exception when excution failed
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 17 +++--
.../apache/doris/nereids/parser/NereidsParser.java | 2 +-
.../trees/plans/commands/CreateTableCommand.java | 48 +++++++++++++--
.../plans/commands/info/ColumnDefinition.java | 9 +++
.../trees/plans/commands/info/CreateTableInfo.java | 3 +-
.../commands/info/DistributionDescriptor.java | 12 +++-
.../org/apache/doris/nereids/types/DataType.java | 14 +++--
.../org/apache/doris/nereids/parser/CTASTest.java | 72 ++++++++++++++++++++++
.../es/test_es_query_nereids.groovy | 4 +-
10 files changed, 163 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index ae717ef0b0..1e717d70db 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -41,12 +41,12 @@ statement
TO user=userIdentify
USING LEFT_PAREN booleanExpression RIGHT_PAREN
#createRowPolicy
| CREATE TABLE (IF NOT EXISTS)? name=multipartIdentifier
- ((LEFT_PAREN columnDefs indexDefs? RIGHT_PAREN) |
(ctasCols=identifierList)?)
+ ((ctasCols=identifierList)? | (LEFT_PAREN columnDefs indexDefs?
RIGHT_PAREN))
(ENGINE EQ engine=identifier)?
((AGGREGATE | UNIQUE | DUPLICATE) KEY keys=identifierList)?
(COMMENT STRING_LITERAL)?
(PARTITION BY (RANGE | LIST) partitionKeys=identifierList LEFT_PAREN
partitions=partitionsDef RIGHT_PAREN)?
- DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) BUCKETS
(INTEGER_VALUE | AUTO)?
+ (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS
INTEGER_VALUE | AUTO)?)?
(ROLLUP LEFT_PAREN rollupDefs RIGHT_PAREN)?
propertyClause?
(AS query)?
#createTable
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index d3acbe1862..213422007b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.AggClauseContext;
@@ -1140,7 +1141,7 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
@Override
public Expression visitCast(DorisParser.CastContext ctx) {
- List<String> types = typedVisit(ctx.dataType());
+ Pair<List<String>, Boolean> types = typedVisit(ctx.dataType());
DataType dataType = DataType.convertPrimitiveFromStrings(types, true);
Expression cast = ParserUtils.withOrigin(ctx, () ->
new Cast(getExpression(ctx.expression()), dataType));
@@ -1536,9 +1537,13 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
keysType = KeysType.UNIQUE_KEYS;
}
String engineName = ctx.engine != null ?
ctx.engine.getText().toLowerCase() : "olap";
- DistributionDescriptor desc = new DistributionDescriptor(ctx.HASH() !=
null, ctx.AUTO() != null,
- Integer.parseInt(ctx.INTEGER_VALUE().getText()),
- ctx.HASH() != null ? visitIdentifierList(ctx.hashKeys) : null);
+ boolean isHash = ctx.HASH() != null || ctx.RANDOM() == null;
+ int bucketNum = FeConstants.default_bucket_num;
+ if (isHash && ctx.INTEGER_VALUE() != null) {
+ bucketNum = Integer.parseInt(ctx.INTEGER_VALUE().getText());
+ }
+ DistributionDescriptor desc = new DistributionDescriptor(isHash,
ctx.AUTO() != null,
+ bucketNum, ctx.HASH() != null ?
visitIdentifierList(ctx.hashKeys) : null);
Map<String, String> properties = ctx.propertyClause() != null
? visitPropertyClause(ctx.propertyClause()) : null;
String partitionType = null;
@@ -2246,11 +2251,11 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
}
@Override
- public List<String> visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
+ public Pair<List<String>, Boolean>
visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
String dataType =
ctx.primitiveColType().type.getText().toLowerCase(Locale.ROOT);
List<String> l = Lists.newArrayList(dataType);
ctx.INTEGER_VALUE().stream().map(ParseTree::getText).forEach(l::add);
- return l;
+ return Pair.of(l, ctx.primitiveColType().UNSIGNED() != null);
}
private Expression parseFunctionWithOrderKeys(String functionName, boolean
isDistinct,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
index 70065c7eda..84b2ee2917 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
@@ -74,7 +74,7 @@ public class NereidsParser {
return parse(expression, DorisParser::expression);
}
- public List<String> parseDataType(String dataType) {
+ public Pair<List<String>, Boolean> parseDataType(String dataType) {
return parse(dataType, DorisParser::dataType);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
index a15dff0dda..e5c4ab3619 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
@@ -21,6 +21,8 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.analyzer.UnboundOlapTableSink;
@@ -36,6 +38,15 @@ import
org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DateTimeType;
+import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateType;
+import org.apache.doris.nereids.types.DateV2Type;
+import org.apache.doris.nereids.types.DecimalV2Type;
+import org.apache.doris.nereids.types.DecimalV3Type;
+import org.apache.doris.nereids.types.TinyIntType;
+import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
@@ -91,11 +102,34 @@ public class CreateTableCommand extends Command implements
ForwardWithSync {
if (slots.size() != ctasCols.size()) {
throw new AnalysisException("ctas column size is not equal to the
query's");
}
- List<ColumnDefinition> columnsOfQuery = slots.stream()
- .map(s -> new ColumnDefinition(s.getName(), s.getDataType(),
s.nullable()))
- .collect(ImmutableList.toImmutableList());
- createTableInfo.validateCreateTableAsSelect(columnsOfQuery, ctx);
-
+ ImmutableList.Builder<ColumnDefinition> columnsOfQuery =
ImmutableList.builder();
+ for (int i = 0; i < slots.size(); i++) {
+ Slot s = slots.get(i);
+ DataType dataType = s.getDataType();
+ if (dataType.isNullType()) {
+ dataType = TinyIntType.INSTANCE;
+ } else if (dataType.isDecimalV2Type()) {
+ if (Config.enable_decimal_conversion) {
+ DecimalV2Type decimalV2Type = (DecimalV2Type) dataType;
+ dataType = DecimalV3Type.createDecimalV3Type(
+ decimalV2Type.getPrecision(),
decimalV2Type.getScale());
+ } else {
+ dataType = DecimalV2Type.SYSTEM_DEFAULT;
+ }
+ } else if (dataType instanceof DateType) {
+ if (Config.enable_date_conversion) {
+ dataType = DateV2Type.INSTANCE;
+ }
+ } else if (dataType instanceof DateTimeType) {
+ if (Config.enable_date_conversion) {
+ dataType = DateTimeV2Type.SYSTEM_DEFAULT;
+ }
+ } else if (i == 0 && dataType.isStringType()) {
+ dataType =
VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH);
+ }
+ columnsOfQuery.add(new ColumnDefinition(s.getName(), dataType,
s.nullable()));
+ }
+ createTableInfo.validateCreateTableAsSelect(columnsOfQuery.build(),
ctx);
CreateTableStmt createTableStmt =
createTableInfo.translateToLegacyStmt();
LOG.debug("Nereids start to execute the ctas command, query id: {},
tableName: {}",
ctx.queryId(), createTableInfo.getTableName());
@@ -114,6 +148,7 @@ public class CreateTableCommand extends Command implements
ForwardWithSync {
}
} catch (Exception e) {
handleFallbackFailedCtas(ctx);
+ throw new AnalysisException("Failed to execute CTAS Reason: " +
e.getMessage(), e);
}
}
@@ -123,7 +158,8 @@ public class CreateTableCommand extends Command implements
ForwardWithSync {
new
TableName(Env.getCurrentEnv().getCurrentCatalog().getName(),
createTableInfo.getDbName(),
createTableInfo.getTableName()), true));
} catch (Exception e) {
- ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected
exception: " + e.getMessage());
+ // TODO: refactor it with normal error process.
+ ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getMessage());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 1aa9f2d6f6..d547a529c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands.info;
+import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
@@ -160,6 +161,14 @@ public class ColumnDefinition {
} else if (type.isArrayType() && !defaultValue.isPresent()) {
defaultValue = Optional.of(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE);
}
+ if (defaultValue.isPresent() &&
type.toCatalogDataType().isScalarType()) {
+ try {
+ ColumnDef.validateDefaultValue(type.toCatalogDataType(),
+ defaultValue.get().getValue(),
defaultValue.get().getDefaultValueExprDef());
+ } catch (Exception e) {
+ throw new AnalysisException(e.getMessage(), e);
+ }
+ }
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 4ae8590c17..f2469c70e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -202,7 +202,7 @@ public class CreateTableInfo {
keyLength += catalogType.getIndexSize();
if (keys.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes)
{
- if (keys.size() == 0 && type.isStringLikeType()) {
+ if (keys.isEmpty() && type.isStringLikeType()) {
keys.add(column.getName());
}
break;
@@ -326,6 +326,7 @@ public class CreateTableInfo {
}
// analyze distribution descriptor
+ distribution.updateCols(columns.get(0).getName());
distribution.validate(columnMap, keysType);
// analyze key set.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java
index f8a3fb14db..397a3f09ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java
@@ -24,10 +24,13 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.nereids.exceptions.AnalysisException;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.commons.collections.CollectionUtils;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
/**
@@ -37,7 +40,7 @@ public class DistributionDescriptor {
private final boolean isHash;
private final boolean isAutoBucket;
private final int bucketNum;
- private final List<String> cols;
+ private List<String> cols;
public DistributionDescriptor(boolean isHash, boolean isAutoBucket, int
bucketNum, List<String> cols) {
this.isHash = isHash;
@@ -50,6 +53,13 @@ public class DistributionDescriptor {
return isHash;
}
+ public void updateCols(String col) {
+ Objects.requireNonNull(col, "col should not be null");
+ if (CollectionUtils.isEmpty(cols)) {
+ cols = Lists.newArrayList(col);
+ }
+ }
+
/**
* analyze distribution descriptor
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 443f4b1004..0b21395948 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
@@ -108,10 +109,11 @@ public abstract class DataType implements
AbstractDataType {
* NOTICE: only used in parser, if u want to convert string to type,
* please use {@link this#convertFromString(String)}
*
- * @param types data type in string representation
+ * @param typesPair data type in string representation
* @return data type in Nereids
*/
- public static DataType convertPrimitiveFromStrings(List<String> types,
boolean tryConvert) {
+ public static DataType convertPrimitiveFromStrings(Pair<List<String>,
Boolean> typesPair, boolean tryConvert) {
+ List<String> types = typesPair.first;
String type = types.get(0).toLowerCase().trim();
switch (type) {
case "bool":
@@ -123,7 +125,11 @@ public abstract class DataType implements AbstractDataType
{
return SmallIntType.INSTANCE;
case "integer":
case "int":
- return IntegerType.INSTANCE;
+ if (typesPair.second) {
+ return BigIntType.INSTANCE;
+ } else {
+ return IntegerType.INSTANCE;
+ }
case "bigint":
return BigIntType.INSTANCE;
case "largeint":
@@ -253,7 +259,7 @@ public abstract class DataType implements AbstractDataType {
*/
public static DataType convertFromString(String type) {
try {
- List<String> types = PARSER.parseDataType(type);
+ Pair<List<String>, Boolean> types = PARSER.parseDataType(type);
return DataType.convertPrimitiveFromStrings(types, false);
} catch (Exception e) {
// TODO: remove it when Nereids parser support array
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/CTASTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/CTASTest.java
new file mode 100644
index 0000000000..909366cb1c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/CTASTest.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser;
+
+import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CTASTest extends ParserTestBase {
+
+ @Test
+ public void testSimpleCtas() {
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan;
+ String cteSql1 = "CREATE TABLE T14 AS SELECT * FROM T10 WHERE DT >
DATE_FORMAT(NOW(), 'YYYYMMDD');";
+ logicalPlan = nereidsParser.parseSingle(cteSql1);
+ Assertions.assertTrue(logicalPlan instanceof CreateTableCommand);
+ }
+
+ @Test
+ public void testCtasWithProperties() {
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan;
+ String cteSql1 = "CREATE TABLE T14 PROPERTIES ('REPLICATION_NUM'='1')
AS SELECT * FROM T10 WHERE DT > DATE_FORMAT(NOW(), 'YYYYMMDD');";
+ logicalPlan = nereidsParser.parseSingle(cteSql1);
+ Assertions.assertTrue(logicalPlan instanceof CreateTableCommand);
+ }
+
+ @Test
+ public void testCtasWithColumn() {
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan;
+ String cteSql1 = "CREATE TABLE T14(ID, AGE, DT) AS SELECT * FROM T10
WHERE DT > DATE_FORMAT(NOW(), 'YYYYMMDD');";
+ logicalPlan = nereidsParser.parseSingle(cteSql1);
+ Assertions.assertTrue(logicalPlan instanceof CreateTableCommand);
+ }
+
+ @Test
+ public void testCtasWithDistributed() {
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan;
+ String cteSql1 = "CREATE TABLE T14 DISTRIBUTED BY RANDOM AS SELECT *
FROM T10 WHERE DT > DATE_FORMAT(NOW(), 'YYYYMMDD');";
+ logicalPlan = nereidsParser.parseSingle(cteSql1);
+ Assertions.assertTrue(logicalPlan instanceof CreateTableCommand);
+ }
+
+ @Test
+ public void testCtasWithbuckets() {
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan;
+ String cteSql1 = "CREATE TABLE T14 DISTRIBUTED BY HASH(ID) BUCKETS 10
AS SELECT * FROM T10 WHERE DT > DATE_FORMAT(NOW(), 'YYYYMMDD');";
+ logicalPlan = nereidsParser.parseSingle(cteSql1);
+ Assertions.assertTrue(logicalPlan instanceof CreateTableCommand);
+ }
+}
diff --git
a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy
b/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy
index 835f78b11d..446c30bf22 100644
--- a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy
+++ b/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy
@@ -29,7 +29,6 @@ suite("test_es_query_nereids", "p0") {
sql """drop table if exists test_v1_nereids;"""
sql """drop table if exists test_v2_nereids;"""
sql """set enable_nereids_planner=true;"""
- sql """set enable_fallback_to_original_planner=false;"""
// test old create-catalog syntax for compatibility
sql """
@@ -133,6 +132,9 @@ suite("test_es_query_nereids", "p0") {
"http_ssl_enabled"="false"
);
"""
+
+ sql """set enable_fallback_to_original_planner=false;"""
+
// TODO(ftw): should open these annotation when nereids support es
external table
// order_qt_sql51 """select * from test_v2_nereids where
test2='text#1'"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
