This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3777ffb43f61608f9859d22585658df52bfc56e9 Author: starocean999 <[email protected]> AuthorDate: Mon Mar 4 16:31:34 2024 +0800 [enhancement](nereids)support null partition for list partition (#31613) --- be/src/exec/tablet_info.cpp | 5 + .../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +- .../org/apache/doris/analysis/PartitionDesc.java | 3 - .../org/apache/doris/analysis/PartitionValue.java | 15 ++- .../org/apache/doris/catalog/PartitionKey.java | 119 +++++++++++++-------- .../doris/nereids/parser/LogicalPlanBuilder.java | 2 + .../nereids/trees/expressions/literal/Literal.java | 2 + .../trees/plans/commands/info/CreateTableInfo.java | 3 - .../plans/commands/info/PartitionDefinition.java | 2 +- .../org/apache/doris/planner/OlapTableSink.java | 9 +- .../nereids_syntax_p0/list_partition_with_null.out | 15 +++ .../list_partition_with_null.groovy | 71 ++++++++++++ .../test_list_partition_datatype.groovy | 19 ---- 13 files changed, 191 insertions(+), 76 deletions(-) diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index facb43ea403..582a97bd681 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -521,6 +521,11 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, column->insert_data(reinterpret_cast<const char*>(&t_expr.bool_literal.value), 0); break; } + case TExprNodeType::NULL_LITERAL: { + // insert a null literal + column->insert_data(nullptr, 0); + break; + } default: { return Status::InternalError("unsupported partition column node type, type={}", t_expr.node_type); diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a56134f00f8..a70df9eec92 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -561,7 +561,7 @@ constantSeq ; partitionValueDef - : INTEGER_VALUE | STRING_LITERAL | MAXVALUE + : INTEGER_VALUE | STRING_LITERAL | MAXVALUE | NULL ; rollupDefs diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 1f68bc2c6b3..39569d0b3e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -208,9 +208,6 @@ public class PartitionDesc { throw new AnalysisException( "The partition column must be NOT NULL with allow_partition_column_nullable OFF"); } - if (this instanceof ListPartitionDesc && columnDef.isAllowNull()) { - throw new AnalysisException("The list partition column must be NOT NULL"); - } if (this instanceof RangePartitionDesc && partitionExprs != null) { if (partitionExprs.get(0) instanceof FunctionCallExpr) { if (!columnDef.getType().isDateType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java index 5dea5c767b7..75e6f54bca2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java @@ -27,6 +27,7 @@ public class PartitionValue { private String value; private boolean isHiveDefaultPartition; + private boolean isNullPartition; private PartitionValue() { @@ -45,6 +46,12 @@ public class PartitionValue { this.isHiveDefaultPartition = isHiveDefaultPartition; } + public PartitionValue(String value, boolean isNullPartition, boolean isHiveDefaultPartition) { + this.value = value; + this.isNullPartition = isNullPartition; + this.isHiveDefaultPartition = isHiveDefaultPartition; + } + public LiteralExpr getValue(Type type) throws AnalysisException { if (isHiveDefaultPartition) { return new StringLiteral(value); @@ -81,12 +88,16 @@ public class PartitionValue { return false; } PartitionValue that = (PartitionValue) o; - return isHiveDefaultPartition == that.isHiveDefaultPartition + return isHiveDefaultPartition == that.isHiveDefaultPartition && isNullPartition == that.isNullPartition && Objects.equal(value, that.value); } @Override public int hashCode() { - return Objects.hashCode(value, isHiveDefaultPartition); + return Objects.hashCode(value, isHiveDefaultPartition, isNullPartition); + } + + public boolean isNullPartition() { + return isNullPartition; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 1677eb6cb5f..9735f11c69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.LargeIntLiteral; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.MaxLiteral; +import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.common.AnalysisException; @@ -137,7 +138,12 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { PartitionKey partitionKey = new PartitionKey(); for (int i = 0; i < values.size(); i++) { - partitionKey.keys.add(values.get(i).getValue(types.get(i))); + if (values.get(i).isNullPartition()) { + partitionKey.keys.add(NullLiteral.create(types.get(i))); + } else { + partitionKey.keys.add(values.get(i).getValue(types.get(i))); + } + if (isHive) { partitionKey.originHiveKeys.add(values.get(i).getStringValue()); } @@ -267,7 +273,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { int i = 0; for (LiteralExpr expr : keys) { Object value = null; - if (expr == MaxLiteral.MAX_VALUE) { + if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) { value = expr.toSql(); sb.append(value); continue; @@ -312,7 +318,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { int i = 0; for (LiteralExpr expr : keys) { Object value = null; - if (expr == MaxLiteral.MAX_VALUE) { + if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) { value = expr.toSql(); } else { value = expr.getRealValue(); @@ -341,9 +347,17 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { out.writeInt(count); for (int i = 0; i < count; i++) { PrimitiveType type = types.get(i); - Text.writeString(out, type.toString()); + if (keys.get(i).isNullLiteral()) { + Text.writeString(out, Type.NULL.toString()); + } else { + Text.writeString(out, type.toString()); + } + if (keys.get(i) == MaxLiteral.MAX_VALUE) { out.writeBoolean(true); + } else if (keys.get(i).isNullLiteral()) { + out.writeBoolean(false); + Text.writeString(out, type.toString()); } else { out.writeBoolean(false); keys.get(i).write(out); @@ -355,10 +369,16 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { int count = in.readInt(); for (int i = 0; i < count; i++) { PrimitiveType type = PrimitiveType.valueOf(Text.readString(in)); - types.add(type); - - LiteralExpr literal = null; boolean isMax = in.readBoolean(); + if (type == PrimitiveType.NULL_TYPE) { + String realType = StringLiteral.read(in).getStringValue(); + type = PrimitiveType.valueOf(realType); + types.add(type); + keys.add(NullLiteral.create(Type.fromPrimitiveType(type))); + continue; + } + LiteralExpr literal = null; + types.add(type); if (isMax) { literal = MaxLiteral.MAX_VALUE; } else { @@ -480,47 +500,54 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable { for (int i = 0; i < count; i++) { JsonArray typeAndKey = new JsonArray(); PrimitiveType type = types.get(i); - typeAndKey.add(new JsonPrimitive(type.toString())); - - if (keys.get(i) == MaxLiteral.MAX_VALUE) { - typeAndKey.add(new JsonPrimitive("MAX_VALUE")); + if (keys.get(i).isNullLiteral()) { + // save NULL_TYPE as type and real type as key + typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString())); + typeAndKey.add(new JsonPrimitive(type.toString())); } else { - switch (type) { - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: { - IntLiteral key = (IntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getLongValue())); - } - break; - case LARGEINT: { - LargeIntLiteral key = (LargeIntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getRealValue().toString())); - } - break; - case DATE: - case DATETIME: - case DATEV2: - case DATETIMEV2: { - DateLiteral key = (DateLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.convertToString(type))); - } - break; - case CHAR: - case VARCHAR: - case STRING: { - StringLiteral key = (StringLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); - } - break; - case BOOLEAN: { - BoolLiteral key = (BoolLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); + typeAndKey.add(new JsonPrimitive(type.toString())); + + if (keys.get(i) == MaxLiteral.MAX_VALUE) { + typeAndKey.add(new JsonPrimitive("MAX_VALUE")); + } else { + switch (type) { + case TINYINT: + case SMALLINT: + case INT: + case BIGINT: { + IntLiteral key = (IntLiteral) keys.get(i); + typeAndKey.add(new JsonPrimitive(key.getLongValue())); + } + break; + case LARGEINT: { + LargeIntLiteral key = (LargeIntLiteral) keys.get(i); + typeAndKey.add(new JsonPrimitive(key.getRealValue().toString())); + } + break; + case DATE: + case DATETIME: + case DATEV2: + case DATETIMEV2: { + DateLiteral key = (DateLiteral) keys.get(i); + typeAndKey.add(new JsonPrimitive(key.convertToString(type))); + } + break; + case CHAR: + case VARCHAR: + case STRING: { + StringLiteral key = (StringLiteral) keys.get(i); + typeAndKey.add(new JsonPrimitive(key.getValue())); + } + break; + case BOOLEAN: { + BoolLiteral key = (BoolLiteral) keys.get(i); + typeAndKey.add(new JsonPrimitive(key.getValue())); + } + break; + default: + throw new JsonParseException( + "type[" + type.name() + "] not supported: "); } - break; - default: - throw new JsonParseException("type[" + type.name() + "] not supported: "); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index df104266e56..03158295a7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2669,6 +2669,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { return Literal.of(toStringValue(ctx.STRING_LITERAL().getText())); } else if (ctx.MAXVALUE() != null) { return MaxValue.INSTANCE; + } else if (ctx.NULL() != null) { + return Literal.of(null); } throw new AnalysisException("Unsupported partition value: " + ctx.getText()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 11c292e3a0b..d51798c1b8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -291,6 +291,8 @@ public abstract class Literal extends Expression implements LeafExpression, Comp DataType dataType = DataType.fromCatalogType(type); if (literalExpr instanceof org.apache.doris.analysis.MaxLiteral) { return new MaxLiteral(dataType); + } else if (literalExpr instanceof org.apache.doris.analysis.NullLiteral) { + return new NullLiteral(dataType); } String stringValue = literalExpr.getStringValue(); if (dataType.isBooleanType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 59f6cc693f7..8ae6ee0da30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -667,9 +667,6 @@ public class CreateTableInfo { throw new AnalysisException( "The partition column must be NOT NULL with allow_partition_column_nullable OFF"); } - if (partitionType.equalsIgnoreCase(PartitionType.LIST.name()) && column.isNullable()) { - throw new AnalysisException("The list partition column must be NOT NULL"); - } } // if auto bucket auto bucket enable, rewrite distribution bucket num && diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java index 81ddd7b2ac2..95c20fdfa8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java @@ -161,7 +161,7 @@ public abstract class PartitionDefinition { */ protected PartitionValue toLegacyPartitionValueStmt(Expression e) { if (e.isLiteral()) { - return new PartitionValue(((Literal) e).getStringValue()); + return new PartitionValue(((Literal) e).getStringValue(), e.isNullLiteral(), false); } else if (e instanceof MaxValue) { return PartitionValue.MAX_VALUE; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 924b64f87cd..f27ed9099dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -20,6 +20,8 @@ package org.apache.doris.planner; import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -473,7 +475,12 @@ public class OlapTableSink extends DataSink { for (PartitionKey partitionKey : partitionKeys) { List<TExprNode> tExprNodes = new ArrayList<>(); for (int i = 0; i < partColNum; i++) { - tExprNodes.add(partitionKey.getKeys().get(i).treeToThrift().getNodes().get(0)); + LiteralExpr literalExpr = partitionKey.getKeys().get(i); + if (literalExpr.isNullLiteral()) { + tExprNodes.add(NullLiteral.create(literalExpr.getType()).treeToThrift().getNodes().get(0)); + } else { + tExprNodes.add(literalExpr.treeToThrift().getNodes().get(0)); + } } tPartition.addToInKeys(tExprNodes); tPartition.setIsDefaultPartition(partitionItem.isDefaultPartition()); diff --git a/regression-test/data/nereids_syntax_p0/list_partition_with_null.out b/regression-test/data/nereids_syntax_p0/list_partition_with_null.out new file mode 100644 index 00000000000..f2b8e48c125 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/list_partition_with_null.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +\N +\N + +-- !select2 -- +\N +\N + +-- !select3 -- +SHANGHAI + +-- !select4 -- +1 + diff --git a/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy b/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy new file mode 100644 index 00000000000..403df939a87 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("list_partition_with_null") { + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_nereids_planner=true" + + sql """DROP TABLE IF EXISTS table_null_list_parition;""" + + sql """ + CREATE TABLE table_null_list_parition + ( + `USER_ID` LARGEINT COMMENT "用户ID", + `CITY` VARCHAR(20) COMMENT "用户所在城市" + ) + ENGINE=OLAP + DUPLICATE KEY(`USER_ID`, `CITY`) + PARTITION BY LIST(`USER_ID`, `CITY`) + ( + PARTITION `P1_CITY` VALUES IN (("1", null), ("1", "SHANGHAI")), + PARTITION `P2_CITY` VALUES IN (("2", "BEIJING"), (null, "SHANGHAI")), + PARTITION `P3_CITY` VALUES IN ((null,null), ("3", "SHANGHAI")) + ) + DISTRIBUTED BY HASH(`USER_ID`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + sql """insert into table_null_list_parition values(1,null),(null,null),(null,'SHANGHAI');""" + explain { + sql("select * from table_null_list_parition where city is null;") + verbose true + contains("partitions=2/3 (P1_CITY,P3_CITY)") + } + + explain { + sql("select * from table_null_list_parition where user_id is null;") + verbose true + contains("partitions=2/3 (P2_CITY,P3_CITY)") + } + explain { + sql("select * from table_null_list_parition where city is not null;") + verbose true + contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)") + } + + explain { + sql("select * from table_null_list_parition where user_id is not null;") + verbose true + contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)") + } + qt_select1 """select city from table_null_list_parition where city is null order by city;""" + qt_select2 """select user_id from table_null_list_parition where user_id is null;""" + qt_select3 """select city from table_null_list_parition where city is not null;""" + qt_select4 """select user_id from table_null_list_parition where user_id is not null;""" +} diff --git a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy index 1a667533dd3..1de3f6588b6 100644 --- a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy +++ b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy @@ -285,25 +285,6 @@ suite("test_list_partition_datatype", "p0") { """ exception "Invalid list value format: errCode = 2, detailMessage = Invalid number format: -10.01" } - test { - sql """ - CREATE TABLE test_list_partition_err_tbl_6 ( - k1 INT NULL, - v1 INT SUM NULL, - v2 INT MAX NULL, - v3 INT MIN NULL, - v4 INT REPLACE NULL ) - AGGREGATE KEY(k1) - PARTITION BY LIST(k1) ( - PARTITION p1 VALUES IN ("1","2","3"), - PARTITION p2 VALUES IN ("10"), - PARTITION p3 VALUES IN ("100","-10"), - PARTITION p4 VALUES IN ("-100","0") ) - DISTRIBUTED BY HASH(k1) BUCKETS 5 - PROPERTIES ("replication_allocation" = "tag.location.default: 1") - """ - exception "The list partition column must be NOT NULL" - } // date/datetime list partition errors like: conflict, invalid format test { sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
