This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new be418110cb4 [enhancement](nereids)support null partition for list
partition (#31613)
be418110cb4 is described below
commit be418110cb45a313281617dc8870c06792399f81
Author: starocean999 <[email protected]>
AuthorDate: Mon Mar 4 16:31:34 2024 +0800
[enhancement](nereids)support null partition for list partition (#31613)
---
be/src/exec/tablet_info.cpp | 5 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../org/apache/doris/analysis/PartitionDesc.java | 3 -
.../org/apache/doris/analysis/PartitionValue.java | 15 ++-
.../org/apache/doris/catalog/PartitionKey.java | 119 +++++++++++++--------
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +
.../nereids/trees/expressions/literal/Literal.java | 2 +
.../trees/plans/commands/info/CreateTableInfo.java | 3 -
.../plans/commands/info/PartitionDefinition.java | 2 +-
.../org/apache/doris/planner/OlapTableSink.java | 9 +-
.../nereids_syntax_p0/list_partition_with_null.out | 15 +++
.../list_partition_with_null.groovy | 71 ++++++++++++
.../test_list_partition_datatype.groovy | 19 ----
13 files changed, 191 insertions(+), 76 deletions(-)
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index facb43ea403..582a97bd681 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -521,6 +521,11 @@ Status
VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr,
column->insert_data(reinterpret_cast<const
char*>(&t_expr.bool_literal.value), 0);
break;
}
+ case TExprNodeType::NULL_LITERAL: {
+ // insert a null literal
+ column->insert_data(nullptr, 0);
+ break;
+ }
default: {
return Status::InternalError("unsupported partition column node type,
type={}",
t_expr.node_type);
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a56134f00f8..a70df9eec92 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -561,7 +561,7 @@ constantSeq
;
partitionValueDef
- : INTEGER_VALUE | STRING_LITERAL | MAXVALUE
+ : INTEGER_VALUE | STRING_LITERAL | MAXVALUE | NULL
;
rollupDefs
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
index 1f68bc2c6b3..39569d0b3e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
@@ -208,9 +208,6 @@ public class PartitionDesc {
throw new AnalysisException(
"The partition column must be NOT NULL with
allow_partition_column_nullable OFF");
}
- if (this instanceof ListPartitionDesc &&
columnDef.isAllowNull()) {
- throw new AnalysisException("The list partition column
must be NOT NULL");
- }
if (this instanceof RangePartitionDesc && partitionExprs
!= null) {
if (partitionExprs.get(0) instanceof FunctionCallExpr)
{
if (!columnDef.getType().isDateType()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java
index 5dea5c767b7..75e6f54bca2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java
@@ -27,6 +27,7 @@ public class PartitionValue {
private String value;
private boolean isHiveDefaultPartition;
+ private boolean isNullPartition;
private PartitionValue() {
@@ -45,6 +46,12 @@ public class PartitionValue {
this.isHiveDefaultPartition = isHiveDefaultPartition;
}
+ public PartitionValue(String value, boolean isNullPartition, boolean
isHiveDefaultPartition) {
+ this.value = value;
+ this.isNullPartition = isNullPartition;
+ this.isHiveDefaultPartition = isHiveDefaultPartition;
+ }
+
public LiteralExpr getValue(Type type) throws AnalysisException {
if (isHiveDefaultPartition) {
return new StringLiteral(value);
@@ -81,12 +88,16 @@ public class PartitionValue {
return false;
}
PartitionValue that = (PartitionValue) o;
- return isHiveDefaultPartition == that.isHiveDefaultPartition
+ return isHiveDefaultPartition == that.isHiveDefaultPartition &&
isNullPartition == that.isNullPartition
&& Objects.equal(value, that.value);
}
@Override
public int hashCode() {
- return Objects.hashCode(value, isHiveDefaultPartition);
+ return Objects.hashCode(value, isHiveDefaultPartition,
isNullPartition);
+ }
+
+ public boolean isNullPartition() {
+ return isNullPartition;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
index 1677eb6cb5f..9735f11c69a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.MaxLiteral;
+import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.common.AnalysisException;
@@ -137,7 +138,12 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
PartitionKey partitionKey = new PartitionKey();
for (int i = 0; i < values.size(); i++) {
- partitionKey.keys.add(values.get(i).getValue(types.get(i)));
+ if (values.get(i).isNullPartition()) {
+ partitionKey.keys.add(NullLiteral.create(types.get(i)));
+ } else {
+ partitionKey.keys.add(values.get(i).getValue(types.get(i)));
+ }
+
if (isHive) {
partitionKey.originHiveKeys.add(values.get(i).getStringValue());
}
@@ -267,7 +273,7 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
int i = 0;
for (LiteralExpr expr : keys) {
Object value = null;
- if (expr == MaxLiteral.MAX_VALUE) {
+ if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
value = expr.toSql();
sb.append(value);
continue;
@@ -312,7 +318,7 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
int i = 0;
for (LiteralExpr expr : keys) {
Object value = null;
- if (expr == MaxLiteral.MAX_VALUE) {
+ if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
value = expr.toSql();
} else {
value = expr.getRealValue();
@@ -341,9 +347,17 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
out.writeInt(count);
for (int i = 0; i < count; i++) {
PrimitiveType type = types.get(i);
- Text.writeString(out, type.toString());
+ if (keys.get(i).isNullLiteral()) {
+ Text.writeString(out, Type.NULL.toString());
+ } else {
+ Text.writeString(out, type.toString());
+ }
+
if (keys.get(i) == MaxLiteral.MAX_VALUE) {
out.writeBoolean(true);
+ } else if (keys.get(i).isNullLiteral()) {
+ out.writeBoolean(false);
+ Text.writeString(out, type.toString());
} else {
out.writeBoolean(false);
keys.get(i).write(out);
@@ -355,10 +369,16 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
int count = in.readInt();
for (int i = 0; i < count; i++) {
PrimitiveType type = PrimitiveType.valueOf(Text.readString(in));
- types.add(type);
-
- LiteralExpr literal = null;
boolean isMax = in.readBoolean();
+ if (type == PrimitiveType.NULL_TYPE) {
+ String realType = StringLiteral.read(in).getStringValue();
+ type = PrimitiveType.valueOf(realType);
+ types.add(type);
+ keys.add(NullLiteral.create(Type.fromPrimitiveType(type)));
+ continue;
+ }
+ LiteralExpr literal = null;
+ types.add(type);
if (isMax) {
literal = MaxLiteral.MAX_VALUE;
} else {
@@ -480,47 +500,54 @@ public class PartitionKey implements
Comparable<PartitionKey>, Writable {
for (int i = 0; i < count; i++) {
JsonArray typeAndKey = new JsonArray();
PrimitiveType type = types.get(i);
- typeAndKey.add(new JsonPrimitive(type.toString()));
-
- if (keys.get(i) == MaxLiteral.MAX_VALUE) {
- typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
+ if (keys.get(i).isNullLiteral()) {
+ // save NULL_TYPE as type and real type as key
+ typeAndKey.add(new
JsonPrimitive(PrimitiveType.NULL_TYPE.toString()));
+ typeAndKey.add(new JsonPrimitive(type.toString()));
} else {
- switch (type) {
- case TINYINT:
- case SMALLINT:
- case INT:
- case BIGINT: {
- IntLiteral key = (IntLiteral) keys.get(i);
- typeAndKey.add(new
JsonPrimitive(key.getLongValue()));
- }
- break;
- case LARGEINT: {
- LargeIntLiteral key = (LargeIntLiteral)
keys.get(i);
- typeAndKey.add(new
JsonPrimitive(key.getRealValue().toString()));
- }
- break;
- case DATE:
- case DATETIME:
- case DATEV2:
- case DATETIMEV2: {
- DateLiteral key = (DateLiteral) keys.get(i);
- typeAndKey.add(new
JsonPrimitive(key.convertToString(type)));
- }
- break;
- case CHAR:
- case VARCHAR:
- case STRING: {
- StringLiteral key = (StringLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getValue()));
- }
- break;
- case BOOLEAN: {
- BoolLiteral key = (BoolLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getValue()));
+ typeAndKey.add(new JsonPrimitive(type.toString()));
+
+ if (keys.get(i) == MaxLiteral.MAX_VALUE) {
+ typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
+ } else {
+ switch (type) {
+ case TINYINT:
+ case SMALLINT:
+ case INT:
+ case BIGINT: {
+ IntLiteral key = (IntLiteral) keys.get(i);
+ typeAndKey.add(new
JsonPrimitive(key.getLongValue()));
+ }
+ break;
+ case LARGEINT: {
+ LargeIntLiteral key = (LargeIntLiteral)
keys.get(i);
+ typeAndKey.add(new
JsonPrimitive(key.getRealValue().toString()));
+ }
+ break;
+ case DATE:
+ case DATETIME:
+ case DATEV2:
+ case DATETIMEV2: {
+ DateLiteral key = (DateLiteral) keys.get(i);
+ typeAndKey.add(new
JsonPrimitive(key.convertToString(type)));
+ }
+ break;
+ case CHAR:
+ case VARCHAR:
+ case STRING: {
+ StringLiteral key = (StringLiteral)
keys.get(i);
+ typeAndKey.add(new
JsonPrimitive(key.getValue()));
+ }
+ break;
+ case BOOLEAN: {
+ BoolLiteral key = (BoolLiteral) keys.get(i);
+ typeAndKey.add(new
JsonPrimitive(key.getValue()));
+ }
+ break;
+ default:
+ throw new JsonParseException(
+ "type[" + type.name() + "] not
supported: ");
}
- break;
- default:
- throw new JsonParseException("type[" + type.name()
+ "] not supported: ");
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 13c54fcb10c..ff8e07f5ed2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2673,6 +2673,8 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return Literal.of(toStringValue(ctx.STRING_LITERAL().getText()));
} else if (ctx.MAXVALUE() != null) {
return MaxValue.INSTANCE;
+ } else if (ctx.NULL() != null) {
+ return Literal.of(null);
}
throw new AnalysisException("Unsupported partition value: " +
ctx.getText());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
index 11c292e3a0b..d51798c1b8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
@@ -291,6 +291,8 @@ public abstract class Literal extends Expression implements
LeafExpression, Comp
DataType dataType = DataType.fromCatalogType(type);
if (literalExpr instanceof org.apache.doris.analysis.MaxLiteral) {
return new MaxLiteral(dataType);
+ } else if (literalExpr instanceof
org.apache.doris.analysis.NullLiteral) {
+ return new NullLiteral(dataType);
}
String stringValue = literalExpr.getStringValue();
if (dataType.isBooleanType()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 400db4fc65f..93a827ed010 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -666,9 +666,6 @@ public class CreateTableInfo {
throw new AnalysisException(
"The partition column must be NOT NULL with
allow_partition_column_nullable OFF");
}
- if (partitionType.equalsIgnoreCase(PartitionType.LIST.name()) &&
column.isNullable()) {
- throw new AnalysisException("The list partition column must be NOT
NULL");
- }
}
// if auto bucket auto bucket enable, rewrite distribution bucket num &&
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
index 81ddd7b2ac2..95c20fdfa8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
@@ -161,7 +161,7 @@ public abstract class PartitionDefinition {
*/
protected PartitionValue toLegacyPartitionValueStmt(Expression e) {
if (e.isLiteral()) {
- return new PartitionValue(((Literal) e).getStringValue());
+ return new PartitionValue(((Literal) e).getStringValue(),
e.isNullLiteral(), false);
} else if (e instanceof MaxValue) {
return PartitionValue.MAX_VALUE;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index c92c08bb7d2..049843f803b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -20,6 +20,8 @@ package org.apache.doris.planner;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@@ -475,7 +477,12 @@ public class OlapTableSink extends DataSink {
for (PartitionKey partitionKey : partitionKeys) {
List<TExprNode> tExprNodes = new ArrayList<>();
for (int i = 0; i < partColNum; i++) {
-
tExprNodes.add(partitionKey.getKeys().get(i).treeToThrift().getNodes().get(0));
+ LiteralExpr literalExpr = partitionKey.getKeys().get(i);
+ if (literalExpr.isNullLiteral()) {
+
tExprNodes.add(NullLiteral.create(literalExpr.getType()).treeToThrift().getNodes().get(0));
+ } else {
+
tExprNodes.add(literalExpr.treeToThrift().getNodes().get(0));
+ }
}
tPartition.addToInKeys(tExprNodes);
tPartition.setIsDefaultPartition(partitionItem.isDefaultPartition());
diff --git
a/regression-test/data/nereids_syntax_p0/list_partition_with_null.out
b/regression-test/data/nereids_syntax_p0/list_partition_with_null.out
new file mode 100644
index 00000000000..f2b8e48c125
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/list_partition_with_null.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+\N
+\N
+
+-- !select2 --
+\N
+\N
+
+-- !select3 --
+SHANGHAI
+
+-- !select4 --
+1
+
diff --git
a/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy
b/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy
new file mode 100644
index 00000000000..403df939a87
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/list_partition_with_null.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("list_partition_with_null") {
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "SET enable_nereids_planner=true"
+
+ sql """DROP TABLE IF EXISTS table_null_list_parition;"""
+
+ sql """
+ CREATE TABLE table_null_list_parition
+ (
+ `USER_ID` LARGEINT COMMENT "用户ID",
+ `CITY` VARCHAR(20) COMMENT "用户所在城市"
+ )
+ ENGINE=OLAP
+ DUPLICATE KEY(`USER_ID`, `CITY`)
+ PARTITION BY LIST(`USER_ID`, `CITY`)
+ (
+ PARTITION `P1_CITY` VALUES IN (("1", null), ("1", "SHANGHAI")),
+ PARTITION `P2_CITY` VALUES IN (("2", "BEIJING"), (null,
"SHANGHAI")),
+ PARTITION `P3_CITY` VALUES IN ((null,null), ("3", "SHANGHAI"))
+ )
+ DISTRIBUTED BY HASH(`USER_ID`) BUCKETS 16
+ PROPERTIES
+ (
+ "replication_num" = "1"
+ );
+ """
+ sql """insert into table_null_list_parition
values(1,null),(null,null),(null,'SHANGHAI');"""
+ explain {
+ sql("select * from table_null_list_parition where city is null;")
+ verbose true
+ contains("partitions=2/3 (P1_CITY,P3_CITY)")
+ }
+
+ explain {
+ sql("select * from table_null_list_parition where user_id is
null;")
+ verbose true
+ contains("partitions=2/3 (P2_CITY,P3_CITY)")
+ }
+ explain {
+ sql("select * from table_null_list_parition where city is not
null;")
+ verbose true
+ contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)")
+ }
+
+ explain {
+ sql("select * from table_null_list_parition where user_id is not
null;")
+ verbose true
+ contains("partitions=3/3 (P1_CITY,P2_CITY,P3_CITY)")
+ }
+ qt_select1 """select city from table_null_list_parition where city is null
order by city;"""
+ qt_select2 """select user_id from table_null_list_parition where user_id
is null;"""
+ qt_select3 """select city from table_null_list_parition where city is not
null;"""
+ qt_select4 """select user_id from table_null_list_parition where user_id
is not null;"""
+}
diff --git
a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy
b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy
index 1a667533dd3..1de3f6588b6 100644
---
a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy
+++
b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy
@@ -285,25 +285,6 @@ suite("test_list_partition_datatype", "p0") {
"""
exception "Invalid list value format: errCode = 2, detailMessage =
Invalid number format: -10.01"
}
- test {
- sql """
- CREATE TABLE test_list_partition_err_tbl_6 (
- k1 INT NULL,
- v1 INT SUM NULL,
- v2 INT MAX NULL,
- v3 INT MIN NULL,
- v4 INT REPLACE NULL )
- AGGREGATE KEY(k1)
- PARTITION BY LIST(k1) (
- PARTITION p1 VALUES IN ("1","2","3"),
- PARTITION p2 VALUES IN ("10"),
- PARTITION p3 VALUES IN ("100","-10"),
- PARTITION p4 VALUES IN ("-100","0") )
- DISTRIBUTED BY HASH(k1) BUCKETS 5
- PROPERTIES ("replication_allocation" = "tag.location.default: 1")
- """
- exception "The list partition column must be NOT NULL"
- }
// date/datetime list partition errors like: conflict, invalid format
test {
sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
