This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bdab3bd4d72 [Fix](Variant) variant should not implicit be short key
column when create mv (#46444)
bdab3bd4d72 is described below
commit bdab3bd4d7268c22fa0531788c8b911b1fd93656
Author: lihangyu <[email protected]>
AuthorDate: Tue Jan 7 16:10:51 2025 +0800
[Fix](Variant) variant should not implicit be short key column when create
mv (#46444)
When creating mv, the variant column should not be part of short key
since it's not supported sorting
---
.../main/java/org/apache/doris/catalog/Type.java | 12 +++++++++
.../doris/alter/MaterializedViewHandler.java | 3 +--
.../doris/analysis/CreateMaterializedViewStmt.java | 6 +++--
.../org/apache/doris/analysis/CreateTableStmt.java | 11 +-------
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 8 +++---
.../trees/plans/commands/info/CreateTableInfo.java | 3 +--
regression-test/data/variant_p0/mv/multi_slot.out | 3 +++
.../test_create_mv_complex_type.groovy | 8 +++---
.../rollup_p0/test_materialized_view_array.groovy | 2 +-
.../rollup_p0/test_materialized_view_struct.groovy | 2 +-
.../suites/variant_p0/mv/multi_slot.groovy | 31 ++++++++++++++++++++--
12 files changed, 62 insertions(+), 29 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 7dfcfd15ebe..d3218203bf6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -382,6 +382,18 @@ public abstract class Type {
return false;
}
+ /**
+ * Return true if this type can be as short key
+ */
+ public boolean couldBeShortKey() {
+ return !(isFloatingPointType()
+ || getPrimitiveType() == PrimitiveType.STRING
+ || isJsonbType()
+ || isComplexType()
+ || isObjectStored()
+ || isVariantType());
+ }
+
/**
* The output of this is stored directly in the hive metastore as the
column type.
* The string must match exactly.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index a6f1cae9987..64a747e99e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -795,10 +795,9 @@ public class MaterializedViewHandler extends AlterHandler {
break;
}
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
-
column.setIsKey(true);
if (column.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 68ec8238702..2922595fbce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -476,7 +476,7 @@ public class CreateMaterializedViewStmt extends DdlStmt
implements NotFallbackIn
}
break;
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
if (column.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
@@ -487,7 +487,9 @@ public class CreateMaterializedViewStmt extends DdlStmt
implements NotFallbackIn
column.setIsKey(true);
}
if (theBeginIndexOfValue == 0) {
- throw new AnalysisException("The first column could not be
float or double type, use decimal instead");
+ throw new AnalysisException(
+ "The first column could not be float, double or complex "
+ + "type like array, struct, map, json, variant.");
}
// supply value
for (; theBeginIndexOfValue < mvColumnItemList.size();
theBeginIndexOfValue++) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 5f173b4a243..9232d66050a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -375,16 +375,7 @@ public class CreateTableStmt extends DdlStmt implements
NotFallbackInParser {
}
break;
}
- if (columnDef.getType().isFloatingPointType()) {
- break;
- }
- if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.STRING) {
- break;
- }
- if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.JSONB) {
- break;
- }
- if (columnDef.getType().isComplexType()) {
+ if (!columnDef.getType().couldBeShortKey()) {
break;
}
if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 19865446456..505970e104c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4699,7 +4699,7 @@ public class Env {
}
break;
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
if (column.getDataType() == PrimitiveType.VARCHAR) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index bab4eb7a314..41a0de198ac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -314,10 +314,10 @@ public class CreateMTMVInfo {
}
break;
}
- if (type.isFloatLikeType() || type.isStringType() ||
type.isJsonType()
- || catalogType.isComplexType() || type.isBitmapType()
|| type.isHllType()
- || type.isQuantileStateType() || type.isJsonType() ||
type.isStructType()
- || column.getAggType() != null ||
type.isVariantType()) {
+ if (column.getAggType() != null) {
+ break;
+ }
+ if (!catalogType.couldBeShortKey()) {
break;
}
keys.add(column.getName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 630aba4dd35..c43fe04fb2c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -382,8 +382,7 @@ public class CreateTableInfo {
}
break;
}
- if (type.isFloatLikeType() || type.isStringType()
|| type.isJsonType()
- || catalogType.isComplexType() ||
catalogType.isVariantType()) {
+ if (!catalogType.couldBeShortKey()) {
break;
}
keys.add(column.getName());
diff --git a/regression-test/data/variant_p0/mv/multi_slot.out
b/regression-test/data/variant_p0/mv/multi_slot.out
index e146ef05e56..259214967d3 100644
--- a/regression-test/data/variant_p0/mv/multi_slot.out
+++ b/regression-test/data/variant_p0/mv/multi_slot.out
@@ -26,3 +26,6 @@
456 \N
789 \N
+-- !sql --
+2021-01-01T11:11:11 "http://xxx.xxx.xxx"; 12
+
diff --git
a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
index 1e07d69ba82..2ce17abcc0b 100644
---
a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
+++
b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
@@ -47,7 +47,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_jsonb, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -65,7 +65,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_array, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -83,7 +83,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_map, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -101,7 +101,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_struct, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
diff --git
a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
index dad735a76ce..52b8a32740d 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
@@ -67,7 +67,7 @@ suite("test_materialized_view_array", "rollup") {
create_test_table.call(tableName)
test {
sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from
${tableName}"
- exception "errCode = 2, detailMessage = The ARRAY column[`mv_k2`
array<smallint> NULL] not support to create materialized view"
+ exception "errCode = 2, detailMessage = The first column could not
be float, double or complex type like array, struct, map, json, variant"
}
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
index 61e8415cacc..2a05b895384 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
@@ -59,7 +59,7 @@ suite("test_materialized_view_struct", "rollup") {
create_test_table.call(tableName)
test {
sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from
${tableName}"
- exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2`
struct<f1:smallint> NULL] not support to create materialized view"
+ exception "errCode = 2, detailMessage = The first column could not
be float, double or complex type like array, struct, map, json, variant."
}
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy
b/regression-test/suites/variant_p0/mv/multi_slot.groovy
index fd1727c571d..1490c30e587 100644
--- a/regression-test/suites/variant_p0/mv/multi_slot.groovy
+++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy
@@ -50,8 +50,35 @@ suite ("multi_slot") {
order_qt_select_star "select abs(cast(v['k1'] as int))+cast(v['k2'] as
int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot;"
order_qt_select_star "select * from multi_slot order by cast(v['k1'] as
int);"
- // TODO fix and remove enable_rewrite_element_at_to_slot
- order_qt_select_star "select
/*+SET_VAR(enable_rewrite_element_at_to_slot=false) */ abs(cast(v['k4']['k44']
as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from
multi_slot group by abs(cast(v['k4']['k44'] as int))"
+ order_qt_select_star "select abs(cast(v['k4']['k44'] as int)),
sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group
by abs(cast(v['k4']['k44'] as int))"
+
+ sql "drop table if exists test_mv"
+ sql """
+ CREATE TABLE `test_mv` (
+ `handle_time` datetime NOT NULL ,
+ `client_request` variant NULL,
+ `status` int NULL
+ )
+ DISTRIBUTED BY HASH(`handle_time`)
+ BUCKETS 10 PROPERTIES (
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V1",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "replication_num" = "1"
+ );
+ """
+ sql """insert into test_mv values ('2021-01-01 11:11:11', '{"url" :
"http://xxx.xxx.xxx"}', 12)"""
+ createMV("create materialized view mv_1 as select `handle_time`,
`client_request`['url'] as `uri`, `status` from test_mv")
+ qt_sql "select `handle_time`, `client_request`['url'] as `uri`, `status`
from test_mv"
+ test {
+ sql "create materialized view mv_x as select `client_request`['url']
as `uri`, `status` from test_mv"
+ exception("The first column could not be float, double or complex type
like array, struct, map, json, variant.")
+ }
+
// def retry_times = 60
// for (def i = 0; i < retry_times; ++i) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
