This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9d9a79c9d09 [fix](variant) Fix metadata compatibility for FE (#53748)
9d9a79c9d09 is described below
commit 9d9a79c9d09c5db980575430d54b39bf783ad9f5
Author: Sun Chenyang <[email protected]>
AuthorDate: Wed Jul 23 14:16:30 2025 +0800
[fix](variant) Fix metadata compatibility for FE (#53748)
### What problem does this PR solve?
- In version 3.1, we used 'VariantType', which inherits from
'ScalarType', as the metadata storage for newly created variants,
differing from the previously used 'ScalarType'.
- To ensure compatibility with previous metadata, convert 'VariantType'
to 'ScalarType' before use.
---
.../java/org/apache/doris/catalog/ScalarType.java | 14 ++++++++
.../org/apache/doris/catalog/VariantField.java | 16 ++++-----
.../main/java/org/apache/doris/catalog/Column.java | 9 +++--
.../apache/doris/nereids/types/VariantField.java | 17 +++++++--
.../org/apache/doris/persist/ScalarTypeTest.java | 40 +++++++++++++++++++++
.../data/variant_p0/predefine/sql/q01.out | Bin 2198 -> 1830 bytes
.../predefine/test_all_prdefine_type_to_sparse.out | Bin 412757 -> 371285 bytes
7 files changed, 81 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index b8b48d390d3..53790fbaa4f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -1221,4 +1221,18 @@ public class ScalarType extends Type {
result = 31 * result + scale;
return result;
}
+
+ public int getVariantMaxSubcolumnsCount() {
+ if (this instanceof VariantType) {
+ return ((VariantType) this).getVariantMaxSubcolumnsCount();
+ }
+ return 0;
+ }
+
+ public boolean getVariantEnableTypedPathsToSparse() {
+ if (this instanceof VariantType) {
+ return ((VariantType) this).getEnableTypedPathsToSparse();
+ }
+ return false;
+ }
}
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantField.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantField.java
index f300916d77d..52d5e60b743 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantField.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantField.java
@@ -67,15 +67,15 @@ public class VariantField {
}
public String toSql(int depth) {
- String typeSql;
- if (depth < Type.MAX_NESTING_DEPTH) {
- typeSql = type.toSql(depth + 1);
- } else {
- typeSql = "...";
+ StringBuilder sb = new StringBuilder();
+ if (patternType == TPatternType.MATCH_NAME) {
+ sb.append(patternType.toString()).append(" ");
}
- StringBuilder sb = new StringBuilder(patternType.toString() + " '");
- if (type != null) {
- sb.append(pattern).append("':").append(typeSql);
+
+ sb.append("'").append(pattern).append("'");
+ sb.append(":").append(type.toSql(depth + 1));
+ if (!comment.isEmpty()) {
+ sb.append(" COMMENT '").append(comment).append("'");
}
return sb.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 51f23264841..fc225412f21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -350,7 +350,7 @@ public class Column implements GsonPostProcessable {
c.setIsAllowNull(field.getContainsNull());
column.addChildrenColumn(c);
}
- } else if (type.isVariantType()) {
+ } else if (type.isVariantType() && type instanceof VariantType) {
// variant may contain predefined structured fields
ArrayList<VariantField> fields = ((VariantType)
type).getPredefinedFields();
for (VariantField field : fields) {
@@ -520,11 +520,11 @@ public class Column implements GsonPostProcessable {
}
public int getVariantMaxSubcolumnsCount() {
- return type.isVariantType() ? ((VariantType)
type).getVariantMaxSubcolumnsCount() : -1;
+ return type.isVariantType() ? ((ScalarType)
type).getVariantMaxSubcolumnsCount() : -1;
}
public boolean getVariantEnableTypedPathsToSparse() {
- return type.isVariantType() ? ((VariantType)
type).getEnableTypedPathsToSparse() : false;
+ return type.isVariantType() ? ((ScalarType)
type).getVariantEnableTypedPathsToSparse() : false;
}
public AggregateType getAggregationType() {
@@ -890,8 +890,7 @@ public class Column implements GsonPostProcessable {
builder.addChildrenColumns(c.toPb(Sets.newHashSet(),
Lists.newArrayList()));
}
} else if (this.type.isVariantType()) {
- VariantType variantType = (VariantType) this.getType();
-
builder.setVariantMaxSubcolumnsCount(variantType.getVariantMaxSubcolumnsCount());
+
builder.setVariantMaxSubcolumnsCount(this.getVariantMaxSubcolumnsCount());
builder.setVariantEnableTypedPathsToSparse(this.getVariantEnableTypedPathsToSparse());
// variant may contain predefined structured fields
addChildren(builder);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java
index 232e7416f3e..03909d8aaf3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantField.java
@@ -72,9 +72,22 @@ public class VariantField {
pattern, dataType.toCatalogDataType(), comment, patternType);
}
+ /**
+ * Convert this VariantField to SQL string representation.
+ * @return SQL string representation of this VariantField
+ */
public String toSql() {
- return pattern + ":" + dataType.toSql()
- + (comment.isEmpty() ? "" : " COMMENT " + comment);
+ StringBuilder sb = new StringBuilder();
+ if (patternType == TPatternType.MATCH_NAME) {
+ sb.append(patternType.toString()).append(" ");
+ }
+
+ sb.append("'").append(pattern).append("'");
+ sb.append(":").append(dataType.toSql());
+ if (!comment.isEmpty()) {
+ sb.append(" COMMENT '").append(comment).append("'");
+ }
+ return sb.toString();
}
public VariantField conversion() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/ScalarTypeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/ScalarTypeTest.java
new file mode 100644
index 00000000000..b1f2039e356
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ScalarTypeTest.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.VariantType;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ScalarTypeTest {
+ @Test
+ public void testScalarType() {
+ ScalarType scalarType = new ScalarType(PrimitiveType.VARIANT);
+ String json = GsonUtils.GSON.toJson(scalarType);
+ System.out.println(json);
+ ScalarType scalarType2 = GsonUtils.GSON.fromJson(json,
ScalarType.class);
+ Assert.assertFalse(scalarType2 instanceof VariantType);
+ Assert.assertEquals(scalarType.getPrimitiveType(),
scalarType2.getPrimitiveType());
+ Assert.assertEquals(scalarType.getVariantMaxSubcolumnsCount(), 0);
+ Assert.assertEquals(scalarType.getVariantEnableTypedPathsToSparse(),
false);
+ }
+}
diff --git a/regression-test/data/variant_p0/predefine/sql/q01.out
b/regression-test/data/variant_p0/predefine/sql/q01.out
index 3514f410e21..2089695abfc 100644
Binary files a/regression-test/data/variant_p0/predefine/sql/q01.out and
b/regression-test/data/variant_p0/predefine/sql/q01.out differ
diff --git
a/regression-test/data/variant_p0/predefine/test_all_prdefine_type_to_sparse.out
b/regression-test/data/variant_p0/predefine/test_all_prdefine_type_to_sparse.out
index 414fd98fc24..8c0bc1d8e8d 100644
Binary files
a/regression-test/data/variant_p0/predefine/test_all_prdefine_type_to_sparse.out
and
b/regression-test/data/variant_p0/predefine/test_all_prdefine_type_to_sparse.out
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
