This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2623282de7f branch-3.1: [Enhancement] reject schema change on hidden
columns #53376 (#55385)
2623282de7f is described below
commit 2623282de7f5e2ff7b8d7d351f06adbfd0a87b38
Author: csding <[email protected]>
AuthorDate: Thu Sep 4 09:53:08 2025 +0800
branch-3.1: [Enhancement] reject schema change on hidden columns #53376
(#55385)
picked from #53376
---
.../java/org/apache/doris/analysis/ColumnDef.java | 56 +++++++++++++---
.../main/java/org/apache/doris/catalog/Column.java | 1 +
.../java/org/apache/doris/common/FeNameFormat.java | 24 +++++--
.../plans/commands/info/ColumnDefinition.java | 78 ++++++++++++++--------
4 files changed, 115 insertions(+), 44 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 7d0d24aa4e5..91812858846 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -199,6 +199,8 @@ public class ColumnDef {
private int clusterKeyId = -1;
private Optional<GeneratedColumnInfo> generatedColumnInfo =
Optional.empty();
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
+ // if add hidden column, must set enableAddHiddenColumn true
+ private boolean enableAddHiddenColumn = false;
public ColumnDef(String name, TypeDef typeDef) {
@@ -262,6 +264,10 @@ public class ColumnDef {
this.visible = visible;
}
+ public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
+ this.enableAddHiddenColumn = enableAddHiddenColumn;
+ }
+
public ColumnDef(String name, TypeDef typeDef, boolean isKey,
ColumnNullableType nullableType, String comment,
Optional<GeneratedColumnInfo> generatedColumnInfo) {
this(name, typeDef, isKey, null, nullableType, -1,
DefaultValue.NOT_SET,
@@ -269,50 +275,72 @@ public class ColumnDef {
}
public static ColumnDef newDeleteSignColumnDef() {
- return new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false, null,
- ColumnNullableType.NOT_NULLABLE, -1, new
ColumnDef.DefaultValue(true, "0"),
+ ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false, null,
+ ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true,
"0"),
"doris delete flag hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newDeleteSignColumnDef(AggregateType
aggregateType) {
- return new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false, aggregateType,
- ColumnNullableType.NOT_NULLABLE, -1, new
ColumnDef.DefaultValue(true, "0"),
+ ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false,
+ aggregateType,
+ ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true,
"0"),
"doris delete flag hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newSequenceColumnDef(Type type) {
- return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
null, ColumnNullableType.NULLABLE, -1,
+ ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new
TypeDef(type),
+ false, null, ColumnNullableType.NULLABLE, -1,
DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden
column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newSequenceColumnDef(Type type, AggregateType
aggregateType) {
- return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
aggregateType, ColumnNullableType.NULLABLE,
+ ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new
TypeDef(type),
+ false, aggregateType, ColumnNullableType.NULLABLE,
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden
column", false,
Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
- return new ColumnDef(Column.ROW_STORE_COL,
TypeDef.create(PrimitiveType.STRING), false, aggregateType,
+ ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL,
TypeDef.create(PrimitiveType.STRING),
+ false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new
ColumnDef.DefaultValue(true, ""),
"doris row store hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newVersionColumnDef() {
- return new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT), false, null,
+ ColumnDef columnDef = new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT), false, null,
ColumnNullableType.NOT_NULLABLE, -1, new
ColumnDef.DefaultValue(true, "0"),
"doris version hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
- return new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT), false, aggregateType,
+ ColumnDef columnDef = new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT),
+ false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new
ColumnDef.DefaultValue(true, "0"),
"doris version hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newSkipBitmapColumnDef(AggregateType
aggregateType) {
- return new ColumnDef(Column.SKIP_BITMAP_COL,
TypeDef.create(PrimitiveType.BITMAP), false, aggregateType,
+ ColumnDef columnDef = new ColumnDef(Column.SKIP_BITMAP_COL,
TypeDef.create(PrimitiveType.BITMAP),
+ false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1,
DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE,
"doris skip bitmap hidden column", false, Optional.empty());
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public boolean isAllowNull() {
@@ -375,7 +403,13 @@ public class ColumnDef {
if (name == null || typeDef == null) {
throw new AnalysisException("No column name or column type in
column definition.");
}
- FeNameFormat.checkColumnName(name);
+
+ if (enableAddHiddenColumn) {
+ FeNameFormat.checkColumnNameBypassHiddenColumn(name);
+ } else {
+ FeNameFormat.checkColumnName(name);
+ }
+
FeNameFormat.checkColumnCommentLength(comment);
typeDef.analyze(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 47cca3b90bb..ab6b423d976 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -62,6 +62,7 @@ import java.util.Set;
*/
public class Column implements GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(Column.class);
+ public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
// NOTE: you should name hidden column start with '__DORIS_'
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
index a7165c3d815..c6c751a88c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
@@ -20,6 +20,7 @@ package org.apache.doris.common;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.ResourceTypeEnum;
+import org.apache.doris.catalog.Column;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.Role;
import org.apache.doris.mysql.privilege.RoleManager;
@@ -93,18 +94,29 @@ public class FeNameFormat {
}
public static void checkColumnName(String columnName) throws
AnalysisException {
+ // if need check another column name prefix, add in
`checkColumnNameBypassHiddenColumn`
+ checkColumnNameBypassHiddenColumn(columnName);
+ checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
+ }
+
+ public static void checkColumnNameBypassHiddenColumn(String columnName)
throws AnalysisException {
if (Strings.isNullOrEmpty(columnName) ||
!columnName.matches(getColumnNameRegex())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, getColumnNameRegex());
}
- if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
-
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
- columnName, getColumnNameRegex());
+ checkColumnNamePrefix(columnName,
SchemaChangeHandler.SHADOW_NAME_PREFIX);
+ checkColumnNamePrefix(columnName,
CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
+ checkColumnNamePrefix(columnName,
CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
+ }
+
+ private static void checkColumnNamePrefix(String columnName, String
prefix) throws AnalysisException {
+ int prefixLength = prefix.length();
+ if (columnName.length() < prefixLength) {
+ return;
}
- if
(columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
- ||
columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX))
{
+ if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
throw new AnalysisException(
- "Incorrect column name " + columnName + ", column name
can't start with 'mv_'/'mva_'");
+ "Incorrect column name " + columnName + ", column name can't
start with '" + prefix + "'");
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 367bdff18fc..77c712de990 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -69,6 +69,8 @@ public class ColumnDefinition {
private int clusterKeyId = -1;
private Optional<GeneratedColumnDesc> generatedColumnDesc =
Optional.empty();
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
+ // if add hidden column, must set enableAddHiddenColumn true
+ private boolean enableAddHiddenColumn = false;
public ColumnDefinition(String name, DataType type, boolean isKey,
AggregateType aggType, boolean isNullable,
Optional<DefaultValue> defaultValue, String comment) {
@@ -232,7 +234,14 @@ public class ColumnDefinition {
public void validate(boolean isOlap, Set<String> keysSet, Set<String>
clusterKeySet, boolean isEnableMergeOnWrite,
KeysType keysType) {
try {
- FeNameFormat.checkColumnName(name);
+ // if enableAddHiddenColumn is true, can add hidden column.
+ // So does not check if the column name starts with __DORIS_
+ if (enableAddHiddenColumn) {
+ FeNameFormat.checkColumnNameBypassHiddenColumn(name);
+ } else {
+ FeNameFormat.checkColumnName(name);
+ }
+
FeNameFormat.checkColumnCommentLength(comment);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e);
@@ -689,43 +698,54 @@ public class ColumnDefinition {
return column;
}
- // hidden column
- public static ColumnDefinition newDeleteSignColumnDefinition() {
- return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE,
false, null, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris delete flag hidden column", false);
- }
-
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris delete flag hidden column", false);
- }
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue(DefaultValue.ZERO_NUMBER)),
+ "doris delete flag hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
- public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
- return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
true,
- Optional.empty(), "sequence column hidden column", false);
- }
-
- public static ColumnDefinition newSequenceColumnDefinition(DataType type,
AggregateType aggregateType) {
- return new ColumnDefinition(Column.SEQUENCE_COL, type, false,
aggregateType, true,
- Optional.empty(), "sequence column hidden column", false);
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue("")), "doris row store hidden
column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue("")),
+ "doris row store hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newVersionColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris version hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue(DefaultValue.ZERO_NUMBER)),
+ "doris version hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
- // used in CreateTableInfo.validate(), specify the default value as
DefaultValue.NULL_DEFAULT_VALUE
- // becasue ColumnDefinition.validate() will check that bitmap type column
don't set default value
- // and then set the default value of that column to bitmap_empty()
+ /**
+ * used in CreateTableInfo.validate(), specify the default value as
DefaultValue.NULL_DEFAULT_VALUE
+ * becasue ColumnDefinition.validate() will check that bitmap type column
don't set default value
+ * and then set the default value of that column to bitmap_empty()
+ */
public static ColumnDefinition newSkipBitmapColumnDef(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.SKIP_BITMAP_COL,
BitmapType.INSTANCE, false, aggregateType, false,
- Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris
skip bitmap hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false,
+ aggregateType, false,
Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE),
+ "doris skip bitmap hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
@@ -740,6 +760,10 @@ public class ColumnDefinition {
generatedColumnsThatReferToThis.addAll(list);
}
+ public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
+ this.enableAddHiddenColumn = enableAddHiddenColumn;
+ }
+
private void validateGeneratedColumnInfo() {
// for generated column
if (generatedColumnDesc.isPresent()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
