This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new c47e61b3ae [#9756] feat(catalogs): Support alternation operations for
ClickHouse tables (#9826)
c47e61b3ae is described below
commit c47e61b3aed896168d7ac2f028f01001004fae8d
Author: Qi Yu <[email protected]>
AuthorDate: Thu Feb 5 21:52:32 2026 +0800
[#9756] feat(catalogs): Support alternation operations for ClickHouse
tables (#9826)
### What changes were proposed in this pull request?
Support alternation operations for ClickHouse tables.
### Why are the changes needed?
It's a feature.
Fix: #9756
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
UTs
---------
Co-authored-by: flaming-archer <[email protected]>
---
.../operations/ClickHouseTableOperations.java | 358 ++++++++++++-
.../operations/TestClickHouseTableOperations.java | 596 +++++++++++++++++++++
.../jdbc/operation/JdbcTableOperations.java | 4 +
3 files changed, 955 insertions(+), 3 deletions(-)
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
index 8356c84291..0fc69d8279 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
@@ -22,6 +22,7 @@ import static
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesM
import static
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata.ENGINE_PROPERTY_ENTRY;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -29,20 +30,27 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.StringIdentifier;
import
org.apache.gravitino.catalog.clickhouse.ClickHouseConstants.TableConstants;
import
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata;
import org.apache.gravitino.catalog.jdbc.JdbcColumn;
+import org.apache.gravitino.catalog.jdbc.JdbcTable;
import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations;
+import org.apache.gravitino.catalog.jdbc.utils.JdbcConnectorUtils;
import org.apache.gravitino.exceptions.NoSuchTableException;
+import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
@@ -55,7 +63,6 @@ import org.apache.gravitino.rel.indexes.Indexes;
public class ClickHouseTableOperations extends JdbcTableOperations {
- @SuppressWarnings("unused")
private static final String CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG =
"Clickhouse does not support nested column names.";
@@ -283,6 +290,23 @@ public class ClickHouseTableOperations extends
JdbcTableOperations {
return "YES".equalsIgnoreCase(resultSet.getString("IS_AUTOINCREMENT"));
}
+ @Override
+ public void alterTable(String databaseName, String tableName, TableChange...
changes)
+ throws NoSuchTableException {
+ LOG.info("Attempting to alter table {} from database {}", tableName,
databaseName);
+ try (Connection connection = getConnection(databaseName)) {
+ String sql = generateAlterTableSql(databaseName, tableName, changes);
+ if (StringUtils.isEmpty(sql)) {
+ LOG.info("No changes to alter table {} from database {}", tableName,
databaseName);
+ return;
+ }
+ JdbcConnectorUtils.executeUpdate(connection, sql);
+ LOG.info("Alter table {} from database {}", tableName, databaseName);
+ } catch (final SQLException se) {
+ throw this.exceptionMapper.toGravitinoException(se);
+ }
+ }
+
@Override
protected Map<String, String> getTableProperties(Connection connection,
String tableName)
throws SQLException {
@@ -327,8 +351,336 @@ public class ClickHouseTableOperations extends
JdbcTableOperations {
@Override
protected String generateAlterTableSql(
String databaseName, String tableName, TableChange... changes) {
- throw new UnsupportedOperationException(
- "ClickHouseTableOperations.generateAlterTableSql is not implemented
yet.");
+ // Not all operations require the original table information, so lazy
loading is used here
+ JdbcTable lazyLoadTable = null;
+ TableChange.UpdateComment updateComment = null;
+ List<TableChange.SetProperty> setProperties = new ArrayList<>();
+ List<String> alterSql = new ArrayList<>();
+
+ for (TableChange change : changes) {
+ if (change instanceof TableChange.UpdateComment) {
+ updateComment = (TableChange.UpdateComment) change;
+
+ } else if (change instanceof TableChange.SetProperty setProperty) {
+ // The set attribute needs to be added at the end.
+ setProperties.add(setProperty);
+
+ } else if (change instanceof TableChange.RemoveProperty) {
+ // Clickhouse does not support deleting table attributes, it can be
replaced by Set Property
+ throw new UnsupportedOperationException(
+ "Remove property for ClickHouse is not supported yet");
+
+ } else if (change instanceof TableChange.AddColumn addColumn) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(addColumnFieldDefinition(addColumn));
+
+ } else if (change instanceof TableChange.RenameColumn renameColumn) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(renameColumnFieldDefinition(renameColumn));
+
+ } else if (change instanceof TableChange.UpdateColumnDefaultValue
updateColumnDefaultValue) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(
+ updateColumnDefaultValueFieldDefinition(updateColumnDefaultValue,
lazyLoadTable));
+
+ } else if (change instanceof TableChange.UpdateColumnType
updateColumnType) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(updateColumnTypeFieldDefinition(updateColumnType,
lazyLoadTable));
+
+ } else if (change instanceof TableChange.UpdateColumnComment
updateColumnComment) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(updateColumnCommentFieldDefinition(updateColumnComment,
lazyLoadTable));
+
+ } else if (change instanceof TableChange.UpdateColumnPosition
updateColumnPosition) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(updateColumnPositionFieldDefinition(updateColumnPosition,
lazyLoadTable));
+
+ } else if (change instanceof TableChange.DeleteColumn deleteColumn) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ String deleteColSql = deleteColumnFieldDefinition(deleteColumn,
lazyLoadTable);
+
+ if (StringUtils.isNotEmpty(deleteColSql)) {
+ alterSql.add(deleteColSql);
+ }
+
+ } else if (change instanceof TableChange.UpdateColumnNullability) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(
+ updateColumnNullabilityDefinition(
+ (TableChange.UpdateColumnNullability) change, lazyLoadTable));
+
+ } else if (change instanceof TableChange.DeleteIndex) {
+ lazyLoadTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ alterSql.add(deleteIndexDefinition(lazyLoadTable,
(TableChange.DeleteIndex) change));
+
+ } else if (change instanceof TableChange.UpdateColumnAutoIncrement) {
+ // Auto increment functionality was added in ClickHouse 25.1. Since
this PR is based on
+ // 23.x, we throw unsupported operation here.
+ throw new UnsupportedOperationException(
+ "ClickHouse auto increment is not supported in this version.");
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported table change type: " + change.getClass().getName());
+ }
+ }
+
+ // Last modified comment
+ if (null != updateComment) {
+ String newComment = updateComment.getNewComment();
+ if (null == StringIdentifier.fromComment(newComment)) {
+ // Detect and add Gravitino id.
+ JdbcTable jdbcTable = getOrCreateTable(databaseName, tableName,
lazyLoadTable);
+ StringIdentifier identifier =
StringIdentifier.fromComment(jdbcTable.comment());
+ if (null != identifier) {
+ newComment = StringIdentifier.addToComment(identifier, newComment);
+ }
+ }
+ String escapedComment = newComment.replace("'", "''");
+ alterSql.add(" MODIFY COMMENT '%s'".formatted(escapedComment));
+ }
+
+ if (!setProperties.isEmpty()) {
+ alterSql.add(generateAlterTableProperties(setProperties));
+ }
+
+ // Remove all empty SQL statements
+ List<String> nonEmptySQLs =
+
alterSql.stream().filter(StringUtils::isNotEmpty).collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(nonEmptySQLs)) {
+ return "";
+ }
+
+ // Return the generated SQL statement
+ String result =
+ "ALTER TABLE %s \n%s;"
+ .formatted(quoteIdentifier(tableName), String.join(",\n",
nonEmptySQLs));
+ LOG.info("Generated alter table:{} sql: {}", databaseName + "." +
tableName, result);
+ return result;
+ }
+
+ @VisibleForTesting
+ private String deleteIndexDefinition(
+ JdbcTable lazyLoadTable, TableChange.DeleteIndex deleteIndex) {
+ boolean indexExists =
+ Arrays.stream(lazyLoadTable.index())
+ .anyMatch(index -> index.name().equals(deleteIndex.getName()));
+
+ // Index does not exist
+ if (!indexExists) {
+ // If ifExists is true, return empty string to skip the operation
+ if (deleteIndex.isIfExists()) {
+ return "";
+ } else {
+ throw new IllegalArgumentException(
+ "Index '%s' does not exist".formatted(deleteIndex.getName()));
+ }
+ }
+
+ return "DROP INDEX %s ".formatted(quoteIdentifier(deleteIndex.getName()));
+ }
+
+ private String updateColumnNullabilityDefinition(
+ TableChange.UpdateColumnNullability change, JdbcTable table) {
+ validateUpdateColumnNullable(change, table);
+
+ String col = change.fieldName()[0];
+ JdbcColumn column = getJdbcColumnFromTable(table, col);
+ JdbcColumn updateColumn =
+ JdbcColumn.builder()
+ .withName(col)
+ .withDefaultValue(column.defaultValue())
+ .withNullable(change.nullable())
+ .withType(column.dataType())
+ .withComment(column.comment())
+ .withAutoIncrement(column.autoIncrement())
+ .build();
+
+ return "%s %s %s"
+ .formatted(
+ MODIFY_COLUMN,
+ quoteIdentifier(col),
+ appendColumnDefinition(updateColumn, new StringBuilder()));
+ }
+
+ private String generateAlterTableProperties(List<TableChange.SetProperty>
setProperties) {
+ if (CollectionUtils.isNotEmpty(setProperties)) {
+ throw new UnsupportedOperationException(
+ "Alter table properties in ClickHouse is not supported");
+ }
+
+ return "";
+ }
+
+ private String updateColumnCommentFieldDefinition(
+ TableChange.UpdateColumnComment updateColumnComment, JdbcTable
jdbcTable) {
+ String newComment = updateColumnComment.getNewComment();
+ if (updateColumnComment.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = updateColumnComment.fieldName()[0];
+ JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col);
+ JdbcColumn updateColumn =
+ JdbcColumn.builder()
+ .withName(col)
+ .withDefaultValue(column.defaultValue())
+ .withNullable(column.nullable())
+ .withType(column.dataType())
+ .withComment(newComment)
+ .withAutoIncrement(column.autoIncrement())
+ .build();
+
+ return "%s %s %s"
+ .formatted(
+ MODIFY_COLUMN,
+ quoteIdentifier(col),
+ appendColumnDefinition(updateColumn, new StringBuilder()));
+ }
+
+ private String addColumnFieldDefinition(TableChange.AddColumn addColumn) {
+ String dataType = typeConverter.fromGravitino(addColumn.getDataType());
+ if (addColumn.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = addColumn.fieldName()[0];
+ StringBuilder columnDefinition = new StringBuilder();
+ // [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after |
FIRST]
+ if (addColumn.isNullable()) {
+ columnDefinition.append(
+ "ADD COLUMN %s Nullable(%s) ".formatted(quoteIdentifier(col),
dataType));
+ } else {
+ columnDefinition.append("ADD COLUMN %s %s
".formatted(quoteIdentifier(col), dataType));
+ }
+
+ if (addColumn.isAutoIncrement()) {
+ throw new UnsupportedOperationException(
+ "ClickHouse does not support adding auto increment column");
+ }
+
+ // Append default value if available
+ if (!Column.DEFAULT_VALUE_NOT_SET.equals(addColumn.getDefaultValue())) {
+ columnDefinition.append(
+ "DEFAULT %s "
+
.formatted(columnDefaultValueConverter.fromGravitino(addColumn.getDefaultValue())));
+ }
+
+ // Append comment if available after default value
+ if (StringUtils.isNotEmpty(addColumn.getComment())) {
+ String escapedComment = StringUtils.replace(addColumn.getComment(), "'",
"''");
+ columnDefinition.append(" COMMENT '%s' ".formatted(escapedComment));
+ }
+
+ // Append position if available
+ if (addColumn.getPosition() instanceof TableChange.First) {
+ columnDefinition.append(" FIRST ");
+ } else if (addColumn.getPosition() instanceof TableChange.After
afterPosition) {
+ columnDefinition.append(" AFTER %s
".formatted(quoteIdentifier(afterPosition.getColumn())));
+ }
+
+ return columnDefinition.toString();
+ }
+
+ private String renameColumnFieldDefinition(TableChange.RenameColumn
renameColumn) {
+ if (renameColumn.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String oldColumnName = renameColumn.fieldName()[0];
+ String newColumnName = renameColumn.getNewName();
+
+ return "RENAME COLUMN %s TO %s"
+ .formatted(quoteIdentifier(oldColumnName),
quoteIdentifier(newColumnName));
+ }
+
+ private String updateColumnPositionFieldDefinition(
+ TableChange.UpdateColumnPosition updateColumnPosition, JdbcTable
jdbcTable) {
+ if (updateColumnPosition.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = updateColumnPosition.fieldName()[0];
+ JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col);
+
+ StringBuilder columnDefinition = new StringBuilder();
+ columnDefinition.append(" %s %s ".formatted(MODIFY_COLUMN,
quoteIdentifier(col)));
+ appendColumnDefinition(column, columnDefinition);
+
+ if (updateColumnPosition.getPosition() instanceof TableChange.First) {
+ columnDefinition.append(" FIRST ");
+ } else if (updateColumnPosition.getPosition() instanceof TableChange.After
afterPosition) {
+ columnDefinition.append(
+ " %s %s ".formatted(AFTER,
quoteIdentifier(afterPosition.getColumn())));
+ } else {
+ Arrays.stream(jdbcTable.columns())
+ .reduce((column1, column2) -> column2)
+ .map(Column::name)
+ .ifPresent(s -> columnDefinition.append(" %s %s ".formatted(AFTER,
quoteIdentifier(s))));
+ }
+ return columnDefinition.toString();
+ }
+
+ private String deleteColumnFieldDefinition(
+ TableChange.DeleteColumn deleteColumn, JdbcTable jdbcTable) {
+ if (deleteColumn.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = deleteColumn.fieldName()[0];
+ boolean colExists = columnExists(jdbcTable, col);
+ if (!colExists) {
+ if (BooleanUtils.isTrue(deleteColumn.getIfExists())) {
+ return "";
+ } else {
+ throw new IllegalArgumentException("Delete column '%s' does not
exist.".formatted(col));
+ }
+ }
+
+ return "DROP COLUMN %s".formatted(quoteIdentifier(col));
+ }
+
+ private String updateColumnDefaultValueFieldDefinition(
+ TableChange.UpdateColumnDefaultValue updateColumnDefaultValue, JdbcTable
jdbcTable) {
+ if (updateColumnDefaultValue.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = updateColumnDefaultValue.fieldName()[0];
+ JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col);
+ StringBuilder sqlBuilder = new StringBuilder(MODIFY_COLUMN +
quoteIdentifier(col));
+ JdbcColumn newColumn =
+ JdbcColumn.builder()
+ .withName(col)
+ .withType(column.dataType())
+ .withNullable(column.nullable())
+ .withComment(column.comment())
+ .withDefaultValue(updateColumnDefaultValue.getNewDefaultValue())
+ .build();
+
+ return appendColumnDefinition(newColumn, sqlBuilder).toString();
+ }
+
+ private String updateColumnTypeFieldDefinition(
+ TableChange.UpdateColumnType updateColumnType, JdbcTable jdbcTable) {
+ if (updateColumnType.fieldName().length > 1) {
+ throw new
UnsupportedOperationException(CLICKHOUSE_NOT_SUPPORT_NESTED_COLUMN_MSG);
+ }
+
+ String col = updateColumnType.fieldName()[0];
+ JdbcColumn column = getJdbcColumnFromTable(jdbcTable, col);
+ StringBuilder sqlBuilder =
+ new StringBuilder("%s %s ".formatted(MODIFY_COLUMN,
quoteIdentifier(col)));
+
+ JdbcColumn newColumn =
+ JdbcColumn.builder()
+ .withName(col)
+ .withType(updateColumnType.getNewDataType())
+ .withComment(column.comment())
+ .withDefaultValue(column.defaultValue())
+ .withNullable(column.nullable())
+ .withAutoIncrement(column.autoIncrement())
+ .build();
+ return appendColumnDefinition(newColumn, sqlBuilder).toString();
}
private StringBuilder appendColumnDefinition(JdbcColumn column,
StringBuilder sqlBuilder) {
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
index b5bc0af80f..1bc181fabf 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.catalog.clickhouse.ClickHouseConstants;
import
org.apache.gravitino.catalog.clickhouse.ClickHouseConstants.TableConstants;
import
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata;
@@ -39,6 +40,9 @@ import
org.apache.gravitino.catalog.clickhouse.converter.ClickHouseExceptionConv
import
org.apache.gravitino.catalog.clickhouse.converter.ClickHouseTypeConverter;
import org.apache.gravitino.catalog.jdbc.JdbcColumn;
import org.apache.gravitino.catalog.jdbc.JdbcTable;
+import org.apache.gravitino.exceptions.GravitinoRuntimeException;
+import org.apache.gravitino.rel.Column;
+import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.NamedReference;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
import org.apache.gravitino.rel.expressions.literals.Literals;
@@ -59,6 +63,377 @@ import org.junit.jupiter.api.Test;
public class TestClickHouseTableOperations extends TestClickHouse {
private static final Type STRING = Types.StringType.get();
private static final Type INT = Types.IntegerType.get();
+ private static final Type LONG = Types.LongType.get();
+
+ @Test
+ public void testCreateAndAlterTable() {
+ String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table";
+ String tableComment = "test_comment";
+ List<JdbcColumn> columns = new ArrayList<>();
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_1")
+ .withType(STRING)
+ .withComment("test_comment")
+ .withNullable(true)
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_2")
+ .withType(INT)
+ .withNullable(false)
+ .withComment("set primary key")
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_3")
+ .withType(INT)
+ .withNullable(true)
+ .withDefaultValue(Literals.NULL)
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_4")
+ .withType(STRING)
+ .withDefaultValue(Literals.of("hello world", STRING))
+ .withNullable(false)
+ .build());
+ Map<String, String> properties = new HashMap<>();
+
+ Index[] indexes = new Index[] {};
+ // create table
+ TABLE_OPERATIONS.create(
+ TEST_DB_NAME.toString(),
+ tableName,
+ columns.toArray(new JdbcColumn[0]),
+ tableComment,
+ properties,
+ null,
+ Distributions.NONE,
+ indexes,
+ getSortOrders("col_2"));
+
+ // list table
+ List<String> tables = TABLE_OPERATIONS.listTables(TEST_DB_NAME.toString());
+ Assertions.assertTrue(tables.contains(tableName));
+
+ // load table
+ JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, indexes,
Transforms.EMPTY_TRANSFORM, load);
+
+ // rename table
+ String newName = "new_table";
+ Assertions.assertDoesNotThrow(
+ () -> TABLE_OPERATIONS.rename(TEST_DB_NAME.toString(), tableName,
newName));
+ Assertions.assertDoesNotThrow(() ->
TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName));
+
+ // alter table add column
+ JdbcColumn newColumn =
+ JdbcColumn.builder()
+ .withName("col_5")
+ .withType(STRING)
+ .withComment("new_add")
+ .withNullable(false) //
+ .build();
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ newName,
+ TableChange.addColumn(
+ new String[] {newColumn.name()},
+ newColumn.dataType(),
+ newColumn.comment(),
+ TableChange.ColumnPosition.after("col_1"),
+ newColumn.nullable(),
+ newColumn.autoIncrement(),
+ newColumn.defaultValue()));
+ load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName);
+ List<JdbcColumn> alterColumns =
+ new ArrayList<JdbcColumn>() {
+ {
+ add(columns.get(0));
+ add(newColumn);
+ add(columns.get(1));
+ add(columns.get(2));
+ add(columns.get(3));
+ }
+ };
+ assertionsTableInfo(
+ newName, tableComment, alterColumns, properties, indexes,
Transforms.EMPTY_TRANSFORM, load);
+
+ // Detect unsupported properties
+ TableChange setProperty = TableChange.setProperty(CLICKHOUSE_ENGINE_KEY,
"ABC");
+ UnsupportedOperationException gravitinoRuntimeException =
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(),
newName, setProperty));
+ Assertions.assertTrue(
+ StringUtils.contains(
+ gravitinoRuntimeException.getMessage(),
+ "Alter table properties in ClickHouse is not supported"));
+
+ // delete column
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ newName,
+ TableChange.deleteColumn(new String[] {newColumn.name()}, true));
+ load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), newName);
+ assertionsTableInfo(
+ newName, tableComment, columns, properties, indexes,
Transforms.EMPTY_TRANSFORM, load);
+
+ TableChange deleteColumn = TableChange.deleteColumn(new String[]
{newColumn.name()}, false);
+ IllegalArgumentException illegalArgumentException =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(),
newName, deleteColumn));
+ Assertions.assertEquals(
+ "Delete column '%s' does not exist.".formatted(newColumn.name()),
+ illegalArgumentException.getMessage());
+ Assertions.assertDoesNotThrow(
+ () ->
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ newName,
+ TableChange.deleteColumn(new String[] {newColumn.name()},
true)));
+
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ newName,
+ TableChange.deleteColumn(new String[] {newColumn.name()}, true));
+ Assertions.assertTrue(
+ TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), newName), "table should
be dropped");
+
+ GravitinoRuntimeException exception =
+ Assertions.assertThrows(
+ GravitinoRuntimeException.class,
+ () -> TABLE_OPERATIONS.drop(TEST_DB_NAME.toString(), newName));
+ Assertions.assertTrue(StringUtils.contains(exception.getMessage(), "does
not exist"));
+ }
+
+ @Test
+ public void testAlterTable() {
+ String tableName = RandomStringUtils.randomAlphabetic(16) + "_al_table";
+ String tableComment = "test_comment";
+ List<JdbcColumn> columns = new ArrayList<>();
+ JdbcColumn col_1 =
+ JdbcColumn.builder()
+ .withName("col_1")
+ .withType(INT)
+ .withComment("id")
+ .withNullable(false)
+ .withDefaultValue(Literals.integerLiteral(0))
+ .build();
+ columns.add(col_1);
+ JdbcColumn col_2 =
+ JdbcColumn.builder()
+ .withName("col_2")
+ .withType(STRING)
+ .withComment("name")
+ .withDefaultValue(Literals.of("hello world", STRING))
+ .withNullable(false)
+ .build();
+ columns.add(col_2);
+ JdbcColumn col_3 =
+ JdbcColumn.builder()
+ .withName("col_3")
+ .withType(STRING)
+ .withComment("name")
+ .withDefaultValue(Literals.NULL)
+ .build();
+ // `col_1` int NOT NULL COMMENT 'id' ,
+ // `col_2` STRING(255) NOT NULL DEFAULT 'hello world' COMMENT 'name' ,
+ // `col_3` STRING(255) NULL DEFAULT NULL COMMENT 'name' ,
+ columns.add(col_3);
+ Map<String, String> properties = new HashMap<>();
+
+ // create table
+ TABLE_OPERATIONS.create(
+ TEST_DB_NAME.toString(),
+ tableName,
+ columns.toArray(new JdbcColumn[0]),
+ tableComment,
+ properties,
+ null,
+ Distributions.NONE,
+ null,
+ getSortOrders("col_2"));
+ JdbcTable load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, null,
Transforms.EMPTY_TRANSFORM, load);
+
+ // Update column type
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ tableName,
+ TableChange.updateColumnType(new String[] {col_1.name()}, LONG));
+
+ load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+
+ // After modifying the type, some attributes of the corresponding column
are not
+ // supported.
+ columns.clear();
+ col_1 =
+ JdbcColumn.builder()
+ .withName(col_1.name())
+ .withType(LONG)
+ .withComment(col_1.comment())
+ .withNullable(col_1.nullable())
+ .withDefaultValue(Literals.longLiteral(0L))
+ .build();
+ columns.add(col_1);
+ columns.add(col_2);
+ columns.add(col_3);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, null,
Transforms.EMPTY_TRANSFORM, load);
+
+ String newComment = "new_comment";
+ // update table comment and column comment
+ // `col_1` int NOT NULL COMMENT 'id' ,
+ // `col_2` STRING(255) NOT NULL DEFAULT 'hello world' COMMENT
'new_comment' ,
+ // `col_3` STRING(255) NULL DEFAULT NULL COMMENT 'name' ,
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ tableName,
+ TableChange.updateColumnType(new String[] {col_1.name()}, INT),
+ TableChange.updateColumnComment(new String[] {col_2.name()},
newComment));
+ load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+
+ columns.clear();
+ col_1 =
+ JdbcColumn.builder()
+ .withName(col_1.name())
+ .withType(INT)
+ .withComment(col_1.comment())
+ .withAutoIncrement(col_1.autoIncrement())
+ .withNullable(col_1.nullable())
+ .withDefaultValue(Literals.integerLiteral(0))
+ .build();
+ col_2 =
+ JdbcColumn.builder()
+ .withName(col_2.name())
+ .withType(col_2.dataType())
+ .withComment(newComment)
+ .withAutoIncrement(col_2.autoIncrement())
+ .withNullable(col_2.nullable())
+ .withDefaultValue(col_2.defaultValue())
+ .build();
+ columns.add(col_1);
+ columns.add(col_2);
+ columns.add(col_3);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, null,
Transforms.EMPTY_TRANSFORM, load);
+
+ String newColName_1 = "new_col_1";
+ // rename column
+ // update table comment and column comment
+ // `new_col_1` int NOT NULL COMMENT 'id' ,
+ // `new_col_2` STRING(255) NOT NULL DEFAULT 'hello world' COMMENT
'new_comment'
+ // ,
+ // `col_3` STRING(255) NULL DEFAULT NULL COMMENT 'name' ,
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ tableName,
+ TableChange.renameColumn(new String[] {col_1.name()}, newColName_1));
+
+ load = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+
+ columns.clear();
+ col_1 =
+ JdbcColumn.builder()
+ .withName(newColName_1)
+ .withType(col_1.dataType())
+ .withComment(col_1.comment())
+ .withAutoIncrement(col_1.autoIncrement())
+ .withNullable(col_1.nullable())
+ .withDefaultValue(col_1.defaultValue())
+ .build();
+ columns.add(col_1);
+ columns.add(col_2);
+ columns.add(col_3);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, null,
Transforms.EMPTY_TRANSFORM, load);
+ }
+
+ @Test
+ public void testAlterTableUpdateColumnDefaultValue() {
+ String tableName = RandomNameUtils.genRandomName("properties_table_");
+ String tableComment = "test_comment";
+ List<JdbcColumn> columns = new ArrayList<>();
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_1")
+ .withType(Types.DecimalType.of(10, 2))
+ .withComment("test_decimal")
+ .withNullable(false)
+ .withDefaultValue(Literals.decimalLiteral(Decimal.of("0.00", 10,
2)))
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_2")
+ .withType(Types.LongType.get())
+ .withNullable(false)
+ .withDefaultValue(Literals.longLiteral(0L))
+ .withComment("long type")
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_3")
+ .withType(Types.TimestampType.withoutTimeZone(0))
+ .withNullable(false)
+ .withComment("timestamp")
+
.withDefaultValue(Literals.timestampLiteral(LocalDateTime.parse("2013-01-01T00:00:00")))
+ .build());
+ columns.add(
+ JdbcColumn.builder()
+ .withName("col_4")
+ .withType(Types.StringType.get())
+ .withNullable(false)
+ .withComment("STRING")
+ .withDefaultValue(Literals.of("hello", Types.StringType.get()))
+ .build());
+ Map<String, String> properties = new HashMap<>();
+
+ Index[] indexes = new Index[0];
+ // create table
+ TABLE_OPERATIONS.create(
+ TEST_DB_NAME.toString(),
+ tableName,
+ columns.toArray(new JdbcColumn[0]),
+ tableComment,
+ properties,
+ null,
+ Distributions.NONE,
+ indexes,
+ getSortOrders("col_2"));
+
+ JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(),
tableName);
+ assertionsTableInfo(
+ tableName, tableComment, columns, properties, indexes,
Transforms.EMPTY_TRANSFORM, loaded);
+
+ TABLE_OPERATIONS.alterTable(
+ TEST_DB_NAME.toString(),
+ tableName,
+ TableChange.updateColumnDefaultValue(
+ new String[] {columns.get(0).name()},
+ Literals.decimalLiteral(Decimal.of("1.23", 10, 2))),
+ TableChange.updateColumnDefaultValue(
+ new String[] {columns.get(1).name()}, Literals.longLiteral(1L)),
+ TableChange.updateColumnDefaultValue(
+ new String[] {columns.get(2).name()},
+
Literals.timestampLiteral(LocalDateTime.parse("2024-04-01T00:00:00"))),
+ TableChange.updateColumnDefaultValue(
+ new String[] {columns.get(3).name()}, Literals.of("world",
Types.StringType.get())));
+
+ loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(), tableName);
+ Assertions.assertEquals(
+ Literals.decimalLiteral(Decimal.of("1.23", 10, 2)),
loaded.columns()[0].defaultValue());
+ Assertions.assertEquals(Literals.longLiteral(1L),
loaded.columns()[1].defaultValue());
+ Assertions.assertEquals(
+ Literals.timestampLiteral(LocalDateTime.parse("2024-04-01T00:00:00")),
+ loaded.columns()[2].defaultValue());
+ Assertions.assertEquals(
+ Literals.of("world", Types.StringType.get()),
loaded.columns()[3].defaultValue());
+ }
@Test
public void testCreateAndLoadTable() {
@@ -633,4 +1008,225 @@ public class TestClickHouseTableOperations extends
TestClickHouse {
tableName, columns, comment, properties, partitioning, distribution,
indexes, sortOrders);
}
}
+
+ @Test
+ public void testGenerateAlterTableSqlCoverage() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+
+ JdbcTable table = buildStubTable();
+ ops.setTable(table);
+
+ TableChange[] changes =
+ new TableChange[] {
+ TableChange.addColumn(
+ new String[] {"c_new"},
+ Types.StringType.get(),
+ "new column",
+ TableChange.ColumnPosition.after("c1"),
+ true,
+ false,
+ Column.DEFAULT_VALUE_NOT_SET),
+ TableChange.updateColumnDefaultValue(
+ new String[] {"c2"}, Literals.of("val", Types.StringType.get())),
+ TableChange.updateColumnType(new String[] {"c1"},
Types.LongType.get()),
+ TableChange.updateColumnComment(new String[] {"c1"}, "c1_comment"),
+ TableChange.updateColumnPosition(new String[] {"c1"},
TableChange.ColumnPosition.first()),
+ TableChange.deleteColumn(new String[] {"c3"}, false),
+ TableChange.updateColumnNullability(new String[] {"c2"}, false),
+ TableChange.deleteIndex("idx1", false),
+ TableChange.renameColumn(new String[] {"c2"}, "c2_new"),
+ TableChange.updateComment("new_table_comment")
+ };
+
+ String sql = ops.buildAlterSql("db", "tbl", changes);
+
+ Assertions.assertTrue(sql.contains("ADD COLUMN `c_new` Nullable(String)"));
+ Assertions.assertTrue(sql.contains("RENAME COLUMN `c2` TO `c2_new`"));
+ Assertions.assertTrue(sql.contains("DEFAULT 'val'"));
+ Assertions.assertTrue(sql.contains("Int64"));
+ Assertions.assertTrue(sql.contains("COMMENT 'c1_comment'"));
+ Assertions.assertTrue(sql.contains("FIRST"));
+ Assertions.assertTrue(sql.contains("DROP COLUMN `c3`"));
+ Assertions.assertTrue(sql.contains("DROP INDEX `idx1`"));
+ Assertions.assertTrue(sql.contains("MODIFY COMMENT 'new_table_comment'"));
+ Assertions.assertTrue(sql.startsWith("ALTER TABLE `tbl`"));
+ }
+
+ @Test
+ public void testAlterTableDeleteColumnIfExistsNoOpReturnsEmpty() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+ ops.setTable(buildStubTable());
+
+ String sql =
+ ops.buildAlterSql(
+ "db",
+ "tbl",
+ new TableChange[] {TableChange.deleteColumn(new String[]
{"missing"}, true)});
+ Assertions.assertEquals("", sql);
+ }
+
+ @Test
+ public void testAlterTableDeleteIndexBranches() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+ ops.setTable(buildStubTable());
+
+ String sqlSkip =
+ ops.buildAlterSql(
+ "db", "tbl", new TableChange[] {TableChange.deleteIndex("missing",
true)});
+ Assertions.assertEquals("", sqlSkip);
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ ops.buildAlterSql(
+ "db", "tbl", new TableChange[]
{TableChange.deleteIndex("missing", false)}));
+ }
+
+ @Test
+ public void testAlterTableNullabilityValidationFails() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+ ops.setTable(buildStubTableWithNullDefault());
+
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ ops.buildAlterSql(
+ "db",
+ "tbl",
+ new TableChange[] {
+ TableChange.updateColumnNullability(new String[] {"c3"},
false)
+ }));
+ }
+
+ @Test
+ public void testAlterTableSetPropertyUnsupported() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+ ops.setTable(buildStubTable());
+
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () ->
+ ops.buildAlterSql("db", "tbl", new TableChange[]
{TableChange.setProperty("k", "v")}));
+ }
+
+ @Test
+ public void testAlterTableRemovePropertyUnsupported() {
+ StubClickHouseTableOperations ops = new StubClickHouseTableOperations();
+ ops.initialize(
+ null,
+ new ClickHouseExceptionConverter(),
+ new ClickHouseTypeConverter(),
+ new ClickHouseColumnDefaultValueConverter(),
+ new HashMap<>());
+ ops.setTable(buildStubTable());
+
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> ops.buildAlterSql("db", "tbl", new TableChange[]
{TableChange.removeProperty("k")}));
+ }
+
+ private static JdbcTable buildStubTable() {
+ JdbcColumn c1 =
+ JdbcColumn.builder()
+ .withName("c1")
+ .withType(Types.IntegerType.get())
+ .withNullable(false)
+ .withDefaultValue(Literals.integerLiteral(1))
+ .build();
+ JdbcColumn c2 =
+ JdbcColumn.builder()
+ .withName("c2")
+ .withType(Types.StringType.get())
+ .withNullable(true)
+ .withDefaultValue(Literals.of("x", Types.StringType.get()))
+ .build();
+ JdbcColumn c3 =
+ JdbcColumn.builder()
+ .withName("c3")
+ .withType(Types.StringType.get())
+ .withNullable(true)
+ .withDefaultValue(DEFAULT_VALUE_NOT_SET)
+ .build();
+ return JdbcTable.builder()
+ .withName("tbl")
+ .withColumns(new JdbcColumn[] {c1, c2, c3})
+ .withIndexes(
+ new Index[] {
+ Indexes.primary(Indexes.DEFAULT_PRIMARY_KEY_NAME, new String[][]
{{"c1"}}),
+ Indexes.unique("idx1", new String[][] {{"c2"}})
+ })
+ .withComment("table_comment")
+ .withTableOperation(null)
+ .build();
+ }
+
+ private static JdbcTable buildStubTableWithNullDefault() {
+ JdbcColumn c1 =
JdbcColumn.builder().withName("c1").withType(Types.IntegerType.get()).build();
+ JdbcColumn c2 =
JdbcColumn.builder().withName("c2").withType(Types.StringType.get()).build();
+ JdbcColumn c3 =
+ JdbcColumn.builder()
+ .withName("c3")
+ .withType(Types.StringType.get())
+ .withNullable(true)
+ .withDefaultValue(Literals.NULL)
+ .build();
+ return JdbcTable.builder()
+ .withName("tbl")
+ .withColumns(new JdbcColumn[] {c1, c2, c3})
+ .withIndexes(
+ new Index[] {
+ Indexes.primary(Indexes.DEFAULT_PRIMARY_KEY_NAME, new String[][]
{{"c1"}})
+ })
+ .withComment("table_comment")
+ .withTableOperation(null)
+ .build();
+ }
+
+ private static final class StubClickHouseTableOperations extends
ClickHouseTableOperations {
+ private JdbcTable table;
+
+ void setTable(JdbcTable table) {
+ this.table = table;
+ }
+
+ @Override
+ protected JdbcTable getOrCreateTable(
+ String databaseName, String tableName, JdbcTable lazyLoadCreateTable) {
+ return table;
+ }
+
+ String buildAlterSql(String db, String tableName, TableChange[] changes) {
+ return generateAlterTableSql(db, tableName, changes);
+ }
+ }
}
diff --git
a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/operation/JdbcTableOperations.java
b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/operation/JdbcTableOperations.java
index 1877ecf519..7af4c94b0c 100644
---
a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/operation/JdbcTableOperations.java
+++
b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/operation/JdbcTableOperations.java
@@ -661,6 +661,10 @@ public abstract class JdbcTableOperations implements
TableOperation {
"Column %s does not exist in table %s", colName,
jdbcTable.name()));
}
+ protected boolean columnExists(JdbcTable table, String columnName) {
+ return Arrays.stream(table.columns()).anyMatch(col ->
col.name().equals(columnName));
+ }
+
protected Connection getConnection(String catalog) throws SQLException {
Connection connection = dataSource.getConnection();
connection.setCatalog(catalog);
