This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new e1afb700e9 [#11881] fix(clickhouse): distinguish MATERIALIZED/ALIAS
from DEFAULT via default_kind (#11886)
e1afb700e9 is described below
commit e1afb700e959035c3062b39b0286f2ddc1b3103a
Author: StormSpirit <[email protected]>
AuthorDate: Mon Jul 27 15:28:03 2026 +0800
[#11881] fix(clickhouse): distinguish MATERIALIZED/ALIAS from DEFAULT via
default_kind (#11886)
### What changes were proposed in this pull request?
Fix column default value handling for MATERIALIZED and ALIAS columns in
the ClickHouse catalog.
ClickHouse distinguishes column default value kinds via
`system.columns.default_kind` (DEFAULT,
MATERIALIZED, ALIAS). The JDBC driver`getColumns()` hardcodes `'NO'` as
`IS_GENERATEDCOLUMN`
(clickhouse-java#1625 pattern), so the catalog cannot distinguish these
kinds through standard
JDBC metadata.
Changes:
- Added `getDefaultKinds()` method that queries `system.columns` for
`default_kind`, following
the same pattern as `getIndexes()` which already bypasses the JDBC
driver for similar limitations
- `load()` now calls `getDefaultKinds()` and uses the result to correct
column metadata inline
during the column loading loop, without exposing instance state
- Fixed `ClickHouseContainer.getJdbcUrl()` to use `localhost` +
`getMappedPort()` instead of
`getContainerIpAddress()`, consistent with how `DorisContainer` handles
macOS Docker Desktop
compatibility
Thread safety: `ClickHouseTableOperations` is a shared singleton (cached
by CatalogManager's
Caffeine cache, accessed concurrently via JettyServer's
QueuedThreadPool). The `default_kinds`
map is a local variable in `load()`, not an instance field, making the
method reentrant.
### Why are the changes needed?
Fix: #11881
MATERIALIZED and ALIAS columns were indistinguishable from DEFAULT
columns on round-trip,
because the catalog never fetched `default_kind` from ClickHouse system
tables.
### Does this PR introduce _any_ user-facing change?
Yes. MATERIALIZED and ALIAS columns now have correct default value kinds
in Gravitino metadata,
enabling consumers to distinguish them from DEFAULT columns.
### How was this patch tested?
**Unit tests** (`TestClickHouseTableOperations`):
- `testLoadTableWithMaterializedAndAliasColumns`: creates table with all
three default kinds
via raw SQL, verifies all three columns return
`UnparsedExpression("today()")` via their
respective code paths
**Integration tests** (`CatalogClickHouseIT`):
- `testMATERIALIZEDAndALIASColumnDefaultKinds`: end-to-end test through
Gravitino API covering
DEFAULT/MATERIALIZED/ALIAS × multiple types (Date, Int64, String) ×
Nullable columns
Both test classes require `-PskipDockerTests=false` to run.
---------
Signed-off-by: jiangxt2 <[email protected]>
---
.../operations/ClickHouseTableOperations.java | 55 +++++++-
.../integration/test/CatalogClickHouseIT.java | 154 +++++++++++++++++++++
.../operations/TestClickHouseTableOperations.java | 66 +++++++++
3 files changed, 270 insertions(+), 5 deletions(-)
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
index 70b8f06346..a4c47811b1 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/main/java/org/apache/gravitino/catalog/clickhouse/operations/ClickHouseTableOperations.java
@@ -61,6 +61,7 @@ import
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata
import
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata.ENGINE;
import org.apache.gravitino.catalog.jdbc.JdbcColumn;
import org.apache.gravitino.catalog.jdbc.JdbcTable;
+import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter;
import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations;
import org.apache.gravitino.catalog.jdbc.utils.JdbcConnectorUtils;
import org.apache.gravitino.exceptions.NoSuchTableException;
@@ -727,15 +728,42 @@ public class ClickHouseTableOperations extends
JdbcTableOperations {
ResultSet tables = getTable(connection, databaseName, tableName);
JdbcTable.Builder jdbcTableBuilder = getTableBuilder(tables,
databaseName, tableName);
+ // Query system.columns for default_kind to correctly identify
MATERIALIZED/ALIAS columns.
+ // The ClickHouse JDBC driver hardcodes IS_GENERATEDCOLUMN to 'NO' for
all columns.
+ // Stored as a local variable (not instance field) to avoid
thread-safety issues,
+ // since ClickHouseTableOperations is a shared singleton across
concurrent requests.
+ Map<String, String> defaultKinds = getDefaultKinds(connection,
databaseName, tableName);
+
+ // NOTE: Cannot use getColumnBuilder() here because we need to override
the default
+ // value for MATERIALIZED/ALIAS columns between getBasicJdbcColumnInfo()
and build().
List<JdbcColumn> jdbcColumns = new ArrayList<>();
ResultSet columns = getColumns(connection, databaseName, tableName);
while (columns.next()) {
- JdbcColumn.Builder columnBuilder = getColumnBuilder(columns,
databaseName, tableName);
- if (columnBuilder != null) {
- boolean autoIncrement = getAutoIncrementInfo(columns);
- columnBuilder.withAutoIncrement(autoIncrement);
- jdbcColumns.add(columnBuilder.build());
+ if (!Objects.equals(columns.getString("TABLE_NAME"), tableName)) {
+ continue;
+ }
+ JdbcColumn.Builder columnBuilder = getBasicJdbcColumnInfo(columns);
+ // Correct default value for MATERIALIZED/ALIAS columns: the JDBC
driver
+ // hardcodes IS_GENERATEDCOLUMN to 'NO', so re-derive with
isExpression=true.
+ String columnName = columns.getString("COLUMN_NAME");
+ String defaultKind = defaultKinds.getOrDefault(columnName, "");
+ if ("MATERIALIZED".equals(defaultKind) || "ALIAS".equals(defaultKind))
{
+ String columnDef = columns.getString("COLUMN_DEF");
+ boolean nullable = columns.getBoolean("NULLABLE");
+ String typeName = columns.getString("TYPE_NAME");
+ int columnSize = columns.getInt("COLUMN_SIZE");
+ int scale = columns.getInt("DECIMAL_DIGITS");
+ JdbcTypeConverter.JdbcTypeBean typeBean = new
JdbcTypeConverter.JdbcTypeBean(typeName);
+ typeBean.setColumnSize(columnSize);
+ typeBean.setScale(scale);
+ typeBean.setDatetimePrecision(calculateDatetimePrecision(typeName,
columnSize, scale));
+ Expression correctDefault =
+ columnDefaultValueConverter.toGravitino(typeBean, columnDef,
true, nullable);
+ columnBuilder.withDefaultValue(correctDefault);
}
+ boolean autoIncrement = getAutoIncrementInfo(columns);
+ columnBuilder.withAutoIncrement(autoIncrement);
+ jdbcColumns.add(columnBuilder.build());
}
jdbcTableBuilder.withColumns(jdbcColumns.toArray(new JdbcColumn[0]));
@@ -773,6 +801,23 @@ public class ClickHouseTableOperations extends
JdbcTableOperations {
}
}
+ @VisibleForTesting
+ Map<String, String> getDefaultKinds(Connection connection, String database,
String table)
+ throws SQLException {
+ Map<String, String> kinds = new HashMap<>();
+ String sql = "SELECT name, default_kind FROM system.columns WHERE database
= ? AND table = ?";
+ try (PreparedStatement stmt = connection.prepareStatement(sql)) {
+ stmt.setString(1, database);
+ stmt.setString(2, table);
+ try (ResultSet rs = stmt.executeQuery()) {
+ while (rs.next()) {
+ kinds.put(rs.getString("name"), rs.getString("default_kind"));
+ }
+ }
+ }
+ return kinds;
+ }
+
@Override
protected Transform[] getTablePartitioning(
Connection connection, String databaseName, String tableName) throws
SQLException {
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
index a433ac058f..7ee3cf52e1 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java
@@ -1104,6 +1104,160 @@ public class CatalogClickHouseIT extends BaseIT {
}
}
+ @Test
+ @EnabledIf("supportColumnDefaultValueExpression")
+ void testMATERIALIZEDAndALIASColumnDefaultKinds() {
+ // Verify that MATERIALIZED and ALIAS columns are correctly distinguished
from DEFAULT
+ // columns on round-trip through the Gravitino API. The ClickHouse JDBC
driver hardcodes
+ // IS_GENERATEDCOLUMN to 'NO', so the catalog must query
system.columns.default_kind
+ // to correctly identify these column kinds.
+ String tableName = GravitinoITUtils.genRandomName("test_default_kind");
+ String fullTableName = schemaName + "." + tableName;
+ String sql =
+ "CREATE TABLE "
+ + fullTableName
+ + " (\n"
+ + " id Int64,\n"
+ + " created_date Date DEFAULT today(),\n"
+ + " computed_date Date MATERIALIZED today(),\n"
+ + " alias_date Date ALIAS today(),\n"
+ + " computed_int Int64 MATERIALIZED id + 1,\n"
+ + " alias_int Int64 ALIAS id * 2,\n"
+ + " materialized_str String MATERIALIZED concat('prefix_',
toString(id)),\n"
+ + " alias_str String ALIAS concat('suffix_', toString(id)),\n"
+ + " nullable_default Nullable(Date) DEFAULT today(),\n"
+ + " nullable_materialized Nullable(Date) MATERIALIZED today(),\n"
+ + " nullable_alias Nullable(Date) ALIAS today()\n"
+ + ") ENGINE = MergeTree ORDER BY id;\n";
+
+ clickhouseService.executeQuery(sql);
+ Table loadedTable =
+ catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName,
tableName));
+
+ for (Column column : loadedTable.columns()) {
+ switch (column.name()) {
+ case "id":
+ // No default value
+ Assertions.assertEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ break;
+ case "created_date":
+ // DEFAULT column: today() is an expression, not a literal
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ Assertions.assertEquals(
+ "today()", ((UnparsedExpression)
column.defaultValue()).unparsedExpression());
+ break;
+ case "computed_date":
+ // MATERIALIZED column: must resolve to UnparsedExpression, not
DEFAULT_VALUE_NOT_SET
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ Assertions.assertEquals(
+ "today()", ((UnparsedExpression)
column.defaultValue()).unparsedExpression());
+ break;
+ case "alias_date":
+ // ALIAS column: must resolve to UnparsedExpression, not
DEFAULT_VALUE_NOT_SET
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ Assertions.assertEquals(
+ "today()", ((UnparsedExpression)
column.defaultValue()).unparsedExpression());
+ break;
+ case "computed_int":
+ // MATERIALIZED with arithmetic expression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ case "alias_int":
+ // ALIAS with arithmetic expression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ case "materialized_str":
+ // MATERIALIZED with function expression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ case "alias_str":
+ // ALIAS with function expression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ case "nullable_default":
+ // Nullable + DEFAULT: should have a default value
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ break;
+ case "nullable_materialized":
+ // Nullable + MATERIALIZED: should have a default value, not
DEFAULT_VALUE_NOT_SET
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ case "nullable_alias":
+ // Nullable + ALIAS: should have a default value, not
DEFAULT_VALUE_NOT_SET
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertTrue(column.defaultValue() instanceof
UnparsedExpression);
+ break;
+ default:
+ Assertions.fail(
+ "Unexpected column name: "
+ + column.name()
+ + ", default value: "
+ + column.defaultValue());
+ }
+ }
+ }
+
+ @Test
+ @EnabledIf("supportColumnDefaultValueExpression")
+ void testDefaultColumnsNotMismarkedAsMaterialized() {
+ // Regression guard: columns with DEFAULT (not MATERIALIZED/ALIAS) must
remain as DEFAULT
+ // on round-trip. The getDefaultKinds() query should not cause plain
DEFAULT columns to be
+ // re-processed through the expression branch intended for
MATERIALIZED/ALIAS.
+ String tableName = GravitinoITUtils.genRandomName("test_default_only");
+ String fullTableName = schemaName + "." + tableName;
+ String sql =
+ "CREATE TABLE "
+ + fullTableName
+ + " (\n"
+ + " id Int64,\n"
+ + " status UInt8 DEFAULT 0,\n"
+ + " name String DEFAULT 'unknown',\n"
+ + " created_at DateTime DEFAULT now()\n"
+ + ") ENGINE = MergeTree ORDER BY id;\n";
+
+ clickhouseService.executeQuery(sql);
+ Table loadedTable =
+ catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName,
tableName));
+
+ for (Column column : loadedTable.columns()) {
+ switch (column.name()) {
+ case "id":
+ Assertions.assertEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ break;
+ case "status":
+ // DEFAULT literal 0 — must NOT be UnparsedExpression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertFalse(
+ column.defaultValue() instanceof UnparsedExpression,
+ "DEFAULT column 'status' should not be treated as
MATERIALIZED/ALIAS");
+ break;
+ case "name":
+ // DEFAULT literal 'unknown' — must NOT be UnparsedExpression
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ Assertions.assertFalse(
+ column.defaultValue() instanceof UnparsedExpression,
+ "DEFAULT column 'name' should not be treated as
MATERIALIZED/ALIAS");
+ break;
+ case "created_at":
+ // DEFAULT now() is a function expression → UnparsedExpression is
acceptable here
+ // (falls back because now() is not a valid timestamp literal).
+ // The key assertion: it must NOT be DEFAULT_VALUE_NOT_SET.
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
column.defaultValue());
+ break;
+ default:
+ Assertions.fail("Unexpected column: " + column.name());
+ }
+ }
+ }
+
@Test
void testColumnTypeConverter() {
// test convert from ClickHouse to Gravitino
diff --git
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
index 5cea6b2801..909b5ec81a 100644
---
a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
+++
b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java
@@ -23,6 +23,9 @@ import static
org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesM
import static
org.apache.gravitino.catalog.clickhouse.ClickHouseUtils.getSortOrders;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@@ -43,6 +46,7 @@ import org.apache.gravitino.catalog.jdbc.JdbcTable;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.NamedReference;
+import org.apache.gravitino.rel.expressions.UnparsedExpression;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
import org.apache.gravitino.rel.expressions.literals.Literals;
import org.apache.gravitino.rel.expressions.sorts.SortOrder;
@@ -491,6 +495,68 @@ public class TestClickHouseTableOperations extends
TestClickHouse {
tableName, tableComment, columns, properties, indexes,
Transforms.EMPTY_TRANSFORM, loaded);
}
+ @Test
+ public void testLoadTableWithMaterializedAndAliasColumns() throws Exception {
+ String tableName = RandomStringUtils.randomAlphabetic(16) +
"_default_kind";
+
+ // Create table with MATERIALIZED and ALIAS columns via raw SQL.
+ // Using DriverManager.getConnection directly because Gravitino's create()
API
+ // does not support specifying MATERIALIZED/ALIAS default value kinds.
+ String jdbcUrl =
containerSuite.getClickHouseContainer().getJdbcUrl(TEST_DB_NAME);
+ try (Connection conn =
+ DriverManager.getConnection(
+ jdbcUrl,
+ containerSuite.getClickHouseContainer().getUsername(),
+ containerSuite.getClickHouseContainer().getPassword());
+ Statement stmt = conn.createStatement()) {
+ stmt.execute(
+ String.format(
+ "CREATE TABLE %s.%s ("
+ + " id Int64,"
+ + " created_date Date DEFAULT today(),"
+ + " computed_date Date MATERIALIZED today(),"
+ + " alias_date Date ALIAS today()"
+ + ") ENGINE = MergeTree ORDER BY id",
+ TEST_DB_NAME, tableName));
+ }
+
+ // Load via table operations
+ JdbcTable loaded = TABLE_OPERATIONS.load(TEST_DB_NAME.toString(),
tableName);
+ Column[] columns = loaded.columns();
+
+ // Find each column and verify default values
+ Column createdDateCol = findColumn(columns, "created_date");
+ Column computedDateCol = findColumn(columns, "computed_date");
+ Column aliasDateCol = findColumn(columns, "alias_date");
+
+ // DEFAULT column: should have a default value (Literal or
UnparsedExpression)
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
createdDateCol.defaultValue());
+ // Verify the default value content is today()
+ UnparsedExpression createdDefault = (UnparsedExpression)
createdDateCol.defaultValue();
+ Assertions.assertEquals("today()", createdDefault.unparsedExpression());
+
+ // MATERIALIZED column: should have UnparsedExpression (not
DEFAULT_VALUE_NOT_SET)
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
computedDateCol.defaultValue());
+ Assertions.assertTrue(computedDateCol.defaultValue() instanceof
UnparsedExpression);
+ UnparsedExpression computedDefault = (UnparsedExpression)
computedDateCol.defaultValue();
+ Assertions.assertEquals("today()", computedDefault.unparsedExpression());
+
+ // ALIAS column: should have UnparsedExpression (not DEFAULT_VALUE_NOT_SET)
+ Assertions.assertNotEquals(DEFAULT_VALUE_NOT_SET,
aliasDateCol.defaultValue());
+ Assertions.assertTrue(aliasDateCol.defaultValue() instanceof
UnparsedExpression);
+ UnparsedExpression aliasDefault = (UnparsedExpression)
aliasDateCol.defaultValue();
+ Assertions.assertEquals("today()", aliasDefault.unparsedExpression());
+ }
+
+ private static Column findColumn(Column[] columns, String name) {
+ for (Column col : columns) {
+ if (col.name().equals(name)) {
+ return col;
+ }
+ }
+ throw new AssertionError("Column not found: " + name);
+ }
+
@Test
public void testTypeConversionAgainstCluster() {
String tableName = RandomStringUtils.randomAlphabetic(16) + "_type_conv";
