LuciferYang commented on code in PR #14658:
URL: https://github.com/apache/gravitino/pull/14658#discussion_r4478400348
##########
flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java:
##########
@@ -425,6 +425,7 @@ public void createTable(ObjectPath tablePath,
CatalogBaseTable table, boolean ig
NameIdentifier.of(tablePath.getDatabaseName(),
tablePath.getObjectName());
ResolvedCatalogBaseTable<?> resolvedTable = (ResolvedCatalogBaseTable<?>)
table;
+ rejectUnsupportedSchemaElements(resolvedTable);
Column[] columns =
Review Comment:
This is intended and listed under user-facing changes. Flink calls
`Catalog.createTable` for `CREATE TABLE IF NOT EXISTS` even when the table
exists, so skipping the check for an existing table would need an extra
`loadTable` request before every create. Added the `ignoreIfExists = true` case
to the test in 3d1279215.
##########
flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java:
##########
@@ -888,6 +889,66 @@ private static Optional<List<String>>
getFlinkPrimaryKey(Table table) {
return Optional.of(primaryKeyFieldList);
}
+ /**
+ * Returns whether this catalog can store a table as a Flink generic table,
which serializes the
+ * full Flink schema, including computed columns, metadata columns and
watermarks, into {@code
+ * flink.*} properties. Only the Hive catalog can.
+ *
+ * @return true if generic tables are supported, false by default
+ */
+ protected boolean supportsGenericTables() {
+ return false;
+ }
+
+ /**
+ * Rejects computed columns, metadata columns and watermarks, which the
Gravitino column model
+ * cannot represent. Storing them anyway would turn each such column into a
plain physical column
+ * and drop the watermark, without any error. This path serves Paimon,
Iceberg and JDBC tables,
+ * and Hive tables with {@code 'connector'='hive'}.
+ *
+ * @param resolvedTable the resolved create-table request
+ */
+ private void rejectUnsupportedSchemaElements(ResolvedCatalogBaseTable<?>
resolvedTable) {
+ for (org.apache.flink.table.catalog.Column column :
+ resolvedTable.getResolvedSchema().getColumns()) {
+ rejectUnsupportedColumn(column);
+ }
+ if (!resolvedTable.getResolvedSchema().getWatermarkSpecs().isEmpty()) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "The watermark cannot be stored in the Gravitino catalog
'%s'.%s",
+ getName(), genericTableAdvice()));
Review Comment:
Flink allows only one watermark per table ("Multiple watermark definitions
are not supported yet."), so the singular is right. The message now names the
rowtime column (3d1279215).
##########
flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java:
##########
@@ -546,6 +550,220 @@ public void
testAlterTableWithCommentChangeForwardsAlterCall() throws Exception
Mockito.verify(tableCatalog, Mockito.times(1)).alterTable(Mockito.any(),
Mockito.any());
}
+ /**
+ * Computed columns, metadata columns and watermarks cannot be represented
by Gravitino columns.
+ * createTable must reject them instead of storing plain physical columns
without the watermark.
+ */
+ @Test
+ void testCreateTableRejectsComputedColumnsAndWatermarks() {
+ TestableBaseCatalog catalog =
+ new TestableBaseCatalog(Mockito.mock(AbstractCatalog.class),
mockedCatalogHandle());
+
+ // computed column: b AS a + 1
+ List<Column> computed =
+ Arrays.asList(
+ Column.physical("a", DataTypes.INT()),
+ Column.computed("b", resolvedExpression(DataTypes.INT())));
+ ResolvedCatalogTable computedTable =
+ resolvedCatalogTable(resolvedSchema(computed,
Collections.emptyList()));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t1"), computedTable,
false));
+
+ // metadata column: m INT METADATA
+ List<Column> metadata =
+ Arrays.asList(
+ Column.physical("a", DataTypes.INT()),
+ Column.metadata("m", DataTypes.INT(), null, false));
+ ResolvedCatalogTable metadataTable =
+ resolvedCatalogTable(resolvedSchema(metadata,
Collections.emptyList()));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t4"), metadataTable,
false));
+
+ // watermark spec on a timestamp column
+ List<WatermarkSpec> watermarkSpecs =
+ Collections.singletonList(
+ WatermarkSpec.of("a",
resolvedExpression(DataTypes.TIMESTAMP_LTZ(3))));
+ ResolvedCatalogTable watermarkedTable =
+ resolvedCatalogTable(
+ resolvedSchema(
+ Collections.singletonList(Column.physical("a",
DataTypes.TIMESTAMP_LTZ(3))),
+ watermarkSpecs));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t2"),
watermarkedTable, false));
+
+ // plain physical column still creates
+ ResolvedCatalogTable plainTable =
+ resolvedCatalogTable(
+ resolvedSchema(
+ Collections.singletonList(Column.physical("a",
DataTypes.INT())),
+ Collections.emptyList()));
+ Assertions.assertDoesNotThrow(
+ () -> catalog.createTable(new ObjectPath("db", "t3"), plainTable,
false));
+ }
+
+ /**
+ * ALTER TABLE ... ADD of a computed column must be rejected like
createTable does, while adding a
+ * physical column still goes through.
+ */
+ @Test
+ void testAlterTableRejectsComputedColumns() throws Exception {
Review Comment:
Added in 3d1279215.
##########
flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java:
##########
@@ -546,6 +550,220 @@ public void
testAlterTableWithCommentChangeForwardsAlterCall() throws Exception
Mockito.verify(tableCatalog, Mockito.times(1)).alterTable(Mockito.any(),
Mockito.any());
}
+ /**
+ * Computed columns, metadata columns and watermarks cannot be represented
by Gravitino columns.
+ * createTable must reject them instead of storing plain physical columns
without the watermark.
+ */
+ @Test
+ void testCreateTableRejectsComputedColumnsAndWatermarks() {
+ TestableBaseCatalog catalog =
+ new TestableBaseCatalog(Mockito.mock(AbstractCatalog.class),
mockedCatalogHandle());
+
+ // computed column: b AS a + 1
+ List<Column> computed =
+ Arrays.asList(
+ Column.physical("a", DataTypes.INT()),
+ Column.computed("b", resolvedExpression(DataTypes.INT())));
+ ResolvedCatalogTable computedTable =
+ resolvedCatalogTable(resolvedSchema(computed,
Collections.emptyList()));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t1"), computedTable,
false));
+
+ // metadata column: m INT METADATA
+ List<Column> metadata =
+ Arrays.asList(
+ Column.physical("a", DataTypes.INT()),
+ Column.metadata("m", DataTypes.INT(), null, false));
+ ResolvedCatalogTable metadataTable =
+ resolvedCatalogTable(resolvedSchema(metadata,
Collections.emptyList()));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t4"), metadataTable,
false));
+
+ // watermark spec on a timestamp column
+ List<WatermarkSpec> watermarkSpecs =
+ Collections.singletonList(
+ WatermarkSpec.of("a",
resolvedExpression(DataTypes.TIMESTAMP_LTZ(3))));
+ ResolvedCatalogTable watermarkedTable =
+ resolvedCatalogTable(
+ resolvedSchema(
+ Collections.singletonList(Column.physical("a",
DataTypes.TIMESTAMP_LTZ(3))),
+ watermarkSpecs));
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> catalog.createTable(new ObjectPath("db", "t2"),
watermarkedTable, false));
+
+ // plain physical column still creates
+ ResolvedCatalogTable plainTable =
+ resolvedCatalogTable(
+ resolvedSchema(
+ Collections.singletonList(Column.physical("a",
DataTypes.INT())),
+ Collections.emptyList()));
+ Assertions.assertDoesNotThrow(
+ () -> catalog.createTable(new ObjectPath("db", "t3"), plainTable,
false));
+ }
+
+ /**
+ * ALTER TABLE ... ADD of a computed column must be rejected like
createTable does, while adding a
+ * physical column still goes through.
+ */
+ @Test
+ void testAlterTableRejectsComputedColumns() throws Exception {
+ Catalog gravitinoCatalogHandle = Mockito.mock(Catalog.class);
+ TableCatalog tableCatalog = Mockito.mock(TableCatalog.class);
+
Mockito.when(gravitinoCatalogHandle.asTableCatalog()).thenReturn(tableCatalog);
+
+ Schema schema = Schema.newBuilder().column("a", DataTypes.INT()).build();
+ CatalogTable existing =
+ catalogCompat().createCatalogTable(schema, null, ImmutableList.of(),
ImmutableMap.of());
+ TestableBaseCatalog catalog =
+ new TestableBaseCatalog(
+ Mockito.mock(AbstractCatalog.class), gravitinoCatalogHandle,
existing);
+
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () ->
+ catalog.alterTable(
+ new ObjectPath("db", "t"),
+ existing,
+ Collections.singletonList(
+ org.apache.flink.table.catalog.TableChange.add(
+ Column.computed("b",
resolvedExpression(DataTypes.INT())))),
+ false));
+
Review Comment:
Added in 3d1279215.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]