LuciferYang commented on code in PR #14658:
URL: https://github.com/apache/gravitino/pull/14658#discussion_r4478400348


##########
flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java:
##########
@@ -425,6 +425,7 @@ public void createTable(ObjectPath tablePath, 
CatalogBaseTable table, boolean ig
         NameIdentifier.of(tablePath.getDatabaseName(), 
tablePath.getObjectName());
 
     ResolvedCatalogBaseTable<?> resolvedTable = (ResolvedCatalogBaseTable<?>) 
table;
+    rejectUnsupportedSchemaElements(resolvedTable);
     Column[] columns =

Review Comment:
   This is intended and listed under user-facing changes. Flink calls 
`Catalog.createTable` for `CREATE TABLE IF NOT EXISTS` even when the table 
exists, so skipping the check for an existing table would need an extra 
`loadTable` request before every create. Added the `ignoreIfExists = true` case 
to the test in 3d1279215.



##########
flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java:
##########
@@ -888,6 +889,66 @@ private static Optional<List<String>> 
getFlinkPrimaryKey(Table table) {
     return Optional.of(primaryKeyFieldList);
   }
 
+  /**
+   * Returns whether this catalog can store a table as a Flink generic table, 
which serializes the
+   * full Flink schema, including computed columns, metadata columns and 
watermarks, into {@code
+   * flink.*} properties. Only the Hive catalog can.
+   *
+   * @return true if generic tables are supported, false by default
+   */
+  protected boolean supportsGenericTables() {
+    return false;
+  }
+
+  /**
+   * Rejects computed columns, metadata columns and watermarks, which the 
Gravitino column model
+   * cannot represent. Storing them anyway would turn each such column into a 
plain physical column
+   * and drop the watermark, without any error. This path serves Paimon, 
Iceberg and JDBC tables,
+   * and Hive tables with {@code 'connector'='hive'}.
+   *
+   * @param resolvedTable the resolved create-table request
+   */
+  private void rejectUnsupportedSchemaElements(ResolvedCatalogBaseTable<?> 
resolvedTable) {
+    for (org.apache.flink.table.catalog.Column column :
+        resolvedTable.getResolvedSchema().getColumns()) {
+      rejectUnsupportedColumn(column);
+    }
+    if (!resolvedTable.getResolvedSchema().getWatermarkSpecs().isEmpty()) {
+      throw new UnsupportedOperationException(
+          String.format(
+              "The watermark cannot be stored in the Gravitino catalog 
'%s'.%s",
+              getName(), genericTableAdvice()));

Review Comment:
   Flink allows only one watermark per table ("Multiple watermark definitions 
are not supported yet."), so the singular is right. The message now names the 
rowtime column (3d1279215).



##########
flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java:
##########
@@ -546,6 +550,220 @@ public void 
testAlterTableWithCommentChangeForwardsAlterCall() throws Exception
     Mockito.verify(tableCatalog, Mockito.times(1)).alterTable(Mockito.any(), 
Mockito.any());
   }
 
+  /**
+   * Computed columns, metadata columns and watermarks cannot be represented 
by Gravitino columns.
+   * createTable must reject them instead of storing plain physical columns 
without the watermark.
+   */
+  @Test
+  void testCreateTableRejectsComputedColumnsAndWatermarks() {
+    TestableBaseCatalog catalog =
+        new TestableBaseCatalog(Mockito.mock(AbstractCatalog.class), 
mockedCatalogHandle());
+
+    // computed column: b AS a + 1
+    List<Column> computed =
+        Arrays.asList(
+            Column.physical("a", DataTypes.INT()),
+            Column.computed("b", resolvedExpression(DataTypes.INT())));
+    ResolvedCatalogTable computedTable =
+        resolvedCatalogTable(resolvedSchema(computed, 
Collections.emptyList()));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t1"), computedTable, 
false));
+
+    // metadata column: m INT METADATA
+    List<Column> metadata =
+        Arrays.asList(
+            Column.physical("a", DataTypes.INT()),
+            Column.metadata("m", DataTypes.INT(), null, false));
+    ResolvedCatalogTable metadataTable =
+        resolvedCatalogTable(resolvedSchema(metadata, 
Collections.emptyList()));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t4"), metadataTable, 
false));
+
+    // watermark spec on a timestamp column
+    List<WatermarkSpec> watermarkSpecs =
+        Collections.singletonList(
+            WatermarkSpec.of("a", 
resolvedExpression(DataTypes.TIMESTAMP_LTZ(3))));
+    ResolvedCatalogTable watermarkedTable =
+        resolvedCatalogTable(
+            resolvedSchema(
+                Collections.singletonList(Column.physical("a", 
DataTypes.TIMESTAMP_LTZ(3))),
+                watermarkSpecs));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t2"), 
watermarkedTable, false));
+
+    // plain physical column still creates
+    ResolvedCatalogTable plainTable =
+        resolvedCatalogTable(
+            resolvedSchema(
+                Collections.singletonList(Column.physical("a", 
DataTypes.INT())),
+                Collections.emptyList()));
+    Assertions.assertDoesNotThrow(
+        () -> catalog.createTable(new ObjectPath("db", "t3"), plainTable, 
false));
+  }
+
+  /**
+   * ALTER TABLE ... ADD of a computed column must be rejected like 
createTable does, while adding a
+   * physical column still goes through.
+   */
+  @Test
+  void testAlterTableRejectsComputedColumns() throws Exception {

Review Comment:
   Added in 3d1279215.



##########
flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java:
##########
@@ -546,6 +550,220 @@ public void 
testAlterTableWithCommentChangeForwardsAlterCall() throws Exception
     Mockito.verify(tableCatalog, Mockito.times(1)).alterTable(Mockito.any(), 
Mockito.any());
   }
 
+  /**
+   * Computed columns, metadata columns and watermarks cannot be represented 
by Gravitino columns.
+   * createTable must reject them instead of storing plain physical columns 
without the watermark.
+   */
+  @Test
+  void testCreateTableRejectsComputedColumnsAndWatermarks() {
+    TestableBaseCatalog catalog =
+        new TestableBaseCatalog(Mockito.mock(AbstractCatalog.class), 
mockedCatalogHandle());
+
+    // computed column: b AS a + 1
+    List<Column> computed =
+        Arrays.asList(
+            Column.physical("a", DataTypes.INT()),
+            Column.computed("b", resolvedExpression(DataTypes.INT())));
+    ResolvedCatalogTable computedTable =
+        resolvedCatalogTable(resolvedSchema(computed, 
Collections.emptyList()));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t1"), computedTable, 
false));
+
+    // metadata column: m INT METADATA
+    List<Column> metadata =
+        Arrays.asList(
+            Column.physical("a", DataTypes.INT()),
+            Column.metadata("m", DataTypes.INT(), null, false));
+    ResolvedCatalogTable metadataTable =
+        resolvedCatalogTable(resolvedSchema(metadata, 
Collections.emptyList()));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t4"), metadataTable, 
false));
+
+    // watermark spec on a timestamp column
+    List<WatermarkSpec> watermarkSpecs =
+        Collections.singletonList(
+            WatermarkSpec.of("a", 
resolvedExpression(DataTypes.TIMESTAMP_LTZ(3))));
+    ResolvedCatalogTable watermarkedTable =
+        resolvedCatalogTable(
+            resolvedSchema(
+                Collections.singletonList(Column.physical("a", 
DataTypes.TIMESTAMP_LTZ(3))),
+                watermarkSpecs));
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> catalog.createTable(new ObjectPath("db", "t2"), 
watermarkedTable, false));
+
+    // plain physical column still creates
+    ResolvedCatalogTable plainTable =
+        resolvedCatalogTable(
+            resolvedSchema(
+                Collections.singletonList(Column.physical("a", 
DataTypes.INT())),
+                Collections.emptyList()));
+    Assertions.assertDoesNotThrow(
+        () -> catalog.createTable(new ObjectPath("db", "t3"), plainTable, 
false));
+  }
+
+  /**
+   * ALTER TABLE ... ADD of a computed column must be rejected like 
createTable does, while adding a
+   * physical column still goes through.
+   */
+  @Test
+  void testAlterTableRejectsComputedColumns() throws Exception {
+    Catalog gravitinoCatalogHandle = Mockito.mock(Catalog.class);
+    TableCatalog tableCatalog = Mockito.mock(TableCatalog.class);
+    
Mockito.when(gravitinoCatalogHandle.asTableCatalog()).thenReturn(tableCatalog);
+
+    Schema schema = Schema.newBuilder().column("a", DataTypes.INT()).build();
+    CatalogTable existing =
+        catalogCompat().createCatalogTable(schema, null, ImmutableList.of(), 
ImmutableMap.of());
+    TestableBaseCatalog catalog =
+        new TestableBaseCatalog(
+            Mockito.mock(AbstractCatalog.class), gravitinoCatalogHandle, 
existing);
+
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () ->
+            catalog.alterTable(
+                new ObjectPath("db", "t"),
+                existing,
+                Collections.singletonList(
+                    org.apache.flink.table.catalog.TableChange.add(
+                        Column.computed("b", 
resolvedExpression(DataTypes.INT())))),
+                false));
+

Review Comment:
   Added in 3d1279215.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to