This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new ec933085f [#4814] Improvement (catalog-lakehouse-paimon): use
purgeTable for Paimon instead of dropTable inGravitino (#4806)
ec933085f is described below
commit ec933085f39ef230e98c4242f4f1674cd38067ae
Author: cai can <[email protected]>
AuthorDate: Fri Aug 30 20:11:59 2024 +0800
[#4814] Improvement (catalog-lakehouse-paimon): use purgeTable for Paimon
instead of dropTable inGravitino (#4806)
### What changes were proposed in this pull request?
the dropTable in Paimon will both delete the metadata and data and skip
the trash, as discussed in
https://github.com/apache/gravitino/issues/1436 , Gravitino Paimon
catalog should use purgeTable not dropTable
### Why are the changes needed?
https://github.com/apache/gravitino/issues/4814
### Does this PR introduce any user-facing change?
yes, updated the doc.
### How was this patch tested?
existing ITs and UTs.
---
.../lakehouse/paimon/PaimonCatalogOperations.java | 21 +++++++++++----------
.../lakehouse/paimon/ops/PaimonCatalogOps.java | 2 +-
.../lakehouse/paimon/TestGravitinoPaimonTable.java | 4 ++--
.../integration/test/CatalogPaimonBaseIT.java | 4 ++--
.../test/CatalogPaimonKerberosFilesystemIT.java | 4 ++--
.../lakehouse/paimon/ops/TestPaimonCatalogOps.java | 4 ++--
docs/lakehouse-paimon-catalog.md | 19 +++++++++++--------
7 files changed, 31 insertions(+), 27 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java
index 99487fe87..fbe6f4be4 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogOperations.java
@@ -438,15 +438,8 @@ public class PaimonCatalogOperations implements
CatalogOperations, SupportsSchem
*/
@Override
public boolean dropTable(NameIdentifier identifier) {
- try {
- NameIdentifier tableIdentifier = buildPaimonNameIdentifier(identifier);
- paimonCatalogOps.dropTable(tableIdentifier.toString());
- } catch (Catalog.TableNotExistException e) {
- LOG.warn("Paimon table {} does not exist.", identifier);
- return false;
- }
- LOG.info("Dropped Paimon table {}.", identifier);
- return true;
+ throw new UnsupportedOperationException(
+ "Paimon dropTable will both remove the metadata and data, please use
purgeTable instead in Gravitino.");
}
/**
@@ -458,7 +451,15 @@ public class PaimonCatalogOperations implements
CatalogOperations, SupportsSchem
*/
@Override
public boolean purgeTable(NameIdentifier identifier) throws
UnsupportedOperationException {
- throw new UnsupportedOperationException("purgeTable is unsupported now for
Paimon Catalog.");
+ try {
+ NameIdentifier tableIdentifier = buildPaimonNameIdentifier(identifier);
+ paimonCatalogOps.purgeTable(tableIdentifier.toString());
+ } catch (Catalog.TableNotExistException e) {
+ LOG.warn("Paimon table {} does not exist.", identifier);
+ return false;
+ }
+ LOG.info("Purged Paimon table {}.", identifier);
+ return true;
}
@Override
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java
index d009c8bb6..a829b322a 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/PaimonCatalogOps.java
@@ -90,7 +90,7 @@ public class PaimonCatalogOps implements AutoCloseable {
catalog.createTable(tableIdentifier(tableName), schema, false);
}
- public void dropTable(String tableName) throws TableNotExistException {
+ public void purgeTable(String tableName) throws TableNotExistException {
catalog.dropTable(tableIdentifier(tableName), false);
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
index 3a522822a..05e36219d 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java
@@ -101,7 +101,7 @@ public class TestGravitinoPaimonTable {
return NameIdentifier.of(
Namespace.of(levels[levels.length - 1]),
nameIdentifier.name());
})
- .forEach(nameIdentifier ->
paimonCatalogOperations.dropTable(nameIdentifier));
+ .forEach(nameIdentifier ->
paimonCatalogOperations.purgeTable(nameIdentifier));
}
paimonCatalogOperations.dropSchema(schemaIdent, false);
initPaimonSchema();
@@ -358,7 +358,7 @@ public class TestGravitinoPaimonTable {
new SortOrder[0]);
Assertions.assertTrue(paimonCatalogOperations.tableExists(tableIdentifier));
- paimonCatalogOperations.dropTable(tableIdentifier);
+ paimonCatalogOperations.purgeTable(tableIdentifier);
Assertions.assertFalse(paimonCatalogOperations.tableExists(tableIdentifier));
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java
index a8f321f42..e2cb6d6e1 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java
@@ -595,13 +595,13 @@ public abstract class CatalogPaimonBaseIT extends
AbstractIT {
Assertions.assertEquals("table_1", tableIdentifiers.get(0));
Assertions.assertEquals("table_2", tableIdentifiers.get(1));
- Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table1));
+ Assertions.assertDoesNotThrow(() -> tableCatalog.purgeTable(table1));
nameIdentifiers = tableCatalog.listTables(Namespace.of(schemaName));
Assertions.assertEquals(1, nameIdentifiers.length);
Assertions.assertEquals("table_2", nameIdentifiers[0].name());
- Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(table2));
+ Assertions.assertDoesNotThrow(() -> tableCatalog.purgeTable(table2));
Namespace schemaNamespace = Namespace.of(schemaName);
nameIdentifiers = tableCatalog.listTables(schemaNamespace);
Assertions.assertEquals(0, nameIdentifiers.length);
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
index 233c8aff7..f9f31cead 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
@@ -266,8 +266,8 @@ public class CatalogPaimonKerberosFilesystemIT extends
AbstractIT {
Table loadTable = catalog.asTableCatalog().loadTable(tableNameIdentifier);
Assertions.assertEquals(loadTable.name(), tableNameIdentifier.name());
- // Drop table
- catalog.asTableCatalog().dropTable(tableNameIdentifier);
+ // Purge table
+ catalog.asTableCatalog().purgeTable(tableNameIdentifier);
Assertions.assertFalse(catalog.asTableCatalog().tableExists(tableNameIdentifier));
// Drop schema
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java
index dbae9c713..ca741c67b 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/ops/TestPaimonCatalogOps.java
@@ -153,10 +153,10 @@ public class TestPaimonCatalogOps {
assertEquals(OPTIONS.get(BUCKET.key()), table.options().get(BUCKET.key()));
// drop table
- Assertions.assertDoesNotThrow(() ->
paimonCatalogOps.dropTable(IDENTIFIER.toString()));
+ Assertions.assertDoesNotThrow(() ->
paimonCatalogOps.purgeTable(IDENTIFIER.toString()));
Assertions.assertThrowsExactly(
Catalog.TableNotExistException.class,
- () -> paimonCatalogOps.dropTable(IDENTIFIER.toString()));
+ () -> paimonCatalogOps.purgeTable(IDENTIFIER.toString()));
// list table again
Assertions.assertEquals(
diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md
index 2b669ff0b..6eabd3e8f 100644
--- a/docs/lakehouse-paimon-catalog.md
+++ b/docs/lakehouse-paimon-catalog.md
@@ -59,9 +59,9 @@ Please refer to [Manage Relational Metadata Using
Gravitino](./manage-relational
### Schema properties
-- Doesn't support specify location and store any schema properties when
createSchema for FilesystemCatalog now.
-- Doesn't return any schema properties when loadSchema for FilesystemCatalog
now.
-- Doesn't support store schema comment for FilesystemCatalog now.
+- Doesn't support specify location and store any schema properties when
createSchema for FilesystemCatalog.
+- Doesn't return any schema properties when loadSchema for FilesystemCatalog.
+- Doesn't support store schema comment for FilesystemCatalog.
### Schema operations
@@ -71,20 +71,23 @@ Please refer to [Manage Relational Metadata Using
Gravitino](./manage-relational
### Table capabilities
-- Supporting createTable, dropTable, alterTable, loadTable and listTable.
-```
-dropTable will delete the table location directly, similar with purgeTable.
-```
+- Supporting createTable, purgeTable, alterTable, loadTable and listTable.
- Supporting Column default value through table properties, such as
`fields.{columnName}.default-value`, not column expression.
-
+
+- Doesn't support dropTable.
- Doesn't support table distribution and sort orders.
+:::info
+Gravitino Paimon Catalog does not support dropTable, because the dropTable in
Paimon will both remove the table metadata and the table location from the file
system and skip the trash, we should use purgeTable instead in Gravitino.
+:::
+
:::info
Paimon does not support auto increment column.
:::
#### Table changes
+- RenameTable
- AddColumn
- DeleteColumn
- RenameColumn
