This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 87121e260b [core] delete external paths when drop table (#5122)
87121e260b is described below
commit 87121e260bb5cf811b1845a47be71a1d1638c00c
Author: wangwj <[email protected]>
AuthorDate: Thu Feb 27 17:19:44 2025 +0800
[core] delete external paths when drop table (#5122)
---
.../org/apache/paimon/catalog/AbstractCatalog.java | 11 +++-
.../apache/paimon/catalog/FileSystemCatalog.java | 5 +-
.../java/org/apache/paimon/jdbc/JdbcCatalog.java | 7 ++-
.../apache/paimon/utils/FileStorePathFactory.java | 4 ++
.../rest/MetadataInMemoryFileSystemCatalog.java | 4 +-
.../flink/PrimaryKeyFileStoreTableITCase.java | 65 ++++++++++++++++++++++
.../java/org/apache/paimon/hive/HiveCatalog.java | 7 ++-
7 files changed, 95 insertions(+), 8 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 67d6b23187..5a5b964393 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -248,8 +248,13 @@ public abstract class AbstractCatalog implements Catalog {
checkNotBranch(identifier, "dropTable");
checkNotSystemTable(identifier, "dropTable");
+ List<Path> externalPaths = new ArrayList<>();
try {
- getTable(identifier);
+ Table table = getTable(identifier);
+ if (table instanceof FileStoreTable) {
+ FileStoreTable fileStoreTable = (FileStoreTable) table;
+ externalPaths =
fileStoreTable.store().pathFactory().getExternalPaths();
+ }
} catch (TableNotExistException e) {
if (ignoreIfNotExists) {
return;
@@ -257,10 +262,10 @@ public abstract class AbstractCatalog implements Catalog {
throw new TableNotExistException(identifier);
}
- dropTableImpl(identifier);
+ dropTableImpl(identifier, externalPaths);
}
- protected abstract void dropTableImpl(Identifier identifier);
+ protected abstract void dropTableImpl(Identifier identifier, List<Path>
externalPaths);
@Override
public void createTable(Identifier identifier, Schema schema, boolean
ignoreIfExists)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java
index 366051fad5..588c801bf8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java
@@ -112,9 +112,12 @@ public class FileSystemCatalog extends AbstractCatalog {
}
@Override
- protected void dropTableImpl(Identifier identifier) {
+ protected void dropTableImpl(Identifier identifier, List<Path>
externalPaths) {
Path path = getTableLocation(identifier);
uncheck(() -> fileIO.delete(path, true));
+ for (Path externalPath : externalPaths) {
+ uncheck(() -> fileIO.delete(externalPath, true));
+ }
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
index 1a5bd7d125..22a9939d20 100644
--- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java
@@ -259,7 +259,7 @@ public class JdbcCatalog extends AbstractCatalog {
}
@Override
- protected void dropTableImpl(Identifier identifier) {
+ protected void dropTableImpl(Identifier identifier, List<Path>
externalPaths) {
try {
int deletedRecords =
execute(
@@ -278,6 +278,11 @@ public class JdbcCatalog extends AbstractCatalog {
if (fileIO.exists(path)) {
fileIO.deleteDirectoryQuietly(path);
}
+ for (Path externalPath : externalPaths) {
+ if (fileIO.exists(externalPath)) {
+ fileIO.deleteDirectoryQuietly(externalPath);
+ }
+ }
} catch (Exception ex) {
LOG.error("Delete directory[{}] fail for table {}", path,
identifier, ex);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
index e8e40b76c1..86ea2e9781 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
+++
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
@@ -171,6 +171,10 @@ public class FileStorePathFactory {
return new ExternalPathProvider(externalPaths,
relativeBucketPath(partition, bucket));
}
+ public List<Path> getExternalPaths() {
+ return externalPaths;
+ }
+
public Path bucketPath(BinaryRow partition, int bucket) {
return new Path(root, relativeBucketPath(partition, bucket));
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/MetadataInMemoryFileSystemCatalog.java
b/paimon-core/src/test/java/org/apache/paimon/rest/MetadataInMemoryFileSystemCatalog.java
index e0c5ab9446..bd52d7bf10 100644
---
a/paimon-core/src/test/java/org/apache/paimon/rest/MetadataInMemoryFileSystemCatalog.java
+++
b/paimon-core/src/test/java/org/apache/paimon/rest/MetadataInMemoryFileSystemCatalog.java
@@ -210,10 +210,10 @@ public class MetadataInMemoryFileSystemCatalog extends
FileSystemCatalog
}
@Override
- protected void dropTableImpl(Identifier identifier) {
+ protected void dropTableImpl(Identifier identifier, List<Path>
externalPaths) {
if (tableMetadataStore.containsKey(identifier.getFullName())) {
tableMetadataStore.remove(identifier.getFullName());
- super.dropTableImpl(identifier);
+ super.dropTableImpl(identifier, externalPaths);
}
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
index 2b73c59ca8..e63a6cba2b 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java
@@ -275,6 +275,71 @@ public class PrimaryKeyFileStoreTableITCase extends
AbstractTestBase {
assertThat(actual).containsExactlyInAnyOrder("+I[1, A]", "+I[2, B]",
"+I[3, C]");
}
+ @Test
+ public void testDropTableWithExternalPaths() throws Exception {
+ TableEnvironment sEnv =
+ tableEnvironmentBuilder()
+ .streamingMode()
+
.checkpointIntervalMs(ThreadLocalRandom.current().nextInt(900) + 100)
+ .parallelism(1)
+ .build();
+
+ sEnv.executeSql(createCatalogSql("testCatalog", path + "/warehouse"));
+ sEnv.executeSql("USE CATALOG testCatalog");
+ String externalPaths =
+ TraceableFileIO.SCHEME
+ + "://"
+ + externalPath1
+ + ","
+ + LocalFileIOLoader.SCHEME
+ + "://"
+ + externalPath2;
+ sEnv.executeSql(
+ "CREATE TABLE T2 ( k INT, v STRING, PRIMARY KEY (k) NOT
ENFORCED ) "
+ + "WITH ( "
+ + "'bucket' = '1',"
+ + "'data-file.external-paths' = '"
+ + externalPaths
+ + "',"
+ + "'data-file.external-paths.strategy' = 'round-robin'"
+ + ")");
+
+ CloseableIterator<Row> it = collect(sEnv.executeSql("SELECT * FROM
T2"));
+
+ // insert data
+ sEnv.executeSql("INSERT INTO T2 VALUES (1, 'A')").await();
+ // read initial data
+ List<String> actual = new ArrayList<>();
+ for (int i = 0; i < 1; i++) {
+ actual.add(it.next().toString());
+ }
+ assertThat(actual).containsExactlyInAnyOrder("+I[1, A]");
+
+ // insert data
+ sEnv.executeSql("INSERT INTO T2 VALUES (2, 'B')").await();
+
+ for (int i = 0; i < 1; i++) {
+ actual.add(it.next().toString());
+ }
+
+ // insert data
+ sEnv.executeSql("INSERT INTO T2 VALUES (3, 'C')").await();
+
+ for (int i = 0; i < 1; i++) {
+ actual.add(it.next().toString());
+ }
+
+ assertThat(actual).containsExactlyInAnyOrder("+I[1, A]", "+I[2, B]",
"+I[3, C]");
+
+ // drop table
+ sEnv.executeSql("DROP TABLE T2");
+
+ LocalFileIO fileIO = LocalFileIO.create();
+ assertThat(fileIO.exists(new Path(path + "/warehouse" + "/default.db"
+ "/T2"))).isFalse();
+ assertThat(fileIO.exists(new Path(externalPath1))).isFalse();
+ assertThat(fileIO.exists(new Path(externalPath2))).isFalse();
+ }
+
@Test
public void testTableReadWriteWithExternalPathSpecificFS() throws
Exception {
TableEnvironment sEnv =
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 057930511b..d102ec0c53 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -920,7 +920,7 @@ public class HiveCatalog extends AbstractCatalog {
}
@Override
- protected void dropTableImpl(Identifier identifier) {
+ protected void dropTableImpl(Identifier identifier, List<Path>
externalPaths) {
try {
boolean externalTable = isExternalTable(getHmsTable(identifier));
clients.execute(
@@ -946,6 +946,11 @@ public class HiveCatalog extends AbstractCatalog {
if (fileIO.exists(path)) {
fileIO.deleteDirectoryQuietly(path);
}
+ for (Path externalPath : externalPaths) {
+ if (fileIO.exists(externalPath)) {
+ fileIO.deleteDirectoryQuietly(externalPath);
+ }
+ }
} catch (Exception ee) {
LOG.error("Delete directory[{}] fail for table {}", path,
identifier, ee);
}
