This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new ee9422bbd [#5465] fix(core): deleting location of managed sub-entities
when deleting a fileset catalog (#5692)
ee9422bbd is described below
commit ee9422bbdee28a860f284896867a261058b46ff2
Author: mchades <[email protected]>
AuthorDate: Thu Nov 28 10:44:00 2024 +0800
[#5465] fix(core): deleting location of managed sub-entities when deleting
a fileset catalog (#5692)
### What changes were proposed in this pull request?
drop the location of managed schemas and filesets when dropping a
fileset catalog
### Why are the changes needed?
Fix: #5465
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
tests added
---
.../client/integration/test/CatalogIT.java | 44 +++++++++++++++++++++-
.../apache/gravitino/catalog/CatalogManager.java | 18 +++++++++
2 files changed, 61 insertions(+), 1 deletion(-)
diff --git
a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java
b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java
index bb6394b6c..8f6a7dd34 100644
---
a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java
+++
b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java
@@ -24,6 +24,9 @@ import static org.apache.gravitino.Catalog.PROPERTY_IN_USE;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Collections;
import java.util.Map;
import org.apache.commons.lang3.ArrayUtils;
@@ -37,6 +40,7 @@ import org.apache.gravitino.client.GravitinoMetalake;
import org.apache.gravitino.exceptions.CatalogAlreadyExistsException;
import org.apache.gravitino.exceptions.CatalogInUseException;
import org.apache.gravitino.exceptions.CatalogNotInUseException;
+import org.apache.gravitino.file.Fileset;
import org.apache.gravitino.file.FilesetCatalog;
import org.apache.gravitino.file.FilesetChange;
import org.apache.gravitino.integration.test.container.ContainerSuite;
@@ -124,7 +128,7 @@ public class CatalogIT extends BaseIT {
}
@Test
- public void testDropCatalog() {
+ public void testDropCatalog() throws IOException {
String catalogName = GravitinoITUtils.genRandomName("catalog");
Assertions.assertFalse(metalake.catalogExists(catalogName));
@@ -153,6 +157,44 @@ public class CatalogIT extends BaseIT {
Assertions.assertDoesNotThrow(() -> metalake.disableCatalog(catalogName));
Assertions.assertTrue(metalake.dropCatalog(catalogName), "catalog should
be dropped");
Assertions.assertFalse(metalake.dropCatalog(catalogName), "catalog should
be non-existent");
+
+ // test drop catalog with managed entity
+ catalog =
+ metalake.createCatalog(
+ catalogName, Catalog.Type.FILESET, "hadoop", null,
ImmutableMap.of());
+
+ String schemaName = GravitinoITUtils.genRandomName("schema");
+ Path schemaDir = Files.createTempDirectory(schemaName);
+ catalog
+ .asSchemas()
+ .createSchema(schemaName, null, ImmutableMap.of("location",
schemaDir.toString()));
+ Assertions.assertTrue(Files.exists(schemaDir));
+ Assertions.assertEquals(
+ schemaDir.toString(),
+
catalog.asSchemas().loadSchema(schemaName).properties().get("location"));
+
+ String filesetName = GravitinoITUtils.genRandomName("fileset");
+ Path filesetDir = Files.createTempDirectory(filesetName);
+ catalog
+ .asFilesetCatalog()
+ .createFileset(
+ NameIdentifier.of(schemaName, filesetName),
+ null,
+ Fileset.Type.MANAGED,
+ filesetDir.toString(),
+ null);
+ Assertions.assertTrue(Files.exists(filesetDir));
+ Assertions.assertEquals(
+ "file:" + filesetDir,
+ catalog
+ .asFilesetCatalog()
+ .loadFileset(NameIdentifier.of(schemaName, filesetName))
+ .storageLocation());
+
+ Assertions.assertTrue(metalake.dropCatalog(catalogName, true));
+ Assertions.assertFalse(metalake.dropCatalog(catalogName));
+ Assertions.assertFalse(Files.exists(schemaDir));
+ Assertions.assertFalse(Files.exists(filesetDir));
}
@Test
diff --git
a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
index 959f91f5c..6f77bb462 100644
--- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
+++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.catalog;
import static org.apache.gravitino.Catalog.PROPERTY_IN_USE;
+import static org.apache.gravitino.Catalog.Type.FILESET;
import static org.apache.gravitino.StringIdentifier.DUMMY_ID;
import static
org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter;
import static
org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate;
@@ -657,6 +658,19 @@ public class CatalogManager implements CatalogDispatcher,
Closeable {
}
}
+ CatalogWrapper catalogWrapper = loadCatalogAndWrap(ident);
+ if (includeManagedEntities(catalogEntity)) {
+ schemas.forEach(
+ schema -> {
+ try {
+ catalogWrapper.doWithSchemaOps(
+ schemaOps -> schemaOps.dropSchema(schema.nameIdentifier(),
true));
+ } catch (Exception e) {
+ LOG.warn("Failed to drop schema {}", schema.nameIdentifier());
+ throw new RuntimeException("Failed to drop schema " +
schema.nameIdentifier(), e);
+ }
+ });
+ }
catalogCache.invalidate(ident);
return store.delete(ident, EntityType.CATALOG, true);
@@ -668,6 +682,10 @@ public class CatalogManager implements CatalogDispatcher,
Closeable {
}
}
+ private boolean includeManagedEntities(CatalogEntity catalogEntity) {
+ return catalogEntity.getType().equals(FILESET);
+ }
+
/**
* Loads the catalog with the specified identifier, wraps it in a
CatalogWrapper, and caches the
* wrapper for reuse.
