This is an automated email from the ASF dual-hosted git repository.

yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git


The following commit(s) were added to refs/heads/main by this push:
     new 8985ffad65 [#13696] fix(lance): require catalog opt-in to purge 
external datasets (#14870)
8985ffad65 is described below

commit 8985ffad65fca3c62a7ddbc32144f58956f2c953
Author: Qi Yu <[email protected]>
AuthorDate: Wed Nov 11 14:37:33 2026 +0800

    [#13696] fix(lance): require catalog opt-in to purge external datasets 
(#14870)
    
    ### What changes were proposed in this pull request?
    
    Add the catalog-only `lance.allow-external-table-purge` property,
    defaulting to `false`. Reject external or registered Lance table purges
    before removing metadata or deleting files, including Lance REST drop
    and create-overwrite of an existing external table. Table properties
    cannot enable the policy.
    
    ### Why are the changes needed?
    
    A caller can own a table registered at another table's location, or
    created at a parent directory containing other datasets. Table ownership
    alone does not authorize deleting that data. This provides a default
    deletion guard; location ownership and storage authorization remain
    follow-up work in #14788.
    
    Refs #13696, #14788.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. External purge is disabled by default; Lance REST returns HTTP 406.
    Catalog administrators must explicitly opt in to retain destructive
    drop/overwrite behavior. The opt-in does not validate location
    ownership. Metadata-only drop/deregister and managed-table deletion
    remain unchanged.
    
    ### How was this patch tested?
    
    - 201 focused unit tests and 48 embedded integration tests passed,
    covering policy defaults, table-property bypass attempts, legacy
    registrations, overwrite, provider callbacks, and catalog opt-in.
    - Two authenticated cross-user HTTP regression cases reproduce #13696.
    Both fail with HTTP 200 when the guard is removed and pass with HTTP 406
    while preserving readable victim datasets when restored.
    - Scoped Spotless checks and `git diff --check` passed. No Docker or
    remote object-store tests were run.
---
 .../generic/GenericCatalogPropertiesMetadata.java  |  11 +++
 .../lakehouse/lance/LanceTableOperations.java      |  11 ++-
 .../generic/TestGenericCatalogOperations.java      |  23 ++---
 .../lakehouse/generic/TestPropertiesMetadata.java  |  15 +++
 .../lakehouse/lance/TestLanceTableOperations.java  | 107 +++++++++++++++++++--
 .../test/CatalogGenericCatalogLanceIT.java         |   2 +
 docs/lakehouse-generic-catalog.md                  |  12 ++-
 docs/lakehouse-generic-lance-table.md              |   9 +-
 docs/lance-rest-service.md                         |  15 ++-
 .../lance/common/utils/LanceConstants.java         |   3 +
 .../lance/integration/test/LanceRESTServiceIT.java |  30 +++++-
 .../test/LanceTableAuthorizationIT.java            |  68 ++++++++++++-
 12 files changed, 269 insertions(+), 37 deletions(-)

diff --git 
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
 
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
index 7c84d3a54d..b63409edea 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
@@ -19,9 +19,11 @@
 
 package org.apache.gravitino.catalog.lakehouse.generic;
 
+import static 
org.apache.gravitino.connector.PropertyEntry.booleanPropertyEntry;
 import static org.apache.gravitino.connector.PropertyEntry.enumPropertyEntry;
 import static 
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyEntry;
 import static 
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyPrefixEntry;
+import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_SCHEMA_REFRESH_MODE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
 
@@ -70,6 +72,15 @@ public class GenericCatalogPropertiesMetadata extends 
BaseCatalogPropertiesMetad
                 true /* immutable */,
                 DefaultTableLocationProvider.NAME, /* defaultValue */
                 false /* hidden */),
+            booleanPropertyEntry(
+                LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+                "Allow physical deletion of external and registered Lance 
datasets. Table"
+                    + " properties cannot enable this catalog-level policy.",
+                false /* required */,
+                false /* immutable */,
+                false /* defaultValue */,
+                false /* hidden */,
+                false /* reserved */),
             enumPropertyEntry(
                 LANCE_SCHEMA_REFRESH_MODE,
                 "Controls when Lance table schemas are refreshed from the 
underlying dataset.",
diff --git 
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
 
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
index 82404cd8b7..dd36f28c19 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
@@ -18,6 +18,7 @@
  */
 package org.apache.gravitino.catalog.lakehouse.lance;
 
+import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_REGISTER;
 import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
@@ -309,6 +310,14 @@ public class LanceTableOperations extends 
ManagedTableOperations {
       Table table = super.loadTable(ident);
       validateLanceTable(ident, table);
       boolean external = 
LancePropertiesUtils.isExternalTable(table.properties());
+      if (external
+          && 
!Boolean.parseBoolean(catalogProperties.get(LANCE_ALLOW_EXTERNAL_TABLE_PURGE))) 
{
+        throw new UnsupportedOperationException(
+            "Purging external Lance tables is disabled. Use dropTable or Lance 
REST deregister"
+                + " to remove metadata only. A catalog administrator may 
enable "
+                + LANCE_ALLOW_EXTERNAL_TABLE_PURGE
+                + " to allow physical deletion of external data.");
+      }
 
       // NOTE: the two steps below (metadata removal and dataset deletion) are 
NOT atomic.
       // If the process crashes between them, the entity-store record will be 
gone but the
@@ -331,7 +340,7 @@ public class LanceTableOperations extends 
ManagedTableOperations {
 
     } catch (NoSuchTableException e) {
       return false;
-    } catch (IllegalArgumentException e) {
+    } catch (IllegalArgumentException | UnsupportedOperationException e) {
       throw e;
     } catch (OptimisticLockException e) {
       // The table metadata conflicted with a concurrent writer, e.g. a lock 
wait timeout, and was
diff --git 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
index 4278939881..ba28e488ad 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
@@ -50,7 +50,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.Arrays;
@@ -316,28 +315,18 @@ public class TestGenericCatalogOperations {
     store.put(entity, false);
 
     if ("purge".equals(operation)) {
-      Assertions.assertTrue(opsWithFakeProvider.purgeTable(ident));
+      Assertions.assertThrowsExactly(
+          UnsupportedOperationException.class, () -> 
opsWithFakeProvider.purgeTable(ident));
     } else if ("cascade".equals(operation)) {
       Assertions.assertTrue(opsWithFakeProvider.dropSchema(schemaIdent, true));
     } else {
       Assertions.assertTrue(opsWithFakeProvider.dropTable(ident));
     }
 
-    Assertions.assertFalse(opsWithFakeProvider.tableExists(ident));
-    if ("purge".equals(operation)) {
-      
Assertions.assertFalse(Files.exists(lanceDataDir.resolve("legacy/_versions")));
-      Assertions.assertEquals(1, 
FakeTableLocationProvider.unprovisioned().size());
-      Assertions.assertEquals(
-          location,
-          FakeTableLocationProvider.unprovisioned()
-              .get(0)
-              .tableProperties()
-              .get(Table.PROPERTY_LOCATION));
-    } else {
-      
Assertions.assertTrue(FakeTableLocationProvider.unprovisioned().isEmpty());
-      try (Dataset dataset = Dataset.open().uri(location).build()) {
-        Assertions.assertEquals("id", 
dataset.getSchema().getFields().get(0).getName());
-      }
+    Assertions.assertEquals("purge".equals(operation), 
opsWithFakeProvider.tableExists(ident));
+    Assertions.assertTrue(FakeTableLocationProvider.unprovisioned().isEmpty());
+    try (Dataset dataset = Dataset.open().uri(location).build()) {
+      Assertions.assertEquals("id", 
dataset.getSchema().getFields().get(0).getName());
     }
   }
 
diff --git 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
index bbc1fbcd13..4f468e1cf1 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
@@ -19,6 +19,8 @@
 
 package org.apache.gravitino.catalog.lakehouse.generic;
 
+import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
+
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;
 import org.apache.gravitino.Schema;
@@ -53,6 +55,19 @@ public class TestPropertiesMetadata {
             GenericCatalogPropertiesMetadata.TABLE_LOCATION_PROVIDER));
   }
 
+  @Test
+  void 
testExternalPurgePolicyDefaultsToDisabledAndCanBeChangedAtCatalogLevel() {
+    PropertiesMetadata metadata = genericCatalog.catalogPropertiesMetadata();
+    
Assertions.assertTrue(metadata.containsProperty(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+    Assertions.assertEquals(
+        false, metadata.getOrDefault(Map.of(), 
LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+    Assertions.assertEquals(
+        true,
+        metadata.getOrDefault(
+            Map.of(LANCE_ALLOW_EXTERNAL_TABLE_PURGE, "true"), 
LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+    
Assertions.assertFalse(metadata.isImmutableProperty(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+  }
+
   @Test
   void testCatalogPropertiesMetadata() {
     PropertiesMetadata catalogPropertiesMetadata = 
genericCatalog.catalogPropertiesMetadata();
diff --git 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
index 310c3dc918..0e1363b416 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
@@ -18,6 +18,7 @@
  */
 package org.apache.gravitino.catalog.lakehouse.lance;
 
+import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_SCHEMA_REFRESH_MODE;
 import static 
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
@@ -167,7 +168,7 @@ public class TestLanceTableOperations {
     }
   }
 
-  /** Legacy registration flags also protect drop and overwrite, while purge 
remains destructive. */
+  /** Legacy registrations retain their data unless the catalog explicitly 
permits purging it. */
   @ParameterizedTest
   @CsvSource(
       value = {
@@ -232,17 +233,111 @@ public class TestLanceTableOperations {
                       new Index[0]));
       Assertions.assertEquals("true", 
replaced.properties().get(Table.PROPERTY_EXTERNAL));
     } else if ("purge".equals(operation)) {
-      Assertions.assertTrue(lanceTableOps.purgeTable(ident));
+      Assertions.assertThrowsExactly(
+          UnsupportedOperationException.class, () -> 
lanceTableOps.purgeTable(ident));
     } else {
       Assertions.assertTrue(lanceTableOps.dropTable(ident));
     }
-    verify(store).delete(ident, Entity.EntityType.TABLE);
-    if ("purge".equals(operation)) {
-      Assertions.assertFalse(Files.exists(Paths.get(location, "_versions")));
-    } else {
+    verify(store, Mockito.times("purge".equals(operation) ? 0 : 1))
+        .delete(ident, Entity.EntityType.TABLE);
+    try (Dataset dataset = Dataset.open().uri(location).build()) {
+      Assertions.assertEquals("id", 
dataset.getSchema().getFields().get(0).getName());
+    }
+  }
+
+  @ParameterizedTest
+  @CsvSource(
+      value = {"true,NULL", "true,false", "true,true", "false,NULL", 
"false,false", "false,true"},
+      nullValues = "NULL")
+  void testPurgeRequiresCatalogPermissionForExternalData(boolean external, 
String catalogPermission)
+      throws Exception {
+    NameIdentifier ident = NameIdentifier.of("schema", "purge_policy");
+    String location = tempDir.resolve("purge_policy").toString();
+    try (Dataset ignored =
+        lanceTableOps.createDataset(
+            location, new Column[] {Column.of("id", Types.IntegerType.get())}, 
Map.of())) {
+      // Purge must preserve both a real dataset and its metadata when the 
policy rejects it.
+    }
+    Map<String, String> properties =
+        Map.of(
+            Table.PROPERTY_TABLE_FORMAT,
+            "lance",
+            Table.PROPERTY_LOCATION,
+            location,
+            Table.PROPERTY_EXTERNAL,
+            Boolean.toString(external),
+            LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+            "true");
+    when(store.get(eq(ident), eq(Entity.EntityType.TABLE), 
eq(TableEntity.class)))
+        .thenReturn(tableEntity(ident, List.of(), properties));
+    when(store.delete(ident, Entity.EntityType.TABLE)).thenReturn(true);
+    if (catalogPermission != null) {
+      lanceTableOps.setCatalogProperties(
+          Map.of(LANCE_ALLOW_EXTERNAL_TABLE_PURGE, catalogPermission));
+    }
+
+    if (external && !Boolean.parseBoolean(catalogPermission)) {
+      UnsupportedOperationException error =
+          Assertions.assertThrowsExactly(
+              UnsupportedOperationException.class, () -> 
lanceTableOps.purgeTable(ident));
+      
Assertions.assertTrue(error.getMessage().contains(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+      verify(store, never()).delete(any(), any());
       try (Dataset dataset = Dataset.open().uri(location).build()) {
         Assertions.assertEquals("id", 
dataset.getSchema().getFields().get(0).getName());
       }
+    } else {
+      Assertions.assertTrue(lanceTableOps.purgeTable(ident));
+      verify(store).delete(ident, Entity.EntityType.TABLE);
+      Assertions.assertFalse(Files.exists(Paths.get(location, "_versions")));
+    }
+  }
+
+  @Test
+  void testOverwriteCannotPurgeExternalDataWithoutCatalogPermission() throws 
Exception {
+    NameIdentifier ident = NameIdentifier.of("schema", "protected_table");
+    String location = tempDir.resolve("protected_table").toString();
+    Column[] columns = {Column.of("id", Types.IntegerType.get())};
+    try (Dataset ignored = lanceTableOps.createDataset(location, columns, 
Map.of())) {
+      // The existing external dataset is not owned by this overwrite request.
+    }
+    when(store.get(eq(ident), eq(Entity.EntityType.TABLE), 
eq(TableEntity.class)))
+        .thenReturn(
+            tableEntity(
+                ident,
+                List.of(),
+                Map.of(
+                    Table.PROPERTY_TABLE_FORMAT,
+                    "lance",
+                    Table.PROPERTY_LOCATION,
+                    location,
+                    Table.PROPERTY_EXTERNAL,
+                    "true")));
+    String replacement = tempDir.resolve("replacement").toString();
+    Assertions.assertThrowsExactly(
+        UnsupportedOperationException.class,
+        () ->
+            lanceTableOps.createTable(
+                ident,
+                columns,
+                null,
+                Map.of(
+                    Table.PROPERTY_TABLE_FORMAT,
+                    "lance",
+                    Table.PROPERTY_LOCATION,
+                    replacement,
+                    LANCE_CREATION_MODE,
+                    "OVERWRITE",
+                    LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+                    "true"),
+                new Transform[0],
+                null,
+                new SortOrder[0],
+                new Index[0]));
+    verify(store, never()).delete(any(), any());
+    verify(store, never()).put(any(), anyBoolean());
+    Assertions.assertFalse(Files.exists(Paths.get(replacement)));
+    try (Dataset dataset = Dataset.open().uri(location).build()) {
+      Assertions.assertEquals("id", 
dataset.getSchema().getFields().get(0).getName());
     }
   }
 
diff --git 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
index fc822a8678..348fb6a38d 100644
--- 
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
+++ 
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
@@ -812,6 +812,8 @@ public class CatalogGenericCatalogLanceIT extends BaseIT {
 
   protected void createCatalog() {
     Map<String, String> properties = Maps.newHashMap();
+    // This suite exercises explicit data deletion, including purge and 
create-overwrite.
+    properties.put("lance.allow-external-table-purge", "true");
     metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, provider, 
"comment", properties);
 
     catalog = metalake.loadCatalog(catalogName);
diff --git a/docs/lakehouse-generic-catalog.md 
b/docs/lakehouse-generic-catalog.md
index a2963807cb..109ddfc8e4 100644
--- a/docs/lakehouse-generic-catalog.md
+++ b/docs/lakehouse-generic-catalog.md
@@ -50,6 +50,7 @@ For detailed information on available operations, see [Manage 
Relational Metadat
 | `location`                  | Root storage path for all schemas and tables   
                                                                                
                                                                                
                                                                                
                           | `s3://bucket/lakehouse` | No       |
 | `table-location-provider`   | Name of the [table location 
provider](#pluggable-table-location-provider) that provisions and unprovisions 
the locations of this catalog's tables. Defaults to `default`, which resolves 
the location from the table, schema and catalog `location` properties as 
described below. Immutable once the catalog is created. | `default`             
  | No       |
 | `lance.schema-refresh-mode` | Lance table schema refresh mode. 
`DECLARED_AND_EMPTY` (default) refreshes declared tables and tables with empty 
stored columns. `VERSION_CHECK` additionally refreshes when the Lance dataset 
version changes.                                                                
                                            | `DECLARED_AND_EMPTY`    | No      
 |
+| `lance.allow-external-table-purge` | Allow physical deletion of external or 
registered Lance tables, including Lance REST `DropTable` and create 
`OVERWRITE`. Catalog-only; table properties cannot enable it. Does not validate 
ownership of storage locations. | `false` | No |
 
 #### Pluggable table location provider
 
@@ -180,12 +181,15 @@ location back would invite it to delete exactly the data 
the catalog just promis
 leak is recoverable and a deletion is not, so `unprovisionTableLocation` is 
not called when a table
 whose `external` property is true is dropped.
 
-A purge is the opposite request. `LanceTableOperations.purgeTable` deletes the 
external dataset that
-its `dropTable` leaves alone, so by the time the callback would run the data 
is gone and the reason
-to skip has gone with it; the location is handed back.
+For Lance, an external-table purge requires 
`lance.allow-external-table-purge=true` in the catalog.
+Without it, purge fails before removing table metadata or deleting data, and 
the provider callback is
+not invoked.
+When allowed, purge deletes the external dataset that drop leaves alone; after 
successful deletion,
+the location is handed back.
 
 The callback is therefore reached in exactly three situations: a dropped 
managed table, a purged
-managed table, and a purged external one. There is deliberately no flag 
distinguishing them, because
+managed table, and an external one whose purge was permitted by its table 
format. There is deliberately
+no flag distinguishing them, because
 the invariant they share is about the location rather than the table: 
**nothing anyone needs is under
 the location named in the context**. In all three the table format has already 
deleted the data under
 it. The catalog makes the drop-versus-purge and external distinctions rather 
than passing them on,
diff --git a/docs/lakehouse-generic-lance-table.md 
b/docs/lakehouse-generic-lance-table.md
index c6b5e55c35..8908a1d20d 100644
--- a/docs/lakehouse-generic-lance-table.md
+++ b/docs/lakehouse-generic-lance-table.md
@@ -148,14 +148,14 @@ Required and optional properties for tables in a Generic 
Lakehouse Catalog:
 
|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------|
 | `format`              | Table format: `lance`, only `lance` is fully 
supported.                                                                      
                                                                                
                                                                                
                                            | (none)   | Yes          |
 | `location`            | Storage path for table metadata and data, Lance 
supports: S3, GCS, OSS, AZ, File, Memory and file-object-store.                 
                                                                                
                                                                                
                                         | (none)   | Conditional* |
-| `external`            | Whether the data directory is an external location. 
If it's `true`, dropping a table will only remove metadata in Gravitino and 
will not delete the data directory, and purge table will delete both. For a 
non-external table, dropping will drop both.                                    
                                             | false    | No           |
+| `external`            | Whether the data directory is an external location. 
If it's `true`, dropping a table will only remove metadata in Gravitino and 
will not delete the data directory, and purge is rejected unless the catalog 
enables `lance.allow-external-table-purge`. For a non-external table, dropping 
will drop both.                                                                 
                | false    | No           |
 | `lance.creation-mode` | Create mode: for create table, it can be `CREATE`, 
`EXIST_OK` or `OVERWRITE`. and it should be `CREATE` or `OVERWRITE` for 
registering tables                                                              
                                                                                
                                              | `CREATE` | No           |
 | `lance.register`      | Whether it is a register table operation. If `true`, 
register an existing, readable Lance dataset without creating data. A missing 
dataset or plain directory is rejected. Registered tables are external. If 
`false`, create a dataset.                                                      
                                           | false    | No           |
 | `lance.storage.xxxx`  | Any additional storage-specific properties required 
by Lance format (e.g., S3 credentials, HDFS configs). Replace `xxxx` with 
actual property names. For example, we can use 
`lance.storage.aws_access_key_id` to set S3 aws_access_key_id when using a S3 
location, for detail, refer to https://lancedb.com/docs/storage/integrations/ | 
(none)   | No           |
 
 - `CREATE`: Create a new table, fail if the table already exists.
 - `EXIST_OK`: Create a new table if it does not exist, otherwise do nothing.
-- `OVERWRITE`: For creation (`lance.register=false`), purge the existing table 
and its dataset before creating a replacement. For registration 
(`lance.register=true`), validate the replacement dataset first and replace 
only external-table metadata, retaining the old dataset. Registration overwrite 
of a managed table is rejected; explicitly dropping or purging a managed table 
deletes its data.
+- `OVERWRITE`: For creation (`lance.register=false`), purge the existing table 
and its dataset before creating a replacement. An existing external table 
requires the catalog to enable `lance.allow-external-table-purge`; otherwise 
the original metadata and data are retained. For registration 
(`lance.register=true`), validate the replacement dataset first and replace 
only external-table metadata, retaining the old dataset. Registration overwrite 
of a managed table is rejected; explicitly d [...]
 
 Registration validation checks dataset existence and readability, not an 
operator-defined storage scope or authorization to purge external data.
 
@@ -302,7 +302,10 @@ tableCatalog.createTable(
 Register existing Lance tables without moving or copying data. A table with 
`lance.register=true`
 is always external, including when the request omits `external` or sets it to 
`false`. Dropping it
 through the Gravitino table API removes only the metadata. Explicit 
`purgeTable` and Lance REST
-`drop` still delete both the metadata and the dataset; use Lance REST 
`deregister` to preserve data.
+`drop` are rejected by default, preserving metadata and data. A catalog 
administrator may set
+`lance.allow-external-table-purge=true` to allow both to be deleted. This 
setting is read only from
+catalog properties; a table property cannot enable it. Use Lance REST 
`deregister` to remove
+metadata while preserving data. The opt-in does not validate location 
ownership.
 
 Existing registrations with a missing or false stored `external` flag receive 
the same protection
 when dropped or deregistered. Their stored properties are not automatically 
migrated.
diff --git a/docs/lance-rest-service.md b/docs/lance-rest-service.md
index e67be78e35..490258d419 100644
--- a/docs/lance-rest-service.md
+++ b/docs/lance-rest-service.md
@@ -127,9 +127,22 @@ versioned metadata reads are not implemented.
 :::
 
 **DropTable vs DeregisterTable**:
-- **DropTable**: Permanently deletes metadata and data files from storage
+- **DropTable**: Permanently deletes metadata and data files when permitted by 
the catalog policy
 - **DeregisterTable**: Removes metadata from Gravitino but preserves Lance 
data files
 
+The generic catalog rejects purging external or registered Lance tables by 
default, before removing
+table metadata or deleting files. Lance REST reports this as HTTP `406` 
(`UnsupportedOperation`). Tables
+created, declared or registered through Lance REST are external, so this 
default also applies to them.
+Use `DeregisterTable` to remove their metadata while retaining their data.
+
+A catalog administrator can set `lance.allow-external-table-purge=true` on the 
`lakehouse-generic`
+catalog to permit physical deletion. This catalog-only setting also applies to 
create `OVERWRITE`
+when it replaces an existing external table. A table property or request 
header cannot enable it.
+Existing catalogs must explicitly opt in to retain their previous destructive 
drop/overwrite behavior.
+Enabling it permits deletion at caller-supplied locations; it does not verify 
storage ownership or
+provide isolation between tables. Enable it only when table owners are trusted 
to delete data at
+locations they can register. Underlying storage permissions still apply.
+
 
 ## Deployment
 
diff --git 
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
 
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
index 197388e8d4..4b238a4504 100644
--- 
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
+++ 
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
@@ -42,5 +42,8 @@ public class LanceConstants {
   public static final String LANCE_TABLE_DECLARED = "lance.declared";
   public static final String LANCE_SCHEMA_REFRESH_MODE = 
"lance.schema-refresh-mode";
 
+  /** Catalog-only opt-in for physically deleting external or registered Lance 
datasets. */
+  public static final String LANCE_ALLOW_EXTERNAL_TABLE_PURGE = 
"lance.allow-external-table-purge";
+
   public static final String LANCE_TABLE_FORMAT = "lance";
 }
diff --git 
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
 
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
index 42c699c5d2..d6cf096c01 100644
--- 
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
+++ 
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
@@ -47,6 +47,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.commons.io.FileUtils;
 import org.apache.gravitino.Catalog;
+import org.apache.gravitino.CatalogChange;
 import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.Schema;
 import org.apache.gravitino.client.GravitinoMetalake;
@@ -574,6 +575,26 @@ public class LanceRESTServiceIT extends BaseIT {
             "lance.storage.a", "value_va",
             "lance.storage.b", "value_vb");
 
+    LanceNamespaceException denied =
+        Assertions.assertThrows(
+            LanceNamespaceException.class,
+            () -> createTable(ids, newLocation, overwriteTableProperties, 
body, "overwrite"));
+    Assertions.assertTrue(
+        
denied.getMessage().contains(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+    Assertions.assertEquals(
+        location,
+        catalog
+            .asTableCatalog()
+            .loadTable(NameIdentifier.of(SCHEMA_NAME, ids.get(2)))
+            .properties()
+            .get(Table.PROPERTY_LOCATION));
+    Assertions.assertTrue(new File(location).exists());
+    Assertions.assertFalse(new File(newLocation).exists());
+    catalog =
+        metalake.alterCatalog(
+            CATALOG_NAME,
+            
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE, 
"true"));
+
     response =
         Assertions.assertDoesNotThrow(
             () -> createTable(ids, newLocation, overwriteTableProperties, 
body, "overwrite"));
@@ -1055,7 +1076,7 @@ public class LanceRESTServiceIT extends BaseIT {
 
   @Test
   void testDropTable() {
-    catalog = createCatalog(CATALOG_NAME);
+    catalog = createCatalog(CATALOG_NAME, true);
     createSchema();
 
     List<String> ids = List.of(CATALOG_NAME, SCHEMA_NAME, "table_to_drop");
@@ -1239,7 +1260,7 @@ public class LanceRESTServiceIT extends BaseIT {
 
   @Test
   void testDropTableWhenLocationMissingOrNotCreated() throws IOException {
-    catalog = createCatalog(CATALOG_NAME);
+    catalog = createCatalog(CATALOG_NAME, true);
     createSchema();
 
     // Register a valid dataset, then remove its location before dropping the 
table
@@ -1293,9 +1314,14 @@ public class LanceRESTServiceIT extends BaseIT {
   }
 
   private Catalog createCatalog(String catalogName) {
+    return createCatalog(catalogName, false);
+  }
+
+  private Catalog createCatalog(String catalogName, boolean 
allowExternalPurge) {
     Map<String, String> catalogProperties =
         ImmutableMap.<String, String>builder()
             .putAll(properties)
+            .put("lance.allow-external-table-purge", 
Boolean.toString(allowExternalPurge))
             .put("lance.storage.endpoint", MINIO_ENDPOINT)
             .put("lance.storage.allow_http", "true")
             .put("lance.storage.access_key_id", MINIO_ACCESS_KEY)
diff --git 
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
 
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
index 400eb40add..cfcbc129b0 100644
--- 
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
+++ 
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
@@ -23,6 +23,7 @@ import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Base64;
@@ -33,6 +34,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.gravitino.Catalog;
+import org.apache.gravitino.CatalogChange;
 import org.apache.gravitino.Configs;
 import org.apache.gravitino.MetadataObject;
 import org.apache.gravitino.MetadataObjects;
@@ -55,6 +57,9 @@ import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.lance.Dataset;
 import org.lance.namespace.model.AlterTableDropColumnsRequest;
 import org.lance.namespace.model.CreateNamespaceRequest;
 import org.lance.namespace.model.DeclareTableRequest;
@@ -334,11 +339,25 @@ public class LanceTableAuthorizationIT extends BaseIT {
     // The denied requests left the table in place.
     assertStatus(200, table(ADMIN, VISIBLE_TABLE, "describe"));
 
-    // Creating assigns ownership, so the creator may use either removal 
operation.
+    // Ownership permits deregistration. Physical deletion also requires 
catalog permission.
     assertStatus(200, registerExistingTable(PROBER, WRITE_SCHEMA, 
DEREGISTER_TABLE));
     assertStatus(200, registerExistingTable(PROBER, WRITE_SCHEMA, DROP_TABLE));
     assertStatus(200, table(PROBER, WRITE_SCHEMA, DEREGISTER_TABLE, 
"deregister"));
-    assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+    assertStatus(406, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+    assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "describe"));
+    Assertions.assertTrue(Files.exists(Path.of(location(DROP_TABLE))));
+    GravitinoMetalake metalake = 
client.loadMetalake(getLanceRESTServerMetalakeName());
+    metalake.alterCatalog(
+        CATALOG,
+        
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE, 
"true"));
+    try {
+      assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+      Assertions.assertFalse(Files.exists(Path.of(location(DROP_TABLE))));
+    } finally {
+      metalake.alterCatalog(
+          CATALOG,
+          
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE, 
"false"));
+    }
     assertStatus(404, table(ADMIN, WRITE_SCHEMA, DEREGISTER_TABLE, 
"describe"));
     assertStatus(404, table(ADMIN, WRITE_SCHEMA, DROP_TABLE, "describe"));
   }
@@ -347,8 +366,11 @@ public class LanceTableAuthorizationIT extends BaseIT {
   public void testSchemaOwnershipRemovesTablesTheOwnerDidNotCreate() throws 
Exception {
     // SCHEMA_OWNER_WITH_USE_CATALOG: the admin registered this table, so the 
only thing the caller
     // has is ownership of the schema holding it, plus USE_CATALOG.
-    assertStatus(200, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE, 
"drop"));
+    assertStatus(406, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE, 
"drop"));
+    assertStatus(200, table(ADMIN, OWNED_SCHEMA, OWNED_SCHEMA_TABLE, 
"describe"));
+    assertStatus(200, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE, 
"deregister"));
     assertStatus(404, table(ADMIN, OWNED_SCHEMA, OWNED_SCHEMA_TABLE, 
"describe"));
+    Assertions.assertTrue(Files.exists(Path.of(location(OWNED_SCHEMA_TABLE))));
   }
 
   @Test
@@ -371,6 +393,46 @@ public class LanceTableAuthorizationIT extends BaseIT {
     assertStatus(404, table(ADMIN, MISSING_TABLE, "deregister"));
   }
 
+  @ParameterizedTest
+  @ValueSource(strings = {"register", "create"})
+  void testTableOwnerCannotPurgeAnotherSchemasData(String operation) throws 
Exception {
+    String victim = "purge_victim_" + operation;
+    String alias = "purge_alias_" + operation;
+    Path parent = tempDir.resolve("purge_parent_" + operation);
+    Path victimLocation = parent.resolve("victim");
+    LanceDatasetTestUtils.createDataset(victimLocation.toString());
+    assertStatus(200, register(ADMIN, SCHEMA, victim, null, 
victimLocation.toString()));
+    try {
+      assertStatus(403, table(PROBER, SCHEMA, victim, "describe"));
+      if ("register".equals(operation)) {
+        assertStatus(200, register(PROBER, WRITE_SCHEMA, alias, null, 
victimLocation.toString()));
+      } else {
+        Schema schema = new Schema(List.of(Field.nullable("id", new 
ArrowType.Int(32, true))));
+        HttpRequest create =
+            request(PROBER, "/v1/table/" + id(CATALOG, WRITE_SCHEMA, alias) + 
"/create", "")
+                .setHeader("Content-Type", 
"application/vnd.apache.arrow.stream")
+                .setHeader(LanceConstants.LANCE_TABLE_LOCATION_HEADER, 
parent.toString())
+                
.POST(HttpRequest.BodyPublishers.ofByteArray(ArrowUtils.generateIpcStream(schema)))
+                .build();
+        assertStatus(200, httpClient.send(create, 
HttpResponse.BodyHandlers.ofString()));
+      }
+      HttpResponse<String> rejected = table(PROBER, WRITE_SCHEMA, alias, 
"drop");
+      assertStatus(406, rejected);
+      Assertions.assertTrue(
+          
rejected.body().contains(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+      assertStatus(200, table(PROBER, WRITE_SCHEMA, alias, "describe"));
+      assertStatus(200, table(ADMIN, SCHEMA, victim, "describe"));
+      try (Dataset dataset = 
Dataset.open().uri(victimLocation.toString()).build()) {
+        Assertions.assertFalse(dataset.getSchema().getFields().isEmpty());
+      }
+      assertStatus(200, table(PROBER, WRITE_SCHEMA, alias, "deregister"));
+      Assertions.assertTrue(Files.exists(victimLocation));
+    } finally {
+      table(ADMIN, WRITE_SCHEMA, alias, "deregister");
+      table(ADMIN, SCHEMA, victim, "deregister");
+    }
+  }
+
   private HttpResponse<String> dropColumns(String user, String tableName, 
String column)
       throws Exception {
     return dropColumns(user, WRITE_SCHEMA, tableName, column);

Reply via email to