This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 8985ffad65 [#13696] fix(lance): require catalog opt-in to purge
external datasets (#14870)
8985ffad65 is described below
commit 8985ffad65fca3c62a7ddbc32144f58956f2c953
Author: Qi Yu <[email protected]>
AuthorDate: Wed Nov 11 14:37:33 2026 +0800
[#13696] fix(lance): require catalog opt-in to purge external datasets
(#14870)
### What changes were proposed in this pull request?
Add the catalog-only `lance.allow-external-table-purge` property,
defaulting to `false`. Reject external or registered Lance table purges
before removing metadata or deleting files, including Lance REST drop
and create-overwrite of an existing external table. Table properties
cannot enable the policy.
### Why are the changes needed?
A caller can own a table registered at another table's location, or
created at a parent directory containing other datasets. Table ownership
alone does not authorize deleting that data. This provides a default
deletion guard; location ownership and storage authorization remain
follow-up work in #14788.
Refs #13696, #14788.
### Does this PR introduce _any_ user-facing change?
Yes. External purge is disabled by default; Lance REST returns HTTP 406.
Catalog administrators must explicitly opt in to retain destructive
drop/overwrite behavior. The opt-in does not validate location
ownership. Metadata-only drop/deregister and managed-table deletion
remain unchanged.
### How was this patch tested?
- 201 focused unit tests and 48 embedded integration tests passed,
covering policy defaults, table-property bypass attempts, legacy
registrations, overwrite, provider callbacks, and catalog opt-in.
- Two authenticated cross-user HTTP regression cases reproduce #13696.
Both fail with HTTP 200 when the guard is removed and pass with HTTP 406
while preserving readable victim datasets when restored.
- Scoped Spotless checks and `git diff --check` passed. No Docker or
remote object-store tests were run.
---
.../generic/GenericCatalogPropertiesMetadata.java | 11 +++
.../lakehouse/lance/LanceTableOperations.java | 11 ++-
.../generic/TestGenericCatalogOperations.java | 23 ++---
.../lakehouse/generic/TestPropertiesMetadata.java | 15 +++
.../lakehouse/lance/TestLanceTableOperations.java | 107 +++++++++++++++++++--
.../test/CatalogGenericCatalogLanceIT.java | 2 +
docs/lakehouse-generic-catalog.md | 12 ++-
docs/lakehouse-generic-lance-table.md | 9 +-
docs/lance-rest-service.md | 15 ++-
.../lance/common/utils/LanceConstants.java | 3 +
.../lance/integration/test/LanceRESTServiceIT.java | 30 +++++-
.../test/LanceTableAuthorizationIT.java | 68 ++++++++++++-
12 files changed, 269 insertions(+), 37 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
index 7c84d3a54d..b63409edea 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
@@ -19,9 +19,11 @@
package org.apache.gravitino.catalog.lakehouse.generic;
+import static
org.apache.gravitino.connector.PropertyEntry.booleanPropertyEntry;
import static org.apache.gravitino.connector.PropertyEntry.enumPropertyEntry;
import static
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyEntry;
import static
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyPrefixEntry;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_SCHEMA_REFRESH_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
@@ -70,6 +72,15 @@ public class GenericCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetad
true /* immutable */,
DefaultTableLocationProvider.NAME, /* defaultValue */
false /* hidden */),
+ booleanPropertyEntry(
+ LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+ "Allow physical deletion of external and registered Lance
datasets. Table"
+ + " properties cannot enable this catalog-level policy.",
+ false /* required */,
+ false /* immutable */,
+ false /* defaultValue */,
+ false /* hidden */,
+ false /* reserved */),
enumPropertyEntry(
LANCE_SCHEMA_REFRESH_MODE,
"Controls when Lance table schemas are refreshed from the
underlying dataset.",
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
index 82404cd8b7..dd36f28c19 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.lakehouse.lance;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_REGISTER;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
@@ -309,6 +310,14 @@ public class LanceTableOperations extends
ManagedTableOperations {
Table table = super.loadTable(ident);
validateLanceTable(ident, table);
boolean external =
LancePropertiesUtils.isExternalTable(table.properties());
+ if (external
+ &&
!Boolean.parseBoolean(catalogProperties.get(LANCE_ALLOW_EXTERNAL_TABLE_PURGE)))
{
+ throw new UnsupportedOperationException(
+ "Purging external Lance tables is disabled. Use dropTable or Lance
REST deregister"
+ + " to remove metadata only. A catalog administrator may
enable "
+ + LANCE_ALLOW_EXTERNAL_TABLE_PURGE
+ + " to allow physical deletion of external data.");
+ }
// NOTE: the two steps below (metadata removal and dataset deletion) are
NOT atomic.
// If the process crashes between them, the entity-store record will be
gone but the
@@ -331,7 +340,7 @@ public class LanceTableOperations extends
ManagedTableOperations {
} catch (NoSuchTableException e) {
return false;
- } catch (IllegalArgumentException e) {
+ } catch (IllegalArgumentException | UnsupportedOperationException e) {
throw e;
} catch (OptimisticLockException e) {
// The table metadata conflicted with a concurrent writer, e.g. a lock
wait timeout, and was
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
index 4278939881..ba28e488ad 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestGenericCatalogOperations.java
@@ -50,7 +50,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Arrays;
@@ -316,28 +315,18 @@ public class TestGenericCatalogOperations {
store.put(entity, false);
if ("purge".equals(operation)) {
- Assertions.assertTrue(opsWithFakeProvider.purgeTable(ident));
+ Assertions.assertThrowsExactly(
+ UnsupportedOperationException.class, () ->
opsWithFakeProvider.purgeTable(ident));
} else if ("cascade".equals(operation)) {
Assertions.assertTrue(opsWithFakeProvider.dropSchema(schemaIdent, true));
} else {
Assertions.assertTrue(opsWithFakeProvider.dropTable(ident));
}
- Assertions.assertFalse(opsWithFakeProvider.tableExists(ident));
- if ("purge".equals(operation)) {
-
Assertions.assertFalse(Files.exists(lanceDataDir.resolve("legacy/_versions")));
- Assertions.assertEquals(1,
FakeTableLocationProvider.unprovisioned().size());
- Assertions.assertEquals(
- location,
- FakeTableLocationProvider.unprovisioned()
- .get(0)
- .tableProperties()
- .get(Table.PROPERTY_LOCATION));
- } else {
-
Assertions.assertTrue(FakeTableLocationProvider.unprovisioned().isEmpty());
- try (Dataset dataset = Dataset.open().uri(location).build()) {
- Assertions.assertEquals("id",
dataset.getSchema().getFields().get(0).getName());
- }
+ Assertions.assertEquals("purge".equals(operation),
opsWithFakeProvider.tableExists(ident));
+ Assertions.assertTrue(FakeTableLocationProvider.unprovisioned().isEmpty());
+ try (Dataset dataset = Dataset.open().uri(location).build()) {
+ Assertions.assertEquals("id",
dataset.getSchema().getFields().get(0).getName());
}
}
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
index bbc1fbcd13..4f468e1cf1 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
@@ -19,6 +19,8 @@
package org.apache.gravitino.catalog.lakehouse.generic;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
+
import com.google.common.collect.ImmutableMap;
import java.util.Map;
import org.apache.gravitino.Schema;
@@ -53,6 +55,19 @@ public class TestPropertiesMetadata {
GenericCatalogPropertiesMetadata.TABLE_LOCATION_PROVIDER));
}
+ @Test
+ void
testExternalPurgePolicyDefaultsToDisabledAndCanBeChangedAtCatalogLevel() {
+ PropertiesMetadata metadata = genericCatalog.catalogPropertiesMetadata();
+
Assertions.assertTrue(metadata.containsProperty(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ Assertions.assertEquals(
+ false, metadata.getOrDefault(Map.of(),
LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ Assertions.assertEquals(
+ true,
+ metadata.getOrDefault(
+ Map.of(LANCE_ALLOW_EXTERNAL_TABLE_PURGE, "true"),
LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+
Assertions.assertFalse(metadata.isImmutableProperty(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ }
+
@Test
void testCatalogPropertiesMetadata() {
PropertiesMetadata catalogPropertiesMetadata =
genericCatalog.catalogPropertiesMetadata();
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
index 310c3dc918..0e1363b416 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino.catalog.lakehouse.lance;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_SCHEMA_REFRESH_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
@@ -167,7 +168,7 @@ public class TestLanceTableOperations {
}
}
- /** Legacy registration flags also protect drop and overwrite, while purge
remains destructive. */
+ /** Legacy registrations retain their data unless the catalog explicitly
permits purging it. */
@ParameterizedTest
@CsvSource(
value = {
@@ -232,17 +233,111 @@ public class TestLanceTableOperations {
new Index[0]));
Assertions.assertEquals("true",
replaced.properties().get(Table.PROPERTY_EXTERNAL));
} else if ("purge".equals(operation)) {
- Assertions.assertTrue(lanceTableOps.purgeTable(ident));
+ Assertions.assertThrowsExactly(
+ UnsupportedOperationException.class, () ->
lanceTableOps.purgeTable(ident));
} else {
Assertions.assertTrue(lanceTableOps.dropTable(ident));
}
- verify(store).delete(ident, Entity.EntityType.TABLE);
- if ("purge".equals(operation)) {
- Assertions.assertFalse(Files.exists(Paths.get(location, "_versions")));
- } else {
+ verify(store, Mockito.times("purge".equals(operation) ? 0 : 1))
+ .delete(ident, Entity.EntityType.TABLE);
+ try (Dataset dataset = Dataset.open().uri(location).build()) {
+ Assertions.assertEquals("id",
dataset.getSchema().getFields().get(0).getName());
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource(
+ value = {"true,NULL", "true,false", "true,true", "false,NULL",
"false,false", "false,true"},
+ nullValues = "NULL")
+ void testPurgeRequiresCatalogPermissionForExternalData(boolean external,
String catalogPermission)
+ throws Exception {
+ NameIdentifier ident = NameIdentifier.of("schema", "purge_policy");
+ String location = tempDir.resolve("purge_policy").toString();
+ try (Dataset ignored =
+ lanceTableOps.createDataset(
+ location, new Column[] {Column.of("id", Types.IntegerType.get())},
Map.of())) {
+ // Purge must preserve both a real dataset and its metadata when the
policy rejects it.
+ }
+ Map<String, String> properties =
+ Map.of(
+ Table.PROPERTY_TABLE_FORMAT,
+ "lance",
+ Table.PROPERTY_LOCATION,
+ location,
+ Table.PROPERTY_EXTERNAL,
+ Boolean.toString(external),
+ LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+ "true");
+ when(store.get(eq(ident), eq(Entity.EntityType.TABLE),
eq(TableEntity.class)))
+ .thenReturn(tableEntity(ident, List.of(), properties));
+ when(store.delete(ident, Entity.EntityType.TABLE)).thenReturn(true);
+ if (catalogPermission != null) {
+ lanceTableOps.setCatalogProperties(
+ Map.of(LANCE_ALLOW_EXTERNAL_TABLE_PURGE, catalogPermission));
+ }
+
+ if (external && !Boolean.parseBoolean(catalogPermission)) {
+ UnsupportedOperationException error =
+ Assertions.assertThrowsExactly(
+ UnsupportedOperationException.class, () ->
lanceTableOps.purgeTable(ident));
+
Assertions.assertTrue(error.getMessage().contains(LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ verify(store, never()).delete(any(), any());
try (Dataset dataset = Dataset.open().uri(location).build()) {
Assertions.assertEquals("id",
dataset.getSchema().getFields().get(0).getName());
}
+ } else {
+ Assertions.assertTrue(lanceTableOps.purgeTable(ident));
+ verify(store).delete(ident, Entity.EntityType.TABLE);
+ Assertions.assertFalse(Files.exists(Paths.get(location, "_versions")));
+ }
+ }
+
+ @Test
+ void testOverwriteCannotPurgeExternalDataWithoutCatalogPermission() throws
Exception {
+ NameIdentifier ident = NameIdentifier.of("schema", "protected_table");
+ String location = tempDir.resolve("protected_table").toString();
+ Column[] columns = {Column.of("id", Types.IntegerType.get())};
+ try (Dataset ignored = lanceTableOps.createDataset(location, columns,
Map.of())) {
+ // The existing external dataset is not owned by this overwrite request.
+ }
+ when(store.get(eq(ident), eq(Entity.EntityType.TABLE),
eq(TableEntity.class)))
+ .thenReturn(
+ tableEntity(
+ ident,
+ List.of(),
+ Map.of(
+ Table.PROPERTY_TABLE_FORMAT,
+ "lance",
+ Table.PROPERTY_LOCATION,
+ location,
+ Table.PROPERTY_EXTERNAL,
+ "true")));
+ String replacement = tempDir.resolve("replacement").toString();
+ Assertions.assertThrowsExactly(
+ UnsupportedOperationException.class,
+ () ->
+ lanceTableOps.createTable(
+ ident,
+ columns,
+ null,
+ Map.of(
+ Table.PROPERTY_TABLE_FORMAT,
+ "lance",
+ Table.PROPERTY_LOCATION,
+ replacement,
+ LANCE_CREATION_MODE,
+ "OVERWRITE",
+ LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
+ "true"),
+ new Transform[0],
+ null,
+ new SortOrder[0],
+ new Index[0]));
+ verify(store, never()).delete(any(), any());
+ verify(store, never()).put(any(), anyBoolean());
+ Assertions.assertFalse(Files.exists(Paths.get(replacement)));
+ try (Dataset dataset = Dataset.open().uri(location).build()) {
+ Assertions.assertEquals("id",
dataset.getSchema().getFields().get(0).getName());
}
}
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
index fc822a8678..348fb6a38d 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
@@ -812,6 +812,8 @@ public class CatalogGenericCatalogLanceIT extends BaseIT {
protected void createCatalog() {
Map<String, String> properties = Maps.newHashMap();
+ // This suite exercises explicit data deletion, including purge and
create-overwrite.
+ properties.put("lance.allow-external-table-purge", "true");
metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, provider,
"comment", properties);
catalog = metalake.loadCatalog(catalogName);
diff --git a/docs/lakehouse-generic-catalog.md
b/docs/lakehouse-generic-catalog.md
index a2963807cb..109ddfc8e4 100644
--- a/docs/lakehouse-generic-catalog.md
+++ b/docs/lakehouse-generic-catalog.md
@@ -50,6 +50,7 @@ For detailed information on available operations, see [Manage
Relational Metadat
| `location` | Root storage path for all schemas and tables
| `s3://bucket/lakehouse` | No |
| `table-location-provider` | Name of the [table location
provider](#pluggable-table-location-provider) that provisions and unprovisions
the locations of this catalog's tables. Defaults to `default`, which resolves
the location from the table, schema and catalog `location` properties as
described below. Immutable once the catalog is created. | `default`
| No |
| `lance.schema-refresh-mode` | Lance table schema refresh mode.
`DECLARED_AND_EMPTY` (default) refreshes declared tables and tables with empty
stored columns. `VERSION_CHECK` additionally refreshes when the Lance dataset
version changes.
| `DECLARED_AND_EMPTY` | No
|
+| `lance.allow-external-table-purge` | Allow physical deletion of external or
registered Lance tables, including Lance REST `DropTable` and create
`OVERWRITE`. Catalog-only; table properties cannot enable it. Does not validate
ownership of storage locations. | `false` | No |
#### Pluggable table location provider
@@ -180,12 +181,15 @@ location back would invite it to delete exactly the data
the catalog just promis
leak is recoverable and a deletion is not, so `unprovisionTableLocation` is
not called when a table
whose `external` property is true is dropped.
-A purge is the opposite request. `LanceTableOperations.purgeTable` deletes the
external dataset that
-its `dropTable` leaves alone, so by the time the callback would run the data
is gone and the reason
-to skip has gone with it; the location is handed back.
+For Lance, an external-table purge requires
`lance.allow-external-table-purge=true` in the catalog.
+Without it, purge fails before removing table metadata or deleting data, and
the provider callback is
+not invoked.
+When allowed, purge deletes the external dataset that drop leaves alone; after
successful deletion,
+the location is handed back.
The callback is therefore reached in exactly three situations: a dropped
managed table, a purged
-managed table, and a purged external one. There is deliberately no flag
distinguishing them, because
+managed table, and an external one whose purge was permitted by its table
format. There is deliberately
+no flag distinguishing them, because
the invariant they share is about the location rather than the table:
**nothing anyone needs is under
the location named in the context**. In all three the table format has already
deleted the data under
it. The catalog makes the drop-versus-purge and external distinctions rather
than passing them on,
diff --git a/docs/lakehouse-generic-lance-table.md
b/docs/lakehouse-generic-lance-table.md
index c6b5e55c35..8908a1d20d 100644
--- a/docs/lakehouse-generic-lance-table.md
+++ b/docs/lakehouse-generic-lance-table.md
@@ -148,14 +148,14 @@ Required and optional properties for tables in a Generic
Lakehouse Catalog:
|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------|
| `format` | Table format: `lance`, only `lance` is fully
supported.
| (none) | Yes |
| `location` | Storage path for table metadata and data, Lance
supports: S3, GCS, OSS, AZ, File, Memory and file-object-store.
| (none) | Conditional* |
-| `external` | Whether the data directory is an external location.
If it's `true`, dropping a table will only remove metadata in Gravitino and
will not delete the data directory, and purge table will delete both. For a
non-external table, dropping will drop both.
| false | No |
+| `external` | Whether the data directory is an external location.
If it's `true`, dropping a table will only remove metadata in Gravitino and
will not delete the data directory, and purge is rejected unless the catalog
enables `lance.allow-external-table-purge`. For a non-external table, dropping
will drop both.
| false | No |
| `lance.creation-mode` | Create mode: for create table, it can be `CREATE`,
`EXIST_OK` or `OVERWRITE`. and it should be `CREATE` or `OVERWRITE` for
registering tables
| `CREATE` | No |
| `lance.register` | Whether it is a register table operation. If `true`,
register an existing, readable Lance dataset without creating data. A missing
dataset or plain directory is rejected. Registered tables are external. If
`false`, create a dataset.
| false | No |
| `lance.storage.xxxx` | Any additional storage-specific properties required
by Lance format (e.g., S3 credentials, HDFS configs). Replace `xxxx` with
actual property names. For example, we can use
`lance.storage.aws_access_key_id` to set S3 aws_access_key_id when using a S3
location, for detail, refer to https://lancedb.com/docs/storage/integrations/ |
(none) | No |
- `CREATE`: Create a new table, fail if the table already exists.
- `EXIST_OK`: Create a new table if it does not exist, otherwise do nothing.
-- `OVERWRITE`: For creation (`lance.register=false`), purge the existing table
and its dataset before creating a replacement. For registration
(`lance.register=true`), validate the replacement dataset first and replace
only external-table metadata, retaining the old dataset. Registration overwrite
of a managed table is rejected; explicitly dropping or purging a managed table
deletes its data.
+- `OVERWRITE`: For creation (`lance.register=false`), purge the existing table
and its dataset before creating a replacement. An existing external table
requires the catalog to enable `lance.allow-external-table-purge`; otherwise
the original metadata and data are retained. For registration
(`lance.register=true`), validate the replacement dataset first and replace
only external-table metadata, retaining the old dataset. Registration overwrite
of a managed table is rejected; explicitly d [...]
Registration validation checks dataset existence and readability, not an
operator-defined storage scope or authorization to purge external data.
@@ -302,7 +302,10 @@ tableCatalog.createTable(
Register existing Lance tables without moving or copying data. A table with
`lance.register=true`
is always external, including when the request omits `external` or sets it to
`false`. Dropping it
through the Gravitino table API removes only the metadata. Explicit
`purgeTable` and Lance REST
-`drop` still delete both the metadata and the dataset; use Lance REST
`deregister` to preserve data.
+`drop` are rejected by default, preserving metadata and data. A catalog
administrator may set
+`lance.allow-external-table-purge=true` to allow both to be deleted. This
setting is read only from
+catalog properties; a table property cannot enable it. Use Lance REST
`deregister` to remove
+metadata while preserving data. The opt-in does not validate location
ownership.
Existing registrations with a missing or false stored `external` flag receive
the same protection
when dropped or deregistered. Their stored properties are not automatically
migrated.
diff --git a/docs/lance-rest-service.md b/docs/lance-rest-service.md
index e67be78e35..490258d419 100644
--- a/docs/lance-rest-service.md
+++ b/docs/lance-rest-service.md
@@ -127,9 +127,22 @@ versioned metadata reads are not implemented.
:::
**DropTable vs DeregisterTable**:
-- **DropTable**: Permanently deletes metadata and data files from storage
+- **DropTable**: Permanently deletes metadata and data files when permitted by
the catalog policy
- **DeregisterTable**: Removes metadata from Gravitino but preserves Lance
data files
+The generic catalog rejects purging external or registered Lance tables by
default, before removing
+table metadata or deleting files. Lance REST reports this as HTTP `406`
(`UnsupportedOperation`). Tables
+created, declared or registered through Lance REST are external, so this
default also applies to them.
+Use `DeregisterTable` to remove their metadata while retaining their data.
+
+A catalog administrator can set `lance.allow-external-table-purge=true` on the
`lakehouse-generic`
+catalog to permit physical deletion. This catalog-only setting also applies to
create `OVERWRITE`
+when it replaces an existing external table. A table property or request
header cannot enable it.
+Existing catalogs must explicitly opt in to retain their previous destructive
drop/overwrite behavior.
+Enabling it permits deletion at caller-supplied locations; it does not verify
storage ownership or
+provide isolation between tables. Enable it only when table owners are trusted
to delete data at
+locations they can register. Underlying storage permissions still apply.
+
## Deployment
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
index 197388e8d4..4b238a4504 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
@@ -42,5 +42,8 @@ public class LanceConstants {
public static final String LANCE_TABLE_DECLARED = "lance.declared";
public static final String LANCE_SCHEMA_REFRESH_MODE =
"lance.schema-refresh-mode";
+ /** Catalog-only opt-in for physically deleting external or registered Lance
datasets. */
+ public static final String LANCE_ALLOW_EXTERNAL_TABLE_PURGE =
"lance.allow-external-table-purge";
+
public static final String LANCE_TABLE_FORMAT = "lance";
}
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
index 42c699c5d2..d6cf096c01 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
@@ -47,6 +47,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.commons.io.FileUtils;
import org.apache.gravitino.Catalog;
+import org.apache.gravitino.CatalogChange;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.Schema;
import org.apache.gravitino.client.GravitinoMetalake;
@@ -574,6 +575,26 @@ public class LanceRESTServiceIT extends BaseIT {
"lance.storage.a", "value_va",
"lance.storage.b", "value_vb");
+ LanceNamespaceException denied =
+ Assertions.assertThrows(
+ LanceNamespaceException.class,
+ () -> createTable(ids, newLocation, overwriteTableProperties,
body, "overwrite"));
+ Assertions.assertTrue(
+
denied.getMessage().contains(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ Assertions.assertEquals(
+ location,
+ catalog
+ .asTableCatalog()
+ .loadTable(NameIdentifier.of(SCHEMA_NAME, ids.get(2)))
+ .properties()
+ .get(Table.PROPERTY_LOCATION));
+ Assertions.assertTrue(new File(location).exists());
+ Assertions.assertFalse(new File(newLocation).exists());
+ catalog =
+ metalake.alterCatalog(
+ CATALOG_NAME,
+
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
"true"));
+
response =
Assertions.assertDoesNotThrow(
() -> createTable(ids, newLocation, overwriteTableProperties,
body, "overwrite"));
@@ -1055,7 +1076,7 @@ public class LanceRESTServiceIT extends BaseIT {
@Test
void testDropTable() {
- catalog = createCatalog(CATALOG_NAME);
+ catalog = createCatalog(CATALOG_NAME, true);
createSchema();
List<String> ids = List.of(CATALOG_NAME, SCHEMA_NAME, "table_to_drop");
@@ -1239,7 +1260,7 @@ public class LanceRESTServiceIT extends BaseIT {
@Test
void testDropTableWhenLocationMissingOrNotCreated() throws IOException {
- catalog = createCatalog(CATALOG_NAME);
+ catalog = createCatalog(CATALOG_NAME, true);
createSchema();
// Register a valid dataset, then remove its location before dropping the
table
@@ -1293,9 +1314,14 @@ public class LanceRESTServiceIT extends BaseIT {
}
private Catalog createCatalog(String catalogName) {
+ return createCatalog(catalogName, false);
+ }
+
+ private Catalog createCatalog(String catalogName, boolean
allowExternalPurge) {
Map<String, String> catalogProperties =
ImmutableMap.<String, String>builder()
.putAll(properties)
+ .put("lance.allow-external-table-purge",
Boolean.toString(allowExternalPurge))
.put("lance.storage.endpoint", MINIO_ENDPOINT)
.put("lance.storage.allow_http", "true")
.put("lance.storage.access_key_id", MINIO_ACCESS_KEY)
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
index 400eb40add..cfcbc129b0 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceTableAuthorizationIT.java
@@ -23,6 +23,7 @@ import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Base64;
@@ -33,6 +34,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.gravitino.Catalog;
+import org.apache.gravitino.CatalogChange;
import org.apache.gravitino.Configs;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.MetadataObjects;
@@ -55,6 +57,9 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.lance.Dataset;
import org.lance.namespace.model.AlterTableDropColumnsRequest;
import org.lance.namespace.model.CreateNamespaceRequest;
import org.lance.namespace.model.DeclareTableRequest;
@@ -334,11 +339,25 @@ public class LanceTableAuthorizationIT extends BaseIT {
// The denied requests left the table in place.
assertStatus(200, table(ADMIN, VISIBLE_TABLE, "describe"));
- // Creating assigns ownership, so the creator may use either removal
operation.
+ // Ownership permits deregistration. Physical deletion also requires
catalog permission.
assertStatus(200, registerExistingTable(PROBER, WRITE_SCHEMA,
DEREGISTER_TABLE));
assertStatus(200, registerExistingTable(PROBER, WRITE_SCHEMA, DROP_TABLE));
assertStatus(200, table(PROBER, WRITE_SCHEMA, DEREGISTER_TABLE,
"deregister"));
- assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+ assertStatus(406, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+ assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "describe"));
+ Assertions.assertTrue(Files.exists(Path.of(location(DROP_TABLE))));
+ GravitinoMetalake metalake =
client.loadMetalake(getLanceRESTServerMetalakeName());
+ metalake.alterCatalog(
+ CATALOG,
+
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
"true"));
+ try {
+ assertStatus(200, table(PROBER, WRITE_SCHEMA, DROP_TABLE, "drop"));
+ Assertions.assertFalse(Files.exists(Path.of(location(DROP_TABLE))));
+ } finally {
+ metalake.alterCatalog(
+ CATALOG,
+
CatalogChange.setProperty(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE,
"false"));
+ }
assertStatus(404, table(ADMIN, WRITE_SCHEMA, DEREGISTER_TABLE,
"describe"));
assertStatus(404, table(ADMIN, WRITE_SCHEMA, DROP_TABLE, "describe"));
}
@@ -347,8 +366,11 @@ public class LanceTableAuthorizationIT extends BaseIT {
public void testSchemaOwnershipRemovesTablesTheOwnerDidNotCreate() throws
Exception {
// SCHEMA_OWNER_WITH_USE_CATALOG: the admin registered this table, so the
only thing the caller
// has is ownership of the schema holding it, plus USE_CATALOG.
- assertStatus(200, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE,
"drop"));
+ assertStatus(406, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE,
"drop"));
+ assertStatus(200, table(ADMIN, OWNED_SCHEMA, OWNED_SCHEMA_TABLE,
"describe"));
+ assertStatus(200, table(SCHEMA_OWNER, OWNED_SCHEMA, OWNED_SCHEMA_TABLE,
"deregister"));
assertStatus(404, table(ADMIN, OWNED_SCHEMA, OWNED_SCHEMA_TABLE,
"describe"));
+ Assertions.assertTrue(Files.exists(Path.of(location(OWNED_SCHEMA_TABLE))));
}
@Test
@@ -371,6 +393,46 @@ public class LanceTableAuthorizationIT extends BaseIT {
assertStatus(404, table(ADMIN, MISSING_TABLE, "deregister"));
}
+ @ParameterizedTest
+ @ValueSource(strings = {"register", "create"})
+ void testTableOwnerCannotPurgeAnotherSchemasData(String operation) throws
Exception {
+ String victim = "purge_victim_" + operation;
+ String alias = "purge_alias_" + operation;
+ Path parent = tempDir.resolve("purge_parent_" + operation);
+ Path victimLocation = parent.resolve("victim");
+ LanceDatasetTestUtils.createDataset(victimLocation.toString());
+ assertStatus(200, register(ADMIN, SCHEMA, victim, null,
victimLocation.toString()));
+ try {
+ assertStatus(403, table(PROBER, SCHEMA, victim, "describe"));
+ if ("register".equals(operation)) {
+ assertStatus(200, register(PROBER, WRITE_SCHEMA, alias, null,
victimLocation.toString()));
+ } else {
+ Schema schema = new Schema(List.of(Field.nullable("id", new
ArrowType.Int(32, true))));
+ HttpRequest create =
+ request(PROBER, "/v1/table/" + id(CATALOG, WRITE_SCHEMA, alias) +
"/create", "")
+ .setHeader("Content-Type",
"application/vnd.apache.arrow.stream")
+ .setHeader(LanceConstants.LANCE_TABLE_LOCATION_HEADER,
parent.toString())
+
.POST(HttpRequest.BodyPublishers.ofByteArray(ArrowUtils.generateIpcStream(schema)))
+ .build();
+ assertStatus(200, httpClient.send(create,
HttpResponse.BodyHandlers.ofString()));
+ }
+ HttpResponse<String> rejected = table(PROBER, WRITE_SCHEMA, alias,
"drop");
+ assertStatus(406, rejected);
+ Assertions.assertTrue(
+
rejected.body().contains(LanceConstants.LANCE_ALLOW_EXTERNAL_TABLE_PURGE));
+ assertStatus(200, table(PROBER, WRITE_SCHEMA, alias, "describe"));
+ assertStatus(200, table(ADMIN, SCHEMA, victim, "describe"));
+ try (Dataset dataset =
Dataset.open().uri(victimLocation.toString()).build()) {
+ Assertions.assertFalse(dataset.getSchema().getFields().isEmpty());
+ }
+ assertStatus(200, table(PROBER, WRITE_SCHEMA, alias, "deregister"));
+ Assertions.assertTrue(Files.exists(victimLocation));
+ } finally {
+ table(ADMIN, WRITE_SCHEMA, alias, "deregister");
+ table(ADMIN, SCHEMA, victim, "deregister");
+ }
+ }
+
private HttpResponse<String> dropColumns(String user, String tableName,
String column)
throws Exception {
return dropColumns(user, WRITE_SCHEMA, tableName, column);