This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 0c28d77875 [#9520] improvement(lance): Refine the concept of
createEmptyTable in Lance REST (#9589)
0c28d77875 is described below
commit 0c28d7787538ab10a382b7eccb137ef2ca1580b5
Author: Mini Yu <[email protected]>
AuthorDate: Fri Jan 9 19:49:50 2026 +0800
[#9520] improvement(lance): Refine the concept of createEmptyTable in Lance
REST (#9589)
### What changes were proposed in this pull request?
This pull request introduces support for creating "empty" tables in the
Lance-backed catalog, where only metadata is stored without creating any
underlying storage or dataset. The implementation includes changes to
the REST API, backend logic, constants, and integration tests to align
with the new specification for empty table creation.
### Why are the changes needed?
To keep updated with the latest docs
Fix: #9520
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
Existing test.
---
.../lakehouse/lance/LanceTableDelegator.java | 11 +++++-
.../lakehouse/lance/LanceTableOperations.java | 13 +++++++
.../test/CatalogGenericCatalogLanceIT.java | 42 ++++++++++++++++++++++
docs/lance-rest-service.md | 27 +++++++-------
.../gravitino/GravitinoLanceTableOperations.java | 12 ++++++-
.../lance/common/utils/LanceConstants.java | 3 ++
.../lance/service/rest/LanceTableOperations.java | 4 +++
.../lance/integration/test/LanceRESTServiceIT.java | 39 ++++++++++----------
8 files changed, 117 insertions(+), 34 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableDelegator.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableDelegator.java
index 6fe7e64fae..0e9a9676df 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableDelegator.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableDelegator.java
@@ -20,6 +20,7 @@ package org.apache.gravitino.catalog.lakehouse.lance;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_CREATE_EMPTY;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_FORMAT;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_REGISTER;
@@ -65,7 +66,15 @@ public class LanceTableDelegator implements
LakehouseTableDelegator {
LanceTableOperations.CreationMode.class,
LanceTableOperations.CreationMode.CREATE,
false /* hidden */,
- false /* reserved */));
+ false /* reserved */),
+ PropertyEntry.booleanPropertyEntry(
+ LANCE_TABLE_CREATE_EMPTY,
+ "Whether this is a lance create empty table (declare table)
operation.",
+ false,
+ true /* immutable */,
+ false /* defaultValue */,
+ false /* hidden */,
+ false));
}
@Override
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
index 529c798c1e..e5f0519a20 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
@@ -45,6 +45,7 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException;
import org.apache.gravitino.exceptions.NoSuchTableException;
import org.apache.gravitino.exceptions.TableAlreadyExistsException;
import org.apache.gravitino.lance.common.ops.gravitino.LanceDataTypeConverter;
+import org.apache.gravitino.lance.common.utils.LanceConstants;
import org.apache.gravitino.lance.common.utils.LancePropertiesUtils;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
@@ -263,6 +264,18 @@ public class LanceTableOperations extends
ManagedTableOperations {
ident, columns, comment, properties, partitions, distribution,
sortOrders, indexes);
}
+ // Check whether it's a create empty table operation.
+ boolean createEmpty =
+
Optional.ofNullable(properties.get(LanceConstants.LANCE_TABLE_CREATE_EMPTY))
+ .map(Boolean::parseBoolean)
+ .orElse(false);
+ if (createEmpty) {
+ // For create empty table, we just create the table metadata in
Gravitino without creating
+ // the underlying Lance dataset.
+ return super.createTable(
+ ident, columns, comment, properties, partitions, distribution,
sortOrders, indexes);
+ }
+
Map<String, String> storageProps =
LancePropertiesUtils.getLanceStorageOptions(properties);
try (Dataset ignored =
Dataset.create(
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
index d3a34193a7..7f4ef535af 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/integration/test/CatalogGenericCatalogLanceIT.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.catalog.lakehouse.lance.integration.test;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_CREATE_EMPTY;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_FORMAT;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_REGISTER;
@@ -63,6 +64,7 @@ import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
+import org.apache.gravitino.rel.TableChange;
import org.apache.gravitino.rel.expressions.NamedReference;
import org.apache.gravitino.rel.expressions.distributions.Distribution;
import org.apache.gravitino.rel.expressions.distributions.Distributions;
@@ -151,6 +153,46 @@ public class CatalogGenericCatalogLanceIT extends BaseIT {
createSchema();
}
+ @Test
+ public void testCrateEmptyTable() {
+ String emptyTableName = GravitinoITUtils.genRandomName(TABLE_PREFIX);
+ NameIdentifier nameIdentifier = NameIdentifier.of(schemaName,
emptyTableName);
+
+ Map<String, String> properties = createProperties();
+ String tableLocation = tempDirectory + "/" + tableName;
+ properties.put("format", "lance");
+ properties.put("location", tableLocation);
+ properties.put(LANCE_TABLE_CREATE_EMPTY, "true");
+ properties.put(Table.PROPERTY_EXTERNAL, "true");
+
+ Table createdTable =
+ catalog
+ .asTableCatalog()
+ .createTable(
+ nameIdentifier,
+ new Column[0],
+ TABLE_COMMENT,
+ properties,
+ Transforms.EMPTY_TRANSFORM,
+ null,
+ null);
+ Assertions.assertEquals(createdTable.name(), emptyTableName);
+
+ // Now try to alter the property LANCE_TABLE_CREATE_EMPTY
+ IllegalArgumentException e =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ catalog
+ .asTableCatalog()
+ .alterTable(
+ nameIdentifier,
+ TableChange.setProperty(LANCE_TABLE_CREATE_EMPTY,
"false")));
+
+ Assertions.assertTrue(
+ e.getMessage().contains("Property lance.create-empty is immutable or
reserved"));
+ }
+
@Test
public void testCreateLanceTable() {
// Create a table from Gravitino API
diff --git a/docs/lance-rest-service.md b/docs/lance-rest-service.md
index 4b0db579f1..f8754d08a9 100644
--- a/docs/lance-rest-service.md
+++ b/docs/lance-rest-service.md
@@ -67,19 +67,20 @@ The Lance REST service acts as a bridge between Lance
datasets and applications:
The Lance REST service provides comprehensive support for namespace
management, table management, and index operations. The table below lists all
supported operations:
-| Operation | Description
| HTTP Method | Endpoint Pattern | Since Version |
-|-------------------|-------------------------------------------------------------------|-------------|---------------------------------------|---------------|
-| CreateNamespace | Create a new Lance namespace
| POST | `/lance/v1/namespace/{id}/create` | 1.1.0 |
-| ListNamespaces | List all namespaces under a parent namespace
| GET | `/lance/v1/namespace/{parent}/list` | 1.1.0 |
-| DescribeNamespace | Retrieve detailed information about a specific namespace
| POST | `/lance/v1/namespace/{id}/describe` | 1.1.0 |
-| DropNamespace | Delete a namespace
| POST | `/lance/v1/namespace/{id}/drop` | 1.1.0 |
-| NamespaceExists | Check whether a namespace exists
| POST | `/lance/v1/namespace/{id}/exists` | 1.1.0 |
-| ListTables | List all tables in a namespace
| GET | `/lance/v1/namespace/{id}/table/list` | 1.1.0 |
-| CreateTable | Create a new table in a namespace
| POST | `/lance/v1/table/{id}/create` | 1.1.0 |
-| DropTable | Delete a table including both metadata and data
| POST | `/lance/v1/table/{id}/drop` | 1.1.0 |
-| TableExists | Check whether a table exists
| POST | `/lance/v1/table/{id}/exists` | 1.1.0 |
-| RegisterTable | Register an existing Lance table to a namespace
| POST | `/lance/v1/table/{id}/register` | 1.1.0 |
-| DeregisterTable | Unregister a table from a namespace (metadata only, data
remains) | POST | `/lance/v1/table/{id}/deregister` | 1.1.0 |
+| Operation | Description
| HTTP Method | Endpoint Pattern
| Since Version |
+|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|---------------------------------------|---------------|
+| CreateNamespace | Create a new Lance namespace
| POST |
`/lance/v1/namespace/{id}/create` | 1.1.0 |
+| ListNamespaces | List all namespaces under a parent namespace
| GET |
`/lance/v1/namespace/{parent}/list` | 1.1.0 |
+| DescribeNamespace | Retrieve detailed information about a specific namespace
| POST |
`/lance/v1/namespace/{id}/describe` | 1.1.0 |
+| DropNamespace | Delete a namespace
| POST |
`/lance/v1/namespace/{id}/drop` | 1.1.0 |
+| NamespaceExists | Check whether a namespace exists
| POST |
`/lance/v1/namespace/{id}/exists` | 1.1.0 |
+| ListTables | List all tables in a namespace
| GET |
`/lance/v1/namespace/{id}/table/list` | 1.1.0 |
+| CreateTable | Create a new table in a namespace
| POST |
`/lance/v1/table/{id}/create` | 1.1.0 |
+| DropTable | Delete a table including both metadata and data
| POST |
`/lance/v1/table/{id}/drop` | 1.1.0 |
+| TableExists | Check whether a table exists
| POST |
`/lance/v1/table/{id}/exists` | 1.1.0 |
+| RegisterTable | Register an existing Lance table to a namespace
| POST |
`/lance/v1/table/{id}/register` | 1.1.0 |
+| DeregisterTable | Unregister a table from a namespace (metadata only, data
remains)
| POST |
`/lance/v1/table/{id}/deregister` | 1.1.0 |
+| CreateEmptyTable | Declare a table and store the metadata without touching
lance table data, for more, please refer to
[doc](https://docs.lancedb.com/api-reference/rest/table/create-an-empty-table)
| POST | `/lance/v1/table/{id}/create-empty` | 1.1.0 |
More details, please refer to the [Lance REST API
specification](https://lance.org/format/namespace/rest/catalog-spec/)
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
index ae0f567215..f94ffdcdbd 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
@@ -22,10 +22,12 @@ package org.apache.gravitino.lance.common.ops.gravitino;
import static
org.apache.gravitino.lance.common.ops.gravitino.LanceDataTypeConverter.CONVERTER;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_LOCATION;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_CREATE_EMPTY;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_TABLE_FORMAT;
import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.lancedb.lance.namespace.LanceNamespaceException;
@@ -154,8 +156,16 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
@Override
public CreateEmptyTableResponse createEmptyTable(
String tableId, String delimiter, String tableLocation, Map<String,
String> tableProperties) {
+ // Empty table creation only supports CREATE mode (not EXIST_OK or
OVERWRITE).
+ ImmutableMap<String, String> props =
+ ImmutableMap.<String, String>builder()
+ .putAll(tableProperties)
+ .put(LANCE_TABLE_CREATE_EMPTY, "true")
+ .put(Table.PROPERTY_EXTERNAL, "true")
+ .build();
+
CreateTableResponse response =
- createTable(tableId, ModeEnum.CREATE, delimiter, tableLocation,
tableProperties, null);
+ createTable(tableId, ModeEnum.CREATE, delimiter, tableLocation, props,
null);
CreateEmptyTableResponse emptyTableResponse = new
CreateEmptyTableResponse();
emptyTableResponse.setProperties(response.getProperties());
emptyTableResponse.setLocation(response.getLocation());
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
index e3be5d0528..3dd5e6fd60 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LanceConstants.java
@@ -37,5 +37,8 @@ public class LanceConstants {
public static final String LANCE_TABLE_REGISTER = "lance.register";
+ // Mark whether it is to create an empty Lance table(no data files)
+ public static final String LANCE_TABLE_CREATE_EMPTY = "lance.create-empty";
+
public static final String LANCE_TABLE_FORMAT = "lance";
}
diff --git
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
index 26f6e6cc90..19d6ae57ec 100644
---
a/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
+++
b/lance/lance-rest-server/src/main/java/org/apache/gravitino/lance/service/rest/LanceTableOperations.java
@@ -122,6 +122,10 @@ public class LanceTableOperations {
}
}
+ /**
+ * According to the spec of lance-namespace with version 0.0.20 to 0.31,
createEmptyTable only
+ * stores the table metadata including its location, and will never touch
lance storage.
+ */
@POST
@Path("/create-empty")
@Produces("application/json")
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
index 716ed83393..2060d93dfd 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
@@ -41,7 +41,6 @@ import
com.lancedb.lance.namespace.model.DescribeTableResponse;
import com.lancedb.lance.namespace.model.DropNamespaceRequest;
import com.lancedb.lance.namespace.model.DropNamespaceResponse;
import com.lancedb.lance.namespace.model.DropTableRequest;
-import com.lancedb.lance.namespace.model.DropTableResponse;
import com.lancedb.lance.namespace.model.ErrorResponse;
import com.lancedb.lance.namespace.model.JsonArrowField;
import com.lancedb.lance.namespace.model.ListNamespacesRequest;
@@ -78,6 +77,8 @@ import org.apache.gravitino.exceptions.NoSuchTableException;
import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.lance.common.utils.ArrowUtils;
+import org.apache.gravitino.lance.common.utils.LanceConstants;
+import org.apache.gravitino.rel.Table;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -434,6 +435,9 @@ public class LanceRESTServiceIT extends BaseIT {
DescribeTableResponse loadTable = ns.describeTable(describeTableRequest);
Assertions.assertNotNull(loadTable);
Assertions.assertEquals(location, loadTable.getLocation());
+ Assertions.assertEquals(
+ "true",
loadTable.getProperties().get(LanceConstants.LANCE_TABLE_CREATE_EMPTY));
+ Assertions.assertEquals("true",
loadTable.getProperties().get(Table.PROPERTY_EXTERNAL));
// Try to create the same table again should fail
LanceNamespaceException exception =
@@ -444,21 +448,23 @@ public class LanceRESTServiceIT extends BaseIT {
});
Assertions.assertEquals(409, exception.getCode());
- // Try to create a table with wrong location should fail
+ // Create an empty table with non-existent location should succeed
+ // since storage is not touched
CreateEmptyTableRequest wrongLocationRequest = new
CreateEmptyTableRequest();
- wrongLocationRequest.setId(List.of(CATALOG_NAME, SCHEMA_NAME,
"wrong_location_table"));
- wrongLocationRequest.setLocation("hdfs://localhost:9000/invalid_path/");
- LanceNamespaceException apiException =
- Assertions.assertThrows(
- LanceNamespaceException.class,
- () -> {
- ns.createEmptyTable(wrongLocationRequest);
- });
- Assertions.assertTrue(apiException.getMessage().contains("Invalid user
input"));
+ wrongLocationRequest.setId(List.of(CATALOG_NAME, SCHEMA_NAME,
"another_table"));
+ String another_location = tempDir + "/" + "another_location/";
+ Assertions.assertFalse(new File(another_location).exists());
+ wrongLocationRequest.setLocation(another_location);
+ response = ns.createEmptyTable(wrongLocationRequest);
+ Assertions.assertNotNull(response);
+ Assertions.assertEquals(another_location, response.getLocation());
+ // Will not touch storage, so the path should not be created.
+ Assertions.assertFalse(new File(another_location).exists());
- // Correct the location and try again
+ // Create another empty table at a new location and verify it succeeds
String correctedLocation = tempDir + "/" + "wrong_location_table/";
wrongLocationRequest.setLocation(correctedLocation);
+ wrongLocationRequest.setId(List.of(CATALOG_NAME, SCHEMA_NAME,
"wrong_location_table"));
CreateEmptyTableResponse wrongLocationResponse =
Assertions.assertDoesNotThrow(() ->
ns.createEmptyTable(wrongLocationRequest));
Assertions.assertNotNull(wrongLocationResponse);
@@ -721,7 +727,7 @@ public class LanceRESTServiceIT extends BaseIT {
Assertions.assertNotNull(deregisterTableResponse);
Assertions.assertEquals(location, deregisterTableResponse.getLocation());
Assertions.assertTrue(Objects.equals(ids,
deregisterTableResponse.getId()));
- Assertions.assertTrue(
+ Assertions.assertFalse(
new File(location).exists(), "Data should still exist after
deregistering the table.");
// Now try to describe the table, should fail
@@ -789,12 +795,7 @@ public class LanceRESTServiceIT extends BaseIT {
// Drop the table
DropTableRequest dropTableRequest = new DropTableRequest();
dropTableRequest.setId(ids);
- DropTableResponse dropTableResponse =
- Assertions.assertDoesNotThrow(() -> ns.dropTable(dropTableRequest));
- Assertions.assertNotNull(dropTableResponse);
- Assertions.assertEquals(location, dropTableResponse.getLocation());
- Assertions.assertFalse(
- new File(location).exists(), "Data should be deleted after dropping
the table.");
+ Assertions.assertThrows(Exception.class, () ->
ns.dropTable(dropTableRequest));
// Describe the dropped table should fail
DescribeTableRequest describeTableRequest = new DescribeTableRequest();
