This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new fb902cccf8 [#11063] fix(lance): manage lance storage config in
Gravitino (#11070)
fb902cccf8 is described below
commit fb902cccf8a95d5839be673ab05e5b55aa63fa64
Author: FANNG <[email protected]>
AuthorDate: Fri May 15 11:48:13 2026 +0800
[#11063] fix(lance): manage lance storage config in Gravitino (#11070)
### What changes were proposed in this pull request?
This PR moves Lance storage configuration into Gravitino-managed catalog
properties and teaches Lance REST to use those catalog defaults when
serving Spark clients. It removes the need to pass
spark.sql.catalog.lance.storage.* in the demo path, while keeping
table-level lance.storage.* overrides supported.
### Why are the changes needed?
Spark currently has to repeat MinIO/S3 settings that are already known
to Gravitino, which makes Lance integration verbose and error-prone.
This change keeps Gravitino as the source of truth for Lance storage
configuration and simplifies Spark usage for Gravitino-managed Lance
catalogs.
Fix: #11063
### Does this PR introduce _any_ user-facing change?
Yes.
- lance.storage.* is now supported as a catalog-level property family
for generic lakehouse catalogs.
- Lance REST responses now resolve storage options from catalog defaults
when table-level values are absent.
- The Spark demo no longer requires explicit Lance storage conf entries.
- Table-level lance.storage.* remains supported as an override.
### How was this patch tested?
- ./gradlew :lance:lance-common:test --tests
org.apache.gravitino.lance.common.utils.TestLancePropertiesUtils
-PskipITs -PskipDockerTests=true
- ./gradlew :catalogs:catalog-lakehouse-generic:test --tests
org.apache.gravitino.catalog.lakehouse.generic.TestPropertiesMetadata
-PskipITs -PskipDockerTests=true
- ./gradlew :lance:lance-rest-server:compileTestJava -PskipITs
-PskipDockerTests=true
---------
Co-authored-by: fanng <“[email protected]”>
---
.../generic/GenericCatalogOperations.java | 16 +++++-
.../generic/GenericCatalogPropertiesMetadata.java | 11 +++-
.../lakehouse/lance/LanceTableOperations.java | 40 +++++++++++---
.../lakehouse/generic/TestPropertiesMetadata.java | 9 +++-
.../lakehouse/lance/TestLanceTableOperations.java | 44 ++++++++++++++-
docs/lakehouse-generic-lance-table.md | 26 ++++++---
docs/lance-rest-integration.md | 35 ++++++++----
.../gravitino/GravitinoLanceTableOperations.java | 9 ++--
.../lance/common/utils/LancePropertiesUtils.java | 38 ++++++++++---
.../common/utils/TestLancePropertiesUtils.java | 63 ++++++++++++++++++++++
.../lance/integration/test/LanceRESTServiceIT.java | 56 ++++++++++++++++++-
11 files changed, 310 insertions(+), 37 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogOperations.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogOperations.java
index e31263fadc..1d5876b6dd 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogOperations.java
@@ -44,6 +44,7 @@ import org.apache.gravitino.Schema;
import org.apache.gravitino.SchemaChange;
import org.apache.gravitino.catalog.ManagedSchemaOperations;
import org.apache.gravitino.catalog.ManagedTableOperations;
+import org.apache.gravitino.catalog.lakehouse.lance.LanceTableOperations;
import org.apache.gravitino.connector.CatalogInfo;
import org.apache.gravitino.connector.CatalogOperations;
import org.apache.gravitino.connector.HasPropertyMetadata;
@@ -77,6 +78,8 @@ public class GenericCatalogOperations implements
CatalogOperations, SupportsSche
private Optional<String> catalogLocation;
+ private Map<String, String> catalogProperties = Map.of();
+
private HasPropertyMetadata propertiesMetadata;
private final Cache<NameIdentifier, String> tableFormatCache;
@@ -125,6 +128,7 @@ public class GenericCatalogOperations implements
CatalogOperations, SupportsSche
public void initialize(
Map<String, String> conf, CatalogInfo info, HasPropertyMetadata
propertiesMetadata)
throws RuntimeException {
+ this.catalogProperties = conf == null ? Map.of() : Maps.newHashMap(conf);
String location =
(String)
propertiesMetadata
@@ -246,7 +250,7 @@ public class GenericCatalogOperations implements
CatalogOperations, SupportsSche
// Get the table operations for the specified table format.
Supplier<ManagedTableOperations> tableOpsSupplier =
tableOpsCache.get(format);
Preconditions.checkArgument(tableOpsSupplier != null, "Unsupported table
format: %s", format);
- ManagedTableOperations tableOps = tableOpsSupplier.get();
+ ManagedTableOperations tableOps =
configureTableOps(tableOpsSupplier.get());
Table createdTable =
tableOps.createTable(
@@ -339,7 +343,7 @@ public class GenericCatalogOperations implements
CatalogOperations, SupportsSche
return format.toLowerCase(Locale.ROOT);
});
- ManagedTableOperations ops = tableOpsCache.get(tableFormat).get();
+ ManagedTableOperations ops =
configureTableOps(tableOpsCache.get(tableFormat).get());
Preconditions.checkArgument(
ops != null, "No table operations found for table format %s",
tableFormat);
return ops;
@@ -360,4 +364,12 @@ public class GenericCatalogOperations implements
CatalogOperations, SupportsSche
}
}
}
+
+ private ManagedTableOperations configureTableOps(ManagedTableOperations ops)
{
+ if (ops instanceof LanceTableOperations) {
+ ((LanceTableOperations) ops).setCatalogProperties(catalogProperties);
+ }
+
+ return ops;
+ }
}
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
index 71daef161b..dd805181f8 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/generic/GenericCatalogPropertiesMetadata.java
@@ -20,6 +20,8 @@
package org.apache.gravitino.catalog.lakehouse.generic;
import static
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyEntry;
+import static
org.apache.gravitino.connector.PropertyEntry.stringOptionalPropertyPrefixEntry;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
@@ -41,7 +43,14 @@ public class GenericCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetad
"The root directory of the generic catalog.",
false /* immutable */,
null, /* defaultValue */
- false /* hidden */));
+ false /* hidden */),
+ stringOptionalPropertyPrefixEntry(
+ LANCE_STORAGE_OPTIONS_PREFIX,
+ "The Lance storage options managed by the catalog.",
+ false /* immutable */,
+ null, /* defaultValue */
+ false /* hidden */,
+ false /* reserved */));
PROPERTIES_METADATA = Maps.uniqueIndex(propertyEntries,
PropertyEntry::getName);
}
diff --git
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
index a1342e1cc9..dccbfe8c04 100644
---
a/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/main/java/org/apache/gravitino/catalog/lakehouse/lance/LanceTableOperations.java
@@ -53,6 +53,7 @@ import
org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
import org.apache.gravitino.storage.IdGenerator;
import org.lance.Dataset;
+import org.lance.ReadOptions;
import org.lance.WriteParams;
import org.lance.index.DistanceType;
import org.lance.index.IndexOptions;
@@ -78,6 +79,8 @@ public class LanceTableOperations extends
ManagedTableOperations {
private final IdGenerator idGenerator;
+ private volatile Map<String, String> catalogProperties = Map.of();
+
public LanceTableOperations(
EntityStore store, ManagedSchemaOperations schemaOps, IdGenerator
idGenerator) {
this.store = store;
@@ -100,6 +103,16 @@ public class LanceTableOperations extends
ManagedTableOperations {
return idGenerator;
}
+ /**
+ * Sets the catalog properties used to resolve Lance storage defaults at
runtime.
+ *
+ * @param catalogProperties the catalog properties
+ */
+ public void setCatalogProperties(Map<String, String> catalogProperties) {
+ this.catalogProperties =
+ catalogProperties == null ? Map.of() :
ImmutableMap.copyOf(catalogProperties);
+ }
+
@Override
public Table createTable(
NameIdentifier ident,
@@ -197,7 +210,9 @@ public class LanceTableOperations extends
ManagedTableOperations {
// Otherwise, we should not delete the dataset.
if (purged) {
// Delete the Lance dataset at the location
- Dataset.drop(location,
LancePropertiesUtils.getLanceStorageOptions(table.properties()));
+ Dataset.drop(
+ location,
+ LancePropertiesUtils.resolveLanceStorageOptions(catalogProperties,
table.properties()));
LOG.info("Deleted Lance dataset at location {}", location);
}
@@ -231,7 +246,9 @@ public class LanceTableOperations extends
ManagedTableOperations {
String location = table.properties().get(Table.PROPERTY_LOCATION);
// Delete the Lance dataset at the location
- Dataset.drop(location,
LancePropertiesUtils.getLanceStorageOptions(table.properties()));
+ Dataset.drop(
+ location,
+ LancePropertiesUtils.resolveLanceStorageOptions(catalogProperties,
table.properties()));
LOG.info("Deleted Lance dataset at location {}", location);
}
@@ -277,7 +294,8 @@ public class LanceTableOperations extends
ManagedTableOperations {
ident, columns, comment, properties, partitions, distribution,
sortOrders, indexes);
}
- Map<String, String> storageProps =
LancePropertiesUtils.getLanceStorageOptions(properties);
+ Map<String, String> storageProps =
+ LancePropertiesUtils.resolveLanceStorageOptions(catalogProperties,
properties);
try (Dataset ignored =
Dataset.write()
.allocator(new RootAllocator())
@@ -308,7 +326,7 @@ public class LanceTableOperations extends
ManagedTableOperations {
} catch (TableAlreadyExistsException e) {
// If the table metadata already exists, but the underlying lance table
was just created
// successfully, we need to clean up the created lance table to avoid
orphaned datasets.
- Dataset.drop(location,
LancePropertiesUtils.getLanceStorageOptions(properties));
+ Dataset.drop(location, storageProps);
throw e;
} catch (IllegalArgumentException e) {
if (e.getMessage().contains("Dataset already exists")) {
@@ -344,7 +362,9 @@ public class LanceTableOperations extends
ManagedTableOperations {
*/
long handleLanceTableChange(Table table, TableChange[] changes) {
String location = table.properties().get(Table.PROPERTY_LOCATION);
- try (Dataset dataset = openDataset(location)) {
+ Map<String, String> storageOptions =
+ LancePropertiesUtils.resolveLanceStorageOptions(catalogProperties,
table.properties());
+ try (Dataset dataset = openDataset(location, storageOptions)) {
for (TableChange change : changes) {
if (change instanceof TableChange.DeleteColumn deleteColumn) {
dataset.dropColumns(List.of(String.join(".",
deleteColumn.fieldName())));
@@ -385,7 +405,15 @@ public class LanceTableOperations extends
ManagedTableOperations {
}
Dataset openDataset(String location) {
- return Dataset.open().allocator(new RootAllocator()).uri(location).build();
+ return openDataset(location, Map.of());
+ }
+
+ Dataset openDataset(String location, Map<String, String> storageOptions) {
+ return Dataset.open()
+ .allocator(new RootAllocator())
+ .uri(location)
+ .readOptions(new
ReadOptions.Builder().setStorageOptions(storageOptions).build())
+ .build();
}
private IndexParams getIndexParamsByIndexType(IndexType indexType) {
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
index 75eb613dc5..5371a63f5a 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/generic/TestPropertiesMetadata.java
@@ -41,13 +41,20 @@ public class TestPropertiesMetadata {
PropertiesMetadata catalogPropertiesMetadata =
genericCatalog.catalogPropertiesMetadata();
Assertions.assertNotNull(catalogPropertiesMetadata);
- Map<String, String> catalogProperties = ImmutableMap.of("location",
"/tmp/test1");
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ "location", "/tmp/test1",
+ "lance.storage.endpoint", "http://minio:9000";);
String catalogLocation =
(String)
catalogPropertiesMetadata.getOrDefault(
catalogProperties, GenericCatalog.PROPERTY_LOCATION);
Assertions.assertEquals("/tmp/test1", catalogLocation);
+
Assertions.assertTrue(catalogPropertiesMetadata.containsProperty("lance.storage.endpoint"));
+ Assertions.assertEquals(
+ "http://minio:9000";,
+ catalogPropertiesMetadata.getOrDefault(catalogProperties,
"lance.storage.endpoint"));
}
@Test
diff --git
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
index c102d321e2..7362f903cb 100644
---
a/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
+++
b/catalogs/catalog-lakehouse-generic/src/test/java/org/apache/gravitino/catalog/lakehouse/lance/TestLanceTableOperations.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.catalog.lakehouse.lance;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_CREATION_MODE;
+import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.mock;
@@ -108,7 +109,7 @@ public class TestLanceTableOperations {
Version version = mock(Version.class);
when(dataset.getVersion()).thenReturn(version);
when(version.getId()).thenReturn(7L);
- Mockito.doReturn(dataset).when(lanceTableOps).openDataset("location");
+ Mockito.doReturn(dataset).when(lanceTableOps).openDataset("location",
Map.of());
TableChange[] changes =
new TableChange[] {
@@ -126,4 +127,45 @@ public class TestLanceTableOperations {
inOrder.verify(dataset).dropColumns(anyList());
inOrder.verify(dataset).getVersion();
}
+
+ @Test
+ public void testHandleLanceTableChangeUsesCatalogStorageOptions() {
+ lanceTableOps.setCatalogProperties(
+ Map.of(
+ LANCE_STORAGE_OPTIONS_PREFIX + "endpoint",
"http://catalog-endpoint";,
+ LANCE_STORAGE_OPTIONS_PREFIX + "secret_access_key",
"catalog-secret"));
+
+ Table table = mock(Table.class);
+ when(table.properties())
+ .thenReturn(
+ Map.of(
+ Table.PROPERTY_LOCATION,
+ "location",
+ LANCE_STORAGE_OPTIONS_PREFIX + "access_key_id",
+ "table-key"));
+
+ Dataset dataset = mock(Dataset.class);
+ Version version = mock(Version.class);
+ when(dataset.getVersion()).thenReturn(version);
+ when(version.getId()).thenReturn(9L);
+ Mockito.doReturn(dataset)
+ .when(lanceTableOps)
+ .openDataset(
+ "location",
+ Map.of(
+ "endpoint",
+ "http://catalog-endpoint";,
+ "secret_access_key",
+ "catalog-secret",
+ "access_key_id",
+ "table-key"));
+
+ TableChange[] changes =
+ new TableChange[] {TableChange.deleteColumn(new String[] {"col1"},
false)};
+ long returnedVersion = lanceTableOps.handleLanceTableChange(table,
changes);
+
+ Assertions.assertEquals(9L, returnedVersion);
+ Mockito.verify(dataset).dropColumns(anyList());
+ Mockito.verify(dataset).getVersion();
+ }
}
diff --git a/docs/lakehouse-generic-lance-table.md
b/docs/lakehouse-generic-lance-table.md
index 12e853fee6..d42e34377a 100644
--- a/docs/lakehouse-generic-lance-table.md
+++ b/docs/lakehouse-generic-lance-table.md
@@ -303,9 +303,26 @@ done
Other table operations (load, alter, drop, truncate) follow standard
relational catalog patterns. See [Table
Operations](./manage-relational-metadata-using-gravitino.md#table-operations)
for details.
### Using Lance table with MinIO
-To use Lance tables stored in MinIO with Gravitino, ensure that the MinIO
storage backend is properly configured. Below is an example of how to set up
and use Lance tables with MinIO.
+To use Lance tables stored in MinIO with Gravitino, configure the MinIO
storage backend once on the Lance catalog. Gravitino will then return those
storage options to Lance clients and Spark does not need to repeat them.
```shell
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "name": "lance_catalog",
+ "type": "RELATIONAL",
+ "provider": "lakehouse-generic",
+ "comment": "catalog for Lance tables on MinIO",
+ "properties": {
+ "location": "s3://bucket1/lance",
+ "lance.storage.endpoint": "http://minio:9000";,
+ "lance.storage.access_key_id": "ak",
+ "lance.storage.secret_access_key": "sk",
+ "lance.storage.allow_http": "true",
+ "lance.storage.region": "us-east-1"
+ }
+}' http://localhost:8090/api/metalakes/test/catalogs
+
curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
-H "Content-Type: application/json" -d '{
"name": "lance_orders",
@@ -320,13 +337,10 @@ curl -X POST -H "Accept:
application/vnd.gravitino.v1+json" \
],
"properties": {
"format": "lance",
- "location": "s3://bucket1/lance_orders",
- "lance.storage.access_key_id": "ak",
- "lance.storage.endpoint": "http://minio:9000";,
- "lance.storage.secret_access_key": "sk",
- "lance.storage.allow_http": "true"
+ "location": "s3://bucket1/lance_orders"
}
}'
http://localhost:8090/api/metalakes/test/catalogs/lance_catalog/schemas/sales/tables
```
+If you need to override storage on a single table, `lance.storage.*` table
properties are still supported.
diff --git a/docs/lance-rest-integration.md b/docs/lance-rest-integration.md
index 101213dfcd..66fbaff4f3 100644
--- a/docs/lance-rest-integration.md
+++ b/docs/lance-rest-integration.md
@@ -126,7 +126,28 @@ spark.sql("SELECT * FROM sales.orders").show()
The `LOCATION` clause in the `CREATE TABLE` statement is optional. When
omitted, lance-spark automatically determines an appropriate storage location
based on catalog properties.
For detailed information on location resolution logic, refer to the [Lakehouse
Generic Catalog
documentation](./lakehouse-generic-catalog.md#key-property-location).
-For cloud storage backends such as Amazon S3 or MinIO, specify credentials and
endpoint configuration in the table properties:
+For Gravitino-managed Lance catalogs, put the storage configuration in the
Gravitino catalog properties so Spark does not need to repeat it.
+
+For example, create the Gravitino catalog with catalog-level Lance storage
properties:
+
+```shell
+curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "name": "lance_catalog",
+ "type": "RELATIONAL",
+ "provider": "lakehouse-generic",
+ "comment": "catalog for Lance tables on MinIO",
+ "properties": {
+ "location": "s3://bucket/tmp",
+ "lance.storage.endpoint": "http://minio:9000";,
+ "lance.storage.access_key_id": "ak",
+ "lance.storage.secret_access_key": "sk",
+ "lance.storage.allow_http": "true",
+ "lance.storage.region": "us-east-1"
+ }
+}' http://localhost:8090/api/metalakes/test/catalogs
+```
```python
spark.sql("""
@@ -136,16 +157,12 @@ spark.sql("""
)
USING lance
LOCATION 's3://bucket/tmp/sales/orders.lance/'
- TBLPROPERTIES (
- 'format' = 'lance',
- 'lance.storage.access_key_id' = 'your_access_key',
- 'lance.storage.secret_access_key' = 'your_secret_key',
- 'lance.storage.endpoint' = 'http://minio:9000',
- 'lance.storage.allow_http' = 'true'
- )
+ TBLPROPERTIES ('format' = 'lance')
""")
```
+If you need a per-table override, `lance.storage.*` table properties are still
supported and take precedence over catalog defaults.
+
## Ray Integration
### Installation
@@ -230,4 +247,4 @@ Refer to each engine's specific documentation for detailed
configuration paramet
- [Lance REST Service Documentation](./lance-rest-service)
- [Lance Format Specification](https://lance.org/)
- [Apache Gravitino Documentation](https://gravitino.apache.org/)
-- [Lakehouse Generic Catalog Guide](./lakehouse-generic-catalog.md)
\ No newline at end of file
+- [Lakehouse Generic Catalog Guide](./lakehouse-generic-catalog.md)
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
index d43aad8f0c..a1ec2ee9bb 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/GravitinoLanceTableOperations.java
@@ -131,7 +131,8 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
Optional.ofNullable(table.properties().get(LANCE_TABLE_VERSION))
.map(Long::valueOf)
.orElse(null));
-
response.setStorageOptions(LancePropertiesUtils.getLanceStorageOptions(table.properties()));
+ response.setStorageOptions(
+ LancePropertiesUtils.resolveLanceStorageOptions(catalog.properties(),
table.properties()));
return response;
}
@@ -179,11 +180,11 @@ public class GravitinoLanceTableOperations implements
LanceTableOperations {
.createTable(
tableIdentifier, columns.toArray(new Column[0]), null,
createTableProperties);
Map<String, String> properties = t.properties();
+ Map<String, String> effectiveStorageOptions =
+ LancePropertiesUtils.resolveLanceStorageOptions(catalog.properties(),
properties);
CreateTableResponse response = new CreateTableResponse();
- // Extract storage options from table properties. All storage options
stores in table
- // properties.
-
response.setStorageOptions(LancePropertiesUtils.getLanceStorageOptions(properties));
+ response.setStorageOptions(effectiveStorageOptions);
response.setVersion(
Optional.ofNullable(properties.get(LANCE_TABLE_VERSION)).map(Long::valueOf).orElse(null));
response.setLocation(properties.get(LANCE_LOCATION));
diff --git
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LancePropertiesUtils.java
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LancePropertiesUtils.java
index e674a7266a..3555fa3f99 100644
---
a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LancePropertiesUtils.java
+++
b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/utils/LancePropertiesUtils.java
@@ -21,25 +21,51 @@ package org.apache.gravitino.lance.common.utils;
import static
org.apache.gravitino.lance.common.utils.LanceConstants.LANCE_STORAGE_OPTIONS_PREFIX;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
-public class LancePropertiesUtils {
+/** Utility methods for Lance storage properties. */
+public final class LancePropertiesUtils {
private LancePropertiesUtils() {
- // Private constructor to prevent instantiation
+ // Utility class.
}
+ /**
+ * Extracts Lance storage options from a property map.
+ *
+ * @param tableProperties the source properties
+ * @return the Lance storage options without the `lance.storage.` prefix
+ */
public static Map<String, String> getLanceStorageOptions(Map<String, String>
tableProperties) {
- if (tableProperties == null) {
+ if (tableProperties == null || tableProperties.isEmpty()) {
return Map.of();
}
return tableProperties.entrySet().stream()
- .filter(e -> e.getKey().startsWith(LANCE_STORAGE_OPTIONS_PREFIX))
+ .filter(entry ->
entry.getKey().startsWith(LANCE_STORAGE_OPTIONS_PREFIX))
.collect(
Collectors.toMap(
- e ->
e.getKey().substring(LANCE_STORAGE_OPTIONS_PREFIX.length()),
- Map.Entry::getValue));
+ entry ->
entry.getKey().substring(LANCE_STORAGE_OPTIONS_PREFIX.length()),
+ Map.Entry::getValue,
+ (left, right) -> right,
+ LinkedHashMap::new));
+ }
+
+ /**
+ * Resolves the effective Lance storage options using table properties first
and catalog
+ * properties as defaults.
+ *
+ * @param catalogProperties the catalog properties
+ * @param tableProperties the table properties
+ * @return the effective storage options
+ */
+ public static Map<String, String> resolveLanceStorageOptions(
+ Map<String, String> catalogProperties, Map<String, String>
tableProperties) {
+ Map<String, String> effectiveStorageOptions =
+ new LinkedHashMap<>(getLanceStorageOptions(catalogProperties));
+ effectiveStorageOptions.putAll(getLanceStorageOptions(tableProperties));
+ return effectiveStorageOptions;
}
}
diff --git
a/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/utils/TestLancePropertiesUtils.java
b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/utils/TestLancePropertiesUtils.java
new file mode 100644
index 0000000000..55bfa6c347
--- /dev/null
+++
b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/utils/TestLancePropertiesUtils.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.lance.common.utils;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestLancePropertiesUtils {
+
+ @Test
+ public void testGetLanceStorageOptions() {
+ Map<String, String> properties =
+ ImmutableMap.of(
+ "lance.storage.endpoint", "http://minio:9000";,
+ "lance.storage.access_key_id", "ak",
+ "not.storage.key", "ignored");
+
+ Map<String, String> storageOptions =
LancePropertiesUtils.getLanceStorageOptions(properties);
+
+ Assertions.assertEquals(2, storageOptions.size());
+ Assertions.assertEquals("http://minio:9000";,
storageOptions.get("endpoint"));
+ Assertions.assertEquals("ak", storageOptions.get("access_key_id"));
+ Assertions.assertFalse(storageOptions.containsKey("not.storage.key"));
+ }
+
+ @Test
+ public void testResolveLanceStorageOptionsPrefersTableProperties() {
+ Map<String, String> catalogProperties =
+ ImmutableMap.of(
+ "lance.storage.endpoint", "http://catalog:9000";,
+ "lance.storage.region", "us-east-1");
+ Map<String, String> tableProperties =
+ ImmutableMap.of(
+ "lance.storage.endpoint", "http://table:9000";,
+ "lance.storage.access_key_id", "table-ak");
+
+ Map<String, String> storageOptions =
+ LancePropertiesUtils.resolveLanceStorageOptions(catalogProperties,
tableProperties);
+
+ Assertions.assertEquals(3, storageOptions.size());
+ Assertions.assertEquals("http://table:9000";,
storageOptions.get("endpoint"));
+ Assertions.assertEquals("us-east-1", storageOptions.get("region"));
+ Assertions.assertEquals("table-ak", storageOptions.get("access_key_id"));
+ }
+}
diff --git
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
index 501f1394e7..3c0661971d 100644
---
a/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
+++
b/lance/lance-rest-server/src/test/java/org/apache/gravitino/lance/integration/test/LanceRESTServiceIT.java
@@ -95,6 +95,10 @@ public class LanceRESTServiceIT extends BaseIT {
private static final String CATALOG_NAME =
GravitinoITUtils.genRandomName("lance_rest_catalog");
private static final String SCHEMA_NAME =
GravitinoITUtils.genRandomName("lance_rest_schema");
private static final String DELIMITER = ".";
+ private static final String MINIO_ENDPOINT = "http://127.0.0.1:9000";;
+ private static final String MINIO_REGION = "us-east-1";
+ private static final String MINIO_ACCESS_KEY = "minioadmin";
+ private static final String MINIO_SECRET_KEY = "minioadmin";
private GravitinoMetalake metalake;
private Catalog catalog;
@@ -661,6 +665,46 @@ public class LanceRESTServiceIT extends BaseIT {
.contains("Column non_existing_column does not exist in the
dataset"));
}
+ @Test
+ void testCreateTableUsesCatalogStorageOptions() throws IOException {
+ catalog =
createCatalog(GravitinoITUtils.genRandomName("lance_rest_catalog"));
+ createSchema();
+
+ String location = tempDir.resolve("catalog_storage_table").toString() +
"/";
+ List<String> ids = List.of(catalog.name(), SCHEMA_NAME,
"catalog_storage_table");
+ org.apache.arrow.vector.types.pojo.Schema schema =
+ new org.apache.arrow.vector.types.pojo.Schema(
+ Arrays.asList(
+ Field.nullable("id", new ArrowType.Int(32, true)),
+ Field.nullable("value", new ArrowType.Utf8())));
+ byte[] body = ArrowUtils.generateIpcStream(schema);
+
+ CreateTableResponse response =
+ createTable(ids, location, ImmutableMap.of("key1", "v1"), body, /*
mode= */ null);
+ Assertions.assertNotNull(response);
+ Assertions.assertEquals(location, response.getLocation());
+ Assertions.assertEquals(MINIO_ENDPOINT,
response.getStorageOptions().get("endpoint"));
+ Assertions.assertEquals("true",
response.getStorageOptions().get("allow_http"));
+ Assertions.assertEquals(MINIO_ACCESS_KEY,
response.getStorageOptions().get("access_key_id"));
+ Assertions.assertEquals(
+ MINIO_SECRET_KEY,
response.getStorageOptions().get("secret_access_key"));
+ Assertions.assertEquals(MINIO_REGION,
response.getStorageOptions().get("region"));
+
+ DescribeTableRequest describeTableRequest = new DescribeTableRequest();
+ describeTableRequest.setId(ids);
+ DescribeTableResponse loadTable = ns.describeTable(describeTableRequest);
+ Assertions.assertNotNull(loadTable);
+ Assertions.assertEquals(MINIO_ENDPOINT,
loadTable.getStorageOptions().get("endpoint"));
+ Assertions.assertEquals("true",
loadTable.getStorageOptions().get("allow_http"));
+ Assertions.assertEquals(MINIO_ACCESS_KEY,
loadTable.getStorageOptions().get("access_key_id"));
+ Assertions.assertEquals(
+ MINIO_SECRET_KEY,
loadTable.getStorageOptions().get("secret_access_key"));
+ Assertions.assertEquals(MINIO_REGION,
loadTable.getStorageOptions().get("region"));
+
Assertions.assertFalse(loadTable.getMetadata().containsKey("lance.storage.endpoint"));
+
Assertions.assertFalse(loadTable.getMetadata().containsKey("lance.storage.access_key_id"));
+
Assertions.assertFalse(loadTable.getMetadata().containsKey("lance.storage.secret_access_key"));
+ }
+
@Test
void testAlterColumns() throws Exception {
catalog = createCatalog(CATALOG_NAME);
@@ -1025,12 +1069,22 @@ public class LanceRESTServiceIT extends BaseIT {
}
private Catalog createCatalog(String catalogName) {
+ Map<String, String> catalogProperties =
+ ImmutableMap.<String, String>builder()
+ .putAll(properties)
+ .put("lance.storage.endpoint", MINIO_ENDPOINT)
+ .put("lance.storage.allow_http", "true")
+ .put("lance.storage.access_key_id", MINIO_ACCESS_KEY)
+ .put("lance.storage.secret_access_key", MINIO_SECRET_KEY)
+ .put("lance.storage.region", MINIO_REGION)
+ .build();
+
return metalake.createCatalog(
catalogName,
Catalog.Type.RELATIONAL,
"lakehouse-generic",
"catalog for lance rest service tests",
- properties);
+ catalogProperties);
}
private void createSchema() {
