This is an automated email from the ASF dual-hosted git repository.
zhoujinsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 505a66e1e [AMORO-4280] Add support for Lance format on REST metastore
(#4334)
505a66e1e is described below
commit 505a66e1e454a4706caca6857e84d4e7f7a39282
Author: WenLingzhang <[email protected]>
AuthorDate: Wed Aug 26 11:04:02 2026 +0800
[AMORO-4280] Add support for Lance format on REST metastore (#4334)
* Add support for Lance format on REST metastore
* skip TableMetaStore auth build for REST + Lance-only catalog
* fixup style
---
.../amoro/server/catalog/CatalogBuilder.java | 9 +-
.../dashboard/controller/CatalogController.java | 16 +-
.../java/org/apache/amoro/utils/CatalogUtil.java | 12 +
.../amoro/formats/lance/AbstractLanceCatalog.java | 111 ++++++++
.../amoro/formats/lance/LanceCatalogFactory.java | 4 +
.../formats/lance/LanceDirectoryV1Catalog.java | 121 ++-------
.../amoro/formats/lance/LanceRestCatalog.java | 134 ++++++++++
amoro-web/src/views/catalogs/Detail.vue | 280 ++++++++++++---------
8 files changed, 471 insertions(+), 216 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
index af39ba533..5c5594e56 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
@@ -59,7 +59,7 @@ public class CatalogBuilder {
CATALOG_TYPE_GLUE,
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
CATALOG_TYPE_REST,
- Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
+ Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.LANCE),
CATALOG_TYPE_CUSTOM,
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
CATALOG_TYPE_HIVE,
@@ -86,6 +86,13 @@ public class CatalogBuilder {
"Table format %s is not supported for metastore type: %s",
tableFormats,
type);
+ Preconditions.checkState(
+ !(CATALOG_TYPE_REST.equals(type)
+ && tableFormats.contains(TableFormat.LANCE)
+ && tableFormats.size() > 1),
+ "REST catalog serves a single protocol per uri,"
+ + " Lance cannot be combined with other table formats: %s",
+ tableFormats);
switch (type) {
case CATALOG_TYPE_HADOOP:
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
index 172b2a0c8..b7dcd5651 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
@@ -208,6 +208,15 @@ public class CatalogController {
CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_OSS, MIXED_ICEBERG));
+
+ // Lance REST is a data proxy: the client never talks to object storage,
so the storage config
+ // is not consumed, but it still reuses the standard REST storage types.
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST, STORAGE_CONFIGS_VALUE_TYPE_S3,
LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, LANCE));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
}
private final PlatformFileManager platformFileInfoService;
@@ -473,8 +482,11 @@ public class CatalogController {
catalogMeta.putToCatalogProperties(
CatalogProperties.CATALOG_IMPL, GlueCatalog.class.getName());
} else if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(metastoreType)) {
- catalogMeta.putToCatalogProperties(
- CatalogProperties.CATALOG_IMPL, RESTCatalog.class.getName());
+ // Only the Iceberg-family formats need the Iceberg RESTCatalog
implementation.
+ if (!info.getTableFormatList().contains(LANCE.name())) {
+ catalogMeta.putToCatalogProperties(
+ CatalogProperties.CATALOG_IMPL, RESTCatalog.class.getName());
+ }
}
catalogMeta.putToCatalogProperties(
diff --git a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
index e35fbc65a..46285b9d0 100644
--- a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
+++ b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
@@ -121,6 +121,13 @@ public class CatalogUtil {
/** Build {@link TableMetaStore} from catalog meta. */
public static TableMetaStore buildMetaStore(CatalogMeta catalogMeta) {
+ // REST + Lance manages storage/auth server-side, so a client-side
TableMetaStore is
+ // unnecessary. Set a local configuration so build() returns a
disabled-auth store.
+ if
(CatalogMetaProperties.CATALOG_TYPE_REST.equalsIgnoreCase(catalogMeta.getCatalogType())
+ && isLanceOnly(catalogMeta)) {
+ return TableMetaStore.builder().withConfiguration(new
Configuration()).build();
+ }
+
// load storage configs
TableMetaStore.Builder builder = TableMetaStore.builder();
boolean isLocalStorage = false;
@@ -215,6 +222,11 @@ public class CatalogUtil {
return builder.build();
}
+ private static boolean isLanceOnly(CatalogMeta catalogMeta) {
+ Set<TableFormat> formats = tableFormats(catalogMeta);
+ return formats.size() == 1 && formats.contains(TableFormat.LANCE);
+ }
+
public static TableIdentifier tableId(TableMeta tableMeta) {
return TableIdentifier.of(
tableMeta.getTableIdentifier().getCatalog(),
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/AbstractLanceCatalog.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/AbstractLanceCatalog.java
new file mode 100644
index 000000000..8ba7e1fc0
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/AbstractLanceCatalog.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.AmoroTable;
+import org.apache.amoro.FormatCatalog;
+import org.apache.amoro.NoSuchDatabaseException;
+import org.apache.amoro.NoSuchTableException;
+import org.apache.amoro.table.TableIdentifier;
+import org.lance.Dataset;
+import org.lance.namespace.LanceNamespace;
+import org.lance.namespace.errors.TableNotFoundException;
+import org.lance.namespace.model.DropTableRequest;
+import org.lance.namespace.model.ListTablesRequest;
+import org.lance.namespace.model.ListTablesResponse;
+import org.lance.namespace.model.TableExistsRequest;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/** Base implementation of {@link FormatCatalog} for Lance. */
+public abstract class AbstractLanceCatalog implements FormatCatalog {
+
+ protected final String catalogName;
+ protected final LanceNamespace namespace;
+
+ protected AbstractLanceCatalog(String catalogName, LanceNamespace namespace)
{
+ this.catalogName = catalogName;
+ this.namespace = namespace;
+ }
+
+ /** Composes the namespace table id used to reference a single table. */
+ protected abstract List<String> tableId(String database, String table);
+
+ /** Composes the namespace id used to list tables of a database. */
+ protected abstract List<String> tableIdForListTables(String database);
+
+ @Override
+ public boolean tableExists(String database, String table) {
+ if (!databaseExists(database)) {
+ return false;
+ }
+
+ try {
+ namespace.tableExists(new TableExistsRequest().id(tableId(database,
table)));
+ return true;
+ } catch (TableNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public AmoroTable<?> loadTable(String database, String tableName) {
+ if (!tableExists(database, tableName)) {
+ throw new NoSuchTableException("Table: " + database + "." + tableName +
" does not exist");
+ }
+
+ TableIdentifier identifier = TableIdentifier.of(catalogName, database,
tableName);
+ Dataset dataset =
+ Dataset.open().namespaceClient(namespace).tableId(tableId(database,
tableName)).build();
+ return new LanceTable(identifier, dataset, Collections.emptyMap());
+ }
+
+ @Override
+ public boolean dropTable(String database, String table, boolean purge) {
+ validateDatabase(database);
+
+ try {
+ namespace.dropTable(new DropTableRequest().id(tableId(database, table)));
+ return true;
+ } catch (TableNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public List<String> listTables(String database) {
+ if (!databaseExists(database)) {
+ return Collections.emptyList();
+ }
+ ListTablesRequest request = new
ListTablesRequest().id(tableIdForListTables(database));
+ ListTablesResponse response = namespace.listTables(request);
+ if (response == null) {
+ return Collections.emptyList();
+ }
+ return new ArrayList<>(response.getTables());
+ }
+
+ protected void validateDatabase(String database) {
+ if (!databaseExists(database)) {
+ throw new NoSuchDatabaseException("Database: " + database + " does not
exist");
+ }
+ }
+}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
index a78fa1ae3..a3faa8b2d 100755
---
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java
@@ -21,6 +21,7 @@ package org.apache.amoro.formats.lance;
import org.apache.amoro.FormatCatalog;
import org.apache.amoro.FormatCatalogFactory;
import org.apache.amoro.TableFormat;
+import org.apache.amoro.properties.CatalogMetaProperties;
import org.apache.amoro.table.TableMetaStore;
import java.util.HashMap;
@@ -38,6 +39,9 @@ public class LanceCatalogFactory implements
FormatCatalogFactory {
String metastoreType,
Map<String, String> properties,
TableMetaStore metaStore) {
+ if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(metastoreType)) {
+ return new LanceRestCatalog(catalogName, properties);
+ }
return new LanceDirectoryV1Catalog(catalogName, properties);
}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
index 926d91ec8..19f8f2b08 100755
---
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java
@@ -18,26 +18,13 @@
package org.apache.amoro.formats.lance;
-import org.apache.amoro.AmoroTable;
-import org.apache.amoro.FormatCatalog;
-import org.apache.amoro.NoSuchDatabaseException;
-import org.apache.amoro.NoSuchTableException;
import org.apache.amoro.properties.CatalogMetaProperties;
-import org.apache.amoro.table.TableIdentifier;
-import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.util.Preconditions;
import org.apache.iceberg.aliyun.AliyunProperties;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
-import org.lance.Dataset;
import org.lance.namespace.LanceNamespace;
-import org.lance.namespace.errors.TableNotFoundException;
-import org.lance.namespace.model.DropTableRequest;
-import org.lance.namespace.model.ListTablesRequest;
-import org.lance.namespace.model.ListTablesResponse;
-import org.lance.namespace.model.TableExistsRequest;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -50,22 +37,22 @@ import java.util.Map;
* root directory, each immediate subdirectory whose name ends with ".lance"
is treated as a Lance
* dataset. All tables live in a single logical database named "default".
*/
-public class LanceDirectoryV1Catalog implements FormatCatalog {
+public class LanceDirectoryV1Catalog extends AbstractLanceCatalog {
private static final String DEFAULT_DATABASE = "default";
private static final String STORAGE_ACCESS_KEY_ID = "storage.access_key_id";
private static final String STORAGE_SECRET_ACCESS_KEY =
"storage.secret_access_key";
private static final String STORAGE_ENDPOINT = "storage.endpoint";
- private final String catalogName;
- private final Map<String, String> namespaceProperties;
- private final LanceNamespace namespace;
public LanceDirectoryV1Catalog(String catalogName, Map<String, String>
catalogProperties) {
+ super(catalogName, buildNamespace(catalogProperties));
+ }
+
+ private static LanceNamespace buildNamespace(Map<String, String>
catalogProperties) {
Preconditions.checkArgument(
catalogProperties != null && !catalogProperties.isEmpty(),
"Catalog properties must be set.");
- this.catalogName = catalogName;
- this.namespaceProperties = new HashMap<>(catalogProperties);
+ Map<String, String> namespaceProperties = new HashMap<>(catalogProperties);
String root =
namespaceProperties.remove(CatalogMetaProperties.KEY_WAREHOUSE);
String storageAccessKey =
removeFirstNonNull(
@@ -80,42 +67,37 @@ public class LanceDirectoryV1Catalog implements
FormatCatalog {
Preconditions.checkArgument(
root != null && !root.isEmpty(), "Warehouse must be set in
catalogProperties.");
- this.namespaceProperties.put("manifest_enabled", "false");
- this.namespaceProperties.put("vend_input_storage_options", "true");
- this.namespaceProperties.put("root", root);
+ namespaceProperties.put("manifest_enabled", "false");
+ namespaceProperties.put("vend_input_storage_options", "true");
+ namespaceProperties.put("root", root);
if (storageAccessKey != null) {
- this.namespaceProperties.put(STORAGE_ACCESS_KEY_ID, storageAccessKey);
+ namespaceProperties.put(STORAGE_ACCESS_KEY_ID, storageAccessKey);
}
if (storageSecretKey != null) {
- this.namespaceProperties.put(STORAGE_SECRET_ACCESS_KEY,
storageSecretKey);
+ namespaceProperties.put(STORAGE_SECRET_ACCESS_KEY, storageSecretKey);
}
putStorageEndpointIfPresent(namespaceProperties);
- this.namespace = initializeNamespace(new RootAllocator(Long.MAX_VALUE));
+ return LanceNamespace.connect("dir", namespaceProperties, new
RootAllocator(Long.MAX_VALUE));
}
@Override
- public List<String> listDatabases() {
- return Collections.singletonList(DEFAULT_DATABASE);
+ protected List<String> tableId(String database, String table) {
+ return Collections.singletonList(table);
}
@Override
- public boolean databaseExists(String database) {
- return DEFAULT_DATABASE.equals(database);
+ protected List<String> tableIdForListTables(String database) {
+ return Collections.emptyList();
}
@Override
- public boolean tableExists(String database, String table) {
- if (!databaseExists(database)) {
- return false;
- }
+ public List<String> listDatabases() {
+ return Collections.singletonList(DEFAULT_DATABASE);
+ }
- try {
- TableExistsRequest request = new
TableExistsRequest().id(Collections.singletonList(table));
- namespace.tableExists(request);
- return true;
- } catch (TableNotFoundException e) {
- return false;
- }
+ @Override
+ public boolean databaseExists(String database) {
+ return DEFAULT_DATABASE.equals(database);
}
@Override
@@ -128,59 +110,6 @@ public class LanceDirectoryV1Catalog implements
FormatCatalog {
throw new UnsupportedOperationException("Dropping Lance databases is not
supported.");
}
- @Override
- public AmoroTable<?> loadTable(String database, String tableName) {
- if (!databaseExists(database) || !tableExists(database, tableName)) {
- throw new NoSuchTableException("Table: " + database + "." + tableName +
" does not exist");
- }
-
- TableIdentifier identifier = TableIdentifier.of(catalogName, database,
tableName);
- Dataset dataset =
- Dataset.open()
- .namespaceClient(namespace)
- .tableId(Collections.singletonList(tableName))
- .build();
- return new LanceTable(identifier, dataset, Collections.emptyMap());
- }
-
- @Override
- public boolean dropTable(String database, String table, boolean purge) {
- validateDatabase(database);
-
- try {
- namespace.dropTable(new
DropTableRequest().id(Collections.singletonList(table)));
- return true;
- } catch (TableNotFoundException e) {
- return false;
- }
- }
-
- @Override
- public List<String> listTables(String database) {
- if (!databaseExists(database)) {
- return Collections.emptyList();
- }
- return listTablesFromNamespace();
- }
-
- private List<String> listTablesFromNamespace() {
- if (namespace == null) {
- return Collections.emptyList();
- }
-
- ListTablesRequest request = new
ListTablesRequest().id(Collections.emptyList());
- ListTablesResponse response = namespace.listTables(request);
- if (response == null) {
- return Collections.emptyList();
- }
-
- return new ArrayList<>(response.getTables());
- }
-
- private LanceNamespace initializeNamespace(BufferAllocator allocator) {
- return LanceNamespace.connect("dir", namespaceProperties, allocator);
- }
-
private static String removeFirstNonNull(Map<String, String> properties,
String... keys) {
String value = null;
for (String key : keys) {
@@ -205,10 +134,4 @@ public class LanceDirectoryV1Catalog implements
FormatCatalog {
properties.put(STORAGE_ENDPOINT, endpoint);
}
}
-
- private void validateDatabase(String database) {
- if (!databaseExists(database)) {
- throw new NoSuchDatabaseException("Database: " + database + " does not
exist");
- }
- }
}
diff --git
a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceRestCatalog.java
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceRestCatalog.java
new file mode 100644
index 000000000..fdf237c12
--- /dev/null
+++
b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceRestCatalog.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.formats.lance;
+
+import org.apache.amoro.AlreadyExistsException;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.Preconditions;
+import org.lance.namespace.LanceNamespace;
+import org.lance.namespace.errors.NamespaceAlreadyExistsException;
+import org.lance.namespace.errors.NamespaceNotFoundException;
+import org.lance.namespace.model.CreateNamespaceRequest;
+import org.lance.namespace.model.DropNamespaceRequest;
+import org.lance.namespace.model.ListNamespacesRequest;
+import org.lance.namespace.model.ListNamespacesResponse;
+import org.lance.namespace.model.NamespaceExistsRequest;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * REST catalog implementation for Lance.
+ *
+ * <p>This catalog connects to a Lance Namespace REST service via {@code
+ * LanceNamespace.connect("rest", ...)} and maps the REST namespace hierarchy
to Amoro's
+ * database/table model.
+ */
+public class LanceRestCatalog extends AbstractLanceCatalog {
+
+ private static final String URI_PROPERTY = "uri";
+
+ /**
+ * Optional catalog name prepended to the REST namespace id, mapping Amoro's
database/table onto
+ * the service's schema/table levels (catalog/schema/table hierarchy). Empty
when the service has
+ * no catalog level.
+ */
+ private static final String CATALOG_PROPERTY = "catalog-name";
+
+ private final List<String> catalogPrefix;
+
+ public LanceRestCatalog(String catalogName, Map<String, String>
catalogProperties) {
+ super(catalogName, buildNamespace(catalogProperties));
+ this.catalogPrefix = catalogPrefix(catalogProperties);
+ }
+
+ private static List<String> catalogPrefix(Map<String, String>
catalogProperties) {
+ String catalog = catalogProperties.get(CATALOG_PROPERTY);
+ return catalog == null || catalog.isEmpty()
+ ? Collections.emptyList()
+ : Collections.singletonList(catalog);
+ }
+
+ private static LanceNamespace buildNamespace(Map<String, String>
namespaceProperties) {
+ Preconditions.checkArgument(
+ namespaceProperties != null && !namespaceProperties.isEmpty(),
+ "Catalog properties must be set.");
+ Preconditions.checkArgument(
+ namespaceProperties.containsKey(URI_PROPERTY),
+ "URI must be set in namespaceProperties for REST metastore type.");
+ Map<String, String> restProperties = new HashMap<>(namespaceProperties);
+ return LanceNamespace.connect("rest", restProperties, new
RootAllocator(Long.MAX_VALUE));
+ }
+
+ @Override
+ protected List<String> tableId(String database, String table) {
+ return concat(catalogPrefix, database, table);
+ }
+
+ @Override
+ protected List<String> tableIdForListTables(String database) {
+ return concat(catalogPrefix, database);
+ }
+
+ private static List<String> concat(List<String> prefix, String... parts) {
+ List<String> id = new ArrayList<>(prefix);
+ Collections.addAll(id, parts);
+ return id;
+ }
+
+ @Override
+ public List<String> listDatabases() {
+ ListNamespacesRequest request = new
ListNamespacesRequest().id(catalogPrefix);
+ ListNamespacesResponse response = namespace.listNamespaces(request);
+ if (response == null) {
+ return Collections.emptyList();
+ }
+ return new ArrayList<>(response.getNamespaces());
+ }
+
+ @Override
+ public boolean databaseExists(String database) {
+ try {
+ NamespaceExistsRequest request =
+ new NamespaceExistsRequest().id(concat(catalogPrefix, database));
+ namespace.namespaceExists(request);
+ return true;
+ } catch (NamespaceNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void createDatabase(String database) {
+ try {
+ namespace.createNamespace(new
CreateNamespaceRequest().id(concat(catalogPrefix, database)));
+ } catch (NamespaceAlreadyExistsException e) {
+ throw new AlreadyExistsException("Database: " + database + " already
exists", e);
+ }
+ }
+
+ @Override
+ public void dropDatabase(String database) {
+ validateDatabase(database);
+ namespace.dropNamespace(new
DropNamespaceRequest().id(concat(catalogPrefix, database)));
+ }
+}
diff --git a/amoro-web/src/views/catalogs/Detail.vue
b/amoro-web/src/views/catalogs/Detail.vue
index e088b2f39..6f78999be 100644
--- a/amoro-web/src/views/catalogs/Detail.vue
+++ b/amoro-web/src/views/catalogs/Detail.vue
@@ -119,6 +119,7 @@ const tableFormatMap = {
MIXED_ICEBERG: 'MIXED_ICEBERG',
PAIMON: 'PAIMON',
HUDI: 'HUDI',
+ LANCE: 'LANCE',
}
const tableFormatText = {
@@ -127,6 +128,7 @@ const tableFormatText = {
[tableFormatMap.MIXED_ICEBERG]: 'Mixed Iceberg',
[tableFormatMap.PAIMON]: 'Paimon',
[tableFormatMap.HUDI]: 'Hudi',
+ [tableFormatMap.LANCE]: 'Lance',
}
const storageConfigFileNameMap = {
@@ -248,7 +250,12 @@ async function loadMetastoreCapabilities(loadTableFormats:
boolean) {
const options = (formats || []) as string[]
tableFormatOptions.value = options
if (loadTableFormats) {
- formState.tableFormatList = [...options]
+ // REST catalog: Lance and Iceberg use different protocols (one protocol
per uri),
+ // so they can't coexist. Default to Iceberg family; Lance stays unchecked
(opt-in).
+ const defaultFormats = type === 'rest'
+ ? options.filter(format => format !== tableFormatMap.LANCE)
+ : options
+ formState.tableFormatList = defaultFormats.length > 0 ?
[...defaultFormats] : [...options]
}
const storageTypes = await getMetastoreStorageTypes(type)
storageTypeOptions.value = (storageTypes || []).map((val: string) => ({
@@ -394,6 +401,11 @@ const authTypeOptions = computed(() => {
const isAuthDisabled = computed(() => formState.storageConfig['storage.type']
=== 'Local')
+// REST + Lance needs no client-side object storage/auth; storage is
server-managed.
+const isRestLance = computed(() => {
+ return formState.catalog.type === 'rest' &&
formState.tableFormatList.includes(tableFormatMap.LANCE)
+})
+
watch(
() => formState.storageConfig['storage.type'],
(storageType) => {
@@ -409,7 +421,37 @@ watch(
formState.authConfig[simpleUsernameKey] = 'local'
}
}
- }
+ },
+)
+
+watch(
+ () => formState.tableFormatList,
+ (formats, oldFormats) => {
+ const isRest = formState.catalog.type === 'rest'
+ const hasLance = formats.includes(tableFormatMap.LANCE)
+
+ if (isRest && hasLance && formats.length > 1) {
+ // If Lance was just added, keep only Lance; otherwise the user switched
away from Lance.
+ const hadLance = oldFormats?.includes(tableFormatMap.LANCE)
+ formState.tableFormatList = hadLance
+ ? formats.filter(format => format !== tableFormatMap.LANCE)
+ : [tableFormatMap.LANCE]
+ return
+ }
+
+ // REST + Lance needs no client-side storage/auth; drop them on selection .
+ if (isRest && hasLance) {
+ Object.keys(formState.storageConfig).forEach((key) => {
+ delete formState.storageConfig[key]
+ })
+ Object.keys(formState.authConfig).forEach((key) => {
+ delete formState.authConfig[key]
+ })
+ }
+ else if (isRest && !formState.storageConfig['storage.type']) {
+ formState.storageConfig['storage.type'] = 'S3'
+ }
+ },
)
async function changeMetastore() {
@@ -488,7 +530,7 @@ function handleSave() {
formRef.value
.validateFields()
.then(async () => {
- const { catalog, tableFormatList, storageConfig, authConfig } = formState
+ const { catalog, tableFormatList } = formState
const properties = await propertiesRef.value.getProperties()
const tableProperties = await tablePropertiesRef.value.getProperties()
if (!properties) {
@@ -500,6 +542,14 @@ function handleSave() {
loading.value = true
const catalogParams = catalog
getFileIdParams()
+ let storageConfig = formState.storageConfig
+ let authConfig = formState.authConfig
+ if (isRestLance.value) {
+ // Lance REST manages storage server-side; send a neutral S3 storage
type
+ // (accepted by the backend REST+S3+LANCE whitelist) and no auth
config.
+ storageConfig = { 'storage.type': 'S3' }
+ authConfig = {}
+ }
await saveCatalogsSetting({
isCreate: isNewCatalog.value,
...catalogParams,
@@ -641,126 +691,128 @@ onMounted(() => {
/>
<span v-else>{{ formState.catalog.optimizerGroup }}</span>
</a-form-item>
- <a-form-item>
- <p class="header">
- {{ $t('storageConfigName') }}
- </p>
- </a-form-item>
- <a-form-item :label="$t('type')" :name="['storageConfig',
'storage.type']" :rules="[{ required: isEdit }]">
- <a-select
- v-if="isEdit"
v-model:value="formState.storageConfig['storage.type']"
- :placeholder="placeholder.selectPh" :options="storageTypeOptions"
- />
- <span v-else class="config-value">{{
formState.storageConfig['storage.type'] }}</span>
- </a-form-item>
- <a-form-item
- v-if="formState.storageConfig['storage.type'] === 'S3'"
:label="$t('endpoint')"
- :name="['storageConfig', 'storage.s3.endpoint']" :rules="[{
required: false }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.s3.endpoint']" />
- <span v-else class="config-value">{{
formState.storageConfig['storage.s3.endpoint'] }}</span>
- </a-form-item>
- <a-form-item
- v-if="formState.storageConfig['storage.type'] === 'S3'"
:label="$t('region')"
- :name="['storageConfig', 'storage.s3.region']" :rules="[{
required: false }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.s3.region']" />
- <span v-else class="config-value">{{
formState.storageConfig['storage.s3.region'] }}</span>
- </a-form-item>
- <a-form-item
+ <template v-if="!isRestLance">
+ <a-form-item>
+ <p class="header">
+ {{ $t('storageConfigName') }}
+ </p>
+ </a-form-item>
+ <a-form-item :label="$t('type')" :name="['storageConfig',
'storage.type']" :rules="[{ required: isEdit }]">
+ <a-select
+ v-if="isEdit"
v-model:value="formState.storageConfig['storage.type']"
+ :placeholder="placeholder.selectPh"
:options="storageTypeOptions"
+ />
+ <span v-else class="config-value">{{
formState.storageConfig['storage.type'] }}</span>
+ </a-form-item>
+ <a-form-item
+ v-if="formState.storageConfig['storage.type'] === 'S3'"
:label="$t('endpoint')"
+ :name="['storageConfig', 'storage.s3.endpoint']" :rules="[{
required: false }]"
+ >
+ <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.s3.endpoint']" />
+ <span v-else class="config-value">{{
formState.storageConfig['storage.s3.endpoint'] }}</span>
+ </a-form-item>
+ <a-form-item
+ v-if="formState.storageConfig['storage.type'] === 'S3'"
:label="$t('region')"
+ :name="['storageConfig', 'storage.s3.region']" :rules="[{
required: false }]"
+ >
+ <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.s3.region']" />
+ <span v-else class="config-value">{{
formState.storageConfig['storage.s3.region'] }}</span>
+ </a-form-item>
+ <a-form-item
v-if="formState.storageConfig['storage.type'] === 'OSS'"
:label="$t('endpoint')"
:name="['storageConfig', 'storage.oss.endpoint']" :rules="[{
required: false }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.oss.endpoint']" />
- <span v-else class="config-value">{{
formState.storageConfig['storage.oss.endpoint'] }}</span>
- </a-form-item>
- <div v-if="formState.storageConfig['storage.type'] === 'Hadoop'">
- <a-form-item
- v-for="config in formState.storageConfigArray"
:key="config.label" :label="config.label"
- class="g-flex-ac"
>
- <a-upload
- v-if="isEdit" v-model:file-list="config.fileList" name="file"
accept=".xml"
- :show-upload-list="false" :action="uploadUrl"
:disabled="config.uploadLoading"
- :headers="uploadHeaders"
- @change="(args: UploadChangeParam<UploadFile<any>>) =>
uploadFile(args, config, 'STORAGE')"
+ <a-input v-if="isEdit"
v-model:value="formState.storageConfig['storage.oss.endpoint']" />
+ <span v-else class="config-value">{{
formState.storageConfig['storage.oss.endpoint'] }}</span>
+ </a-form-item>
+ <div v-if="formState.storageConfig['storage.type'] === 'Hadoop'">
+ <a-form-item
+ v-for="config in formState.storageConfigArray"
:key="config.label" :label="config.label"
+ class="g-flex-ac"
>
- <a-button type="primary" ghost :loading="config.uploadLoading"
class="g-mr-12">
- {{ $t('upload')
- }}
- </a-button>
- </a-upload>
- <span
- v-if="config.isSuccess || config.fileName" class="config-value"
- :class="{ 'view-active': !!config.fileUrl }"
@click="viewFileDetail(config.fileUrl, config.fileName)"
- >{{ config.fileName
- }}</span>
+ <a-upload
+ v-if="isEdit" v-model:file-list="config.fileList"
name="file" accept=".xml"
+ :show-upload-list="false" :action="uploadUrl"
:disabled="config.uploadLoading"
+ :headers="uploadHeaders"
+ @change="(args: UploadChangeParam<UploadFile<any>>) =>
uploadFile(args, config, 'STORAGE')"
+ >
+ <a-button type="primary" ghost
:loading="config.uploadLoading" class="g-mr-12">
+ {{ $t('upload')
+ }}
+ </a-button>
+ </a-upload>
+ <span
+ v-if="config.isSuccess || config.fileName"
class="config-value"
+ :class="{ 'view-active': !!config.fileUrl }"
@click="viewFileDetail(config.fileUrl, config.fileName)"
+ >{{ config.fileName
+ }}</span>
+ </a-form-item>
+ </div>
+ <a-form-item>
+ <p class="header">
+ {{ $t('authenticationConfig') }}
+ </p>
+ </a-form-item>
+ <a-form-item :label="$t('type')" :name="['authConfig',
'auth.type']" :rules="[{ required: isEdit && !isAuthDisabled }]">
+ <a-select
+ v-if="isEdit" v-model:value="formState.authConfig['auth.type']"
+ :placeholder="placeholder.selectPh" :options="authTypeOptions"
:disabled="isAuthDisabled"
+ />
+ <span v-else class="config-value">{{
formState.authConfig['auth.type'] }}</span>
</a-form-item>
- </div>
- <a-form-item>
- <p class="header">
- {{ $t('authenticationConfig') }}
- </p>
- </a-form-item>
- <a-form-item :label="$t('type')" :name="['authConfig', 'auth.type']"
:rules="[{ required: isEdit && !isAuthDisabled }]">
- <a-select
- v-if="isEdit" v-model:value="formState.authConfig['auth.type']"
- :placeholder="placeholder.selectPh" :options="authTypeOptions"
:disabled="isAuthDisabled"
- />
- <span v-else class="config-value">{{
formState.authConfig['auth.type'] }}</span>
- </a-form-item>
- <a-form-item
- v-if="formState.authConfig['auth.type'] === 'SIMPLE'"
:label="$t('hadoopUsername')"
- :name="['authConfig', 'auth.simple.hadoop_username']" :rules="[{
required: isEdit && !isAuthDisabled }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.simple.hadoop_username']"
:disabled="isAuthDisabled" />
- <span v-else class="config-value">{{
formState.authConfig['auth.simple.hadoop_username'] }}</span>
- </a-form-item>
- <a-form-item
- v-if="formState.authConfig['auth.type'] === 'KERBEROS'"
:label="$t('kerberosPrincipal')"
- :name="['authConfig', 'auth.kerberos.principal']" :rules="[{
required: isEdit && !isAuthDisabled }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.kerberos.principal']"
:disabled="isAuthDisabled" />
- <span v-else class="config-value">{{
formState.authConfig['auth.kerberos.principal'] }}</span>
- </a-form-item>
- <div v-if="formState.authConfig['auth.type'] === 'KERBEROS'">
<a-form-item
- v-for="config in formState.authConfigArray" :key="config.label"
:label="config.label"
- class="g-flex-ac"
+ v-if="formState.authConfig['auth.type'] === 'SIMPLE'"
:label="$t('hadoopUsername')"
+ :name="['authConfig', 'auth.simple.hadoop_username']" :rules="[{
required: isEdit && !isAuthDisabled }]"
>
- <a-upload
- v-if="isEdit" v-model:file-list="config.fileList" name="file"
- :accept="config.key === 'auth.kerberos.keytab' ? '.keytab' :
'.conf'" :show-upload-list="false"
- :action="uploadUrl" :disabled="config.uploadLoading"
- :headers="uploadHeaders"
- @change="(args: UploadChangeParam<UploadFile<any>>) =>
uploadFile(args, config)"
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.simple.hadoop_username']"
:disabled="isAuthDisabled" />
+ <span v-else class="config-value">{{
formState.authConfig['auth.simple.hadoop_username'] }}</span>
+ </a-form-item>
+ <a-form-item
+ v-if="formState.authConfig['auth.type'] === 'KERBEROS'"
:label="$t('kerberosPrincipal')"
+ :name="['authConfig', 'auth.kerberos.principal']" :rules="[{
required: isEdit && !isAuthDisabled }]"
+ >
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.kerberos.principal']"
:disabled="isAuthDisabled" />
+ <span v-else class="config-value">{{
formState.authConfig['auth.kerberos.principal'] }}</span>
+ </a-form-item>
+ <div v-if="formState.authConfig['auth.type'] === 'KERBEROS'">
+ <a-form-item
+ v-for="config in formState.authConfigArray"
:key="config.label" :label="config.label"
+ class="g-flex-ac"
>
- <a-button type="primary" ghost :loading="config.uploadLoading"
class="g-mr-12">
- {{ $t('upload')
- }}
- </a-button>
- </a-upload>
- <span
- v-if="config.isSuccess || config.fileName" class="config-value
auth-filename"
- :class="{ 'view-active': !!config.fileUrl }"
:title="config.fileName"
- @click="viewFileDetail(config.fileUrl, config.fileName)"
- >{{ config.fileName }}</span>
+ <a-upload
+ v-if="isEdit" v-model:file-list="config.fileList" name="file"
+ :accept="config.key === 'auth.kerberos.keytab' ? '.keytab' :
'.conf'" :show-upload-list="false"
+ :action="uploadUrl" :disabled="config.uploadLoading"
+ :headers="uploadHeaders"
+ @change="(args: UploadChangeParam<UploadFile<any>>) =>
uploadFile(args, config)"
+ >
+ <a-button type="primary" ghost
:loading="config.uploadLoading" class="g-mr-12">
+ {{ $t('upload')
+ }}
+ </a-button>
+ </a-upload>
+ <span
+ v-if="config.isSuccess || config.fileName"
class="config-value auth-filename"
+ :class="{ 'view-active': !!config.fileUrl }"
:title="config.fileName"
+ @click="viewFileDetail(config.fileUrl, config.fileName)"
+ >{{ config.fileName }}</span>
+ </a-form-item>
+ </div>
+ <a-form-item
+ v-if="formState.authConfig['auth.type'] === 'AK/SK'"
:label="$t('accessKey')"
+ :name="['authConfig', 'auth.ak_sk.access_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
+ >
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.access_key']"
:disabled="isAuthDisabled" />
+ <span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.access_key'] }}</span>
</a-form-item>
- </div>
- <a-form-item
- v-if="formState.authConfig['auth.type'] === 'AK/SK'"
:label="$t('accessKey')"
- :name="['authConfig', 'auth.ak_sk.access_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.access_key']"
:disabled="isAuthDisabled" />
- <span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.access_key'] }}</span>
- </a-form-item>
- <a-form-item
- v-if="formState.authConfig['auth.type'] === 'AK/SK'"
:label="$t('secretKey')"
- :name="['authConfig', 'auth.ak_sk.secret_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
- >
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.secret_key']"
:disabled="isAuthDisabled" />
- <span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.secret_key'] }}</span>
- </a-form-item>
+ <a-form-item
+ v-if="formState.authConfig['auth.type'] === 'AK/SK'"
:label="$t('secretKey')"
+ :name="['authConfig', 'auth.ak_sk.secret_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
+ >
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.secret_key']"
:disabled="isAuthDisabled" />
+ <span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.secret_key'] }}</span>
+ </a-form-item>
+ </template>
<a-form-item>
<p class="header">
{{ $t('properties') }}
