This is an automated email from the ASF dual-hosted git repository.
nathanma pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new f5b0ac564 [Feature]: unify filesystem alias and add metastore matrix
apis (#4060)
f5b0ac564 is described below
commit f5b0ac5641942368b8fb763ab2ba85f570529355
Author: nathan.ma <[email protected]>
AuthorDate: Mon Jan 26 16:46:01 2026 +0800
[Feature]: unify filesystem alias and add metastore matrix apis (#4060)
* [Feature]: unify filesystem alias and add metastore matrix apis
* spotless
* fix ci
---------
Co-authored-by: majin.nathan <[email protected]>
---
.../amoro/server/catalog/CatalogBuilder.java | 5 +
.../server/catalog/DefaultCatalogManager.java | 5 +-
.../amoro/server/dashboard/DashboardServer.java | 2 +
.../dashboard/controller/CatalogController.java | 59 +++++-
.../amoro/server/terminal/TerminalManager.java | 13 +-
.../org/apache/amoro/CommonUnifiedCatalog.java | 9 +-
.../amoro/properties/CatalogMetaProperties.java | 9 +-
.../java/org/apache/amoro/utils/CatalogUtil.java | 195 ++++++++++++-------
.../amoro/formats/hudi/HudiCatalogFactory.java | 3 +-
.../formats/mixed/MixedIcebergCatalogFactory.java | 3 +-
.../java/org/apache/amoro/mixed/CatalogLoader.java | 3 +-
.../apache/amoro/utils/MixedFormatCatalogUtil.java | 3 +-
amoro-web/src/services/setting.services.ts | 6 +
amoro-web/src/views/catalogs/Detail.vue | 215 +++++++++------------
.../src/views/tables/components/TableExplorer.vue | 2 +-
amoro-web/src/views/tables/index.vue | 5 +
16 files changed, 316 insertions(+), 221 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
index d405bc1c3..31c8240ca 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
@@ -20,6 +20,7 @@ package org.apache.amoro.server.catalog;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_AMS;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_CUSTOM;
+import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_GLUE;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HADOOP;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HIVE;
@@ -42,6 +43,9 @@ public class CatalogBuilder {
private static final Map<String, Set<TableFormat>> formatSupportedMatrix =
ImmutableMap.of(
CATALOG_TYPE_HADOOP,
+ Sets.newHashSet(
+ TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.PAIMON, TableFormat.HUDI),
+ CATALOG_TYPE_FILESYSTEM,
Sets.newHashSet(
TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.PAIMON, TableFormat.HUDI),
CATALOG_TYPE_GLUE,
@@ -77,6 +81,7 @@ public class CatalogBuilder {
switch (type) {
case CATALOG_TYPE_HADOOP:
+ case CATALOG_TYPE_FILESYSTEM:
case CATALOG_TYPE_GLUE:
case CATALOG_TYPE_REST:
case CATALOG_TYPE_CUSTOM:
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/DefaultCatalogManager.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/DefaultCatalogManager.java
index 1c3dc52a5..ce56959be 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/DefaultCatalogManager.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/DefaultCatalogManager.java
@@ -40,6 +40,7 @@ import
org.apache.amoro.shade.guava32.com.google.common.cache.CacheLoader;
import org.apache.amoro.shade.guava32.com.google.common.cache.LoadingCache;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableIdentifier;
+import org.apache.amoro.utils.CatalogUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -230,7 +231,9 @@ public class DefaultCatalogManager extends PersistentBase
implements CatalogMana
}
private void validateCatalogUpdate(CatalogMeta oldMeta, CatalogMeta newMeta)
{
- if (!oldMeta.getCatalogType().equals(newMeta.getCatalogType())) {
+ String oldType =
CatalogUtil.normalizeMetastoreType(oldMeta.getCatalogType());
+ String newType =
CatalogUtil.normalizeMetastoreType(newMeta.getCatalogType());
+ if (!oldType.equals(newType)) {
throw new IllegalMetadataException("Cannot update catalog type");
}
}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
index 5233b7676..e465e65e7 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java
@@ -299,6 +299,8 @@ public class DashboardServer {
get("", tableController::getCatalogs);
post("", catalogController::createCatalog);
get("metastore/types", catalogController::getCatalogTypeList);
+ get("metastore/{type}/table-formats",
catalogController::getMetastoreTableFormats);
+ get("metastore/{type}/storage-types",
catalogController::getMetastoreStorageTypes);
get("/{catalogName}", catalogController::getCatalogDetail);
delete("/{catalogName}", catalogController::deleteCatalog);
put("/{catalogName}", catalogController::updateCatalog);
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
index 8a316e15f..d688f9e29 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
@@ -35,6 +35,7 @@ import static
org.apache.amoro.properties.CatalogMetaProperties.AUTH_CONFIGS_VAL
import static
org.apache.amoro.properties.CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_AMS;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_CUSTOM;
+import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_GLUE;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HADOOP;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HIVE;
@@ -48,6 +49,7 @@ import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_S3_ENDPOINT;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP;
+import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_OSS;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3;
import static org.apache.amoro.properties.CatalogMetaProperties.TABLE_FORMATS;
@@ -106,6 +108,8 @@ public class CatalogController {
CATALOG_REQUIRED_PROPERTIES.put(CATALOG_TYPE_AMS,
Lists.newArrayList(KEY_WAREHOUSE));
CATALOG_REQUIRED_PROPERTIES.put(
CATALOG_TYPE_HADOOP,
Lists.newArrayList(CatalogProperties.WAREHOUSE_LOCATION));
+ CATALOG_REQUIRED_PROPERTIES.put(
+ CATALOG_TYPE_FILESYSTEM,
Lists.newArrayList(CatalogProperties.WAREHOUSE_LOCATION));
CATALOG_REQUIRED_PROPERTIES.put(
CATALOG_TYPE_GLUE,
Lists.newArrayList(CatalogProperties.WAREHOUSE_LOCATION));
CATALOG_REQUIRED_PROPERTIES.put(
@@ -146,6 +150,17 @@ public class CatalogController {
CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(
+ CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP,
MIXED_ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_LOCAL, ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3,
ICEBERG));
VALIDATE_CATALOGS.add(
@@ -156,6 +171,8 @@ public class CatalogController {
CatalogDescriptor.of(CATALOG_TYPE_CUSTOM,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, ICEBERG));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_HADOOP,
STORAGE_CONFIGS_VALUE_TYPE_OSS, PAIMON));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM,
STORAGE_CONFIGS_VALUE_TYPE_OSS, PAIMON));
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(CATALOG_TYPE_GLUE,
STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG));
VALIDATE_CATALOGS.add(
@@ -232,9 +249,9 @@ public class CatalogController {
List<ImmutableMap<String, String>> catalogTypes = new ArrayList<>();
String valueKey = "value";
String displayKey = "display";
- catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_AMS, displayKey,
"Amoro Metastore"));
- catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HIVE, displayKey,
"Hive Metastore"));
- catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP,
displayKey, "Filesystem"));
+ catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_AMS, displayKey,
"Internal"));
+ catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HIVE, displayKey,
"Hive"));
+ catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_FILESYSTEM,
displayKey, "FileSystem"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_GLUE, displayKey,
"Glue"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_REST, displayKey,
"REST"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_CUSTOM,
displayKey, "Custom"));
@@ -432,14 +449,15 @@ public class CatalogController {
private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info,
CatalogMeta oldCatalogMeta) {
CatalogMeta catalogMeta = new CatalogMeta();
catalogMeta.setCatalogName(info.getName());
- catalogMeta.setCatalogType(info.getType());
+ String metastoreType = CatalogUtil.normalizeMetastoreType(info.getType());
+ catalogMeta.setCatalogType(metastoreType);
catalogMeta.setCatalogProperties(
PropertiesUtil.unionCatalogProperties(info.getTableProperties(),
info.getProperties()));
// fill catalog impl when catalog type is glue or rest
- if (CatalogMetaProperties.CATALOG_TYPE_GLUE.equals(info.getType())) {
+ if (CatalogMetaProperties.CATALOG_TYPE_GLUE.equals(metastoreType)) {
catalogMeta.putToCatalogProperties(
CatalogProperties.CATALOG_IMPL, GlueCatalog.class.getName());
- } else if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(info.getType()))
{
+ } else if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(metastoreType)) {
catalogMeta.putToCatalogProperties(
CatalogProperties.CATALOG_IMPL, RESTCatalog.class.getName());
}
@@ -519,6 +537,8 @@ public class CatalogController {
STORAGE_CONFIGS_KEY_OSS_ENDPOINT,
"fs.oss.endpoint");
}
+ } else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_LOCAL)) {
+ // Local storage type does not require additional storage configs.
} else {
throw new RuntimeException("Invalid storage type " + storageType);
}
@@ -622,12 +642,11 @@ public class CatalogController {
if (catalogService.catalogExist(catalogName)) {
info.setName(catalogMeta.getCatalogName());
- // We create ams catalog with type hadoop in v0.3, we should be
compatible with it.
if (CATALOG_TYPE_HADOOP.equals(catalogMeta.getCatalogType())
&& !catalogMeta.getCatalogProperties().containsKey(TABLE_FORMATS)) {
info.setType(CATALOG_TYPE_AMS);
} else {
- info.setType(catalogMeta.getCatalogType());
+
info.setType(CatalogUtil.normalizeMetastoreType(catalogMeta.getCatalogType()));
}
// we put the table format single
@@ -732,6 +751,30 @@ public class CatalogController {
}
}
+ public void getMetastoreTableFormats(Context ctx) {
+ String metastoreType = ctx.pathParam("type");
+ String normalizedType = CatalogUtil.normalizeMetastoreType(metastoreType);
+ List<String> tableFormats =
+ VALIDATE_CATALOGS.stream()
+ .filter(d -> d.catalogType.equalsIgnoreCase(normalizedType))
+ .map(d -> d.tableFormat.name())
+ .distinct()
+ .collect(Collectors.toList());
+ ctx.json(OkResponse.of(tableFormats));
+ }
+
+ public void getMetastoreStorageTypes(Context ctx) {
+ String metastoreType = ctx.pathParam("type");
+ String normalizedType = CatalogUtil.normalizeMetastoreType(metastoreType);
+ List<String> storageTypes =
+ VALIDATE_CATALOGS.stream()
+ .filter(d -> d.catalogType.equalsIgnoreCase(normalizedType))
+ .map(d -> d.storageType)
+ .distinct()
+ .collect(Collectors.toList());
+ ctx.json(OkResponse.of(storageTypes));
+ }
+
private static class CatalogDescriptor {
private final String catalogType;
private final String storageType;
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
b/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
index 2596d0428..4ab386bf0 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
@@ -37,6 +37,7 @@ import
org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.CatalogUtil;
+import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,7 +251,7 @@ public class TerminalManager {
// ========================== private method =========================
private String catalogConnectorType(CatalogMeta catalogMeta) {
- String catalogType = catalogMeta.getCatalogType();
+ String catalogType =
CatalogUtil.normalizeCatalogType(catalogMeta.getCatalogType());
Set<TableFormat> tableFormatSet = CatalogUtil.tableFormats(catalogMeta);
if (catalogType.equalsIgnoreCase(CatalogType.AMS.name())) {
if (tableFormatSet.size() > 1) {
@@ -301,8 +302,12 @@ public class TerminalManager {
TableMetaStore.Builder builder = TableMetaStore.builder();
if (catalogMeta.getStorageConfigs() != null) {
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
- if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
- CatalogUtil.getCompatibleStorageType(storageConfigs))) {
+ String storageType =
CatalogUtil.getCompatibleStorageType(storageConfigs);
+ if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL.equalsIgnoreCase(storageType))
{
+ builder.withConfiguration(new Configuration());
+ return builder.build();
+ }
+ if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(storageType))
{
builder
.withBase64MetaStoreSite(
catalogMeta
@@ -388,7 +393,7 @@ public class TerminalManager {
private void applyClientProperties(CatalogMeta catalogMeta) {
Set<TableFormat> formats = CatalogUtil.tableFormats(catalogMeta);
- String catalogType = catalogMeta.getCatalogType();
+ String catalogType =
CatalogUtil.normalizeCatalogType(catalogMeta.getCatalogType());
if (formats.contains(TableFormat.ICEBERG)) {
if
(CatalogMetaProperties.CATALOG_TYPE_AMS.equalsIgnoreCase(catalogType)) {
catalogMeta.putToCatalogProperties(
diff --git
a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
index 573cf42a0..b631c595f 100644
--- a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
+++ b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java
@@ -202,14 +202,17 @@ public class CommonUnifiedCatalog implements
UnifiedCatalog {
protected void initializeFormatCatalogs() {
ServiceLoader<FormatCatalogFactory> loader =
ServiceLoader.load(FormatCatalogFactory.class);
- Set<TableFormat> formats = CatalogUtil.tableFormats(metaStoreType,
catalogProperties);
+ String normalizedMetastoreType =
CatalogUtil.normalizeMetastoreType(metaStoreType);
+ Set<TableFormat> formats =
CatalogUtil.tableFormats(normalizedMetastoreType, catalogProperties);
Map<TableFormat, FormatCatalog> formatCatalogs = Maps.newConcurrentMap();
for (FormatCatalogFactory factory : loader) {
if (formats.contains(factory.format())) {
Map<String, String> formatCatalogProperties =
- factory.convertCatalogProperties(name(), metaStoreType,
this.catalogProperties);
+ factory.convertCatalogProperties(
+ name(), normalizedMetastoreType, this.catalogProperties);
FormatCatalog catalog =
- factory.create(name(), metaStoreType, formatCatalogProperties,
tableMetaStore);
+ factory.create(
+ name(), normalizedMetastoreType, formatCatalogProperties,
tableMetaStore);
formatCatalogs.put(factory.format(), catalog);
}
}
diff --git
a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
index 875a7c0f3..10f055b80 100644
---
a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
+++
b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
@@ -33,6 +33,7 @@ public class CatalogMetaProperties {
public static final String STORAGE_CONFIGS_VALUE_TYPE_HADOOP = "Hadoop";
public static final String STORAGE_CONFIGS_VALUE_TYPE_S3 = "S3";
public static final String STORAGE_CONFIGS_VALUE_TYPE_OSS = "OSS";
+ public static final String STORAGE_CONFIGS_VALUE_TYPE_LOCAL = "Local";
public static final String AUTH_CONFIGS_KEY_TYPE = "auth.type";
public static final String AUTH_CONFIGS_KEY_PRINCIPAL =
"auth.kerberos.principal";
@@ -53,7 +54,13 @@ public class CatalogMetaProperties {
public static final String KEY_TABLE_FILTER = "table-filter";
- public static final String CATALOG_TYPE_HADOOP = "hadoop";
+ /**
+ * Use {@link #CATALOG_TYPE_FILESYSTEM} instead for front But we can't
remove this until we could
+ * use filesystem and storage to determine the right catalog-impl
+ */
+ @Deprecated public static final String CATALOG_TYPE_HADOOP = "hadoop";
+
+ public static final String CATALOG_TYPE_FILESYSTEM = "filesystem";
public static final String CATALOG_TYPE_HIVE = "hive";
public static final String CATALOG_TYPE_AMS = "ams";
public static final String CATALOG_TYPE_GLUE = "glue";
diff --git a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
index 862ca8516..19694f835 100644
--- a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
+++ b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
@@ -27,12 +27,14 @@ import
org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -48,30 +50,40 @@ public class CatalogUtil {
/** Return table format set catalog supported. */
public static Set<TableFormat> tableFormats(
String metastoreType, Map<String, String> catalogProperties) {
+ Set<TableFormat> parsedFormats = null;
if (catalogProperties != null
&& catalogProperties.containsKey(CatalogMetaProperties.TABLE_FORMATS))
{
String tableFormatsProperty =
catalogProperties.get(CatalogMetaProperties.TABLE_FORMATS);
- return Arrays.stream(tableFormatsProperty.split(","))
- .map(
- tableFormatString ->
-
TableFormat.valueOf(tableFormatString.trim().toUpperCase(Locale.ROOT)))
- .collect(Collectors.toSet());
- } else {
- // Generate table format from catalog type for compatibility with older
versions
- switch (metastoreType) {
- case CatalogMetaProperties.CATALOG_TYPE_AMS:
- return Sets.newHashSet(TableFormat.MIXED_ICEBERG);
- case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
- case CatalogMetaProperties.CATALOG_TYPE_REST:
- case CatalogMetaProperties.CATALOG_TYPE_HADOOP:
- case CatalogMetaProperties.CATALOG_TYPE_GLUE:
- return Sets.newHashSet(TableFormat.ICEBERG);
- case CatalogMetaProperties.CATALOG_TYPE_HIVE:
- return Sets.newHashSet(TableFormat.MIXED_HIVE);
- default:
- throw new IllegalArgumentException("Unsupported catalog type:" +
metastoreType);
+ if (tableFormatsProperty != null) {
+ parsedFormats =
+ Arrays.stream(tableFormatsProperty.split(","))
+ .map(String::trim)
+ .filter(s -> !s.isEmpty())
+ .map(s -> TableFormat.valueOf(s.toUpperCase(Locale.ROOT)))
+ .filter(Objects::nonNull)
+ .collect(Collectors.toSet());
}
}
+
+ if (parsedFormats != null && !parsedFormats.isEmpty()) {
+ return parsedFormats;
+ }
+
+ // Generate table format from catalog type for compatibility with older
versions
+ switch (metastoreType) {
+ case CatalogMetaProperties.CATALOG_TYPE_AMS:
+ return Sets.newHashSet(TableFormat.MIXED_ICEBERG);
+ case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
+ case CatalogMetaProperties.CATALOG_TYPE_REST:
+ case CatalogMetaProperties.CATALOG_TYPE_HADOOP:
+ case CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM:
+ case CatalogMetaProperties.CATALOG_TYPE_GLUE:
+ return Sets.newHashSet(TableFormat.ICEBERG);
+ case CatalogMetaProperties.CATALOG_TYPE_HIVE:
+ return Sets.newHashSet(TableFormat.MIXED_HIVE);
+ default:
+ throw new IllegalArgumentException("Unsupported catalog type:" +
metastoreType);
+ }
}
/** Merge catalog properties in client side into catalog meta. */
@@ -111,10 +123,13 @@ public class CatalogUtil {
public static TableMetaStore buildMetaStore(CatalogMeta catalogMeta) {
// load storage configs
TableMetaStore.Builder builder = TableMetaStore.builder();
+ boolean isLocalStorage = false;
if (catalogMeta.getStorageConfigs() != null) {
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
- if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
- CatalogUtil.getCompatibleStorageType(storageConfigs))) {
+ String storageType =
CatalogUtil.getCompatibleStorageType(storageConfigs);
+ isLocalStorage =
+
CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_LOCAL.equalsIgnoreCase(storageType);
+ if
(CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(storageType))
{
String coreSite =
storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE);
String hdfsSite =
storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE);
String hiveSite =
storageConfigs.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HIVE_SITE);
@@ -122,72 +137,77 @@ public class CatalogUtil {
.withBase64CoreSite(coreSite)
.withBase64MetaStoreSite(hiveSite)
.withBase64HdfsSite(hdfsSite);
+ } else if (isLocalStorage) {
+ builder.withConfiguration(new Configuration());
}
}
- boolean loadAuthFromAMS =
- propertyAsBoolean(
- catalogMeta.getCatalogProperties(),
- CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
- CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
- // load auth configs from ams
- if (loadAuthFromAMS) {
- if (catalogMeta.getAuthConfigs() != null) {
- Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
- String authType =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
- LOG.info("TableMetaStore use auth config in catalog meta, authType is
{}", authType);
+ if (!isLocalStorage) {
+ boolean loadAuthFromAMS =
+ propertyAsBoolean(
+ catalogMeta.getCatalogProperties(),
+ CatalogMetaProperties.LOAD_AUTH_FROM_AMS,
+ CatalogMetaProperties.LOAD_AUTH_FROM_AMS_DEFAULT);
+ // load auth configs from ams
+ if (loadAuthFromAMS) {
+ if (catalogMeta.getAuthConfigs() != null) {
+ Map<String, String> authConfigs = catalogMeta.getAuthConfigs();
+ String authType =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
+ LOG.info("TableMetaStore use auth config in catalog meta, authType
is {}", authType);
+ if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType))
{
+ String hadoopUsername =
+
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
+ builder.withSimpleAuth(hadoopUsername);
+ } else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
+ authType)) {
+ String krb5 =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
+ String keytab =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
+ String principal =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
+ builder.withBase64KrbAuth(keytab, krb5, principal);
+ } else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(
+ authType)) {
+ String accessKey =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
+ String secretKey =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
+ builder.withAkSkAuth(accessKey, secretKey);
+ }
+ }
+ }
+
+ // cover auth configs from ams with auth configs in properties
+ String authType =
+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
+ if (StringUtils.isNotEmpty(authType)) {
+ LOG.info("TableMetaStore use auth config in properties, authType is
{}", authType);
if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType))
{
String hadoopUsername =
-
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
+ catalogMeta
+ .getCatalogProperties()
+ .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
builder.withSimpleAuth(hadoopUsername);
} else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
authType)) {
- String krb5 =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
- String keytab =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
- String principal =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
+ String krb5 =
+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
+ String keytab =
+
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
+ String principal =
+ catalogMeta
+ .getCatalogProperties()
+ .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
builder.withBase64KrbAuth(keytab, krb5, principal);
} else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType))
{
- String accessKey =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
- String secretKey =
authConfigs.get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
+ String accessKey =
+ catalogMeta
+ .getCatalogProperties()
+ .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
+ String secretKey =
+ catalogMeta
+ .getCatalogProperties()
+ .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
builder.withAkSkAuth(accessKey, secretKey);
}
}
}
-
- // cover auth configs from ams with auth configs in properties
- String authType =
-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
- if (StringUtils.isNotEmpty(authType)) {
- LOG.info("TableMetaStore use auth config in properties, authType is {}",
authType);
- if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE.equalsIgnoreCase(authType))
{
- String hadoopUsername =
- catalogMeta
- .getCatalogProperties()
- .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME);
- builder.withSimpleAuth(hadoopUsername);
- } else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS.equalsIgnoreCase(
- authType)) {
- String krb5 =
-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5);
- String keytab =
-
catalogMeta.getCatalogProperties().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB);
- String principal =
- catalogMeta
- .getCatalogProperties()
- .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL);
- builder.withBase64KrbAuth(keytab, krb5, principal);
- } else if
(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_AK_SK.equalsIgnoreCase(authType))
{
- String accessKey =
- catalogMeta
- .getCatalogProperties()
- .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_ACCESS_KEY);
- String secretKey =
- catalogMeta
- .getCatalogProperties()
- .get(CatalogMetaProperties.AUTH_CONFIGS_KEY_SECRET_KEY);
- builder.withAkSkAuth(accessKey, secretKey);
- }
- }
return builder.build();
}
@@ -238,6 +258,35 @@ public class CatalogUtil {
}
}
+ public static String normalizeMetastoreType(String type) {
+ if (type == null) {
+ return null;
+ }
+ if (CatalogMetaProperties.CATALOG_TYPE_HADOOP.equalsIgnoreCase(type)
+ ||
CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM.equalsIgnoreCase(type)) {
+ return CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM;
+ }
+ return type;
+ }
+
+ /**
+ * Normalize catalog type for client-side and factory usage.
+ *
+ * <p>Iceberg only recognizes short catalog types like "hadoop" or "hive" in
its {@code
+ * CatalogUtil}, and does not recognize "filesystem" yet. To keep AMS using
"filesystem" in
+ * metadata while still working with Iceberg, we normalize "filesystem" back
to "hadoop" before
+ * passing it down to Iceberg.
+ */
+ public static String normalizeCatalogType(String type) {
+ if (type == null) {
+ return null;
+ }
+ if (CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM.equalsIgnoreCase(type)) {
+ return CatalogMetaProperties.CATALOG_TYPE_HADOOP;
+ }
+ return type;
+ }
+
private static boolean propertyAsBoolean(
Map<String, String> properties, String property, boolean defaultValue) {
String value = properties.get(property);
diff --git
a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiCatalogFactory.java
b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiCatalogFactory.java
index 86537594a..d50cfcdcd 100644
---
a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiCatalogFactory.java
+++
b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiCatalogFactory.java
@@ -34,7 +34,8 @@ public class HudiCatalogFactory implements
FormatCatalogFactory {
String metastoreType,
Map<String, String> properties,
TableMetaStore metaStore) {
- if
(CatalogMetaProperties.CATALOG_TYPE_HADOOP.equalsIgnoreCase(metastoreType)) {
+ if
(CatalogMetaProperties.CATALOG_TYPE_HADOOP.equalsIgnoreCase(metastoreType)
+ ||
CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM.equalsIgnoreCase(metastoreType)) {
return new HudiHadoopCatalog(catalogName, properties, metaStore);
} else if
(CatalogMetaProperties.CATALOG_TYPE_HIVE.equalsIgnoreCase(metastoreType)) {
return new HudiHiveCatalog(catalogName, properties, metaStore);
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/mixed/MixedIcebergCatalogFactory.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/mixed/MixedIcebergCatalogFactory.java
index 90fea3e7e..4135a52ce 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/mixed/MixedIcebergCatalogFactory.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/mixed/MixedIcebergCatalogFactory.java
@@ -28,6 +28,7 @@ import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.properties.CatalogMetaProperties;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
+import org.apache.amoro.utils.CatalogUtil;
import java.util.Map;
@@ -52,7 +53,7 @@ public class MixedIcebergCatalogFactory implements
FormatCatalogFactory {
public Map<String, String> convertCatalogProperties(
String catalogName, String metastoreType, Map<String, String>
unifiedCatalogProperties) {
Map<String, String> properties = Maps.newHashMap(unifiedCatalogProperties);
- properties.put(ICEBERG_CATALOG_TYPE, metastoreType);
+ properties.put(ICEBERG_CATALOG_TYPE,
CatalogUtil.normalizeCatalogType(metastoreType));
properties.put(CatalogMetaProperties.TABLE_FORMATS, format().name());
return properties;
}
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
index 277edfc3a..339608d26 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
@@ -82,6 +82,7 @@ public class CatalogLoader {
String catalogImpl;
switch (metastoreType) {
case CatalogMetaProperties.CATALOG_TYPE_HADOOP:
+ case CatalogMetaProperties.CATALOG_TYPE_FILESYSTEM:
case CatalogMetaProperties.CATALOG_TYPE_GLUE:
case CatalogMetaProperties.CATALOG_TYPE_REST:
case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
@@ -168,7 +169,7 @@ public class CatalogLoader {
String catalogImpl = catalogImpl(metastoreType, properties);
MixedFormatCatalog catalog = buildCatalog(catalogImpl);
if (!properties.containsKey(ICEBERG_CATALOG_TYPE)) {
- properties.put(ICEBERG_CATALOG_TYPE, metastoreType);
+ properties.put(ICEBERG_CATALOG_TYPE,
CatalogUtil.normalizeCatalogType(metastoreType));
}
catalog.initialize(catalogName, properties, metaStore);
return catalog;
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
index c612b7bc2..71b060349 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
@@ -80,7 +80,8 @@ public class MixedFormatCatalogUtil {
String catalogName, String metastoreType, Map<String, String>
properties) {
Map<String, String> icebergCatalogProperties = Maps.newHashMap(properties);
icebergCatalogProperties.put(
- org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE, metastoreType);
+ org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE,
+ CatalogUtil.normalizeCatalogType(metastoreType));
if (CatalogMetaProperties.CATALOG_TYPE_GLUE.equals(metastoreType)) {
icebergCatalogProperties.put(CatalogProperties.CATALOG_IMPL,
GlueCatalog.class.getName());
}
diff --git a/amoro-web/src/services/setting.services.ts
b/amoro-web/src/services/setting.services.ts
index e7ce6652d..6098e1303 100644
--- a/amoro-web/src/services/setting.services.ts
+++ b/amoro-web/src/services/setting.services.ts
@@ -22,6 +22,12 @@ import request from '@/utils/request'
export function getCatalogsTypes() {
return request.get('api/ams/v1/catalogs/metastore/types')
}
+export function getMetastoreTableFormats(type: string) {
+ return request.get(`api/ams/v1/catalogs/metastore/${type}/table-formats`)
+}
+export function getMetastoreStorageTypes(type: string) {
+ return request.get(`api/ams/v1/catalogs/metastore/${type}/storage-types`)
+}
export function getCatalogsSetting(catalogName: string) {
return request.get(`api/ams/v1/catalogs/${catalogName}`)
}
diff --git a/amoro-web/src/views/catalogs/Detail.vue
b/amoro-web/src/views/catalogs/Detail.vue
index 37073257e..12a523658 100644
--- a/amoro-web/src/views/catalogs/Detail.vue
+++ b/amoro-web/src/views/catalogs/Detail.vue
@@ -29,7 +29,7 @@ import {
message,
} from 'ant-design-vue'
import Properties from './Properties.vue'
-import { checkCatalogStatus, delCatalog, getCatalogsSetting, getCatalogsTypes,
saveCatalogsSetting } from '@/services/setting.services'
+import { checkCatalogStatus, delCatalog, getCatalogsSetting, getCatalogsTypes,
getMetastoreStorageTypes, getMetastoreTableFormats, saveCatalogsSetting } from
'@/services/setting.services'
import type { ICatalogItem, IIOptimizeGroupItem, ILableAndValue, IMap } from
'@/types/common.type'
import { usePlaceholder } from '@/hooks/usePlaceholder'
import { getResourceGroupsListAPI } from '@/services/optimize.service'
@@ -65,13 +65,10 @@ const emit = defineEmits<{
(e: 'updateCatalogs'): void
}>()
-const typeShowMap = { 'Internal Catalog': 'Internal Catalog', 'External
Catalog': 'External Catalog' }
-
const formState: FormState = reactive({
catalog: {
name: '',
type: 'ams',
- typeshow: typeShowMap['Internal Catalog'],
optimizerGroup: undefined,
},
tableFormatList: [],
@@ -129,14 +126,6 @@ const tableFormatText = {
[tableFormatMap.PAIMON]: 'Paimon',
[tableFormatMap.HUDI]: 'Hudi',
}
-const storeSupportFormat: { [prop: string]: string[] } = {
- ams: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
- hive: [tableFormatMap.MIXED_HIVE, tableFormatMap.MIXED_ICEBERG,
tableFormatMap.ICEBERG, tableFormatMap.PAIMON, tableFormatMap.HUDI],
- hadoop: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG,
tableFormatMap.PAIMON],
- glue: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
- rest: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
- custom: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
-}
const storageConfigFileNameMap = {
'hadoop.core.site': 'core-site.xml',
@@ -153,10 +142,6 @@ const newCatalogConfig = {
'auth.kerberos.krb5': '',
},
}
-const typwShowOptions = ref([
- { label: typeShowMap['Internal Catalog'], value: typeShowMap['Internal
Catalog'] },
- { label: typeShowMap['External Catalog'], value: typeShowMap['External
Catalog'] },
-])
const hadoopConfigTypeOps = reactive<ILableAndValue[]>([{
label: 'SIMPLE',
@@ -195,6 +180,7 @@ const authConfigMap = {
const defaultPropertiesMap = {
ams: ['warehouse'],
hadoop: ['warehouse'],
+ filesystem: ['warehouse'],
custom: ['catalog-impl'],
glue: ['warehouse', 'lock-impl', 'lock.table'],
rest: ['uri'],
@@ -208,6 +194,8 @@ watch(() => route.query, (value) => {
deep: true,
})
const catalogTypeOps = reactive<ILableAndValue[]>([])
+const storageTypeOptions = ref<ILableAndValue[]>([])
+const tableFormatOptions = ref<string[]>([])
const optimizerGroupList = ref<ILableAndValue[]>([])
function initData() {
getConfigInfo()
@@ -237,19 +225,40 @@ async function getOptimizerGroupList() {
}
async function getCatalogTypeOps() {
const res = await getCatalogsTypes();
- (res || []).forEach((ele: any) => {
- if (ele.value !== 'ams') {
- catalogTypeOps.push({
- label: ele.display,
- value: ele.value,
- })
- }
+ catalogTypeOps.length = 0
+ ;(res || []).forEach((ele: any) => {
+ catalogTypeOps.push({
+ label: ele.display,
+ value: ele.value,
+ })
})
getMetastoreType()
}
function getMetastoreType() {
metastoreType.value = (catalogTypeOps.find(ele => ele.value ===
formState.catalog.type) || {}).label || ''
}
+async function loadMetastoreCapabilities(loadTableFormats: boolean) {
+ const type = formState.catalog.type as string
+ if (!type) {
+ return
+ }
+ const formats = await getMetastoreTableFormats(type)
+ const options = (formats || []) as string[]
+ tableFormatOptions.value = options
+ if (loadTableFormats) {
+ formState.tableFormatList = [...options]
+ }
+ const storageTypes = await getMetastoreStorageTypes(type)
+ storageTypeOptions.value = (storageTypes || []).map((val: string) => ({
+ label: val,
+ value: val,
+ }))
+ const availableTypes = storageTypeOptions.value.map(option => option.value)
+ const currentStorageType = formState.storageConfig['storage.type']
+ if (!currentStorageType || !availableTypes.includes(currentStorageType)) {
+ formState.storageConfig['storage.type'] = availableTypes.length > 0 ?
availableTypes[0] : ''
+ }
+}
async function getConfigInfo() {
try {
loading.value = true
@@ -261,7 +270,7 @@ async function getConfigInfo() {
formState.catalog.name = ''
formState.catalog.type = (type || 'ams') as string
formState.catalog.optimizerGroup = undefined
- formState.tableFormatList = [tableFormatMap.MIXED_ICEBERG]
+ formState.tableFormatList = []
formState.authConfig = { ...newCatalogConfig.authConfig }
formState.storageConfig = { ...newCatalogConfig.storageConfig }
const keys = defaultPropertiesMap[formState.catalog.type as keyof typeof
defaultPropertiesMap] || []
@@ -272,6 +281,7 @@ async function getConfigInfo() {
formState.tableProperties = {}
formState.storageConfigArray.length = 0
formState.authConfigArray.length = 0
+ await changeMetastore()
}
else {
const res = await getCatalogsSetting(catalogname as string)
@@ -290,8 +300,8 @@ async function getConfigInfo() {
formState.storageConfigArray.length = 0
formState.authConfigArray.length = 0
getMetastoreType()
+ await loadMetastoreCapabilities(false)
}
- formState.catalog.typeshow = formState.catalog.type === 'ams' ?
typeShowMap['Internal Catalog'] : typeShowMap['External Catalog']
const { storageConfig, authConfig } = formState
Object.keys(storageConfig).forEach((key) => {
@@ -338,20 +348,7 @@ async function getConfigInfo() {
}
}
-function changeTypeShow(val: string) {
- if (val === typeShowMap['Internal Catalog']) {
- formState.catalog.type = 'ams'
- }
- else {
- formState.catalog.type = catalogTypeOps[0].value
- }
- changeMetastore()
-}
-const formatOptions = computed(() => {
- const type = formState.catalog.type
- return storeSupportFormat[type as keyof typeof storeSupportFormat] || []
-})
async function changeProperties() {
const properties = await
propertiesRef.value.getPropertiesWithoputValidation()
@@ -377,53 +374,6 @@ async function changeProperties() {
formState.properties = properties
}
-const storageConfigTypeS3Oss = reactive<ILableAndValue[]>([{
- label: 'S3',
- value: 'S3',
-}, {
- label: 'OSS',
- value: 'OSS',
-}])
-
-const storageConfigTypeOSS = reactive<ILableAndValue[]>([{
- label: 'OSS',
- value: 'OSS',
-}])
-
-const storageConfigTypeHadoop = reactive<ILableAndValue[]>([{
- label: 'Hadoop',
- value: 'Hadoop',
-}])
-
-const storageConfigTypeHadoopS3Oss = reactive<ILableAndValue[]>([{
- label: 'Hadoop',
- value: 'Hadoop',
-}, {
- label: 'S3',
- value: 'S3',
-}, {
- label: 'OSS',
- value: 'OSS',
-}])
-
-const storageConfigTypeOps = computed(() => {
- const type = formState.catalog.type
- if (type === 'ams' || type === 'custom' || type === 'rest') {
- return storageConfigTypeHadoopS3Oss
- }
- else if (type === 'glue') {
- return storageConfigTypeS3Oss
- }
- else if (type === 'hive') {
- return storageConfigTypeHadoop
- }
- else if (type === 'hadoop') {
- return storageConfigTypeHadoopS3Oss
- }
- else {
- return null
- }
-})
const authTypeOptions = computed(() => {
const type = formState.storageConfig['storage.type']
@@ -440,10 +390,28 @@ const authTypeOptions = computed(() => {
return null
})
+const isAuthDisabled = computed(() => formState.storageConfig['storage.type']
=== 'Local')
+
+watch(
+ () => formState.storageConfig['storage.type'],
+ (storageType) => {
+ if (storageType === 'Local') {
+ if (isNewCatalog.value) {
+ formState.tableFormatList = [tableFormatMap.ICEBERG]
+ }
+ if (!formState.authConfig['auth.type']) {
+ formState.authConfig['auth.type'] = 'SIMPLE'
+ }
+ const simpleUsernameKey = 'auth.simple.hadoop_username'
+ if (!formState.authConfig[simpleUsernameKey]) {
+ formState.authConfig[simpleUsernameKey] = 'local'
+ }
+ }
+ }
+)
+
async function changeMetastore() {
- formState.tableFormatList = [formatOptions.value[0]]
- if (!isNewCatalog.value)
- return
+ await loadMetastoreCapabilities(isNewCatalog.value)
const index = formState.storageConfigArray.findIndex(item => item.key ===
'hive.site')
if (isHiveMetastore.value) {
@@ -472,9 +440,6 @@ async function changeMetastore() {
await changeProperties()
}
-async function changeTableFormat() {
- await changeProperties()
-}
function handleEdit() {
emit('updateEdit', true)
@@ -531,7 +496,7 @@ function handleSave() {
return
}
loading.value = true
- const { typeshow, ...catalogParams } = catalog
+ const catalogParams = catalog
getFileIdParams()
await saveCatalogsSetting({
isCreate: isNewCatalog.value,
@@ -635,37 +600,35 @@ onMounted(() => {
<a-input v-if="isEdit && isNewCatalog"
v-model:value="formState.catalog.name" />
<span v-else class="config-value">{{ formState.catalog.name
}}</span>
</a-form-item>
- <a-form-item :label="$t('type')" :name="['catalog', 'typeshow']">
- <a-select
- v-if="isEdit && isNewCatalog"
v-model:value="formState.catalog.typeshow"
- :options="typwShowOptions" :placeholder="placeholder.selectPh"
@change="changeTypeShow"
- />
- <span v-else>{{ formState.catalog.typeshow }}</span>
- </a-form-item>
<a-form-item
- v-if="formState.catalog.typeshow === typeShowMap['External
Catalog']" :label="$t('metastore')"
+ :label="$t('metastore')"
:name="['catalog', 'type']" :rules="[{ required: isEdit &&
isNewCatalog }]"
>
- <a-select
- v-if="isEdit && isNewCatalog"
v-model:value="formState.catalog.type" :options="catalogTypeOps"
- :placeholder="placeholder.selectPh" @change="changeMetastore"
- />
- <span v-else>{{ metastoreType }}</span>
- </a-form-item>
- <a-form-item
+ <a-select
+ v-if="isEdit && isNewCatalog"
v-model:value="formState.catalog.type" :options="catalogTypeOps"
+ :placeholder="placeholder.selectPh" @change="changeMetastore"
+ />
+ <span v-else>{{ metastoreType }}</span>
+ </a-form-item>
+ <a-form-item
:label="$t('tableFormat')" :name="['tableFormatList']"
- :rules="[{ required: isEdit && isNewCatalog }]"
- >
+ >
<a-checkbox-group
- v-model:value="formState.tableFormatList" :disabled="!isEdit ||
!isNewCatalog"
- @change="changeTableFormat"
+ v-if="isEdit"
+ v-model:value="formState.tableFormatList"
>
- <a-checkbox v-for="item in formatOptions" :key="item"
:value="item">
- {{ tableFormatText[item]
- }}
+ <a-checkbox
+ v-for="item in tableFormatOptions"
+ :key="item"
+ :value="item"
+ >
+ {{ tableFormatText[item] || item }}
</a-checkbox>
</a-checkbox-group>
- </a-form-item>
+ <span v-else class="config-value">
+ {{ formState.tableFormatList.map((item) => tableFormatText[item]
|| item).join(', ') }}
+ </span>
+ </a-form-item>
<a-form-item
:label="$t('optimizerGroup')" :name="['catalog', 'optimizerGroup']"
:rules="[{ required: isEdit }]"
@@ -684,7 +647,7 @@ onMounted(() => {
<a-form-item label="Type" :name="['storageConfig', 'storage.type']"
:rules="[{ required: isEdit }]">
<a-select
v-if="isEdit"
v-model:value="formState.storageConfig['storage.type']"
- :placeholder="placeholder.selectPh"
:options="storageConfigTypeOps"
+ :placeholder="placeholder.selectPh" :options="storageTypeOptions"
/>
<span v-else class="config-value">{{
formState.storageConfig['storage.type'] }}</span>
</a-form-item>
@@ -737,25 +700,25 @@ onMounted(() => {
{{ $t('authenticationConfig') }}
</p>
</a-form-item>
- <a-form-item label="Type" :name="['authConfig', 'auth.type']"
:rules="[{ required: isEdit }]">
+ <a-form-item label="Type" :name="['authConfig', 'auth.type']"
:rules="[{ required: isEdit && !isAuthDisabled }]">
<a-select
v-if="isEdit" v-model:value="formState.authConfig['auth.type']"
- :placeholder="placeholder.selectPh" :options="authTypeOptions"
+ :placeholder="placeholder.selectPh" :options="authTypeOptions"
:disabled="isAuthDisabled"
/>
<span v-else class="config-value">{{
formState.authConfig['auth.type'] }}</span>
</a-form-item>
<a-form-item
v-if="formState.authConfig['auth.type'] === 'SIMPLE'"
label="Hadoop Username"
- :name="['authConfig', 'auth.simple.hadoop_username']" :rules="[{
required: isEdit }]"
+ :name="['authConfig', 'auth.simple.hadoop_username']" :rules="[{
required: isEdit && !isAuthDisabled }]"
>
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.simple.hadoop_username']" />
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.simple.hadoop_username']"
:disabled="isAuthDisabled" />
<span v-else class="config-value">{{
formState.authConfig['auth.simple.hadoop_username'] }}</span>
</a-form-item>
<a-form-item
v-if="formState.authConfig['auth.type'] === 'KERBEROS'"
label="Kerberos Principal"
- :name="['authConfig', 'auth.kerberos.principal']" :rules="[{
required: isEdit }]"
+ :name="['authConfig', 'auth.kerberos.principal']" :rules="[{
required: isEdit && !isAuthDisabled }]"
>
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.kerberos.principal']" />
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.kerberos.principal']"
:disabled="isAuthDisabled" />
<span v-else class="config-value">{{
formState.authConfig['auth.kerberos.principal'] }}</span>
</a-form-item>
<div v-if="formState.authConfig['auth.type'] === 'KERBEROS'">
@@ -784,16 +747,16 @@ onMounted(() => {
</div>
<a-form-item
v-if="formState.authConfig['auth.type'] === 'AK/SK'" label="Access
Key"
- :name="['authConfig', 'auth.ak_sk.access_key']" :rules="[{
required: isEdit }]"
+ :name="['authConfig', 'auth.ak_sk.access_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
>
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.access_key']" />
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.access_key']"
:disabled="isAuthDisabled" />
<span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.access_key'] }}</span>
</a-form-item>
<a-form-item
v-if="formState.authConfig['auth.type'] === 'AK/SK'" label="Secret
Key"
- :name="['authConfig', 'auth.ak_sk.secret_key']" :rules="[{
required: isEdit }]"
+ :name="['authConfig', 'auth.ak_sk.secret_key']" :rules="[{
required: isEdit && !isAuthDisabled }]"
>
- <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.secret_key']" />
+ <a-input v-if="isEdit"
v-model:value="formState.authConfig['auth.ak_sk.secret_key']"
:disabled="isAuthDisabled" />
<span v-else class="config-value">{{
formState.authConfig['auth.ak_sk.secret_key'] }}</span>
</a-form-item>
<a-form-item>
diff --git a/amoro-web/src/views/tables/components/TableExplorer.vue
b/amoro-web/src/views/tables/components/TableExplorer.vue
index 0e36851cc..fb6d5b68e 100755
--- a/amoro-web/src/views/tables/components/TableExplorer.vue
+++ b/amoro-web/src/views/tables/components/TableExplorer.vue
@@ -182,7 +182,7 @@ async function loadChildren(node: any) {
const tables = (res || []) as TableItem[]
state.tablesByCatalogDb[cacheKey] = tables
if (!tables.length) {
- data.isLeaf = true
+ data.isLeaf = false
updateTreeNodeChildren(data.key as string, [])
return
}
diff --git a/amoro-web/src/views/tables/index.vue
b/amoro-web/src/views/tables/index.vue
index e1f1cbe9a..8fdd485f1 100644
--- a/amoro-web/src/views/tables/index.vue
+++ b/amoro-web/src/views/tables/index.vue
@@ -164,6 +164,11 @@ export default defineComponent({
const { catalog: oldCatalog, db: oldDb, table: oldTable } = oldVal
if (`${catalog}${db}${table}` !== `${oldCatalog}${oldDb}${oldTable}`) {
state.activeKey = 'Details'
+ nextTick(() => {
+ if (detailRef.value) {
+ detailRef.value.getTableDetails()
+ }
+ })
return
}
state.activeKey = value.tab as string
