This is an automated email from the ASF dual-hosted git repository.
xuba pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 6e5ce3983 [AMORO-3799] Add REST metastore type (#3819)
6e5ce3983 is described below
commit 6e5ce398380f7191731991f64ac63b976c14762e
Author: zhangwl9 <[email protected]>
AuthorDate: Thu Oct 23 13:45:35 2025 +0800
[AMORO-3799] Add REST metastore type (#3819)
* Add rest metastore type
* Update
amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
* Update
amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
* fixup
---
.../amoro/server/catalog/CatalogBuilder.java | 32 ++++++++++++----------
.../apache/amoro/server/catalog/CatalogType.java | 1 +
.../dashboard/controller/CatalogController.java | 23 +++++++++++++++-
.../amoro/server/terminal/TerminalManager.java | 6 ++++
.../amoro/properties/CatalogMetaProperties.java | 1 +
.../java/org/apache/amoro/utils/CatalogUtil.java | 1 +
.../java/org/apache/amoro/mixed/CatalogLoader.java | 1 +
.../apache/amoro/utils/MixedFormatCatalogUtil.java | 3 ++
amoro-web/src/views/catalogs/Detail.vue | 4 ++-
9 files changed, 56 insertions(+), 16 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
index 2fb8b4e29..d405bc1c3 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
@@ -23,6 +23,7 @@ import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_CUS
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_GLUE;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HADOOP;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HIVE;
+import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_REST;
import org.apache.amoro.TableFormat;
import org.apache.amoro.api.CatalogMeta;
@@ -41,21 +42,23 @@ public class CatalogBuilder {
private static final Map<String, Set<TableFormat>> formatSupportedMatrix =
ImmutableMap.of(
CATALOG_TYPE_HADOOP,
- Sets.newHashSet(
- TableFormat.ICEBERG,
- TableFormat.MIXED_ICEBERG,
- TableFormat.PAIMON,
- TableFormat.HUDI),
- CATALOG_TYPE_GLUE, Sets.newHashSet(TableFormat.ICEBERG,
TableFormat.MIXED_ICEBERG),
- CATALOG_TYPE_CUSTOM, Sets.newHashSet(TableFormat.ICEBERG,
TableFormat.MIXED_ICEBERG),
+ Sets.newHashSet(
+ TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG,
TableFormat.PAIMON, TableFormat.HUDI),
+ CATALOG_TYPE_GLUE,
+ Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
+ CATALOG_TYPE_REST,
+ Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
+ CATALOG_TYPE_CUSTOM,
+ Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
CATALOG_TYPE_HIVE,
- Sets.newHashSet(
- TableFormat.ICEBERG,
- TableFormat.MIXED_ICEBERG,
- TableFormat.MIXED_HIVE,
- TableFormat.PAIMON,
- TableFormat.HUDI),
- CATALOG_TYPE_AMS, Sets.newHashSet(TableFormat.ICEBERG,
TableFormat.MIXED_ICEBERG));
+ Sets.newHashSet(
+ TableFormat.ICEBERG,
+ TableFormat.MIXED_ICEBERG,
+ TableFormat.MIXED_HIVE,
+ TableFormat.PAIMON,
+ TableFormat.HUDI),
+ CATALOG_TYPE_AMS,
+ Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG));
public static ServerCatalog buildServerCatalog(
CatalogMeta catalogMeta, Configurations serverConfiguration) {
@@ -75,6 +78,7 @@ public class CatalogBuilder {
switch (type) {
case CATALOG_TYPE_HADOOP:
case CATALOG_TYPE_GLUE:
+ case CATALOG_TYPE_REST:
case CATALOG_TYPE_CUSTOM:
case CATALOG_TYPE_HIVE:
return new ExternalCatalog(catalogMeta);
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogType.java
b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogType.java
index f44a1d498..8609f1db9 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogType.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogType.java
@@ -23,5 +23,6 @@ public enum CatalogType {
HIVE,
HADOOP,
GLUE,
+ REST,
CUSTOM
}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
index 7c3e044ab..cfd8f4bcd 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java
@@ -38,6 +38,7 @@ import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_CUS
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_GLUE;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HADOOP;
import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_HIVE;
+import static
org.apache.amoro.properties.CatalogMetaProperties.CATALOG_TYPE_REST;
import static org.apache.amoro.properties.CatalogMetaProperties.KEY_WAREHOUSE;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE;
import static
org.apache.amoro.properties.CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE;
@@ -80,6 +81,7 @@ import org.apache.iceberg.aliyun.AliyunProperties;
import org.apache.iceberg.aws.AwsClientProperties;
import org.apache.iceberg.aws.glue.GlueCatalog;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.iceberg.rest.RESTCatalog;
import java.util.ArrayList;
import java.util.Arrays;
@@ -108,6 +110,7 @@ public class CatalogController {
CATALOG_TYPE_GLUE,
Lists.newArrayList(CatalogProperties.WAREHOUSE_LOCATION));
CATALOG_REQUIRED_PROPERTIES.put(
CATALOG_TYPE_CUSTOM,
Lists.newArrayList(CatalogProperties.CATALOG_IMPL));
+ CATALOG_REQUIRED_PROPERTIES.put(CATALOG_TYPE_REST,
Lists.newArrayList(CatalogProperties.URI));
}
static {
@@ -162,6 +165,19 @@ public class CatalogController {
VALIDATE_CATALOGS.add(
CatalogDescriptor.of(
CATALOG_TYPE_CUSTOM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP,
MIXED_ICEBERG));
+
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST, STORAGE_CONFIGS_VALUE_TYPE_S3,
ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST, STORAGE_CONFIGS_VALUE_TYPE_S3,
MIXED_ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_HADOOP, MIXED_ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_OSS, ICEBERG));
+ VALIDATE_CATALOGS.add(
+ CatalogDescriptor.of(CATALOG_TYPE_REST,
STORAGE_CONFIGS_VALUE_TYPE_OSS, MIXED_ICEBERG));
}
private final PlatformFileManager platformFileInfoService;
@@ -220,6 +236,7 @@ public class CatalogController {
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HIVE, displayKey,
"Hive Metastore"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_HADOOP,
displayKey, "Filesystem"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_GLUE, displayKey,
"Glue"));
+ catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_REST, displayKey,
"REST"));
catalogTypes.add(ImmutableMap.of(valueKey, CATALOG_TYPE_CUSTOM,
displayKey, "Custom"));
ctx.json(OkResponse.of(catalogTypes));
}
@@ -414,11 +431,15 @@ public class CatalogController {
catalogMeta.setCatalogType(info.getType());
catalogMeta.setCatalogProperties(
PropertiesUtil.unionCatalogProperties(info.getTableProperties(),
info.getProperties()));
- // fill catalog impl when catalog type is glue
+ // fill catalog impl when catalog type is glue or rest
if (CatalogMetaProperties.CATALOG_TYPE_GLUE.equals(info.getType())) {
catalogMeta.putToCatalogProperties(
CatalogProperties.CATALOG_IMPL, GlueCatalog.class.getName());
+ } else if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(info.getType()))
{
+ catalogMeta.putToCatalogProperties(
+ CatalogProperties.CATALOG_IMPL, RESTCatalog.class.getName());
}
+
catalogMeta.putToCatalogProperties(
CatalogMetaProperties.TABLE_PROPERTIES_PREFIX +
TableProperties.SELF_OPTIMIZING_GROUP,
info.getOptimizerGroup());
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
b/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
index 1396bf5c4..2596d0428 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
@@ -275,6 +275,12 @@ public class TerminalManager {
}
} else if (catalogType.equalsIgnoreCase(CatalogType.CUSTOM.name())) {
return "iceberg";
+ } else if (catalogType.equalsIgnoreCase(CatalogType.REST.name())) {
+ if (tableFormatSet.contains(TableFormat.MIXED_ICEBERG)) {
+ return "mixed_iceberg";
+ } else if (tableFormatSet.contains(TableFormat.ICEBERG)) {
+ return "iceberg";
+ }
} else if (catalogType.equalsIgnoreCase(CatalogType.GLUE.name())) {
return "iceberg";
}
diff --git
a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
index 31fb1ffb3..875a7c0f3 100644
---
a/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
+++
b/amoro-common/src/main/java/org/apache/amoro/properties/CatalogMetaProperties.java
@@ -57,6 +57,7 @@ public class CatalogMetaProperties {
public static final String CATALOG_TYPE_HIVE = "hive";
public static final String CATALOG_TYPE_AMS = "ams";
public static final String CATALOG_TYPE_GLUE = "glue";
+ public static final String CATALOG_TYPE_REST = "rest";
public static final String CATALOG_TYPE_CUSTOM = "custom";
public static final String TABLE_FORMATS = "table-formats";
diff --git a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
index bd49ea931..862ca8516 100644
--- a/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
+++ b/amoro-common/src/main/java/org/apache/amoro/utils/CatalogUtil.java
@@ -62,6 +62,7 @@ public class CatalogUtil {
case CatalogMetaProperties.CATALOG_TYPE_AMS:
return Sets.newHashSet(TableFormat.MIXED_ICEBERG);
case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
+ case CatalogMetaProperties.CATALOG_TYPE_REST:
case CatalogMetaProperties.CATALOG_TYPE_HADOOP:
case CatalogMetaProperties.CATALOG_TYPE_GLUE:
return Sets.newHashSet(TableFormat.ICEBERG);
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
index 1afc5e31f..277edfc3a 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/mixed/CatalogLoader.java
@@ -83,6 +83,7 @@ public class CatalogLoader {
switch (metastoreType) {
case CatalogMetaProperties.CATALOG_TYPE_HADOOP:
case CatalogMetaProperties.CATALOG_TYPE_GLUE:
+ case CatalogMetaProperties.CATALOG_TYPE_REST:
case CatalogMetaProperties.CATALOG_TYPE_CUSTOM:
catalogImpl = MIXED_ICEBERG_CATALOG_IMP;
break;
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
index 8028ef568..c612b7bc2 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/MixedFormatCatalogUtil.java
@@ -90,6 +90,9 @@ public class MixedFormatCatalogUtil {
icebergCatalogProperties.put(CatalogProperties.CATALOG_IMPL,
RESTCatalog.class.getName());
}
}
+ if (CatalogMetaProperties.CATALOG_TYPE_REST.equals(metastoreType)) {
+ icebergCatalogProperties.put(CatalogProperties.CATALOG_IMPL,
RESTCatalog.class.getName());
+ }
if
(CatalogMetaProperties.CATALOG_TYPE_CUSTOM.equalsIgnoreCase(metastoreType)) {
Preconditions.checkArgument(
diff --git a/amoro-web/src/views/catalogs/Detail.vue
b/amoro-web/src/views/catalogs/Detail.vue
index 0a2caac0e..37073257e 100644
--- a/amoro-web/src/views/catalogs/Detail.vue
+++ b/amoro-web/src/views/catalogs/Detail.vue
@@ -134,6 +134,7 @@ const storeSupportFormat: { [prop: string]: string[] } = {
hive: [tableFormatMap.MIXED_HIVE, tableFormatMap.MIXED_ICEBERG,
tableFormatMap.ICEBERG, tableFormatMap.PAIMON, tableFormatMap.HUDI],
hadoop: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG,
tableFormatMap.PAIMON],
glue: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
+ rest: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
custom: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
}
@@ -196,6 +197,7 @@ const defaultPropertiesMap = {
hadoop: ['warehouse'],
custom: ['catalog-impl'],
glue: ['warehouse', 'lock-impl', 'lock.table'],
+ rest: ['uri'],
PAIMON: ['warehouse'],
}
@@ -406,7 +408,7 @@ const storageConfigTypeHadoopS3Oss =
reactive<ILableAndValue[]>([{
const storageConfigTypeOps = computed(() => {
const type = formState.catalog.type
- if (type === 'ams' || type === 'custom') {
+ if (type === 'ams' || type === 'custom' || type === 'rest') {
return storageConfigTypeHadoopS3Oss
}
else if (type === 'glue') {
