This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 23f7f4caa [#4580] feat(catalog): unify cloud storage configurations
(#4897)
23f7f4caa is described below
commit 23f7f4caa666e48b7b58f49fc04ee2a37559f5ee
Author: FANNG <[email protected]>
AuthorDate: Fri Sep 13 10:18:02 2024 +0800
[#4580] feat(catalog): unify cloud storage configurations (#4897)
### What changes were proposed in this pull request?
unify the storage configurations
### Why are the changes needed?
Fix: #4580
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
existing tests
---
.../lakehouse/iceberg/IcebergConstants.java | 7 -----
.../lakehouse/iceberg/IcebergPropertiesUtils.java | 18 +++++------
.../apache/gravitino/storage/OSSProperties.java | 32 ++++++++++++++++++++
.../org/apache/gravitino/storage/S3Properties.java | 35 ++++++++++++++++++++++
.../iceberg/IcebergCatalogPropertiesMetadata.java | 18 ++++++-----
.../gravitino/iceberg/common/IcebergConfig.java | 18 ++++++-----
6 files changed, 96 insertions(+), 32 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
index cb5a23369..21462b9ca 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -37,20 +37,13 @@ public class IcebergConstants {
// IO properties
public static final String IO_IMPL = "io-impl";
- public static final String GRAVITINO_S3_ENDPOINT = "s3-endpoint";
public static final String ICEBERG_S3_ENDPOINT = "s3.endpoint";
- public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id";
public static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id";
- public static final String GRAVITINO_S3_SECRET_ACCESS_KEY =
"s3-secret-access-key";
public static final String ICEBERG_S3_SECRET_ACCESS_KEY =
"s3.secret-access-key";
- public static final String GRAVITINO_S3_REGION = "s3-region";
public static final String AWS_S3_REGION = "client.region";
- public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint";
public static final String ICEBERG_OSS_ENDPOINT = "oss.endpoint";
- public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
public static final String ICEBERG_OSS_ACCESS_KEY_ID =
"client.access-key-id";
- public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET =
"oss-access-key-secret";
public static final String ICEBERG_OSS_ACCESS_KEY_SECRET =
"client.access-key-secret";
// Iceberg Table properties constants
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
index b016dd6e7..2a92f545e 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
@@ -21,6 +21,8 @@ package org.apache.gravitino.catalog.lakehouse.iceberg;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.gravitino.storage.OSSProperties;
+import org.apache.gravitino.storage.S3Properties;
public class IcebergPropertiesUtils {
@@ -40,18 +42,16 @@ public class IcebergPropertiesUtils {
map.put(IcebergConstants.CATALOG_BACKEND_NAME,
IcebergConstants.CATALOG_BACKEND_NAME);
map.put(IcebergConstants.IO_IMPL, IcebergConstants.IO_IMPL);
// S3
- map.put(IcebergConstants.GRAVITINO_S3_ENDPOINT,
IcebergConstants.ICEBERG_S3_ENDPOINT);
- map.put(IcebergConstants.GRAVITINO_S3_REGION,
IcebergConstants.AWS_S3_REGION);
- map.put(IcebergConstants.GRAVITINO_S3_ACCESS_KEY_ID,
IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID);
+ map.put(S3Properties.GRAVITINO_S3_ENDPOINT,
IcebergConstants.ICEBERG_S3_ENDPOINT);
+ map.put(S3Properties.GRAVITINO_S3_REGION, IcebergConstants.AWS_S3_REGION);
+ map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID,
IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID);
map.put(
- IcebergConstants.GRAVITINO_S3_SECRET_ACCESS_KEY,
- IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY);
+ S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY,
IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY);
// OSS
- map.put(IcebergConstants.GRAVITINO_OSS_ENDPOINT,
IcebergConstants.ICEBERG_OSS_ENDPOINT);
+ map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT,
IcebergConstants.ICEBERG_OSS_ENDPOINT);
+ map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID);
map.put(
- IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_ID,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID);
- map.put(
- IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_SECRET,
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_SECRET);
GRAVITINO_CONFIG_TO_ICEBERG = Collections.unmodifiableMap(map);
}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
new file mode 100644
index 000000000..8ceae5b82
--- /dev/null
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage;
+
+// Defines the unified OSS properties for different catalogs and connectors.
+public class OSSProperties {
+
+ // The endpoint of Aliyun OSS service.
+ public static final String GRAVITINO_OSS_ENDPOINT = "oss-endpoint";
+ // The static access key ID used to access OSS data.
+ public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
+ // The static access key secret used to access OSS data.
+ public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET =
"oss-access-key-secret";
+
+ private OSSProperties() {}
+}
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java
new file mode 100644
index 000000000..9775bebc8
--- /dev/null
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.storage;
+
+// Defines the unified S3 properties for different catalogs and connectors.
+public class S3Properties {
+ // An alternative endpoint of the S3 service, This could be used to for
S3FileIO with any
+ // s3-compatible object storage service that has a different endpoint, or
access a private S3
+ // endpoint in a virtual private cloud
+ public static final String GRAVITINO_S3_ENDPOINT = "s3-endpoint";
+ // The static access key ID used to access S3 data.
+ public static final String GRAVITINO_S3_ACCESS_KEY_ID = "s3-access-key-id";
+ // The static secret access key used to access S3 data.
+ public static final String GRAVITINO_S3_SECRET_ACCESS_KEY =
"s3-secret-access-key";
+ // The region of the S3 service.
+ public static final String GRAVITINO_S3_REGION = "s3-region";
+
+ private S3Properties() {}
+}
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
index 0165d09d3..6d61a6220 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
@@ -33,6 +33,8 @@ import org.apache.gravitino.connector.PropertyEntry;
import org.apache.gravitino.iceberg.common.IcebergCatalogBackend;
import org.apache.gravitino.iceberg.common.authentication.AuthenticationConfig;
import
org.apache.gravitino.iceberg.common.authentication.kerberos.KerberosConfig;
+import org.apache.gravitino.storage.OSSProperties;
+import org.apache.gravitino.storage.S3Properties;
public class IcebergCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetadata {
public static final String CATALOG_BACKEND =
IcebergConstants.CATALOG_BACKEND;
@@ -84,26 +86,26 @@ public class IcebergCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetad
null /* defaultValue */,
false /* hidden */),
stringOptionalPropertyEntry(
- IcebergConstants.GRAVITINO_S3_ACCESS_KEY_ID,
- "s3 access-key-id",
+ S3Properties.GRAVITINO_S3_ACCESS_KEY_ID,
+ "s3 access key ID",
false /* immutable */,
null /* defaultValue */,
true /* hidden */),
stringOptionalPropertyEntry(
- IcebergConstants.GRAVITINO_S3_SECRET_ACCESS_KEY,
- "s3 secret-access-key",
+ S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY,
+ "s3 secret access key",
false /* immutable */,
null /* defaultValue */,
true /* hidden */),
stringOptionalPropertyEntry(
- IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_ID,
- "OSS access-key-id",
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID,
+ "OSS access key ID",
false /* immutable */,
null /* defaultValue */,
true /* hidden */),
stringOptionalPropertyEntry(
- IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_SECRET,
- "OSS access-key-secret",
+ OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
+ "OSS access key secret",
false /* immutable */,
null /* defaultValue */,
true /* hidden */));
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
index cc8921979..ed4306b8c 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -31,6 +31,8 @@ import org.apache.gravitino.config.ConfigConstants;
import org.apache.gravitino.config.ConfigEntry;
import org.apache.gravitino.server.web.JettyServerConfig;
import org.apache.gravitino.server.web.OverwriteDefaultConfig;
+import org.apache.gravitino.storage.OSSProperties;
+import org.apache.gravitino.storage.S3Properties;
public class IcebergConfig extends Config implements OverwriteDefaultConfig {
@@ -105,7 +107,7 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.create();
public static final ConfigEntry<String> S3_ENDPOINT =
- new ConfigBuilder(IcebergConstants.GRAVITINO_S3_ENDPOINT)
+ new ConfigBuilder(S3Properties.GRAVITINO_S3_ENDPOINT)
.doc(
"An alternative endpoint of the S3 service, This could be used
to for S3FileIO with "
+ "any s3-compatible object storage service that has a
different endpoint, or "
@@ -115,7 +117,7 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.create();
public static final ConfigEntry<String> S3_REGION =
- new ConfigBuilder(IcebergConstants.GRAVITINO_S3_REGION)
+ new ConfigBuilder(S3Properties.GRAVITINO_S3_REGION)
.doc("The region of the S3 service")
.version(ConfigConstants.VERSION_0_6_0)
.stringConf()
@@ -123,36 +125,36 @@ public class IcebergConfig extends Config implements
OverwriteDefaultConfig {
.create();
public static final ConfigEntry<String> S3_ACCESS_KEY_ID =
- new ConfigBuilder(IcebergConstants.GRAVITINO_S3_ACCESS_KEY_ID)
+ new ConfigBuilder(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID)
.doc("The static access key ID used to access S3 data")
.version(ConfigConstants.VERSION_0_6_0)
.stringConf()
.create();
public static final ConfigEntry<String> S3_SECRET_ACCESS_KEY =
- new ConfigBuilder(IcebergConstants.GRAVITINO_S3_SECRET_ACCESS_KEY)
+ new ConfigBuilder(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY)
.doc("The static secret access key used to access S3 data")
.version(ConfigConstants.VERSION_0_6_0)
.stringConf()
.create();
public static final ConfigEntry<String> OSS_ENDPOINT =
- new ConfigBuilder(IcebergConstants.GRAVITINO_OSS_ENDPOINT)
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ENDPOINT)
.doc("The endpoint of Aliyun OSS service")
.version(ConfigConstants.VERSION_0_7_0)
.stringConf()
.create();
public static final ConfigEntry<String> OSS_ACCESS_KEY_ID =
- new ConfigBuilder(IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_ID)
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID)
.doc("The static access key ID used to access OSS data")
.version(ConfigConstants.VERSION_0_7_0)
.stringConf()
.create();
public static final ConfigEntry<String> OSS_ACCESS_KEY_SECRET =
- new ConfigBuilder(IcebergConstants.GRAVITINO_OSS_ACCESS_KEY_SECRET)
- .doc("The static secret access key used to access OSS data")
+ new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET)
+ .doc("The static access key secret used to access OSS data")
.version(ConfigConstants.VERSION_0_7_0)
.stringConf()
.create();
