This is an automated email from the ASF dual-hosted git repository.
yuqi4733 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new b21d88487 [#4084] feat(bundled-catalog): remove IcebergPropertiesMeta
from bundled catalog (#4122)
b21d88487 is described below
commit b21d88487156c95a149b7bc142dcd3a0a3807099
Author: FANNG <[email protected]>
AuthorDate: Tue Jul 16 12:44:24 2024 +0800
[#4084] feat(bundled-catalog): remove IcebergPropertiesMeta from bundled
catalog (#4122)
### What changes were proposed in this pull request?
remove IcebergPropertiesMeta from bundled catalog, `Iceberg catalog`,
`trino-connector`, `spark-connector` use IcebergConstants defined in
bundled-catalog.
### Why are the changes needed?
Fix: #4084
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
existing tests
---
catalogs/bundled-catalog/build.gradle.kts | 2 -
.../gravitino/catalog/common/ClassProvider.java | 12 ------
.../lakehouse/iceberg/IcebergConstants.java | 49 ++++++++++++++++++++++
.../catalog/property/PropertyConverter.java | 30 +------------
.../catalog-lakehouse-iceberg/build.gradle.kts | 1 +
.../iceberg/IcebergCatalogPropertiesMetadata.java | 20 ++++-----
.../iceberg/IcebergSchemaPropertiesMetadata.java | 2 +-
.../iceberg/IcebergTablePropertiesMetadata.java | 20 ++++-----
.../iceberg/IcebergPropertiesConstants.java | 44 +++++++------------
spark-connector/v3.3/spark/build.gradle.kts | 1 +
spark-connector/v3.4/spark/build.gradle.kts | 1 +
spark-connector/v3.5/spark/build.gradle.kts | 1 +
.../catalog/hive/HiveCatalogPropertyConverter.java | 7 ----
.../catalog/hive/HiveSchemaPropertyConverter.java | 11 -----
.../catalog/hive/HiveTablePropertyConverter.java | 10 -----
.../iceberg/IcebergCatalogPropertyConverter.java | 6 ---
.../iceberg/IcebergSchemaPropertyConverter.java | 12 ------
.../iceberg/IcebergTablePropertyConverter.java | 14 +------
.../catalog/jdbc/JDBCCatalogPropertyConverter.java | 6 ---
.../jdbc/mysql/MySQLTablePropertyConverter.java | 11 -----
.../TestIcebergCatalogPropertyConverter.java | 14 -------
21 files changed, 93 insertions(+), 181 deletions(-)
diff --git a/catalogs/bundled-catalog/build.gradle.kts
b/catalogs/bundled-catalog/build.gradle.kts
index 66aadcf8a..a9cc1141b 100644
--- a/catalogs/bundled-catalog/build.gradle.kts
+++ b/catalogs/bundled-catalog/build.gradle.kts
@@ -29,7 +29,6 @@ dependencies {
implementation(project(":catalogs:catalog-jdbc-common"))
implementation(project(":catalogs:catalog-jdbc-mysql"))
implementation(project(":catalogs:catalog-jdbc-postgresql"))
- implementation(project(":catalogs:catalog-lakehouse-iceberg"))
implementation(project(":catalogs:catalog-lakehouse-paimon"))
implementation(project(":core"))
implementation(libs.slf4j.api)
@@ -94,7 +93,6 @@ tasks.jar {
tasks.compileJava {
dependsOn(":catalogs:catalog-jdbc-postgresql:runtimeJars")
- dependsOn(":catalogs:catalog-lakehouse-iceberg:runtimeJars")
dependsOn(":catalogs:catalog-lakehouse-paimon:runtimeJars")
dependsOn(":catalogs:catalog-jdbc-mysql:runtimeJars")
dependsOn(":catalogs:catalog-hive:runtimeJars")
diff --git
a/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/common/ClassProvider.java
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/common/ClassProvider.java
index 6b81ce3fa..cc09dce2d 100644
---
a/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/common/ClassProvider.java
+++
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/common/ClassProvider.java
@@ -25,9 +25,6 @@ import
org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta;
import org.apache.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
import org.apache.gravitino.catalog.hive.HiveTablePropertiesMetadata;
import org.apache.gravitino.catalog.jdbc.JdbcTablePropertiesMetadata;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
import org.apache.gravitino.catalog.mysql.MysqlTablePropertiesMetadata;
import org.apache.gravitino.connector.BasePropertiesMetadata;
import org.apache.gravitino.connector.PropertiesMetadata;
@@ -71,15 +68,6 @@ public class ClassProvider {
}
};
- private static final Set<Class<?>> ICEBERG_NEED_CLASS =
- new HashSet<Class<?>>() {
- {
- add(IcebergTablePropertiesMetadata.class);
- add(IcebergSchemaPropertiesMetadata.class);
- add(IcebergCatalogPropertiesMetadata.class);
- }
- };
-
private static final Set<Class<?>> MYSQL_NEED_CLASS =
new HashSet<Class<?>>() {
{
diff --git
a/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
new file mode 100644
index 000000000..abb3a39eb
--- /dev/null
+++
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.iceberg;
+
+public class IcebergConstants {
+ // Iceberg catalog properties constants
+ public static final String CATALOG_BACKEND = "catalog-backend";
+
+ public static final String GRAVITINO_JDBC_USER = "jdbc-user";
+ public static final String ICEBERG_JDBC_USER = "jdbc.user";
+
+ public static final String GRAVITINO_JDBC_PASSWORD = "jdbc-password";
+ public static final String ICEBERG_JDBC_PASSWORD = "jdbc.password";
+ public static final String ICEBERG_JDBC_INITIALIZE = "jdbc-initialize";
+
+ public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver";
+ public static final String WAREHOUSE = "warehouse";
+ public static final String URI = "uri";
+ public static final String CATALOG_BACKEND_NAME = "catalog-backend-name";
+
+ // Iceberg Table properties constants
+
+ public static final String COMMENT = "comment";
+ public static final String CREATOR = "creator";
+ public static final String LOCATION = "location";
+ public static final String CURRENT_SNAPSHOT_ID = "current-snapshot-id";
+ public static final String CHERRY_PICK_SNAPSHOT_ID =
"cherry-pick-snapshot-id";
+ public static final String SORT_ORDER = "sort-order";
+ public static final String IDENTIFIER_FIELDS = "identifier-fields";
+ public static final String PROVIDER = "provider";
+ public static final String FORMAT = "format";
+ public static final String FORMAT_VERSION = "format-version";
+}
diff --git
a/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/property/PropertyConverter.java
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/property/PropertyConverter.java
index 40fd37ac1..c326a6dc0 100644
---
a/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/property/PropertyConverter.java
+++
b/catalogs/bundled-catalog/src/main/java/org/apache/gravitino/catalog/property/PropertyConverter.java
@@ -21,9 +21,6 @@ package org.apache.gravitino.catalog.property;
import java.util.HashMap;
import java.util.Map;
-import org.apache.gravitino.catalog.hive.HiveTablePropertiesMetadata;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,14 +38,7 @@ public abstract class PropertyConverter {
*/
public abstract Map<String, String> engineToGravitinoMapping();
- /**
- * Get the property metadata for the catalog. for more please see {@link
- * HiveTablePropertiesMetadata#propertyEntries()} or {@link
- * IcebergTablePropertiesMetadata#propertyEntries()}
- */
- public abstract Map<String, PropertyEntry<?>> gravitinoPropertyMeta();
-
- Map<String, String> revsereMap(Map<String, String> map) {
+ Map<String, String> reverseMap(Map<String, String> map) {
Map<String, String> res = new HashMap<>();
for (Map.Entry<String, String> entry : map.entrySet()) {
res.put(entry.getValue(), entry.getKey());
@@ -60,7 +50,7 @@ public abstract class PropertyConverter {
/** Convert Gravitino properties to engine properties. */
public Map<String, String> gravitinoToEngineProperties(Map<String, String>
gravitinoProperties) {
Map<String, String> engineProperties = new HashMap<>();
- Map<String, String> gravitinoToEngineMapping =
revsereMap(engineToGravitinoMapping());
+ Map<String, String> gravitinoToEngineMapping =
reverseMap(engineToGravitinoMapping());
for (Map.Entry<String, String> entry : gravitinoProperties.entrySet()) {
String engineKey = gravitinoToEngineMapping.get(entry.getKey());
if (engineKey != null) {
@@ -82,30 +72,14 @@ public abstract class PropertyConverter {
Map<String, Object> gravitinoProperties = new HashMap<>();
Map<String, String> engineToGravitinoMapping = engineToGravitinoMapping();
- Map<String, PropertyEntry<?>> propertyEntryMap = gravitinoPropertyMeta();
for (Map.Entry<String, Object> entry : engineProperties.entrySet()) {
String gravitinoKey = engineToGravitinoMapping.get(entry.getKey());
if (gravitinoKey != null) {
- PropertyEntry<?> propertyEntry = propertyEntryMap.get(gravitinoKey);
- if (propertyEntry != null) {
- // Check value is valid.
- propertyEntry.decode(entry.getValue().toString());
- }
gravitinoProperties.put(gravitinoKey, entry.getValue());
} else {
LOG.info("Property {} is not supported by Gravitino", entry.getKey());
}
}
-
- // Check the required properties.
- for (Map.Entry<String, PropertyEntry<?>> propertyEntry :
propertyEntryMap.entrySet()) {
- if (propertyEntry.getValue().isRequired()
- && !gravitinoProperties.containsKey(propertyEntry.getKey())) {
- throw new IllegalArgumentException(
- "Property " + propertyEntry.getKey() + " is required, you should
");
- }
- }
-
return gravitinoProperties;
}
}
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
index 5aa6f8cde..eadd43f68 100644
--- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -32,6 +32,7 @@ val scalaCollectionCompatVersion: String =
libs.versions.scala.collection.compat
dependencies {
implementation(project(":api"))
+ implementation(project(":catalogs:bundled-catalog", configuration =
"shadow"))
implementation(project(":common"))
implementation(project(":core"))
implementation(project(":server-common"))
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
index b5d5e8be8..a94a1e8d3 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java
@@ -33,19 +33,19 @@ import
org.apache.gravitino.connector.BaseCatalogPropertiesMetadata;
import org.apache.gravitino.connector.PropertyEntry;
public class IcebergCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetadata {
- public static final String CATALOG_BACKEND = "catalog-backend";
+ public static final String CATALOG_BACKEND =
IcebergConstants.CATALOG_BACKEND;
- public static final String GRAVITINO_JDBC_USER = "jdbc-user";
- public static final String ICEBERG_JDBC_USER = "jdbc.user";
+ public static final String GRAVITINO_JDBC_USER =
IcebergConstants.GRAVITINO_JDBC_USER;
+ public static final String ICEBERG_JDBC_USER =
IcebergConstants.ICEBERG_JDBC_USER;
- public static final String GRAVITINO_JDBC_PASSWORD = "jdbc-password";
- public static final String ICEBERG_JDBC_PASSWORD = "jdbc.password";
- public static final String ICEBERG_JDBC_INITIALIZE = "jdbc-initialize";
+ public static final String GRAVITINO_JDBC_PASSWORD =
IcebergConstants.GRAVITINO_JDBC_PASSWORD;
+ public static final String ICEBERG_JDBC_PASSWORD =
IcebergConstants.ICEBERG_JDBC_PASSWORD;
+ public static final String ICEBERG_JDBC_INITIALIZE =
IcebergConstants.ICEBERG_JDBC_INITIALIZE;
- public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver";
- public static final String WAREHOUSE = "warehouse";
- public static final String URI = "uri";
- public static final String CATALOG_BACKEND_NAME = "catalog-backend-name";
+ public static final String GRAVITINO_JDBC_DRIVER =
IcebergConstants.GRAVITINO_JDBC_DRIVER;
+ public static final String WAREHOUSE = IcebergConstants.WAREHOUSE;
+ public static final String URI = IcebergConstants.URI;
+ public static final String CATALOG_BACKEND_NAME =
IcebergConstants.CATALOG_BACKEND_NAME;
private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java
index ad0866c57..ae659e06c 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java
@@ -29,7 +29,7 @@ import org.apache.gravitino.connector.PropertyEntry;
public class IcebergSchemaPropertiesMetadata extends BasePropertiesMetadata {
- public static final String COMMENT = "comment";
+ public static final String COMMENT = IcebergConstants.COMMENT;
private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
static {
diff --git
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java
index 9caf4e461..d336ad15e 100644
---
a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java
@@ -30,16 +30,16 @@ import org.apache.gravitino.connector.PropertyEntry;
import org.apache.iceberg.TableProperties;
public class IcebergTablePropertiesMetadata extends BasePropertiesMetadata {
- public static final String COMMENT = "comment";
- public static final String CREATOR = "creator";
- public static final String LOCATION = "location";
- public static final String CURRENT_SNAPSHOT_ID = "current-snapshot-id";
- public static final String CHERRY_PICK_SNAPSHOT_ID =
"cherry-pick-snapshot-id";
- public static final String SORT_ORDER = "sort-order";
- public static final String IDENTIFIER_FIELDS = "identifier-fields";
- public static final String PROVIDER = "provider";
- public static final String FORMAT = "format";
- public static final String FORMAT_VERSION = "format-version";
+ public static final String COMMENT = IcebergConstants.COMMENT;
+ public static final String CREATOR = IcebergConstants.CREATOR;
+ public static final String LOCATION = IcebergConstants.LOCATION;
+ public static final String CURRENT_SNAPSHOT_ID =
IcebergConstants.CURRENT_SNAPSHOT_ID;
+ public static final String CHERRY_PICK_SNAPSHOT_ID =
IcebergConstants.CHERRY_PICK_SNAPSHOT_ID;
+ public static final String SORT_ORDER = IcebergConstants.SORT_ORDER;
+ public static final String IDENTIFIER_FIELDS =
IcebergConstants.IDENTIFIER_FIELDS;
+ public static final String PROVIDER = IcebergConstants.PROVIDER;
+ public static final String FORMAT = IcebergConstants.FORMAT;
+ public static final String FORMAT_VERSION = IcebergConstants.FORMAT_VERSION;
public static final String DISTRIBUTION_MODE =
TableProperties.WRITE_DISTRIBUTION_MODE;
private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/IcebergPropertiesConstants.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/IcebergPropertiesConstants.java
index 7e2dab545..caabc6dba 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/IcebergPropertiesConstants.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/IcebergPropertiesConstants.java
@@ -19,38 +19,32 @@
package org.apache.gravitino.spark.connector.iceberg;
import com.google.common.annotations.VisibleForTesting;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
+import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
public class IcebergPropertiesConstants {
@VisibleForTesting
- public static final String GRAVITINO_ICEBERG_CATALOG_BACKEND =
- IcebergCatalogPropertiesMetadata.CATALOG_BACKEND;
+ public static final String GRAVITINO_ICEBERG_CATALOG_BACKEND =
IcebergConstants.CATALOG_BACKEND;
static final String ICEBERG_CATALOG_TYPE = CatalogUtil.ICEBERG_CATALOG_TYPE;
@VisibleForTesting
- public static final String GRAVITINO_ICEBERG_CATALOG_WAREHOUSE =
- IcebergCatalogPropertiesMetadata.WAREHOUSE;
+ public static final String GRAVITINO_ICEBERG_CATALOG_WAREHOUSE =
IcebergConstants.WAREHOUSE;
static final String ICEBERG_CATALOG_WAREHOUSE =
CatalogProperties.WAREHOUSE_LOCATION;
@VisibleForTesting
- public static final String GRAVITINO_ICEBERG_CATALOG_URI =
IcebergCatalogPropertiesMetadata.URI;
+ public static final String GRAVITINO_ICEBERG_CATALOG_URI =
IcebergConstants.URI;
static final String ICEBERG_CATALOG_URI = CatalogProperties.URI;
- static final String GRAVITINO_ICEBERG_CATALOG_JDBC_USER =
- IcebergCatalogPropertiesMetadata.GRAVITINO_JDBC_USER;
- static final String ICEBERG_CATALOG_JDBC_USER =
- IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_USER;
+ static final String GRAVITINO_ICEBERG_CATALOG_JDBC_USER =
IcebergConstants.GRAVITINO_JDBC_USER;
+ static final String ICEBERG_CATALOG_JDBC_USER =
IcebergConstants.ICEBERG_JDBC_USER;
static final String GRAVITINO_ICEBERG_CATALOG_JDBC_PASSWORD =
- IcebergCatalogPropertiesMetadata.GRAVITINO_JDBC_PASSWORD;
- static final String ICEBERG_CATALOG_JDBC_PASSWORD =
- IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_PASSWORD;
+ IcebergConstants.GRAVITINO_JDBC_PASSWORD;
+ static final String ICEBERG_CATALOG_JDBC_PASSWORD =
IcebergConstants.ICEBERG_JDBC_PASSWORD;
@VisibleForTesting
public static final String ICEBERG_CATALOG_BACKEND_HIVE =
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE;
@@ -65,34 +59,28 @@ public class IcebergPropertiesConstants {
static final String GRAVITINO_ICEBERG_CATALOG_BACKEND_REST = "rest";
- @VisibleForTesting
- public static final String ICEBERG_LOCATION =
IcebergTablePropertiesMetadata.LOCATION;
+ @VisibleForTesting public static final String ICEBERG_LOCATION =
IcebergConstants.LOCATION;
@VisibleForTesting
- public static final String ICEBERG_CURRENT_SNAPSHOT_ID =
- IcebergTablePropertiesMetadata.CURRENT_SNAPSHOT_ID;
+ public static final String ICEBERG_CURRENT_SNAPSHOT_ID =
IcebergConstants.CURRENT_SNAPSHOT_ID;
- @VisibleForTesting
- public static final String ICEBERG_SORT_ORDER =
IcebergTablePropertiesMetadata.SORT_ORDER;
+ @VisibleForTesting public static final String ICEBERG_SORT_ORDER =
IcebergConstants.SORT_ORDER;
@VisibleForTesting
- public static final String ICEBERG_IDENTIFIER_FIELDS =
- IcebergTablePropertiesMetadata.IDENTIFIER_FIELDS;
+ public static final String ICEBERG_IDENTIFIER_FIELDS =
IcebergConstants.IDENTIFIER_FIELDS;
- @VisibleForTesting
- public static final String ICEBERG_PROVIDER =
IcebergTablePropertiesMetadata.PROVIDER;
+ @VisibleForTesting public static final String ICEBERG_PROVIDER =
IcebergConstants.PROVIDER;
- @VisibleForTesting
- public static final String ICEBERG_FILE_FORMAT =
IcebergTablePropertiesMetadata.FORMAT;
+ @VisibleForTesting public static final String ICEBERG_FILE_FORMAT =
IcebergConstants.FORMAT;
@VisibleForTesting
- public static final String ICEBERG_FORMAT_VERSION =
IcebergTablePropertiesMetadata.FORMAT_VERSION;
+ public static final String ICEBERG_FORMAT_VERSION =
IcebergConstants.FORMAT_VERSION;
@VisibleForTesting
public static final String ICEBERG_CATALOG_CACHE_ENABLED =
CatalogProperties.CACHE_ENABLED;
static final String GRAVITINO_ICEBERG_CATALOG_BACKEND_NAME =
- IcebergCatalogPropertiesMetadata.CATALOG_BACKEND_NAME;
+ IcebergConstants.CATALOG_BACKEND_NAME;
private IcebergPropertiesConstants() {}
}
diff --git a/spark-connector/v3.3/spark/build.gradle.kts
b/spark-connector/v3.3/spark/build.gradle.kts
index f66da56e6..98c05f359 100644
--- a/spark-connector/v3.3/spark/build.gradle.kts
+++ b/spark-connector/v3.3/spark/build.gradle.kts
@@ -149,6 +149,7 @@ tasks.test {
exclude("**/integration/**")
} else {
dependsOn(tasks.jar)
+ dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.12")
diff --git a/spark-connector/v3.4/spark/build.gradle.kts
b/spark-connector/v3.4/spark/build.gradle.kts
index 467d52a0b..449e3c76b 100644
--- a/spark-connector/v3.4/spark/build.gradle.kts
+++ b/spark-connector/v3.4/spark/build.gradle.kts
@@ -149,6 +149,7 @@ tasks.test {
exclude("**/integration/**")
} else {
dependsOn(tasks.jar)
+ dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.12")
diff --git a/spark-connector/v3.5/spark/build.gradle.kts
b/spark-connector/v3.5/spark/build.gradle.kts
index f3b288949..125ff5fa6 100644
--- a/spark-connector/v3.5/spark/build.gradle.kts
+++ b/spark-connector/v3.5/spark/build.gradle.kts
@@ -151,6 +151,7 @@ tasks.test {
exclude("**/integration/**")
} else {
dependsOn(tasks.jar)
+ dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.12")
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java
index d3ffaf3dc..0a76c03b3 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java
@@ -20,10 +20,8 @@
package org.apache.gravitino.trino.connector.catalog.hive;
import com.google.common.collect.ImmutableMap;
-import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.PropertyEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -241,9 +239,4 @@ public class HiveCatalogPropertyConverter extends
PropertyConverter {
public TreeBidiMap<String, String> engineToGravitinoMapping() {
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return ImmutableMap.of();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java
index dd03e9eca..3050e63b0 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java
@@ -21,17 +21,11 @@ package org.apache.gravitino.trino.connector.catalog.hive;
import static
org.apache.gravitino.trino.connector.catalog.hive.HivePropertyMeta.HIVE_SCHEMA_LOCATION;
import com.google.common.collect.ImmutableMap;
-import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.BasePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
public class HiveSchemaPropertyConverter extends PropertyConverter {
- private final BasePropertiesMetadata hiveSchemaPropertiesMetadata =
- new HiveSchemaPropertiesMetadata();
-
// Trino property key does not allow upper case character and '-', so we
need to map it to
// Gravitino
private static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
@@ -44,9 +38,4 @@ public class HiveSchemaPropertyConverter extends
PropertyConverter {
public TreeBidiMap<String, String> engineToGravitinoMapping() {
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return hiveSchemaPropertiesMetadata.propertyEntries();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java
index 1d956baea..9bd3180d2 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java
@@ -20,16 +20,11 @@ package org.apache.gravitino.trino.connector.catalog.hive;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
-import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.hive.HiveTablePropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.BasePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
public class HiveTablePropertyConverter extends PropertyConverter {
- private final BasePropertiesMetadata hiveTablePropertiesMetadata =
- new HiveTablePropertiesMetadata();
// Trino property key does not allow upper case character and '-', so we
need to map it to
// Gravitino
@VisibleForTesting
@@ -56,9 +51,4 @@ public class HiveTablePropertyConverter extends
PropertyConverter {
public TreeBidiMap<String, String> engineToGravitinoMapping() {
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return hiveTablePropertiesMetadata.propertyEntries();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
index 1e58fa519..367c03db3 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.PropertyEntry;
import org.apache.gravitino.trino.connector.GravitinoErrorCode;
public class IcebergCatalogPropertyConverter extends PropertyConverter {
@@ -356,9 +355,4 @@ public class IcebergCatalogPropertyConverter extends
PropertyConverter {
return jdbcProperties;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return ImmutableMap.of();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java
index f194c0bfa..ca520165e 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java
@@ -20,17 +20,10 @@
package org.apache.gravitino.trino.connector.catalog.iceberg;
import com.google.common.collect.ImmutableMap;
-import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.BasePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
public class IcebergSchemaPropertyConverter extends PropertyConverter {
-
- private final BasePropertiesMetadata icebergSchemaPropertiesMetadata =
- new IcebergSchemaPropertiesMetadata();
// TODO (yuqi) add more properties
private static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
new TreeBidiMap<>(new ImmutableMap.Builder<String, String>().build());
@@ -41,9 +34,4 @@ public class IcebergSchemaPropertyConverter extends
PropertyConverter {
// no matter what the mapping is, the properties will be empty.
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return icebergSchemaPropertiesMetadata.propertyEntries();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
index 323d7a3ec..a44524dcd 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
@@ -19,23 +19,16 @@
package org.apache.gravitino.trino.connector.catalog.iceberg;
-import static
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata.LOCATION;
+import static
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.LOCATION;
import static
org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.ICEBERG_LOCATION_PROPERTY;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
-import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.BasePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
public class IcebergTablePropertyConverter extends PropertyConverter {
- private final BasePropertiesMetadata icebergTablePropertiesMetadata =
- new IcebergTablePropertiesMetadata();
-
// TODO (yuqi) add more properties
@VisibleForTesting
static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
@@ -50,9 +43,4 @@ public class IcebergTablePropertyConverter extends
PropertyConverter {
// no matter what the mapping is, the properties will be empty.
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return icebergTablePropertiesMetadata.propertyEntries();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java
index 64547b8cb..5c244ed5c 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.PropertyEntry;
public class JDBCCatalogPropertyConverter extends PropertyConverter {
@@ -125,9 +124,4 @@ public class JDBCCatalogPropertyConverter extends
PropertyConverter {
return trinoProperties;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return ImmutableMap.of();
- }
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLTablePropertyConverter.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLTablePropertyConverter.java
index 3326f5505..c8a3a8677 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLTablePropertyConverter.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/mysql/MySQLTablePropertyConverter.java
@@ -24,15 +24,9 @@ import java.util.Map;
import org.apache.commons.collections4.bidimap.TreeBidiMap;
import org.apache.gravitino.catalog.mysql.MysqlTablePropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
-import org.apache.gravitino.connector.BasePropertiesMetadata;
-import org.apache.gravitino.connector.PropertyEntry;
import org.apache.gravitino.shaded.com.google.common.collect.ImmutableMap;
public class MySQLTablePropertyConverter extends PropertyConverter {
-
- private final BasePropertiesMetadata mysqlTablePropertiesMetadata =
- new MysqlTablePropertiesMetadata();
-
// TODO (yuqi) add more properties
@VisibleForTesting
static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
@@ -49,9 +43,4 @@ public class MySQLTablePropertyConverter extends
PropertyConverter {
public Map<String, String> engineToGravitinoMapping() {
return TRINO_KEY_TO_GRAVITINO_KEY;
}
-
- @Override
- public Map<String, PropertyEntry<?>> gravitinoPropertyMeta() {
- return mysqlTablePropertiesMetadata.propertyEntries();
- }
}
diff --git
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java
index cc3a19219..d5862ba59 100644
---
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java
+++
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java
@@ -21,12 +21,9 @@ package org.apache.gravitino.trino.connector.catalog.iceberg;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import io.trino.spi.TrinoException;
import java.util.Map;
-import java.util.Set;
import org.apache.gravitino.Catalog;
-import
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
import org.apache.gravitino.catalog.property.PropertyConverter;
import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog;
import org.apache.gravitino.trino.connector.metadata.TestGravitinoCatalog;
@@ -95,17 +92,6 @@ public class TestIcebergCatalogPropertyConverter {
"Missing required property for JDBC backend: [jdbc-driver]");
}
- // To test whether we load jar `bundled-catalog` successfully.
- @Test
- public void testPropertyMetadata() {
- Set<String> gravitinoHiveKeys =
-
Sets.newHashSet(IcebergTablePropertyConverter.TRINO_KEY_TO_GRAVITINO_KEY.values());
- Set<String> actualGravitinoKeys =
- Sets.newHashSet(new
IcebergTablePropertiesMetadata().propertyEntries().keySet());
-
- Assertions.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys));
- }
-
@Test
@SuppressWarnings("unchecked")
public void testBuildConnectorPropertiesWithHiveBackend() throws Exception {
