This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 86a235bf9c [MINOR] refactor(spark-connector): Align JDBC connector
common logic (#11847)
86a235bf9c is described below
commit 86a235bf9c9335fd4f07ae6d95ad5d8f2cf22fa8
Author: Yuhui <[email protected]>
AuthorDate: Thu Jul 2 09:45:29 2026 +0800
[MINOR] refactor(spark-connector): Align JDBC connector common logic
(#11847)
### What changes were proposed in this pull request?
Align common JDBC Spark connector logic and related tests.
### Why are the changes needed?
Avoid PostgreSQL-specific naming in shared JDBC logic.
Fix: #N/A
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing targeted Spark connector tests.
---
.../spark/connector/jdbc/GravitinoJdbcCatalog.java | 5 +++++
.../connector/jdbc/JdbcPropertiesConverter.java | 8 +++++---
.../connector/version/CatalogNameAdaptor.java | 8 ++++----
.../jdbc/TestJdbcPropertiesConverter.java | 23 ++++++++++++++++++++++
.../GravitinoPostgreSqlCatalogSpark33.java | 2 +-
.../connector/version/TestCatalogNameAdaptor.java | 8 ++++++++
.../GravitinoPostgreSqlCatalogSpark34.java | 2 +-
.../connector/version/TestCatalogNameAdaptor.java | 8 ++++++++
.../GravitinoPostgreSqlCatalogSpark35.java | 2 +-
.../connector/version/TestCatalogNameAdaptor.java | 8 ++++++++
10 files changed, 64 insertions(+), 10 deletions(-)
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java
index 6e4f3e4975..aef1b5e250 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalog.java
@@ -39,9 +39,13 @@ import org.apache.spark.sql.errors.QueryCompilationErrors;
import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTable;
import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class GravitinoJdbcCatalog extends BaseCatalog {
+ private static final Logger LOG =
LoggerFactory.getLogger(GravitinoJdbcCatalog.class);
+
@Override
protected TableCatalog createAndInitSparkCatalog(
String name, CaseInsensitiveStringMap options, Map<String, String>
properties) {
@@ -66,6 +70,7 @@ public class GravitinoJdbcCatalog extends BaseCatalog {
try {
credentials = catalog.supportsCredentials().getCredentials();
} catch (UnsupportedOperationException e) {
+ LOG.debug("Catalog {} does not support credential vending, skipping.",
catalog.name(), e);
return;
}
for (Credential credential : credentials) {
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java
index 7bd2a86b85..cbcac2fce6 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/JdbcPropertiesConverter.java
@@ -31,7 +31,8 @@ public class JdbcPropertiesConverter implements
PropertiesConverter {
public static class JdbcPropertiesConverterHolder {
private static final JdbcPropertiesConverter INSTANCE = new
JdbcPropertiesConverter();
- private static final JdbcPropertiesConverter PG_INSTANCE = new
JdbcPropertiesConverter(false);
+ private static final JdbcPropertiesConverter NO_TABLE_PROPERTIES_INSTANCE =
+ new JdbcPropertiesConverter(false);
}
private JdbcPropertiesConverter() {
@@ -46,8 +47,9 @@ public class JdbcPropertiesConverter implements
PropertiesConverter {
return JdbcPropertiesConverterHolder.INSTANCE;
}
- public static JdbcPropertiesConverter getPGInstance() {
- return JdbcPropertiesConverterHolder.PG_INSTANCE;
+ /** Returns the singleton instance that does not support table-level
properties. */
+ public static JdbcPropertiesConverter getNoTablePropertiesInstance() {
+ return JdbcPropertiesConverterHolder.NO_TABLE_PROPERTIES_INSTANCE;
}
private static final Map<String, String> GRAVITINO_CONFIG_TO_JDBC =
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java
index c5c4bbcc78..4583a8e0ea 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/version/CatalogNameAdaptor.java
@@ -87,14 +87,14 @@ public class CatalogNameAdaptor {
}
private static String getCatalogName(String provider, int majorVersion, int
minorVersion) {
+ String versionKey = String.format("%d.%d", majorVersion, minorVersion);
if (provider.startsWith("jdbc")) {
if (provider.startsWith("jdbc-postgresql")) {
- return pgCatalogNames.get(String.format("%d.%d", majorVersion,
minorVersion));
+ return pgCatalogNames.get(versionKey);
}
- return jdbcCatalogNames.get(String.format("%d.%d", majorVersion,
minorVersion));
+ return jdbcCatalogNames.get(versionKey);
}
- String key =
- String.format("%s-%d.%d", provider.toLowerCase(Locale.ROOT),
majorVersion, minorVersion);
+ String key = String.format("%s-%s", provider.toLowerCase(Locale.ROOT),
versionKey);
return catalogNames.get(key);
}
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java
index 5d3e4d065f..33b704f11f 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestJdbcPropertiesConverter.java
@@ -28,6 +28,29 @@ public class TestJdbcPropertiesConverter {
private final JdbcPropertiesConverter jdbcPropertiesConverter =
JdbcPropertiesConverter.getInstance();
+ @Test
+ void testGetNoTablePropertiesInstance() {
+ JdbcPropertiesConverter instance =
JdbcPropertiesConverter.getNoTablePropertiesInstance();
+ Assertions.assertNotNull(instance);
+ Assertions.assertSame(instance,
JdbcPropertiesConverter.getNoTablePropertiesInstance());
+ Assertions.assertNotSame(instance, JdbcPropertiesConverter.getInstance());
+
+ Map<String, String> emptyProps =
instance.toGravitinoTableProperties(ImmutableMap.of());
+ Assertions.assertTrue(emptyProps.isEmpty());
+
+ Map<String, String> ownerProps =
+ instance.toGravitinoTableProperties(ImmutableMap.of("owner", "alice"));
+ Assertions.assertTrue(ownerProps.isEmpty());
+
+ Map<String, String> upperOwnerProps =
+ instance.toGravitinoTableProperties(ImmutableMap.of("OWNER", "bob"));
+ Assertions.assertTrue(upperOwnerProps.isEmpty());
+
+ Assertions.assertThrows(
+ UnsupportedOperationException.class,
+ () -> instance.toGravitinoTableProperties(ImmutableMap.of("someKey",
"value")));
+ }
+
@Test
void testCatalogProperties() {
String url = "jdbc-url";
diff --git
a/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark33.java
b/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark33.java
index 147e23a889..6614a1111c 100644
---
a/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark33.java
+++
b/spark-connector/v3.3/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark33.java
@@ -25,6 +25,6 @@ import
org.apache.gravitino.spark.connector.jdbc.JdbcPropertiesConverter;
public class GravitinoPostgreSqlCatalogSpark33 extends
GravitinoJdbcCatalogSpark33 {
@Override
protected PropertiesConverter getPropertiesConverter() {
- return JdbcPropertiesConverter.getPGInstance();
+ return JdbcPropertiesConverter.getNoTablePropertiesInstance();
}
}
diff --git
a/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
b/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
index c600844506..2d82386d02 100644
---
a/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
+++
b/spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
@@ -20,6 +20,8 @@ package org.apache.gravitino.spark.connector.version;
import org.apache.gravitino.spark.connector.hive.GravitinoHiveCatalogSpark33;
import
org.apache.gravitino.spark.connector.iceberg.GravitinoIcebergCatalogSpark33;
+import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark33;
+import
org.apache.gravitino.spark.connector.jdbc.postgresql.GravitinoPostgreSqlCatalogSpark33;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -36,5 +38,11 @@ public class TestCatalogNameAdaptor {
Assertions.assertEquals(
"org.apache.gravitino.spark.connector.paimon.GravitinoPaimonCatalogSpark33",
paimonCatalogName);
+
+ String pgCatalogName =
CatalogNameAdaptor.getCatalogName("jdbc-postgresql");
+ Assertions.assertEquals(GravitinoPostgreSqlCatalogSpark33.class.getName(),
pgCatalogName);
+
+ String jdbcCatalogName = CatalogNameAdaptor.getCatalogName("jdbc");
+ Assertions.assertEquals(GravitinoJdbcCatalogSpark33.class.getName(),
jdbcCatalogName);
}
}
diff --git
a/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark34.java
b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark34.java
index 545f85378c..61c1c32751 100644
---
a/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark34.java
+++
b/spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark34.java
@@ -25,6 +25,6 @@ import
org.apache.gravitino.spark.connector.jdbc.JdbcPropertiesConverter;
public class GravitinoPostgreSqlCatalogSpark34 extends GravitinoJdbcCatalog {
@Override
protected PropertiesConverter getPropertiesConverter() {
- return JdbcPropertiesConverter.getPGInstance();
+ return JdbcPropertiesConverter.getNoTablePropertiesInstance();
}
}
diff --git
a/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
b/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
index 1924a2c4e8..ad50652ffc 100644
---
a/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
+++
b/spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
@@ -20,6 +20,8 @@ package org.apache.gravitino.spark.connector.version;
import org.apache.gravitino.spark.connector.hive.GravitinoHiveCatalogSpark34;
import
org.apache.gravitino.spark.connector.iceberg.GravitinoIcebergCatalogSpark34;
+import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark34;
+import
org.apache.gravitino.spark.connector.jdbc.postgresql.GravitinoPostgreSqlCatalogSpark34;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -36,5 +38,11 @@ public class TestCatalogNameAdaptor {
Assertions.assertEquals(
"org.apache.gravitino.spark.connector.paimon.GravitinoPaimonCatalogSpark34",
paimonCatalogName);
+
+ String pgCatalogName =
CatalogNameAdaptor.getCatalogName("jdbc-postgresql");
+ Assertions.assertEquals(GravitinoPostgreSqlCatalogSpark34.class.getName(),
pgCatalogName);
+
+ String jdbcCatalogName = CatalogNameAdaptor.getCatalogName("jdbc");
+ Assertions.assertEquals(GravitinoJdbcCatalogSpark34.class.getName(),
jdbcCatalogName);
}
}
diff --git
a/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark35.java
b/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark35.java
index 709013786c..fc8a4367ec 100644
---
a/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark35.java
+++
b/spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/postgresql/GravitinoPostgreSqlCatalogSpark35.java
@@ -25,6 +25,6 @@ import
org.apache.gravitino.spark.connector.jdbc.JdbcPropertiesConverter;
public class GravitinoPostgreSqlCatalogSpark35 extends GravitinoJdbcCatalog {
@Override
protected PropertiesConverter getPropertiesConverter() {
- return JdbcPropertiesConverter.getPGInstance();
+ return JdbcPropertiesConverter.getNoTablePropertiesInstance();
}
}
diff --git
a/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
b/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
index 119817d39f..28d9337242 100644
---
a/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
+++
b/spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/version/TestCatalogNameAdaptor.java
@@ -20,6 +20,8 @@ package org.apache.gravitino.spark.connector.version;
import org.apache.gravitino.spark.connector.hive.GravitinoHiveCatalogSpark35;
import
org.apache.gravitino.spark.connector.iceberg.GravitinoIcebergCatalogSpark35;
+import org.apache.gravitino.spark.connector.jdbc.GravitinoJdbcCatalogSpark35;
+import
org.apache.gravitino.spark.connector.jdbc.postgresql.GravitinoPostgreSqlCatalogSpark35;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -36,5 +38,11 @@ public class TestCatalogNameAdaptor {
Assertions.assertEquals(
"org.apache.gravitino.spark.connector.paimon.GravitinoPaimonCatalogSpark35",
paimonCatalogName);
+
+ String pgCatalogName =
CatalogNameAdaptor.getCatalogName("jdbc-postgresql");
+ Assertions.assertEquals(GravitinoPostgreSqlCatalogSpark35.class.getName(),
pgCatalogName);
+
+ String jdbcCatalogName = CatalogNameAdaptor.getCatalogName("jdbc");
+ Assertions.assertEquals(GravitinoJdbcCatalogSpark35.class.getName(),
jdbcCatalogName);
}
}
