(gravitino) branch main updated: [#11231] fix(spark-connector): handle JDBC timestamp type (#11424)

[roryqi]

This is an automated email from the ASF dual-hosted git repository.

roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git


The following commit(s) were added to refs/heads/main by this push:
     new 3ce2387bcd [#11231] fix(spark-connector): handle JDBC timestamp type 
(#11424)
3ce2387bcd is described below

commit 3ce2387bcd93c74eeaf69116c246fa725042e47d
Author: YangJie <yangji...@baidu.com>
AuthorDate: Thu Jun 4 20:46:14 2026 +0800

    [#11231] fix(spark-connector): handle JDBC timestamp type (#11424)
    
    ### What changes were proposed in this pull request?
    
    This PR updates the Spark JDBC type converter to map Gravitino timestamp
    types to Spark `TimestampType` in Spark common.
    
    It also adds unit tests for JDBC timestamp type conversion.
    
    ### Why are the changes needed?
    
    Spark 3.3 does not expose `TimestampNTZType` as a normal production
    timestamp type for this JDBC path. When PostgreSQL tables contain
    `timestamp without time zone`, Gravitino maps it to
    `TimestampType{withTimeZone=false}`, but Spark JDBC catalog schema
    rendering can fail because the common JDBC converter did not handle
    Gravitino timestamp types.
    
    Fix: #11231
    
    ### Does this PR introduce _any_ user-facing change?
    
    No API or configuration changes.
    
    This fixes Spark JDBC catalog behavior for timestamp columns.
    
    ### How was this patch tested?
    
    ```bash
    ./gradlew :spark-connector:spark-common:test \
        --tests 
"org.apache.gravitino.spark.connector.jdbc.TestSparkJdbcTypeConverter" \
        -PskipITs \
        -PskipDockerTests=true
    ```
    
    ```bash
    ./gradlew :spark-connector:spark-common:test \
        --tests "org.apache.gravitino.spark.connector.TestSparkTypeConverter" \
        -PskipITs \
        -PskipDockerTests=true
    ```
    
    Also ran:
    
    ```bash
    git diff --check
    ```
---
 .../connector/jdbc/SparkJdbcTypeConverter.java     |  2 ++
 .../jdbc/TestSparkJdbcTypeConverter.java}          | 33 +++++++++++++---------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
index 56e2734a7f..95a9c0c408 100644
--- 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
+++ 
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
@@ -33,6 +33,8 @@ public class SparkJdbcTypeConverter extends 
SparkTypeConverter {
     // type varchar.
     if (gravitinoType instanceof Types.VarCharType) {
       return DataTypes.StringType;
+    } else if (gravitinoType instanceof Types.TimestampType) {
+      return DataTypes.TimestampType;
     } else {
       return super.toSparkType(gravitinoType);
     }
diff --git 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
similarity index 54%
copy from 
spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
copy to 
spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
index 56e2734a7f..00fd1c7784 100644
--- 
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
+++ 
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
@@ -19,22 +19,29 @@
 
 package org.apache.gravitino.spark.connector.jdbc;
 
-import org.apache.gravitino.rel.types.Type;
 import org.apache.gravitino.rel.types.Types;
-import org.apache.gravitino.spark.connector.SparkTypeConverter;
-import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.DataTypes;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
 
-public class SparkJdbcTypeConverter extends SparkTypeConverter {
+/** Unit tests for {@link SparkJdbcTypeConverter}. */
+public class TestSparkJdbcTypeConverter {
 
-  @Override
-  public DataType toSparkType(Type gravitinoType) {
-    // if spark version lower than 3.4.4, using VarCharType will throw an 
exception: Unsupported
-    // type varchar.
-    if (gravitinoType instanceof Types.VarCharType) {
-      return DataTypes.StringType;
-    } else {
-      return super.toSparkType(gravitinoType);
-    }
+  private final SparkJdbcTypeConverter sparkJdbcTypeConverter = new 
SparkJdbcTypeConverter();
+
+  @Test
+  void testConvertTimestampTypesToSparkTimestamp() {
+    Assertions.assertEquals(
+        DataTypes.TimestampType,
+        
sparkJdbcTypeConverter.toSparkType(Types.TimestampType.withTimeZone()));
+    Assertions.assertEquals(
+        DataTypes.TimestampType,
+        
sparkJdbcTypeConverter.toSparkType(Types.TimestampType.withoutTimeZone()));
+  }
+
+  @Test
+  void testConvertVarCharTypeToSparkString() {
+    Assertions.assertEquals(
+        DataTypes.StringType, 
sparkJdbcTypeConverter.toSparkType(Types.VarCharType.of(10)));
   }
 }