This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 9b7bbc9d0ca3 [SPARK-60507][SQL] Reuse schema resolution connection to 
read JDBC database metadata
9b7bbc9d0ca3 is described below

commit 9b7bbc9d0ca3aa9376ef37081c09345e1fa9e143
Author: Constantin-Tiberiu Craiu 
<[email protected]>
AuthorDate: Thu Oct 29 19:48:14 2026 +0000

    [SPARK-60507][SQL] Reuse schema resolution connection to read JDBC database 
metadata
    
    ### What changes were proposed in this pull request?
    Read the JDBC database metadata on the connection that resolves the schema 
and avoid spawning two connections for schema and metadata resolution.
    
    ### Why are the changes needed?
    Reduces resource usage, relevant in resource-constrained environments (e.g: 
limited connection slots, rate-limits etc.)
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Unit test in `JDBCV2Suite`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Generated-by: Claude Code (Claude Opus 5.5)
    
    Closes #59748 from craiuconstantintiberiu/jdbc-metadata-connection-reuse.
    
    Authored-by: Constantin-Tiberiu Craiu 
<[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
    (cherry picked from commit 2cb4c3a4ef77702ba031c440b386c1f1ceea5f1e)
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .../datasources/jdbc/JDBCDatabaseMetadata.scala    | 32 +++++++++++++++------
 .../sql/execution/datasources/jdbc/JDBCRDD.scala   | 12 +++-----
 .../execution/datasources/jdbc/JDBCRelation.scala  | 31 ++++++++++++--------
 .../datasources/jdbc/JdbcRelationProvider.scala    |  6 ++--
 .../datasources/v2/jdbc/JDBCScanBuilder.scala      |  7 +++--
 .../execution/datasources/v2/jdbc/JDBCTable.scala  |  7 +++--
 .../datasources/v2/jdbc/JDBCTableCatalog.scala     |  5 ++--
 .../org/apache/spark/sql/jdbc/JDBCV2Suite.scala    | 33 ++++++++++++++++++++++
 8 files changed, 94 insertions(+), 39 deletions(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
index 7aa184f57fc0..79c73a2a0f0b 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
@@ -59,6 +59,28 @@ object JDBCDatabaseMetadata extends Logging {
     }
   }
 
+  /**
+   * Reads the metadata from an open JDBC Connection without closing it,
+   * handling errors for each field individually.
+   */
+  def fromJDBCConnection(conn: Connection): JDBCDatabaseMetadata = {
+    try {
+      // getMetaData itself can throw, so we catch that and return None for 
all fields
+      val databaseMetadata = conn.getMetaData
+
+      JDBCDatabaseMetadata(
+        databaseMajorVersion = 
safeGet(databaseMetadata.getDatabaseMajorVersion),
+        databaseMinorVersion = 
safeGet(databaseMetadata.getDatabaseMinorVersion),
+        databaseDriverMajorVersion = 
safeGet(databaseMetadata.getDriverMajorVersion),
+        databaseDriverMinorVersion = 
safeGet(databaseMetadata.getDriverMinorVersion)
+      )
+    } catch {
+      case NonFatal(e) =>
+        logWarning(log"Exception while getting database metadata object from 
connection", e)
+        JDBCDatabaseMetadata(None, None, None, None)
+    }
+  }
+
   /**
    * Creates a DatabaseMetadata instance from a JDBC Connection,
    * handling errors for each field individually.
@@ -82,15 +104,7 @@ object JDBCDatabaseMetadata extends Logging {
 
     try {
       conn = getConnection(-1)
-      // getMetaData itself can throw, so we catch that and return None for 
all fields
-      val databaseMetadata = conn.getMetaData
-
-      JDBCDatabaseMetadata(
-        databaseMajorVersion = 
safeGet(databaseMetadata.getDatabaseMajorVersion),
-        databaseMinorVersion = 
safeGet(databaseMetadata.getDatabaseMinorVersion),
-        databaseDriverMajorVersion = 
safeGet(databaseMetadata.getDriverMajorVersion),
-        databaseDriverMinorVersion = 
safeGet(databaseMetadata.getDriverMinorVersion)
-      )
+      fromJDBCConnection(conn)
     } catch {
       case NonFatal(e) =>
         logWarning(log"Exception while getting database metadata object from 
connection", e)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 0fbc2aa8643e..0c1bffa1a7fc 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -94,12 +94,6 @@ object JDBCRDD extends Logging {
     }
   }
 
-  def resolveTable(options: JDBCOptions): StructType = {
-    JdbcUtils.withConnection(options) {
-      resolveTable(options, _)
-    }
-  }
-
   def getQueryOutputSchema(
       query: String, options: JDBCOptions, dialect: JdbcDialect, conn: 
Connection): StructType = {
     logInfo(log"Generated JDBC query to get scan output schema: 
${MDC(SQL_TEXT, query)}")
@@ -171,7 +165,8 @@ object JDBCRDD extends Logging {
       limit: Int = 0,
       sortOrders: Array[String] = Array.empty[String],
       offset: Int = 0,
-      additionalMetrics: Map[String, SQLMetric] = Map()): RDD[InternalRow] = {
+      additionalMetrics: Map[String, SQLMetric] = Map(),
+      databaseMetadata: Option[JDBCDatabaseMetadata] = None): RDD[InternalRow] 
= {
     val url = options.url
     val dialect = JdbcDialects.get(url)
     val quotedColumns = if (groupByColumns.isEmpty) {
@@ -191,7 +186,8 @@ object JDBCRDD extends Logging {
       parts,
       url,
       options,
-      databaseMetadata = 
JDBCDatabaseMetadata.fromJDBCConnectionFactory(connectionFactory),
+      databaseMetadata = databaseMetadata.getOrElse(
+        JDBCDatabaseMetadata.fromJDBCConnectionFactory(connectionFactory)),
       groupByColumns,
       sampleClause,
       limit,
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index 91e45d0a99e8..f5e0d5763f50 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -244,19 +244,24 @@ private[sql] object JDBCRelation extends Logging {
   /**
    * Takes a (schema, table) specification and returns the table's Catalyst 
schema.
    * If `customSchema` defined in the JDBC options, replaces the schema's 
dataType with the
-   * custom schema's type.
+   * custom schema's type. Also reads the database metadata over the same 
connection.
    *
    * @param resolver function used to determine if two identifiers are equal
    * @param jdbcOptions JDBC options that contains url, table and other 
information.
-   * @return resolved Catalyst schema of a JDBC table
+   * @return resolved Catalyst schema of a JDBC table and the database metadata
    */
-  def getSchema(resolver: Resolver, jdbcOptions: JDBCOptions): StructType = {
-    val tableSchema = JDBCRDD.resolveTable(jdbcOptions)
-    jdbcOptions.customSchema match {
+  def getSchemaAndDatabaseMetadata(
+      resolver: Resolver,
+      jdbcOptions: JDBCOptions): (StructType, JDBCDatabaseMetadata) = {
+    val (tableSchema, databaseMetadata) = 
JdbcUtils.withConnection(jdbcOptions) { conn =>
+      (JDBCRDD.resolveTable(jdbcOptions, conn), 
JDBCDatabaseMetadata.fromJDBCConnection(conn))
+    }
+    val schema = jdbcOptions.customSchema match {
       case Some(customSchema) => JdbcUtils.getCustomSchema(
         tableSchema, customSchema, resolver)
       case None => tableSchema
     }
+    (schema, databaseMetadata)
   }
 
   /**
@@ -267,11 +272,11 @@ private[sql] object JDBCRelation extends Logging {
       jdbcOptions: JDBCOptions)(
       sparkSession: SparkSession): JDBCRelation = {
     val remoteSchemaFetchMetric = 
JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
-    val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
-      JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver, 
jdbcOptions)
+    val (schema, databaseMetadata) = 
SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+      getSchemaAndDatabaseMetadata(sparkSession.sessionState.conf.resolver, 
jdbcOptions)
     }
     JDBCRelation(schema, parts, jdbcOptions,
-      Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession)
+      Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession, 
Some(databaseMetadata))
   }
 }
 
@@ -279,7 +284,9 @@ private[sql] case class JDBCRelation(
     override val schema: StructType,
     parts: Array[Partition],
     jdbcOptions: JDBCOptions,
-    additionalMetrics: Map[String, SQLMetric] = Map())(@transient val 
sparkSession: SparkSession)
+    additionalMetrics: Map[String, SQLMetric] = Map())(
+    @transient val sparkSession: SparkSession,
+    val databaseMetadata: Option[JDBCDatabaseMetadata] = None)
   extends BaseRelation
   with PrunedFilteredScan
   with InsertableRelation {
@@ -313,7 +320,8 @@ private[sql] case class JDBCRelation(
       pushedPredicates,
       parts,
       jdbcOptions,
-      additionalMetrics = additionalMetrics).asInstanceOf[RDD[Row]]
+      additionalMetrics = additionalMetrics,
+      databaseMetadata = databaseMetadata).asInstanceOf[RDD[Row]]
   }
 
   def buildScan(
@@ -339,7 +347,8 @@ private[sql] case class JDBCRelation(
       limit,
       sortOrders,
       offset,
-      additionalMetrics).asInstanceOf[RDD[Row]]
+      additionalMetrics,
+      databaseMetadata).asInstanceOf[RDD[Row]]
   }
 
   override def insert(data: DataFrame, overwrite: Boolean): Unit = {
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index 244064762f28..590e7568f074 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -37,8 +37,8 @@ class JdbcRelationProvider extends CreatableRelationProvider
     val resolver = sparkSession.sessionState.conf.resolver
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
     val remoteSchemaFetchMetric = 
JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
-    val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
-      JDBCRelation.getSchema(resolver, jdbcOptions)
+    val (schema, databaseMetadata) = 
SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+      JDBCRelation.getSchemaAndDatabaseMetadata(resolver, jdbcOptions)
     }
     val parts = JDBCRelation.columnPartition(schema, resolver, timeZoneId, 
jdbcOptions)
     JDBCRelation(
@@ -46,7 +46,7 @@ class JdbcRelationProvider extends CreatableRelationProvider
       parts,
       jdbcOptions,
       Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric)
-    )(sparkSession)
+    )(sparkSession, Some(databaseMetadata))
   }
 
   override def createRelation(
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
index ef663e034295..18a952677ef3 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
@@ -29,7 +29,7 @@ import 
org.apache.spark.sql.connector.expressions.filter.Predicate
 import org.apache.spark.sql.connector.join.JoinType
 import org.apache.spark.sql.connector.read.{SampleMethod, ScanBuilder, 
SupportsPushDownAggregates, SupportsPushDownJoin, SupportsPushDownLimit, 
SupportsPushDownOffset, SupportsPushDownRequiredColumns, 
SupportsPushDownTableSample, SupportsPushDownTopN, SupportsPushDownV2Filters}
 import org.apache.spark.sql.execution.datasources.PartitioningUtils
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, 
JDBCPartition, JDBCRDD, JDBCRelation}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata, 
JDBCOptions, JDBCPartition, JDBCRDD, JDBCRelation}
 import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcSQLQueryBuilder, 
JoinPushdownAliasGenerator}
@@ -39,7 +39,8 @@ case class JDBCScanBuilder(
     session: SparkSession,
     schema: StructType,
     var jdbcOptions: JDBCOptions,
-    additionalMetrics: Map[String, SQLMetric] = Map())
+    additionalMetrics: Map[String, SQLMetric] = Map(),
+    databaseMetadata: Option[JDBCDatabaseMetadata] = None)
   extends ScanBuilder
     with SupportsPushDownV2Filters
     with SupportsPushDownRequiredColumns
@@ -375,7 +376,7 @@ case class JDBCScanBuilder(
     // "DEPT","NAME",MAX("SALARY"),MIN("BONUS"), instead of getting column 
names from
     // prunedSchema and quote them (will become "MAX(SALARY)", "MIN(BONUS)" 
and can't
     // be used in sql string.
-    JDBCScan(JDBCRelation(schema, parts, jdbcOptions, 
additionalMetrics)(session),
+    JDBCScan(JDBCRelation(schema, parts, jdbcOptions, 
additionalMetrics)(session, databaseMetadata),
       finalSchema, pushedPredicate, pushedAggregateList, pushedGroupBys,
       tableSampleClause, pushedLimit, sortOrders, pushedOffset)
   }
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
index a242a24baf9c..02197ba3d9df 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
@@ -27,7 +27,7 @@ import 
org.apache.spark.sql.connector.catalog.index.{SupportsIndex, TableIndex}
 import org.apache.spark.sql.connector.expressions.NamedReference
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
 import org.apache.spark.sql.errors.DataTypeErrorsBase
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, 
JdbcOptionsInWrite, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata, 
JDBCOptions, JdbcOptionsInWrite, JdbcUtils}
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.apache.spark.sql.types.StructType
@@ -37,7 +37,8 @@ case class JDBCTable(
     ident: Identifier,
     override val schema: StructType,
     jdbcOptions: JDBCOptions,
-    additionalMetrics: Map[String, SQLMetric] = Map())
+    additionalMetrics: Map[String, SQLMetric] = Map(),
+    databaseMetadata: Option[JDBCDatabaseMetadata] = None)
   extends Table
   with SupportsRead
   with SupportsWrite
@@ -61,7 +62,7 @@ case class JDBCTable(
   override def newScanBuilder(options: CaseInsensitiveStringMap): 
JDBCScanBuilder = {
     val mergedOptions = new JDBCOptions(
       jdbcOptions.parameters.originalMap ++ 
options.asCaseSensitiveMap().asScala)
-    JDBCScanBuilder(SparkSession.active, schema, mergedOptions, 
additionalMetrics)
+    JDBCScanBuilder(SparkSession.active, schema, mergedOptions, 
additionalMetrics, databaseMetadata)
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
index dc7009d7c6b4..ee1794b63c7f 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.connector.catalog.{CatalogV2Util, 
Column, FunctionCa
 import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.errors.{DataTypeErrorsBase, 
QueryCompilationErrors, QueryExecutionErrors}
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, 
JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata, 
JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
@@ -171,7 +171,8 @@ class JDBCTableCatalog extends TableCatalog
           JDBCRDD.resolveTable(tableOptions, conn, Some(ident), Some(name()))
         }
         JDBCTable(ident, schema, tableOptions,
-          Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric))
+          Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric),
+          Some(JDBCDatabaseMetadata.fromJDBCConnection(conn)))
       }
     }
   }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
index 49a354e412d2..52502683d20d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.jdbc
 import java.sql.{Connection, DriverManager}
 import java.util.{HexFormat, Properties}
 import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.atomic.AtomicInteger
 
 import scala.jdk.CollectionConverters._
 import scala.util.control.NonFatal
@@ -3350,6 +3351,38 @@ class JDBCV2Suite extends SharedSparkSession with 
ExplainSuiteHelper {
     assertResult(expectedMetadata) { jdbcRdd.getDatabaseMetadata }
   }
 
+  test("SPARK-60507: JDBC read opens one driver connection") {
+    val driverConnections = new AtomicInteger()
+    val countingDialect = new H2Dialect() {
+      override def createConnectionFactory(options: JDBCOptions): Int => 
Connection = {
+        val factory = super.createConnectionFactory(options)
+        partitionId => {
+          if (partitionId == -1) driverConnections.incrementAndGet()
+          factory(partitionId)
+        }
+      }
+    }
+    JdbcDialects.unregisterDialect(h2Dialect)
+    try {
+      JdbcDialects.registerDialect(countingDialect)
+      val reads = Seq[(String, () => Unit)](
+        "v1" -> { () =>
+          val df = spark.read.jdbc(url, "\"test\".\"people\"", new 
Properties())
+          df.count()
+          df.collect()
+        },
+        "v2" -> (() => sql("SELECT * FROM h2.test.people").collect()))
+      reads.foreach { case (name, read) =>
+        driverConnections.set(0)
+        read()
+        assert(driverConnections.get() == 1, s"$name read opened 
${driverConnections.get()}")
+      }
+    } finally {
+      JdbcDialects.unregisterDialect(countingDialect)
+      JdbcDialects.registerDialect(h2Dialect)
+    }
+  }
+
   test("SPARK-58782: null literal in aggregate should render as NULL not 
'null'") {
     val df = sql("SELECT NULLIF(s, '') AS g, COUNT(*) FROM 
h2.test.null_literal GROUP BY g")
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to