This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2cb4c3a4ef77 [SPARK-60507][SQL] Reuse schema resolution connection to
read JDBC database metadata
2cb4c3a4ef77 is described below
commit 2cb4c3a4ef77702ba031c440b386c1f1ceea5f1e
Author: Constantin-Tiberiu Craiu
<[email protected]>
AuthorDate: Thu Oct 29 19:48:14 2026 +0000
[SPARK-60507][SQL] Reuse schema resolution connection to read JDBC database
metadata
### What changes were proposed in this pull request?
Read the JDBC database metadata on the connection that resolves the schema
and avoid spawning two connections for schema and metadata resolution.
### Why are the changes needed?
Reduces resource usage, relevant in resource-constrained environments (e.g:
limited connection slots, rate-limits etc.)
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test in `JDBCV2Suite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5.5)
Closes #59748 from craiuconstantintiberiu/jdbc-metadata-connection-reuse.
Authored-by: Constantin-Tiberiu Craiu
<[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../datasources/jdbc/JDBCDatabaseMetadata.scala | 32 +++++++++++++++------
.../sql/execution/datasources/jdbc/JDBCRDD.scala | 12 +++-----
.../execution/datasources/jdbc/JDBCRelation.scala | 31 ++++++++++++--------
.../datasources/jdbc/JdbcRelationProvider.scala | 6 ++--
.../datasources/v2/jdbc/JDBCScanBuilder.scala | 7 +++--
.../execution/datasources/v2/jdbc/JDBCTable.scala | 7 +++--
.../datasources/v2/jdbc/JDBCTableCatalog.scala | 5 ++--
.../org/apache/spark/sql/jdbc/JDBCV2Suite.scala | 33 ++++++++++++++++++++++
8 files changed, 94 insertions(+), 39 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
index 7aa184f57fc0..79c73a2a0f0b 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCDatabaseMetadata.scala
@@ -59,6 +59,28 @@ object JDBCDatabaseMetadata extends Logging {
}
}
+ /**
+ * Reads the metadata from an open JDBC Connection without closing it,
+ * handling errors for each field individually.
+ */
+ def fromJDBCConnection(conn: Connection): JDBCDatabaseMetadata = {
+ try {
+ // getMetaData itself can throw, so we catch that and return None for
all fields
+ val databaseMetadata = conn.getMetaData
+
+ JDBCDatabaseMetadata(
+ databaseMajorVersion =
safeGet(databaseMetadata.getDatabaseMajorVersion),
+ databaseMinorVersion =
safeGet(databaseMetadata.getDatabaseMinorVersion),
+ databaseDriverMajorVersion =
safeGet(databaseMetadata.getDriverMajorVersion),
+ databaseDriverMinorVersion =
safeGet(databaseMetadata.getDriverMinorVersion)
+ )
+ } catch {
+ case NonFatal(e) =>
+ logWarning(log"Exception while getting database metadata object from
connection", e)
+ JDBCDatabaseMetadata(None, None, None, None)
+ }
+ }
+
/**
* Creates a DatabaseMetadata instance from a JDBC Connection,
* handling errors for each field individually.
@@ -82,15 +104,7 @@ object JDBCDatabaseMetadata extends Logging {
try {
conn = getConnection(-1)
- // getMetaData itself can throw, so we catch that and return None for
all fields
- val databaseMetadata = conn.getMetaData
-
- JDBCDatabaseMetadata(
- databaseMajorVersion =
safeGet(databaseMetadata.getDatabaseMajorVersion),
- databaseMinorVersion =
safeGet(databaseMetadata.getDatabaseMinorVersion),
- databaseDriverMajorVersion =
safeGet(databaseMetadata.getDriverMajorVersion),
- databaseDriverMinorVersion =
safeGet(databaseMetadata.getDriverMinorVersion)
- )
+ fromJDBCConnection(conn)
} catch {
case NonFatal(e) =>
logWarning(log"Exception while getting database metadata object from
connection", e)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 0fbc2aa8643e..0c1bffa1a7fc 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -94,12 +94,6 @@ object JDBCRDD extends Logging {
}
}
- def resolveTable(options: JDBCOptions): StructType = {
- JdbcUtils.withConnection(options) {
- resolveTable(options, _)
- }
- }
-
def getQueryOutputSchema(
query: String, options: JDBCOptions, dialect: JdbcDialect, conn:
Connection): StructType = {
logInfo(log"Generated JDBC query to get scan output schema:
${MDC(SQL_TEXT, query)}")
@@ -171,7 +165,8 @@ object JDBCRDD extends Logging {
limit: Int = 0,
sortOrders: Array[String] = Array.empty[String],
offset: Int = 0,
- additionalMetrics: Map[String, SQLMetric] = Map()): RDD[InternalRow] = {
+ additionalMetrics: Map[String, SQLMetric] = Map(),
+ databaseMetadata: Option[JDBCDatabaseMetadata] = None): RDD[InternalRow]
= {
val url = options.url
val dialect = JdbcDialects.get(url)
val quotedColumns = if (groupByColumns.isEmpty) {
@@ -191,7 +186,8 @@ object JDBCRDD extends Logging {
parts,
url,
options,
- databaseMetadata =
JDBCDatabaseMetadata.fromJDBCConnectionFactory(connectionFactory),
+ databaseMetadata = databaseMetadata.getOrElse(
+ JDBCDatabaseMetadata.fromJDBCConnectionFactory(connectionFactory)),
groupByColumns,
sampleClause,
limit,
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index 91e45d0a99e8..f5e0d5763f50 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -244,19 +244,24 @@ private[sql] object JDBCRelation extends Logging {
/**
* Takes a (schema, table) specification and returns the table's Catalyst
schema.
* If `customSchema` defined in the JDBC options, replaces the schema's
dataType with the
- * custom schema's type.
+ * custom schema's type. Also reads the database metadata over the same
connection.
*
* @param resolver function used to determine if two identifiers are equal
* @param jdbcOptions JDBC options that contains url, table and other
information.
- * @return resolved Catalyst schema of a JDBC table
+ * @return resolved Catalyst schema of a JDBC table and the database metadata
*/
- def getSchema(resolver: Resolver, jdbcOptions: JDBCOptions): StructType = {
- val tableSchema = JDBCRDD.resolveTable(jdbcOptions)
- jdbcOptions.customSchema match {
+ def getSchemaAndDatabaseMetadata(
+ resolver: Resolver,
+ jdbcOptions: JDBCOptions): (StructType, JDBCDatabaseMetadata) = {
+ val (tableSchema, databaseMetadata) =
JdbcUtils.withConnection(jdbcOptions) { conn =>
+ (JDBCRDD.resolveTable(jdbcOptions, conn),
JDBCDatabaseMetadata.fromJDBCConnection(conn))
+ }
+ val schema = jdbcOptions.customSchema match {
case Some(customSchema) => JdbcUtils.getCustomSchema(
tableSchema, customSchema, resolver)
case None => tableSchema
}
+ (schema, databaseMetadata)
}
/**
@@ -267,11 +272,11 @@ private[sql] object JDBCRelation extends Logging {
jdbcOptions: JDBCOptions)(
sparkSession: SparkSession): JDBCRelation = {
val remoteSchemaFetchMetric =
JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
- val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
- JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver,
jdbcOptions)
+ val (schema, databaseMetadata) =
SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+ getSchemaAndDatabaseMetadata(sparkSession.sessionState.conf.resolver,
jdbcOptions)
}
JDBCRelation(schema, parts, jdbcOptions,
- Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession)
+ Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession,
Some(databaseMetadata))
}
}
@@ -279,7 +284,9 @@ private[sql] case class JDBCRelation(
override val schema: StructType,
parts: Array[Partition],
jdbcOptions: JDBCOptions,
- additionalMetrics: Map[String, SQLMetric] = Map())(@transient val
sparkSession: SparkSession)
+ additionalMetrics: Map[String, SQLMetric] = Map())(
+ @transient val sparkSession: SparkSession,
+ val databaseMetadata: Option[JDBCDatabaseMetadata] = None)
extends BaseRelation
with PrunedFilteredScan
with InsertableRelation {
@@ -313,7 +320,8 @@ private[sql] case class JDBCRelation(
pushedPredicates,
parts,
jdbcOptions,
- additionalMetrics = additionalMetrics).asInstanceOf[RDD[Row]]
+ additionalMetrics = additionalMetrics,
+ databaseMetadata = databaseMetadata).asInstanceOf[RDD[Row]]
}
def buildScan(
@@ -339,7 +347,8 @@ private[sql] case class JDBCRelation(
limit,
sortOrders,
offset,
- additionalMetrics).asInstanceOf[RDD[Row]]
+ additionalMetrics,
+ databaseMetadata).asInstanceOf[RDD[Row]]
}
override def insert(data: DataFrame, overwrite: Boolean): Unit = {
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index 244064762f28..590e7568f074 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -37,8 +37,8 @@ class JdbcRelationProvider extends CreatableRelationProvider
val resolver = sparkSession.sessionState.conf.resolver
val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
val remoteSchemaFetchMetric =
JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
- val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
- JDBCRelation.getSchema(resolver, jdbcOptions)
+ val (schema, databaseMetadata) =
SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+ JDBCRelation.getSchemaAndDatabaseMetadata(resolver, jdbcOptions)
}
val parts = JDBCRelation.columnPartition(schema, resolver, timeZoneId,
jdbcOptions)
JDBCRelation(
@@ -46,7 +46,7 @@ class JdbcRelationProvider extends CreatableRelationProvider
parts,
jdbcOptions,
Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric)
- )(sparkSession)
+ )(sparkSession, Some(databaseMetadata))
}
override def createRelation(
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
index ef663e034295..18a952677ef3 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
@@ -29,7 +29,7 @@ import
org.apache.spark.sql.connector.expressions.filter.Predicate
import org.apache.spark.sql.connector.join.JoinType
import org.apache.spark.sql.connector.read.{SampleMethod, ScanBuilder,
SupportsPushDownAggregates, SupportsPushDownJoin, SupportsPushDownLimit,
SupportsPushDownOffset, SupportsPushDownRequiredColumns,
SupportsPushDownTableSample, SupportsPushDownTopN, SupportsPushDownV2Filters}
import org.apache.spark.sql.execution.datasources.PartitioningUtils
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions,
JDBCPartition, JDBCRDD, JDBCRelation}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata,
JDBCOptions, JDBCPartition, JDBCRDD, JDBCRelation}
import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcSQLQueryBuilder,
JoinPushdownAliasGenerator}
@@ -39,7 +39,8 @@ case class JDBCScanBuilder(
session: SparkSession,
schema: StructType,
var jdbcOptions: JDBCOptions,
- additionalMetrics: Map[String, SQLMetric] = Map())
+ additionalMetrics: Map[String, SQLMetric] = Map(),
+ databaseMetadata: Option[JDBCDatabaseMetadata] = None)
extends ScanBuilder
with SupportsPushDownV2Filters
with SupportsPushDownRequiredColumns
@@ -375,7 +376,7 @@ case class JDBCScanBuilder(
// "DEPT","NAME",MAX("SALARY"),MIN("BONUS"), instead of getting column
names from
// prunedSchema and quote them (will become "MAX(SALARY)", "MIN(BONUS)"
and can't
// be used in sql string.
- JDBCScan(JDBCRelation(schema, parts, jdbcOptions,
additionalMetrics)(session),
+ JDBCScan(JDBCRelation(schema, parts, jdbcOptions,
additionalMetrics)(session, databaseMetadata),
finalSchema, pushedPredicate, pushedAggregateList, pushedGroupBys,
tableSampleClause, pushedLimit, sortOrders, pushedOffset)
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
index a242a24baf9c..02197ba3d9df 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
@@ -27,7 +27,7 @@ import
org.apache.spark.sql.connector.catalog.index.{SupportsIndex, TableIndex}
import org.apache.spark.sql.connector.expressions.NamedReference
import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
import org.apache.spark.sql.errors.DataTypeErrorsBase
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions,
JdbcOptionsInWrite, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata,
JDBCOptions, JdbcOptionsInWrite, JdbcUtils}
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.types.StructType
@@ -37,7 +37,8 @@ case class JDBCTable(
ident: Identifier,
override val schema: StructType,
jdbcOptions: JDBCOptions,
- additionalMetrics: Map[String, SQLMetric] = Map())
+ additionalMetrics: Map[String, SQLMetric] = Map(),
+ databaseMetadata: Option[JDBCDatabaseMetadata] = None)
extends Table
with SupportsRead
with SupportsWrite
@@ -61,7 +62,7 @@ case class JDBCTable(
override def newScanBuilder(options: CaseInsensitiveStringMap):
JDBCScanBuilder = {
val mergedOptions = new JDBCOptions(
jdbcOptions.parameters.originalMap ++
options.asCaseSensitiveMap().asScala)
- JDBCScanBuilder(SparkSession.active, schema, mergedOptions,
additionalMetrics)
+ JDBCScanBuilder(SparkSession.active, schema, mergedOptions,
additionalMetrics, databaseMetadata)
}
override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
index dc7009d7c6b4..ee1794b63c7f 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.connector.catalog.{CatalogV2Util,
Column, FunctionCa
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.errors.{DataTypeErrorsBase,
QueryCompilationErrors, QueryExecutionErrors}
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions,
JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCDatabaseMetadata,
JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
import org.apache.spark.sql.execution.metric.SQLMetrics
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
@@ -171,7 +171,8 @@ class JDBCTableCatalog extends TableCatalog
JDBCRDD.resolveTable(tableOptions, conn, Some(ident), Some(name()))
}
JDBCTable(ident, schema, tableOptions,
- Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric))
+ Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric),
+ Some(JDBCDatabaseMetadata.fromJDBCConnection(conn)))
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
index 49a354e412d2..52502683d20d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.jdbc
import java.sql.{Connection, DriverManager}
import java.util.{HexFormat, Properties}
import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.atomic.AtomicInteger
import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal
@@ -3350,6 +3351,38 @@ class JDBCV2Suite extends SharedSparkSession with
ExplainSuiteHelper {
assertResult(expectedMetadata) { jdbcRdd.getDatabaseMetadata }
}
+ test("SPARK-60507: JDBC read opens one driver connection") {
+ val driverConnections = new AtomicInteger()
+ val countingDialect = new H2Dialect() {
+ override def createConnectionFactory(options: JDBCOptions): Int =>
Connection = {
+ val factory = super.createConnectionFactory(options)
+ partitionId => {
+ if (partitionId == -1) driverConnections.incrementAndGet()
+ factory(partitionId)
+ }
+ }
+ }
+ JdbcDialects.unregisterDialect(h2Dialect)
+ try {
+ JdbcDialects.registerDialect(countingDialect)
+ val reads = Seq[(String, () => Unit)](
+ "v1" -> { () =>
+ val df = spark.read.jdbc(url, "\"test\".\"people\"", new
Properties())
+ df.count()
+ df.collect()
+ },
+ "v2" -> (() => sql("SELECT * FROM h2.test.people").collect()))
+ reads.foreach { case (name, read) =>
+ driverConnections.set(0)
+ read()
+ assert(driverConnections.get() == 1, s"$name read opened
${driverConnections.get()}")
+ }
+ } finally {
+ JdbcDialects.unregisterDialect(countingDialect)
+ JdbcDialects.registerDialect(h2Dialect)
+ }
+ }
+
test("SPARK-58782: null literal in aggregate should render as NULL not
'null'") {
val df = sql("SELECT NULLIF(s, '') AS g, COUNT(*) FROM
h2.test.null_literal GROUP BY g")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]