This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 07d9f341b3c1 [SPARK-55707][SQL][TEST] Re-enable DB2 JDBC tests by
upgrading jcc to 12.1.4.0
07d9f341b3c1 is described below
commit 07d9f341b3c1bfba7a279e4b31cd30625d0eeeb5
Author: Wenchen Fan <[email protected]>
AuthorDate: Tue Jun 16 17:20:28 2026 -0700
[SPARK-55707][SQL][TEST] Re-enable DB2 JDBC tests by upgrading jcc to
12.1.4.0
SPARK-55706 disabled the DB2 JDBC driver tests because `com.ibm.db2:jcc`
11.5.x
bundles an old, unshaded copy of lz4-java (`net.jpountz.*`) that predates
`LZ4BlockInputStream.newBuilder()`. On the shared test classpath it shadows
the
lz4-java Spark ships and breaks the builder API adopted in SPARK-54571.
The jcc 12.1.x driver bundles a newer lz4-java that already provides the
full
`newBuilder()` builder API (including the `LZ4SafeDecompressor` overload),
so the
conflict no longer occurs. This:
- bumps `db2.jcc.version` 11.5.9.0 -> 12.1.4.0, and
- reverts the test-disabling changes from SPARK-55706.
To restore the DB2 JDBC test coverage disabled in SPARK-55706.
No. `com.ibm.db2:jcc` is a test-only dependency.
The DB2 integration / connection-provider suites disabled by SPARK-55706 are
re-enabled. The 12.1.x driver is backward-compatible with the DB2 11 test
container.
This pull request and its description were written by Isaac.
Closes #56534 from cloud-fan/SPARK-55707-shade-db2-jcc-lz4.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 2fb4a1bf2b1b504d5211c7f681fe774587db5b09)
Signed-off-by: Wenchen Fan <[email protected]>
---
connector/docker-integration-tests/pom.xml | 5 ++---
.../apache/spark/sql/jdbc/DB2IntegrationSuite.scala | 5 ++---
.../spark/sql/jdbc/DB2KrbIntegrationSuite.scala | 2 --
.../spark/sql/jdbc/v2/DB2IntegrationSuite.scala | 5 ++---
.../apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala | 9 ++++-----
pom.xml | 7 +++----
sql/core/pom.xml | 5 ++---
.../jdbc/connection/ConnectionProviderSuite.scala | 20 ++++++++++----------
.../jdbc/connection/DB2ConnectionProviderSuite.scala | 3 ---
.../scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 10 +++-------
10 files changed, 28 insertions(+), 43 deletions(-)
diff --git a/connector/docker-integration-tests/pom.xml
b/connector/docker-integration-tests/pom.xml
index 1c99c5494195..7d3f129d2805 100644
--- a/connector/docker-integration-tests/pom.xml
+++ b/connector/docker-integration-tests/pom.xml
@@ -100,12 +100,11 @@
<artifactId>ojdbc17</artifactId>
<scope>test</scope>
</dependency>
- <!-- TODO(SPARK-55707: Re-enable DB2 JDBC Driver tests) -->
- <!--dependency>
+ <dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>test</scope>
- </dependency-->
+ </dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index 39aef6d7c90b..6a489ffb2d42 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -21,13 +21,12 @@ import java.math.BigDecimal
import java.sql.{Connection, Date, Timestamp}
import java.util.Properties
-import org.scalatest.Ignore
-
import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{ByteType, ShortType, StructType}
+import org.apache.spark.tags.DockerTest
/**
* To run this test suite for a specific version (e.g.,
icr.io/db2_community/db2:11.5.9.0):
@@ -37,7 +36,7 @@ import org.apache.spark.sql.types.{ByteType, ShortType,
StructType}
* "docker-integration-tests/testOnly
org.apache.spark.sql.jdbc.DB2IntegrationSuite"
* }}}
*/
-@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
+@DockerTest
class DB2IntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new DB2DatabaseOnDocker
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
index a34c1f5590a0..c7d8fc43393e 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala
@@ -24,7 +24,6 @@ import javax.security.auth.login.Configuration
import com.github.dockerjava.api.model.{AccessMode, Bind, ContainerConfig,
HostConfig, Volume}
import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation}
import
org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS
-import org.scalatest.Ignore
import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
import
org.apache.spark.sql.execution.datasources.jdbc.connection.{DB2ConnectionProvider,
SecureConnectionProvider}
@@ -38,7 +37,6 @@ import org.apache.spark.tags.DockerTest
* "docker-integration-tests/testOnly *DB2KrbIntegrationSuite"
* }}}
*/
-@Ignore // TODO(SPARK-55707: Re-enable DB2 JDBC Driver tests)
@DockerTest
class DB2KrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
override protected val userName = s"db2/$dockerIp"
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
index e0b937a61fec..ecc02f705178 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala
@@ -20,13 +20,12 @@ package org.apache.spark.sql.jdbc.v2
import java.sql.Connection
import java.util.Locale
-import org.scalatest.Ignore
-
import org.apache.spark.SparkConf
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
import org.apache.spark.sql.jdbc.DB2DatabaseOnDocker
import org.apache.spark.sql.types._
+import org.apache.spark.tags.DockerTest
/**
* To run this test suite for a specific version (e.g.,
icr.io/db2_community/db2:11.5.9.0):
@@ -35,7 +34,7 @@ import org.apache.spark.sql.types._
* ./build/sbt -Pdocker-integration-tests "testOnly
*v2.DB2IntegrationSuite"
* }}}
*/
-@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
+@DockerTest
class DB2IntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest
{
// Following tests are disabled for both single and multiple partition read
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala
index d96f4bf7011e..52ef902d9c00 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala
@@ -21,8 +21,6 @@ import java.sql.Connection
import scala.jdk.CollectionConverters._
-import org.scalatest.Ignore
-
import org.apache.spark.sql.jdbc.{DB2DatabaseOnDocker,
DockerJDBCIntegrationSuite}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.tags.DockerTest
@@ -34,7 +32,6 @@ import org.apache.spark.tags.DockerTest
* ./build/sbt -Pdocker-integration-tests "testOnly *v2.DB2NamespaceSuite"
* }}}
*/
-@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
@DockerTest
class DB2NamespaceSuite extends DockerJDBCIntegrationSuite with
V2JDBCNamespaceTest {
override val db = new DB2DatabaseOnDocker
@@ -42,8 +39,10 @@ class DB2NamespaceSuite extends DockerJDBCIntegrationSuite
with V2JDBCNamespaceT
Map("url" -> db.getJdbcUrl(dockerIp, externalPort),
"driver" -> "com.ibm.db2.jcc.DB2Driver").asJava)
- // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
- // catalog.initialize("db2", map)
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ catalog.initialize("db2", map)
+ }
override def dataPreparation(conn: Connection): Unit = {}
diff --git a/pom.xml b/pom.xml
index d2d4f53e6da1..9f1665d342b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -348,7 +348,7 @@
<mariadb.java.client.version>3.5.7</mariadb.java.client.version>
<mysql.connector.version>9.6.0</mysql.connector.version>
<postgresql.version>42.7.10</postgresql.version>
- <db2.jcc.version>11.5.9.0</db2.jcc.version>
+ <db2.jcc.version>12.1.4.0</db2.jcc.version>
<mssql.jdbc.version>13.2.1.jre11</mssql.jdbc.version>
<ojdbc17.version>23.26.1.0.0</ojdbc17.version>
<databricks.jdbc.version>3.3.1</databricks.jdbc.version>
@@ -1392,13 +1392,12 @@
<version>${postgresql.version}</version>
<scope>test</scope>
</dependency>
- <!-- TODO(SPARK-55707: Re-enable DB2 JDBC Driver tests) -->
- <!--dependency>
+ <dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<version>${db2.jcc.version}</version>
<scope>test</scope>
- </dependency-->
+ </dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 19d23b2cbc30..b545b516b05e 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -211,12 +211,11 @@
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
- <!-- TODO(SPARK-55707: Re-enable DB2 JDBC Driver tests) -->
- <!--dependency>
+ <dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<scope>test</scope>
- </dependency-->
+ </dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
index b2e788b52109..0d7b133f0e15 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala
@@ -185,28 +185,28 @@ class ConnectionProviderSuite
val postgresDriver = registerDriver(postgresProvider.driverClass)
val postgresOptions = options("jdbc:postgresql://localhost/postgres")
val postgresAppEntry = postgresProvider.appEntry(postgresDriver,
postgresOptions)
- val mysqlProvider = new MariaDBConnectionProvider()
- val mysqlDriver = registerDriver(mysqlProvider.driverClass)
- val mysqlOptions = options("jdbc:mysql://localhost/db")
- val mysqlAppEntry = mysqlProvider.appEntry(mysqlDriver, mysqlOptions)
+ val db2Provider = new DB2ConnectionProvider()
+ val db2Driver = registerDriver(db2Provider.driverClass)
+ val db2Options = options("jdbc:db2://localhost/db2")
+ val db2AppEntry = db2Provider.appEntry(db2Driver, db2Options)
// Make sure no authentication for the databases are set
val rootConfig = Configuration.getConfiguration
assert(rootConfig.getAppConfigurationEntry(postgresAppEntry) == null)
- assert(rootConfig.getAppConfigurationEntry(mysqlAppEntry) == null)
+ assert(rootConfig.getAppConfigurationEntry(db2AppEntry) == null)
postgresProvider.setAuthenticationConfig(postgresDriver, postgresOptions)
val postgresConfig = Configuration.getConfiguration
- mysqlProvider.setAuthenticationConfig(mysqlDriver, mysqlOptions)
- val mysqlConfig = Configuration.getConfiguration
+ db2Provider.setAuthenticationConfig(db2Driver, db2Options)
+ val db2Config = Configuration.getConfiguration
// Make sure authentication for the databases are set
assert(rootConfig != postgresConfig)
- assert(rootConfig != mysqlConfig)
+ assert(rootConfig != db2Config)
// The topmost config in the chain is linked with all the subsequent
entries
- assert(mysqlConfig.getAppConfigurationEntry(postgresAppEntry) != null)
- assert(mysqlConfig.getAppConfigurationEntry(mysqlAppEntry) != null)
+ assert(db2Config.getAppConfigurationEntry(postgresAppEntry) != null)
+ assert(db2Config.getAppConfigurationEntry(db2AppEntry) != null)
Configuration.setConfiguration(null)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
index f29a1b1e7ab6..895b3d85d960 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala
@@ -17,9 +17,6 @@
package org.apache.spark.sql.execution.datasources.jdbc.connection
-import org.scalatest.Ignore
-
-@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
class DB2ConnectionProviderSuite extends ConnectionProviderSuiteBase {
test("setAuthenticationConfig must set authentication all the time") {
val provider = new DB2ConnectionProvider()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 3a83a2549f92..3c04c517a670 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1265,8 +1265,7 @@ class JDBCSuite extends SharedSparkSession {
"SELECT TOP (123) a,b FROM test")
}
- // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
- ignore("SPARK-42534: DB2Dialect Limit query test") {
+ test("SPARK-42534: DB2Dialect Limit query test") {
// JDBC url is a required option but is not used in this test.
val options = new JDBCOptions(Map("url" -> "jdbc:db2://host:port",
"dbtable" -> "test"))
assert(
@@ -2439,9 +2438,7 @@ class JDBCSuite extends SharedSparkSession {
}
// not supported
Seq(
- // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
- // "jdbc:db2://host:port",
- "jdbc:derby:memory", "jdbc:h2://host:port",
+ "jdbc:db2://host:port", "jdbc:derby:memory", "jdbc:h2://host:port",
"jdbc:sqlserver://host:port", "jdbc:postgresql://host:5432/postgres",
"jdbc:snowflake://host:443?account=test",
"jdbc:teradata://host:port").foreach { url =>
val options = new JDBCOptions(baseParameters + ("url" -> url))
@@ -2462,8 +2459,7 @@ class JDBCSuite extends SharedSparkSession {
"jdbc:mysql",
"jdbc:postgresql",
"jdbc:sqlserver",
- // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests
- // "jdbc:db2",
+ "jdbc:db2",
"jdbc:h2",
"jdbc:teradata",
"jdbc:databricks"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
