This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6f79faadc016 [SPARK-44977][SPARK-46257][BUILD][CORE][SQL] Upgrade
Derby to 10.16.1.1
6f79faadc016 is described below
commit 6f79faadc0166eb357df00fafd84e5bc24558cb4
Author: yangjie01 <[email protected]>
AuthorDate: Wed Dec 6 11:14:24 2023 +0800
[SPARK-44977][SPARK-46257][BUILD][CORE][SQL] Upgrade Derby to 10.16.1.1
### What changes were proposed in this pull request?
This PR aims to upgrade Derby from 10.14.2.0 to 10.16.1.1：
1. The version in `pom.xml` and related comments were updated. After the
update, Spark added a dependency on the `org.apache.derby:derbyshared` module
because it is a cascading dependency of `derby`.
2. Since `org.apache.derby.jdbc.EmbeddedDriver` has been moved to the
`org.apache.derby:derbytools` module, the corresponding dependency was added to
the project.
3. Since `org.apache.derby.jdbc.AutoloadedDriver` has been moved to
`org.apache.derby.iapi.jdbc.AutoloadedDriver`, the corresponding test
configuration in `o.a.spark.sql.hive.thriftserver.CliSuite` was modified.
### Why are the changes needed?
Derby 10.16 release family [supports Java
17](https://issues.apache.org/jira/browse/DERBY-7137) and higher with JDBC 4.2,
and this version [no longer supports the Java
SecurityManager](https://issues.apache.org/jira/browse/DERBY-7138).
The full release notes as follows:
- https://db.apache.org/derby/releases/release-10_16_1_1.cgi
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GitHub Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #44174 from LuciferYang/SPARK-46257.
Lead-authored-by: yangjie01 <[email protected]>
Co-authored-by: YangJie <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
core/pom.xml | 5 +++++
dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 +++-
pom.xml | 9 +++++++--
sql/core/pom.xml | 5 +++++
.../scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +-
sql/hive/pom.xml | 4 ++++
6 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 4cb0fe055fd0..f2d1f4eb144a 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -347,6 +347,11 @@
<artifactId>derby</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derbytools</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index ebfe6acad960..ba7c7aee894a 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -59,7 +59,9 @@ datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
datasketches-java/3.3.0//datasketches-java-3.3.0.jar
datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar
-derby/10.14.2.0//derby-10.14.2.0.jar
+derby/10.16.1.1//derby-10.16.1.1.jar
+derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
+derbytools/10.16.1.1//derbytools-10.16.1.1.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar
gcs-connector/hadoop3-2.2.18/shaded/gcs-connector-hadoop3-2.2.18-shaded.jar
diff --git a/pom.xml b/pom.xml
index 27ee42f103dd..33cb6cf257f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -138,8 +138,8 @@
<hive.version.short>2.3</hive.version.short>
<!-- note that this should be compatible with Kafka brokers version 0.10
and up -->
<kafka.version>3.4.1</kafka.version>
- <!-- After 10.15.1.3, the minimum required version is JDK9 -->
- <derby.version>10.14.2.0</derby.version>
+ <!-- After 10.17.1.0, the minimum required version is JDK19 -->
+ <derby.version>10.16.1.1</derby.version>
<parquet.version>1.13.1</parquet.version>
<orc.version>1.9.2</orc.version>
<orc.classifier>shaded-protobuf</orc.classifier>
@@ -962,6 +962,11 @@
<artifactId>derby</artifactId>
<version>${derby.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derbytools</artifactId>
+ <version>${derby.version}</version>
+ </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 185d50c018c6..c83104a6bd45 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -194,6 +194,11 @@
<artifactId>derby</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derbytools</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 649c985cade9..bbcbf80126b0 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -812,7 +812,7 @@ class CliSuite extends SparkFunSuite {
val catalogUrl =
s"spark.sql.catalog.$catalogName.url=jdbc:derby:memory:$catalogName;create=true"
val catalogDriver =
-
s"spark.sql.catalog.$catalogName.driver=org.apache.derby.jdbc.AutoloadedDriver"
+
s"spark.sql.catalog.$catalogName.driver=org.apache.derby.iapi.jdbc.AutoloadedDriver"
val catalogConfigs =
Seq(catalogImpl, catalogDriver, catalogUrl,
"spark.sql.catalogImplementation=in-memory")
.flatMap(Seq("--conf", _))
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 8b8842333793..42618fe5bc22 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -172,6 +172,10 @@
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derbytools</artifactId>
+ </dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
