This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 8afa4418ad16 [SPARK-59966][SQL] Only add explicitly configured remote
repositories with custom Ivy settings
8afa4418ad16 is described below
commit 8afa4418ad16162a310d1d3d81ad66a57790b95e
Author: qianlan717 <[email protected]>
AuthorDate: Tue Oct 6 10:41:51 2026 +0000
[SPARK-59966][SQL] Only add explicitly configured remote repositories with
custom Ivy settings
### What changes were proposed in this pull request?
This PR changes Hive metastore dependency resolution when a custom
`spark.jars.ivySettings` file is configured:
- Add `spark.sql.maven.additionalRemoteRepositories` only when the user
explicitly sets it.
- Preserve the existing effective default repository behavior when no
custom Ivy settings file
is used.
- Document the custom Ivy settings behavior.
- Add regression coverage for default and explicitly configured
repositories.
### Why are the changes needed?
`spark.sql.maven.additionalRemoteRepositories` has a default Google Cloud
Storage Maven mirror.
`IsolatedClientLoader` previously read the effective value unconditionally
and appended that
mirror even when users supplied a custom Ivy settings file.
Users who provide custom Ivy settings expect those settings to control
repository resolution.
Implicitly adding another repository can violate network or repository
policy and cause
unexpected outbound access.
### Does this PR introduce _any_ user-facing change?
Yes.
Previously, setting `spark.jars.ivySettings` implicitly appended the default
`spark.sql.maven.additionalRemoteRepositories` value, including the default
Google Cloud Storage
mirror.
After this change, a custom Ivy settings file is used without additional
remote repositories
unless `spark.sql.maven.additionalRemoteRepositories` is explicitly
configured. Behavior without
a custom Ivy settings file remains unchanged.
### How was this patch tested?
Added assertions to `IsolatedClientLoaderIvySettingsSuite` covering:
- Custom Ivy settings with no explicit additional repositories.
- Resolution without custom Ivy settings, which retains the effective
default.
- Explicit additional repositories with and without custom Ivy settings.
Ran:
```bash
build/sbt 'hive/testOnly
org.apache.spark.sql.hive.client.IsolatedClientLoaderIvySettingsSuite'
```
Result: 2 tests passed, 0 failed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #59220 from
qianlan717/qianlan-chen_data/hive-ivy-settings-remote-repos.
Authored-by: qianlan717 <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
(cherry picked from commit 7d30a7e0e047040b4bf2ebd864fe23bdfd9a1de2)
Signed-off-by: Uros Bojanic <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 3 ++-
.../sql/hive/client/IsolatedClientLoader.scala | 23 ++++++++++++++-----
.../IsolatedClientLoaderIvySettingsSuite.scala | 26 ++++++++++++++++++++++
3 files changed, 45 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index ce88b88bdf4c..5ec5479683b0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -6913,7 +6913,8 @@ object SQLConf {
buildConf("spark.sql.maven.additionalRemoteRepositories")
.doc("A comma-delimited string config of the optional additional remote
Maven mirror " +
"repositories. This is only used for downloading Hive jars in
IsolatedClientLoader " +
- "if the default Maven Central repo is unreachable.")
+ "if the default Maven Central repo is unreachable. When
spark.jars.ivySettings is " +
+ "set, the repositories are added only if this configuration is
explicitly set.")
.version("3.0.0")
.stringConf
.createWithDefault(
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 950f7bc752a8..fcd3c5834f82 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -57,8 +57,8 @@ private[hive] object IsolatedClientLoader extends Logging {
val files = if (resolvedVersions.contains((resolvedVersion,
hadoopVersion))) {
resolvedVersions((resolvedVersion, hadoopVersion))
} else {
- val remoteRepos = sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)
val ivySettingsPath =
sparkConf.getOption(MavenUtils.JAR_IVY_SETTING_PATH_KEY)
+ val remoteRepos = getRemoteRepos(sparkConf, ivySettingsPath)
val (downloadedFiles, actualHadoopVersion) =
try {
(downloadVersion(resolvedVersion, hadoopVersion, ivyPath,
ivySettingsPath, remoteRepos),
@@ -121,12 +121,23 @@ private[hive] object IsolatedClientLoader extends Logging
{
}
}
+ private[hive] def getRemoteRepos(
+ sparkConf: SparkConf,
+ ivySettingsPath: Option[String]): Option[String] = {
+ if (ivySettingsPath.isEmpty ||
+ sparkConf.contains(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)) {
+ Some(sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES))
+ } else {
+ None
+ }
+ }
+
private def downloadVersion(
version: HiveVersion,
hadoopVersion: String,
ivyPath: Option[String],
ivySettingsPath: Option[String],
- remoteRepos: String): Seq[URL] = {
+ remoteRepos: Option[String]): Seq[URL] = {
val hadoopJarNames = if (supportsHadoopShadedClient(hadoopVersion)) {
Seq(s"org.apache.hadoop:hadoop-client-api:$hadoopVersion",
s"org.apache.hadoop:hadoop-client-runtime:$hadoopVersion")
@@ -140,15 +151,15 @@ private[hive] object IsolatedClientLoader extends Logging
{
implicit val printStream: PrintStream = SparkSubmit.printStream
val ivySettings = ivySettingsPath match {
case Some(path) =>
- MavenUtils.loadIvySettings(path, Some(remoteRepos), ivyPath)
+ MavenUtils.loadIvySettings(path, remoteRepos, ivyPath)
case None =>
- MavenUtils.buildIvySettings(Some(remoteRepos), ivyPath)
+ MavenUtils.buildIvySettings(remoteRepos, ivyPath)
}
val noCacheIvySettings = ivySettingsPath match {
case Some(path) =>
- Some(MavenUtils.loadIvySettings(path, Some(remoteRepos), ivyPath))
+ Some(MavenUtils.loadIvySettings(path, remoteRepos, ivyPath))
case None =>
- Some(MavenUtils.buildIvySettings(Some(remoteRepos), ivyPath,
useLocalM2AsCache = false))
+ Some(MavenUtils.buildIvySettings(remoteRepos, ivyPath,
useLocalM2AsCache = false))
}
val classpaths = quietly {
MavenUtils.resolveMavenCoordinates(
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
index 265ced9065f4..72b1d413c8c2 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
@@ -23,11 +23,37 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.util.VersionInfo
import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.util.{MavenUtils, Utils}
class IsolatedClientLoaderIvySettingsSuite extends SparkFunSuite {
override protected val enableAutoThreadAudit = false
+ test("only add remote repositories explicitly configured with custom Ivy
settings") {
+ val sparkConf = new SparkConf()
+ val ivySettingsPath = Some("ivysettings.xml")
+
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
ivySettingsPath).isEmpty)
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf, None).contains(
+ sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)))
+
+ val remoteRepos = "https://repository.example.com/maven2/";
+ sparkConf.set(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key, remoteRepos)
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
ivySettingsPath).contains(remoteRepos))
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
None).contains(remoteRepos))
+ }
+
+ test("substitute variables in remote repositories with custom Ivy settings")
{
+ val sparkConf = new SparkConf()
+ val remoteRepos = "https://repository.example.com/maven2/";
+ sparkConf.set("spark.test.remoteRepository", remoteRepos)
+ sparkConf.set(
+ SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key,
"${spark.test.remoteRepository}")
+
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
Some("ivysettings.xml"))
+ .contains(remoteRepos))
+ }
+
test("SPARK-56867: respect spark.jars.ivySettings when downloading Hive
metastore jars") {
val ivyPath = Utils.createTempDir(namePrefix = "ivy-settings-test")
val ivySettingsFile = new File(ivyPath, "ivysettings.xml")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
