This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7d30a7e0e047 [SPARK-59966][SQL] Only add explicitly configured remote
repositories with custom Ivy settings
7d30a7e0e047 is described below
commit 7d30a7e0e047040b4bf2ebd864fe23bdfd9a1de2
Author: qianlan717 <[email protected]>
AuthorDate: Tue Oct 6 10:41:51 2026 +0000
[SPARK-59966][SQL] Only add explicitly configured remote repositories with
custom Ivy settings
### What changes were proposed in this pull request?
This PR changes Hive metastore dependency resolution when a custom
`spark.jars.ivySettings` file is configured:
- Add `spark.sql.maven.additionalRemoteRepositories` only when the user
explicitly sets it.
- Preserve the existing effective default repository behavior when no
custom Ivy settings file
is used.
- Document the custom Ivy settings behavior.
- Add regression coverage for default and explicitly configured
repositories.
### Why are the changes needed?
`spark.sql.maven.additionalRemoteRepositories` has a default Google Cloud
Storage Maven mirror.
`IsolatedClientLoader` previously read the effective value unconditionally
and appended that
mirror even when users supplied a custom Ivy settings file.
Users who provide custom Ivy settings expect those settings to control
repository resolution.
Implicitly adding another repository can violate network or repository
policy and cause
unexpected outbound access.
### Does this PR introduce _any_ user-facing change?
Yes.
Previously, setting `spark.jars.ivySettings` implicitly appended the default
`spark.sql.maven.additionalRemoteRepositories` value, including the default
Google Cloud Storage
mirror.
After this change, a custom Ivy settings file is used without additional
remote repositories
unless `spark.sql.maven.additionalRemoteRepositories` is explicitly
configured. Behavior without
a custom Ivy settings file remains unchanged.
### How was this patch tested?
Added assertions to `IsolatedClientLoaderIvySettingsSuite` covering:
- Custom Ivy settings with no explicit additional repositories.
- Resolution without custom Ivy settings, which retains the effective
default.
- Explicit additional repositories with and without custom Ivy settings.
Ran:
```bash
build/sbt 'hive/testOnly
org.apache.spark.sql.hive.client.IsolatedClientLoaderIvySettingsSuite'
```
Result: 2 tests passed, 0 failed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #59220 from
qianlan717/qianlan-chen_data/hive-ivy-settings-remote-repos.
Authored-by: qianlan717 <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 3 ++-
.../sql/hive/client/IsolatedClientLoader.scala | 23 ++++++++++++++-----
.../IsolatedClientLoaderIvySettingsSuite.scala | 26 ++++++++++++++++++++++
3 files changed, 45 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 5801d8e399c4..0bfbb510fe21 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -6914,7 +6914,8 @@ object SQLConf {
buildConf("spark.sql.maven.additionalRemoteRepositories")
.doc("A comma-delimited string config of the optional additional remote
Maven mirror " +
"repositories. This is only used for downloading Hive jars in
IsolatedClientLoader " +
- "if the default Maven Central repo is unreachable.")
+ "if the default Maven Central repo is unreachable. When
spark.jars.ivySettings is " +
+ "set, the repositories are added only if this configuration is
explicitly set.")
.version("3.0.0")
.stringConf
.createWithDefault(
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index a1358d369d27..86c831a50949 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -57,8 +57,8 @@ private[hive] object IsolatedClientLoader extends Logging {
val files = if (resolvedVersions.contains((resolvedVersion,
hadoopVersion))) {
resolvedVersions((resolvedVersion, hadoopVersion))
} else {
- val remoteRepos = sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)
val ivySettingsPath =
sparkConf.getOption(MavenUtils.JAR_IVY_SETTING_PATH_KEY)
+ val remoteRepos = getRemoteRepos(sparkConf, ivySettingsPath)
val (downloadedFiles, actualHadoopVersion) =
try {
(downloadVersion(resolvedVersion, hadoopVersion, ivyPath,
ivySettingsPath, remoteRepos),
@@ -122,12 +122,23 @@ private[hive] object IsolatedClientLoader extends Logging
{
}
}
+ private[hive] def getRemoteRepos(
+ sparkConf: SparkConf,
+ ivySettingsPath: Option[String]): Option[String] = {
+ if (ivySettingsPath.isEmpty ||
+ sparkConf.contains(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)) {
+ Some(sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES))
+ } else {
+ None
+ }
+ }
+
private def downloadVersion(
version: HiveVersion,
hadoopVersion: String,
ivyPath: Option[String],
ivySettingsPath: Option[String],
- remoteRepos: String): Seq[URL] = {
+ remoteRepos: Option[String]): Seq[URL] = {
val hadoopJarNames = if (supportsHadoopShadedClient(hadoopVersion)) {
Seq(s"org.apache.hadoop:hadoop-client-api:$hadoopVersion",
s"org.apache.hadoop:hadoop-client-runtime:$hadoopVersion")
@@ -141,15 +152,15 @@ private[hive] object IsolatedClientLoader extends Logging
{
implicit val printStream: PrintStream = SparkSubmit.printStream
val ivySettings = ivySettingsPath match {
case Some(path) =>
- MavenUtils.loadIvySettings(path, Some(remoteRepos), ivyPath)
+ MavenUtils.loadIvySettings(path, remoteRepos, ivyPath)
case None =>
- MavenUtils.buildIvySettings(Some(remoteRepos), ivyPath)
+ MavenUtils.buildIvySettings(remoteRepos, ivyPath)
}
val noCacheIvySettings = ivySettingsPath match {
case Some(path) =>
- Some(MavenUtils.loadIvySettings(path, Some(remoteRepos), ivyPath))
+ Some(MavenUtils.loadIvySettings(path, remoteRepos, ivyPath))
case None =>
- Some(MavenUtils.buildIvySettings(Some(remoteRepos), ivyPath,
useLocalM2AsCache = false))
+ Some(MavenUtils.buildIvySettings(remoteRepos, ivyPath,
useLocalM2AsCache = false))
}
val classpaths = quietly {
MavenUtils.resolveMavenCoordinates(
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
index 265ced9065f4..72b1d413c8c2 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/IsolatedClientLoaderIvySettingsSuite.scala
@@ -23,11 +23,37 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.util.VersionInfo
import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.util.{MavenUtils, Utils}
class IsolatedClientLoaderIvySettingsSuite extends SparkFunSuite {
override protected val enableAutoThreadAudit = false
+ test("only add remote repositories explicitly configured with custom Ivy
settings") {
+ val sparkConf = new SparkConf()
+ val ivySettingsPath = Some("ivysettings.xml")
+
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
ivySettingsPath).isEmpty)
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf, None).contains(
+ sparkConf.get(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES)))
+
+ val remoteRepos = "https://repository.example.com/maven2/";
+ sparkConf.set(SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key, remoteRepos)
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
ivySettingsPath).contains(remoteRepos))
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
None).contains(remoteRepos))
+ }
+
+ test("substitute variables in remote repositories with custom Ivy settings")
{
+ val sparkConf = new SparkConf()
+ val remoteRepos = "https://repository.example.com/maven2/";
+ sparkConf.set("spark.test.remoteRepository", remoteRepos)
+ sparkConf.set(
+ SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.key,
"${spark.test.remoteRepository}")
+
+ assert(IsolatedClientLoader.getRemoteRepos(sparkConf,
Some("ivysettings.xml"))
+ .contains(remoteRepos))
+ }
+
test("SPARK-56867: respect spark.jars.ivySettings when downloading Hive
metastore jars") {
val ivyPath = Utils.createTempDir(namePrefix = "ivy-settings-test")
val ivySettingsFile = new File(ivyPath, "ivysettings.xml")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
