This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f4ec35647d35 [SPARK-60012][CONNECT][SQL] Avoid copying Ivy caches when
cloning ArtifactManager
f4ec35647d35 is described below
commit f4ec35647d35e346341f3f19807ddf87b71601b1
Author: qianlan717 <[email protected]>
AuthorDate: Thu Oct 8 09:55:24 2026 +0000
[SPARK-60012][CONNECT][SQL] Avoid copying Ivy caches when cloning
ArtifactManager
### What changes were proposed in this pull request?
Update `ArtifactManager.clone()` to copy the parent artifact directory
entries individually while excluding the top-level `.ivy-cache` directory.
Installed session artifacts, including resolved JARs, continue to be copied.
The cloned artifact manager creates its own Ivy cache lazily if it
subsequently performs Maven resolution. A regression test verifies that
installed JARs are retained, the parent cache is skipped, and resolution in the
clone creates a fresh cache.
### Why are the changes needed?
The per-session Maven resolver stores disposable Ivy state under
`artifactPath/.ivy-cache`. Recursively copying this cache into every cloned
session duplicates downloaded dependencies, increasing clone latency and disk
usage for large dependency trees.
### Does this PR introduce _any_ user-facing change?
Yes. Cloning a session after Maven dependency resolution no longer copies
the parent session's disposable Ivy cache. Installed artifacts remain
available, and later Maven resolution in the cloned session continues to work
with a fresh cache.
### How was this patch tested?
Added a regression test to `ArtifactManagerIvySettingsSuite`.
The following suites passed:
```
build/sbt 'sql/testOnly
org.apache.spark.sql.artifact.ArtifactManagerIvySettingsSuite'
build/sbt 'sql/testOnly org.apache.spark.sql.artifact.ArtifactManagerSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #59263 from qianlan717/SPARK-60012-cache.
Authored-by: qianlan717 <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../spark/sql/artifact/ArtifactManager.scala | 21 ++++++++++++--
.../artifact/ArtifactManagerIvySettingsSuite.scala | 32 ++++++++++++++++++++++
2 files changed, 51 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
index c82acef5818b..2a03a58502c6 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
@@ -71,9 +71,11 @@ class ArtifactManager(session: SparkSession) extends
AutoCloseable with Logging
(ArtifactUtils.concatenatePaths(artifactRootPath, session.sessionUUID),
s"$artifactRootURI/${session.sessionUUID}")
+ private val ivyCachePath = artifactPath.resolve(".ivy-cache")
+
private lazy val runtimeDependencyResolver = {
val sparkConf = session.sparkContext.getConf
- val ivyPath =
Files.createDirectories(artifactPath.resolve(".ivy-cache")).toString
+ val ivyPath = Files.createDirectories(ivyCachePath).toString
new RuntimeDependencyResolver(
ivySettingsPath = sparkConf.get(JAR_IVY_SETTING_PATH),
ivyPath = Some(ivyPath),
@@ -400,11 +402,26 @@ class ArtifactManager(session: SparkSession) extends
AutoCloseable with Logging
loader
}
+ private def copyArtifacts(targetPath: Path): Unit = {
+ Files.createDirectories(targetPath)
+ Utils.tryWithResource(Files.newDirectoryStream(artifactPath)) { entries =>
+ // The Ivy cache is session-local scratch data. A cloned session
recreates it lazily.
+ entries.asScala.filterNot(_ == ivyCachePath).foreach { source =>
+ val target = targetPath.resolve(source.getFileName)
+ if (Files.isDirectory(source)) {
+ Utils.copyDirectory(source.toFile, target.toFile)
+ } else {
+ Utils.copyFile(source.toFile, target.toFile)
+ }
+ }
+ }
+ }
+
private[sql] def clone(newSession: SparkSession): ArtifactManager = {
val sparkContext = session.sparkContext
val newArtifactManager = new ArtifactManager(newSession)
if (artifactPath.toFile.exists()) {
- Utils.copyDirectory(artifactPath.toFile,
newArtifactManager.artifactPath.toFile)
+ copyArtifacts(newArtifactManager.artifactPath)
}
// Share cached blocks with the cloned session by copying the references
and incrementing
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala
index 889e37c6afc8..efb57fd0e478 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerIvySettingsSuite.scala
@@ -84,4 +84,36 @@ class ArtifactManagerIvySettingsSuite extends
SharedSparkSession {
assert(spark.artifactManager.ivyConnectTimeoutMs == 7000)
assert(spark.artifactManager.ivyReadTimeoutMs == 11000)
}
+
+ test("SPARK-60012: cloning skips the Ivy cache and creates a fresh cache on
resolution") {
+ val ivyUri = URI.create(s"ivy://${coordinate.toString}")
+ val jarName =
s"${coordinate.groupId}_${coordinate.artifactId}-${coordinate.version}.jar"
+ val parentSession = spark.newSession()
+
+ try {
+ parentSession.addArtifact(ivyUri)
+ val parentArtifactPath = parentSession.artifactManager.artifactPath
+ val parentIvyCachePath = parentArtifactPath.resolve(".ivy-cache")
+ val parentCacheMarker = parentIvyCachePath.resolve("parent-only")
+ Files.writeString(parentCacheMarker, "parent cache")
+
+ val clonedSession = parentSession.cloneSession()
+ try {
+ val clonedArtifactPath = clonedSession.artifactManager.artifactPath
+ val clonedIvyCachePath = clonedArtifactPath.resolve(".ivy-cache")
+
+
assert(Files.exists(clonedArtifactPath.resolve("jars").resolve(jarName)))
+ assert(!Files.exists(clonedIvyCachePath))
+
+ clonedSession.addArtifact(ivyUri)
+
+ assert(Files.isDirectory(clonedIvyCachePath))
+
assert(!Files.exists(clonedIvyCachePath.resolve(parentCacheMarker.getFileName)))
+ } finally {
+ clonedSession.artifactManager.cleanUpResourcesForTesting()
+ }
+ } finally {
+ parentSession.artifactManager.cleanUpResourcesForTesting()
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
