This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7af0d94c1765 [SPARK-60984][CORE][TEST] Fix
`HighlyCompressedMapStatusBenchmark` failure when `mockito` is not on the
classpath
7af0d94c1765 is described below
commit 7af0d94c1765102a49c2e28da3d754686e0f641a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Nov 5 22:15:27 2026 -0800
[SPARK-60984][CORE][TEST] Fix `HighlyCompressedMapStatusBenchmark` failure
when `mockito` is not on the classpath
### What changes were proposed in this pull request?
This PR removes the Mockito dependency from
`HighlyCompressedMapStatusBenchmark`.
- Instead of mocking `SparkEnv` for each case, the benchmark now starts a
single `local[1]` `SparkContext` and changes
`spark.shuffle.accurateBlockSkewedFactor` on `SparkEnv.get.conf` before each
case. `HighlyCompressedMapStatus` reads this value from `SparkEnv.get.conf` on
every call, so the behavior is the same as the mock.
- The `SparkContext` is stopped before the "Driver heap retained by
MapOutputTracker" report, which creates its own `SparkContext`.
### Why are the changes needed?
The benchmark job fails with:
```
java.lang.NoClassDefFoundError: org/mockito/Mockito
at
org.apache.spark.HighlyCompressedMapStatusBenchmark$.envWithSkewedFactor(HighlyCompressedMapStatusBenchmark.scala:52)
```
(https://github.com/dongjoon-hyun/spark/actions/runs/41684094207/job/127686502284)
The GitHub Actions benchmark workflow runs `Benchmarks` through
`spark-submit --jars <test jars>`. `mockito-core` is a test-scope dependency of
`core`, so it is not on that classpath. `build/sbt "core/Test/runMain ..."`
does include test dependencies, which is why the failure does not reproduce
locally.
The benchmark was added by SPARK-48290 (a1253377c4d, 2026-10-19). Other
benchmarks do not depend on Mockito.
- https://github.com/apache/spark/pull/57528
### Does this PR introduce _any_ user-facing change?
No. This only changes a benchmark.
### How was this patch tested?
Ran the benchmark locally and confirmed that it completes successfully and
that the reported numbers (for example, the bytes retained per map task) are
the same as before the change:
```
build/sbt "core/Test/runMain
org.apache.spark.HighlyCompressedMapStatusBenchmark"
```
The benchmark has not been run through `spark-submit --jars` yet, so the
GitHub Actions benchmark job needs to confirm the fix.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Sonnet 5.5
Closes #60160 from dongjoon-hyun/SPARK-60984.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/HighlyCompressedMapStatusBenchmark.scala | 68 +++++++++++-----------
1 file changed, 35 insertions(+), 33 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
b/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
index 1e7bc69d0ee5..ed60cc65171c 100644
---
a/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
+++
b/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
@@ -20,7 +20,6 @@ package org.apache.spark
import java.io.PrintStream
import org.apache.commons.io.output.TeeOutputStream
-import org.mockito.Mockito.{doReturn, mock}
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
import org.apache.spark.internal.config
@@ -46,12 +45,15 @@ object HighlyCompressedMapStatusBenchmark extends
BenchmarkBase {
private val loc = BlockManagerId("a", "host", 1000)
- private def envWithSkewedFactor(skewedFactor: Double): SparkEnv = {
- val conf = new SparkConf()
- .set(config.SHUFFLE_ACCURATE_BLOCK_SKEWED_FACTOR.key,
skewedFactor.toString)
- val env = mock(classOf[SparkEnv])
- doReturn(conf, Seq.empty: _*).when(env).conf
- env
+ // HighlyCompressedMapStatus reads the skewed factor from SparkEnv.get.conf
on every call.
+ private def setSkewedFactor(skewedFactor: Double): Unit = {
+ SparkEnv.get.conf.set(config.SHUFFLE_ACCURATE_BLOCK_SKEWED_FACTOR.key,
skewedFactor.toString)
+ }
+
+ private def withLocalSparkContext(f: => Unit): Unit = {
+ val sc = new SparkContext(
+ new
SparkConf().setMaster("local[1]").setAppName("HighlyCompressedMapStatusBenchmark"))
+ try f finally sc.stop()
}
private def skewedBlockSizes(numPartitions: Int): Array[Long] = {
@@ -88,10 +90,10 @@ object HighlyCompressedMapStatusBenchmark extends
BenchmarkBase {
* adaptive query execution plans on, so the lookup cost is paid
numPartitions * numMapTasks
* times per shuffle.
*/
- private def runLookupBenchmark(envWithSkewedSizes: SparkEnv): Unit = {
+ private def runLookupBenchmark(): Unit = {
val numMapTasks = 100
Seq(2048, 50000).foreach { numPartitions =>
- SparkEnv.set(envWithSkewedSizes)
+ setSkewedFactor(5.0)
val sizes = tiedSkewedBlockSizes(numPartitions)
val statuses =
Array.tabulate(numMapTasks)(id => HighlyCompressedMapStatus(loc,
sizes, id.toLong))
@@ -182,33 +184,33 @@ object HighlyCompressedMapStatusBenchmark extends
BenchmarkBase {
}
override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
- val envWithoutSkewedSizes = envWithSkewedFactor(-1.0)
- val envWithSkewedSizes = envWithSkewedFactor(5.0)
- Seq("skewed" -> skewedBlockSizes _, "organ pipe" -> organPipeBlockSizes
_).foreach {
- case (distribution, blockSizes) =>
- runBenchmark(s"Build HighlyCompressedMapStatus, $distribution block
sizes") {
- Seq(2048, 10000, 50000).foreach { numPartitions =>
- val benchmark = new Benchmark(s"$numPartitions shuffle
partitions", 1, output = output)
- val sizes = blockSizes(numPartitions)
-
- benchmark.addCase("skewed block sizes not recorded", 10) { _ =>
- SparkEnv.set(envWithoutSkewedSizes)
- HighlyCompressedMapStatus(loc, sizes, 0)
- }
-
- benchmark.addCase("skewed block sizes recorded accurately", 10) {
_ =>
- SparkEnv.set(envWithSkewedSizes)
- HighlyCompressedMapStatus(loc, sizes, 0)
+ withLocalSparkContext {
+ Seq("skewed" -> skewedBlockSizes _, "organ pipe" -> organPipeBlockSizes
_).foreach {
+ case (distribution, blockSizes) =>
+ runBenchmark(s"Build HighlyCompressedMapStatus, $distribution block
sizes") {
+ Seq(2048, 10000, 50000).foreach { numPartitions =>
+ val benchmark =
+ new Benchmark(s"$numPartitions shuffle partitions", 1, output
= output)
+ val sizes = blockSizes(numPartitions)
+
+ benchmark.addCase("skewed block sizes not recorded", 10) { _ =>
+ setSkewedFactor(-1.0)
+ HighlyCompressedMapStatus(loc, sizes, 0)
+ }
+
+ benchmark.addCase("skewed block sizes recorded accurately", 10)
{ _ =>
+ setSkewedFactor(5.0)
+ HighlyCompressedMapStatus(loc, sizes, 0)
+ }
+
+ benchmark.run()
}
-
- benchmark.run()
}
- }
- }
- runBenchmark("Read block sizes out of HighlyCompressedMapStatus") {
- runLookupBenchmark(envWithSkewedSizes)
+ }
+ runBenchmark("Read block sizes out of HighlyCompressedMapStatus") {
+ runLookupBenchmark()
+ }
}
- SparkEnv.set(null)
runBenchmark("Driver heap retained by MapOutputTracker") {
runDriverHeapReport()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]