This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 7af0d94c1765 [SPARK-60984][CORE][TEST] Fix 
`HighlyCompressedMapStatusBenchmark` failure when `mockito` is not on the 
classpath
7af0d94c1765 is described below

commit 7af0d94c1765102a49c2e28da3d754686e0f641a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Nov 5 22:15:27 2026 -0800

    [SPARK-60984][CORE][TEST] Fix `HighlyCompressedMapStatusBenchmark` failure 
when `mockito` is not on the classpath
    
    ### What changes were proposed in this pull request?
    
    This PR removes the Mockito dependency from 
`HighlyCompressedMapStatusBenchmark`.
    
    - Instead of mocking `SparkEnv` for each case, the benchmark now starts a 
single `local[1]` `SparkContext` and changes 
`spark.shuffle.accurateBlockSkewedFactor` on `SparkEnv.get.conf` before each 
case. `HighlyCompressedMapStatus` reads this value from `SparkEnv.get.conf` on 
every call, so the behavior is the same as the mock.
    - The `SparkContext` is stopped before the "Driver heap retained by 
MapOutputTracker" report, which creates its own `SparkContext`.
    
    ### Why are the changes needed?
    
    The benchmark job fails with:
    
    ```
    java.lang.NoClassDefFoundError: org/mockito/Mockito
      at 
org.apache.spark.HighlyCompressedMapStatusBenchmark$.envWithSkewedFactor(HighlyCompressedMapStatusBenchmark.scala:52)
    ```
    
    
(https://github.com/dongjoon-hyun/spark/actions/runs/41684094207/job/127686502284)
    
    The GitHub Actions benchmark workflow runs `Benchmarks` through 
`spark-submit --jars <test jars>`. `mockito-core` is a test-scope dependency of 
`core`, so it is not on that classpath. `build/sbt "core/Test/runMain ..."` 
does include test dependencies, which is why the failure does not reproduce 
locally.
    
    The benchmark was added by SPARK-48290 (a1253377c4d, 2026-10-19). Other 
benchmarks do not depend on Mockito.
    - https://github.com/apache/spark/pull/57528
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This only changes a benchmark.
    
    ### How was this patch tested?
    
    Ran the benchmark locally and confirmed that it completes successfully and 
that the reported numbers (for example, the bytes retained per map task) are 
the same as before the change:
    
    ```
    build/sbt "core/Test/runMain 
org.apache.spark.HighlyCompressedMapStatusBenchmark"
    ```
    
    The benchmark has not been run through `spark-submit --jars` yet, so the 
GitHub Actions benchmark job needs to confirm the fix.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Sonnet 5.5
    
    Closes #60160 from dongjoon-hyun/SPARK-60984.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../spark/HighlyCompressedMapStatusBenchmark.scala | 68 +++++++++++-----------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git 
a/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala 
b/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
index 1e7bc69d0ee5..ed60cc65171c 100644
--- 
a/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
+++ 
b/core/src/test/scala/org/apache/spark/HighlyCompressedMapStatusBenchmark.scala
@@ -20,7 +20,6 @@ package org.apache.spark
 import java.io.PrintStream
 
 import org.apache.commons.io.output.TeeOutputStream
-import org.mockito.Mockito.{doReturn, mock}
 
 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
 import org.apache.spark.internal.config
@@ -46,12 +45,15 @@ object HighlyCompressedMapStatusBenchmark extends 
BenchmarkBase {
 
   private val loc = BlockManagerId("a", "host", 1000)
 
-  private def envWithSkewedFactor(skewedFactor: Double): SparkEnv = {
-    val conf = new SparkConf()
-      .set(config.SHUFFLE_ACCURATE_BLOCK_SKEWED_FACTOR.key, 
skewedFactor.toString)
-    val env = mock(classOf[SparkEnv])
-    doReturn(conf, Seq.empty: _*).when(env).conf
-    env
+  // HighlyCompressedMapStatus reads the skewed factor from SparkEnv.get.conf 
on every call.
+  private def setSkewedFactor(skewedFactor: Double): Unit = {
+    SparkEnv.get.conf.set(config.SHUFFLE_ACCURATE_BLOCK_SKEWED_FACTOR.key, 
skewedFactor.toString)
+  }
+
+  private def withLocalSparkContext(f: => Unit): Unit = {
+    val sc = new SparkContext(
+      new 
SparkConf().setMaster("local[1]").setAppName("HighlyCompressedMapStatusBenchmark"))
+    try f finally sc.stop()
   }
 
   private def skewedBlockSizes(numPartitions: Int): Array[Long] = {
@@ -88,10 +90,10 @@ object HighlyCompressedMapStatusBenchmark extends 
BenchmarkBase {
    * adaptive query execution plans on, so the lookup cost is paid 
numPartitions * numMapTasks
    * times per shuffle.
    */
-  private def runLookupBenchmark(envWithSkewedSizes: SparkEnv): Unit = {
+  private def runLookupBenchmark(): Unit = {
     val numMapTasks = 100
     Seq(2048, 50000).foreach { numPartitions =>
-      SparkEnv.set(envWithSkewedSizes)
+      setSkewedFactor(5.0)
       val sizes = tiedSkewedBlockSizes(numPartitions)
       val statuses =
         Array.tabulate(numMapTasks)(id => HighlyCompressedMapStatus(loc, 
sizes, id.toLong))
@@ -182,33 +184,33 @@ object HighlyCompressedMapStatusBenchmark extends 
BenchmarkBase {
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
-    val envWithoutSkewedSizes = envWithSkewedFactor(-1.0)
-    val envWithSkewedSizes = envWithSkewedFactor(5.0)
-    Seq("skewed" -> skewedBlockSizes _, "organ pipe" -> organPipeBlockSizes 
_).foreach {
-      case (distribution, blockSizes) =>
-        runBenchmark(s"Build HighlyCompressedMapStatus, $distribution block 
sizes") {
-          Seq(2048, 10000, 50000).foreach { numPartitions =>
-            val benchmark = new Benchmark(s"$numPartitions shuffle 
partitions", 1, output = output)
-            val sizes = blockSizes(numPartitions)
-
-            benchmark.addCase("skewed block sizes not recorded", 10) { _ =>
-              SparkEnv.set(envWithoutSkewedSizes)
-              HighlyCompressedMapStatus(loc, sizes, 0)
-            }
-
-            benchmark.addCase("skewed block sizes recorded accurately", 10) { 
_ =>
-              SparkEnv.set(envWithSkewedSizes)
-              HighlyCompressedMapStatus(loc, sizes, 0)
+    withLocalSparkContext {
+      Seq("skewed" -> skewedBlockSizes _, "organ pipe" -> organPipeBlockSizes 
_).foreach {
+        case (distribution, blockSizes) =>
+          runBenchmark(s"Build HighlyCompressedMapStatus, $distribution block 
sizes") {
+            Seq(2048, 10000, 50000).foreach { numPartitions =>
+              val benchmark =
+                new Benchmark(s"$numPartitions shuffle partitions", 1, output 
= output)
+              val sizes = blockSizes(numPartitions)
+
+              benchmark.addCase("skewed block sizes not recorded", 10) { _ =>
+                setSkewedFactor(-1.0)
+                HighlyCompressedMapStatus(loc, sizes, 0)
+              }
+
+              benchmark.addCase("skewed block sizes recorded accurately", 10) 
{ _ =>
+                setSkewedFactor(5.0)
+                HighlyCompressedMapStatus(loc, sizes, 0)
+              }
+
+              benchmark.run()
             }
-
-            benchmark.run()
           }
-        }
-    }
-    runBenchmark("Read block sizes out of HighlyCompressedMapStatus") {
-      runLookupBenchmark(envWithSkewedSizes)
+      }
+      runBenchmark("Read block sizes out of HighlyCompressedMapStatus") {
+        runLookupBenchmark()
+      }
     }
-    SparkEnv.set(null)
     runBenchmark("Driver heap retained by MapOutputTracker") {
       runDriverHeapReport()
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to