This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 511f9d5b6747 [SPARK-54449][CORE] Storage Memory off heap size should 
be considered only when off heap is enabled
511f9d5b6747 is described below

commit 511f9d5b6747c719b68ed8238f320a07b53476d8
Author: VindhyaG <[email protected]>
AuthorDate: Tue Nov 3 14:31:15 2026 -0800

    [SPARK-54449][CORE] Storage Memory off heap size should be considered only 
when off heap is enabled
    
    ### What changes were proposed in this pull request?
    
    Currently,  even if spark.memory.offHeap.enabled=false
    spark.memory.offHeap.size is calculated for SPARK UI and wherever the 
config is used. This PR changes the logic to take the value set in 
spark.memory.offHeap.size only when spark.memory.offHeap.enabled
    
    ### Why are the changes needed?
    
    Change the logic to get the config of spark.memory.offHeap.size  by 
checking spark.memory.offHeap.enabled  first
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Spark UI will show the total of Onheap and Off heap memory only if 
spark.memory.offHeap.enabled = true.
    
    ### How was this patch tested?
    
    Existing UTs
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #53383 from VindhyaG/SPARK-54449.
    
    Authored-by: VindhyaG <[email protected]>
    Signed-off-by: Holden Karau <[email protected]>
---
 .../org/apache/spark/memory/MemoryManager.scala    |  6 ++++-
 .../apache/spark/status/AppStatusListener.scala    |  6 ++++-
 .../executor_memory_usage_expectation.json         |  2 +-
 ...executor_node_excludeOnFailure_expectation.json |  2 +-
 .../internal/plugin/PluginContainerSuite.scala     | 14 ++++++++++-
 .../apache/spark/memory/MemoryManagerSuite.scala   | 28 ++++++++++++++++++----
 .../spark/memory/UnifiedMemoryManagerSuite.scala   |  4 +++-
 .../spark/status/AppStatusListenerSuite.scala      |  1 +
 .../storage/BlockManagerReplicationSuite.scala     |  8 +++++--
 .../apache/spark/storage/BlockManagerSuite.scala   |  3 +++
 .../org/apache/spark/ui/UISeleniumSuite.scala      |  3 ++-
 11 files changed, 64 insertions(+), 13 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala 
b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
index dac82cc76357..88d973756e21 100644
--- a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
@@ -58,7 +58,8 @@ private[spark] abstract class MemoryManager(
   onHeapStorageMemoryPool.incrementPoolSize(onHeapStorageMemory)
   onHeapExecutionMemoryPool.incrementPoolSize(onHeapExecutionMemory)
 
-  protected[this] val maxOffHeapMemory = conf.get(MEMORY_OFFHEAP_SIZE)
+  protected[this] val maxOffHeapMemory =
+    if (conf.get(MEMORY_OFFHEAP_ENABLED))  conf.get(MEMORY_OFFHEAP_SIZE) else 0
   protected[this] val offHeapStorageMemory =
     (maxOffHeapMemory * conf.get(MEMORY_STORAGE_FRACTION)).toLong
 
@@ -241,6 +242,9 @@ private[spark] abstract class MemoryManager(
         "No support for unaligned Unsafe. Set spark.memory.offHeap.enabled to 
false.")
       MemoryMode.OFF_HEAP
     } else {
+      if (conf.get(MEMORY_OFFHEAP_SIZE) > 0) {
+        logWarning(s"spark.memory.offHeap.size is > 0 when 
spark.memory.offHeap.enabled == false")
+      }
       MemoryMode.ON_HEAP
     }
   }
diff --git 
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala 
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 87eed5c6b2c5..a0dc75fc14fc 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -912,7 +912,11 @@ private[spark] class AppStatusListener(
     exec.hostPort = event.blockManagerId.hostPort
     event.maxOnHeapMem.foreach { _ =>
       exec.totalOnHeap = event.maxOnHeapMem.get
-      exec.totalOffHeap = event.maxOffHeapMem.get
+      if (event.blockManagerId.executorId == SparkContext.DRIVER_IDENTIFIER) {
+        exec.totalOffHeap = 0
+      } else {
+        exec.totalOffHeap = event.maxOffHeapMem.get
+      }
       // SPARK-30594: whenever(first time or re-register) a BlockManager 
added, all blocks
       // from this BlockManager will be reported to driver later. So, we 
should clean up
       // used memory to avoid overlapped count.
diff --git 
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
 
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index fc63185b4c4a..999f7ff5fc1a 100644
--- 
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ 
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
   "maxMemory" : 908381388,
   "maxTasks" : 0,
   "memoryMetrics" : {
-    "totalOffHeapStorageMemory" : 524288000,
+    "totalOffHeapStorageMemory" : 0,
     "totalOnHeapStorageMemory" : 384093388,
     "usedOffHeapStorageMemory" : 0,
     "usedOnHeapStorageMemory" : 0
diff --git 
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
 
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
index fc63185b4c4a..999f7ff5fc1a 100644
--- 
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
+++ 
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -16,7 +16,7 @@
   "maxMemory" : 908381388,
   "maxTasks" : 0,
   "memoryMetrics" : {
-    "totalOffHeapStorageMemory" : 524288000,
+    "totalOffHeapStorageMemory" : 0,
     "totalOnHeapStorageMemory" : 384093388,
     "usedOffHeapStorageMemory" : 0,
     "usedOnHeapStorageMemory" : 0
diff --git 
a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
 
b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
index 99165585aefa..8718a1c6119b 100644
--- 
a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
@@ -272,7 +272,19 @@ class PluginContainerSuite extends SparkFunSuite with 
LocalSparkContext {
           .get(ResourceProfile.OFFHEAP_MEM).map(_.amount).getOrElse(-1L))
 
       // Ensure all executors has started
-      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 120000)
+      Thread.sleep(10000)
+
+      // Check executor memory is also updated for all executors except driver
+      val executorInfos = sc.statusTracker.getExecutorInfos
+      val execWithMemory = executorInfos.filter(_.totalOffHeapStorageMemory() 
> 0)
+
+      assert(execWithMemory.length >= 2,
+       s"Expected atleast 2 executors with memory, got 
${execWithMemory.length}")
+      execWithMemory.foreach { exec =>
+        assert(exec.totalOffHeapStorageMemory() == 
MemoryOverridePlugin.offHeapMemory)
+      }
+
 
       // Check executor memory is also updated
       eventually(timeout(10.seconds), interval(100.milliseconds)) {
diff --git 
a/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala 
b/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
index 1f40ef944a84..63e92d0cefda 100644
--- a/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
@@ -143,7 +143,9 @@ private[memory] trait MemoryManagerSuite extends 
SparkFunSuite {
    */
   protected def createMemoryManager(
      maxOnHeapExecutionMemory: Long,
-     maxOffHeapExecutionMemory: Long = 0L): MemoryManager
+     maxOffHeapExecutionMemory: Long = 0L,
+     isOffHeapEnabled: Boolean = true ): MemoryManager
+
 
   // -- Tests of sharing of execution memory between tasks 
----------------------------------------
   // Prior to Spark 1.6, these tests were part of ShuffleMemoryManagerSuite.
@@ -318,7 +320,7 @@ private[memory] trait MemoryManagerSuite extends 
SparkFunSuite {
   test("off-heap execution allocations cannot exceed limit") {
     val memoryManager = createMemoryManager(
       maxOnHeapExecutionMemory = 2L,
-      maxOffHeapExecutionMemory = 1000L)
+      maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = true)
 
     val tMemManager = new TaskMemoryManager(memoryManager, 1)
     val c = new TestMemoryConsumer(tMemManager, MemoryMode.OFF_HEAP)
@@ -336,10 +338,10 @@ private[memory] trait MemoryManagerSuite extends 
SparkFunSuite {
     assert(tMemManager.getMemoryConsumptionForThisTask === 0L)
   }
 
-  test("task peak execution memory usage") {
+  test("task peak execution memory usage when offheap memory is enabled") {
     val memoryManager = createMemoryManager(
       maxOnHeapExecutionMemory = 1000L,
-      maxOffHeapExecutionMemory = 1000L)
+      maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = true)
 
     val tMemManager = new TaskMemoryManager(memoryManager, 1)
     val offHeapConsumer = new TestMemoryConsumer(tMemManager, 
MemoryMode.OFF_HEAP)
@@ -353,6 +355,24 @@ private[memory] trait MemoryManagerSuite extends 
SparkFunSuite {
     assert(tMemManager.getPeakOnHeapExecutionMemory === 400L)
     assert(tMemManager.getPeakOffHeapExecutionMemory === 500L)
   }
+
+  test("task peak execution memory usage when offheap memory is disabled") {
+    val memoryManager = createMemoryManager(
+      maxOnHeapExecutionMemory = 1000L,
+      maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = false)
+
+    val tMemManager = new TaskMemoryManager(memoryManager, 1)
+    val offHeapConsumer = new TestMemoryConsumer(tMemManager, 
MemoryMode.OFF_HEAP)
+    val onHeapConsumer = new TestMemoryConsumer(tMemManager, 
MemoryMode.ON_HEAP)
+
+    val result1 = tMemManager.acquireExecutionMemory(500L, offHeapConsumer)
+    val result2 = tMemManager.acquireExecutionMemory(400L, onHeapConsumer)
+    assert(result1 === 0L)
+    assert(result2 === 400L)
+    assert(tMemManager.getMemoryConsumptionForThisTask === 400L)
+    assert(tMemManager.getPeakOnHeapExecutionMemory === 400L)
+    assert(tMemManager.getPeakOffHeapExecutionMemory === 0L)
+  }
 }
 
 private object MemoryManagerSuite {
diff --git 
a/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala 
b/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
index 9a21fa25b35b..580972b7621b 100644
--- 
a/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
@@ -42,12 +42,14 @@ class UnifiedMemoryManagerSuite extends MemoryManagerSuite 
with PrivateMethodTes
 
   override protected def createMemoryManager(
       maxOnHeapExecutionMemory: Long,
-      maxOffHeapExecutionMemory: Long): UnifiedMemoryManager = {
+      maxOffHeapExecutionMemory: Long,
+      isOffHeapEnabled: Boolean = false ): UnifiedMemoryManager = {
     val conf = new SparkConf()
       .set(MEMORY_FRACTION, 1.0)
       .set(TEST_MEMORY, maxOnHeapExecutionMemory)
       .set(MEMORY_OFFHEAP_SIZE, maxOffHeapExecutionMemory)
       .set(MEMORY_STORAGE_FRACTION, storageFraction)
+      .set(MEMORY_OFFHEAP_ENABLED, isOffHeapEnabled)
     UnifiedMemoryManager(conf, numCores = 1)
   }
 
diff --git 
a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala 
b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 86d6fecfb30a..5939f4b94af0 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -1875,6 +1875,7 @@ abstract class AppStatusListenerSuite extends 
SparkFunSuite with BeforeAndAfter
       val memoryMetrics = d.info.memoryMetrics.get
       assert(memoryMetrics.usedOffHeapStorageMemory == 0)
       assert(memoryMetrics.usedOnHeapStorageMemory == 0)
+      assert(memoryMetrics.totalOffHeapStorageMemory == 0)
     }
   }
 
diff --git 
a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
 
b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
index 07eb7b3da65c..be4ae5589a4a 100644
--- 
a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
@@ -75,9 +75,11 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
   protected def makeBlockManager(
       maxMem: Long,
       name: String = SparkContext.DRIVER_IDENTIFIER,
-      memoryManager: Option[UnifiedMemoryManager] = None): BlockManager = {
+      memoryManager: Option[UnifiedMemoryManager] = None,
+      isOffHeapEnabled: Boolean = false ): BlockManager = {
     conf.set(TEST_MEMORY, maxMem)
     conf.set(MEMORY_OFFHEAP_SIZE, maxMem)
+    conf.set(MEMORY_OFFHEAP_ENABLED, isOffHeapEnabled)
     val serializerManager = new SerializerManager(serializer, conf)
     val transfer = new NettyBlockTransferService(
       conf, securityMgr, serializerManager, "localhost", "localhost", 0, 1)
@@ -98,6 +100,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
     conf.set(MEMORY_FRACTION, 1.0)
     conf.set(MEMORY_STORAGE_FRACTION, 0.999)
     conf.set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
+    conf.set(MEMORY_OFFHEAP_ENABLED, false)
 
     // to make cached peers refresh frequently
     conf.set(STORAGE_CACHED_PEERS_TTL, 10)
@@ -403,7 +406,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
 
     // As many stores as the replication factor
     val stores = (1 to maxReplication).map {
-      i => makeBlockManager(storeSize, s"store$i")
+      i => makeBlockManager(storeSize, s"store$i", isOffHeapEnabled = true)
     }
 
     storageLevels.foreach { storageLevel =>
@@ -587,6 +590,7 @@ class BlockManagerBasicStrategyReplicationSuite extends 
BlockManagerReplicationB
     new SparkConf(false)
       .set("spark.app.id", "test")
       .set(Kryo.KRYO_SERIALIZER_BUFFER_SIZE.key, "1m")
+      .set(MEMORY_OFFHEAP_ENABLED, true)
       .set(
         STORAGE_REPLICATION_POLICY,
         classOf[BasicBlockReplicationPolicy].getName)
diff --git 
a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala 
b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 70501afd9d92..71e38d937c44 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -129,6 +129,9 @@ class BlockManagerSuite extends SparkFunSuite with Matchers 
with PrivateMethodTe
     val bmConf = testConf.map(_.setAll(conf.getAll)).getOrElse(conf)
     bmConf.set(TEST_MEMORY, maxMem)
     bmConf.set(MEMORY_OFFHEAP_SIZE, maxMem)
+    if(maxMem > 0) {
+      bmConf.set(MEMORY_OFFHEAP_ENABLED, true)
+    }
     val serializer = new KryoSerializer(bmConf)
     val encryptionKey = if (bmConf.get(IO_ENCRYPTION_ENABLED)) {
       Some(CryptoStreamUtils.createKey(bmConf))
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala 
b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index aa9db55df91e..51a994a20790 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -123,6 +123,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser 
with Matchers {
       .set(UI_KILL_ENABLED, killEnabled)
       .set(UI_TIMELINE_ENABLED, timelineEnabled)
       .set(MEMORY_OFFHEAP_SIZE.key, "64m")
+      .set(MEMORY_OFFHEAP_ENABLED, true)
     additionalConfs.foreach { case (k, v) => conf.set(k, v) }
     val sc = new SparkContext(conf)
     assert(sc.ui.isDefined)
@@ -152,7 +153,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser 
with Matchers {
 
   test("effects of unpersist() / persist() should be reflected") {
     // Regression test for SPARK-2527
-    withSpark(newSparkContext()) { sc =>
+    withSpark(newSparkContext(master = "local-cluster[1,1,1024]")) { sc =>
       val ui = sc.ui.get
       val rdd = sc.parallelize(Seq(1, 2, 3))
       rdd.persist(StorageLevels.DISK_ONLY).count()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to