This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 511f9d5b6747 [SPARK-54449][CORE] Storage Memory off heap size should
be considered only when off heap is enabled
511f9d5b6747 is described below
commit 511f9d5b6747c719b68ed8238f320a07b53476d8
Author: VindhyaG <[email protected]>
AuthorDate: Tue Nov 3 14:31:15 2026 -0800
[SPARK-54449][CORE] Storage Memory off heap size should be considered only
when off heap is enabled
### What changes were proposed in this pull request?
Currently, even if spark.memory.offHeap.enabled=false
spark.memory.offHeap.size is calculated for SPARK UI and wherever the
config is used. This PR changes the logic to take the value set in
spark.memory.offHeap.size only when spark.memory.offHeap.enabled
### Why are the changes needed?
Change the logic to get the config of spark.memory.offHeap.size by
checking spark.memory.offHeap.enabled first
### Does this PR introduce _any_ user-facing change?
Yes. Spark UI will show the total of Onheap and Off heap memory only if
spark.memory.offHeap.enabled = true.
### How was this patch tested?
Existing UTs
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53383 from VindhyaG/SPARK-54449.
Authored-by: VindhyaG <[email protected]>
Signed-off-by: Holden Karau <[email protected]>
---
.../org/apache/spark/memory/MemoryManager.scala | 6 ++++-
.../apache/spark/status/AppStatusListener.scala | 6 ++++-
.../executor_memory_usage_expectation.json | 2 +-
...executor_node_excludeOnFailure_expectation.json | 2 +-
.../internal/plugin/PluginContainerSuite.scala | 14 ++++++++++-
.../apache/spark/memory/MemoryManagerSuite.scala | 28 ++++++++++++++++++----
.../spark/memory/UnifiedMemoryManagerSuite.scala | 4 +++-
.../spark/status/AppStatusListenerSuite.scala | 1 +
.../storage/BlockManagerReplicationSuite.scala | 8 +++++--
.../apache/spark/storage/BlockManagerSuite.scala | 3 +++
.../org/apache/spark/ui/UISeleniumSuite.scala | 3 ++-
11 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
index dac82cc76357..88d973756e21 100644
--- a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala
@@ -58,7 +58,8 @@ private[spark] abstract class MemoryManager(
onHeapStorageMemoryPool.incrementPoolSize(onHeapStorageMemory)
onHeapExecutionMemoryPool.incrementPoolSize(onHeapExecutionMemory)
- protected[this] val maxOffHeapMemory = conf.get(MEMORY_OFFHEAP_SIZE)
+ protected[this] val maxOffHeapMemory =
+ if (conf.get(MEMORY_OFFHEAP_ENABLED)) conf.get(MEMORY_OFFHEAP_SIZE) else 0
protected[this] val offHeapStorageMemory =
(maxOffHeapMemory * conf.get(MEMORY_STORAGE_FRACTION)).toLong
@@ -241,6 +242,9 @@ private[spark] abstract class MemoryManager(
"No support for unaligned Unsafe. Set spark.memory.offHeap.enabled to
false.")
MemoryMode.OFF_HEAP
} else {
+ if (conf.get(MEMORY_OFFHEAP_SIZE) > 0) {
+ logWarning(s"spark.memory.offHeap.size is > 0 when
spark.memory.offHeap.enabled == false")
+ }
MemoryMode.ON_HEAP
}
}
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 87eed5c6b2c5..a0dc75fc14fc 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -912,7 +912,11 @@ private[spark] class AppStatusListener(
exec.hostPort = event.blockManagerId.hostPort
event.maxOnHeapMem.foreach { _ =>
exec.totalOnHeap = event.maxOnHeapMem.get
- exec.totalOffHeap = event.maxOffHeapMem.get
+ if (event.blockManagerId.executorId == SparkContext.DRIVER_IDENTIFIER) {
+ exec.totalOffHeap = 0
+ } else {
+ exec.totalOffHeap = event.maxOffHeapMem.get
+ }
// SPARK-30594: whenever(first time or re-register) a BlockManager
added, all blocks
// from this BlockManager will be reported to driver later. So, we
should clean up
// used memory to avoid overlapped count.
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index fc63185b4c4a..999f7ff5fc1a 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
"maxMemory" : 908381388,
"maxTasks" : 0,
"memoryMetrics" : {
- "totalOffHeapStorageMemory" : 524288000,
+ "totalOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
"usedOffHeapStorageMemory" : 0,
"usedOnHeapStorageMemory" : 0
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
index fc63185b4c4a..999f7ff5fc1a 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -16,7 +16,7 @@
"maxMemory" : 908381388,
"maxTasks" : 0,
"memoryMetrics" : {
- "totalOffHeapStorageMemory" : 524288000,
+ "totalOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
"usedOffHeapStorageMemory" : 0,
"usedOnHeapStorageMemory" : 0
diff --git
a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
index 99165585aefa..8718a1c6119b 100644
---
a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
@@ -272,7 +272,19 @@ class PluginContainerSuite extends SparkFunSuite with
LocalSparkContext {
.get(ResourceProfile.OFFHEAP_MEM).map(_.amount).getOrElse(-1L))
// Ensure all executors has started
- TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+ TestUtils.waitUntilExecutorsUp(sc, 2, 120000)
+ Thread.sleep(10000)
+
+ // Check executor memory is also updated for all executors except driver
+ val executorInfos = sc.statusTracker.getExecutorInfos
+ val execWithMemory = executorInfos.filter(_.totalOffHeapStorageMemory()
> 0)
+
+ assert(execWithMemory.length >= 2,
+ s"Expected atleast 2 executors with memory, got
${execWithMemory.length}")
+ execWithMemory.foreach { exec =>
+ assert(exec.totalOffHeapStorageMemory() ==
MemoryOverridePlugin.offHeapMemory)
+ }
+
// Check executor memory is also updated
eventually(timeout(10.seconds), interval(100.milliseconds)) {
diff --git
a/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
b/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
index 1f40ef944a84..63e92d0cefda 100644
--- a/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/memory/MemoryManagerSuite.scala
@@ -143,7 +143,9 @@ private[memory] trait MemoryManagerSuite extends
SparkFunSuite {
*/
protected def createMemoryManager(
maxOnHeapExecutionMemory: Long,
- maxOffHeapExecutionMemory: Long = 0L): MemoryManager
+ maxOffHeapExecutionMemory: Long = 0L,
+ isOffHeapEnabled: Boolean = true ): MemoryManager
+
// -- Tests of sharing of execution memory between tasks
----------------------------------------
// Prior to Spark 1.6, these tests were part of ShuffleMemoryManagerSuite.
@@ -318,7 +320,7 @@ private[memory] trait MemoryManagerSuite extends
SparkFunSuite {
test("off-heap execution allocations cannot exceed limit") {
val memoryManager = createMemoryManager(
maxOnHeapExecutionMemory = 2L,
- maxOffHeapExecutionMemory = 1000L)
+ maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = true)
val tMemManager = new TaskMemoryManager(memoryManager, 1)
val c = new TestMemoryConsumer(tMemManager, MemoryMode.OFF_HEAP)
@@ -336,10 +338,10 @@ private[memory] trait MemoryManagerSuite extends
SparkFunSuite {
assert(tMemManager.getMemoryConsumptionForThisTask === 0L)
}
- test("task peak execution memory usage") {
+ test("task peak execution memory usage when offheap memory is enabled") {
val memoryManager = createMemoryManager(
maxOnHeapExecutionMemory = 1000L,
- maxOffHeapExecutionMemory = 1000L)
+ maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = true)
val tMemManager = new TaskMemoryManager(memoryManager, 1)
val offHeapConsumer = new TestMemoryConsumer(tMemManager,
MemoryMode.OFF_HEAP)
@@ -353,6 +355,24 @@ private[memory] trait MemoryManagerSuite extends
SparkFunSuite {
assert(tMemManager.getPeakOnHeapExecutionMemory === 400L)
assert(tMemManager.getPeakOffHeapExecutionMemory === 500L)
}
+
+ test("task peak execution memory usage when offheap memory is disabled") {
+ val memoryManager = createMemoryManager(
+ maxOnHeapExecutionMemory = 1000L,
+ maxOffHeapExecutionMemory = 1000L, isOffHeapEnabled = false)
+
+ val tMemManager = new TaskMemoryManager(memoryManager, 1)
+ val offHeapConsumer = new TestMemoryConsumer(tMemManager,
MemoryMode.OFF_HEAP)
+ val onHeapConsumer = new TestMemoryConsumer(tMemManager,
MemoryMode.ON_HEAP)
+
+ val result1 = tMemManager.acquireExecutionMemory(500L, offHeapConsumer)
+ val result2 = tMemManager.acquireExecutionMemory(400L, onHeapConsumer)
+ assert(result1 === 0L)
+ assert(result2 === 400L)
+ assert(tMemManager.getMemoryConsumptionForThisTask === 400L)
+ assert(tMemManager.getPeakOnHeapExecutionMemory === 400L)
+ assert(tMemManager.getPeakOffHeapExecutionMemory === 0L)
+ }
}
private object MemoryManagerSuite {
diff --git
a/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
b/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
index 9a21fa25b35b..580972b7621b 100644
---
a/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/memory/UnifiedMemoryManagerSuite.scala
@@ -42,12 +42,14 @@ class UnifiedMemoryManagerSuite extends MemoryManagerSuite
with PrivateMethodTes
override protected def createMemoryManager(
maxOnHeapExecutionMemory: Long,
- maxOffHeapExecutionMemory: Long): UnifiedMemoryManager = {
+ maxOffHeapExecutionMemory: Long,
+ isOffHeapEnabled: Boolean = false ): UnifiedMemoryManager = {
val conf = new SparkConf()
.set(MEMORY_FRACTION, 1.0)
.set(TEST_MEMORY, maxOnHeapExecutionMemory)
.set(MEMORY_OFFHEAP_SIZE, maxOffHeapExecutionMemory)
.set(MEMORY_STORAGE_FRACTION, storageFraction)
+ .set(MEMORY_OFFHEAP_ENABLED, isOffHeapEnabled)
UnifiedMemoryManager(conf, numCores = 1)
}
diff --git
a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 86d6fecfb30a..5939f4b94af0 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -1875,6 +1875,7 @@ abstract class AppStatusListenerSuite extends
SparkFunSuite with BeforeAndAfter
val memoryMetrics = d.info.memoryMetrics.get
assert(memoryMetrics.usedOffHeapStorageMemory == 0)
assert(memoryMetrics.usedOnHeapStorageMemory == 0)
+ assert(memoryMetrics.totalOffHeapStorageMemory == 0)
}
}
diff --git
a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
index 07eb7b3da65c..be4ae5589a4a 100644
---
a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
+++
b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
@@ -75,9 +75,11 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
protected def makeBlockManager(
maxMem: Long,
name: String = SparkContext.DRIVER_IDENTIFIER,
- memoryManager: Option[UnifiedMemoryManager] = None): BlockManager = {
+ memoryManager: Option[UnifiedMemoryManager] = None,
+ isOffHeapEnabled: Boolean = false ): BlockManager = {
conf.set(TEST_MEMORY, maxMem)
conf.set(MEMORY_OFFHEAP_SIZE, maxMem)
+ conf.set(MEMORY_OFFHEAP_ENABLED, isOffHeapEnabled)
val serializerManager = new SerializerManager(serializer, conf)
val transfer = new NettyBlockTransferService(
conf, securityMgr, serializerManager, "localhost", "localhost", 0, 1)
@@ -98,6 +100,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
conf.set(MEMORY_FRACTION, 1.0)
conf.set(MEMORY_STORAGE_FRACTION, 0.999)
conf.set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
+ conf.set(MEMORY_OFFHEAP_ENABLED, false)
// to make cached peers refresh frequently
conf.set(STORAGE_CACHED_PEERS_TTL, 10)
@@ -403,7 +406,7 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
// As many stores as the replication factor
val stores = (1 to maxReplication).map {
- i => makeBlockManager(storeSize, s"store$i")
+ i => makeBlockManager(storeSize, s"store$i", isOffHeapEnabled = true)
}
storageLevels.foreach { storageLevel =>
@@ -587,6 +590,7 @@ class BlockManagerBasicStrategyReplicationSuite extends
BlockManagerReplicationB
new SparkConf(false)
.set("spark.app.id", "test")
.set(Kryo.KRYO_SERIALIZER_BUFFER_SIZE.key, "1m")
+ .set(MEMORY_OFFHEAP_ENABLED, true)
.set(
STORAGE_REPLICATION_POLICY,
classOf[BasicBlockReplicationPolicy].getName)
diff --git
a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 70501afd9d92..71e38d937c44 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -129,6 +129,9 @@ class BlockManagerSuite extends SparkFunSuite with Matchers
with PrivateMethodTe
val bmConf = testConf.map(_.setAll(conf.getAll)).getOrElse(conf)
bmConf.set(TEST_MEMORY, maxMem)
bmConf.set(MEMORY_OFFHEAP_SIZE, maxMem)
+ if(maxMem > 0) {
+ bmConf.set(MEMORY_OFFHEAP_ENABLED, true)
+ }
val serializer = new KryoSerializer(bmConf)
val encryptionKey = if (bmConf.get(IO_ENCRYPTION_ENABLED)) {
Some(CryptoStreamUtils.createKey(bmConf))
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index aa9db55df91e..51a994a20790 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -123,6 +123,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser
with Matchers {
.set(UI_KILL_ENABLED, killEnabled)
.set(UI_TIMELINE_ENABLED, timelineEnabled)
.set(MEMORY_OFFHEAP_SIZE.key, "64m")
+ .set(MEMORY_OFFHEAP_ENABLED, true)
additionalConfs.foreach { case (k, v) => conf.set(k, v) }
val sc = new SparkContext(conf)
assert(sc.ui.isDefined)
@@ -152,7 +153,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser
with Matchers {
test("effects of unpersist() / persist() should be reflected") {
// Regression test for SPARK-2527
- withSpark(newSparkContext()) { sc =>
+ withSpark(newSparkContext(master = "local-cluster[1,1,1024]")) { sc =>
val ui = sc.ui.get
val rdd = sc.parallelize(Seq(1, 2, 3))
rdd.persist(StorageLevels.DISK_ONLY).count()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]