This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 039fd13eacb1 [SPARK-49749][CORE] Change log level to debug in
BlockManagerInfo
039fd13eacb1 is described below
commit 039fd13eacb1cef835045e3a60cebf958589e1a2
Author: Yuming Wang <[email protected]>
AuthorDate: Sat Sep 28 19:45:52 2024 -0700
[SPARK-49749][CORE] Change log level to debug in BlockManagerInfo
### What changes were proposed in this pull request?
This PR changes the log level to debug in `BlockManagerInfo`.
### Why are the changes needed?
Before this PR:
Logging in `BlockManagerMasterEndpoint` uses 3.25% of the CPU and generates
60.5% of the logs.
<img width="1186" alt="image"
src="https://github.com/user-attachments/assets/81013d98-8700-4e4f-a1aa-aa17fc0c5c52";>
```
cat spark.20240921-09.log | grep "in memory on" | wc -l
8587851
cat spark.20240921-09.log | wc -l
14185544
```
After this PR:
<img width="1196" alt="image"
src="https://github.com/user-attachments/assets/6d52cd2c-de3b-4a32-a6e8-a98c42909faa";>
```
cat spark.20240926-09.log | grep "in memory on" | wc -l
0
cat spark.20240926-09.log | wc -l
2224037
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
N/A.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48197 from wangyum/SPARK-49749.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/storage/BlockManagerMasterEndpoint.scala | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 73f89ea0e86e..fc4e6e771aad 100644
---
a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++
b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -1059,13 +1059,13 @@ private[spark] class BlockManagerInfo(
_blocks.put(blockId, blockStatus)
_remainingMem -= memSize
if (blockExists) {
- logInfo(log"Updated ${MDC(BLOCK_ID, blockId)} in memory on " +
+ logDebug(log"Updated ${MDC(BLOCK_ID, blockId)} in memory on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} (current size: " +
log"${MDC(CURRENT_MEMORY_SIZE, Utils.bytesToString(memSize))},
original " +
log"size: ${MDC(ORIGINAL_MEMORY_SIZE,
Utils.bytesToString(originalMemSize))}, " +
log"free: ${MDC(FREE_MEMORY_SIZE,
Utils.bytesToString(_remainingMem))})")
} else {
- logInfo(log"Added ${MDC(BLOCK_ID, blockId)} in memory on " +
+ logDebug(log"Added ${MDC(BLOCK_ID, blockId)} in memory on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} " +
log"(size: ${MDC(CURRENT_MEMORY_SIZE,
Utils.bytesToString(memSize))}, " +
log"free: ${MDC(FREE_MEMORY_SIZE,
Utils.bytesToString(_remainingMem))})")
@@ -1075,12 +1075,12 @@ private[spark] class BlockManagerInfo(
blockStatus = BlockStatus(storageLevel, memSize = 0, diskSize =
diskSize)
_blocks.put(blockId, blockStatus)
if (blockExists) {
- logInfo(log"Updated ${MDC(BLOCK_ID, blockId)} on disk on " +
+ logDebug(log"Updated ${MDC(BLOCK_ID, blockId)} on disk on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} " +
log"(current size: ${MDC(CURRENT_DISK_SIZE,
Utils.bytesToString(diskSize))}," +
log" original size: ${MDC(ORIGINAL_DISK_SIZE,
Utils.bytesToString(originalDiskSize))})")
} else {
- logInfo(log"Added ${MDC(BLOCK_ID, blockId)} on disk on " +
+ logDebug(log"Added ${MDC(BLOCK_ID, blockId)} on disk on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} (size: " +
log"${MDC(CURRENT_DISK_SIZE, Utils.bytesToString(diskSize))})")
}
@@ -1098,13 +1098,13 @@ private[spark] class BlockManagerInfo(
blockStatus.remove(blockId)
}
if (originalLevel.useMemory) {
- logInfo(log"Removed ${MDC(BLOCK_ID, blockId)} on " +
+ logDebug(log"Removed ${MDC(BLOCK_ID, blockId)} on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} in memory " +
log"(size: ${MDC(ORIGINAL_MEMORY_SIZE,
Utils.bytesToString(originalMemSize))}, " +
log"free: ${MDC(FREE_MEMORY_SIZE,
Utils.bytesToString(_remainingMem))})")
}
if (originalLevel.useDisk) {
- logInfo(log"Removed ${MDC(BLOCK_ID, blockId)} on " +
+ logDebug(log"Removed ${MDC(BLOCK_ID, blockId)} on " +
log"${MDC(HOST_PORT, blockManagerId.hostPort)} on disk" +
log" (size: ${MDC(ORIGINAL_DISK_SIZE,
Utils.bytesToString(originalDiskSize))})")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
