AngersZhuuuu commented on PR #57716:
URL: https://github.com/apache/spark/pull/57716#issuecomment-5173900444
This fix have a problem that may cause gluten failed to init, since we
change teh method's parameter. So there have two way to fix this:
1. Add a compatible method and set its default value to true.
```
def apply(conf: SparkConf, numCores: Int): UnifiedMemoryManager = {
apply(conf, numCores, isDriver = true)
}
def apply(
conf: SparkConf,
numCores: Int,
isDriver: Boolean = true): UnifiedMemoryManager = {
isDriver: Boolean): UnifiedMemoryManager = {
val maxMemory = getMaxMemory(conf, isDriver)
new UnifiedMemoryManager(
conf,
maxHeapMemory = maxMemory,
onHeapStorageRegionSize =
(maxMemory * conf.get(config.MEMORY_STORAGE_FRACTION)).toLong,
numCores = numCores)
}
```
2. add
```
// Compatible with legacy calls, such as Gluten
def apply(conf: SparkConf, numCores: Int): UnifiedMemoryManager = {
applyInternal(conf, numCores, None)
}
// Spark internally explicitly passes in roles
private def applyInternal(
conf: SparkConf,
numCores: Int,
isDriver: Option[Boolean]): UnifiedMemoryManager = {
val maxMemory = getMaxMemory(conf, isDriver)
...
}
isDriver match {
case Some(true) =>
// Only check Driver memory
case Some(false) =>
// Only check Executor memory
case None =>
// both checkļ¼Driver & Executor
}
```
Which one do you think is better? @uros-b
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
