This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5cb7f6388ac0 [SPARK-59059][CORE] Support per-logger log level
configuration with static Spark conf
5cb7f6388ac0 is described below
commit 5cb7f6388ac0401a7984bc5bcead799b1d057b56
Author: John Zhuge <[email protected]>
AuthorDate: Wed Oct 28 17:15:32 2026 -0700
[SPARK-59059][CORE] Support per-logger log level configuration with static
Spark conf
### What changes were proposed in this pull request?
Add `spark.log.level.<logger>`, a conf that sets the level of a single
logger on the driver and the
executors:
```
--conf spark.log.level=WARN
--conf spark.log.level.org.apache.spark.sql.execution=DEBUG
```
`Utils.setLoggerLevels` validates each value against
`SparkContext.VALID_LOG_LEVELS` and applies it
with log4j2 `Configurator.setLevel`. It is called from `SparkContext` right
after the existing
`spark.log.level` handling, and from `CoarseGrainedExecutorBackend.run`
right after the executor
applies the root level, so a per-logger level always wins over the root
level.
### Why are the changes needed?
SPARK-43782 added `spark.log.level`, but it only sets the root logger.
Turning on DEBUG for one class
still means shipping a `log4j2.properties` and pointing both the driver and
the executor JVMs at it
with `-Dlog4j.configurationFile`. That is awkward in cluster mode, and a
Spark Connect client cannot
do it at all: it has no `SparkContext`, and usually cannot change the log4j
configuration of the
platform it connects to. That was the motivation accepted for SPARK-43782.
### Does this PR introduce _any_ user-facing change?
Yes, a new configuration property, documented in `docs/configuration.md`.
No change when it is unset.
### How was this patch tested?
New tests in `SparkContextSuite`: the named logger ends at `DEBUG` while
the root logger stays at the
`ERROR` from `spark.log.level`, and an invalid level is rejected with a
message naming the config key.
The executor path is not covered by a unit test.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5
Closes #58365 from jzhuge/SPARK-59059-per-logger-level.
Authored-by: John Zhuge <[email protected]>
Signed-off-by: Holden Karau <[email protected]>
---
.../main/scala/org/apache/spark/SparkContext.scala | 1 +
.../executor/CoarseGrainedExecutorBackend.scala | 1 +
.../org/apache/spark/internal/config/package.scala | 2 ++
.../main/scala/org/apache/spark/util/Utils.scala | 19 +++++++++++++++-
.../scala/org/apache/spark/SparkContextSuite.scala | 25 ++++++++++++++++++++++
docs/configuration.md | 11 ++++++++++
6 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index d7ea13d8487a..8fd728ba6e2e 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -437,6 +437,7 @@ class SparkContext(config: SparkConf) extends Logging {
}
setLogLevel(level)
}
+ Utils.setLoggerLevels(_conf)
_conf.validateSettings()
_conf.set("spark.app.startTime", startTime.toString)
diff --git
a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index 375682383b1f..525d9fff4f5d 100644
---
a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -485,6 +485,7 @@ private[spark] object CoarseGrainedExecutorBackend extends
Logging {
driverConf.set(EXECUTOR_ID, arguments.executorId)
cfg.logLevel.foreach(logLevel => Utils.setLogLevelIfNeeded(logLevel))
+ Utils.setLoggerLevels(driverConf)
// Set executor memory related config here according to resource profile
if (cfg.resourceProfile.id !=
ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID) {
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 8759613490d8..f1e5828e5899 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1314,6 +1314,8 @@ package object config {
SparkContext.VALID_LOG_LEVELS.mkString(","))
.createOptional
+ private[spark] val SPARK_LOG_LEVEL_PREFIX = "spark.log.level."
+
private[spark] val FILES_MAX_PARTITION_BYTES =
ConfigBuilder("spark.files.maxPartitionBytes")
.doc("The maximum number of bytes to pack into a single partition when
reading files.")
.version("2.1.0")
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 8c7829541fc4..a6f103f4314b 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -60,7 +60,7 @@ import org.apache.hadoop.util.RunJar
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.logging.log4j.{Level, LogManager}
import org.apache.logging.log4j.core.LoggerContext
-import org.apache.logging.log4j.core.config.LoggerConfig
+import org.apache.logging.log4j.core.config.{Configurator, LoggerConfig}
import org.slf4j.Logger
import org.apache.spark.{SPARK_VERSION, _}
@@ -2365,6 +2365,23 @@ private[spark] object Utils
}
}
+ /**
+ * Set the level of the loggers named by the `spark.log.level.<logger>`
configs. Applied after the
+ * root level from `spark.log.level`, so it wins for those loggers.
+ */
+ def setLoggerLevels(conf: SparkConf): Unit = {
+ val levels = conf.getAllWithPrefix(SPARK_LOG_LEVEL_PREFIX).map { case
(logger, level) =>
+ val upperCased = level.toUpperCase(Locale.ROOT)
+ require(SparkContext.VALID_LOG_LEVELS.contains(upperCased),
+ s"Invalid value for '$SPARK_LOG_LEVEL_PREFIX$logger'. Valid values are
" +
+ SparkContext.VALID_LOG_LEVELS.mkString(","))
+ logger -> Level.toLevel(upperCased)
+ }.toMap
+ if (levels.nonEmpty) {
+ Configurator.setLevel(levels.asJava)
+ }
+ }
+
private lazy val getLogContext: (LoggerContext, LoggerConfig) = {
val ctx = LogManager.getContext(false).asInstanceOf[LoggerContext]
(ctx, ctx.getConfiguration().getLoggerConfig(LogManager.ROOT_LOGGER_NAME))
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index f7d668377c1c..4ec8b15c55c8 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.ipc.{CallerContext =>
HadoopCallerContext}
import org.apache.hadoop.mapred.TextInputFormat
import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat =>
NewTextInputFormat}
import org.apache.logging.log4j.{Level, LogManager}
+import org.apache.logging.log4j.core.config.Configurator
import org.json4s.{DefaultFormats, Extraction}
import org.mockito.ArgumentMatchers.{any, eq => meq}
import org.mockito.Mockito.{mock, verify, when}
@@ -647,6 +648,30 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
sc.stop()
}
+ test("SPARK-59059: conf to override log level of a specific logger") {
+ val logger = "org.apache.spark.SparkContextSuite.testLogger"
+ val rootLevel = LogManager.getRootLogger().getLevel
+
+ try {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local")
+ .set(SPARK_LOG_LEVEL, "ERROR")
+ .set(SPARK_LOG_LEVEL_PREFIX + logger, "DEBUG"))
+ assert(LogManager.getLogger(logger).getLevel === Level.DEBUG)
+ assert(LogManager.getRootLogger().getLevel === Level.ERROR)
+ } finally {
+ Configurator.setLevel(logger, rootLevel)
+ Utils.setLogLevel(rootLevel)
+ }
+ }
+
+ test("SPARK-59059: invalid per-logger log level is rejected") {
+ val e = intercept[IllegalArgumentException] {
+ Utils.setLoggerLevels(
+ new SparkConf().set(SPARK_LOG_LEVEL_PREFIX + "org.apache.spark",
"NOT_A_LEVEL"))
+ }
+ assert(e.getMessage.contains("spark.log.level.org.apache.spark"))
+ }
+
test("register and deregister Spark listener from SparkContext") {
sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local"))
val sparkListener1 = new SparkListener { }
diff --git a/docs/configuration.md b/docs/configuration.md
index 3ff96ad8c5d0..8c87ebede11d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -449,6 +449,17 @@ of the most common options to set are:
</td>
<td>3.5.0</td>
</tr>
+<tr>
+ <td><code>spark.log.level.<logger></code></td>
+ <td>(none)</td>
+ <td>
+ When set, overrides the level of the named logger on the driver and the
executors, for example
+ <code>spark.log.level.org.apache.spark.sql.execution=DEBUG</code>. Applied
after
+ <code>spark.log.level</code>, which sets the root logger, so it wins for
that logger. Valid log
+ levels include: "ALL", "DEBUG", "ERROR", "FATAL", "INFO", "OFF", "TRACE",
"WARN".
+ </td>
+ <td>5.0.0</td>
+</tr>
<tr>
<td><code>spark.driver.supervise</code></td>
<td>false</td>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
