This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e61d1e7 [SPARK-37794][BUILD] Remove internal log4j bridge api usage
e61d1e7 is described below
commit e61d1e7894dced952ac63992752dcde8aa086b93
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Dec 31 16:01:21 2021 -0800
[SPARK-37794][BUILD] Remove internal log4j bridge api usage
### What changes were proposed in this pull request?
This patch removes log4j 1.x bridge api usage.
### Why are the changes needed?
Although for compatibility reason, we include log4j 1.x bridge api
dependency. But for Spark itself, it is better to remove the bridge api usage.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass all tests
Closes #35075 from viirya/minor.
Authored-by: Liang-Chi Hsieh <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/internal/Logging.scala | 33 +++++----
.../examples/streaming/StreamingExamples.scala | 6 +-
.../examples/streaming/KinesisWordCountASL.scala | 6 +-
.../spark/deploy/yarn/SparkRackResolver.scala | 8 ++-
.../v2/jdbc/JDBCTableCatalogSuite.scala | 2 +-
.../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 3 +-
.../HiveMetastoreLazyInitializationSuite.scala | 5 +-
.../test/resources/data/conf/hive-log4j.properties | 78 ----------------------
.../org/apache/spark/sql/hive/test/TestHive.scala | 9 +--
9 files changed, 39 insertions(+), 111 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala
b/core/src/main/scala/org/apache/spark/internal/Logging.scala
index 08e8f8b..5191be4 100644
--- a/core/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -20,7 +20,7 @@ package org.apache.spark.internal
import scala.collection.JavaConverters._
import org.apache.logging.log4j.{Level, LogManager}
-import org.apache.logging.log4j.core.{Filter, LifeCycle, LogEvent,
LoggerContext}
+import org.apache.logging.log4j.core.{Filter, LifeCycle, LogEvent, Logger =>
Log4jLogger, LoggerContext}
import org.apache.logging.log4j.core.appender.ConsoleAppender
import org.apache.logging.log4j.core.config.DefaultConfiguration
import org.apache.logging.log4j.core.filter.AbstractFilter
@@ -127,17 +127,11 @@ trait Logging {
private def initializeLogging(isInterpreter: Boolean, silent: Boolean): Unit
= {
if (Logging.isLog4j2()) {
- val rootLogger =
LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger]
+ val rootLogger = LogManager.getRootLogger.asInstanceOf[Log4jLogger]
// If Log4j 2 is used but is initialized by default configuration,
// load a default properties file
- // (see org.apache.logging.log4j.core.config.DefaultConfiguration)
- val needToInitializeLog4j2 = rootLogger.getAppenders.isEmpty ||
- (rootLogger.getAppenders.size() == 1 &&
- rootLogger.getLevel == Level.ERROR &&
- LogManager.getContext.asInstanceOf[LoggerContext]
- .getConfiguration.isInstanceOf[DefaultConfiguration])
// scalastyle:off println
- if (needToInitializeLog4j2) {
+ if (Logging.islog4j2DefaultConfigured()) {
Logging.defaultSparkLog4jConfig = true
val defaultLogProps = "org/apache/spark/log4j2-defaults.properties"
Option(Utils.getSparkClassLoader.getResource(defaultLogProps)) match {
@@ -159,8 +153,7 @@ trait Logging {
if (isInterpreter) {
// Use the repl's main class to define the default log level when
running the shell,
// overriding the root logger's config if they're different.
- val replLogger = LogManager.getLogger(logName)
- .asInstanceOf[org.apache.logging.log4j.core.Logger]
+ val replLogger =
LogManager.getLogger(logName).asInstanceOf[Log4jLogger]
val replLevel = Option(replLogger.getLevel()).getOrElse(Level.WARN)
// Update the consoleAppender threshold to replLevel
if (replLevel != rootLogger.getLevel()) {
@@ -220,8 +213,7 @@ private[spark] object Logging {
val context = LogManager.getContext(false).asInstanceOf[LoggerContext]
context.reconfigure()
} else {
- val rootLogger = LogManager.getRootLogger()
- .asInstanceOf[org.apache.logging.log4j.core.Logger]
+ val rootLogger = LogManager.getRootLogger().asInstanceOf[Log4jLogger]
rootLogger.setLevel(defaultRootLevel)
sparkShellThresholdLevel = null
}
@@ -237,6 +229,19 @@ private[spark] object Logging {
"org.apache.logging.slf4j.Log4jLoggerFactory".equals(binderClass)
}
+ /**
+ * Return true if log4j2 is initialized by default configuration which has
one
+ * appender with error level. See
`org.apache.logging.log4j.core.config.DefaultConfiguration`.
+ */
+ private[spark] def islog4j2DefaultConfigured(): Boolean = {
+ val rootLogger = LogManager.getRootLogger.asInstanceOf[Log4jLogger]
+ rootLogger.getAppenders.isEmpty ||
+ (rootLogger.getAppenders.size() == 1 &&
+ rootLogger.getLevel == Level.ERROR &&
+ LogManager.getContext.asInstanceOf[LoggerContext]
+ .getConfiguration.isInstanceOf[DefaultConfiguration])
+ }
+
private[spark] class SparkShellLoggingFilter extends AbstractFilter {
private var status = LifeCycle.State.INITIALIZING
@@ -255,7 +260,7 @@ private[spark] object Logging {
Filter.Result.NEUTRAL
} else {
var logger = LogManager.getLogger(logEvent.getLoggerName)
- .asInstanceOf[org.apache.logging.log4j.core.Logger]
+ .asInstanceOf[Log4jLogger]
while (logger.getParent() != null) {
if (logger.getLevel != null || !logger.getAppenders.isEmpty) {
return Filter.Result.NEUTRAL
diff --git
a/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
b/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
index fe336b5..20c5eb1 100644
---
a/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
+++
b/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
@@ -17,8 +17,7 @@
package org.apache.spark.examples.streaming
-import org.apache.logging.log4j.{Level, LogManager}
-import org.apache.logging.log4j.core.Logger
+import org.apache.logging.log4j.Level
import org.apache.logging.log4j.core.config.Configurator
import org.apache.spark.internal.Logging
@@ -28,8 +27,7 @@ object StreamingExamples extends Logging {
/** Set reasonable logging levels for streaming if the user has not
configured log4j. */
def setStreamingLogLevels(): Unit = {
- val log4jInitialized =
!LogManager.getRootLogger.asInstanceOf[Logger].getAppenders.isEmpty
- if (!log4jInitialized) {
+ if (Logging.islog4j2DefaultConfigured()) {
// We first log something to initialize Spark's default logging, then we
override the
// logging level.
logInfo("Setting log level to [WARN] for streaming example." +
diff --git
a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index ea47665..e96a669 100644
---
a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++
b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -25,8 +25,7 @@ import scala.util.Random
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
import com.amazonaws.services.kinesis.AmazonKinesisClient
import com.amazonaws.services.kinesis.model.PutRecordRequest
-import org.apache.logging.log4j.{Level, LogManager}
-import org.apache.logging.log4j.core.Logger
+import org.apache.logging.log4j.Level
import org.apache.logging.log4j.core.config.Configurator
import org.apache.spark.SparkConf
@@ -272,8 +271,7 @@ object KinesisWordProducerASL {
private[streaming] object StreamingExamples extends Logging {
// Set reasonable logging levels for streaming if the user has not
configured log4j.
def setStreamingLogLevels(): Unit = {
- val log4jInitialized =
!LogManager.getRootLogger.asInstanceOf[Logger].getAppenders.isEmpty
- if (!log4jInitialized) {
+ if (Logging.islog4j2DefaultConfigured()) {
// We first log something to initialize Spark's default logging, then we
override the
// logging level.
logInfo("Setting log level to [WARN] for streaming example." +
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/SparkRackResolver.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/SparkRackResolver.scala
index 51be932..f3e8a9f 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/SparkRackResolver.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/SparkRackResolver.scala
@@ -26,7 +26,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic
import org.apache.hadoop.net._
import org.apache.hadoop.util.ReflectionUtils
import org.apache.hadoop.yarn.util.RackResolver
-import org.apache.log4j.{Level, Logger}
+import org.apache.logging.log4j.{Level, LogManager}
+import org.apache.logging.log4j.core.Logger
import org.apache.spark.internal.Logging
@@ -38,8 +39,9 @@ import org.apache.spark.internal.Logging
private[spark] class SparkRackResolver(conf: Configuration) extends Logging {
// RackResolver logs an INFO message whenever it resolves a rack, which is
way too often.
- if (Logger.getLogger(classOf[RackResolver]).getLevel == null) {
- Logger.getLogger(classOf[RackResolver]).setLevel(Level.WARN)
+ val logger = LogManager.getLogger(classOf[RackResolver])
+ if (logger.getLevel != Level.WARN) {
+ logger.asInstanceOf[Logger].setLevel(Level.WARN)
}
private val dnsToSwitchMapping: DNSToSwitchMapping = {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
index 43b8686..8d8d132 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.v2.jdbc
import java.sql.{Connection, DriverManager}
import java.util.Properties
-import org.apache.log4j.Level
+import org.apache.logging.log4j.Level
import org.apache.spark.SparkConf
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index d53e09b..e17b748 100644
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.Driver
import org.apache.hadoop.hive.ql.processors._
import org.apache.hadoop.hive.ql.session.SessionState
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
-import org.apache.log4j.Level
import org.apache.thrift.transport.TSocket
import org.slf4j.LoggerFactory
import sun.misc.{Signal, SignalHandler}
@@ -327,7 +326,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
if (!isRemoteMode) {
SparkSQLEnv.init()
if (sessionState.getIsSilent) {
- SparkSQLEnv.sparkContext.setLogLevel(Level.WARN.toString)
+ SparkSQLEnv.sparkContext.setLogLevel("warn")
}
} else {
// Hive 1.2 + not supported in CLI
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
index 951f927..af11b81 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreLazyInitializationSuite.scala
@@ -17,6 +17,9 @@
package org.apache.spark.sql.hive
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.core.Logger
+
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.util.Utils
@@ -30,7 +33,7 @@ class HiveMetastoreLazyInitializationSuite extends
SparkFunSuite {
.enableHiveSupport()
.config("spark.hadoop.hive.metastore.uris", "thrift://127.0.0.1:11111")
.getOrCreate()
- val originalLevel = org.apache.log4j.Logger.getRootLogger().getLevel
+ val originalLevel = LogManager.getRootLogger.asInstanceOf[Logger].getLevel
val originalClassLoader = Thread.currentThread().getContextClassLoader
try {
// Avoid outputting a lot of expected warning logs
diff --git a/sql/hive/src/test/resources/data/conf/hive-log4j.properties
b/sql/hive/src/test/resources/data/conf/hive-log4j.properties
deleted file mode 100644
index 83fd03a..0000000
--- a/sql/hive/src/test/resources/data/conf/hive-log4j.properties
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.root.logger=DEBUG,DRFA
-hive.log.dir=${build.dir.hive}/ql/tmp/
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=WARN
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Roll over at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}
(%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p
%c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop
Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
-
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 3769de0..07361cf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -535,10 +535,11 @@ private[hive] class TestHiveSparkSession(
def reset(): Unit = {
try {
// HACK: Hive is too noisy by default.
- org.apache.log4j.LogManager.getCurrentLoggers.asScala.foreach { log =>
- val logger = log.asInstanceOf[org.apache.log4j.Logger]
- if (!logger.getName.contains("org.apache.spark")) {
- logger.setLevel(org.apache.log4j.Level.WARN)
+ org.apache.logging.log4j.LogManager.getContext(false)
+
.asInstanceOf[org.apache.logging.log4j.core.LoggerContext].getConfiguration.getLoggers()
+ .asScala.foreach { case (_, log) =>
+ if (!log.getName.contains("org.apache.spark")) {
+ log.setLevel(org.apache.logging.log4j.Level.WARN)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
