This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 8e66ac833 [CELEBORN-1994] Introduce disruptor dependency to support
asynchronous logging of log4j2
8e66ac833 is described below
commit 8e66ac833aa4665d139775287d9b9b2c5ebf193d
Author: SteNicholas <[email protected]>
AuthorDate: Tue May 13 19:45:51 2025 +0800
[CELEBORN-1994] Introduce disruptor dependency to support asynchronous
logging of log4j2
### What changes were proposed in this pull request?
Introduce disruptor dependency to support asynchronous logging of log4j2.
### Why are the changes needed?
We add
`-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector`
in `CELEBORN_MASTER_JAVA_OPTS` and `CELEBORN_WOKRER_JAVA_OPTS` for production
environment. `AsyncLoggerContextSelector` depends on disruptor dependency.
Therefore, it's recommend to introduce disruptor dependency to support log4j2
asynchronous loggers.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Cluster test.
Closes #3246 from SteNicholas/CELEBORN-1994.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: mingji <[email protected]>
---
LICENSE-binary | 1 +
dev/deps/dependencies-server | 1 +
master/pom.xml | 4 ++++
pom.xml | 7 +++++++
project/CelebornBuild.scala | 4 ++++
worker/pom.xml | 4 ++++
6 files changed, 21 insertions(+)
diff --git a/LICENSE-binary b/LICENSE-binary
index 20d746b1b..8228d461e 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -216,6 +216,7 @@ com.fasterxml.jackson.module:jackson-module-scala
com.google.code.findbugs:jsr305
com.google.guava:failureaccess
com.google.guava:guava
+com.lmax:disruptor
com.thoughtworks.paranamer:paranamer
commons-cli:commons-cli
commons-io:commons-io
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index 8f24085e3..eb08ed80f 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -25,6 +25,7 @@ commons-crypto/1.0.0//commons-crypto-1.0.0.jar
commons-io/2.17.0//commons-io-2.17.0.jar
commons-lang3/3.17.0//commons-lang3-3.17.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
+disruptor/3.4.4//disruptor-3.4.4.jar
failureaccess/1.0.2//failureaccess-1.0.2.jar
guava/33.1.0-jre//guava-33.1.0-jre.jar
hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
diff --git a/master/pom.xml b/master/pom.xml
index 94858ba63..b2b2b06cd 100644
--- a/master/pom.xml
+++ b/master/pom.xml
@@ -86,6 +86,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.celeborn</groupId>
diff --git a/pom.xml b/pom.xml
index 0b7d37af0..31a38cda1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,6 +90,7 @@
<junit.version>4.13.2</junit.version>
<leveldb.version>1.8</leveldb.version>
<log4j2.version>2.24.3</log4j2.version>
+ <disruptor.version>3.4.4</disruptor.version>
<lz4-java.version>1.8.0</lz4-java.version>
<mockito.version>4.11.0</mockito.version>
<mockito-scalatest.version>1.17.14</mockito-scalatest.version>
@@ -245,6 +246,12 @@
<artifactId>log4j-1.2-api</artifactId>
<version>${log4j2.version}</version>
</dependency>
+ <dependency>
+ <!-- log4j2 has an optional dependency on disruptor like
AsyncLoggerContextSelector. -->
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index e8818977f..ecd4fb4be 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -58,6 +58,7 @@ object Dependencies {
val junitVersion = "4.13.2"
val leveldbJniVersion = "1.8"
val log4j2Version = "2.24.3"
+ val disruptorVersion = "3.4.4"
val jdkToolsVersion = "0.1"
val metricsVersion = "4.2.25"
val mockitoVersion = "4.11.0"
@@ -148,6 +149,7 @@ object Dependencies {
val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4j2Version
val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version
val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" %
log4j2Version
+ val disruptor = "com.lmax" % "disruptor" % disruptorVersion
val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion
val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion
val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion
@@ -748,6 +750,7 @@ object CelebornMaster {
Dependencies.hadoopClientApi,
Dependencies.log4j12Api,
Dependencies.log4jSlf4jImpl,
+ Dependencies.disruptor,
Dependencies.ratisClient,
Dependencies.ratisCommon,
Dependencies.ratisGrpc,
@@ -783,6 +786,7 @@ object CelebornWorker {
Dependencies.ioNetty,
Dependencies.log4j12Api,
Dependencies.log4jSlf4jImpl,
+ Dependencies.disruptor,
Dependencies.leveldbJniAll,
Dependencies.roaringBitmap,
Dependencies.rocksdbJni,
diff --git a/worker/pom.xml b/worker/pom.xml
index 6d9c858a1..ad6b1156b 100644
--- a/worker/pom.xml
+++ b/worker/pom.xml
@@ -66,6 +66,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ </dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
