This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cedac51f29 Upgrade spark from 3.2 to 3.5 (#11702)
cedac51f29 is described below
commit cedac51f2950e75a585297250157cc6ae5a969e1
Author: Gonzalo Ortiz Jaureguizar <[email protected]>
AuthorDate: Fri Oct 13 12:31:50 2023 +0200
Upgrade spark from 3.2 to 3.5 (#11702)
---
pinot-common/pom.xml | 2 +-
pinot-connectors/pinot-spark-3-connector/pom.xml | 2 +-
pinot-distribution/pinot-assembly.xml | 4 ++--
.../pom.xml | 6 +++---
.../batch/spark3/SparkSegmentGenerationJobRunner.java | 0
.../batch/spark3/SparkSegmentMetadataPushJobRunner.java | 0
.../batch/spark3/SparkSegmentTarPushJobRunner.java | 0
.../batch/spark3/SparkSegmentUriPushJobRunner.java | 0
.../resources/segmentCreationAndMetadataPushJobSpec.yaml | 0
.../main/resources/segmentCreationAndTarPushJobSpec.yaml | 0
.../main/resources/segmentCreationAndUriPushJobSpec.yaml | 0
.../src/main/resources/segmentCreationJobSpec.yaml | 0
.../src/main/resources/segmentTarPushJobSpec.yaml | 0
.../src/main/resources/segmentUriPushJobSpec.yaml | 0
.../batch/spark3/SparkSegmentGenerationJobRunnerTest.java | 0
pinot-plugins/pinot-batch-ingestion/pom.xml | 2 +-
pinot-spi/pom.xml | 2 +-
pom.xml | 15 +++++++++++++--
18 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index 570d83dff2..1ea5f53b67 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -257,7 +257,7 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
diff --git a/pinot-connectors/pinot-spark-3-connector/pom.xml
b/pinot-connectors/pinot-spark-3-connector/pom.xml
index 5efe8a3a9a..10923484b8 100644
--- a/pinot-connectors/pinot-spark-3-connector/pom.xml
+++ b/pinot-connectors/pinot-spark-3-connector/pom.xml
@@ -33,7 +33,7 @@
<url>https://pinot.apache.org/</url>
<properties>
<pinot.root>${basedir}/../..</pinot.root>
- <spark.version>3.4.0</spark.version>
+ <spark.version>3.5.0</spark.version>
<antlr-runtime.version>4.8</antlr-runtime.version>
<scalatest.version>3.1.1</scalatest.version>
<shadeBase>org.apache.pinot.\$internal</shadeBase>
diff --git a/pinot-distribution/pinot-assembly.xml
b/pinot-distribution/pinot-assembly.xml
index eef6100d27..16b1051d20 100644
--- a/pinot-distribution/pinot-assembly.xml
+++ b/pinot-distribution/pinot-assembly.xml
@@ -94,10 +94,10 @@
</file>
<file>
<source>
-
${pinot.root}/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/target/pinot-batch-ingestion-spark-3.2-${project.version}-shaded.jar
+
${pinot.root}/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/target/pinot-batch-ingestion-spark-3-${project.version}-shaded.jar
</source>
<destName>
-
plugins-external/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pinot-batch-ingestion-spark-3.2-${project.version}-shaded.jar
+
plugins-external/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pinot-batch-ingestion-spark-3-${project.version}-shaded.jar
</destName>
</file>
<!-- End Include Pinot Batch Ingestion Plugins-->
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
similarity index 97%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
index 0d11fe655e..95f3fcdbc2 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
@@ -30,14 +30,14 @@
<relativePath>..</relativePath>
</parent>
- <artifactId>pinot-batch-ingestion-spark-3.2</artifactId>
- <name>Pinot Batch Ingestion for Spark 3.2</name>
+ <artifactId>pinot-batch-ingestion-spark-3</artifactId>
+ <name>Pinot Batch Ingestion for Spark 3</name>
<url>https://pinot.apache.org/</url>
<properties>
<pinot.root>${basedir}/../../..</pinot.root>
<phase.prop>package</phase.prop>
<scala.major.version>2.12</scala.major.version>
- <spark.version>3.2.1</spark.version>
+ <spark.version>3.5.0</spark.version>
<scala.minor.version>2.12.15</scala.minor.version>
<commons-lang3.version>3.11</commons-lang3.version>
</properties>
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunner.java
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentMetadataPushJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentMetadataPushJobRunner.java
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentMetadataPushJobRunner.java
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentMetadataPushJobRunner.java
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentUriPushJobRunner.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentUriPushJobRunner.java
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentUriPushJobRunner.java
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentUriPushJobRunner.java
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndMetadataPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndMetadataPushJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndMetadataPushJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndMetadataPushJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentCreationJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentCreationJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentTarPushJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentTarPushJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentTarPushJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentUriPushJobSpec.yaml
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/main/resources/segmentUriPushJobSpec.yaml
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/resources/segmentUriPushJobSpec.yaml
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunnerTest.java
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunnerTest.java
similarity index 100%
rename from
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunnerTest.java
rename to
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentGenerationJobRunnerTest.java
diff --git a/pinot-plugins/pinot-batch-ingestion/pom.xml
b/pinot-plugins/pinot-batch-ingestion/pom.xml
index 26bc514af7..faed51339d 100644
--- a/pinot-plugins/pinot-batch-ingestion/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pom.xml
@@ -41,7 +41,7 @@
<modules>
<module>pinot-batch-ingestion-common</module>
<module>pinot-batch-ingestion-spark-2.4</module>
- <module>pinot-batch-ingestion-spark-3.2</module>
+ <module>pinot-batch-ingestion-spark-3</module>
<module>pinot-batch-ingestion-hadoop</module>
<module>pinot-batch-ingestion-standalone</module>
diff --git a/pinot-spi/pom.xml b/pinot-spi/pom.xml
index 956665d3d2..08e5f6709d 100644
--- a/pinot-spi/pom.xml
+++ b/pinot-spi/pom.xml
@@ -96,7 +96,7 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
diff --git a/pom.xml b/pom.xml
index ea8efa66da..0e2d425cf0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -151,7 +151,7 @@
<snappy-java.version>1.1.10.4</snappy-java.version>
<zstd-jni.version>1.5.2-3</zstd-jni.version>
<lz4-java.version>1.8.0</lz4-java.version>
- <log4j.version>2.17.1</log4j.version>
+ <log4j.version>2.20.0</log4j.version>
<netty.version>4.1.94.Final</netty.version>
<reactivestreams.version>1.0.3</reactivestreams.version>
<jts.version>1.19.0</jts.version>
@@ -605,7 +605,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.25</version>
+ <version>2.0.9</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -618,9 +618,20 @@
<version>${log4j.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+ <dependency>
+ <!--
+ We don't use slf4j but slf4j2, so we can just ignore this dependency.
+ Instead of just remove this dependency, we set the scope as provided.
+ We cannot just remove the dependency because that would create
conflicts in the enforcer
+ -->
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
