This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push: new 0eccc3a145 [spark] Clean up maven dependencies (#6082) 0eccc3a145 is described below commit 0eccc3a1453a894e6c95303d2c6f80df50650300 Author: Zouxxyy <zouxinyu....@alibaba-inc.com> AuthorDate: Mon Aug 18 10:05:02 2025 +0800 [spark] Clean up maven dependencies (#6082) --- paimon-spark/paimon-spark-3.2/pom.xml | 32 +------ paimon-spark/paimon-spark-3.3/pom.xml | 32 +------ paimon-spark/paimon-spark-3.4/pom.xml | 31 +------ paimon-spark/paimon-spark-3.5/pom.xml | 58 +----------- paimon-spark/paimon-spark-4.0/pom.xml | 24 +---- paimon-spark/paimon-spark-common/pom.xml | 5 - paimon-spark/paimon-spark-ut/pom.xml | 148 +----------------------------- paimon-spark/paimon-spark4-common/pom.xml | 2 +- paimon-spark/pom.xml | 102 ++++++++++---------- 9 files changed, 71 insertions(+), 363 deletions(-) diff --git a/paimon-spark/paimon-spark-3.2/pom.xml b/paimon-spark/paimon-spark-3.2/pom.xml index cc62cde3b3..440731776b 100644 --- a/paimon-spark/paimon-spark-3.2/pom.xml +++ b/paimon-spark/paimon-spark-3.2/pom.xml @@ -38,15 +38,13 @@ under the License. <dependencies> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark3-common</artifactId> - <version>${project.version}</version> + <artifactId>paimon-format</artifactId> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark-common_${scala.binary.version}</artifactId> + <artifactId>paimon-spark3-common</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> @@ -73,11 +71,6 @@ under the License. <version>${spark.version}</version> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> - <!-- test --> <dependency> @@ -88,13 +81,6 @@ under the License. <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-format</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> @@ -118,20 +104,6 @@ under the License. <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-3.3/pom.xml b/paimon-spark/paimon-spark-3.3/pom.xml index 22e4cf269c..b198223c0b 100644 --- a/paimon-spark/paimon-spark-3.3/pom.xml +++ b/paimon-spark/paimon-spark-3.3/pom.xml @@ -38,15 +38,13 @@ under the License. <dependencies> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark3-common</artifactId> - <version>${project.version}</version> + <artifactId>paimon-format</artifactId> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark-common_${scala.binary.version}</artifactId> + <artifactId>paimon-spark3-common</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> @@ -73,11 +71,6 @@ under the License. <version>${spark.version}</version> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> - <!-- test --> <dependency> @@ -88,13 +81,6 @@ under the License. <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-format</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> @@ -118,20 +104,6 @@ under the License. <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-3.4/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml index 14d39806e7..1d852d1a9f 100644 --- a/paimon-spark/paimon-spark-3.4/pom.xml +++ b/paimon-spark/paimon-spark-3.4/pom.xml @@ -38,13 +38,12 @@ under the License. <dependencies> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark3-common</artifactId> - <version>${project.version}</version> + <artifactId>paimon-format</artifactId> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark-common_${scala.binary.version}</artifactId> + <artifactId>paimon-spark3-common</artifactId> <version>${project.version}</version> </dependency> @@ -72,11 +71,6 @@ under the License. <version>${spark.version}</version> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> - <!-- test --> <dependency> @@ -87,13 +81,6 @@ under the License. <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-format</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> @@ -117,20 +104,6 @@ under the License. <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-3.5/pom.xml b/paimon-spark/paimon-spark-3.5/pom.xml index ee87e3e8da..7aa7d4d674 100644 --- a/paimon-spark/paimon-spark-3.5/pom.xml +++ b/paimon-spark/paimon-spark-3.5/pom.xml @@ -38,13 +38,12 @@ under the License. <dependencies> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark3-common</artifactId> - <version>${project.version}</version> + <artifactId>paimon-format</artifactId> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark-common_${scala.binary.version}</artifactId> + <artifactId>paimon-spark3-common</artifactId> <version>${project.version}</version> </dependency> @@ -72,11 +71,6 @@ under the License. <version>${spark.version}</version> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> - <!-- test --> <dependency> @@ -93,16 +87,6 @@ under the License. <version>${spark.version}</version> <classifier>tests</classifier> <scope>test</scope> - <exclusions> - <exclusion> - <artifactId>parquet-hadoop</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - <exclusion> - <artifactId>parquet-column</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - </exclusions> </dependency> <dependency> @@ -120,44 +104,6 @@ under the License. <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-format</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-hadoop</artifactId> - <version>${parquet.version}</version> - <exclusions> - <exclusion> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - </exclusion> - <exclusion> - <artifactId>zstd-jni</artifactId> - <groupId>com.github.luben</groupId> - </exclusion> - </exclusions> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-4.0/pom.xml b/paimon-spark/paimon-spark-4.0/pom.xml index 50158bbeb6..8d0e2097f9 100644 --- a/paimon-spark/paimon-spark-4.0/pom.xml +++ b/paimon-spark/paimon-spark-4.0/pom.xml @@ -36,6 +36,11 @@ under the License. </properties> <dependencies> + <dependency> + <groupId>org.apache.paimon</groupId> + <artifactId>paimon-format</artifactId> + </dependency> + <dependency> <groupId>org.apache.paimon</groupId> <artifactId>paimon-spark4-common</artifactId> @@ -72,11 +77,6 @@ under the License. <version>${spark.version}</version> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> - <!-- test --> <dependency> @@ -116,20 +116,6 @@ under the License. <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-common/pom.xml b/paimon-spark/paimon-spark-common/pom.xml index ad352e32d3..e73f7f62b5 100644 --- a/paimon-spark/paimon-spark-common/pom.xml +++ b/paimon-spark/paimon-spark-common/pom.xml @@ -67,11 +67,6 @@ under the License. <artifactId>antlr4-runtime</artifactId> <version>${antlr4.version}</version> </dependency> - - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - </dependency> </dependencies> <build> diff --git a/paimon-spark/paimon-spark-ut/pom.xml b/paimon-spark/paimon-spark-ut/pom.xml index d1ceafae23..6a8713ac84 100644 --- a/paimon-spark/paimon-spark-ut/pom.xml +++ b/paimon-spark/paimon-spark-ut/pom.xml @@ -33,77 +33,23 @@ under the License. <properties> <spark.version>${paimon-spark-common.spark.version}</spark.version> - <jackson.version>2.15.2</jackson.version> - <janino.test.version>3.1.9</janino.test.version> </properties> <dependencies> - - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-common</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - + <!-- Add to the beginning, just to make IDEA prioritize loading Paimon's conflicting classes, + such as Parquet and ORC. Remove this once IDEA's bug is fixed. --> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-core</artifactId> - <version>${project.version}</version> - <scope>test</scope> + <artifactId>paimon-format</artifactId> </dependency> <dependency> <groupId>org.apache.paimon</groupId> <artifactId>${paimon-sparkx-common}</artifactId> <version>${project.version}</version> - <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-spark-common_${scala.binary.version}</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-sql_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <artifactId>parquet-column</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - <exclusion> - <artifactId>parquet-hadoop</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - </exclusions> - </dependency> + <!-- Below are test dependencies which need to be distinguished for different Spark versions. --> <dependency> <groupId>org.apache.spark</groupId> @@ -111,20 +57,6 @@ under the License. <version>${spark.version}</version> <classifier>tests</classifier> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <artifactId>parquet-column</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - <exclusion> - <artifactId>parquet-hadoop</artifactId> - <groupId>org.apache.parquet</groupId> - </exclusion> - </exclusions> </dependency> <dependency> @@ -141,16 +73,6 @@ under the License. <version>${spark.version}</version> <classifier>tests</classifier> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> @@ -158,16 +80,6 @@ under the License. <artifactId>spark-hive_${scala.binary.version}</artifactId> <version>${spark.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> @@ -176,57 +88,6 @@ under the License. <version>${spark.version}</version> <scope>test</scope> </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_${scala.binary.version}</artifactId> - <version>${jackson.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>${protobuf-java.version}</version> - </dependency> - - <dependency> - <groupId>com.squareup.okhttp3</groupId> - <artifactId>mockwebserver</artifactId> - <version>${okhttp.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-format</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-hadoop</artifactId> - <version>${parquet.version}</version> - <exclusions> - <exclusion> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - </exclusion> - <exclusion> - <artifactId>zstd-jni</artifactId> - <groupId>com.github.luben</groupId> - </exclusion> - </exclusions> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.codehaus.janino</groupId> - <artifactId>janino</artifactId> - <version>${janino.test.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> @@ -244,7 +105,6 @@ under the License. </execution> </executions> </plugin> - </plugins> </build> </project> diff --git a/paimon-spark/paimon-spark4-common/pom.xml b/paimon-spark/paimon-spark4-common/pom.xml index 916d6423e0..9206fbd0da 100644 --- a/paimon-spark/paimon-spark4-common/pom.xml +++ b/paimon-spark/paimon-spark4-common/pom.xml @@ -46,7 +46,7 @@ under the License. <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-sql-api_2.13</artifactId> + <artifactId>spark-sql-api_${scala.binary.version}</artifactId> <version>${spark.version}</version> <exclusions> <exclusion> diff --git a/paimon-spark/pom.xml b/paimon-spark/pom.xml index a546d8d889..4d688ac146 100644 --- a/paimon-spark/pom.xml +++ b/paimon-spark/pom.xml @@ -44,6 +44,23 @@ under the License. <dependencyManagement> <dependencies> + <dependency> + <groupId>org.apache.paimon</groupId> + <artifactId>paimon-format</artifactId> + <version>${project.version}</version> + <exclusions> + <!-- todo(fix me): we should use the jackson libraries shaded by Paimon. --> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> @@ -60,22 +77,6 @@ under the License. <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-slf4j2-impl</artifactId> </exclusion> - <exclusion> - <groupId>org.apache.orc</groupId> - <artifactId>orc-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.orc</groupId> - <artifactId>orc-mapreduce</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-column</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-hadoop</artifactId> - </exclusion> </exclusions> </dependency> @@ -96,18 +97,6 @@ under the License. <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-slf4j2-impl</artifactId> </exclusion> - <exclusion> - <groupId>org.apache.orc</groupId> - <artifactId>orc-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.orc</groupId> - <artifactId>orc-mapreduce</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-column</artifactId> - </exclusion> <exclusion> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> @@ -145,12 +134,6 @@ under the License. </exclusions> </dependency> - <dependency> - <groupId>org.apache.paimon</groupId> - <artifactId>paimon-bundle</artifactId> - <version>${project.version}</version> - </dependency> - <!-- test --> <dependency> @@ -215,6 +198,8 @@ under the License. </dependencyManagement> <dependencies> + <!-- Common dependencies --> + <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> @@ -233,51 +218,56 @@ under the License. <version>${scala.version}</version> </dependency> - <!-- Test --> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.binary.version}</artifactId> - <version>3.1.0</version> - <scope>test</scope> + <groupId>org.apache.paimon</groupId> + <artifactId>paimon-bundle</artifactId> + <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-common</artifactId> + <artifactId>paimon-hive-common</artifactId> <version>${project.version}</version> - <type>test-jar</type> + </dependency> + + <!-- Below are common test dependencies, place them in this parent module to facilitate + reuse by all downstream test modules. --> + + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.binary.version}</artifactId> + <version>3.1.0</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-core</artifactId> + <artifactId>paimon-test-utils</artifactId> <version>${project.version}</version> <scope>test</scope> - <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-hive-common</artifactId> + <artifactId>paimon-common</artifactId> <version>${project.version}</version> + <classifier>tests</classifier> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-hive-common</artifactId> + <artifactId>paimon-core</artifactId> <version>${project.version}</version> <classifier>tests</classifier> - <type>test-jar</type> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.paimon</groupId> - <artifactId>paimon-test-utils</artifactId> + <artifactId>paimon-hive-common</artifactId> <version>${project.version}</version> + <classifier>tests</classifier> <scope>test</scope> </dependency> @@ -286,13 +276,13 @@ under the License. <artifactId>paimon-s3</artifactId> <version>${project.version}</version> <scope>test</scope> - <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.paimon</groupId> <artifactId>paimon-s3</artifactId> <version>${project.version}</version> + <classifier>tests</classifier> <scope>test</scope> </dependency> @@ -315,6 +305,20 @@ under the License. <version>${aws.version}</version> <scope>test</scope> </dependency> + + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <version>${protobuf-java.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.squareup.okhttp3</groupId> + <artifactId>mockwebserver</artifactId> + <version>${okhttp.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build>