This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.10.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/release-0.10.1-rc1 by this
push:
new eac1a02 [HUDI-3195] Fix spark 3 pom (#4555)
eac1a02 is described below
commit eac1a02a53e6ae501eee986dd75cf1cc907f3fcf
Author: Raymond Xu <[email protected]>
AuthorDate: Mon Jan 10 16:37:55 2022 -0800
[HUDI-3195] Fix spark 3 pom (#4555)
* [HUDI-3195] Fix spark 3 pom
- drop 3.0.x profile
- update readme
- update build CI bot.yml
* fix spark 3 bundle name
---
.github/workflows/bot.yml | 4 ++--
README.md | 5 +----
pom.xml | 25 +++++++++++++++++++------
3 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index f1f178e..52c077a 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -21,9 +21,9 @@ jobs:
- scala: "scala-2.11"
spark: "spark2,spark-shade-unbundle-avro"
- scala: "scala-2.12"
- spark: "spark3,spark3.0.x"
+ spark: "spark3.0.x"
- scala: "scala-2.12"
- spark: "spark3,spark3.0.x,spark-shade-unbundle-avro"
+ spark: "spark3.0.x,spark-shade-unbundle-avro"
- scala: "scala-2.12"
spark: "spark3"
- scala: "scala-2.12"
diff --git a/README.md b/README.md
index 4f48fc6..4bc2708 100644
--- a/README.md
+++ b/README.md
@@ -83,12 +83,9 @@ mvn clean package -DskipTests -Dscala-2.12
The default Spark version supported is 2.4.4. To build for different Spark 3
versions, use the corresponding profile
```
-# Build against Spark 3.2.0 (default build shipped with the public jars)
+# Build against Spark 3.1.2 (the default build shipped with the public Spark 3
bundle)
mvn clean package -DskipTests -Dspark3
-# Build against Spark 3.1.2
-mvn clean package -DskipTests -Dspark3.1.x
-
# Build against Spark 3.0.3
mvn clean package -DskipTests -Dspark3.0.x
```
diff --git a/pom.xml b/pom.xml
index cd1825d..da92455 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,12 +113,10 @@
<prometheus.version>0.8.0</prometheus.version>
<http.version>4.4.1</http.version>
<spark.version>${spark2.version}</spark.version>
- <sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
+ <sparkbundle.version></sparkbundle.version>
<flink.version>1.13.1</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.1.2</spark3.version>
- <spark2bundle.version></spark2bundle.version>
- <spark3bundle.version>3</spark3bundle.version>
<hudi.spark.module>hudi-spark2</hudi.spark.module>
<avro.version>1.8.2</avro.version>
<scala11.version>2.11.12</scala11.version>
@@ -1568,7 +1566,7 @@
<id>spark3</id>
<properties>
<spark.version>${spark3.version}</spark.version>
- <sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
+ <sparkbundle.version>${spark3.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<hudi.spark.module>hudi-spark3</hudi.spark.module>
@@ -1590,12 +1588,27 @@
<profile>
<id>spark3.0.x</id>
-<!-- for spark 3.0.x we need override the follow propeprties to package
and run test-->
<properties>
- <spark3.version>3.0.0</spark3.version>
+ <spark3.version>3.0.3</spark3.version>
<spark.version>${spark3.version}</spark.version>
+ <sparkbundle.version>${spark3.version}</sparkbundle.version>
+ <scala.version>${scala12.version}</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
+ <hudi.spark.module>hudi-spark3</hudi.spark.module>
<scalatest.version>3.0.1</scalatest.version>
+ <kafka.version>2.4.1</kafka.version>
+ <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
+
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
+
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
+
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+ <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
+ <skipITs>true</skipITs>
</properties>
+ <activation>
+ <property>
+ <name>spark3.0.x</name>
+ </property>
+ </activation>
</profile>
<profile>
