This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new f0e4f73 [refactor](script) optimize build scala version (#198)
f0e4f73 is described below
commit f0e4f7384fc190b34777cbaf46f731bb5990fd7b
Author: gnehil <[email protected]>
AuthorDate: Wed May 8 10:17:50 2024 +0800
[refactor](script) optimize build scala version (#198)
---
.github/workflows/build-extension.yml | 35 ++++++++++++++++++------------
spark-doris-connector/build.sh | 19 ++++++++++++-----
spark-doris-connector/pom.xml | 40 +++++++++++++++++++++++++++++------
3 files changed, 68 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/build-extension.yml
b/.github/workflows/build-extension.yml
index bbc68f7..5651c5a 100644
--- a/.github/workflows/build-extension.yml
+++ b/.github/workflows/build-extension.yml
@@ -46,47 +46,54 @@ jobs:
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.3.4 \
- -Dscala.version=2.11 \
- -Dspark.major.version=2.3
+ -Dscala.version=2.11.8 \
+ -Dspark.major.version=2.3 \
+ -Dscala.major.version=2.11
- name: Build spark connector 2.4 2.11
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.4.0 \
- -Dscala.version=2.11 \
- -Dspark.major.version=2.4
+ -Dscala.version=2.11.8 \
+ -Dspark.major.version=2.4 \
+ -Dscala.major.version=2.11
- name: Build spark connector 2.4 2.12
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.4.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=2.4
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=2.4 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.1
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.1.2 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.1
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.1 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.2
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.2.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.2
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.2 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.3
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.3.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.3
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.3 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.4
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.4.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.4
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.4 \
+ -Dscala.major.version=2.12
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
index ab2f770..3c125e8 100755
--- a/spark-doris-connector/build.sh
+++ b/spark-doris-connector/build.sh
@@ -170,9 +170,15 @@ selectSpark() {
done
}
+SCALA_VERSION=0
selectScala
ScalaVer=$?
-SCALA_VERSION="2.1${ScalaVer}"
+if [ ${ScalaVer} -eq 1 ]; then
+ SCALA_VERSION="2.11.8"
+elif [ ${ScalaVer} -eq 2 ]; then
+ SCALA_VERSION="2.12.10"
+fi
+
SPARK_VERSION=0
selectSpark
@@ -196,27 +202,30 @@ elif [ ${SparkVer} -eq 7 ]; then
SPARK_VERSION=$ver
fi
-if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12" ]]; then
+if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12.10" ]]; then
echo_r "Spark 2.3 is not compatible with scala 2.12, will exit."
exit 1
-elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11" ]]; then
+elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.8" ]]; then
echo_r "Spark 3.x is not compatible with scala 2.11, will exit."
exit 1
fi
# extract major version:
# eg: 3.1.2 -> 3.1
+SCALA_MAJOR_VERSION=0
+[ ${SCALA_VERSION} != 0 ] && SCALA_MAJOR_VERSION=${SCALA_VERSION%.*}
SPARK_MAJOR_VERSION=0
[ ${SPARK_VERSION} != 0 ] && SPARK_MAJOR_VERSION=${SPARK_VERSION%.*}
-echo_g " scala version: ${SCALA_VERSION}"
+echo_g " scala version: ${SCALA_VERSION}, major version:
${SCALA_MAJOR_VERSION}"
echo_g " spark version: ${SPARK_VERSION}, major version:
${SPARK_MAJOR_VERSION}"
echo_g " build starting..."
${MVN_BIN} clean package \
-Dspark.version=${SPARK_VERSION} \
-Dscala.version=${SCALA_VERSION} \
- -Dspark.major.version=${SPARK_MAJOR_VERSION} "$@"
+ -Dspark.major.version=${SPARK_MAJOR_VERSION} \
+ -Dscala.major.version=${SCALA_MAJOR_VERSION} "$@"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
index fef519d..a9fd180 100644
--- a/spark-doris-connector/pom.xml
+++ b/spark-doris-connector/pom.xml
@@ -26,7 +26,7 @@
<version>29</version>
</parent>
<groupId>org.apache.doris</groupId>
-
<artifactId>spark-doris-connector-${spark.major.version}_${scala.version}</artifactId>
+
<artifactId>spark-doris-connector-${spark.major.version}_${scala.major.version}</artifactId>
<version>${revision}</version>
<name>Spark Doris Connector</name>
<url>https://doris.apache.org/</url>
@@ -70,7 +70,8 @@
<revision>1.4.0-SNAPSHOT</revision>
<spark.version>3.1.2</spark.version>
<spark.major.version>3.1</spark.major.version>
- <scala.version>2.12</scala.version>
+ <scala.version>2.12.10</scala.version>
+ <scala.major.version>2.12</scala.major.version>
<libthrift.version>0.16.0</libthrift.version>
<arrow.version>13.0.0</arrow.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -86,6 +87,12 @@
<groupId>org.apache.doris</groupId>
<artifactId>thrift-service</artifactId>
<version>${thrift-service.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
@@ -96,14 +103,14 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.version}</artifactId>
+ <artifactId>spark-core_${scala.major.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.version}</artifactId>
+ <artifactId>spark-sql_${scala.major.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
@@ -151,7 +158,7 @@
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-scala_${scala.version}</artifactId>
+ <artifactId>mockito-scala_${scala.major.version}</artifactId>
<version>1.4.7</version>
<exclusions>
<exclusion>
@@ -177,7 +184,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql-kafka-0-10_${scala.version}</artifactId>
+
<artifactId>spark-sql-kafka-0-10_${scala.major.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
@@ -202,7 +209,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.version}</artifactId>
+
<artifactId>jackson-module-scala_${scala.major.version}</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
@@ -226,6 +233,21 @@
<version>4.2.0</version>
<scope>test</scope>
</dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.scala-lang/scala-library
-->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.13</version>
+ </dependency>
+
</dependencies>
<build>
@@ -329,6 +351,10 @@
<pattern>org.apache.thrift</pattern>
<shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+
<shadedPattern>org.apache.doris.shaded.org.apache.http</shadedPattern>
+ </relocation>
</relocations>
</configuration>
<executions>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
