This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cc78e17b6 [GLUTEN-7023][CH] Shade dependency jars (#7027)
cc78e17b6 is described below
commit cc78e17b6569aa2bc45a0b7f620dac7104910a2e
Author: loudongfeng <[email protected]>
AuthorDate: Mon Sep 9 16:37:18 2024 +0800
[GLUTEN-7023][CH] Shade dependency jars (#7027)
* [GLUTEN-7023][CH] Shade dependency jars
* [GLUTEN-7023][CH] Shade dependency jars(update pacakge script and
getstart doc)
---
backends-clickhouse/pom.xml | 53 ++++++++++++++++++++++++++++++++------
docs/get-started/ClickHouse.md | 8 ++----
ep/build-clickhouse/src/package.sh | 2 --
3 files changed, 47 insertions(+), 16 deletions(-)
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index a2b685490..130fe8855 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -266,20 +266,57 @@
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>3.3.0</version>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
<configuration>
-
<finalName>${jar.assembly.name.prefix}-${project.version}-spark-${sparkbundle.version}</finalName>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+
<outputFile>target/${jar.assembly.name.prefix}-${project.version}-spark-${sparkbundle.version}-jar-with-dependencies.jar</outputFile>
+ <relocations>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+
<shadedPattern>${gluten.shade.packageName}.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>google.protobuf</pattern>
+
<shadedPattern>${gluten.shade.packageName}.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.gson</pattern>
+
<shadedPattern>${gluten.shade.packageName}.com.google.gson</shadedPattern>
+ <includes>
+ <include>com.google.gson.**</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>com.google.errorprone.annotations</pattern>
+
<shadedPattern>${gluten.shade.packageName}.com.google.errorprone.annotations</shadedPattern>
+ <includes>
+ <include>com.google.errorprone.annotations.**</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>com.google.j2objc</pattern>
+
<shadedPattern>${gluten.shade.packageName}.com.google.j2objc</shadedPattern>
+ <includes>
+ <include>com.google.j2objc.**</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>com.github.benmanes.caffeine</pattern>
+
<shadedPattern>${gluten.shade.packageName}.com.github.benmanes.caffeine</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.checkerframework</pattern>
+
<shadedPattern>${gluten.shade.packageName}.org.checkerframework</shadedPattern>
+ </relocation>
+ </relocations>
</configuration>
<executions>
<execution>
- <id>make-assembly</id>
<phase>package</phase>
<goals>
- <goal>single</goal>
+ <goal>shade</goal>
</goals>
</execution>
</executions>
diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md
index 966d146b3..a4b3174d1 100644
--- a/docs/get-started/ClickHouse.md
+++ b/docs/get-started/ClickHouse.md
@@ -148,9 +148,7 @@ The prerequisites are the same as the one mentioned above.
Compile Gluten with C
```
tar zxf spark-3.2.2-bin-hadoop2.7.tgz
cd spark-3.2.2-bin-hadoop2.7
-rm -f jars/protobuf-java-2.5.0.jar
-#download protobuf-java-3.23.4.jar, delta-core_2.12-2.0.1.jar and
delta-storage-2.0.1.jar
-wget
https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.23.4/protobuf-java-3.23.4.jar
-P ./jars
+#download delta-core_2.12-2.0.1.jar and delta-storage-2.0.1.jar
wget
https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.0.1/delta-core_2.12-2.0.1.jar
-P ./jars
wget
https://repo1.maven.org/maven2/io/delta/delta-storage/2.0.1/delta-storage-2.0.1.jar
-P ./jars
cp gluten-XXXXX-spark-3.2-jar-with-dependencies.jar jars/
@@ -161,9 +159,7 @@ cp gluten-XXXXX-spark-3.2-jar-with-dependencies.jar jars/
```
tar zxf spark-3.3.1-bin-hadoop2.7.tgz
cd spark-3.3.1-bin-hadoop2.7
-rm -f jars/protobuf-java-2.5.0.jar
-#download protobuf-java-3.23.4.jar, delta-core_2.12-2.2.0.jar and
delta-storage-2.2.0.jar
-wget
https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.23.4/protobuf-java-3.23.4.jar
-P ./jars
+#download delta-core_2.12-2.2.0.jar and delta-storage-2.2.0.jar
wget
https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.2.0/delta-core_2.12-2.2.0.jar
-P ./jars
wget
https://repo1.maven.org/maven2/io/delta/delta-storage/2.2.0/delta-storage-2.2.0.jar
-P ./jars
cp gluten-XXXXX-spark-3.3-jar-with-dependencies.jar jars/
diff --git a/ep/build-clickhouse/src/package.sh
b/ep/build-clickhouse/src/package.sh
index 242f7913a..8a3f00684 100755
--- a/ep/build-clickhouse/src/package.sh
+++ b/ep/build-clickhouse/src/package.sh
@@ -98,8 +98,6 @@ function build_gluten_by_spark_version() {
}
# download common 3rd party jars
-protobuf_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo"
-Dexec.args='${protobuf.version}' --non-recursive exec:exec)
-wget
https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/${protobuf_version}/protobuf-java-${protobuf_version}.jar
-P "${PACKAGE_DIR_PATH}"/jars/spark32
celeborn_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo"
-Dexec.args='${celeborn.version}' --non-recursive exec:exec)
wget
https://repo1.maven.org/maven2/org/apache/celeborn/celeborn-client-spark-3-shaded_2.12/${celeborn_version}/celeborn-client-spark-3-shaded_2.12-${celeborn_version}.jar
-P "${PACKAGE_DIR_PATH}"/jars/spark32
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
