This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 9ebda605f [#1409] fix(client): Netty Epoll is unavailable for the RSS
Client. (#1410)
9ebda605f is described below
commit 9ebda605fb25065339311d834d8a1d29472f9e50
Author: RickyMa <[email protected]>
AuthorDate: Tue Jan 2 13:56:41 2024 +0800
[#1409] fix(client): Netty Epoll is unavailable for the RSS Client. (#1410)
### What changes were proposed in this pull request?
Fix a bug that was introduced from
[#343](https://github.com/apache/incubator-uniffle/issues/343).
The Netty native libraries have never been successfully shaded. PR
[#924](https://github.com/apache/incubator-uniffle/pull/924) is not working at
all.
### Why are the changes needed?
For [#1409](https://github.com/apache/incubator-uniffle/issues/1409)
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
By running the RSS Netty client with Epoll mode successfully.
---
client-mr/core/pom.xml | 74 +++++++++++++++++++++++++++-----
client-spark/spark2-shaded/pom.xml | 74 +++++++++++++++++++++++++++-----
client-spark/spark3-shaded/pom.xml | 74 +++++++++++++++++++++++++++-----
client-tez/pom.xml | 86 ++++++++++++++++++++++++++++++++++++++
4 files changed, 278 insertions(+), 30 deletions(-)
diff --git a/client-mr/core/pom.xml b/client-mr/core/pom.xml
index f8ac47bce..d860b0b68 100644
--- a/client-mr/core/pom.xml
+++ b/client-mr/core/pom.xml
@@ -209,35 +209,89 @@
</relocations>
</configuration>
</execution>
- <!-- shade the native netty libs as well -->
+ </executions>
+ </plugin>
+ <!-- shade the native netty libs as well -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>rss.shade.native.packageName</name>
+ <value>${rss.shade.packageName}</value>
+ <regex>\.</regex>
+ <replacement>_</replacement>
+ <failIfNoMatch>true</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
<execution>
<id>unpack</id>
<phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
<configuration>
<target>
- <echo message="Shade netty native libraries to
${rss.shade.packageName}" />
- <unzip
src="${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/" />
+ <echo message="Shade netty native libraries to
${rss.shade.native.packageName}"/>
+ <unzip
src="${project.build.directory}/shaded/${project.artifactId}-${project.version}-shaded.jar"
+
dest="${project.build.directory}/unpacked/"/>
<echo message="renaming native epoll
library"></echo>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper
from="libnetty_transport_native_epoll_x86_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_x86_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_x86_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper
from="libnetty_transport_native_epoll_aarch_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_aarch_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper
from="libnetty_transport_native_kqueue_x86_64.jnilib"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_kqueue_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_aarch_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_kqueue_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib"
type="glob"></mapper>
</move>
<echo message="repackaging netty jar"></echo>
- <jar
destfile="${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked" />
+ <jar
destfile="${project.build.directory}/shaded/${project.artifactId}-${project.version}-shaded.jar"
+
basedir="${project.build.directory}/unpacked"/>
</target>
</configuration>
</execution>
diff --git a/client-spark/spark2-shaded/pom.xml
b/client-spark/spark2-shaded/pom.xml
index 471926234..74bea5399 100644
--- a/client-spark/spark2-shaded/pom.xml
+++ b/client-spark/spark2-shaded/pom.xml
@@ -126,35 +126,89 @@
</relocations>
</configuration>
</execution>
- <!-- shade the native netty libs as well -->
+ </executions>
+ </plugin>
+ <!-- shade the native netty libs as well -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>rss.shade.native.packageName</name>
+ <value>${rss.shade.packageName}</value>
+ <regex>\.</regex>
+ <replacement>_</replacement>
+ <failIfNoMatch>true</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
<execution>
<id>unpack</id>
<phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
<configuration>
<target>
- <echo message="Shade netty native libraries to
${rss.shade.packageName}" />
- <unzip src="${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/" />
+ <echo message="Shade netty native libraries to
${rss.shade.native.packageName}"/>
+ <unzip
src="${project.build.directory}/${project.artifactId}-${project.version}.jar"
+ dest="${project.build.directory}/unpacked/"/>
<echo message="renaming native epoll library"></echo>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_x86_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_x86_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_x86_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_aarch_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_aarch_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_kqueue_x86_64.jnilib"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_kqueue_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_aarch_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_kqueue_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib"
type="glob"></mapper>
</move>
<echo message="repackaging netty jar"></echo>
- <jar destfile="${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked" />
+ <jar
destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"
+ basedir="${project.build.directory}/unpacked"/>
</target>
</configuration>
</execution>
diff --git a/client-spark/spark3-shaded/pom.xml
b/client-spark/spark3-shaded/pom.xml
index 163ea0a96..615adb15d 100644
--- a/client-spark/spark3-shaded/pom.xml
+++ b/client-spark/spark3-shaded/pom.xml
@@ -126,35 +126,89 @@
</relocations>
</configuration>
</execution>
- <!-- shade the native netty libs as well -->
+ </executions>
+ </plugin>
+ <!-- shade the native netty libs as well -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>rss.shade.native.packageName</name>
+ <value>${rss.shade.packageName}</value>
+ <regex>\.</regex>
+ <replacement>_</replacement>
+ <failIfNoMatch>true</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
<execution>
<id>unpack</id>
<phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
<configuration>
<target>
- <echo message="Shade netty native libraries to
${rss.shade.packageName}" />
- <unzip src="${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/" />
+ <echo message="Shade netty native libraries to
${rss.shade.native.packageName}"/>
+ <unzip
src="${project.build.directory}/${project.artifactId}-${project.version}.jar"
+ dest="${project.build.directory}/unpacked/"/>
<echo message="renaming native epoll library"></echo>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_x86_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_x86_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_x86_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_aarch_64.so"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_epoll_aarch_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so"
type="glob"></mapper>
</move>
- <move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_kqueue_x86_64.jnilib"
-
to="lib${rss.shade.packageName}_shaded_netty_transport_native_kqueue_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_aarch_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_kqueue_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib"
type="glob"></mapper>
</move>
<echo message="repackaging netty jar"></echo>
- <jar destfile="${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked" />
+ <jar
destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"
+ basedir="${project.build.directory}/unpacked"/>
</target>
</configuration>
</execution>
diff --git a/client-tez/pom.xml b/client-tez/pom.xml
index edab39797..a632eea08 100644
--- a/client-tez/pom.xml
+++ b/client-tez/pom.xml
@@ -224,6 +224,92 @@
</execution>
</executions>
</plugin>
+ <!-- shade the native netty libs as well -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>regex-property</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <configuration>
+ <name>rss.shade.native.packageName</name>
+ <value>${rss.shade.packageName}</value>
+ <regex>\.</regex>
+ <replacement>_</replacement>
+ <failIfNoMatch>true</failIfNoMatch>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo message="Shade netty native libraries to
${rss.shade.native.packageName}"/>
+ <unzip
src="${project.build.directory}/shaded/${project.artifactId}-${project.version}-shaded.jar"
+
dest="${project.build.directory}/unpacked/"/>
+ <echo message="renaming native epoll
library"></echo>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_epoll_x86_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_x86_64.so"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_epoll_aarch_64.so"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_kqueue_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_aarch_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_resolver_dns_native_macos_x86_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_resolver_dns_native_macos_x86_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <move includeemptydirs="false"
+
todir="${project.build.directory}/unpacked/META-INF/native">
+ <fileset
dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
+ <mapper
from="libnetty_transport_native_kqueue_aarch_64.jnilib"
+
to="lib${rss.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib"
+ type="glob"></mapper>
+ </move>
+ <echo message="repackaging netty jar"></echo>
+ <jar
destfile="${project.build.directory}/shaded/${project.artifactId}-${project.version}-shaded.jar"
+
basedir="${project.build.directory}/unpacked"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
