This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6428153bae1f [SPARK-55935][BUILD] Exclude unused Netty dependencies
6428153bae1f is described below
commit 6428153bae1f7e6ea510246826f96cf298f43bca
Author: Cheng Pan <[email protected]>
AuthorDate: Wed Mar 11 10:18:23 2026 +0800
[SPARK-55935][BUILD] Exclude unused Netty dependencies
### What changes were proposed in this pull request?
Exclude unused Netty dependencies.
### Why are the changes needed?
Reduce binary distribution size, ~24m.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
```
$ dev/make-distribution.sh --name master --tgz
-Phive,hive-thriftserver,yarn,kubernetes
$ ll spark-4.2.0-SNAPSHOT-bin-master.tgz
-rw-r--r-- 1 chengpan staff 546M Mar 10 23:43
spark-4.2.0-SNAPSHOT-bin-master.tgz
$ dev/make-distribution.sh --name SPARK-55935 --tgz
-Phive,hive-thriftserver,yarn,kubernetes
$ ll spark-4.2.0-SNAPSHOT-bin-SPARK-55935.tgz
-rw-r--r-- 1 chengpan staff 522M Mar 11 00:03
spark-4.2.0-SNAPSHOT-bin-SPARK-55935.tgz
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54732 from pan3793/exclude-unused-netty.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
common/network-yarn/pom.xml | 14 --------------
dev/deps/spark-deps-hadoop-3-hive-2.3 | 13 -------------
pom.xml | 28 ++++++++++++++++++++++++++++
3 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 7ff63f8430d6..f91069f32c2c 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -176,20 +176,14 @@
<unzip src="${shuffle.jar}"
dest="${project.build.directory}/exploded/" />
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_x86_64.so"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_x86_64.so"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_io_uring42_x86_64.so"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_io_uring42_x86_64.so"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_aarch_64.so"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_io_uring42_aarch_64.so"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_io_uring42_aarch_64.so"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_riscv64.so"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_riscv64.so"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_io_uring42_riscv64.so"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_io_uring42_riscv64.so"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_tcnative_linux_x86_64.so"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_tcnative_linux_x86_64.so"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_tcnative_linux_aarch_64.so"
@@ -198,14 +192,6 @@
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_tcnative_osx_x86_64.jnilib"
/>
<move
file="${project.build.directory}/exploded/META-INF/native/libnetty_tcnative_osx_aarch_64.jnilib"
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_tcnative_osx_aarch_64.jnilib"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_quiche42_linux_x86_64.so"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_quiche42_linux_x86_64.so"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_quiche42_linux_aarch_64.so"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_quiche42_linux_aarch_64.so"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_quiche42_osx_x86_64.jnilib"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_quiche42_osx_x86_64.jnilib"
/>
- <move
file="${project.build.directory}/exploded/META-INF/native/libnetty_quiche42_osx_aarch_64.jnilib"
-
tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_quiche42_osx_aarch_64.jnilib"
/>
<jar destfile="${shuffle.jar}"
basedir="${project.build.directory}/exploded" />
</target>
</configuration>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 5dad066e1c53..c38658a1ebf1 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -197,19 +197,10 @@ minlog/1.3.0//minlog-1.3.0.jar
netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar
netty-codec-base/4.2.10.Final//netty-codec-base-4.2.10.Final.jar
-netty-codec-classes-quic/4.2.10.Final//netty-codec-classes-quic-4.2.10.Final.jar
netty-codec-compression/4.2.10.Final//netty-codec-compression-4.2.10.Final.jar
netty-codec-dns/4.2.10.Final//netty-codec-dns-4.2.10.Final.jar
netty-codec-http/4.2.10.Final//netty-codec-http-4.2.10.Final.jar
netty-codec-http2/4.2.10.Final//netty-codec-http2-4.2.10.Final.jar
-netty-codec-http3/4.2.10.Final//netty-codec-http3-4.2.10.Final.jar
-netty-codec-marshalling/4.2.10.Final//netty-codec-marshalling-4.2.10.Final.jar
-netty-codec-native-quic/4.2.10.Final/linux-aarch_64/netty-codec-native-quic-4.2.10.Final-linux-aarch_64.jar
-netty-codec-native-quic/4.2.10.Final/linux-x86_64/netty-codec-native-quic-4.2.10.Final-linux-x86_64.jar
-netty-codec-native-quic/4.2.10.Final/osx-aarch_64/netty-codec-native-quic-4.2.10.Final-osx-aarch_64.jar
-netty-codec-native-quic/4.2.10.Final/osx-x86_64/netty-codec-native-quic-4.2.10.Final-osx-x86_64.jar
-netty-codec-native-quic/4.2.10.Final/windows-x86_64/netty-codec-native-quic-4.2.10.Final-windows-x86_64.jar
-netty-codec-protobuf/4.2.10.Final//netty-codec-protobuf-4.2.10.Final.jar
netty-codec-socks/4.2.10.Final//netty-codec-socks-4.2.10.Final.jar
netty-codec/4.2.10.Final//netty-codec-4.2.10.Final.jar
netty-common/4.2.10.Final//netty-common-4.2.10.Final.jar
@@ -224,14 +215,10 @@
netty-tcnative-boringssl-static/2.0.74.Final/osx-x86_64/netty-tcnative-boringssl
netty-tcnative-boringssl-static/2.0.74.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar
netty-tcnative-classes/2.0.74.Final//netty-tcnative-classes-2.0.74.Final.jar
netty-transport-classes-epoll/4.2.10.Final//netty-transport-classes-epoll-4.2.10.Final.jar
-netty-transport-classes-io_uring/4.2.10.Final//netty-transport-classes-io_uring-4.2.10.Final.jar
netty-transport-classes-kqueue/4.2.10.Final//netty-transport-classes-kqueue-4.2.10.Final.jar
netty-transport-native-epoll/4.2.10.Final/linux-aarch_64/netty-transport-native-epoll-4.2.10.Final-linux-aarch_64.jar
netty-transport-native-epoll/4.2.10.Final/linux-riscv64/netty-transport-native-epoll-4.2.10.Final-linux-riscv64.jar
netty-transport-native-epoll/4.2.10.Final/linux-x86_64/netty-transport-native-epoll-4.2.10.Final-linux-x86_64.jar
-netty-transport-native-io_uring/4.2.10.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.10.Final-linux-aarch_64.jar
-netty-transport-native-io_uring/4.2.10.Final/linux-riscv64/netty-transport-native-io_uring-4.2.10.Final-linux-riscv64.jar
-netty-transport-native-io_uring/4.2.10.Final/linux-x86_64/netty-transport-native-io_uring-4.2.10.Final-linux-x86_64.jar
netty-transport-native-kqueue/4.2.10.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.10.Final-osx-aarch_64.jar
netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqueue-4.2.10.Final-osx-x86_64.jar
netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
diff --git a/pom.xml b/pom.xml
index a85b661b1965..84b2eb54f36c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -958,6 +958,26 @@
<groupId>io.netty</groupId>
<artifactId>netty-codec-xml</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-marshalling</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-classes-quic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-native-quic</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-protobuf</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-resolver-dns-classes-macos</artifactId>
@@ -978,6 +998,14 @@
<groupId>io.netty</groupId>
<artifactId>netty-transport-udt</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-classes-io_uring</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-io_uring</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty-handler-ssl-ocsp</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
