This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch reduce-distro-size in repository https://gitbox.apache.org/repos/asf/storm.git
commit 97d292229d0c994eb65747dcc12444a1c98dc85a Author: Richard Zowalla <[email protected]> AuthorDate: Tue Jun 30 20:13:57 2026 +0200 build: wire lib-common de-duplication into the binary assembly final-package now stages the daemon jars (copy-dependencies -> staging/lib) and the shared jars (storm-client-bin's lib-worker -> staging/lib-common), runs dedup-libs.py to remove the byte-identical copies from lib, and the assembly packages the staged lib/ and lib-common/ directories. The storm-client-bin tree (which only carried lib-worker) is no longer copied directly. Verified through the prepare-package phase: staging/lib = 48 jars, staging/ lib-common = 40 jars, zero overlap, full 88-jar daemon set preserved across lib + lib-common, ~71 MB reclaimed. The final tar/zip packaging requires a full -Pdist (native) distribution build and must be validated in CI / on Linux. --- storm-dist/binary/final-package/pom.xml | 67 ++++++++++++++++++++++ .../final-package/src/main/assembly/binary.xml | 26 +++++---- 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/storm-dist/binary/final-package/pom.xml b/storm-dist/binary/final-package/pom.xml index 8471b307c..dce19eaf6 100644 --- a/storm-dist/binary/final-package/pom.xml +++ b/storm-dist/binary/final-package/pom.xml @@ -43,6 +43,73 @@ <build> <finalName>apache-storm-${project.version}</finalName> <plugins> + <!-- + Stage the daemon (lib) and shared (lib-common) jars before assembly and de-duplicate + them. The worker classpath is a byte-identical subset of the daemon classpath, so the + shared jars are kept once in lib-common (referenced by both classpaths in bin/storm.py) + and removed from lib. See src/main/scripts/dedup-libs.py. + --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>stage-daemon-lib</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <outputDirectory>${project.build.directory}/staging/lib</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>stage-common-lib</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/staging/lib-common</outputDirectory> + <resources> + <resource> + <directory>${project.basedir}/../storm-client-bin/target/client/client/lib-worker</directory> + <includes> + <include>*.jar</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>dedup-libs</id> + <phase>prepare-package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>python3</executable> + <arguments> + <argument>${project.basedir}/src/main/scripts/dedup-libs.py</argument> + <argument>${project.build.directory}/staging</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index e41226110..5e62e2bc0 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -24,21 +24,23 @@ <format>zip</format> </formats> - <!-- put deps in the lib folder --> - <dependencySets> - <dependencySet> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>lib</outputDirectory> - <unpack>false</unpack> - </dependencySet> - </dependencySets> - <fileSets> + <!-- Daemon jars (lib) and shared jars (lib-common) are staged and de-duplicated by + final-package's build (maven-dependency-plugin + dedup-libs.py) before assembly. + The worker jars now live in lib-common, so the storm-client-bin tree (which only + contained lib-worker) is no longer copied here. --> <fileSet> - <directory>${project.basedir}/../storm-client-bin/target/client/client/</directory> - <outputDirectory>.</outputDirectory> + <directory>${project.build.directory}/staging/lib</directory> + <outputDirectory>lib</outputDirectory> <includes> - <include>*/**</include> + <include>*.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${project.build.directory}/staging/lib-common</directory> + <outputDirectory>lib-common</outputDirectory> + <includes> + <include>*.jar</include> </includes> </fileSet> <fileSet>
