This is an automated email from the ASF dual-hosted git repository.
sxnan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new c17c9a2 [infra] Deduplicate common jar dependency to reduce wheel
size (#557)
c17c9a2 is described below
commit c17c9a2a113b8d3ad023d75d5472b48cb6a9d6a9
Author: Xuannan <[email protected]>
AuthorDate: Thu Mar 5 21:59:53 2026 +0800
[infra] Deduplicate common jar dependency to reduce wheel size (#557)
---
dist/common/pom.xml | 122 +++++++++++++++++++++++
dist/flink-1.20/pom.xml | 44 ++++++--
dist/flink-2.0/pom.xml | 42 +++++++-
dist/flink-2.1/pom.xml | 43 +++++++-
dist/flink-2.2/pom.xml | 43 ++++++++
dist/pom.xml | 1 +
python/flink_agents/api/execution_environment.py | 15 ++-
tools/build.sh | 26 +++--
tools/releasing/create_binary_release.sh | 6 +-
9 files changed, 321 insertions(+), 21 deletions(-)
diff --git a/dist/common/pom.xml b/dist/common/pom.xml
new file mode 100644
index 0000000..2be34af
--- /dev/null
+++ b/dist/common/pom.xml
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-agents-dist</artifactId>
+ <version>0.3-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>flink-agents-dist-common</artifactId>
+ <name>Flink Agents : Dist : Common</name>
+ <description>Common dependencies shared across all Flink versions for
Python wheel</description>
+
+ <dependencies>
+ <!-- All integrations dependencies - same as parent pom -->
+ <!-- These will be bundled into the common JAR -->
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Skip deployment - this module is for Python wheel
optimization only -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Copy the shared LICENSE/NOTICE files from parent -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-shared-resources</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>${project.build.outputDirectory}</outputDirectory>
+ <resources>
+ <resource>
+
<directory>${project.parent.basedir}/src/main/resources</directory>
+ <includes>
+ <include>META-INF/**</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Build uber jar with all dependencies except flink-agents own
code -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-common</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+ <filters>
+ <!-- Exclude signature files -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ <!-- Exclude flink-agents own code from common
JAR -->
+ <filter>
+
<artifact>org.apache.flink:flink-agents-*</artifact>
+ <excludes>
+
<exclude>org/apache/flink/agents/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/dist/flink-1.20/pom.xml b/dist/flink-1.20/pom.xml
index 1827e11..891d5c6 100644
--- a/dist/flink-1.20/pom.xml
+++ b/dist/flink-1.20/pom.xml
@@ -138,7 +138,7 @@ under the License.
</executions>
</plugin>
- <!-- Extend parent's shade plugin with flink-1.20 specific
exclusions -->
+ <!-- Extend parent's shade plugin with flink-1.20 specific
configurations -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -146,17 +146,43 @@ under the License.
<execution>
<id>shade-flink-agents</id>
<configuration>
- <!-- Append flink-1.20 specific filter to parent's
filters -->
- <filters combine.children="append">
- <!-- Exclude the conflict classes of the
runtime module, using version flink-1.20. -->
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ </configuration>
+ </execution>
+ <!-- Build thin jar for Python wheel (only flink-agents
code) -->
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
<filter>
-
<artifact>org.apache.flink:flink-agents-runtime</artifact>
- <excludes>
-
<exclude>org/apache/flink/agents/runtime/operator/OperatorUtils.class</exclude>
- </excludes>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
</filter>
</filters>
-
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
</configuration>
</execution>
</executions>
diff --git a/dist/flink-2.0/pom.xml b/dist/flink-2.0/pom.xml
index d9d6471..50bb37c 100644
--- a/dist/flink-2.0/pom.xml
+++ b/dist/flink-2.0/pom.xml
@@ -84,7 +84,47 @@
</executions>
</plugin>
- <!-- Shade plugin configuration inherits from the parent pom
(dist/pom.xml) -->
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/dist/flink-2.1/pom.xml b/dist/flink-2.1/pom.xml
index 8a00986..126e45f 100644
--- a/dist/flink-2.1/pom.xml
+++ b/dist/flink-2.1/pom.xml
@@ -84,7 +84,48 @@
</executions>
</plugin>
- <!-- Shade plugin configuration inherits from the parent pom
(dist/pom.xml) -->
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <!-- Not attached as Maven artifact to avoid deployment -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/dist/flink-2.2/pom.xml b/dist/flink-2.2/pom.xml
index e5dd9e9..29a3e25 100644
--- a/dist/flink-2.2/pom.xml
+++ b/dist/flink-2.2/pom.xml
@@ -54,6 +54,49 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <!-- Not attached as Maven artifact to avoid deployment -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/dist/pom.xml b/dist/pom.xml
index 16ed7fc..f7e064f 100644
--- a/dist/pom.xml
+++ b/dist/pom.xml
@@ -37,6 +37,7 @@ under the License.
</properties>
<modules>
+ <module>common</module>
<module>flink-1.20</module>
<module>flink-2.0</module>
<module>flink-2.1</module>
diff --git a/python/flink_agents/api/execution_environment.py
b/python/flink_agents/api/execution_environment.py
index 4b4c80b..11de9ab 100644
--- a/python/flink_agents/api/execution_environment.py
+++ b/python/flink_agents/api/execution_environment.py
@@ -130,9 +130,20 @@ class AgentsExecutionEnvironment(ABC):
else:
major_version = flink_version_manager.major_version
if major_version:
- # Determine the version-specific lib directory
- version_dir = f"flink-{major_version}"
lib_base = files("flink_agents.lib")
+
+ # Load the common JAR (shared dependencies)
+ common_lib = lib_base / "common"
+ if common_lib.is_dir():
+ for jar_file in common_lib.iterdir():
+ if jar_file.is_file() and
str(jar_file).endswith(".jar"):
+ env.add_jars(f"file://{jar_file}")
+ else:
+ err_msg = "Flink Agents common JAR not found."
+ raise FileNotFoundError(err_msg)
+
+ # Load the version-specific thin JAR
+ version_dir = f"flink-{major_version}"
version_lib = lib_base / version_dir
# Check if version-specific directory exists
diff --git a/tools/build.sh b/tools/build.sh
index 718adf5..0e17a8f 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -53,8 +53,20 @@ if $build_python; then
PROJECT_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml |
head -n 2 | tail -n 1)
- # Automatically detect and copy all Flink version JARs from dist
subdirectories
DIST_DIR="${PROJECT_ROOT}/dist"
+
+ # Copy common JAR (shared dependencies, ~110MB)
+ echo "Processing common dependencies..."
+ mkdir -p "${PYTHON_LIB_DIR}/common"
+
common_jar="${DIST_DIR}/common/target/flink-agents-dist-common-${PROJECT_VERSION}.jar"
+ if [ -f "$common_jar" ]; then
+ cp "$common_jar" "${PYTHON_LIB_DIR}/common/"
+ echo " Copied: flink-agents-dist-common-${PROJECT_VERSION}.jar"
+ else
+ echo " Warning: Common JAR file not found at $common_jar"
+ fi
+
+ # Copy thin JARs for each Flink version (only flink-agents code, ~1MB each)
for version_dir in "${DIST_DIR}"/flink-*; do
if [ -d "$version_dir" ]; then
version_name=$(basename "$version_dir")
@@ -63,13 +75,13 @@ if $build_python; then
# Create corresponding lib subdirectory
mkdir -p "${PYTHON_LIB_DIR}/${version_name}"
- # Find and copy the JAR file
-
jar_file="${version_dir}/target/flink-agents-dist-${version_name}-${PROJECT_VERSION}.jar"
- if [ -f "$jar_file" ]; then
- cp "$jar_file" "${PYTHON_LIB_DIR}/${version_name}/"
- echo " Copied:
flink-agents-dist-${version_name}-${PROJECT_VERSION}.jar"
+ # Find and copy the thin JAR file
+
thin_jar="${version_dir}/target/flink-agents-dist-${version_name}-${PROJECT_VERSION}-thin.jar"
+ if [ -f "$thin_jar" ]; then
+ cp "$thin_jar" "${PYTHON_LIB_DIR}/${version_name}/"
+ echo " Copied:
flink-agents-dist-${version_name}-${PROJECT_VERSION}-thin.jar"
else
- echo " Warning: JAR file not found at $jar_file"
+ echo " Warning: Thin JAR file not found at $thin_jar"
fi
fi
done
diff --git a/tools/releasing/create_binary_release.sh
b/tools/releasing/create_binary_release.sh
index eee8f53..8c053c7 100755
--- a/tools/releasing/create_binary_release.sh
+++ b/tools/releasing/create_binary_release.sh
@@ -61,10 +61,14 @@ make_binary_release() {
echo "Creating binary release"
# Dynamically discover dist sub-modules (directories containing pom.xml)
+ # Exclude 'common' module - it's for Python wheel optimization only, not for
Maven release
DIST_MODULES=()
for module_dir in dist/*/; do
if [ -f "${module_dir}pom.xml" ]; then
- DIST_MODULES+=("$(basename "${module_dir}")")
+ module_name=$(basename "${module_dir}")
+ if [ "$module_name" != "common" ]; then
+ DIST_MODULES+=("$module_name")
+ fi
fi
done
