This is an automated email from the ASF dual-hosted git repository.
lehelb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new d9187f3ad1 NIFI-14323 Added Support for Astral uv to Install Python
Dependencies
d9187f3ad1 is described below
commit d9187f3ad1c0912434879b9f9fb0bdd5ab2d9244
Author: exceptionfactory <[email protected]>
AuthorDate: Tue Mar 4 19:10:52 2025 -0600
NIFI-14323 Added Support for Astral uv to Install Python Dependencies
- Added setup-uv to system-tests workflow
- Preferred uv for installing Python dependencies when command is found
- Installed Astral uv in Docker image build
- Removed python3-pip package from Docker image build
This closes #9771.
Signed-off-by: Lehel Boer <[email protected]>
---
.github/workflows/system-tests.yml | 5 +++++
nifi-docker/dockerhub/Dockerfile | 3 ++-
nifi-docker/dockerhub/sh/common.sh | 3 +++
nifi-docker/dockermaven/Dockerfile | 3 ++-
.../src/main/python/framework/ExtensionManager.py | 12 ++++++++++++
.../src/test/resources/conf/pythonic/logback.xml | 2 +-
6 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/system-tests.yml
b/.github/workflows/system-tests.yml
index abbe4c46fe..257682abea 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -102,6 +102,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: '3.10'
+ - name: Set up Astral uv
+ uses: astral-sh/setup-uv@v5
+ with:
+ python-version: '3.10'
+ enable-cache: false
- name: Build Dependencies
env:
diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile
index 2bf07adee0..4ba4c5f2bf 100644
--- a/nifi-docker/dockerhub/Dockerfile
+++ b/nifi-docker/dockerhub/Dockerfile
@@ -52,7 +52,6 @@ RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group
${GID} | cut -d: -
&& apt-get update \
&& apt-get install -y jq xmlstarlet procps unzip \
&& apt-get install -y python3 \
- && apt-get install -y python3-pip \
&& apt-get install -y python3-venv \
&& apt-get -y autoremove \
&& apt-get clean autoclean \
@@ -60,6 +59,8 @@ RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group
${GID} | cut -d: -
USER nifi
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
# Download, validate, and expand Apache NiFi Toolkit binary.
RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH} -o
${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \
&& echo "$(curl ${BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH}.sha512)
*${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip" | sha512sum -c - \
diff --git a/nifi-docker/dockerhub/sh/common.sh
b/nifi-docker/dockerhub/sh/common.sh
index d78aec494c..9481480bf5 100755
--- a/nifi-docker/dockerhub/sh/common.sh
+++ b/nifi-docker/dockerhub/sh/common.sh
@@ -48,3 +48,6 @@ export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf
export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties
export hostname=$(hostname)
+
+# Set Path to include local directory for Astral uv
+export PATH=${PATH}:~/.local/bin
diff --git a/nifi-docker/dockermaven/Dockerfile
b/nifi-docker/dockermaven/Dockerfile
index 901ec18427..1771ee59fa 100644
--- a/nifi-docker/dockermaven/Dockerfile
+++ b/nifi-docker/dockermaven/Dockerfile
@@ -84,7 +84,6 @@ RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group
${GID} | cut -d: -
&& apt-get update \
&& apt-get install -y jq xmlstarlet procps \
&& apt-get install -y python3 \
- && apt-get install -y python3-pip \
&& apt-get install -y python3-venv \
&& apt-get -y autoremove \
&& apt-get clean autoclean \
@@ -104,6 +103,8 @@ VOLUME ${NIFI_LOG_DIR} \
USER nifi
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
# Clear nifi-env.sh in favour of configuring all environment variables in the
Dockerfile
RUN echo "#!/bin/sh\n" > "${NIFI_HOME}/bin/nifi-env.sh"
diff --git
a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
index a47e0d2d2b..a6bb03f4ac 100644
---
a/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
+++
b/nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ExtensionManager.py
@@ -19,6 +19,7 @@ import inspect
import logging
import os
import pkgutil
+import shutil
import subprocess
import sys
from pathlib import Path
@@ -60,9 +61,11 @@ class ExtensionManager:
processor_class_by_name = {}
module_files_by_extension_type = {}
dependency_directories = {}
+ astral_uv_installed = False
def __init__(self, gateway):
self.gateway = gateway
+ self.astral_uv_installed = self.__is_astral_uv_installed()
def get_processor_details(self, classname, version):
@@ -301,6 +304,11 @@ class ExtensionManager:
if len(dependency_references) > 0:
python_cmd = os.getenv("PYTHON_CMD")
args = [python_cmd, '-m', 'pip', 'install', '--no-cache-dir',
'--target', target_dir] + dependency_references
+
+ if self.astral_uv_installed:
+ uv_command = shutil.which("uv")
+ args = [uv_command, 'pip', 'install', '--no-cache',
'--target', target_dir] + dependency_references
+
logger.info(f"Installing dependencies {dependency_references} for
{class_name} to {target_dir} using command {args}")
result = subprocess.run(args)
@@ -316,6 +324,10 @@ class ExtensionManager:
file.write("True")
+ def __is_astral_uv_installed(self):
+ uv_command = shutil.which("uv")
+ return uv_command is not None
+
def __load_extension_module(self, file, local_dependencies):
# If there are any local dependencies (i.e., other python files in the
same directory), load those modules first
if local_dependencies:
diff --git
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml
index a46edc5e07..975e940e21 100644
---
a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml
+++
b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml
@@ -101,7 +101,7 @@
<logger
name="org.apache.nifi.controller.repository.StandardProcessSession"
level="WARN" />
<!-- Py4J set to WARN to avoid verbose socket communication messages -->
- <logger name="py4j" level="WARN" />
+ <logger name="py4j.java_gateway" level="WARN" />
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
