This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12464-987715e4a92ae46f391cb6a291c0beef0625fff3 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit a140a0015baf0d0e78b23d12b53e7f9e0d701f91 Author: hutiefang76 <[email protected]> AuthorDate: Fri Oct 9 09:25:59 2026 +0000 [Improve][CLI] Install source setup in a project virtual environment (#12464) --- .gitignore | 3 + docs/en/ai-cli/quickstart.md | 1 + docs/zh/ai-cli/quickstart.md | 1 + seatunnel-cli/README.md | 3 +- seatunnel-cli/README.zh-CN.md | 3 +- seatunnel-cli/setup.sh | 52 +++++++++++-- seatunnel-cli/tests/test_setup_script.py | 129 +++++++++++++++++++++++++++++++ 7 files changed, 184 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index b1eba9b18d..6e54678864 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ apidoc # Python *.py[cod] +seatunnel-cli/.venv +seatunnel-cli/.venv.invalid.* +seatunnel-cli/*.egg-info/ Test.java Test.scala diff --git a/docs/en/ai-cli/quickstart.md b/docs/en/ai-cli/quickstart.md index a513ec07b9..d6f0bab50d 100644 --- a/docs/en/ai-cli/quickstart.md +++ b/docs/en/ai-cli/quickstart.md @@ -36,6 +36,7 @@ bin/seatunnel-ai.sh ```bash cd seatunnel-cli bash setup.sh # installs all providers + dev tools +source .venv/bin/activate seatunnel --init # interactive provider setup ``` diff --git a/docs/zh/ai-cli/quickstart.md b/docs/zh/ai-cli/quickstart.md index 02806de4bc..e1ce2dccc3 100644 --- a/docs/zh/ai-cli/quickstart.md +++ b/docs/zh/ai-cli/quickstart.md @@ -36,6 +36,7 @@ bin/seatunnel-ai.sh ```bash cd seatunnel-cli bash setup.sh # 安装全部提供商依赖 + 开发工具 +source .venv/bin/activate seatunnel --init # 交互式配置提供商 ``` diff --git a/seatunnel-cli/README.md b/seatunnel-cli/README.md index ccb6cf87c8..9db8ce6314 100644 --- a/seatunnel-cli/README.md +++ b/seatunnel-cli/README.md @@ -55,9 +55,10 @@ cd seatunnel-cli # Quick setup (installs all providers + dev tools) bash setup.sh +source .venv/bin/activate ``` -Then configure your LLM provider: +The setup script installs into `seatunnel-cli/.venv` and reuses that environment on later runs. Activate it before using `seatunnel`, or call `.venv/bin/seatunnel` directly. It does not install packages into the system Python. Then configure your LLM provider: ```bash seatunnel --init diff --git a/seatunnel-cli/README.zh-CN.md b/seatunnel-cli/README.zh-CN.md index 7745f891a2..585d591b10 100644 --- a/seatunnel-cli/README.zh-CN.md +++ b/seatunnel-cli/README.zh-CN.md @@ -54,9 +54,10 @@ cd seatunnel-cli # 快速安装（安装所有提供商 + 开发工具） bash setup.sh +source .venv/bin/activate ``` -然后配置 LLM 提供商： +脚本会将依赖安装到 `seatunnel-cli/.venv`，再次运行时复用该环境。使用 `seatunnel` 前请先激活环境，也可直接运行 `.venv/bin/seatunnel`；脚本不会把依赖安装到系统 Python。然后配置 LLM 提供商： ```bash seatunnel --init diff --git a/seatunnel-cli/setup.sh b/seatunnel-cli/setup.sh index 07703bdde6..e01dea5dc1 100644 --- a/seatunnel-cli/setup.sh +++ b/seatunnel-cli/setup.sh @@ -18,7 +18,9 @@ # SeaTunnel CLI - Quick Setup Script -set -e +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" echo "=== SeaTunnel CLI Setup ===" echo "" @@ -27,22 +29,60 @@ echo "" PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) || { echo "Error: Python 3.10+ required but python3 not found"; exit 1; } -PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d. -f2) -if [ "$PYTHON_MINOR" -lt 10 ]; then +PYTHON_MAJOR=${PYTHON_VERSION%%.*} +PYTHON_MINOR=${PYTHON_VERSION#*.} +if [ "$PYTHON_MAJOR" -lt 3 ] || { [ "$PYTHON_MAJOR" -eq 3 ] && [ "$PYTHON_MINOR" -lt 10 ]; }; then echo "Error: Python >= 3.10 required, but found Python $PYTHON_VERSION" exit 1 fi echo "Python $PYTHON_VERSION" -# Install in development mode -echo "Installing seatunnel CLI..." -python3 -m pip install -e ".[dev]" +VENV_DIR="$PWD/.venv" +venv_usable() { + [ -x "$VENV_DIR/bin/python" ] && + "$VENV_DIR/bin/python" -c 'import sys; assert sys.prefix != sys.base_prefix and sys.version_info >= (3, 10)' >/dev/null 2>&1 && + "$VENV_DIR/bin/python" -m pip --version >/dev/null 2>&1 +} + +if venv_usable; then + echo "Reusing $VENV_DIR" +else + # Preserve a pre-existing but unusable environment for inspection. + VENV_BACKUP="" + if [ -e "$VENV_DIR" ] || [ -L "$VENV_DIR" ]; then + VENV_BACKUP="${VENV_DIR}.invalid.$(date +%s).$$" + mv "$VENV_DIR" "$VENV_BACKUP" + echo "Preserved unusable environment at $VENV_BACKUP" + fi + + if ! python3 -m venv "$VENV_DIR" || ! venv_usable; then + # A failed venv attempt can leave files that virtualenv will not overwrite. + rm -rf -- "$VENV_DIR" + if ! command -v virtualenv >/dev/null 2>&1 || + ! virtualenv --python=python3 "$VENV_DIR" || ! venv_usable; then + rm -rf -- "$VENV_DIR" + if [ -n "$VENV_BACKUP" ]; then + mv "$VENV_BACKUP" "$VENV_DIR" + fi + echo "Error: Python virtual environments require python3-venv (ensurepip) or virtualenv." >&2 + exit 1 + fi + fi + +fi + +echo "Installing seatunnel CLI in $VENV_DIR..." +"$VENV_DIR/bin/python" -m pip install --upgrade pip setuptools wheel +"$VENV_DIR/bin/python" -m pip install -e ".[dev]" echo "" echo "=== Setup Complete ===" echo "" echo "Next steps:" echo "" +echo " Activate this checkout's environment, or call .venv/bin/seatunnel directly:" +echo " source .venv/bin/activate" +echo "" echo " 1. Set SeaTunnel engine path (for /check, /run, connector metadata):" echo "" echo " export SEATUNNEL_HOME=/path/to/apache-seatunnel" diff --git a/seatunnel-cli/tests/test_setup_script.py b/seatunnel-cli/tests/test_setup_script.py new file mode 100644 index 0000000000..8bc1daeeea --- /dev/null +++ b/seatunnel-cli/tests/test_setup_script.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Behavioral tests for the source-install setup script without downloading packages.""" + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SETUP_SCRIPT = Path(__file__).resolve().parents[1] / "setup.sh" + + +class SetupScriptTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.root = Path(self.temp_dir.name) + self.cli_dir = self.root / "seatunnel-cli" + self.cli_dir.mkdir() + shutil.copy2(SETUP_SCRIPT, self.cli_dir / "setup.sh") + self.bin_dir = self.root / "bin" + self.bin_dir.mkdir() + self.log = self.root / "calls.log" + self._write_executable( + "venv-python", + """#!/usr/bin/env bash +echo "venv-python $*" >> "$SETUP_TEST_LOG" +exit 0 +""", + ) + self._write_executable( + "python3", + """#!/usr/bin/env bash +if [ "$1" = "-c" ]; then echo 3.11; exit 0; fi +echo "python3 $*" >> "$SETUP_TEST_LOG" +if [ "$1" = "-m" ] && [ "$2" = "venv" ]; then + mkdir -p "$3/bin" + if [ "$SETUP_TEST_MODE" = "fallback" ] || [ "$SETUP_TEST_MODE" = "fail" ]; then + touch "$3/partial" + exit 1 + fi + cp "$SETUP_TEST_BIN/venv-python" "$3/bin/python" + exit 0 +fi +exit 1 +""", + ) + self._write_executable( + "virtualenv", + """#!/usr/bin/env bash +echo "virtualenv $*" >> "$SETUP_TEST_LOG" +dest="${@: -1}" +if [ -e "$dest/partial" ] || [ "$SETUP_TEST_MODE" = "fail" ]; then exit 1; fi +mkdir -p "$dest/bin" +cp "$SETUP_TEST_BIN/venv-python" "$dest/bin/python" +""", + ) + + def _write_executable(self, name, content): + path = self.bin_dir / name + path.write_text(content, encoding="utf-8") + path.chmod(0o755) + + def _run_setup(self, mode="normal"): + env = os.environ.copy() + env.update({ + "PATH": str(self.bin_dir) + os.pathsep + env["PATH"], + "SETUP_TEST_LOG": str(self.log), + "SETUP_TEST_BIN": str(self.bin_dir), + "SETUP_TEST_MODE": mode, + }) + return subprocess.run( + ["bash", str(self.cli_dir / "setup.sh")], + cwd=self.root, + env=env, + text=True, + capture_output=True, + check=False, + ) + + def test_creates_isolated_environment_and_reuses_it(self): + first = self._run_setup() + self.assertEqual(first.returncode, 0, first.stderr) + self.assertTrue((self.cli_dir / ".venv/bin/python").exists()) + self.assertIn("source .venv/bin/activate", first.stdout) + second = self._run_setup() + self.assertEqual(second.returncode, 0, second.stderr) + calls = self.log.read_text(encoding="utf-8") + self.assertEqual(calls.count("python3 -m venv"), 1) + self.assertEqual(calls.count("venv-python -m pip install --upgrade"), 2) + self.assertEqual(calls.count("venv-python -m pip install -e .[dev]"), 2) + self.assertNotIn("python3 -m pip", calls) + + def test_removes_partial_venv_before_virtualenv_fallback(self): + result = self._run_setup("fallback") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((self.cli_dir / ".venv/bin/python").exists()) + self.assertFalse((self.cli_dir / ".venv/partial").exists()) + self.assertIn("virtualenv --python=python3", self.log.read_text(encoding="utf-8")) + + def test_reports_missing_venv_support_without_global_install(self): + old = self.cli_dir / ".venv" + old.mkdir() + (old / "important-marker").write_text("keep", encoding="utf-8") + result = self._run_setup("fail") + self.assertNotEqual(result.returncode, 0) + self.assertIn("python3-venv (ensurepip) or virtualenv", result.stderr) + self.assertEqual((old / "important-marker").read_text(encoding="utf-8"), "keep") + self.assertNotIn("python3 -m pip", self.log.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()
