chaokunyang commented on code in PR #2406:
URL: https://github.com/apache/fory/pull/2406#discussion_r2206169489
##########
ci/run_ci.py:
##########
@@ -15,202 +15,270 @@
# specific language governing permissions and limitations
# under the License.
-
import argparse
+import logging
+import os
import shutil
import subprocess
-import platform
-import urllib.request as ulib
-import os
-import logging
-import importlib
-
-
-def _get_bazel_version():
- with open(os.path.join(PROJECT_ROOT_DIR, ".bazelversion")) as f:
- return f.read().strip()
-
+import sys
-PYARROW_VERSION = "15.0.0"
-
-PROJECT_ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../")
+from tasks import cpp, java, javascript, kotlin, rust, python, go, format
+from tasks.common import is_windows
+# Configure logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
-
-def _bazel(cmd: str):
- bazel_cmd = "bazel" if _is_windows() else "~/bin/bazel"
- return _exec_cmd(f"{bazel_cmd} {cmd}")
-
-
-def _exec_cmd(cmd: str):
- logging.info(f"running command: {cmd}")
- try:
- result = subprocess.check_output(cmd, shell=True,
universal_newlines=True)
- except subprocess.CalledProcessError as error:
- logging.error(error.stdout)
- raise
-
- logging.info(f"command result: {result}")
- return result
-
-
-def _get_os_name_lower():
- return platform.system().lower()
-
-
-def _is_windows():
- return _get_os_name_lower() == "windows"
-
-
-def _get_os_machine():
- machine = platform.machine().lower()
- # Unified to x86_64(Windows return AMD64, others return x86_64).
- return machine.replace("amd64", "x86_64")
-
-
-def _get_bazel_download_url():
- bazel_version = _get_bazel_version()
- download_url_base = (
-
f"https://github.com/bazelbuild/bazel/releases/download/{bazel_version}";
- )
- suffix = "exe" if _is_windows() else "sh"
- return (
- f"{download_url_base}/bazel-{bazel_version}{'' if _is_windows() else
'-installer'}-"
- f"{_get_os_name_lower()}-{_get_os_machine()}.{suffix}"
- )
-
-
-def _cd_project_subdir(subdir):
- os.chdir(os.path.join(PROJECT_ROOT_DIR, subdir))
-
-
-def _run_cpp():
- _install_cpp_deps()
- # collect all C++ targets
- query_result = _bazel("query //...")
- targets = query_result.replace("\n", " ").replace("\r", " ")
- test_command = "test"
- if _get_os_machine() == "x86_64":
- test_command += " --config=x86_64"
- _bazel(f"{test_command} {targets}")
-
-
-def _run_rust():
- logging.info("Executing fory rust tests")
- _cd_project_subdir("rust")
-
- cmds = (
- "cargo doc --no-deps --document-private-items --all-features --open",
- "cargo fmt --all -- --check",
- "cargo fmt --all",
- "cargo clippy --workspace --all-features --all-targets -- -D warnings",
- "cargo doc",
- "cargo build --all-features --all-targets",
- "cargo test",
- "cargo clean",
- )
- for cmd in cmds:
- _exec_cmd(cmd)
- logging.info("Executing fory rust tests succeeds")
-
-
-def _run_js():
- logging.info("Executing fory javascript tests.")
- _cd_project_subdir("javascript")
- _exec_cmd("npm install")
- _exec_cmd("npm run test")
- logging.info("Executing fory javascript tests succeeds.")
-
-
-def _install_cpp_deps():
- _exec_cmd(f"pip install pyarrow=={PYARROW_VERSION}")
- # Automatically install numpy
- _exec_cmd("pip install psutil")
- _install_bazel()
-
-
-def _install_bazel():
- local_name = "bazel.exe" if _is_windows() else "bazel-installer.sh"
- bazel_download_url = _get_bazel_download_url()
- logging.info(bazel_download_url)
- ulib.urlretrieve(bazel_download_url, local_name)
- os.chmod(local_name, 0o777)
-
- if _is_windows():
- bazel_path = os.path.join(os.getcwd(), local_name)
- _exec_cmd(f'setx path "%PATH%;{bazel_path}"')
- else:
- if shutil.which("bazel"):
- os.remove(shutil.which("bazel"))
- _exec_cmd(f"./{local_name} --user")
- _update_shell_profile()
- os.remove(local_name)
-
- # bazel install status check
- _bazel("--version")
-
- # default is byte
- psutil = importlib.import_module("psutil")
- total_mem = psutil.virtual_memory().total
- limit_jobs = int(total_mem / 1024 / 1024 / 1024 / 3)
- with open(".bazelrc", "a") as file:
- file.write(f"\nbuild --jobs={limit_jobs}")
-
-
-def _update_shell_profile():
- home = os.path.expanduser("~")
- profiles = [".bashrc", ".bash_profile", ".zshrc"]
- path_export = 'export PATH="$PATH:$HOME/bin" # Add Bazel to PATH\n'
- for profile in profiles:
- profile_path = os.path.join(home, profile)
- if os.path.exists(profile_path):
- with open(profile_path, "a") as f:
- f.write(path_export)
- logging.info(f"Updated {profile} to include Bazel PATH.")
- break
+# Migration Strategy:
+# This script supports a gradual migration from the shell script (run_ci.sh)
to Python.
+# You can control which languages use the Python implementation by setting
environment variables.
+#
+# To use the shell script for a specific language, set the corresponding
environment variable to "0".
+# To use the Python implementation, set it to "1" or leave it unset (default
is Python).
+#
+# Example:
+# # Use shell script for Java, Python implementation for everything else
+# export USE_PYTHON_JAVA=0
+# python run_ci.py java --version 17
+#
+# # Use shell script for multiple languages
+# export USE_PYTHON_JAVA=0 USE_PYTHON_CPP=0 USE_PYTHON_RUST=0
+# python run_ci.py cpp
+#
+# Available environment variables:
+# USE_PYTHON_CPP - C++ implementation
+# USE_PYTHON_RUST - Rust implementation
+# USE_PYTHON_JAVASCRIPT - JavaScript implementation
+# USE_PYTHON_JAVA - Java implementation
+# USE_PYTHON_KOTLIN - Kotlin implementation
+# USE_PYTHON_PYTHON - Python implementation
+# USE_PYTHON_GO - Go implementation
+# USE_PYTHON_FORMAT - Format implementation
+#
+# By default, JavaScript, Rust, and C++ use the Python implementation,
+# while Java, Kotlin, Python, Go, and Format use the shell script
implementation.
+
+# Environment variables to control which languages use the Python
implementation
+# Default to the status quo before migration:
+# - JavaScript, Rust, and C++ use Python implementation
+# - Java, Kotlin, Python, Go, and Format use shell script implementation
+USE_PYTHON_CPP = os.environ.get("USE_PYTHON_CPP", "1") == "1"
Review Comment:
Where we set `USE_PYTHON_CPP` to 1, I don't see such code in this PR
##########
ci/run_ci.py:
##########
@@ -15,202 +15,270 @@
# specific language governing permissions and limitations
# under the License.
-
import argparse
+import logging
+import os
import shutil
import subprocess
-import platform
-import urllib.request as ulib
-import os
-import logging
-import importlib
-
-
-def _get_bazel_version():
- with open(os.path.join(PROJECT_ROOT_DIR, ".bazelversion")) as f:
- return f.read().strip()
-
+import sys
-PYARROW_VERSION = "15.0.0"
-
-PROJECT_ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../")
+from tasks import cpp, java, javascript, kotlin, rust, python, go, format
+from tasks.common import is_windows
+# Configure logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
-
-def _bazel(cmd: str):
- bazel_cmd = "bazel" if _is_windows() else "~/bin/bazel"
- return _exec_cmd(f"{bazel_cmd} {cmd}")
-
-
-def _exec_cmd(cmd: str):
- logging.info(f"running command: {cmd}")
- try:
- result = subprocess.check_output(cmd, shell=True,
universal_newlines=True)
- except subprocess.CalledProcessError as error:
- logging.error(error.stdout)
- raise
-
- logging.info(f"command result: {result}")
- return result
-
-
-def _get_os_name_lower():
- return platform.system().lower()
-
-
-def _is_windows():
- return _get_os_name_lower() == "windows"
-
-
-def _get_os_machine():
- machine = platform.machine().lower()
- # Unified to x86_64(Windows return AMD64, others return x86_64).
- return machine.replace("amd64", "x86_64")
-
-
-def _get_bazel_download_url():
- bazel_version = _get_bazel_version()
- download_url_base = (
-
f"https://github.com/bazelbuild/bazel/releases/download/{bazel_version}";
- )
- suffix = "exe" if _is_windows() else "sh"
- return (
- f"{download_url_base}/bazel-{bazel_version}{'' if _is_windows() else
'-installer'}-"
- f"{_get_os_name_lower()}-{_get_os_machine()}.{suffix}"
- )
-
-
-def _cd_project_subdir(subdir):
- os.chdir(os.path.join(PROJECT_ROOT_DIR, subdir))
-
-
-def _run_cpp():
- _install_cpp_deps()
- # collect all C++ targets
- query_result = _bazel("query //...")
- targets = query_result.replace("\n", " ").replace("\r", " ")
- test_command = "test"
- if _get_os_machine() == "x86_64":
- test_command += " --config=x86_64"
- _bazel(f"{test_command} {targets}")
-
-
-def _run_rust():
- logging.info("Executing fory rust tests")
- _cd_project_subdir("rust")
-
- cmds = (
- "cargo doc --no-deps --document-private-items --all-features --open",
- "cargo fmt --all -- --check",
- "cargo fmt --all",
- "cargo clippy --workspace --all-features --all-targets -- -D warnings",
- "cargo doc",
- "cargo build --all-features --all-targets",
- "cargo test",
- "cargo clean",
- )
- for cmd in cmds:
- _exec_cmd(cmd)
- logging.info("Executing fory rust tests succeeds")
-
-
-def _run_js():
- logging.info("Executing fory javascript tests.")
- _cd_project_subdir("javascript")
- _exec_cmd("npm install")
- _exec_cmd("npm run test")
- logging.info("Executing fory javascript tests succeeds.")
-
-
-def _install_cpp_deps():
- _exec_cmd(f"pip install pyarrow=={PYARROW_VERSION}")
- # Automatically install numpy
- _exec_cmd("pip install psutil")
- _install_bazel()
-
-
-def _install_bazel():
- local_name = "bazel.exe" if _is_windows() else "bazel-installer.sh"
- bazel_download_url = _get_bazel_download_url()
- logging.info(bazel_download_url)
- ulib.urlretrieve(bazel_download_url, local_name)
- os.chmod(local_name, 0o777)
-
- if _is_windows():
- bazel_path = os.path.join(os.getcwd(), local_name)
- _exec_cmd(f'setx path "%PATH%;{bazel_path}"')
- else:
- if shutil.which("bazel"):
- os.remove(shutil.which("bazel"))
- _exec_cmd(f"./{local_name} --user")
- _update_shell_profile()
- os.remove(local_name)
-
- # bazel install status check
- _bazel("--version")
-
- # default is byte
- psutil = importlib.import_module("psutil")
- total_mem = psutil.virtual_memory().total
- limit_jobs = int(total_mem / 1024 / 1024 / 1024 / 3)
- with open(".bazelrc", "a") as file:
- file.write(f"\nbuild --jobs={limit_jobs}")
-
-
-def _update_shell_profile():
- home = os.path.expanduser("~")
- profiles = [".bashrc", ".bash_profile", ".zshrc"]
- path_export = 'export PATH="$PATH:$HOME/bin" # Add Bazel to PATH\n'
- for profile in profiles:
- profile_path = os.path.join(home, profile)
- if os.path.exists(profile_path):
- with open(profile_path, "a") as f:
- f.write(path_export)
- logging.info(f"Updated {profile} to include Bazel PATH.")
- break
+# Migration Strategy:
+# This script supports a gradual migration from the shell script (run_ci.sh)
to Python.
+# You can control which languages use the Python implementation by setting
environment variables.
+#
+# To use the shell script for a specific language, set the corresponding
environment variable to "0".
+# To use the Python implementation, set it to "1" or leave it unset (default
is Python).
+#
+# Example:
+# # Use shell script for Java, Python implementation for everything else
+# export USE_PYTHON_JAVA=0
+# python run_ci.py java --version 17
+#
+# # Use shell script for multiple languages
+# export USE_PYTHON_JAVA=0 USE_PYTHON_CPP=0 USE_PYTHON_RUST=0
+# python run_ci.py cpp
+#
+# Available environment variables:
+# USE_PYTHON_CPP - C++ implementation
+# USE_PYTHON_RUST - Rust implementation
+# USE_PYTHON_JAVASCRIPT - JavaScript implementation
+# USE_PYTHON_JAVA - Java implementation
+# USE_PYTHON_KOTLIN - Kotlin implementation
+# USE_PYTHON_PYTHON - Python implementation
+# USE_PYTHON_GO - Go implementation
+# USE_PYTHON_FORMAT - Format implementation
+#
+# By default, JavaScript, Rust, and C++ use the Python implementation,
+# while Java, Kotlin, Python, Go, and Format use the shell script
implementation.
+
+# Environment variables to control which languages use the Python
implementation
+# Default to the status quo before migration:
+# - JavaScript, Rust, and C++ use Python implementation
+# - Java, Kotlin, Python, Go, and Format use shell script implementation
+USE_PYTHON_CPP = os.environ.get("USE_PYTHON_CPP", "1") == "1"
+USE_PYTHON_RUST = os.environ.get("USE_PYTHON_RUST", "1") == "1"
+USE_PYTHON_JAVASCRIPT = os.environ.get("USE_PYTHON_JAVASCRIPT", "1") == "1"
+USE_PYTHON_JAVA = os.environ.get("USE_PYTHON_JAVA", "0") == "1"
+USE_PYTHON_KOTLIN = os.environ.get("USE_PYTHON_KOTLIN", "0") == "1"
+USE_PYTHON_PYTHON = os.environ.get("USE_PYTHON_PYTHON", "0") == "1"
+USE_PYTHON_GO = os.environ.get("USE_PYTHON_GO", "0") == "1"
+USE_PYTHON_FORMAT = os.environ.get("USE_PYTHON_FORMAT", "0") == "1"
+
+
+def run_shell_script(command, *args):
+ """Run the shell script with the given command and arguments."""
+ script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"run_ci.sh")
+
+ if is_windows():
+ # On Windows, try to use bash if available
+ bash_path = shutil.which("bash")
+ if bash_path:
+ cmd = [bash_path, script_path, command]
+ cmd.extend(args)
+ logging.info(f"Falling back to shell script with bash: {'
'.join(cmd)}")
+ return subprocess.call(cmd)
+ else:
+ logging.error(
+ "Bash is not available on this Windows system. Cannot run
shell script."
+ )
+ logging.error(
+ "Please install Git Bash, WSL, or Cygwin to run shell scripts
on Windows."
+ )
+ logging.error(
+ "Alternatively, set USE_PYTHON_JAVA=1 to use the Python
implementation."
+ )
+ sys.exit(1)
else:
- logging.info("No shell profile found. Please add Bazel to PATH
manually.")
+ # On Unix-like systems, run the script directly
+ cmd = [script_path, command]
+ cmd.extend(args)
+ logging.info(f"Falling back to shell script: {' '.join(cmd)}")
+ return subprocess.call(cmd)
-def _parse_args():
+def parse_args():
+ """Parse command-line arguments and dispatch to the appropriate task
module."""
parser = argparse.ArgumentParser(
- formatter_class=argparse.ArgumentDefaultsHelpFormatter
+ description="Fory CI Runner",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
- parser.set_defaults(func=parser.print_help)
- subparsers = parser.add_subparsers()
+ parser.set_defaults(func=lambda: parser.print_help())
+ subparsers = parser.add_subparsers(dest="command")
+ # C++ subparser
cpp_parser = subparsers.add_parser(
"cpp",
description="Run C++ CI",
help="Run C++ CI",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
- cpp_parser.set_defaults(func=_run_cpp)
+ cpp_parser.add_argument(
+ "--install-deps-only",
+ action="store_true",
+ help="Only install dependencies without running tests",
+ )
+ cpp_parser.set_defaults(func=lambda install_deps_only:
cpp.run(install_deps_only))
+ # Rust subparser
rust_parser = subparsers.add_parser(
"rust",
description="Run Rust CI",
help="Run Rust CI",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
- rust_parser.set_defaults(func=_run_rust)
+ rust_parser.set_defaults(func=rust.run)
+ # JavaScript subparser
js_parser = subparsers.add_parser(
"javascript",
- description="Run Javascript CI",
- help="Run Javascript CI",
+ description="Run JavaScript CI",
+ help="Run JavaScript CI",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ js_parser.set_defaults(func=javascript.run)
+
+ # Java subparser
+ java_parser = subparsers.add_parser(
+ "java",
+ description="Run Java CI",
+ help="Run Java CI",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ java_parser.add_argument(
+ "--version",
+ choices=[
+ "8",
+ "11",
+ "17",
+ "21",
+ "24",
+ "windows_java21",
+ "integration_tests",
+ "graalvm",
+ ],
+ default="17",
Review Comment:
we need to use java8, jars built usxing higher jdk may not run on lower jdk
version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
