This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch pkg-ain-via-pyinstaller in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f5766cb889fb8abbd052be0ffa6bc006f8b0f585 Author: Yongzao <[email protected]> AuthorDate: Thu Nov 6 11:36:43 2025 +0800 trigger ci --- iotdb-core/ainode/.gitignore | 6 +- iotdb-core/ainode/ainode.spec | 194 +++++++++++++++ iotdb-core/ainode/ainode.xml | 15 +- iotdb-core/ainode/build_binary.py | 259 +++++++++++++++++++++ iotdb-core/ainode/iotdb/ainode/core/config.py | 10 +- iotdb-core/ainode/iotdb/ainode/core/constant.py | 4 - .../iotdb/ainode/core/inference/pool_controller.py | 47 ++-- .../pool_scheduler/basic_pool_scheduler.py | 7 +- iotdb-core/ainode/iotdb/ainode/core/log.py | 4 +- iotdb-core/ainode/iotdb/ainode/core/script.py | 20 ++ iotdb-core/ainode/poetry.lock | 109 ++++++++- iotdb-core/ainode/pom.xml | 92 +------- iotdb-core/ainode/pyproject.toml | 16 +- .../ainode/resources/syncPythonVersion.groovy | 33 --- scripts/conf/ainode-env.sh | 138 ----------- scripts/conf/windows/ainode-env.bat | 129 ---------- scripts/sbin/start-ainode.sh | 39 +--- scripts/sbin/windows/start-ainode.bat | 46 +--- 18 files changed, 633 insertions(+), 535 deletions(-) diff --git a/iotdb-core/ainode/.gitignore b/iotdb-core/ainode/.gitignore index 8cc2098c3fd..bdb2698ec78 100644 --- a/iotdb-core/ainode/.gitignore +++ b/iotdb-core/ainode/.gitignore @@ -14,8 +14,6 @@ # generated by maven /iotdb/ainode/conf/ -# .whl of ainode, generated by Poetry +# generated by pyinstaller /dist/ - -# the config to build ainode, it will be generated automatically -pyproject.toml +/build/ diff --git a/iotdb-core/ainode/ainode.spec b/iotdb-core/ainode/ainode.spec new file mode 100644 index 00000000000..d14546249e4 --- /dev/null +++ b/iotdb-core/ainode/ainode.spec @@ -0,0 +1,194 @@ +# -*- mode: python ; coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under this License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from pathlib import Path + +# Get project root directory +project_root = Path(SPECPATH).parent + +block_cipher = None + +# Auto-collect all submodules of large dependency libraries +# Using collect_all automatically includes all dependencies and avoids manual maintenance of hiddenimports +from PyInstaller.utils.hooks import collect_all, collect_submodules, collect_data_files + +# Collect only essential data files and binaries for large libraries +# Using collect_all for all submodules slows down startup significantly +# Instead, we collect only what's needed and rely on PyInstaller's dependency analysis +all_datas = [] +all_binaries = [] +all_hiddenimports = [] + +# Only collect essential data files and binaries for critical libraries +# This reduces startup time by avoiding unnecessary module imports +essential_libraries = { + 'torch': True, # Keep collect_all for torch as it has many dynamic imports + 'transformers': True, # Keep collect_all for transformers + 'safetensors': True, # Keep collect_all for safetensors +} + +# For other libraries, use selective collection to speed up startup +other_libraries = ['sktime', 'scipy', 'pandas', 'sklearn', 'statsmodels', 'optuna'] + +for lib in essential_libraries: + try: + lib_datas, lib_binaries, lib_hiddenimports = collect_all(lib) + all_datas.extend(lib_datas) + all_binaries.extend(lib_binaries) + all_hiddenimports.extend(lib_hiddenimports) + except Exception: + pass + +# For other libraries, only collect submodules (lighter weight) +# This relies on PyInstaller's dependency analysis to include what's actually used +for lib in other_libraries: + try: + submodules = collect_submodules(lib) + all_hiddenimports.extend(submodules) + # Only collect essential data files and binaries, not all submodules + # This significantly reduces startup time + try: + lib_datas, lib_binaries, _ = collect_all(lib) + all_datas.extend(lib_datas) + all_binaries.extend(lib_binaries) + except Exception: + # If collect_all fails, try collect_data_files for essential data only + try: + lib_datas = collect_data_files(lib) + all_datas.extend(lib_datas) + except Exception: + pass + except Exception: + pass + +# Project-specific packages that need their submodules collected +# Only list top-level packages - collect_submodules will recursively collect all submodules +TOP_LEVEL_PACKAGES = [ + 'iotdb.ainode.core', # This will include all sub-packages: manager, model, inference, etc. + 'iotdb.thrift', # This will include all thrift sub-packages +] + +# Collect all submodules for project packages automatically +# Using top-level packages avoids duplicate collection +for package in TOP_LEVEL_PACKAGES: + try: + submodules = collect_submodules(package) + all_hiddenimports.extend(submodules) + except Exception: + # If package doesn't exist or collection fails, add the package itself + all_hiddenimports.append(package) + +# Add parent packages to ensure they are included +all_hiddenimports.extend(['iotdb', 'iotdb.ainode']) + +# Multiprocessing support for PyInstaller +# When using multiprocessing with PyInstaller, we need to ensure proper handling +multiprocessing_modules = [ + 'multiprocessing', + 'multiprocessing.spawn', + 'multiprocessing.popen_spawn_posix', + 'multiprocessing.popen_spawn_win32', + 'multiprocessing.popen_fork', + 'multiprocessing.popen_forkserver', + 'multiprocessing.context', + 'multiprocessing.reduction', + 'multiprocessing.util', + 'torch.multiprocessing', + 'torch.multiprocessing.spawn', +] + +# Additional dependencies that may need explicit import +# These are external libraries that might use dynamic imports +external_dependencies = [ + 'huggingface_hub', + 'tokenizers', + 'hf_xet', + 'einops', + 'dynaconf', + 'tzlocal', + 'thrift', + 'psutil', + 'requests', +] + +all_hiddenimports.extend(multiprocessing_modules) +all_hiddenimports.extend(external_dependencies) + +# Analyze main entry file +a = Analysis( + ['iotdb/ainode/core/script.py'], + pathex=[str(project_root)], + binaries=all_binaries, + datas=all_datas, + hiddenimports=all_hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[ + # Exclude unnecessary modules to reduce size and improve startup time + # Note: Do not exclude unittest, as torch and other libraries require it + # Only exclude modules that are definitely not used and not required by dependencies + 'matplotlib', + 'IPython', + 'jupyter', + 'notebook', + 'pytest', + 'test', + 'tests' + ], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=True, # Set to True to speed up startup - files are not archived into PYZ +) + +# Package all PYZ files +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +# Create executable (onedir mode for faster startup) +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='ainode', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) + +# Collect all files into a directory (onedir mode) +coll = COLLECT( + exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='ainode', +) \ No newline at end of file diff --git a/iotdb-core/ainode/ainode.xml b/iotdb-core/ainode/ainode.xml index beab4b69c01..4cf3037f4b0 100644 --- a/iotdb-core/ainode/ainode.xml +++ b/iotdb-core/ainode/ainode.xml @@ -42,6 +42,10 @@ </file> </files> <fileSets> + <fileSet> + <directory>iotdb/ainode/conf</directory> + <outputDirectory>conf</outputDirectory> + </fileSet> <fileSet> <directory>resources/conf</directory> <outputDirectory>conf</outputDirectory> @@ -54,17 +58,6 @@ <fileSet> <directory>dist</directory> <outputDirectory>lib</outputDirectory> - <includes> - <include>*.whl</include> - </includes> - </fileSet> - <fileSet> - <directory>${project.basedir}/../../scripts/conf</directory> - <outputDirectory>conf</outputDirectory> - <includes> - <include>ainode-env.*</include> - <include>**/ainode-env.*</include> - </includes> <fileMode>0755</fileMode> </fileSet> <fileSet> diff --git a/iotdb-core/ainode/build_binary.py b/iotdb-core/ainode/build_binary.py new file mode 100644 index 00000000000..3c5d153a244 --- /dev/null +++ b/iotdb-core/ainode/build_binary.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +""" +PyInstaller build script (Python version) +""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +def setup_venv(): + """Create virtual environment if it doesn't exist""" + script_dir = Path(__file__).parent + venv_dir = script_dir / ".venv" + + if venv_dir.exists(): + print(f"Virtual environment already exists at: {venv_dir}") + return venv_dir + + print(f"Creating virtual environment at: {venv_dir}") + subprocess.run([sys.executable, "-m", "venv", str(venv_dir)], check=True) + print("Virtual environment created successfully") + return venv_dir + + +def get_venv_python(venv_dir): + """Get Python executable path in virtual environment""" + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + else: + return venv_dir / "bin" / "python" + + +def update_pip(venv_python): + """ + Update pip in virtual environment. + + Note: subprocess.run() is synchronous and blocks until the subprocess completes. + This ensures pip upgrade finishes before the script continues. + """ + print("Updating pip...") + subprocess.run( + [str(venv_python), "-m", "pip", "install", "--upgrade", "pip"], check=True + ) + print("pip updated successfully") + + +def get_venv_env(venv_dir): + """Get environment variables for virtual environment activation""" + env = os.environ.copy() + venv_path = str(venv_dir.absolute()) + env["VIRTUAL_ENV"] = venv_path + + # Add venv bin directory to PATH + if sys.platform == "win32": + venv_bin = str(venv_dir / "Scripts") + else: + venv_bin = str(venv_dir / "bin") + + # Prepend venv bin to PATH to ensure venv tools take precedence + env["PATH"] = f"{venv_bin}{os.pathsep}{env.get('PATH', '')}" + + return env + + +def install_dependencies(venv_python, venv_dir, script_dir): + """ + Install dependencies using poetry. + + Note: subprocess.run() is synchronous and blocks until each command completes. + This ensures each step (poetry lock, install) finishes before proceeding. + + We configure poetry to use our .venv directory by: + 1. Configuring poetry to use in-project virtualenvs + 2. Setting poetry to use our .venv via poetry env use + 3. Running poetry lock and install which will use our .venv + """ + print("Installing dependencies with poetry...") + + # Get environment with VIRTUAL_ENV set + venv_env = get_venv_env(venv_dir) + + # Configure poetry to use in-project virtualenvs + # This makes poetry create/use .venv in the project directory + print("Configuring poetry to use in-project virtualenvs...") + try: + subprocess.run( + ["poetry", "config", "virtualenvs.in-project", "true"], + cwd=str(script_dir), + env=venv_env, + check=True, + capture_output=True, + text=True, + ) + except Exception: + pass # Configuration may already be set + + # Configure poetry to use our existing virtual environment + # This links poetry's management to our .venv directory + print(f"Configuring poetry to use virtual environment at: {venv_dir}") + result = subprocess.run( + ["poetry", "env", "use", str(venv_python)], + cwd=str(script_dir), + env=venv_env, + check=False, # Don't fail if venv is already configured + capture_output=True, + text=True, + ) + + # Check output - if poetry tries to recreate venv, that's okay as it will use our path + if result.stdout: + output = result.stdout.strip() + print(output) + if result.stderr: + stderr = result.stderr.strip() + # Ignore warnings about venv already existing or being created + if ( + "already been activated" not in stderr.lower() + and "already in use" not in stderr.lower() + ): + print(stderr) + + # Run poetry lock + print("Running poetry lock...") + result = subprocess.run( + ["poetry", "lock"], + cwd=str(script_dir), + env=venv_env, + check=True, + capture_output=True, + text=True, + ) + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + + # Run poetry install + # With VIRTUAL_ENV set and poetry env use configured, this should install into our .venv + print("Running poetry install...") + result = subprocess.run( + ["poetry", "install"], + cwd=str(script_dir), + env=venv_env, + check=True, + capture_output=True, + text=True, + ) + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + + print("Dependencies installed successfully") + + +def check_pyinstaller(venv_python): + """Check if PyInstaller is installed""" + try: + result = subprocess.run( + [ + str(venv_python), + "-c", + "import PyInstaller; print(PyInstaller.__version__)", + ], + capture_output=True, + text=True, + check=True, + ) + version = result.stdout.strip() + print(f"PyInstaller version: {version}") + return True + except (subprocess.CalledProcessError, FileNotFoundError): + print("Error: PyInstaller is not installed") + print("Please run: pip install pyinstaller") + return False + + +def build(): + """Execute build process""" + script_dir = Path(__file__).parent + + # Setup virtual environment + venv_dir = setup_venv() + venv_python = get_venv_python(venv_dir) + + # Update pip + update_pip(venv_python) + + # Install dependencies + install_dependencies(venv_python, venv_dir, script_dir) + + # Check PyInstaller + if not check_pyinstaller(venv_python): + sys.exit(1) + + print("=" * 50) + print("IoTDB AINode PyInstaller Build Script") + print("=" * 50) + print() + + # Execute build (incremental build - no cleanup for faster rebuilds) + print("Starting build...") + print() + + spec_file = script_dir / "ainode.spec" + if not spec_file.exists(): + print(f"Error: Spec file not found: {spec_file}") + sys.exit(1) + + cmd = [ + str(venv_python), + "-m", + "PyInstaller", + "--noconfirm", + str(spec_file), + ] + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"\nError: Build failed: {e}") + sys.exit(1) + + print() + print("=" * 50) + print("Build completed!") + print("=" * 50) + print() + print("Executable location: dist/ainode/ainode") + print() + print("Usage:") + print(" ./dist/ainode/ainode start # Start AINode") + print(" ./dist/ainode/ainode remove # Remove AINode") + print() + + +if __name__ == "__main__": + build() diff --git a/iotdb-core/ainode/iotdb/ainode/core/config.py b/iotdb-core/ainode/iotdb/ainode/core/config.py index 328f4a5faa6..44a21c72f81 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/config.py +++ b/iotdb-core/ainode/iotdb/ainode/core/config.py @@ -37,8 +37,6 @@ from iotdb.ainode.core.constant import ( AINODE_INFERENCE_MODEL_MEM_USAGE_MAP, AINODE_LOG_DIR, AINODE_MODELS_DIR, - AINODE_ROOT_CONF_DIRECTORY_NAME, - AINODE_ROOT_DIR, AINODE_RPC_ADDRESS, AINODE_RPC_PORT, AINODE_SYSTEM_DIR, @@ -315,9 +313,7 @@ class AINodeDescriptor(object): if "ainode_id" in system_configs: self._config.set_ainode_id(int(system_configs["ainode_id"])) - git_file = os.path.join( - AINODE_ROOT_DIR, AINODE_ROOT_CONF_DIRECTORY_NAME, AINODE_CONF_GIT_FILE_NAME - ) + git_file = os.path.join(AINODE_CONF_DIRECTORY_NAME, AINODE_CONF_GIT_FILE_NAME) if os.path.exists(git_file): git_configs = load_properties(git_file) if "git.commit.id.abbrev" in git_configs: @@ -327,9 +323,7 @@ class AINodeDescriptor(object): build_info += "-dev" self._config.set_build_info(build_info) - pom_file = os.path.join( - AINODE_ROOT_DIR, AINODE_ROOT_CONF_DIRECTORY_NAME, AINODE_CONF_POM_FILE_NAME - ) + pom_file = os.path.join(AINODE_CONF_DIRECTORY_NAME, AINODE_CONF_POM_FILE_NAME) if os.path.exists(pom_file): pom_configs = load_properties(pom_file) if "version" in pom_configs: diff --git a/iotdb-core/ainode/iotdb/ainode/core/constant.py b/iotdb-core/ainode/iotdb/ainode/core/constant.py index f4547e99803..96eca136368 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/constant.py +++ b/iotdb-core/ainode/iotdb/ainode/core/constant.py @@ -27,7 +27,6 @@ from iotdb.thrift.common.ttypes import TEndPoint AINODE_VERSION_INFO = "UNKNOWN" AINODE_BUILD_INFO = "UNKNOWN" AINODE_CONF_DIRECTORY_NAME = "conf" -AINODE_ROOT_CONF_DIRECTORY_NAME = "conf" AINODE_CONF_FILE_NAME = "iotdb-ainode.properties" AINODE_CONF_GIT_FILE_NAME = "git.properties" AINODE_CONF_POM_FILE_NAME = "pom.properties" @@ -62,9 +61,6 @@ AINODE_INFERENCE_EXTRA_MEMORY_RATIO = ( ) # AINode folder structure -AINODE_ROOT_DIR = os.path.dirname( - os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -) AINODE_MODELS_DIR = "data/ainode/models" AINODE_BUILTIN_MODELS_DIR = "data/ainode/models/weights" # For built-in models, we only need to store their weights and config. AINODE_SYSTEM_DIR = "data/ainode/system" diff --git a/iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py b/iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py index 069a6b9ced6..00bb3b5568b 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py +++ b/iotdb-core/ainode/iotdb/ainode/core/inference/pool_controller.py @@ -50,7 +50,6 @@ from iotdb.ainode.core.util.decorator import synchronized from iotdb.ainode.core.util.thread_name import ThreadName logger = Logger() -MODEL_MANAGER = ModelManager() class PoolController: @@ -59,6 +58,7 @@ class PoolController: """ def __init__(self, result_queue: mp.Queue): + self._model_manager = ModelManager() # structure: {model_id: {device_id: PoolGroup}} self._request_pool_map: Dict[str, Dict[str, PoolGroup]] = {} self._new_pool_id = AtomicInt() @@ -82,24 +82,25 @@ class PoolController: """ Initialize the pools when the first request for the given model_id arrives. """ - if not self.has_request_pools(model_id, device.index): - # TODO: choose a device based on some strategy - device = self.DEFAULT_DEVICE - actions = self._pool_scheduler.schedule(model_id, device) - for action in actions: - if action.action == ScaleActionType.SCALE_UP: - # initialize the first pool - self._first_pool_init(action.model_id, str(device)) - # start a background thread to expand pools - expand_thread = threading.Thread( - target=self._expand_pools_on_device, - args=(action.model_id, str(device), action.amount - 1), - daemon=True, - ) - expand_thread.start() - elif action.action == ScaleActionType.SCALE_DOWN: - # TODO: implement scale down logic - pass + pass + # if not self.has_request_pools(model_id, device.index): + # # TODO: choose a device based on some strategy + # device = self.DEFAULT_DEVICE + # actions = self._pool_scheduler.schedule(model_id, device) + # for action in actions: + # if action.action == ScaleActionType.SCALE_UP: + # # initialize the first pool + # self._first_pool_init(action.model_id, str(device)) + # # start a background thread to expand pools + # expand_thread = threading.Thread( + # target=self._expand_pools_on_device, + # args=(action.model_id, str(device), action.amount - 1), + # daemon=True, + # ) + # expand_thread.start() + # elif action.action == ScaleActionType.SCALE_DOWN: + # # TODO: implement scale down logic + # pass def _first_pool_init(self, model_id: str, device_str: str): """ @@ -194,7 +195,7 @@ class PoolController: def _load_model_on_device_task(device_id: str): if not self.has_request_pools(model_id, device_id): actions = self._pool_scheduler.schedule_load_model_to_device( - MODEL_MANAGER.get_model_info(model_id), device_id + self._model_manager.get_model_info(model_id), device_id ) for action in actions: if action.action == ScaleActionType.SCALE_UP: @@ -221,7 +222,7 @@ class PoolController: def _unload_model_on_device_task(device_id: str): if self.has_request_pools(model_id, device_id): actions = self._pool_scheduler.schedule_unload_model_from_device( - MODEL_MANAGER.get_model_info(model_id), device_id + self._model_manager.get_model_info(model_id), device_id ) for action in actions: if action.action == ScaleActionType.SCALE_DOWN: @@ -256,7 +257,7 @@ class PoolController: def _expand_pool_on_device(*_): result_queue = mp.Queue() pool_id = self._new_pool_id.get_and_increment() - model_info = MODEL_MANAGER.get_model_info(model_id) + model_info = self._model_manager.get_model_info(model_id) model_type = model_info.model_type if model_type == BuiltInModelType.SUNDIAL.value: config = SundialConfig() @@ -277,7 +278,7 @@ class PoolController: ) pool.start() self._register_pool(model_id, device_id, pool_id, pool, result_queue) - if not pool.ready_event.wait(timeout=30): + if not pool.ready_event.wait(timeout=300): logger.error( f"[Inference][Device-{device_id}][Pool-{pool_id}] Pool failed to be ready in time" ) diff --git a/iotdb-core/ainode/iotdb/ainode/core/inference/pool_scheduler/basic_pool_scheduler.py b/iotdb-core/ainode/iotdb/ainode/core/inference/pool_scheduler/basic_pool_scheduler.py index 5ee1b4f0c9a..6a2bd2b619a 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/inference/pool_scheduler/basic_pool_scheduler.py +++ b/iotdb-core/ainode/iotdb/ainode/core/inference/pool_scheduler/basic_pool_scheduler.py @@ -41,8 +41,6 @@ from iotdb.ainode.core.util.gpu_mapping import convert_device_id_to_torch_device logger = Logger() -MODEL_MANAGER = ModelManager() - def _estimate_shared_pool_size_by_total_mem( device: torch.device, @@ -106,6 +104,7 @@ class BasicPoolScheduler(AbstractPoolScheduler): def __init__(self, request_pool_map: Dict[str, Dict[str, PoolGroup]]): super().__init__(request_pool_map) + self._model_manager = ModelManager() def schedule(self, model_id: str) -> List[ScaleAction]: """ @@ -123,7 +122,7 @@ class BasicPoolScheduler(AbstractPoolScheduler): self, model_info: ModelInfo, device_id: str ) -> List[ScaleAction]: existing_model_infos = [ - MODEL_MANAGER.get_model_info(existing_model_id) + self._model_manager.get_model_info(existing_model_id) for existing_model_id, pool_group_map in self._request_pool_map.items() if existing_model_id != model_info.model_id and device_id in pool_group_map ] @@ -140,7 +139,7 @@ class BasicPoolScheduler(AbstractPoolScheduler): self, model_info: ModelInfo, device_id: str ) -> List[ScaleAction]: existing_model_infos = [ - MODEL_MANAGER.get_model_info(existing_model_id) + self._model_manager.get_model_info(existing_model_id) for existing_model_id, pool_group_map in self._request_pool_map.items() if existing_model_id != model_info.model_id and device_id in pool_group_map ] diff --git a/iotdb-core/ainode/iotdb/ainode/core/log.py b/iotdb-core/ainode/iotdb/ainode/core/log.py index a7a05d0ea92..fd121b26349 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/log.py +++ b/iotdb-core/ainode/iotdb/ainode/core/log.py @@ -116,7 +116,9 @@ class BaseLogger: file_handler.setFormatter(self.logger_format) self.logger.addHandler(file_handler) - self.info(f"Logger init successfully.") + self.info( + f"Logger init successfully, log file prefix name {log_file_name_prefix}." + ) # interfaces def debug(self, *msg): diff --git a/iotdb-core/ainode/iotdb/ainode/core/script.py b/iotdb-core/ainode/iotdb/ainode/core/script.py index 82ed32bc5a2..6b7e6ec54e4 100644 --- a/iotdb-core/ainode/iotdb/ainode/core/script.py +++ b/iotdb-core/ainode/iotdb/ainode/core/script.py @@ -19,6 +19,7 @@ import os import shutil import sys +import multiprocessing import torch.multiprocessing as mp from iotdb.ainode.core.ai_node import AINode @@ -74,6 +75,14 @@ def remove_ainode(arguments): def main(): + # Handle PyInstaller: filter out Python arguments that might be passed to subprocesses + # These arguments are not needed in frozen executables and cause warnings + # Note: This filtering should happen AFTER freeze_support() has handled child processes + if getattr(sys, "frozen", False): + python_args_to_filter = ["-I", "-B", "-S", "-E", "-O", "-OO"] + sys.argv = [arg for arg in sys.argv if arg not in python_args_to_filter] + + logger.info(f"Starting IoTDB-AINode process with sys argv {sys.argv}.") arguments = sys.argv # load config AINodeDescriptor() @@ -104,4 +113,15 @@ def main(): if __name__ == "__main__": + # PyInstaller multiprocessing support + # freeze_support() is essential for PyInstaller frozen executables on all platforms + # It detects if the current process is a multiprocessing child process + # If it is, it executes the child process target function and exits + # If it's not, it returns immediately and continues with main() execution + # This prevents child processes from executing the main application logic + if getattr(sys, "frozen", False): + # Call freeze_support() for both standard multiprocessing and torch.multiprocessing + multiprocessing.freeze_support() + mp.freeze_support() + main() diff --git a/iotdb-core/ainode/poetry.lock b/iotdb-core/ainode/poetry.lock index 7a22506d06f..a2bcb1d31d4 100644 --- a/iotdb-core/ainode/poetry.lock +++ b/iotdb-core/ainode/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "alembic" @@ -20,6 +20,18 @@ typing-extensions = ">=4.12" [package.extras] tz = ["tzdata"] +[[package]] +name = "altgraph" +version = "0.17.4" +description = "Python graph (network) package" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, + {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, +] + [[package]] name = "black" version = "25.1.0" @@ -510,6 +522,22 @@ cli = ["jsonargparse[signatures] (>=4.38.0)", "tomlkit"] docs = ["requests (>=2.0.0)"] typing = ["mypy (>=1.0.0)", "types-setuptools"] +[[package]] +name = "macholib" +version = "1.16.3" +description = "Mach-O header analysis and editing" +optional = false +python-versions = "*" +groups = ["main"] +markers = "sys_platform == \"darwin\"" +files = [ + {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, + {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, +] + +[package.dependencies] +altgraph = ">=0.17" + [[package]] name = "mako" version = "1.3.10" @@ -1141,6 +1169,19 @@ numpy = ">=1.4" [package.extras] test = ["pytest", "pytest-cov", "scipy"] +[[package]] +name = "pefile" +version = "2023.2.7" +description = "Python PE parsing module" +optional = false +python-versions = ">=3.6.0" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, +] + [[package]] name = "platformdirs" version = "4.4.0" @@ -1181,6 +1222,57 @@ files = [ dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pyreadline ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-subtests", "pytest-xdist", "pywin32 ; os_name == \"nt\" and platform_python_implementation != \"PyPy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and platform_python_implementation != \ [...] test = ["pytest", "pytest-instafail", "pytest-subtests", "pytest-xdist", "pywin32 ; os_name == \"nt\" and platform_python_implementation != \"PyPy\"", "setuptools", "wheel ; os_name == \"nt\" and platform_python_implementation != \"PyPy\"", "wmi ; os_name == \"nt\" and platform_python_implementation != \"PyPy\""] +[[package]] +name = "pyinstaller" +version = "6.16.0" +description = "PyInstaller bundles a Python application and all its dependencies into a single package." +optional = false +python-versions = "<3.15,>=3.8" +groups = ["main"] +files = [ + {file = "pyinstaller-6.16.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:7fd1c785219a87ca747c21fa92f561b0d2926a7edc06d0a0fe37f3736e00bd7a"}, + {file = "pyinstaller-6.16.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b756ddb9007b8141c5476b553351f9d97559b8af5d07f9460869bfae02be26b0"}, + {file = "pyinstaller-6.16.0-py3-none-manylinux2014_i686.whl", hash = "sha256:0a48f55b85ff60f83169e10050f2759019cf1d06773ad1c4da3a411cd8751058"}, + {file = "pyinstaller-6.16.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:73ba72e04fcece92e32518bbb1e1fb5ac2892677943dfdff38e01a06e8742851"}, + {file = "pyinstaller-6.16.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:b1752488248f7899281b17ca3238eefb5410521291371a686a4f5830f29f52b3"}, + {file = "pyinstaller-6.16.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ba618a61627ee674d6d68e5de084ba17c707b59a4f2a856084b3999bdffbd3f0"}, + {file = "pyinstaller-6.16.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:c8b7ef536711617e12fef4673806198872033fa06fa92326ad7fd1d84a9fa454"}, + {file = "pyinstaller-6.16.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:d1ebf84d02c51fed19b82a8abb4df536923abd55bb684d694e1356e4ae2a0ce5"}, + {file = "pyinstaller-6.16.0-py3-none-win32.whl", hash = "sha256:6d5f8617f3650ff9ef893e2ab4ddbf3c0d23d0c602ef74b5df8fbef4607840c8"}, + {file = "pyinstaller-6.16.0-py3-none-win_amd64.whl", hash = "sha256:bc10eb1a787f99fea613509f55b902fbd2d8b73ff5f51ff245ea29a481d97d41"}, + {file = "pyinstaller-6.16.0-py3-none-win_arm64.whl", hash = "sha256:d0af8a401de792c233c32c44b16d065ca9ab8262ee0c906835c12bdebc992a64"}, + {file = "pyinstaller-6.16.0.tar.gz", hash = "sha256:53559fe1e041a234f2b4dcc3288ea8bdd57f7cad8a6644e422c27bb407f3edef"}, +] + +[package.dependencies] +altgraph = "*" +macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +packaging = ">=22.0" +pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2025.8" +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} +setuptools = ">=42.0.0" + +[package.extras] +completion = ["argcomplete"] +hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2025.9" +description = "Community maintained hooks for PyInstaller" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyinstaller_hooks_contrib-2025.9-py3-none-any.whl", hash = "sha256:ccbfaa49399ef6b18486a165810155e5a8d4c59b41f20dc5da81af7482aaf038"}, + {file = "pyinstaller_hooks_contrib-2025.9.tar.gz", hash = "sha256:56e972bdaad4e9af767ed47d132362d162112260cbe488c9da7fee01f228a5a6"}, +] + +[package.dependencies] +packaging = ">=22.0" +setuptools = ">=42.0.0" + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1208,6 +1300,19 @@ files = [ {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -2223,4 +2328,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<=3.13.5" -content-hash = "08746b5f45ee9836e2d4368f8e394d55d27f556789fb1254e7584b9eedd49d5b" +content-hash = "5a09e61ffb4faed0c9f56fe9518d2f3f9d89405465ec2efb8b939586955d890b" diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml index 63c8f2e3f55..1a3be76b633 100644 --- a/iotdb-core/ainode/pom.xml +++ b/iotdb-core/ainode/pom.xml @@ -87,9 +87,6 @@ <fileset> <directory>target</directory> </fileset> - <fileset> - <directory>venv</directory> - </fileset> </filesets> </configuration> </plugin> @@ -142,6 +139,9 @@ <encoding>${project.build.sourceEncoding}</encoding> </configuration> <executions> + <!-- + Copy the python api of IoTDB from iotdb-client. + --> <execution> <id>copy-python-dependencies</id> <phase>generate-sources</phase> @@ -212,55 +212,6 @@ <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> <executions> - <!-- Prepare a local installation of python --> - <execution> - <id>python-venv</id> - <phase>initialize</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.exe.bin}</executable> - <arguments> - <argument>-m</argument> - <argument>venv</argument> - <argument>./venv</argument> - </arguments> - </configuration> - </execution> - <!-- Update pip in our local working environment --> - <execution> - <id>python-upgrade-pip</id> - <phase>initialize</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.venv.bin}${python.exe.bin}</executable> - <arguments> - <argument>-m</argument> - <argument>pip</argument> - <argument>install</argument> - <argument>--upgrade</argument> - <argument>pip</argument> - </arguments> - </configuration> - </execution> - <!-- Install poetry in our local working environment --> - <execution> - <id>python-install-poetry</id> - <phase>initialize</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.venv.bin}pip3</executable> - <arguments> - <argument>install</argument> - <argument>poetry</argument> - </arguments> - </configuration> - </execution> <execution> <id>python-compile</id> <phase>compile</phase> @@ -268,44 +219,13 @@ <goal>exec</goal> </goals> <configuration> - <executable>${python.venv.bin}poetry</executable> + <executable>${python.exe.bin}</executable> + <workingDirectory>${project.basedir}</workingDirectory> <arguments> - <argument>build</argument> + <argument>build_binary.py</argument> </arguments> </configuration> </execution> - <!--execution> - <id>python-test</id> - <phase>test</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.venv.bin}${python.exe.bin}</executable> - <arguments> - <argument>-m</argument> - <argument>pytest</argument> - <argument>tests</argument> - </arguments> - </configuration> - </execution--> - <!--execution> - <id>python-package</id> - <phase>package</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${python.venv.bin}pip3</executable> - <arguments> - <argument>wheel</argument> - <argument>.</argument> - <argument>-w</argument> - <argument>dist</argument> - <argument>- -no-deps</argument> - </arguments> - </configuration> - </execution--> </executions> </plugin> <plugin> diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml index 634f3e09a74..f1d197b0165 100644 --- a/iotdb-core/ainode/pyproject.toml +++ b/iotdb-core/ainode/pyproject.toml @@ -79,19 +79,10 @@ exclude = [ python = ">=3.11,<=3.13.5" # Core scientific stack -numpy = [ - { version = "^2.3.2", python = ">=3.10" }, - { version = "^1.26.4", python = ">=3.9,<3.10" } -] -scipy = [ - { version = "^1.12.0", python = ">=3.10" }, - { version = "^1.11.4", python = ">=3.9,<3.10" } -] +numpy = "^2.3.2" +scipy = "^1.12.0" pandas = "^2.3.2" -scikit-learn = [ - { version = "^1.7.1", python = ">=3.10" }, - { version = "^1.5.2", python = ">=3.9,<3.10" } -] +scikit-learn = "^1.7.1" statsmodels = "^0.14.5" sktime = "0.38.5" @@ -116,6 +107,7 @@ tzlocal = "^5.3.1" hf_xet = ">=1.1.9" # ---- Tooling ---- +pyinstaller = "^6.13.0" black = "25.1.0" isort = "6.0.1" setuptools = ">=75.3.0" diff --git a/iotdb-core/ainode/resources/syncPythonVersion.groovy b/iotdb-core/ainode/resources/syncPythonVersion.groovy index 0061930bfca..ecd0f2bdf21 100644 --- a/iotdb-core/ainode/resources/syncPythonVersion.groovy +++ b/iotdb-core/ainode/resources/syncPythonVersion.groovy @@ -120,41 +120,8 @@ def checkPython() { } } - -// On Ubuntu it seems that venv is generally available, but the 'ensurepip' command fails. -// In this case we need to install the python3-venv package. Unfortunately checking the -// venv is successful in this case, so we need this slightly odd test. -def checkPythonVenv() { - print "Detecting venv: " - try { - def python = project.properties['python.exe.bin'] - def cmdArray = [python, "-Im", "ensurepip"] - def process = cmdArray.execute() - def stdOut = new StringBuilder() - def stdErr = new StringBuilder() - process.waitForProcessOutput(stdOut, stdErr) - if (stdErr.contains("No module named")) { - println "missing" - println "--- output of version `python -Im \"ensurepip\"` command ---" - println output - println "------------------------------------------------------------" - allConditionsMet = false - } else { - println " OK" - } - } catch (Exception e) { - println "missing" - println "--- failed with exception ---" - println e - e.printStackTrace() - println "----------------------------------------------------" - allConditionsMet = false - } -} - // Check the python environment is setup correctly. checkPython() -checkPythonVenv() if (!allConditionsMet) { throw new RuntimeException("Not all conditions met, see log for details.") diff --git a/scripts/conf/ainode-env.sh b/scripts/conf/ainode-env.sh deleted file mode 100644 index 1ec434ad2d9..00000000000 --- a/scripts/conf/ainode-env.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# The defaulte venv environment is used if ain_interpreter_dir is not set. Please use absolute path without quotation mark -# ain_interpreter_dir= - -# Set ain_force_reinstall to 1 to force reinstall AINode -ain_force_reinstall=0 - -# don't install dependencies online -ain_install_offline=0 - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - -# fetch parameters with names -while getopts "i:t:rnm:" opt; do - case $opt in - i) - p_ain_interpreter_dir="$OPTARG" - ;; - r) - p_ain_force_reinstall=1 - ;; - t) ;; - n) - p_ain_no_dependencies="--no-dependencies" - ;; - m) - p_pypi_mirror="$OPTARG" - ;; - \?) - echo "Invalid option -$OPTARG" >&2 - exit 1 - ;; - esac -done - -if [ -z "$p_ain_interpreter_dir" ]; then - echo "No interpreter_dir is set, use default value." -else - ain_interpreter_dir="$p_ain_interpreter_dir" -fi - -if [ -z "$p_ain_force_reinstall" ]; then - echo "No check_version is set, use default value." -else - ain_force_reinstall="$p_ain_force_reinstall" -fi -echo Script got inputs: "ain_interpreter_dir: $ain_interpreter_dir", "ain_force_reinstall: $ain_force_reinstall" - -if [ -z $ain_interpreter_dir ]; then - $(dirname "$0")/../venv/bin/python3 -c "import sys; print(sys.executable)" && - echo "Activate default venv environment" || ( - echo "Creating default venv environment" && python3 -m venv "$(dirname "$0")/../venv" - ) - ain_interpreter_dir="$SCRIPT_DIR/../venv/bin/python3" -fi -echo "Calling venv to check: $ain_interpreter_dir" - -# Change the working directory to the parent directory -cd "$SCRIPT_DIR/.." - -echo "Confirming AINode..." -$ain_interpreter_dir -m pip config set global.disable-pip-version-check true -$ain_interpreter_dir -m pip list | grep "apache-iotdb-ainode" >/dev/null -if [ $? -eq 0 ]; then - if [ $ain_force_reinstall -eq 0 ]; then - echo "AINode is already installed" - exit 0 - fi -fi - -ain_only_ainode=1 - -# if $ain_install_offline is 1 then do not install dependencies -if [ $ain_install_offline -eq 1 ]; then - # if offline and not -n, then install dependencies - if [ -z "$p_ain_no_dependencies" ]; then - ain_only_ainode=0 - else - ain_only_ainode=1 - fi - p_ain_no_dependencies="--no-dependencies" - echo "Installing AINode offline----without dependencies..." -fi - -if [ $ain_force_reinstall -eq 1 ]; then - p_ain_force_reinstall="--force-reinstall" -else - p_ain_force_reinstall="" -fi - -echo "Installing AINode..." -cd "$SCRIPT_DIR/../lib/" -shopt -s nullglob -for i in *.whl; do - if [[ $i =~ "ainode" ]]; then - echo Installing AINode body: $i - if [ -z "$p_pypi_mirror" ]; then - $ain_interpreter_dir -m pip install "$i" $p_ain_force_reinstall --no-warn-script-location $p_ain_no_dependencies --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - else - $ain_interpreter_dir -m pip install "$i" $p_ain_force_reinstall -i $p_pypi_mirror --no-warn-script-location $p_ain_no_dependencies --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - fi - else - # if ain_only_ainode is 0 then install dependencies - if [ $ain_only_ainode -eq 0 ]; then - echo Installing dependencies $i - if [ -z "$p_pypi_mirror" ]; then - $ain_interpreter_dir -m pip install "$i" $p_ain_force_reinstall --no-warn-script-location $p_ain_no_dependencies - else - $ain_interpreter_dir -m pip install "$i" $p_ain_force_reinstall -i $p_pypi_mirror --no-warn-script-location $p_ain_no_dependencies - fi - fi - fi - if [ $? -eq 1 ]; then - echo "Failed to install AINode" - exit 1 - fi -done -echo "AINode is installed successfully" -exit 0 diff --git a/scripts/conf/windows/ainode-env.bat b/scripts/conf/windows/ainode-env.bat deleted file mode 100644 index 2c01d411a2c..00000000000 --- a/scripts/conf/windows/ainode-env.bat +++ /dev/null @@ -1,129 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM - -@echo off - -@REM The defaulte venv environment is used if ain_interpreter_dir is not set. Please use absolute path without quotation mark -@REM set ain_interpreter_dir= - -@REM Set ain_force_reinstall to 1 to force reinstall ainode -set ain_force_reinstall=0 - -@REM don't install dependencies online -set ain_install_offline=0 - -pushd %~dp0..\.. -if NOT DEFINED IOTDB_AINODE_HOME set IOTDB_AINODE_HOME=%cd% - -:initial -if "%1"=="" goto done -set aux=%1 -if "%aux:~0,2%"=="-r" ( - set ain_force_reinstall=1 - shift - goto initial -) -if "%aux:~0,2%"=="-n" ( - set ain_no_dependencies=--no-dependencies - shift - goto initial -) -if "%aux:~0,1%"=="-" ( - set nome=%aux:~1,250% -) else ( - set "%nome%=%1" - set nome= -) -shift -goto initial - -:done -@REM check if the parameters are set -if "%i%"=="" ( - echo No interpreter_dir is set, use default value. -) else ( - set ain_interpreter_dir=%i% -) - -echo Script got inputs: ain_interpreter_dir: %ain_interpreter_dir% , ain_force_reinstall: %ain_force_reinstall% -if "%ain_interpreter_dir%"=="" ( - %IOTDB_AINODE_HOME%//venv//Scripts//python.exe -c "import sys; print(sys.executable)" && ( - echo Activate default venv environment - ) || ( - echo Creating default venv environment - python -m venv "%IOTDB_AINODE_HOME%//venv" - ) - set ain_interpreter_dir="%IOTDB_AINODE_HOME%//venv//Scripts//python.exe" -) - -@REM Switch the working directory to the directory one level above the script -cd %IOTDB_AINODE_HOME% - -echo Confirming ainode -%ain_interpreter_dir% -m pip config set global.disable-pip-version-check true -%ain_interpreter_dir% -m pip list | findstr /C:"apache-iotdb-ainode" >nul -if %errorlevel% == 0 ( - if %ain_force_reinstall% == 0 ( - echo ainode is already installed - exit /b 0 - ) -) - -set ain_only_ainode=1 -@REM if $ain_install_offline is 1 then do not install dependencies -if %ain_install_offline% == 1 ( - @REM if offline and not -n, then install dependencies - if "%ain_no_dependencies%"=="" ( - set ain_only_ainode=0 - ) else ( - set ain_only_ainode=1 - ) - set ain_no_dependencies=--no-dependencies - echo Installing ainode offline----without dependencies... -) - -if %ain_force_reinstall% == 1 ( - set ain_force_reinstall=--force-reinstall -) else ( - set ain_force_reinstall= -) - -echo Installing ainode... -@REM Print current work dir -cd lib -for %%i in (*.whl *.tar.gz) do ( - echo %%i | findstr "ainode" >nul && ( - echo Installing ainode body: %%i - %ain_interpreter_dir% -m pip install %%i %ain_force_reinstall% --no-warn-script-location %ain_no_dependencies% --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - ) || ( - @REM if ain_only_ainode is 0 then install dependencies - if %ain_only_ainode% == 0 ( - echo Installing dependencies: %%i - set ain_force_reinstall=--force-reinstall - %ain_interpreter_dir% -m pip install %%i %ain_force_reinstall% --no-warn-script-location %ain_no_dependencies% --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - ) - ) - if %errorlevel% == 1 ( - echo Failed to install ainode - exit /b 1 - ) -) -echo ainode is installed successfully -cd .. -exit /b 0 diff --git a/scripts/sbin/start-ainode.sh b/scripts/sbin/start-ainode.sh index 4ab202a209e..8011d2f03b0 100644 --- a/scripts/sbin/start-ainode.sh +++ b/scripts/sbin/start-ainode.sh @@ -25,22 +25,10 @@ echo --------------------------- IOTDB_AINODE_HOME="$(cd "`dirname "$0"`"/..; pwd)" echo "IOTDB_AINODE_HOME: $IOTDB_AINODE_HOME" -chmod u+x $IOTDB_AINODE_HOME/conf/ainode-env.sh -ain_interpreter_dir=$(sed -n 's/^ain_interpreter_dir=\(.*\)$/\1/p' $IOTDB_AINODE_HOME/conf/ainode-env.sh) -bash $IOTDB_AINODE_HOME/conf/ainode-env.sh $* -if [ $? -eq 1 ]; then - echo "Environment check failed. Exiting..." - exit 1 -fi - # fetch parameters with names while getopts "i:rn" opt; do case $opt in - i) p_ain_interpreter_dir="$OPTARG" - ;; - r) p_ain_force_reinstall="$OPTARG" - ;; n) ;; \?) echo "Invalid option -$OPTARG" >&2 @@ -49,31 +37,10 @@ while getopts "i:rn" opt; do esac done -# If ain_interpreter_dir in parameters is empty: -if [ -z "$p_ain_interpreter_dir" ]; then - # If ain_interpreter_dir in ../conf/ainode-env.sh is empty, set default value to ../venv/bin/python3 - if [ -z "$ain_interpreter_dir" ]; then - ain_interpreter_dir="$IOTDB_AINODE_HOME/venv/bin/python3" - fi -else - # If ain_interpreter_dir in parameters is not empty, set ain_interpreter_dir to the value in parameters - ain_interpreter_dir="$p_ain_interpreter_dir" -fi - -# check if ain_interpreter_dir is an absolute path -if [[ "$ain_interpreter_dir" != /* ]]; then - ain_interpreter_dir="$IOTDB_AINODE_HOME/$ain_interpreter_dir" -fi - -echo Script got parameter: ain_interpreter_dir: $ain_interpreter_dir - -# Change the working directory to the parent directory -cd "$IOTDB_AINODE_HOME" - -ain_ainode_dir=$(dirname "$ain_interpreter_dir")/ainode +ain_ainode_executable="$IOTDB_AINODE_HOME/lib/ainode/ainode" -echo Script got ainode dir: ain_ainode_dir: $ain_ainode_dir +echo Script got ainode executable: "$ain_ainode_executable" echo Starting AINode... -$ain_ainode_dir start +$ain_ainode_executable start diff --git a/scripts/sbin/windows/start-ainode.bat b/scripts/sbin/windows/start-ainode.bat index 0a83865fd23..bda9aedf6b2 100644 --- a/scripts/sbin/windows/start-ainode.bat +++ b/scripts/sbin/windows/start-ainode.bat @@ -26,54 +26,12 @@ echo ``````````````````````````` pushd %~dp0..\.. if NOT DEFINED IOTDB_AINODE_HOME set IOTDB_AINODE_HOME=%cd% -call %IOTDB_AINODE_HOME%\\conf\\windows\\ainode-env.bat %* -if %errorlevel% neq 0 ( - echo Environment check failed. Exiting... - exit /b 1 -) +set $ain_ainode_executable=%IOTDB_AINODE_HOME%\\lib\\ainode\\ainode -for /f "tokens=2 delims==" %%a in ('findstr /i /c:"^ain_interpreter_dir" "%IOTDB_AINODE_HOME%\\conf\\windows\\ainode-env.bat"') do ( - set _ain_interpreter_dir=%%a - goto :done -) - -:initial -if "%1"=="" goto done -set aux=%1 -if "%aux:~0,1%"=="-" ( - set nome=%aux:~1,250% -) else ( - set "%nome%=%1" - set nome= -) -shift -goto initial - -:done -if "%i%"=="" ( - if "%_ain_interpreter_dir%"=="" ( - set _ain_interpreter_dir=%IOTDB_AINODE_HOME%\\venv\\Scripts\\python.exe - ) -) else ( - set _ain_interpreter_dir=%i% -) - -echo Script got parameter: ain_interpreter_dir: %_ain_interpreter_dir% - -cd %IOTDB_AINODE_HOME% - -for %%i in ("%_ain_interpreter_dir%") do set "parent=%%~dpi" - -set ain_ainode_dir=%parent%\ainode.exe - -set ain_ainode_dir_new=%parent%\Scripts\\ainode.exe +echo Script got ainode executable: "$ain_ainode_executable" echo Starting AINode... %ain_ainode_dir% start -if %errorlevel% neq 0 ( - echo ain_ainode_dir_new is %ain_ainode_dir_new% - %ain_ainode_dir_new% start -) pause \ No newline at end of file
