This is an automated email from the ASF dual-hosted git repository. msciabarra pushed a commit to branch 0.1.0-testing in repository https://gitbox.apache.org/repos/asf/openserverless-task.git
commit 984b82da42766d387e7b29492315c03810f54bcc Author: d4rkstar <[email protected]> AuthorDate: Tue Jul 9 01:39:10 2024 +0200 feat: moved config to module. added pydoc strings Created a config.py module and moved there all the common defaults params. Added pydoc fucntions documentation --- ide/deploy/client.py | 59 +++++++++++++++++++++++------------ ide/deploy/config.py | 65 +++++++++++++++++++++++++++++++++++++++ ide/deploy/deploy.py | 87 +++++++++++++++++++++++++++++++++++++++++++--------- ide/deploy/scan.py | 51 +++++++++++++++++++----------- ide/deploy/watch.py | 39 +++++++++++++++++------ 5 files changed, 239 insertions(+), 62 deletions(-) diff --git a/ide/deploy/client.py b/ide/deploy/client.py index 21631fd..895df97 100644 --- a/ide/deploy/client.py +++ b/ide/deploy/client.py @@ -15,47 +15,66 @@ # specific language governing permissions and limitations # under the License. -from pathlib import Path from subprocess import Popen, PIPE -import os, os.path, json +import os +import os.path import threading -import asyncio - -def get_nuvolaris_config(key, default): - try: - dir = os.environ.get("NUV_PWD", "/do_not_exists") - file = f"{dir}/package.json" - info = json.loads(Path(file).read_text()) - return info.get("nuvolaris", {}).get(key, default) - except: - return default - -def readlines(inp): +from typing import IO +from .config import get_nuvolaris_config + + +def readlines(inp: IO[str]): + """Read line from an file descriptor + + Args: + inp (IO[str]): the file descriptor + """ for line in iter(inp.readline, ''): print(line, end='') # serve web area -def launch(key, default): + + +def launch(key: str, default: (str | list)): + """Launch a command in a process, reading. + The command is extracted from nuvolaris config in package.json + or a default command is used + + Args: + key (str): the key from which read the command + default (str | list): the default if the key is not found + """ cmd = get_nuvolaris_config(key, default) proc = Popen( - cmd, shell=True, - cwd=os.environ.get("NUV_PWD"), env=os.environ, + cmd, shell=True, + cwd=os.environ.get("NUV_PWD"), env=os.environ, stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True ) threading.Thread(target=readlines, args=(proc.stdout,)).start() threading.Thread(target=readlines, args=(proc.stderr,)).start() -def serve(): + +def serve(): + """Serve the web area + """ launch("devel", "nuv ide serve") -def logs(): + +def logs(): + """Serve the openwhisk activation's logs + """ launch("logs", "nuv activation poll") # build + + def build(): + """Try to build the frontend application, if the deploy command is set. + + """ deploy = get_nuvolaris_config("deploy", "true") proc = Popen( - deploy, shell=True, + deploy, shell=True, env=os.environ, cwd=os.environ.get("NUV_PWD"), stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True diff --git a/ide/deploy/config.py b/ide/deploy/config.py new file mode 100644 index 0000000..e060e16 --- /dev/null +++ b/ide/deploy/config.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import json +from pathlib import Path + +MAINS = ["__main__.py", + "index.js", + "index.php", + "main.go"] + +SKIPDIR = ["virtualenv", + "node_modules", + "__pycache__"] + +DEFAULT_REQ_GLOBS = ["packages/*/*/requirements.txt", + "packages/*/*/package.json", + "packages/*/*/composer.json", + "packages/*/*/go.mod"] + +DEFAULT_MAIN_GLOBS = ["packages/*/*/index.js", + "packages/*/*/__main__.py", + "packages/*/*/index.php", + "packages/*/*/main.go"] + +DEFAULT_SINGLES_GLOBS = ["packages/*/*.py", + "packages/*/*.js", + "packages/*/*.php", + "packages/*/*.go"] + + +def get_nuvolaris_config(key: str, default: list | str) -> (list | str): + """Read package.json if exists and retrieve the required + value for passed key (if defined) + + Args: + key (str): the key of the parameter to retrieve + default (list | str): the default value to return + + Returns: + _type_: the list of values or the single string value. + if not defined, the default value is returned + """ + try: + dir = os.environ.get("NUV_PWD", "/do_not_exists") + file = f"{dir}/package.json" + info = json.loads(Path(file).read_text()) + return info.get("nuvolaris", {}).get(key, default) + except: + return default diff --git a/ide/deploy/deploy.py b/ide/deploy/deploy.py index e698a15..9baa9fc 100644 --- a/ide/deploy/deploy.py +++ b/ide/deploy/deploy.py @@ -15,28 +15,49 @@ # specific language governing permissions and limitations # under the License. -MAINS = ["__main__.py", "index.js", "index.php", "main.go"] - import os -from os.path import exists, isdir +from os.path import exists, isdir from subprocess import Popen +from .config import MAINS dry_run = False -def set_dry_run(b): + +def set_dry_run(b: bool): + """Set global dry run + + Args: + b (bool): true for dry run enabled + """ global dry_run dry_run = b -def exec(cmd): + +def exec(cmd: str): + """Exec a shell command and wait for it to complete. + If dryrun is set, the command is not executed. + + Args: + cmd (str): command line to execue + """ global dry_run print("$", cmd) if not dry_run: Popen(cmd, shell=True, env=os.environ).wait() -def extract_args(files): + +def extract_args(files: list) -> list: + """Extract openwhisk args from files + + Args: + files (list): the list of files to inspect + + Returns: + list: a list of parameters + """ res = [] for file in files: - #if dry_run: + # if dry_run: # print(f": inspecting {file}") if exists(file): with open(file, "r") as f: @@ -47,9 +68,16 @@ def extract_args(files): res.append(line.strip()[2:]) return res + package_done = set() -def deploy_package(package): + +def deploy_package(package: str): + """Deploy a package on nuvolaris + + Args: + package (str): the name of package + """ global package_done # package args ppath = f"packages/{package}.args" @@ -59,15 +87,38 @@ def deploy_package(package): exec(cmd) package_done.add(cmd) -def build_zip(package, action): + +def build_zip(package: str, action: str) -> str: + """Builds a zip for the package / action + + Args: + package (str): package + action (str): action + + Returns: + str: the path of the built zip file + """ exec(f"nuv ide util zip A={package}/{action}") return f"packages/{package}/{action}.zip" -def build_action(package, action): + +def build_action(package: str, action: str) -> str: + """Invoke the nuv ide util action command on package / action + + Args: + package (_type_): _description_ + action (_type_): _description_ + + Returns: + str: the zip with the built action + """ exec(f"nuv ide util action A={package}/{action}") return f"packages/{package}/{action}.zip" -def deploy_action(artifact): + +def deploy_action(artifact: str): + """Deploy an artifact calling nuv action update + """ try: sp = artifact.split("/") [name, typ] = sp[-1].rsplit(".", 1) @@ -83,18 +134,26 @@ def deploy_action(artifact): to_inspect = [f"{base}/{x}" for x in MAINS] else: to_inspect = [artifact] - + args = " ".join(extract_args(to_inspect)) exec(f"nuv action update {package}/{name} {artifact} {args}") + """ file = "packages/deploy/hello.py" file = "packages/deploy/multi.zip" file = "packages/deploy/multi/__main__.py" file = "packages/deploy/multi/requirements.txt" """ -def deploy(file): - #print(f"*** {file}") + + +def deploy(file: str): + """Deploy a package on nuvolaris + + Args: + file (str): the file to deploy + """ + # print(f"*** {file}") if isdir(file): for start in MAINS: sub = f"{file}/{start}" diff --git a/ide/deploy/scan.py b/ide/deploy/scan.py index 4b33250..0be87e7 100644 --- a/ide/deploy/scan.py +++ b/ide/deploy/scan.py @@ -18,22 +18,44 @@ from glob import glob from .deploy import * from .client import get_nuvolaris_config +from .config import DEFAULT_REQ_GLOBS, DEFAULT_MAIN_GLOBS, DEFAULT_SINGLES_GLOBS def scan(): + """This function has two stages:\n + Scan + === + A) Packages requirements\n + Check for package requirements. It will look in a specific requirement file + (from web package.json) or a default set of requirements (requirements.txt, + composer.json etc. etc).\n + For each requirement found, a zip is built calling the task: + ``nuv ide util zip A={package}/{action}``\n + The zip is added to actions (deployments); the directory is added to + packages (packages)\n + B) Mains for packages\n + After that, the scan continue checking the "mains". For each main, an + action is built calling the task:\n + ``nuv ide util action A={package}/{action}``\n + where package is the base directory of main file and the action is the + basename of the main file.\n + C) Singles\n + Finally the scan will take care of single file functions\n + + Deploy + === + Each package is deployed with the command:\n + ``nuv package update {package} {pargs}``\n + Each deployment is deployed with the command:\n + ``nuv action update {package}/{name} {artifact} {args}``\n + """ + # first look for requirements.txt and build the venv (add in set) deployments = set() packages = set() print("> Scan:") - - # => REQUIREMENTS - default_reqs_globs = ["packages/*/*/requirements.txt", - "packages/*/*/package.json", - "packages/*/*/composer.json", - "packages/*/*/go.mod"] - - package_globs = get_nuvolaris_config("requirements", default_reqs_globs) + package_globs = get_nuvolaris_config("requirements", DEFAULT_REQ_GLOBS) reqs = list() for pkg_glob in package_globs: @@ -51,11 +73,7 @@ def scan(): packages.add(sp[1]) # => MAINS - default_mains_globs = ["packages/*/*/index.js", - "packages/*/*/__main__.py", - "packages/*/*/index.php", - "packages/*/*/main.go"] - mains_globs = get_nuvolaris_config("mains", default_mains_globs) + mains_globs = get_nuvolaris_config("mains", DEFAULT_MAIN_GLOBS) mains = list() for main_glob in mains_globs: items = glob(main_glob) @@ -71,11 +89,8 @@ def scan(): packages.add(sp[1]) # => SINGLES - default_singles_globs = ["packages/*/*.py", - "packages/*/*.js", - "packages/*/*.php", - "packages/*/*.go"] - singles_globs = get_nuvolaris_config("singles", default_singles_globs) + + singles_globs = get_nuvolaris_config("singles", DEFAULT_SINGLES_GLOBS) singles = list() for single_glob in singles_globs: items = glob(single_glob) diff --git a/ide/deploy/watch.py b/ide/deploy/watch.py index 99d4234..7559958 100644 --- a/ide/deploy/watch.py +++ b/ide/deploy/watch.py @@ -15,48 +15,66 @@ # specific language governing permissions and limitations # under the License. -SKIPDIR = ["virtualenv", "node_modules", "__pycache__"] - +from typing import Tuple import watchfiles + import asyncio import os.path import signal - +from .config import SKIPDIR from .deploy import deploy from .client import serve, logs -def check_and_deploy(change): + +def check_and_deploy(change: Tuple[watchfiles.Change, str]): + """Deploy a file if the received change is a file modification + + Args: + change (Tuple[watchfiles.Change, str]): the change type and filename + """ change_type, path = change path = os.path.abspath(path) cur_dir_len = len(os.getcwd())+1 src = path[cur_dir_len:] # only modified - if change_type != watchfiles.Change.modified: return + if change_type != watchfiles.Change.modified: + return # no directories - if os.path.isdir(src): return + if os.path.isdir(src): + return # no missing files - if not os.path.exists(src): return + if not os.path.exists(src): + return # no generated directories for dir in src.split("/")[:-1]: - if dir in SKIPDIR: return + if dir in SKIPDIR: + return # no generated files - if src.endswith(".zip"): return + if src.endswith(".zip"): + return # now you can deploy deploy(src) + async def redeploy(): + """Install the filesystem watcher over the packages directory + """ print("> Watching:") iterator = watchfiles.awatch("packages", recursive=True) async for changes in iterator: for change in changes: try: - #print(change) + # print(change) check_and_deploy(change) except Exception as e: print(e) + def watch(): + """This function will start the webserver and show logs. + After that it will watch the filesystem in a loop, until SIGTERM received. + """ # start web server serve() # show logs @@ -64,6 +82,7 @@ def watch(): loop = asyncio.get_event_loop() task = loop.create_task(redeploy()) + def end_loop(): print("Ending task.") task.cancel()
