areusch commented on a change in pull request #9534: URL: https://github.com/apache/tvm/pull/9534#discussion_r762048729
########## File path: tests/scripts/ci.py ########## @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import multiprocessing +import os +import click Review comment: possible to avoid using non-stdlib stuff here? for Python deps, we need to first finish the work to populate pyproject.toml and then it will be easier to add dependencies here. since this is just for CLI parsing, and we use argparse most other places, is it a big ask to change to argparse here? ########## File path: tests/scripts/ci.py ########## @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import multiprocessing +import os +import click +import getpass +import contextlib +import subprocess +from pathlib import Path +from typing import List, Dict, Any + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +NPROC = multiprocessing.cpu_count() + + +class col: + BLUE = "\033[94m" + CYAN = "\033[96m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + RESET = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def print_color(color: str, msg: str, **kwargs: Any) -> None: + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + print(col.BOLD + color + msg + col.RESET, **kwargs) + else: + print(msg, **kwargs) + + +def clean_exit(msg: str) -> None: + print_color(col.RED, msg, file=sys.stderr) + exit(1) + + +def cmd(commands: List[Any], **kwargs: Any): + commands = [str(s) for s in commands] + command_str = " ".join(commands) + print_color(col.BLUE, command_str) + proc = subprocess.run(commands, **kwargs) + if proc.returncode != 0: + raise RuntimeError(f"Command failed: '{command_str}'") + + +def cleanup(): + # chown the directories back to the current user in case Docker generated + # files with a different user ID + cmd(["sudo", "chown", getpass.getuser(), REPO_ROOT, "-R"]) + + +def docker(name: str, image: str, scripts: List[str], env: Dict[str, str]): + """ + Invoke a set of bash scripts through docker/bash.sh + """ + docker_bash = REPO_ROOT / "docker" / "bash.sh" + command = ["sudo", docker_bash, "--name", name] + for key, value in env.items(): + command.append("--env") + command.append(f"{key}={value}") + command += [image, "bash", "-c", " && ".join(scripts)] + + try: + cmd(command) + except RuntimeError as e: + cleanup() + clean_exit(f"Error invoking Docker: {e}") + except KeyboardInterrupt: + cmd(["sudo", "docker", "stop", name]) + + cleanup() + + +@click.group() +def cli() -> None: + """ + Run CI scripts locally via Docker + """ + pass + + +@cli.command() +@click.option( + "--full", default=False, is_flag=True, help="Build all language docs, not just Python" +) +@click.option("--precheck", default=False, is_flag=True, help="Run Sphinx precheck script") +def docs(full: bool, precheck: bool) -> None: + """ + Build the documentation from gallery/ and docs/ + """ + scripts = [] + if precheck: + scripts += [ + "./tests/scripts/task_sphinx_precheck.sh", + ] + scripts += [ + "./tests/scripts/task_config_build_gpu.sh", + f"./tests/scripts/task_build.sh build -j{NPROC}", + "./tests/scripts/task_ci_setup.sh", + "./tests/scripts/task_python_docs.sh", + ] + env = { + "TVM_TUTORIAL_EXEC_PATTERN": os.getenv( + "TVM_TUTORIAL_EXEC_PATTERN", ".py" if full else "None" + ), + "PYTHON_DOCS_ONLY": "0" if full else "1", + "IS_LOCAL": "1", + } + docker(name="ci-docs", image="tlcpack/ci-gpu:v0.78", scripts=scripts, env=env) + + +@cli.command() +@click.option("--directory", default="_docs") +def serve_docs(directory) -> None: + """ + Serve the docs using Python's http server + """ + directory = Path(directory) + if not directory.exists(): + clean_exit("Docs have not been build, run 'ci.py docs' first") + cmd([sys.executable, "-m", "http.server"], cwd=directory) + + +@cli.command() +def lint() -> None: + """ + Serve the docs using Python's http server + """ + docker( + name="ci-lint", + image="tlcpack/ci-lint:v0.67", Review comment: for `image` in this file, simply state `ci_lint`, and `docker/bash.sh` will cross-reference the Jenkinsfile. ########## File path: tests/scripts/ci.py ########## @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import multiprocessing +import os +import click +import getpass +import contextlib +import subprocess +from pathlib import Path +from typing import List, Dict, Any + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +NPROC = multiprocessing.cpu_count() + + +class col: + BLUE = "\033[94m" + CYAN = "\033[96m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + RESET = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def print_color(color: str, msg: str, **kwargs: Any) -> None: + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + print(col.BOLD + color + msg + col.RESET, **kwargs) + else: + print(msg, **kwargs) + + +def clean_exit(msg: str) -> None: + print_color(col.RED, msg, file=sys.stderr) + exit(1) + + +def cmd(commands: List[Any], **kwargs: Any): + commands = [str(s) for s in commands] + command_str = " ".join(commands) + print_color(col.BLUE, command_str) + proc = subprocess.run(commands, **kwargs) + if proc.returncode != 0: + raise RuntimeError(f"Command failed: '{command_str}'") + + +def cleanup(): + # chown the directories back to the current user in case Docker generated + # files with a different user ID + cmd(["sudo", "chown", getpass.getuser(), REPO_ROOT, "-R"]) + + +def docker(name: str, image: str, scripts: List[str], env: Dict[str, str]): + """ + Invoke a set of bash scripts through docker/bash.sh + """ + docker_bash = REPO_ROOT / "docker" / "bash.sh" + command = ["sudo", docker_bash, "--name", name] + for key, value in env.items(): + command.append("--env") + command.append(f"{key}={value}") + command += [image, "bash", "-c", " && ".join(scripts)] + + try: + cmd(command) + except RuntimeError as e: + cleanup() + clean_exit(f"Error invoking Docker: {e}") + except KeyboardInterrupt: + cmd(["sudo", "docker", "stop", name]) + + cleanup() + + +@click.group() +def cli() -> None: + """ + Run CI scripts locally via Docker + """ + pass + + +@cli.command() +@click.option( + "--full", default=False, is_flag=True, help="Build all language docs, not just Python" +) +@click.option("--precheck", default=False, is_flag=True, help="Run Sphinx precheck script") +def docs(full: bool, precheck: bool) -> None: + """ + Build the documentation from gallery/ and docs/ + """ + scripts = [] + if precheck: + scripts += [ + "./tests/scripts/task_sphinx_precheck.sh", + ] + scripts += [ + "./tests/scripts/task_config_build_gpu.sh", + f"./tests/scripts/task_build.sh build -j{NPROC}", + "./tests/scripts/task_ci_setup.sh", + "./tests/scripts/task_python_docs.sh", + ] + env = { + "TVM_TUTORIAL_EXEC_PATTERN": os.getenv( + "TVM_TUTORIAL_EXEC_PATTERN", ".py" if full else "None" Review comment: i think this might be good to consume via options, it's a fairly common workflow. ########## File path: Makefile ########## @@ -19,7 +19,7 @@ .PHONY: all \ runtime vta cpptest crttest \ lint pylint cpplint scalalint \ - doc \ + doc docs \ Review comment: could we change `doc` to `make cppdoc`? i think in the future we can probably move all of the non-user-facing targets elsewhere, but let's start with something less controversial for this PR. ########## File path: docs/README.md ########## @@ -0,0 +1,101 @@ +<!--- Licensed to the Apache Software Foundation (ASF) under one --> +<!--- or more contributor license agreements. See the NOTICE file --> +<!--- distributed with this work for additional information --> +<!--- regarding copyright ownership. The ASF licenses this file --> +<!--- to you under the Apache License, Version 2.0 (the --> +<!--- "License"); you may not use this file except in compliance --> +<!--- with the License. You may obtain a copy of the License at --> + +<!--- http://www.apache.org/licenses/LICENSE-2.0 --> + +<!--- Unless required by applicable law or agreed to in writing, --> +<!--- software distributed under the License is distributed on an --> +<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --> +<!--- KIND, either express or implied. See the License for the --> +<!--- specific language governing permissions and limitations --> +<!--- under the License. --> + +# TVM Documentation +This folder contains the source of TVM's documentation, hosted at https://tvm.apache.org/docs + +## Build Locally + +See also the instructions below to run a specific tutorial. Note that some of the tutorials need GPU support. Once build, either of these can be served using Python's build in HTTP server: + +```bash +# Run this and then visit http://localhost:8000 in your browser +cd docs/_build/html && python3 -m http.server +``` + +### With Docker (recommended) + +1. Build TVM and the docs inside the [tlcpack/ci-gpu image](https://hub.docker.com/r/tlcpack/ci-gpu) + + ```bash + # If this runs into errors, try cleaning your 'build' directory + make docs + ``` + + +### Native + +1. [Build TVM](https://tvm.apache.org/docs/install/from_source.html) first in the repo root folder +2. Install dependencies + + ```bash + # Pillow on Ubuntu may require libjpeg-dev from apt + sudo docker run tlcpack/ci-gpu:v0.78 bash -c \ + 'python3 -m pip install --quiet tlcpack-sphinx-addon==0.2.1 synr==0.5.0 && python3 -m pip freeze' > frozen-requirements.txt + + pip install -r frozen-requirements.txt + ``` + +3. Generate the docs + + ```bash + # TVM_TUTORIAL_EXEC_PATTERN=none skips the tutorial execution to the build + # work on most environments (e.g. MacOS). + export TVM_TUTORIAL_EXEC_PATTERN=none + + make html + ``` + + +## Only Execute Specified Tutorials +The document build process will execute all the tutorials in the sphinx gallery. +This will cause failure in some cases when certain machines do not have necessary +environment. You can set `TVM_TUTORIAL_EXEC_PATTERN` to only execute +the path that matches the regular expression pattern. + +For example, to only build tutorials under `/vta/tutorials`, run + +```bash +TVM_TUTORIAL_EXEC_PATTERN=/vta/tutorials make html +``` + +To only build one specific file, do + +```bash +# The slash \ is used to get . in regular expression +TVM_TUTORIAL_EXEC_PATTERN=file_name\.py make html +``` + +## Helper Scripts Review comment: should the ci.py be mentioned here too? ########## File path: tests/scripts/ci.py ########## @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import multiprocessing +import os +import click +import getpass +import contextlib +import subprocess +from pathlib import Path +from typing import List, Dict, Any + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +NPROC = multiprocessing.cpu_count() + + +class col: + BLUE = "\033[94m" + CYAN = "\033[96m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + RESET = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def print_color(color: str, msg: str, **kwargs: Any) -> None: + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + print(col.BOLD + color + msg + col.RESET, **kwargs) + else: + print(msg, **kwargs) + + +def clean_exit(msg: str) -> None: + print_color(col.RED, msg, file=sys.stderr) + exit(1) + + +def cmd(commands: List[Any], **kwargs: Any): + commands = [str(s) for s in commands] + command_str = " ".join(commands) + print_color(col.BLUE, command_str) + proc = subprocess.run(commands, **kwargs) + if proc.returncode != 0: + raise RuntimeError(f"Command failed: '{command_str}'") + + +def cleanup(): + # chown the directories back to the current user in case Docker generated + # files with a different user ID Review comment: this shouldn't happen--docker/with_the_same_user intentionally creates a user with UID to match yours. are you seeing this? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@tvm.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
