leandron commented on a change in pull request #6703: URL: https://github.com/apache/incubator-tvm/pull/6703#discussion_r512553762
########## File path: apps/microtvm/reference-vm/zephyr/pyproject.toml ########## @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[tool.black] +line-length = 100 +target-version = ['py36'] +include = '(\.pyi?$)' +exclude = ''' + +( + /( + \.github + | \.tvm + | \.tvm_test_data + | \.vscode + | \.venv + | 3rdparty + | build\/ + | cmake\/ + | conda\/ + | docker\/ + | docs\/ + | golang\/ + | include\/ + | jvm\/ + | licenses\/ + | nnvm\/ + | rust\/ + | src\/ + | vta\/ + | web\/ + )/ +) +''' +[tool.poetry] +name = "incubator-tvm" +version = "0.1.0" +description = "" +authors = ["Your Name <[email protected]>"] +packages = [ + { include = "tvm", from = "python" }, +] + +[tool.poetry.dependencies] +attrs = "^19" +decorator = "^4.4" +numpy = "~1.19" +psutil = "^5" +scipy = "^1.4" +python = "^3.6" +tornado = "^6" +typed_ast = "^1.4" Review comment: Many dependencies listed here overlap with the ones described in TVM's `setup.py`, can we try to install TVM in a way that we get the dependencies from there, rather than duplicating them here? I think it would reduce a lot the maintenance needed to keep this working over time. ########## File path: apps/microtvm/reference-vm/base-box-tool.py ########## @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import abc +import argparse +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys + + +_LOG = logging.getLogger(__name__) + + +THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".") + + +# List of vagrant providers supported by this tool +ALL_PROVIDERS = ( + "parallels", + "virtualbox", +) + + +def parse_virtualbox_devices(): + output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") + devices = [] + current_dev = {} + for line in output.split("\n"): + if not line.strip(): + if current_dev: + if "VendorId" in current_dev and "ProductId" in current_dev: + devices.append(current_dev) + current_dev = {} + + continue + + key, value = line.split(":", 1) + value = value.lstrip(" ") + current_dev[key] = value + + if current_dev: + devices.append(current_dev) + return devices + + +VIRTUALBOX_VID_PID_RE = re.compile(r"0x([0-9A-Fa-f]{4}).*") + + +def attach_virtualbox(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = parse_virtualbox_devices() + for dev in usb_devices: + m = VIRTUALBOX_VID_PID_RE.match(dev["VendorId"]) + if not m: + _LOG.warning("Malformed VendorId: %s", dev["VendorId"]) + continue + + dev_vid_hex = m.group(1).lower() + + m = VIRTUALBOX_VID_PID_RE.match(dev["ProductId"]) + if not m: + _LOG.warning("Malformed ProductId: %s", dev["ProductId"]) + continue + + dev_pid_hex = m.group(1).lower() + + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev["SerialNumber"]) + ): + rule_args = [ + "VBoxManage", + "usbfilter", + "add", + "0", + "--action", + "hold", + "--name", + "test device", + "--target", + uuid, + "--vendorid", + vid_hex, + "--productid", + pid_hex, + ] + if serial is not None: + rule_args.extend(["--serialnumber", serial]) + subprocess.check_call(rule_args) + subprocess.check_call(["VBoxManage", "controlvm", uuid, "usbattach", dev["UUID"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +def attach_parallels(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = json.loads( + subprocess.check_output(["prlsrvctl", "usb", "list", "-j"], encoding="utf-8") + ) + for dev in usb_devices: + _, dev_vid_hex, dev_pid_hex, _, _, dev_serial = dev["System name"].split("|") + dev_vid_hex = dev_vid_hex.lower() + dev_pid_hex = dev_pid_hex.lower() + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev_serial) + ): + subprocess.check_call(["prlsrvctl", "usb", "set", dev["Name"], uuid]) + subprocess.check_call(["prlctl", "set", uuid, "--device-connect", dev["Name"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +ATTACH_USB_DEVICE = { + "parallels": attach_parallels, + "virtualbox": attach_virtualbox, +} + + +def generate_packer_config(file_path, providers): + builders = [] + for provider_name in providers: + builders.append( + { + "type": "vagrant", + "output_dir": f"output-packer-{provider_name}", + "communicator": "ssh", + "source_path": "generic/ubuntu1804", + "provider": provider_name, + "template": "Vagrantfile.packer-template", + } + ) + + with open(file_path, "w") as f: + json.dump( + { + "builders": builders, + }, + f, + sort_keys=True, + indent=2, + ) + + +def build_command(args): + generate_packer_config( + os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"), + args.provider.split(",") or ALL_PROVIDERS, + ) + subprocess.check_call( + ["packer", "build", "packer.json"], cwd=os.path.join(THIS_DIR, args.platform, "base-box") + ) + + +REQUIRED_TEST_CONFIG_KEYS = { + "vid_hex": str, + "pid_hex": str, + "test_cmd": list, +} + + +VM_BOX_RE = re.compile(r'(.*\.vm\.box) = "(.*)"') + + +# Paths, relative to the platform box directory, which will not be copied to release-test dir. +SKIP_COPY_PATHS = [".vagrant", "base-box"] + + +def test_command(args): + user_box_dir = os.path.join(THIS_DIR, args.platform) + base_box_dir = os.path.join(THIS_DIR, args.platform, "base-box") + test_config_file = os.path.join(base_box_dir, "test-config.json") + with open(test_config_file) as f: + test_config = json.load(f) + for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): + assert key in test_config and isinstance( + test_config[key], expected_type + ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" + + test_config["vid_hex"] = test_config["vid_hex"].lower() + test_config["pid_hex"] = test_config["pid_hex"].lower() + + providers = args.provider.split(",") + provider_passed = {p: False for p in providers} + + release_test_dir = os.path.join(THIS_DIR, "release-test") + + for provider_name in providers: + if os.path.exists(release_test_dir): + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + for dirpath, _, filenames in os.walk(user_box_dir): + rel_path = os.path.relpath(dirpath, user_box_dir) + if any( + rel_path == scp or rel_path.startswith(f"{scp}{os.path.sep}") + for scp in SKIP_COPY_PATHS + ): + continue + + dest_dir = os.path.join(release_test_dir, rel_path) + os.makedirs(dest_dir) + for filename in filenames: + shutil.copy2(os.path.join(dirpath, filename), os.path.join(dest_dir, filename)) + + release_test_vagrantfile = os.path.join(release_test_dir, "Vagrantfile") + with open(release_test_vagrantfile) as f: + lines = list(f) + + found_box_line = False + with open(release_test_vagrantfile, "w") as f: + for line in lines: + m = VM_BOX_RE.match(line) + if not m: + f.write(line) + continue + + box_package = os.path.join( + base_box_dir, f"output-packer-{provider_name}", "package.box" + ) + f.write(f'{m.group(1)} = "{os.path.relpath(box_package, release_test_dir)}"\n') + found_box_line = True + + if not found_box_line: + _LOG.error( + "testing provider %s: couldn't find config.box.vm = line in Vagrantfile; unable to test", + provider_name, + ) + continue + + subprocess.check_call( + ["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir + ) + try: + with open( + os.path.join( + release_test_dir, ".vagrant", "machines", "default", provider_name, "id" + ) + ) as f: + machine_uuid = f.read() + ATTACH_USB_DEVICE[provider_name]( + machine_uuid, + vid_hex=test_config["vid_hex"], + pid_hex=test_config["pid_hex"], + serial=args.test_device_serial, + ) + tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", "..")) + + def _quote_cmd(cmd): + return " ".join(shlex.quote(a) for a in cmd) + + test_cmd = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd"]) + subprocess.check_call( + ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir + ) + provider_passed[provider_name] = True + + finally: + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + if not all(provider_passed[p] for p in provider_passed.keys()): + sys.exit( + "some providers failed release test: " + + ",".join(name for name, passed in provider_passed if not passed) + ) + + +def release_command(args): + # subprocess.check_call(["vagrant", "cloud", "version", "create", f"tlcpack/microtvm-{args.platform}", args.version]) + if not args.version: + sys.exit(f"--version must be specified") + + for provider_name in args.provider.split(","): + subprocess.check_call( + [ + "vagrant", + "cloud", + "publish", + "-f", + f"tlcpack/microtvm-{args.platform}", + args.version, + provider_name, + os.path.join( + THIS_DIR, + args.platform, + "base-box", + f"output-packer-{provider_name}/package.box", + ), + ] + ) + + +ALL_COMMANDS = { + "build": build_command, + "test": test_command, + "release": release_command, +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Automates building, testing, and releasing a base box" + ) + parser.add_argument( + "command", default=",".join(ALL_COMMANDS), choices=ALL_COMMANDS, help="Action to perform." + ) + parser.add_argument( + "platform", + help="Name of the platform VM to act on. Must be a sub-directory of this directory.", + ) + parser.add_argument( + "--provider", + help="Name of the provider or providers to act on; if not specified, act on all", Review comment: ```suggestion parser.add_argument( "--provider", choices=ALL_PROVIDERS, help="Name of the provider or providers (comma-separated) to act on.If not specified, act on all", ``` ########## File path: python/tvm/exec/microtvm_debug_shell.py ########## @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=redefined-outer-name, invalid-name +"""Start an RPC server intended for use as a microTVM debugger. + +microTVM aims to be runtime-agnostic, and to that end, frameworks often define command-line tools +used to launch a debug flow. These tools often manage the process of connecting to an attached +device using a hardware debugger, exposing a GDB server, and launching GDB connected to that +server with a source file attached. It's also true that this debugger can typically not be executed +concurrently with any flash tool, so this integration point is provided to allow TVM to launch and +terminate any debuggers integrated with the larger microTVM compilation/autotuning flow. + +To use this tool, first launch this script in a separate terminal window. Then, provide the hostport +to your compiler's Flasher instance. +""" + +import argparse +import logging +import socket +import struct + +import tvm.micro.debugger as _ # NOTE: imported to expose global PackedFuncs over RPC. + +from .._ffi.base import py_str +from ..rpc import base +from ..rpc import _ffi_api + + +_LOG = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments to this script.""" + parser = argparse.ArgumentParser(description="microTVM debug-tool runner") + parser.add_argument("--host", default="0.0.0.0", help="hostname to listen on") + parser.add_argument("--port", type=int, default=9090, help="hostname to listen on") + parser.add_argument( + "--impl", + help=( + "If given, name of a module underneath tvm.micro.contrib " + "which contains the Debugger implementation to use." Review comment: It looks like the module described here can go with full absolute namespace as well as relative (based on my interpretation of code on line ~125 here). Maybe an example here would be beneficial for users to understand this option. ########## File path: apps/microtvm/reference-vm/base-box-tool.py ########## @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import abc +import argparse +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys + + +_LOG = logging.getLogger(__name__) + + +THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".") + + +# List of vagrant providers supported by this tool +ALL_PROVIDERS = ( + "parallels", + "virtualbox", +) + + +def parse_virtualbox_devices(): + output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") + devices = [] + current_dev = {} + for line in output.split("\n"): + if not line.strip(): + if current_dev: + if "VendorId" in current_dev and "ProductId" in current_dev: + devices.append(current_dev) + current_dev = {} + + continue + + key, value = line.split(":", 1) + value = value.lstrip(" ") + current_dev[key] = value + + if current_dev: + devices.append(current_dev) + return devices + + +VIRTUALBOX_VID_PID_RE = re.compile(r"0x([0-9A-Fa-f]{4}).*") + + +def attach_virtualbox(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = parse_virtualbox_devices() + for dev in usb_devices: + m = VIRTUALBOX_VID_PID_RE.match(dev["VendorId"]) + if not m: + _LOG.warning("Malformed VendorId: %s", dev["VendorId"]) + continue + + dev_vid_hex = m.group(1).lower() + + m = VIRTUALBOX_VID_PID_RE.match(dev["ProductId"]) + if not m: + _LOG.warning("Malformed ProductId: %s", dev["ProductId"]) + continue + + dev_pid_hex = m.group(1).lower() + + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev["SerialNumber"]) + ): + rule_args = [ + "VBoxManage", + "usbfilter", + "add", + "0", + "--action", + "hold", + "--name", + "test device", + "--target", + uuid, + "--vendorid", + vid_hex, + "--productid", + pid_hex, + ] + if serial is not None: + rule_args.extend(["--serialnumber", serial]) + subprocess.check_call(rule_args) + subprocess.check_call(["VBoxManage", "controlvm", uuid, "usbattach", dev["UUID"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +def attach_parallels(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = json.loads( + subprocess.check_output(["prlsrvctl", "usb", "list", "-j"], encoding="utf-8") + ) + for dev in usb_devices: + _, dev_vid_hex, dev_pid_hex, _, _, dev_serial = dev["System name"].split("|") + dev_vid_hex = dev_vid_hex.lower() + dev_pid_hex = dev_pid_hex.lower() + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev_serial) + ): + subprocess.check_call(["prlsrvctl", "usb", "set", dev["Name"], uuid]) + subprocess.check_call(["prlctl", "set", uuid, "--device-connect", dev["Name"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +ATTACH_USB_DEVICE = { + "parallels": attach_parallels, + "virtualbox": attach_virtualbox, +} + + +def generate_packer_config(file_path, providers): + builders = [] + for provider_name in providers: + builders.append( + { + "type": "vagrant", + "output_dir": f"output-packer-{provider_name}", + "communicator": "ssh", + "source_path": "generic/ubuntu1804", + "provider": provider_name, + "template": "Vagrantfile.packer-template", + } + ) + + with open(file_path, "w") as f: + json.dump( + { + "builders": builders, + }, + f, + sort_keys=True, + indent=2, + ) + + +def build_command(args): + generate_packer_config( + os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"), + args.provider.split(",") or ALL_PROVIDERS, + ) + subprocess.check_call( + ["packer", "build", "packer.json"], cwd=os.path.join(THIS_DIR, args.platform, "base-box") + ) + + +REQUIRED_TEST_CONFIG_KEYS = { + "vid_hex": str, + "pid_hex": str, + "test_cmd": list, +} + + +VM_BOX_RE = re.compile(r'(.*\.vm\.box) = "(.*)"') + + +# Paths, relative to the platform box directory, which will not be copied to release-test dir. +SKIP_COPY_PATHS = [".vagrant", "base-box"] + + +def test_command(args): + user_box_dir = os.path.join(THIS_DIR, args.platform) + base_box_dir = os.path.join(THIS_DIR, args.platform, "base-box") + test_config_file = os.path.join(base_box_dir, "test-config.json") + with open(test_config_file) as f: + test_config = json.load(f) + for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): + assert key in test_config and isinstance( + test_config[key], expected_type + ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" + + test_config["vid_hex"] = test_config["vid_hex"].lower() + test_config["pid_hex"] = test_config["pid_hex"].lower() + + providers = args.provider.split(",") + provider_passed = {p: False for p in providers} + + release_test_dir = os.path.join(THIS_DIR, "release-test") + + for provider_name in providers: + if os.path.exists(release_test_dir): + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + for dirpath, _, filenames in os.walk(user_box_dir): + rel_path = os.path.relpath(dirpath, user_box_dir) + if any( + rel_path == scp or rel_path.startswith(f"{scp}{os.path.sep}") + for scp in SKIP_COPY_PATHS + ): + continue + + dest_dir = os.path.join(release_test_dir, rel_path) + os.makedirs(dest_dir) + for filename in filenames: + shutil.copy2(os.path.join(dirpath, filename), os.path.join(dest_dir, filename)) + + release_test_vagrantfile = os.path.join(release_test_dir, "Vagrantfile") + with open(release_test_vagrantfile) as f: + lines = list(f) + + found_box_line = False + with open(release_test_vagrantfile, "w") as f: + for line in lines: + m = VM_BOX_RE.match(line) + if not m: + f.write(line) + continue + + box_package = os.path.join( + base_box_dir, f"output-packer-{provider_name}", "package.box" + ) + f.write(f'{m.group(1)} = "{os.path.relpath(box_package, release_test_dir)}"\n') + found_box_line = True + + if not found_box_line: + _LOG.error( + "testing provider %s: couldn't find config.box.vm = line in Vagrantfile; unable to test", + provider_name, + ) + continue + + subprocess.check_call( + ["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir + ) + try: + with open( + os.path.join( + release_test_dir, ".vagrant", "machines", "default", provider_name, "id" + ) + ) as f: + machine_uuid = f.read() + ATTACH_USB_DEVICE[provider_name]( + machine_uuid, + vid_hex=test_config["vid_hex"], + pid_hex=test_config["pid_hex"], + serial=args.test_device_serial, + ) + tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", "..")) + + def _quote_cmd(cmd): + return " ".join(shlex.quote(a) for a in cmd) + + test_cmd = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd"]) + subprocess.check_call( + ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir + ) + provider_passed[provider_name] = True + + finally: + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + if not all(provider_passed[p] for p in provider_passed.keys()): + sys.exit( + "some providers failed release test: " + + ",".join(name for name, passed in provider_passed if not passed) + ) + + +def release_command(args): + # subprocess.check_call(["vagrant", "cloud", "version", "create", f"tlcpack/microtvm-{args.platform}", args.version]) + if not args.version: + sys.exit(f"--version must be specified") + + for provider_name in args.provider.split(","): + subprocess.check_call( + [ + "vagrant", + "cloud", + "publish", + "-f", + f"tlcpack/microtvm-{args.platform}", + args.version, + provider_name, + os.path.join( + THIS_DIR, + args.platform, + "base-box", + f"output-packer-{provider_name}/package.box", + ), + ] + ) + + +ALL_COMMANDS = { + "build": build_command, + "test": test_command, + "release": release_command, +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Automates building, testing, and releasing a base box" + ) + parser.add_argument( + "command", default=",".join(ALL_COMMANDS), choices=ALL_COMMANDS, help="Action to perform." Review comment: ```suggestion "command", default=",".join(ALL_COMMANDS), choices=ALL_COMMANDS, help="Action or actions (comma-separated) to perform." ``` ########## File path: apps/microtvm/reference-vm/base-box-tool.py ########## @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import abc Review comment: I think `abc` is not used here, and can be removed? ########## File path: apps/microtvm/reference-vm/base-box-tool.py ########## @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import abc +import argparse +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys + + +_LOG = logging.getLogger(__name__) + + +THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".") + + +# List of vagrant providers supported by this tool +ALL_PROVIDERS = ( + "parallels", + "virtualbox", +) + + +def parse_virtualbox_devices(): + output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") + devices = [] + current_dev = {} + for line in output.split("\n"): + if not line.strip(): + if current_dev: + if "VendorId" in current_dev and "ProductId" in current_dev: + devices.append(current_dev) + current_dev = {} + + continue + + key, value = line.split(":", 1) + value = value.lstrip(" ") + current_dev[key] = value + + if current_dev: + devices.append(current_dev) + return devices + + +VIRTUALBOX_VID_PID_RE = re.compile(r"0x([0-9A-Fa-f]{4}).*") + + +def attach_virtualbox(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = parse_virtualbox_devices() + for dev in usb_devices: + m = VIRTUALBOX_VID_PID_RE.match(dev["VendorId"]) + if not m: + _LOG.warning("Malformed VendorId: %s", dev["VendorId"]) + continue + + dev_vid_hex = m.group(1).lower() + + m = VIRTUALBOX_VID_PID_RE.match(dev["ProductId"]) + if not m: + _LOG.warning("Malformed ProductId: %s", dev["ProductId"]) + continue + + dev_pid_hex = m.group(1).lower() + + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev["SerialNumber"]) + ): + rule_args = [ + "VBoxManage", + "usbfilter", + "add", + "0", + "--action", + "hold", + "--name", + "test device", + "--target", + uuid, + "--vendorid", + vid_hex, + "--productid", + pid_hex, + ] + if serial is not None: + rule_args.extend(["--serialnumber", serial]) + subprocess.check_call(rule_args) + subprocess.check_call(["VBoxManage", "controlvm", uuid, "usbattach", dev["UUID"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +def attach_parallels(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = json.loads( + subprocess.check_output(["prlsrvctl", "usb", "list", "-j"], encoding="utf-8") + ) + for dev in usb_devices: + _, dev_vid_hex, dev_pid_hex, _, _, dev_serial = dev["System name"].split("|") + dev_vid_hex = dev_vid_hex.lower() + dev_pid_hex = dev_pid_hex.lower() + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev_serial) + ): + subprocess.check_call(["prlsrvctl", "usb", "set", dev["Name"], uuid]) + subprocess.check_call(["prlctl", "set", uuid, "--device-connect", dev["Name"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +ATTACH_USB_DEVICE = { + "parallels": attach_parallels, + "virtualbox": attach_virtualbox, +} + + +def generate_packer_config(file_path, providers): + builders = [] + for provider_name in providers: + builders.append( + { + "type": "vagrant", + "output_dir": f"output-packer-{provider_name}", + "communicator": "ssh", + "source_path": "generic/ubuntu1804", + "provider": provider_name, + "template": "Vagrantfile.packer-template", + } + ) + + with open(file_path, "w") as f: + json.dump( + { + "builders": builders, + }, + f, + sort_keys=True, + indent=2, + ) + + +def build_command(args): + generate_packer_config( + os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"), + args.provider.split(",") or ALL_PROVIDERS, + ) + subprocess.check_call( + ["packer", "build", "packer.json"], cwd=os.path.join(THIS_DIR, args.platform, "base-box") + ) + + +REQUIRED_TEST_CONFIG_KEYS = { + "vid_hex": str, + "pid_hex": str, + "test_cmd": list, +} + + +VM_BOX_RE = re.compile(r'(.*\.vm\.box) = "(.*)"') + + +# Paths, relative to the platform box directory, which will not be copied to release-test dir. +SKIP_COPY_PATHS = [".vagrant", "base-box"] + + +def test_command(args): + user_box_dir = os.path.join(THIS_DIR, args.platform) + base_box_dir = os.path.join(THIS_DIR, args.platform, "base-box") + test_config_file = os.path.join(base_box_dir, "test-config.json") + with open(test_config_file) as f: + test_config = json.load(f) + for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): + assert key in test_config and isinstance( + test_config[key], expected_type + ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" + + test_config["vid_hex"] = test_config["vid_hex"].lower() + test_config["pid_hex"] = test_config["pid_hex"].lower() + + providers = args.provider.split(",") + provider_passed = {p: False for p in providers} + + release_test_dir = os.path.join(THIS_DIR, "release-test") + + for provider_name in providers: + if os.path.exists(release_test_dir): + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + for dirpath, _, filenames in os.walk(user_box_dir): + rel_path = os.path.relpath(dirpath, user_box_dir) + if any( + rel_path == scp or rel_path.startswith(f"{scp}{os.path.sep}") + for scp in SKIP_COPY_PATHS + ): + continue + + dest_dir = os.path.join(release_test_dir, rel_path) + os.makedirs(dest_dir) + for filename in filenames: + shutil.copy2(os.path.join(dirpath, filename), os.path.join(dest_dir, filename)) + + release_test_vagrantfile = os.path.join(release_test_dir, "Vagrantfile") + with open(release_test_vagrantfile) as f: + lines = list(f) + + found_box_line = False + with open(release_test_vagrantfile, "w") as f: + for line in lines: + m = VM_BOX_RE.match(line) + if not m: + f.write(line) + continue + + box_package = os.path.join( + base_box_dir, f"output-packer-{provider_name}", "package.box" + ) + f.write(f'{m.group(1)} = "{os.path.relpath(box_package, release_test_dir)}"\n') + found_box_line = True + + if not found_box_line: + _LOG.error( + "testing provider %s: couldn't find config.box.vm = line in Vagrantfile; unable to test", + provider_name, + ) + continue + + subprocess.check_call( + ["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir + ) + try: + with open( + os.path.join( + release_test_dir, ".vagrant", "machines", "default", provider_name, "id" + ) + ) as f: + machine_uuid = f.read() + ATTACH_USB_DEVICE[provider_name]( + machine_uuid, + vid_hex=test_config["vid_hex"], + pid_hex=test_config["pid_hex"], + serial=args.test_device_serial, + ) + tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", "..")) + + def _quote_cmd(cmd): + return " ".join(shlex.quote(a) for a in cmd) + + test_cmd = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd"]) + subprocess.check_call( + ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir + ) + provider_passed[provider_name] = True + + finally: + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + if not all(provider_passed[p] for p in provider_passed.keys()): + sys.exit( + "some providers failed release test: " + + ",".join(name for name, passed in provider_passed if not passed) + ) + + +def release_command(args): + # subprocess.check_call(["vagrant", "cloud", "version", "create", f"tlcpack/microtvm-{args.platform}", args.version]) + if not args.version: + sys.exit(f"--version must be specified") + + for provider_name in args.provider.split(","): + subprocess.check_call( + [ + "vagrant", + "cloud", + "publish", + "-f", + f"tlcpack/microtvm-{args.platform}", + args.version, + provider_name, + os.path.join( + THIS_DIR, + args.platform, + "base-box", + f"output-packer-{provider_name}/package.box", + ), + ] + ) + + +ALL_COMMANDS = { + "build": build_command, + "test": test_command, + "release": release_command, +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Automates building, testing, and releasing a base box" + ) + parser.add_argument( + "command", default=",".join(ALL_COMMANDS), choices=ALL_COMMANDS, help="Action to perform." + ) + parser.add_argument( + "platform", + help="Name of the platform VM to act on. Must be a sub-directory of this directory.", + ) + parser.add_argument( + "--provider", + help="Name of the provider or providers to act on; if not specified, act on all", + ) + parser.add_argument( + "--test-device-serial", help="If given, attach the test device with this USB serial number" Review comment: I think an example is needed here, to clarify how this option looks like, from the user point of view. ########## File path: apps/microtvm/reference-vm/base-box-tool.py ########## @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import abc +import argparse +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys + + +_LOG = logging.getLogger(__name__) + + +THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".") + + +# List of vagrant providers supported by this tool +ALL_PROVIDERS = ( + "parallels", + "virtualbox", +) + + +def parse_virtualbox_devices(): + output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") + devices = [] + current_dev = {} + for line in output.split("\n"): + if not line.strip(): + if current_dev: + if "VendorId" in current_dev and "ProductId" in current_dev: + devices.append(current_dev) + current_dev = {} + + continue + + key, value = line.split(":", 1) + value = value.lstrip(" ") + current_dev[key] = value + + if current_dev: + devices.append(current_dev) + return devices + + +VIRTUALBOX_VID_PID_RE = re.compile(r"0x([0-9A-Fa-f]{4}).*") + + +def attach_virtualbox(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = parse_virtualbox_devices() + for dev in usb_devices: + m = VIRTUALBOX_VID_PID_RE.match(dev["VendorId"]) + if not m: + _LOG.warning("Malformed VendorId: %s", dev["VendorId"]) + continue + + dev_vid_hex = m.group(1).lower() + + m = VIRTUALBOX_VID_PID_RE.match(dev["ProductId"]) + if not m: + _LOG.warning("Malformed ProductId: %s", dev["ProductId"]) + continue + + dev_pid_hex = m.group(1).lower() + + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev["SerialNumber"]) + ): + rule_args = [ + "VBoxManage", + "usbfilter", + "add", + "0", + "--action", + "hold", + "--name", + "test device", + "--target", + uuid, + "--vendorid", + vid_hex, + "--productid", + pid_hex, + ] + if serial is not None: + rule_args.extend(["--serialnumber", serial]) + subprocess.check_call(rule_args) + subprocess.check_call(["VBoxManage", "controlvm", uuid, "usbattach", dev["UUID"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +def attach_parallels(uuid, vid_hex=None, pid_hex=None, serial=None): + usb_devices = json.loads( + subprocess.check_output(["prlsrvctl", "usb", "list", "-j"], encoding="utf-8") + ) + for dev in usb_devices: + _, dev_vid_hex, dev_pid_hex, _, _, dev_serial = dev["System name"].split("|") + dev_vid_hex = dev_vid_hex.lower() + dev_pid_hex = dev_pid_hex.lower() + if ( + vid_hex == dev_vid_hex + and pid_hex == dev_pid_hex + and (serial is None or serial == dev_serial) + ): + subprocess.check_call(["prlsrvctl", "usb", "set", dev["Name"], uuid]) + subprocess.check_call(["prlctl", "set", uuid, "--device-connect", dev["Name"]]) + return + + raise Exception( + f"Device with vid={vid_hex}, pid={pid_hex}, serial={serial!r} not found:\n{usb_devices!r}" + ) + + +ATTACH_USB_DEVICE = { + "parallels": attach_parallels, + "virtualbox": attach_virtualbox, +} + + +def generate_packer_config(file_path, providers): + builders = [] + for provider_name in providers: + builders.append( + { + "type": "vagrant", + "output_dir": f"output-packer-{provider_name}", + "communicator": "ssh", + "source_path": "generic/ubuntu1804", + "provider": provider_name, + "template": "Vagrantfile.packer-template", + } + ) + + with open(file_path, "w") as f: + json.dump( + { + "builders": builders, + }, + f, + sort_keys=True, + indent=2, + ) + + +def build_command(args): + generate_packer_config( + os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"), + args.provider.split(",") or ALL_PROVIDERS, + ) + subprocess.check_call( + ["packer", "build", "packer.json"], cwd=os.path.join(THIS_DIR, args.platform, "base-box") + ) + + +REQUIRED_TEST_CONFIG_KEYS = { + "vid_hex": str, + "pid_hex": str, + "test_cmd": list, +} + + +VM_BOX_RE = re.compile(r'(.*\.vm\.box) = "(.*)"') + + +# Paths, relative to the platform box directory, which will not be copied to release-test dir. +SKIP_COPY_PATHS = [".vagrant", "base-box"] + + +def test_command(args): + user_box_dir = os.path.join(THIS_DIR, args.platform) + base_box_dir = os.path.join(THIS_DIR, args.platform, "base-box") + test_config_file = os.path.join(base_box_dir, "test-config.json") + with open(test_config_file) as f: + test_config = json.load(f) + for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): + assert key in test_config and isinstance( + test_config[key], expected_type + ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" + + test_config["vid_hex"] = test_config["vid_hex"].lower() + test_config["pid_hex"] = test_config["pid_hex"].lower() + + providers = args.provider.split(",") + provider_passed = {p: False for p in providers} + + release_test_dir = os.path.join(THIS_DIR, "release-test") + + for provider_name in providers: + if os.path.exists(release_test_dir): + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + for dirpath, _, filenames in os.walk(user_box_dir): + rel_path = os.path.relpath(dirpath, user_box_dir) + if any( + rel_path == scp or rel_path.startswith(f"{scp}{os.path.sep}") + for scp in SKIP_COPY_PATHS + ): + continue + + dest_dir = os.path.join(release_test_dir, rel_path) + os.makedirs(dest_dir) + for filename in filenames: + shutil.copy2(os.path.join(dirpath, filename), os.path.join(dest_dir, filename)) + + release_test_vagrantfile = os.path.join(release_test_dir, "Vagrantfile") + with open(release_test_vagrantfile) as f: + lines = list(f) + + found_box_line = False + with open(release_test_vagrantfile, "w") as f: + for line in lines: + m = VM_BOX_RE.match(line) + if not m: + f.write(line) + continue + + box_package = os.path.join( + base_box_dir, f"output-packer-{provider_name}", "package.box" + ) + f.write(f'{m.group(1)} = "{os.path.relpath(box_package, release_test_dir)}"\n') + found_box_line = True + + if not found_box_line: + _LOG.error( + "testing provider %s: couldn't find config.box.vm = line in Vagrantfile; unable to test", + provider_name, + ) + continue + + subprocess.check_call( + ["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir + ) + try: + with open( + os.path.join( + release_test_dir, ".vagrant", "machines", "default", provider_name, "id" + ) + ) as f: + machine_uuid = f.read() + ATTACH_USB_DEVICE[provider_name]( + machine_uuid, + vid_hex=test_config["vid_hex"], + pid_hex=test_config["pid_hex"], + serial=args.test_device_serial, + ) + tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", "..")) + + def _quote_cmd(cmd): + return " ".join(shlex.quote(a) for a in cmd) + + test_cmd = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd"]) + subprocess.check_call( + ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir + ) + provider_passed[provider_name] = True + + finally: + subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) + shutil.rmtree(release_test_dir) + + if not all(provider_passed[p] for p in provider_passed.keys()): + sys.exit( + "some providers failed release test: " + + ",".join(name for name, passed in provider_passed if not passed) + ) + + +def release_command(args): + # subprocess.check_call(["vagrant", "cloud", "version", "create", f"tlcpack/microtvm-{args.platform}", args.version]) + if not args.version: + sys.exit(f"--version must be specified") + + for provider_name in args.provider.split(","): + subprocess.check_call( + [ + "vagrant", + "cloud", + "publish", + "-f", + f"tlcpack/microtvm-{args.platform}", + args.version, + provider_name, + os.path.join( + THIS_DIR, + args.platform, + "base-box", + f"output-packer-{provider_name}/package.box", + ), + ] + ) + + +ALL_COMMANDS = { + "build": build_command, + "test": test_command, + "release": release_command, +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Automates building, testing, and releasing a base box" + ) + parser.add_argument( + "command", default=",".join(ALL_COMMANDS), choices=ALL_COMMANDS, help="Action to perform." + ) + parser.add_argument( + "platform", + help="Name of the platform VM to act on. Must be a sub-directory of this directory.", + ) + parser.add_argument( + "--provider", + help="Name of the provider or providers to act on; if not specified, act on all", + ) + parser.add_argument( + "--test-device-serial", help="If given, attach the test device with this USB serial number" + ) + parser.add_argument("--version", help="Version to release. Must be specified with release.") Review comment: I think an example is needed here, to clarify how `version` looks like. Apart from that, maybe `--version` is expected by users to mean "what is the version of this tool?". I suggest renaming this to something more meaningful to the end user. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
