This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/buildstream-plugins.git
commit 8b030e9eb33cad067011db90a3da91fab35518bd Author: Tristan van Berkom <[email protected]> AuthorDate: Mon Mar 21 16:54:35 2022 +0900 tests/elements/pip.py: Adding tests for pip element --- tests/elements/pip.py | 132 ++++++++++++++++++++++ tests/elements/pip/elements/base.bst | 3 + tests/elements/pip/elements/base/alpine-image.bst | 6 + tests/elements/pip/files/piphello.tar.xz | Bin 0 -> 628 bytes tests/elements/pip/project.conf | 15 +++ tests/testutils/python_repo.py | 132 ++++++++++++++++++++++ 6 files changed, 288 insertions(+) diff --git a/tests/elements/pip.py b/tests/elements/pip.py new file mode 100644 index 0000000..1664063 --- /dev/null +++ b/tests/elements/pip.py @@ -0,0 +1,132 @@ +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os + +import pytest + +from buildstream import _yaml + +from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.integration import assert_contains +from buildstream._testing.integration import integration_cache # pylint: disable=unused-import +from buildstream._testing._utils.site import HAVE_SANDBOX + +from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unused-import + + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip") + + [email protected](DATA_DIR) [email protected](not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_pip_build(cli, datafiles): + project = str(datafiles) + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" + + element = { + "kind": "pip", + "variables": {"pip": "pip3"}, + "depends": [{"filename": "base.bst"}], + "sources": [ + { + "kind": "tar", + "url": "file://{}/files/piphello.tar.xz".format(project), + "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d", + } + ], + } + os.makedirs( + os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True, + ) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=["build", element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout],) + assert result.exit_code == 0 + + assert_contains( + checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/bin/hello", "/usr/lib/python3.6",], + ) + + +# Test running an executable built with pip [email protected](DATA_DIR) [email protected](not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_pip_run(cli, datafiles): + # Create and build our test element + test_pip_build(cli, datafiles) + + project = str(datafiles) + element_name = "pip/hello.bst" + + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) + assert result.exit_code == 0 + assert result.output == "Hello, world!\n" + + [email protected](DATA_DIR) [email protected](not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo): + project = str(datafiles) + elements_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" + + # check that exotically named packages are imported correctly + myreqs_packages = "alohalib" + dependencies = [ + "app2", + "app.3", + "app-4", + "app_5", + "app.no.6", + "app-no-7", + "app_no_8", + ] + mock_packages = {myreqs_packages: {package: {} for package in dependencies}} + + # set up directories + pypi_repo = os.path.join(project, "files", "pypi-repo") + os.makedirs(pypi_repo, exist_ok=True) + os.makedirs( + os.path.dirname(os.path.join(elements_path, element_name)), exist_ok=True, + ) + setup_pypi_repo(mock_packages, pypi_repo) + + # create pip element + element = { + "kind": "pip", + "variables": {"pip": "pip3"}, + "depends": [{"filename": "base.bst"}], + "sources": [ + { + "kind": "tar", + "url": "file://{}/files/piphello.tar.xz".format(project), + # FIXME: remove hardcoded ref once issue #1010 is closed + "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d", + }, + {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages],}, + ], + } + _yaml.roundtrip_dump(element, os.path.join(elements_path, element_name)) + + result = cli.run(project=project, args=["source", "track", element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=["build", element_name]) + assert result.exit_code == 0 + + # get installed packages in sandbox + installed_packages = set( + cli.run(project=project, args=["shell", element_name, "pip3", "freeze"]).output.split("\n") + ) + # compare with packages that are expected to be installed + pip_source_packages = {package.replace("_", "-") + "==0.1" for package in dependencies + [myreqs_packages]} + assert pip_source_packages.issubset(installed_packages) diff --git a/tests/elements/pip/elements/base.bst b/tests/elements/pip/elements/base.bst new file mode 100644 index 0000000..da7c70b --- /dev/null +++ b/tests/elements/pip/elements/base.bst @@ -0,0 +1,3 @@ +kind: stack +depends: +- base/alpine-image.bst diff --git a/tests/elements/pip/elements/base/alpine-image.bst b/tests/elements/pip/elements/base/alpine-image.bst new file mode 100644 index 0000000..f8e00ba --- /dev/null +++ b/tests/elements/pip/elements/base/alpine-image.bst @@ -0,0 +1,6 @@ +kind: import +description: Import an alpine image as the platform +sources: +- kind: tar + url: alpine:integration-tests-base.v1.x86_64.tar.xz + ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 diff --git a/tests/elements/pip/files/piphello.tar.xz b/tests/elements/pip/files/piphello.tar.xz new file mode 100644 index 0000000..72ec9b3 Binary files /dev/null and b/tests/elements/pip/files/piphello.tar.xz differ diff --git a/tests/elements/pip/project.conf b/tests/elements/pip/project.conf new file mode 100644 index 0000000..f04cd98 --- /dev/null +++ b/tests/elements/pip/project.conf @@ -0,0 +1,15 @@ +# test project config +name: test +min-version: 2.0 + +element-path: elements + +plugins: +- origin: pip + package-name: buildstream-plugins + elements: + - pip + +aliases: + alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ + project_dir: file://{project_dir} diff --git a/tests/testutils/python_repo.py b/tests/testutils/python_repo.py new file mode 100644 index 0000000..52dd8a6 --- /dev/null +++ b/tests/testutils/python_repo.py @@ -0,0 +1,132 @@ +import os +import re +import shutil +import subprocess +import sys + +import pytest + + +SETUP_TEMPLATE = """\ +from setuptools import setup + +setup( + name='{name}', + version='{version}', + description='{name}', + packages=['{pkgdirname}'], + install_requires={pkgdeps}, + entry_points={{ + 'console_scripts': [ + '{pkgdirname}={pkgdirname}:main' + ] + }} +) +""" + +# All packages generated via generate_pip_package will have the functions below +INIT_TEMPLATE = """\ +def main(): + print('This is {name}') + +def hello(actor='world'): + print('Hello {{}}! This is {name}'.format(actor)) +""" + +HTML_TEMPLATE = """\ +<html> + <head> + <title>Links for {name}</title> + </head> + <body> + <a href='{name}-{version}.tar.gz'>{name}-{version}.tar.gz</a><br /> + </body> +</html> +""" + + +# Creates a simple python source distribution and copies this into a specified +# directory which is to serve as a mock python repository +# +# Args: +# tmpdir (str): Directory in which the source files will be created +# pypi (str): Directory serving as a mock python repository +# name (str): The name of the package to be created +# version (str): The version of the package to be created +# +# Returns: +# None +# +def generate_pip_package(tmpdir, pypi, name, version="0.1", dependencies=None): + if dependencies is None: + dependencies = [] + # check if package already exists in pypi + pypi_package = os.path.join(pypi, re.sub("[^0-9a-zA-Z]+", "-", name)) + if os.path.exists(pypi_package): + return + + # create the package source files in tmpdir resulting in a directory + # tree resembling the following structure: + # + # tmpdir + # |-- setup.py + # `-- package + # `-- __init__.py + # + setup_file = os.path.join(tmpdir, "setup.py") + pkgdirname = re.sub("[^0-9a-zA-Z]+", "", name) + with open(setup_file, "w", encoding="utf-8") as f: + f.write(SETUP_TEMPLATE.format(name=name, version=version, pkgdirname=pkgdirname, pkgdeps=dependencies,)) + os.chmod(setup_file, 0o755) + + package = os.path.join(tmpdir, pkgdirname) + os.makedirs(package) + + main_file = os.path.join(package, "__init__.py") + with open(main_file, "w", encoding="utf-8") as f: + f.write(INIT_TEMPLATE.format(name=name)) + os.chmod(main_file, 0o644) + + # Run sdist with a fresh process + subprocess.run([sys.executable, "setup.py", "sdist"], cwd=tmpdir, check=True) + + # create directory for this package in pypi resulting in a directory + # tree resembling the following structure: + # + # pypi + # `-- pypi_package + # |-- index.html + # `-- foo-0.1.tar.gz + # + os.makedirs(pypi_package) + + # add an index html page + index_html = os.path.join(pypi_package, "index.html") + with open(index_html, "w", encoding="utf-8") as f: + f.write(HTML_TEMPLATE.format(name=name, version=version)) + + # copy generated tarfile to pypi package + dist_dir = os.path.join(tmpdir, "dist") + for tar in os.listdir(dist_dir): + tarpath = os.path.join(dist_dir, tar) + shutil.copy(tarpath, pypi_package) + + [email protected] +def setup_pypi_repo(tmpdir): + def create_pkgdir(package): + pkgdirname = re.sub("[^0-9a-zA-Z]+", "", package) + pkgdir = os.path.join(str(tmpdir), pkgdirname) + os.makedirs(pkgdir) + return pkgdir + + def add_packages(packages, pypi_repo): + for package, dependencies in packages.items(): + pkgdir = create_pkgdir(package) + generate_pip_package( + pkgdir, pypi_repo, package, dependencies=list(dependencies.keys()), + ) + for dependency, dependency_dependencies in dependencies.items(): + add_packages({dependency: dependency_dependencies}, pypi_repo) + + return add_packages
