Addressed review comments plus:
- dropped argparse filtering from read-dependencies and brpm now that master
has baked argparse direclty into the spec for py26
- added dry-run param to read-dependencies to allow for viewing intended
install actions
- made sure Requires in redhat and suse specs no longer contained any
dependencies from test-requirements.txt
Diff comments:
> diff --git a/Makefile b/Makefile
> index a3bfaf7..faa391c 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -78,6 +86,7 @@ clean_pyc:
> clean: clean_pyc
> rm -rf /var/log/cloud-init.log /var/lib/cloud/
>
> +
accidental, dropped.
> yaml:
> @$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES)
>
> diff --git a/tools/read-dependencies b/tools/read-dependencies
> index f434905..149c2b0 100755
> --- a/tools/read-dependencies
> +++ b/tools/read-dependencies
> @@ -1,43 +1,194 @@
> #!/usr/bin/env python
> +"""List pip dependencies or system package dependencies for cloud-init."""
>
> # You might be tempted to rewrite this as a shell script, but you
> # would be surprised to discover that things like 'egrep' or 'sed' may
> # differ between Linux and *BSD.
>
> +try:
> + from argparse import ArgumentParser
> +except ImportError:
> + raise RuntimeError(
> + 'Could not import python-argparse. Please install python-argparse '
> + 'package to continue')
> +
> +import json
> import os
> import re
> -import sys
> import subprocess
> +import sys
>
> -if 'CLOUD_INIT_TOP_D' in os.environ:
> - topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
> -else:
> - topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
>
> -for fname in ("setup.py", "requirements.txt"):
> - if not os.path.isfile(os.path.join(topd, fname)):
> - sys.stderr.write("Unable to locate '%s' file that should "
> - "exist in cloud-init root directory." % fname)
> - sys.exit(1)
> -
> -if len(sys.argv) > 1:
> - reqfile = sys.argv[1]
> -else:
> - reqfile = "requirements.txt"
> -
> -with open(os.path.join(topd, reqfile), "r") as fp:
> - for line in fp:
> - line = line.strip()
> - if not line or line.startswith("#"):
> +# Map the appropriate package dir needed for each distro choice
> +DISTRO_PKG_TYPE_MAP = {
> + 'centos': 'redhat',
> + 'redhat': 'redhat',
> + 'debian': 'debian',
> + 'ubuntu': 'debian',
> + 'opensuse': 'suse',
> + 'suse': 'suse'
> +}
> +
> +DISTRO_INSTALL_PKG_CMD = {
> + 'centos': ['yum', 'install', '--assumeyes'],
> + 'redhat': ['yum', 'install', '--assumeyes'],
> + 'debian': ['apt', 'install', '-y'],
> + 'ubuntu': ['apt', 'install', '-y'],
> + 'opensuse': ['zypper', 'install'],
> + 'suse': ['zypper', 'install']
> +}
> +
> +PY_26_ONLY = ['argparse']
> +
> +# List of base system packages required to start using make
> +EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar']
> +
> +
> +# JSON definition of distro-specific package dependencies
> +DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json"
> +
> +
> +def get_parser():
> + """Return an argument parser for this command."""
> + parser = ArgumentParser(description=__doc__)
> + parser.add_argument(
> + '-r', '--requirements-file', type=str, dest='req_file',
> + default='requirements.txt', help='The pip-style requirements file')
> + parser.add_argument(
> + '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(),
> + help='The name of the distro to generate package deps for.')
> + parser.add_argument(
> + '-s', '--system-pkg-names', action='store_true', default=False,
> + dest='system_pkg_names',
> + help='The name of the distro to generate package deps for.')
> + parser.add_argument(
> + '-i', '--install', action='store_true', default=False,
> + dest='install',
> + help='When specified, install the required system packages.')
> + parser.add_argument(
> + '-v', '--python-version', type=str, dest='python_version',
> default="2",
> + choices=["2", "3"],
> + help='The version of python we want to generate system package '
> + 'dependencies for.')
> + return parser
> +
> +
> +def get_package_deps_from_json(topdir, distro):
> + """Get a dict of build and runtime package requirements for a distro.
> +
> + @param topdir: The root directory in which to search for the
> + DISTRO_PKG_DEPS_PATH json blob of package requirements information.
> + @param distro: The specific distribution shortname to pull dependencies
> + for.
> + @return: Dict containing "requires", "build-requires" and "rename" lists
> + for a given distribution.
> + """
> + with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream:
> + deps = json.loads(stream.read())
> + if distro is None:
> + return {}
> + return deps[DISTRO_PKG_TYPE_MAP[distro]]
> +
> +
> +def parse_pip_requirements(requirements_path):
> + """Return the pip requirement names from pip-style requirements_path."""
> + dep_names = []
> + with open(requirements_path, "r") as fp:
> + for line in fp:
> + line = line.strip()
> + if not line or line.startswith("#"):
> + continue
> +
> + # remove pip-style markers
> + dep = line.split(';')[0]
> +
> + # remove version requirements
> + if re.search('[>=.<]+', dep):
> + dep_names.append(re.split("[>=.<]*", dep)[0].strip())
> + else:
> + dep_names.append(dep)
> + return dep_names
> +
> +
> +def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"):
> + """Translate pip package names to distro-specific package names.
> +
> + @param pip_requires: List of versionless pip package names to translate.
> + @param renames: Dict containg special case renames from pip name to
> system
> + package name for the distro.
> + """
> + if python_ver == "2":
> + prefix = "python-"
> + else:
> + prefix = "python3-"
> + standard_pkg_name = "{0}{1}"
> + translated_names = []
> + for pip_name in pip_requires:
> + pip_name = pip_name.lower()
> + if pip_name in PY_26_ONLY and sys.version_info[0:2] > (2, 6):
> + # Skip PY26 pkg deps unless we are actually on python 2.6
> continue
> + # Find a rename if present for the distro package and python version
> + rename = renames.get(pip_name, {}).get(python_ver, None)
> + if rename:
> + translated_names.append(rename)
> + else:
> + translated_names.append(
> + standard_pkg_name.format(prefix, pip_name))
> + return translated_names
> +
> +
> +def main(distro):
> + parser = get_parser()
> + args = parser.parse_args()
> + if 'CLOUD_INIT_TOP_D' in os.environ:
> + topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
> + else:
> + topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
> + req_path = os.path.join(topd, args.req_file)
> + if not os.path.isfile(req_path):
> + sys.stderr.write("Unable to locate '%s' file that should "
> + "exist in cloud-init root directory." % req_path)
> + return 1
> + pip_pkg_names = parse_pip_requirements(req_path)
> + deps_from_json = get_package_deps_from_json(topd, args.distro)
> + renames = deps_from_json.get('renames', {})
> + translated_pip_names = translate_pip_to_system_pkg(
> + pip_pkg_names, renames, args.python_version)
> + all_deps = []
> + if args.distro:
> + all_deps.extend(
> + translated_pip_names + deps_from_json['requires'] +
> + deps_from_json['build-requires'])
> + else:
> + if args.system_pkg_names:
> + all_deps = translated_pip_names
> + else:
> + all_deps = pip_pkg_names
> + if args.install:
> + pkg_install(all_deps, args.distro)
> + else:
> + print('\n'.join(all_deps))
> +
>
> - # remove pip-style markers
> - dep = line.split(';')[0]
> +def pkg_install(pkg_list, distro):
> + """Install a list of packages using the DISTRO_INSTALL_PKG_CMD"""
> + print("Installing deps:", ' '.join(pkg_list))
> + pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS)
> + if distro == 'centos':
fixed.
> + # CentOs needs epel-release to access oauthlib and jsonschema
fixed.
> + subprocess.check_call(DISTRO_INSTALL_PKG_CMD[distro] +
> ['epel-release'])
> + if distro in ['suse', 'opensuse', 'redhat', 'centos']:
> + pkg_list.append('rpm-build')
> + cmd = DISTRO_INSTALL_PKG_CMD[distro] + pkg_list
> + if os.getuid() != 0:
> + cmd.insert(0, 'sudo') # We aren't root so let's try sudo
> + subprocess.check_call(cmd)
>
> - # remove version requirements
> - dep = re.split("[>=.<]*", dep)[0].strip()
> - print(dep)
>
> -sys.exit(0)
> +if __name__ == "__main__":
> + parser = get_parser()
> + args = parser.parse_args()
> + sys.exit(main(args.distro))
>
> # vi: ts=4 expandtab
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/325342
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:ci-deps into cloud-init:master.
_______________________________________________
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
