Hello community, here is the log from the commit of package sesdev for openSUSE:Factory checked in at 2020-08-12 10:33:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/sesdev (Old) and /work/SRC/openSUSE:Factory/.sesdev.new.3399 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "sesdev" Wed Aug 12 10:33:29 2020 rev:15 rq:825782 version:1.6.1+1597147109.g204bf79 Changes: -------- --- /work/SRC/openSUSE:Factory/sesdev/sesdev.changes 2020-07-27 17:43:30.159174583 +0200 +++ /work/SRC/openSUSE:Factory/.sesdev.new.3399/sesdev.changes 2020-08-12 10:37:26.988319623 +0200 @@ -1,0 +2,13 @@ +Tue Aug 11 12:02:56 UTC 2020 - Nathan Cutler <ncut...@suse.com> + +- Update to 1.6.1+1597147109.g204bf79 + + upstream 1.6.1 release (2020-08-11) + * ceph_salt_deployment: implement "--stop-before-ceph-orch-apply" (PR #415) + * settings: rename straggler version_os_repo_mappings (PR #419) + * setup.cfg: fix "options.package_data" file list (PR #424) + (fixes a packaging regression introduced in 1.6.0) + * sesdev: non-create, non-box functions in alphabetical order (PR #414) + * Unify naming of role-related constants (PR #416) + * CaaSP: remove hard dependency on loadbalancer (PR #422) + +------------------------------------------------------------------- Old: ---- sesdev-1.6.0+1595760860.gc353ea1.tar.gz New: ---- sesdev-1.6.1+1597147109.g204bf79.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ sesdev.spec ++++++ --- /var/tmp/diff_new_pack.cjA9oX/_old 2020-08-12 10:37:28.216320232 +0200 +++ /var/tmp/diff_new_pack.cjA9oX/_new 2020-08-12 10:37:28.220320234 +0200 @@ -16,7 +16,7 @@ # Name: sesdev -Version: 1.6.0+1595760860.gc353ea1 +Version: 1.6.1+1597147109.g204bf79 Release: 1%{?dist} Summary: CLI tool to deploy and manage SES clusters License: MIT ++++++ sesdev-1.6.0+1595760860.gc353ea1.tar.gz -> sesdev-1.6.1+1597147109.g204bf79.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/CHANGELOG.md new/sesdev-1.6.1+1597147109.g204bf79/CHANGELOG.md --- old/sesdev-1.6.0+1595760860.gc353ea1/CHANGELOG.md 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/CHANGELOG.md 2020-08-11 13:58:29.347702885 +0200 @@ -7,6 +7,17 @@ ## [Unreleased] +## [1.6.1] - 2020-08-11 + +### Fixed +- ceph_salt_deployment: implement "--stop-before-ceph-orch-apply" (PR #415) +- settings: rename straggler version_os_repo_mappings (PR #419) +- setup.cfg: fix "options.package_data" file list (PR #424) + +### Changed +- sesdev: non-create, non-box functions in alphabetical order (PR #414) +- Unify naming of role-related constants (PR #416) + ## [1.6.0] - 2020-07-26 ### Added @@ -473,7 +484,8 @@ - Minimal README with a few usage instructions. - The CHANGELOG file. -[unreleased]: https://github.com/SUSE/sesdev/compare/v1.6.0...HEAD +[unreleased]: https://github.com/SUSE/sesdev/compare/v1.6.1...HEAD +[1.6.1]: https://github.com/SUSE/sesdev/releases/tag/v1.6.1 [1.6.0]: https://github.com/SUSE/sesdev/releases/tag/v1.6.0 [1.5.0]: https://github.com/SUSE/sesdev/releases/tag/v1.5.0 [1.4.0]: https://github.com/SUSE/sesdev/releases/tag/v1.4.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/README.md new/sesdev-1.6.1+1597147109.g204bf79/README.md --- old/sesdev-1.6.0+1595760860.gc353ea1/README.md 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/README.md 2020-08-11 13:58:29.347702885 +0200 @@ -42,6 +42,7 @@ * [Running the unit tests](#running-the-unit-tests) * [Usage](#usage) * [Create/deploy a Ceph cluster](#createdeploy-a-ceph-cluster) + * [Rook and CaaSP based Ceph cluster](#rook-and-caasp-based-ceph-cluster) * [On a remote libvirt server via SSH](#on-a-remote-libvirt-server-via-ssh) * [Using salt instead of DeepSea/ceph-salt CLI](#using-salt-instead-of-deepseaceph-salt-cli) * [Without the devel repo](#without-the-devel-repo) @@ -381,6 +382,25 @@ [storage, mon, mgr, mds], [igw, nfs, rgw]" ``` +#### Rook and CaaSP based Ceph cluster + +To create CaaSP k8s cluster that has loadbalancer, 2 worker nodes and master: + +``` +$ sesdev create caasp4 +``` + +By default it just creates and configures CaaSP cluster and workers don't have any disks. + +To create workers with disks and without loadbalancer role: + +``` +$ sesdev create caasp4 --roles="[master], [worker], [worker]" --disk-size 6 --num-disks 2 +``` + +To deploy Rook on that cluster use `--deploy-ses` option, also be aware about minimum requirement for the disk sizes (>5G). + + #### On a remote libvirt server via SSH If you would like to start the cluster VMs on a remote server via libvirt/SSH, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/contrib/standalone.sh new/sesdev-1.6.1+1597147109.g204bf79/contrib/standalone.sh --- old/sesdev-1.6.0+1595760860.gc353ea1/contrib/standalone.sh 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/contrib/standalone.sh 2020-08-11 13:58:29.347702885 +0200 @@ -205,6 +205,7 @@ fi set -x +touch $HOME/.sesdev/config.yaml if [ "$SES5" ] ; then sesdev --verbose box remove --non-interactive sles-12-sp3 @@ -334,4 +335,6 @@ exit 1 fi +rm $HOME/.sesdev/config.yaml + final_report diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/sesdev/__init__.py new/sesdev-1.6.1+1597147109.g204bf79/sesdev/__init__.py --- old/sesdev-1.6.0+1595760860.gc353ea1/sesdev/__init__.py 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/sesdev/__init__.py 2020-08-11 13:58:29.347702885 +0200 @@ -1,13 +1,14 @@ import fnmatch import json import logging +from os import environ, path import re import sys -from os.path import isabs, exists, isdir + +from prettytable import PrettyTable import click import pkg_resources -from prettytable import PrettyTable from seslib.box import Box from seslib.constant import Constant @@ -24,7 +25,7 @@ VersionNotKnown from seslib.log import Log from seslib.settings import Settings -from seslib.tools import gen_random_string +from seslib import tools from seslib.zypper import ZypperRepo @@ -286,75 +287,6 @@ Constant.CONFIG_FILE = config_file # set once here, never to change again -@click.option('--format', 'format_opt', type=str, default=None) -@cli.command(name='list') -def list_deps(format_opt): - """ - Lists all the available deployments. - """ - p_table = None - deployments_list = [] - deps = Deployment.list(True) - if deps: - Log.info("list_deps: Found deployments: {}".format(", ".join(d.dep_id for d in deps))) - else: - msg = "No deployments found" - Log.info("list_deps: {}".format(msg)) - if format_opt in ['json']: - click.echo(json.dumps([], sort_keys=True, indent=4)) - else: - click.echo(msg) - return None - - def _status(nodes): - status = None - for node in nodes.values(): - if status is None: - status = node.status - elif node.status == 'running' and status == 'not deployed': - status = 'partially deployed' - elif node.status == 'stopped' and status == 'running': - status = 'partially running' - elif node.status == 'suspended' and status == 'running': - status = 'partially running' - elif node.status == 'running' and status == 'stopped': - status = 'partially running' - elif node.status == 'running' and status == 'suspended': - status = 'partially running' - return status - - if format_opt not in ['json']: - p_table = PrettyTable(["ID", "Version", "Status", "Nodes"]) - p_table.align = "l" - - for dep in deps: - Log.debug("_status: Looping over deployments ({})".format(dep.dep_id)) - status = str(_status(dep.nodes)) - Log.debug("_status: -> status: {}".format(status)) - version = getattr(dep.settings, 'version', None) - Log.debug("_status: -> version: {}".format(version)) - nodes = getattr(dep, 'nodes', None) - node_names = '(unknown)' if nodes is None else ', '.join(nodes) - Log.debug("_status: -> node_names: {}".format(node_names)) - if format_opt in ['json']: - deployments_list.append({ - "id": dep.dep_id, - "version": version, - "status": status, - "nodes": list(nodes), - }) - else: - p_table.add_row([dep.dep_id, version, status, node_names]) - if format_opt in ['json']: - click.echo(json.dumps(deployments_list, sort_keys=True, indent=4)) - else: - deployment_word = "deployments" if len(deps) > 1 else "deployment" - click.echo("Found {} {}:".format(len(deps), deployment_word)) - click.echo() - click.echo(p_table) - click.echo() - - @cli.group() def box(): """ @@ -535,11 +467,11 @@ elif single_node: roles_string = "" if version in ['ses7', 'octopus', 'pacific']: - roles_string = Constant.ROLES_SINGLE_NODE_OCTOPUS + roles_string = Constant.ROLES_SINGLE_NODE['octopus'] elif version in ['ses6', 'nautilus']: - roles_string = Constant.ROLES_SINGLE_NODE_NAUTILUS + roles_string = Constant.ROLES_SINGLE_NODE['nautilus'] elif version in ['ses5']: - roles_string = Constant.ROLES_SINGLE_NODE_LUMINOUS + roles_string = Constant.ROLES_SINGLE_NODE['luminous'] else: raise VersionNotKnown(version) settings_dict['roles'] = _parse_roles(roles_string) @@ -713,12 +645,12 @@ for folder in synced_folder: try: src, dst = folder.split(':') - if not all([isabs(x) for x in [src, dst]]): + if not all([path.isabs(x) for x in [src, dst]]): raise OptionValueError('--synced-folder', "Please provide absolute paths for " "synced folder paths", folder) - if not exists(src): + if not path.exists(src): raise OptionValueError('--synced-folder', "Path to the source synced folder must exist", src) @@ -1007,6 +939,33 @@ return "clusters" +@cli.command(name='add-repo') +@click.option('--repo-priority/--no-repo-priority', default=False, + help="Set elevated priority on custom zypper repos") +@click.option('--update/--no-update', is_flag=True, + help='Update packages after adding devel repo') +@click.argument('deployment_id') +@click.argument('custom_repo', required=False) +def add_repo(deployment_id, **kwargs): + """ + Add a custom repo to all nodes of an already-deployed cluster. The repo + should be specified in the form of an URL, but it is optional: if it is + omitted, the "devel" repo (which has a specific meaning depending on the + deployment version) will be added. + """ + if kwargs['update'] and kwargs['custom_repo']: + raise AddRepoNoUpdateWithExplicitRepo() + dep = Deployment.load(deployment_id) + custom_repo = None + if kwargs['custom_repo']: + custom_repo = ZypperRepo( + name='custom_repo_{}'.format(tools.gen_random_string(6)), + url=kwargs['custom_repo'], + priority=Constant.ZYPPER_PRIO_ELEVATED if kwargs['repo_priority'] else None + ) + dep.add_repo_subcommand(custom_repo, kwargs['update'], _print_log) + + @cli.command() @click.argument('deployment_id') @click.option('--non-interactive', '-n', '--force', '-f', @@ -1040,25 +999,147 @@ click.echo("Deployment {} destroyed!".format(dep.dep_id)) +@click.option('--format', 'format_opt', type=str, default=None) +@cli.command(name='list') +def list_deps(format_opt): + """ + Lists all the available deployments. + """ + p_table = None + deployments_list = [] + deps = Deployment.list(True) + if deps: + Log.info("list_deps: Found deployments: {}".format(", ".join(d.dep_id for d in deps))) + else: + msg = "No deployments found" + Log.info("list_deps: {}".format(msg)) + if format_opt in ['json']: + click.echo(json.dumps([], sort_keys=True, indent=4)) + else: + click.echo(msg) + return None + + def _status(nodes): + status = None + for node in nodes.values(): + if status is None: + status = node.status + elif node.status == 'running' and status == 'not deployed': + status = 'partially deployed' + elif node.status == 'stopped' and status == 'running': + status = 'partially running' + elif node.status == 'suspended' and status == 'running': + status = 'partially running' + elif node.status == 'running' and status == 'stopped': + status = 'partially running' + elif node.status == 'running' and status == 'suspended': + status = 'partially running' + return status + + if format_opt not in ['json']: + p_table = PrettyTable(["ID", "Version", "Status", "Nodes"]) + p_table.align = "l" + + for dep in deps: + Log.debug("_status: Looping over deployments ({})".format(dep.dep_id)) + status = str(_status(dep.nodes)) + Log.debug("_status: -> status: {}".format(status)) + version = getattr(dep.settings, 'version', None) + Log.debug("_status: -> version: {}".format(version)) + nodes = getattr(dep, 'nodes', None) + node_names = '(unknown)' if nodes is None else ', '.join(nodes) + Log.debug("_status: -> node_names: {}".format(node_names)) + if format_opt in ['json']: + deployments_list.append({ + "id": dep.dep_id, + "version": version, + "status": status, + "nodes": list(nodes), + }) + else: + p_table.add_row([dep.dep_id, version, status, node_names]) + if format_opt in ['json']: + click.echo(json.dumps(deployments_list, sort_keys=True, indent=4)) + else: + deployment_word = "deployments" if len(deps) > 1 else "deployment" + click.echo("Found {} {}:".format(len(deps), deployment_word)) + click.echo() + click.echo(p_table) + click.echo() + + +@cli.command(name='qa-test') +@click.argument('deployment_id') +def qa_test(deployment_id): + """ + Runs QA test on an already-deployed cluster. + """ + dep = Deployment.load(deployment_id) + dep.qa_test(_print_log) + + @cli.command() @click.argument('deployment_id') -@click.argument('node', required=False) -@click.argument('command', required=False, nargs=-1) -def ssh(deployment_id, node=None, command=None): +@click.option('--non-interactive', '-n', '--force', '-f', + is_flag=True, + callback=_abort_if_false, + default=False, + expose_value=False, + help='Allow to redeploy the cluster without user confirmation', + prompt='Are you sure you want to redeploy the cluster?') +def redeploy(deployment_id, **kwargs): """ - Opens an SSH shell to, or runs optional COMMAND on, node NODE in deployment - DEPLOYMENT_ID. + Destroys the VMs of the deployment DEPLOYMENT_ID and deploys again the cluster + from scratch with the same configuration. + """ + interactive = not (kwargs.get('non_interactive', False) or kwargs.get('force', False)) + if interactive: + really_want_to = True + if interactive: + really_want_to = click.confirm( + 'Do you want to continue with the deployment?', + default=True, + ) + if not really_want_to: + raise click.Abort() + dep = Deployment.load(deployment_id) + dep.destroy(_print_log) + dep = Deployment.create(deployment_id, dep.settings) + dep.start(_print_log) - If the node is not specified, it defaults to "master". - Note: You can check the existing node names with the command - "sesdev show <deployment_id>" +@cli.command(name='replace-ceph-salt') +@click.argument('deployment_id') +@click.option('--local', default=None, type=str, show_default=True, + help='The local path for "ceph-salt" source') +def replace_ceph_salt(deployment_id, local=None): + """ + Install ceph-salt from source """ dep = Deployment.load(deployment_id) - node_name = 'master' if node is None else node - if command: - Log.info("Running SSH command on {}: {}".format(node_name, command)) - dep.ssh(node_name, command) + dep.replace_ceph_salt(local) + + +@cli.command(name='replace-mgr-modules') +@click.argument('deployment_id') +@click.option('--local', type=str, help='The local repository path. E.g.: ~/ceph') +@click.option('--pr', type=int, help='The PR to be fetched from a remote repository') +@click.option('--branch', default='master', type=str, show_default=True, + help='The branch to be fetched from a remote repository') +@click.option('--repo', default='ceph', type=str, show_default=True, + help='The remote repository from which to fetch PRs or branches') +@click.option('--langs', default='en-US', type=str, show_default=True, + help='Dashboard languages to be built') +def replace_mgr_modules(deployment_id, **kwargs): + """ + Fetches a different version of Ceph MGR modules from a local repository or github, + replacing the installed ones. + + --local, --pr and --branch conflict with each other, + when the first is found the remaining are ignored. + """ + dep = Deployment.load(deployment_id) + dep.replace_mgr_modules(**kwargs) @cli.command() @@ -1102,71 +1183,55 @@ dep.scp(source, destination, recurse=recursive) -@cli.command(name='qa-test') -@click.argument('deployment_id') -def qa_test(deployment_id): - """ - Runs QA test on an already-deployed cluster. - """ - dep = Deployment.load(deployment_id) - dep.qa_test(_print_log) - - -@cli.command(name='add-repo') -@click.option('--repo-priority/--no-repo-priority', default=False, - help="Set elevated priority on custom zypper repos") -@click.option('--update/--no-update', is_flag=True, - help='Update packages after adding devel repo') +@cli.command() +@click.option('--detail/--no-detail', is_flag=True, default=False, + help='Display details of each VM in additional to deployment-wide configuration') @click.argument('deployment_id') -@click.argument('custom_repo', required=False) -def add_repo(deployment_id, **kwargs): +def show(deployment_id, **kwargs): """ - Add a custom repo to all nodes of an already-deployed cluster. The repo - should be specified in the form of an URL, but it is optional: if it is - omitted, the "devel" repo (which has a specific meaning depending on the - deployment version) will be added. + Display the configuration of a running deployment - this is the same + information that is displayed by the "create" command before asking the user + whether they are really sure they want to create the cluster). Use "--detail" + to get information on individual VMs in the deployment. """ - if kwargs['update'] and kwargs['custom_repo']: - raise AddRepoNoUpdateWithExplicitRepo() dep = Deployment.load(deployment_id) - custom_repo = None - if kwargs['custom_repo']: - custom_repo = ZypperRepo( - name='custom_repo_{}'.format(gen_random_string(6)), - url=kwargs['custom_repo'], - priority=Constant.ZYPPER_PRIO_ELEVATED if kwargs['repo_priority'] else None - ) - dep.add_repo_subcommand(custom_repo, kwargs['update'], _print_log) + click.echo(dep.configuration_report(show_individual_vms=kwargs['detail'])) @cli.command() @click.argument('deployment_id') @click.argument('node', required=False) -def supportconfig(deployment_id, node): +@click.argument('command', required=False, nargs=-1) +def ssh(deployment_id, node=None, command=None): """ - Runs supportconfig on a node within an already-deployed cluster. Dumps the - resulting tarball in the current working directory. + Opens an SSH shell to, or runs optional COMMAND on, node NODE in deployment + DEPLOYMENT_ID. If the node is not specified, it defaults to "master". - NOTE: supportconfig is only available in deployments running on SUSE Linux - Enterprise. + Note: You can check the existing node names with the command + "sesdev show <deployment_id>" """ dep = Deployment.load(deployment_id) - _node = 'master' if node is None else node - dep.supportconfig(_print_log, _node) + node_name = 'master' if node is None else node + if command: + Log.info("Running SSH command on {}: {}".format(node_name, command)) + dep.ssh(node_name, command) @cli.command() @click.argument('deployment_id') @click.argument('node', required=False) -def stop(deployment_id, node=None): +def start(deployment_id, node=None): """ - Stops the VMs of the deployment DEPLOYMENT_SPEC, where DEPLOYMENT_SPEC + Starts the VMs of the deployment DEPLOYMENT_SPEC, where DEPLOYMENT_SPEC might be either a literal deployment ID or a glob ("octopus_*"). + + If cluster was not yet deployed (if was created with the --no-deploy flag), it will + start the deployment of the cluster. """ matching_deployments = _maybe_glob_deps(deployment_id) - click.echo("Stopping {} {}".format( + click.echo("Starting {} {}".format( len(matching_deployments), _cluster_singular_or_plural(matching_deployments), )) @@ -1174,23 +1239,20 @@ click.echo("Ignoring node advice because DEPLOYMENT_SPEC is a glob") node = None for dep in matching_deployments: - dep.stop(_print_log, node) - click.echo("Deployment {} stopped!".format(dep.dep_id)) + dep.start(_print_log, node) + click.echo("Deployment {} started!".format(dep.dep_id)) @cli.command() @click.argument('deployment_id') @click.argument('node', required=False) -def start(deployment_id, node=None): +def stop(deployment_id, node=None): """ - Starts the VMs of the deployment DEPLOYMENT_SPEC, where DEPLOYMENT_SPEC + Stops the VMs of the deployment DEPLOYMENT_SPEC, where DEPLOYMENT_SPEC might be either a literal deployment ID or a glob ("octopus_*"). - - If cluster was not yet deployed (if was created with the --no-deploy flag), it will - start the deployment of the cluster. """ matching_deployments = _maybe_glob_deps(deployment_id) - click.echo("Starting {} {}".format( + click.echo("Stopping {} {}".format( len(matching_deployments), _cluster_singular_or_plural(matching_deployments), )) @@ -1198,53 +1260,26 @@ click.echo("Ignoring node advice because DEPLOYMENT_SPEC is a glob") node = None for dep in matching_deployments: - dep.start(_print_log, node) - click.echo("Deployment {} started!".format(dep.dep_id)) + dep.stop(_print_log, node) + click.echo("Deployment {} stopped!".format(dep.dep_id)) @cli.command() -@click.option('--detail/--no-detail', is_flag=True, default=False, - help='Display details of each VM in additional to deployment-wide configuration') @click.argument('deployment_id') -def show(deployment_id, **kwargs): - """ - Display the configuration of a running deployment - this is the same - information that is displayed by the "create" command before asking the user - whether they are really sure they want to create the cluster). Use "--detail" - to get information on individual VMs in the deployment. +@click.argument('node', required=False) +def supportconfig(deployment_id, node): """ - dep = Deployment.load(deployment_id) - click.echo(dep.configuration_report(show_individual_vms=kwargs['detail'])) + Runs supportconfig on a node within an already-deployed cluster. Dumps the + resulting tarball in the current working directory. + If the node is not specified, it defaults to "master". -@cli.command() -@click.argument('deployment_id') -@click.option('--non-interactive', '-n', '--force', '-f', - is_flag=True, - callback=_abort_if_false, - default=False, - expose_value=False, - help='Allow to redeploy the cluster without user confirmation', - prompt='Are you sure you want to redeploy the cluster?') -def redeploy(deployment_id, **kwargs): - """ - Destroys the VMs of the deployment DEPLOYMENT_ID and deploys again the cluster - from scratch with the same configuration. + NOTE: supportconfig is only available in deployments running on SUSE Linux + Enterprise. """ - interactive = not (kwargs.get('non_interactive', False) or kwargs.get('force', False)) - if interactive: - really_want_to = True - if interactive: - really_want_to = click.confirm( - 'Do you want to continue with the deployment?', - default=True, - ) - if not really_want_to: - raise click.Abort() dep = Deployment.load(deployment_id) - dep.destroy(_print_log) - dep = Deployment.create(deployment_id, dep.settings) - dep.start(_print_log) + _node = 'master' if node is None else node + dep.supportconfig(_print_log, _node) @cli.command() @@ -1286,41 +1321,3 @@ dep.dep_id) ) dep.start_port_forwarding(service, node, remote_port, local_port, local_address) - - -@cli.command(name='replace-ceph-salt') -@click.argument('deployment_id') -@click.option('--local', default=None, type=str, show_default=True, - help='The local path for "ceph-salt" source') -def replace_ceph_salt(deployment_id, local=None): - """ - Install ceph-salt from source - """ - dep = Deployment.load(deployment_id) - dep.replace_ceph_salt(local) - - -@cli.command(name='replace-mgr-modules') -@click.argument('deployment_id') -@click.option('--local', type=str, help='The local repository path. E.g.: ~/ceph') -@click.option('--pr', type=int, help='The PR to be fetched from a remote repository') -@click.option('--branch', default='master', type=str, show_default=True, - help='The branch to be fetched from a remote repository') -@click.option('--repo', default='ceph', type=str, show_default=True, - help='The remote repository from which to fetch PRs or branches') -@click.option('--langs', default='en-US', type=str, show_default=True, - help='Dashboard languages to be built') -def replace_mgr_modules(deployment_id, **kwargs): - """ - Fetches a different version of Ceph MGR modules from a local repository or github, - replacing the installed ones. - - --local, --pr and --branch conflict with each other, - when the first is found the remaining are ignored. - """ - dep = Deployment.load(deployment_id) - dep.replace_mgr_modules(**kwargs) - - -if __name__ == '__main__': - sys.exit(sesdev_main()) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/sesdev.spec new/sesdev-1.6.1+1597147109.g204bf79/sesdev.spec --- old/sesdev-1.6.0+1595760860.gc353ea1/sesdev.spec 2020-07-26 12:54:20.210214000 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/sesdev.spec 2020-08-11 13:58:29.647702503 +0200 @@ -16,7 +16,7 @@ # Name: sesdev -Version: 1.6.0+1595760860.gc353ea1 +Version: 1.6.1+1597147109.g204bf79 Release: 1%{?dist} Summary: CLI tool to deploy and manage SES clusters License: MIT diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/constant.py new/sesdev-1.6.1+1597147109.g204bf79/seslib/constant.py --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/constant.py 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/constant.py 2020-08-11 13:58:29.347702885 +0200 @@ -18,23 +18,6 @@ DEBUG = False - DEFAULT_ROLES = { - "caasp4": [["master"], ["worker"], ["worker"], ["loadbalancer"]], - "luminous": [["master", "client", "openattic"], - ["storage", "mon", "mgr", "rgw", "igw"], - ["storage", "mon", "mgr", "mds", "nfs"], - ["storage", "mon", "mgr", "mds", "rgw", "nfs"]], - "makecheck": [["makecheck"]], - "nautilus": [["master", "client", "prometheus", "grafana"], - ["storage", "mon", "mgr", "rgw", "igw"], - ["storage", "mon", "mgr", "mds", "igw", "nfs"], - ["storage", "mon", "mgr", "mds", "rgw", "nfs"]], - "octopus": [["master", "client", "prometheus", "grafana"], - ["bootstrap", "storage", "mon", "mgr", "rgw", "igw"], - ["storage", "mon", "mgr", "mds", "igw", "nfs"], - ["storage", "mon", "mgr", "mds", "rgw", "nfs"]] - } - IMAGE_PATHS = { 'ses7': 'registry.suse.de/devel/storage/7.0/containers/ses/7/ceph/ceph', 'octopus': 'registry.opensuse.org/filesystems/ceph/octopus/images/ceph/ceph', @@ -43,28 +26,6 @@ JINJA_ENV = Environment(loader=PackageLoader('seslib', 'templates'), trim_blocks=True) - KNOWN_ROLES = [ - "admin", - "bootstrap", - "client", - "ganesha", # deprecated (replaced by "nfs") - "grafana", - "igw", - "loadbalancer", - "makecheck", - "master", - "mds", - "mgr", - "mon", - "nfs", - "openattic", - "prometheus", - "rgw", - "storage", - "suma", - "worker", - ] - LOGFILE = None MAKECHECK_DEFAULT_RAM = 8 @@ -177,19 +138,65 @@ }, } - ROLES_SINGLE_NODE_LUMINOUS = ( - "[ master, storage, mon, mgr, mds, igw, rgw, nfs, openattic ]" - ) - - ROLES_SINGLE_NODE_NAUTILUS = ( - "[ master, storage, mon, mgr, prometheus, grafana, mds, igw, rgw, " - "nfs ]" - ) - - ROLES_SINGLE_NODE_OCTOPUS = ( - "[ master, bootstrap, storage, mon, mgr, prometheus, grafana, mds, " - "igw, rgw, nfs ]" - ) + ROLES_DEFAULT = { + "caasp4": [["master"], ["worker"], ["worker"], ["loadbalancer"]], + "luminous": [["master", "client", "openattic"], + ["storage", "mon", "mgr", "rgw", "igw"], + ["storage", "mon", "mgr", "mds", "nfs"], + ["storage", "mon", "mgr", "mds", "rgw", "nfs"]], + "makecheck": [["makecheck"]], + "nautilus": [["master", "client", "prometheus", "grafana"], + ["storage", "mon", "mgr", "rgw", "igw"], + ["storage", "mon", "mgr", "mds", "igw", "nfs"], + ["storage", "mon", "mgr", "mds", "rgw", "nfs"]], + "octopus": [["master", "client", "prometheus", "grafana", "alertmanager", "node-exporter"], + ["bootstrap", "storage", "mon", "mgr", "rgw", "igw", "node-exporter"], + ["storage", "mon", "mgr", "mds", "igw", "nfs", "node-exporter"], + ["storage", "mon", "mgr", "mds", "rgw", "nfs", "node-exporter"]] + } + + ROLES_DEFAULT_BY_VERSION = { + 'caasp4': ROLES_DEFAULT["caasp4"], + 'makecheck': ROLES_DEFAULT["makecheck"], + 'nautilus': ROLES_DEFAULT["nautilus"], + 'octopus': ROLES_DEFAULT["octopus"], + 'pacific': ROLES_DEFAULT["octopus"], + 'ses5': ROLES_DEFAULT["luminous"], + 'ses6': ROLES_DEFAULT["nautilus"], + 'ses7': ROLES_DEFAULT["octopus"], + } + + ROLES_KNOWN = [ + "admin", + "alertmanager", + "bootstrap", + "client", + "ganesha", # deprecated (replaced by "nfs") + "grafana", + "igw", + "loadbalancer", + "makecheck", + "master", + "mds", + "mgr", + "mon", + "nfs", + "node-exporter", + "openattic", + "prometheus", + "rgw", + "storage", + "suma", + "worker", + ] + + ROLES_SINGLE_NODE = { + "luminous": "[ master, storage, mon, mgr, mds, igw, rgw, nfs, openattic ]", + "nautilus": "[ master, storage, mon, mgr, prometheus, grafana, mds, igw, rgw, " + "nfs ]", + "octopus": "[ master, bootstrap, storage, mon, mgr, mds, igw, rgw, nfs, " + "prometheus, grafana, alertmanager, node-exporter ]", + } SSH_KEY_NAME = 'sesdev' # do NOT use 'id_rsa' @@ -197,17 +204,6 @@ VERBOSE = None - VERSION_DEFAULT_ROLES = { - 'caasp4': DEFAULT_ROLES["caasp4"], - 'makecheck': DEFAULT_ROLES["makecheck"], - 'nautilus': DEFAULT_ROLES["nautilus"], - 'octopus': DEFAULT_ROLES["octopus"], - 'pacific': DEFAULT_ROLES["octopus"], - 'ses5': DEFAULT_ROLES["luminous"], - 'ses6': DEFAULT_ROLES["nautilus"], - 'ses7': DEFAULT_ROLES["octopus"], - } - VERSION_DEVEL_REPOS = { 'ses5': { 'sles-12-sp3': [ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/deployment.py new/sesdev-1.6.1+1597147109.g204bf79/seslib/deployment.py --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/deployment.py 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/deployment.py 2020-08-11 13:58:29.347702885 +0200 @@ -70,6 +70,7 @@ class Deployment(): + def __init__(self, dep_id, settings, existing=False): Log.info("Instantiating deployment {}".format(dep_id)) if existing: @@ -82,7 +83,7 @@ self.node_counts = {} self.nodes_with_role = {} self.roles_of_nodes = {} - for role in Constant.KNOWN_ROLES: + for role in Constant.ROLES_KNOWN: self.node_counts[role] = 0 self.nodes_with_role[role] = [] Log.debug("Deployment ctor: node_counts: {}".format(self.node_counts)) @@ -114,7 +115,7 @@ if self.settings.version == 'makecheck': self.settings.override('single_node', True) - self.settings.override('roles', Constant.VERSION_DEFAULT_ROLES['makecheck']) + self.settings.override('roles', Constant.ROLES_DEFAULT_BY_VERSION['makecheck']) if not self.settings.explicit_num_disks: self.settings.override('num_disks', 0) self.settings.override('explicit_num_disks', True) @@ -191,9 +192,9 @@ .format(self.settings.roles)) for node_roles in self.settings.roles: # loop once for every node in cluster for role in node_roles: - if role not in Constant.KNOWN_ROLES: + if role not in Constant.ROLES_KNOWN: raise RoleNotKnown(role) - for role_type in Constant.KNOWN_ROLES: + for role_type in Constant.ROLES_KNOWN: if role_type in node_roles: self.node_counts[role_type] += 1 @@ -844,7 +845,7 @@ raise RoleNotSupported('loadbalancer', self.settings.version) # no node may have more than one of any role for node in self.settings.roles: - for role in Constant.KNOWN_ROLES: + for role in Constant.ROLES_KNOWN: if node.count(role) > 1: raise DuplicateRolesNotSupported(role) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/node.py new/sesdev-1.6.1+1597147109.g204bf79/seslib/node.py --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/node.py 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/node.py 2020-08-11 13:58:29.347702885 +0200 @@ -32,7 +32,7 @@ self.custom_repos = [] def has_role(self, role): - if role not in Constant.KNOWN_ROLES: + if role not in Constant.ROLES_KNOWN: raise RoleNotKnown(role) return role in self.roles @@ -43,7 +43,7 @@ def has_exclusive_role(self, role): Log.debug("Node {}: has_exclusive_role: self.roles: {}" .format(self.fqdn, self.roles)) - if role not in Constant.KNOWN_ROLES: + if role not in Constant.ROLES_KNOWN: raise RoleNotKnown(role) return self.roles == [role] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/settings.py new/sesdev-1.6.1+1597147109.g204bf79/seslib/settings.py --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/settings.py 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/settings.py 2020-08-11 13:58:29.347702885 +0200 @@ -278,7 +278,7 @@ 'version_default_roles': { 'type': dict, 'help': 'Default roles for each node - one set of default roles per deployment version', - 'default': Constant.VERSION_DEFAULT_ROLES, + 'default': Constant.ROLES_DEFAULT_BY_VERSION, }, 'version_devel_repos': { 'type': dict, @@ -366,9 +366,9 @@ Log.debug("_load_config_file: config_tree: {}".format(config_tree)) assert isinstance(config_tree, dict), "yaml.load() of config file misbehaved!" __fill_in_config_tree('os_repos', Constant.OS_REPOS) - __fill_in_config_tree('version_os_repo_mapping', Constant.VERSION_OS_REPO_MAPPING) + __fill_in_config_tree('version_devel_repos', Constant.VERSION_DEVEL_REPOS) __fill_in_config_tree('image_paths', Constant.IMAGE_PATHS) - __fill_in_config_tree('version_default_roles', Constant.VERSION_DEFAULT_ROLES) + __fill_in_config_tree('version_default_roles', Constant.ROLES_DEFAULT_BY_VERSION) return config_tree diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/caasp/master.sh.j2 new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/caasp/master.sh.j2 --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/caasp/master.sh.j2 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/caasp/master.sh.j2 2020-08-11 13:58:29.347702885 +0200 @@ -43,7 +43,12 @@ mkdir -p ~/cluster cd ~/cluster +{% if node_manager.get_one_by_role('loadbalancer') %} skuba -v ${SKUBA_VERBOSITY} cluster init --control-plane {{ node_manager.get_one_by_role('loadbalancer').name }} caasp4-cluster +{% else %} +skuba -v ${SKUBA_VERBOSITY} cluster init --control-plane {{ node_manager.get_one_by_role('master').name }} caasp4-cluster +{% endif %} + chmod g+rx caasp4-cluster cd caasp4-cluster skuba -v ${SKUBA_VERBOSITY} node bootstrap --user sles --sudo --target {{ node.name }} {{ node.name }} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/makecheck/provision.sh.j2 new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/makecheck/provision.sh.j2 --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/makecheck/provision.sh.j2 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/makecheck/provision.sh.j2 2020-08-11 13:58:29.351702880 +0200 @@ -30,20 +30,31 @@ EOF {% if makecheck_stop_before_git_clone %} +set +x +echo "Stopping the deployment now because --stop-before-git-clone option was given." +set -x exit 0 -{% endif %} +{% endif %} {# makecheck_stop_before_git_clone #} + {% if os == "sles-12-sp3" %} su {{ makecheck_username }} -c 'git config --global http.sslVerify false' {% endif %} su {{ makecheck_username }} -c 'git clone --branch {{ makecheck_ceph_branch }} --progress {{ makecheck_ceph_repo }} /home/{{ makecheck_username }}/ceph' {% if makecheck_stop_before_install_deps %} +set +x +echo "Stopping the deployment now because --stop-before-install-deps option was given." +set -x exit 0 -{% endif %} +{% endif %} {# makecheck_stop_before_install_deps #} su {{ makecheck_username }} -c 'cd /home/{{ makecheck_username }}/ceph ; sed -i -e "s/^set -e$/set -ex/" install-deps.sh ; FOR_MAKE_CHECK=true ./install-deps.sh' {% if makecheck_stop_before_run_make_check %} +set +x +echo "Stopping the deployment now because --stop-before-run-make-check option was given." +set -x exit 0 -{% endif %} +{% endif %} {# makecheck_stop_before_run_make_check #} + su {{ makecheck_username }} -c 'cd /home/{{ makecheck_username }}/ceph ; sed -i -e "s/^set -e$/set -ex/" run-make-check.sh ; ./run-make-check.sh' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/salt/ceph-salt/ceph_salt_deployment.sh.j2 new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/salt/ceph-salt/ceph_salt_deployment.sh.j2 --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/salt/ceph-salt/ceph_salt_deployment.sh.j2 2020-07-26 12:54:20.022212776 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/salt/ceph-salt/ceph_salt_deployment.sh.j2 2020-08-11 13:58:29.351702880 +0200 @@ -38,8 +38,11 @@ {% endif %} {% if stop_before_ceph_salt_config %} +set +x +echo "Stopping the deployment now because --stop-before-ceph-salt-config option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_ceph_salt_config #} echo "PATH is $PATH" type ceph-salt @@ -83,6 +86,9 @@ ceph-salt --version {% if stop_before_ceph_salt_apply %} +set +x +echo "Stopping the deployment now because --stop-before-ceph-salt-apply option was given." +set -x exit 0 {% endif %} {# stop_before_ceph_salt_apply #} @@ -92,6 +98,13 @@ stdbuf -o0 ceph-salt -ldebug apply --non-interactive {% endif %} +{% if stop_before_ceph_orch_apply %} +set +x +echo "Stopping the deployment now because --stop-before-ceph-orch-apply option was given." +set -x +exit 0 +{% endif %} {# stop_before_ceph_orch_apply #} + {% set service_spec_core = "/root/service_spec_core.yml" %} rm -f {{ service_spec_core }} touch {{ service_spec_core }} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/salt/deepsea/deepsea_deployment.sh.j2 new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/salt/deepsea/deepsea_deployment.sh.j2 --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/templates/salt/deepsea/deepsea_deployment.sh.j2 2020-07-26 12:54:20.026212803 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/templates/salt/deepsea/deepsea_deployment.sh.j2 2020-08-11 13:58:29.351702880 +0200 @@ -34,8 +34,11 @@ {% endif %} {% if stop_before_stage == 0 %} +set +x +echo "Stopping the deployment now because --stop-before-deepsea-stage=0 option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_stage == 0 #} echo "PATH is $PATH" {% if use_salt %} @@ -56,8 +59,11 @@ {% include "salt/deepsea/" + version + "_pre_stage_1.sh.j2" ignore missing %} {% if stop_before_stage == 1 %} +set +x +echo "Stopping the deployment now because --stop-before-deepsea-stage=1 option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_stage == 1 #} echo "" echo "***** RUNNING stage.1 *******" @@ -71,8 +77,11 @@ {% include "salt/deepsea/" + version + "_pre_stage_2.sh.j2" ignore missing %} {% if stop_before_stage == 2 %} +set +x +echo "Stopping the deployment now because --stop-before-deepsea-stage=2 option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_stage == 2 #} echo "" echo "***** RUNNING stage.2 *******" @@ -86,8 +95,11 @@ {% include "salt/deepsea/" + version + "_pre_stage_3.sh.j2" ignore missing %} {% if stop_before_stage == 3 %} +set +x +echo "Stopping the deployment now because --stop-before-deepsea-stage=3 option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_stage == 3 #} {% if version != 'ses5' %} {% set drive_groups_yml = "/srv/salt/ceph/configuration/files/drive_groups.yml" %} @@ -131,8 +143,11 @@ {% include "salt/deepsea/" + version + "_pre_stage_4.sh.j2" ignore missing %} {% if stop_before_stage == 4 %} +set +x +echo "Stopping the deployment now because --stop-before-deepsea-stage=4 option was given." +set -x exit 0 -{% endif %} +{% endif %} {# stop_before_stage == 4 #} {% if deepsea_need_stage_4 %} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/seslib/tools.py new/sesdev-1.6.1+1597147109.g204bf79/seslib/tools.py --- old/sesdev-1.6.0+1595760860.gc353ea1/seslib/tools.py 2020-07-26 12:54:20.026212803 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/seslib/tools.py 2020-08-11 13:58:29.351702880 +0200 @@ -15,7 +15,7 @@ def run_sync(command, cwd=None): - Log.debug("Running command in directory {}: {}".format( + Log.info("Running command in directory {}: {}".format( cwd if cwd else ".", command)) with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) as proc: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/sesdev-1.6.0+1595760860.gc353ea1/setup.cfg new/sesdev-1.6.1+1597147109.g204bf79/setup.cfg --- old/sesdev-1.6.0+1595760860.gc353ea1/setup.cfg 2020-07-26 12:54:20.026212803 +0200 +++ new/sesdev-1.6.1+1597147109.g204bf79/setup.cfg 2020-08-11 13:58:29.351702880 +0200 @@ -36,11 +36,13 @@ [options.package_data] seslib = templates/*.j2 - templates/deepsea/*.j2 templates/caasp/*.j2 - templates/ceph-salt/*.j2 - templates/suma/*.j2 templates/engine/libvirt/*.j2 + templates/makecheck/*.j2 + templates/salt/*.j2 + templates/salt/ceph-salt/*.j2 + templates/salt/deepsea/*.j2 + templates/salt/suma/*.j2 [options.entry_points] console_scripts =