Thcipriani has uploaded a new change for review.
https://gerrit.wikimedia.org/r/240292
Change subject: Add config deployment
......................................................................
Add config deployment
What happens on tin
---
1. Looks for environment-specific `./scap/config-files.yaml` with target
files in the format:
/path/to/target:
template: env-specific-template.yaml.j2
remote_vars: /optional/remote/variable/file.yaml
2. Looks for environment-specific `./scap/vars.yaml` that includes
variables used to render the template. These variables will be overridden
by any conflicting variables in the file specified by `remote_vars`
3. Variables from any environment-specific `vars.yaml` file are combined
with variables from the root `vars.yaml` file.
4. A json file is created at `[repo]/.git/config-files/[revision].json`
that contains the final path to any environment-specific templates as
well as a final list of combined variables.
What happens on targets
---
1. Download the file from `tin/[repo]/.git/config-files/[revision].json`
2. Loop through the config files, and render each template using the
variables from the downloaded json file and the variables from the (now)
local remove_vars file
3. Links rendered file (in `[repo]/.git/config-files/[path]`) to final
location
Bug: T109512
Change-Id: I3a7c116aa90a0cc3e470931bfd7423634f075b19
---
M scap/config.py
M scap/main.py
M scap/tasks.py
A scap/template.py
M scap/utils.py
5 files changed, 258 insertions(+), 14 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/92/240292/1
diff --git a/scap/config.py b/scap/config.py
index b7a7d81..34d8f26 100644
--- a/scap/config.py
+++ b/scap/config.py
@@ -37,6 +37,7 @@
'git_server': 'tin.eqiad.wmnet',
'git_scheme': 'http',
'git_submodules': False,
+ 'config_deploy': False,
}
diff --git a/scap/main.py b/scap/main.py
index e707533..f2aab49 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -7,17 +7,22 @@
"""
import argparse
import errno
+import json
import multiprocessing
import netifaces
import os
import psutil
+import requests
import subprocess
+import yaml
from . import cli
from . import log
from . import ssh
from . import tasks
+from . import template
from . import utils
+
from datetime import datetime
@@ -578,7 +583,7 @@
class DeployLocal(cli.Application):
"""Deploy service code via git"""
- STAGES = ['fetch', 'promote', 'check']
+ STAGES = ['fetch', 'config_deploy', 'promote', 'check']
EX_STAGES = ['rollback']
rev = None
@@ -611,6 +616,15 @@
self.user = self.config['git_repo_user']
+ # only supports http from tin for the moment
+ scheme = 'http'
+ repo = self.config['git_repo']
+ server = self.config['git_server']
+
+ url = os.path.normpath('{0}/{1}'.format(server, repo))
+
+ self.server_url = "{0}://{1}".format(scheme, url)
+
getattr(self, self.arguments.stage)()
def fetch(self):
@@ -625,23 +639,17 @@
logger = self.get_logger()
- repo = self.config['git_repo']
- server = self.config['git_server']
has_submodules = self.config['git_submodules']
# create deployment directories if they don't already exist
for d in [self.cache_dir, self.revs_dir]:
utils.mkdir_p(d, logger=logger)
- # only supports http from tin for the moment
- scheme = 'http'
-
- url = os.path.normpath('{0}/{1}'.format(server, repo))
- url = "{0}://{1}/.git".format(scheme, url)
- self.get_logger().debug('Fetching from: {}'.format(url))
+ git_remote = os.path.join(self.server_url, '.git')
+ self.get_logger().debug('Fetching from: {}'.format(git_remote))
# clone/fetch from the repo to the cache directory
- tasks.git_fetch(self.cache_dir, url, user=self.user)
+ tasks.git_fetch(self.cache_dir, git_remote, user=self.user)
# clone/fetch from the local cache directory to the revision directory
tasks.git_fetch(self.rev_dir, self.cache_dir, user=self.user)
@@ -654,6 +662,61 @@
# link the .in-progress flag to the rev directory
self._link_rev_dir(self.progress_flag)
+ def config_deploy(self):
+ """Renders config files
+
+ Grabs the current config json file from the deploy git server, and
+ renders the final template inside the current revision's
+ `.git/config-files` directory
+ """
+ logger = self.get_logger()
+ if not self.config['config_deploy']:
+ return
+
+ deploy_dir = self.config['deploy_dir']
+ config_url = os.path.join(self.server_url, '.git', 'config-files',
+ '{}.json'.format(self.rev))
+
+ logger.debug('Get config json: {}'.format(config_url))
+ r = requests.get(config_url)
+ if r.status_code != requests.codes.ok:
+ raise IOError(errno.ENOENT, 'Config file not found', config_url)
+
+ config_files = r.json()
+ overrides = config_files.get('override_vars', {})
+
+ # `revs/[revision]/.git/config-files` directory to render config_files
+ source_basepath = os.path.join(self.rev_dir, '.git', 'config-files')
+ logger.debug('Source basepath: {}'.format(source_basepath))
+
+ for config_file in config_files['files']:
+ name = config_file['template']
+ if os.path.commonprefix([name, deploy_dir]) == deploy_dir:
+ name = os.path.relpath(name, deploy_dir)
+
+ tmpl = template.Template(
+ name=name,
+ location='{}://{}'.format(
+ self.config['git_scheme'],
+ self.config['git_server'],
+ ),
+ var_file=config_file.get('remote_vars', None),
+ overrides=overrides
+ )
+
+ filename = config_file['name']
+ if filename.startswith('/'):
+ filename = filename[1:]
+
+ utils.mkdir_p(os.path.join(
+ source_basepath, os.path.dirname(filename)))
+
+ source = os.path.join(source_basepath, filename)
+ logger.debug('Rendering config_file: {}'.format(source))
+
+ with open(source, 'w') as f:
+ f.write(tmpl.render())
+
def promote(self):
"""Promote the current deployment.
@@ -664,6 +727,7 @@
service = self.config.get('service_name', None)
self._link_rev_dir(self.cur_link)
+ self._link_config_files()
if service is not None:
tasks.restart_service(service, user=self.config['git_repo_user'])
@@ -724,6 +788,22 @@
self.promote()
self._remove_progress_link()
+ def _link_config_files(self):
+ """Links rendered config files to their final destination"""
+ logger = self.get_logger()
+
+ config_base = os.path.join(self.cur_link, '.git', 'config-files')
+ logger.debug('Linking config files at: {}'.format(config_base))
+
+ for dir_path, _, conf_files in os.walk(config_base):
+ for conf_file in conf_files:
+ full_path = os.path.normpath(
+ '{}/{}'.format(dir_path, conf_file))
+
+ rel_path = os.path.relpath(full_path, config_base)
+ final_path = os.path.join('/', rel_path)
+ tasks.move_symlink(full_path, final_path, user=self.user)
+
def _link_rev_dir(self, symlink_path):
tasks.move_symlink(self.rev_dir, symlink_path, user=self.user)
@@ -751,6 +831,7 @@
'git_submodules',
'service_name',
'service_port',
+ 'config_deploy',
]
repo = None
@@ -811,6 +892,8 @@
tasks.git_update_deploy_head(deploy_info, location=cwd)
tasks.git_tag_repo(deploy_info, location=cwd)
+ self.config_deploy_setup(commit)
+
self.config['git_rev'] = commit
# Run git update-server-info because git repo is a dumb
@@ -825,6 +908,80 @@
return 0
+ def config_deploy_setup(self, commit):
+ """Generate environment-specific config file and variable template list
+
+ Builds a json file that contains:
+ #. A list of file objects containing template files to be deployed
+ #. An object containing variables specificed in the
+ environment-specific `vars.yaml` file and inheriting from the
+ `vars.yaml` file
+ """
+ logger = self.get_logger()
+
+ if not self.config['config_deploy']:
+ return
+
+ logger.debug('Deploy config: True')
+ path_root = os.path.join(self.config['git_deploy_dir'], self.repo)
+ scap_path = os.path.join(path_root, 'scap')
+
+ cfg_file = utils.get_env_specific_filename(
+ os.path.join(scap_path, 'config-files.yaml'),
+ self.arguments.environment
+ )
+
+ logger.debug('Config deploy file: {}'.format(cfg_file))
+ if not os.path.isfile(cfg_file):
+ return
+
+ config_file_path = os.path.join(path_root, '.git', 'config-files')
+ utils.mkdir_p(config_file_path)
+ tmp_cfg_file = os.path.join(config_file_path, '{}.json'.format(commit))
+ tmp_cfg = {}
+
+ with open(cfg_file, 'r') as cf:
+ config_files = yaml.load(cf.read())
+
+ tmp_cfg['files'] = []
+ # Get an environment specific template
+ for config_file in config_files:
+ f = {}
+ f['name'] = config_file
+ template_name = config_files[config_file]['template']
+ template = utils.get_env_specific_filename(
+ os.path.join(scap_path, 'templates', template_name))
+ f['template'] = os.path.relpath(
+ template, self.config['git_deploy_dir'])
+
+ # Remote var file is optional
+ if config_files[config_file].get('remote_vars', None):
+ f['remote_vars'] = config_files[config_file]['remote_vars']
+
+ tmp_cfg['files'].append(f)
+
+ tmp_cfg['override_vars'] = {}
+
+ # Build vars to override remote
+ vars_files = [
+ utils.get_env_specific_filename(
+ 'vars.yaml',
+ self.arguments.environment
+ ),
+ os.path.join(os.getcwd(), 'scap/vars.yaml')
+ ]
+
+ for vars_file in vars_files:
+ try:
+ with open(vars_file, 'r') as vf:
+ tmp_cfg['override_vars'].update(yaml.load(vf.read()))
+ except IOError:
+ pass # don't worry if a vars.yaml doesn't exist
+
+ with open(tmp_cfg_file, 'w') as tc:
+ json.dump(tmp_cfg, tc)
+ logger.debug('Wrote config deploy file: {}'.format(tmp_cfg_file))
+
def execute_rollback(self, stage):
prompt = "Stage '{}' failed. Perform rollback?".format(stage)
diff --git a/scap/tasks.py b/scap/tasks.py
index 1dddd11..20e4394 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -703,11 +703,14 @@
def move_symlink(source, dest, user='mwdeploy', logger=None):
- common_path = os.path.commonprefix([source, dest])
- rsource = os.path.relpath(source, common_path)
- rdest = os.path.relpath(dest, common_path)
+ dest_dir = os.path.dirname(dest)
+ rsource = os.path.relpath(source, dest_dir)
+ rdest = os.path.relpath(dest, dest_dir)
- with utils.cd(common_path):
+ # Make link target's parent directory if it doesn't exist
+ utils.mkdir_p(dest_dir, user=user, logger=logger)
+
+ with utils.cd(dest_dir):
utils.sudo_check_call(user,
"ln -sfT '{}' '{}'".format(rsource, rdest),
logger)
diff --git a/scap/template.py b/scap/template.py
new file mode 100644
index 0000000..88bbd2b
--- /dev/null
+++ b/scap/template.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""
+ scap.template
+ ~~~~~~~~~~
+ Module for working with file templates
+"""
+
+import jinja2
+import requests
+import yaml
+
+
+class ScapTemplateLoader(jinja2.BaseLoader):
+ """Overrides the jinja2 baseloader with one that will fetch template
+ files from tin
+ """
+ def __init__(self, location):
+ self.location = location
+
+ def get_source(self, environment, template):
+ """Grab a template file from the staging server"""
+ uri = '{}/{}'.format(self.location, template)
+ template = self._get_remote_file(uri)
+ if not template:
+ raise jinja2.TemplateNotFound(template)
+
+ source, etag = template
+ uptodate = self._uptodate(uri, etag)
+ return source, uri, uptodate
+
+ def _get_remote_file(self, uri):
+ """Helper method to return etag and response body"""
+ r = requests.get(uri)
+ if r.status_code != requests.codes.ok:
+ return None
+ return r.text, r.headers.get('etag', '')
+
+ def _uptodate(self, uri, etag):
+ """Returns a functions that takes no arguments that will check if
+ a given template is up-to-date
+ """
+ def uptodate():
+ import requests
+ r = requests.get(uri)
+ if r.status_code == requests.codes.ok:
+ return etag == r.headers.get('etag', '')
+ # If we got a non-200 response, assume the template we have is fine
+ return True
+ return uptodate
+
+
+class Template(object):
+ """Adapter class that wraps jinja2 templates
+ """
+ def __init__(self, name, location, var_file=None, overrides=None):
+ loader = ScapTemplateLoader(location)
+ self._env = jinja2.Environment(loader=loader)
+ self._template = self._env.get_template(name)
+ self._overrides = overrides
+ self.var_file = var_file
+
+ def _get_file_vars(self):
+ if not self.var_file:
+ return {}
+
+ with open(self.var_file, 'r') as variables:
+ return yaml.load(variables.read())
+
+ def render(self):
+ """Renders the templates specified by `self.name` using the
+ variables sourced from the import yaml file specified by
+ `self.var_file`
+ """
+ template_vars = self._get_file_vars()
+ if self._overrides:
+ overrides = self._overrides
+ overrides.update(template_vars)
+ template_vars = overrides
+ return self._template.render(template_vars)
diff --git a/scap/utils.py b/scap/utils.py
index 77eccd2..0e8a95e 100644
--- a/scap/utils.py
+++ b/scap/utils.py
@@ -117,6 +117,10 @@
base = os.path.dirname(path)
filename = os.path.basename(path)
+ if base.endswith('/templates'):
+ base = os.path.dirname(base)
+ filename = os.path.join('templates', filename)
+
env_filename = os.path.join(base, 'environments', env, filename)
if os.path.isfile(env_filename):
--
To view, visit https://gerrit.wikimedia.org/r/240292
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3a7c116aa90a0cc3e470931bfd7423634f075b19
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
