Thcipriani has uploaded a new change for review.
https://gerrit.wikimedia.org/r/235385
Change subject: Add config deployment
......................................................................
Add config deployment
Adds a section inside the local scap/scap.cfg for configuration files to
be deployed, formatted like:
[config_files]
/path/to/dest_config.file:
template: file_in_the_template_dir.yaml.j2
vars_file: /path/to/vars_file.yaml
The deploy command renders a json file in a known location on tin that
is later fetched by each deploy target.
The deploy target loops through the config files and grabs the
`template` and renders it using the `vars_file` (which is put into place
by puppet on the target and contains puppet private repo information)
Current limitations:
- Files created are created by the ssh_user specified in the config
and therefore must be created in locations to which that user has
read access
Change-Id: I64aa464039c0471f156795a3042ea44d48c8622a
---
A bin/deploy-local-config
M scap/__init__.py
M scap/config.py
M scap/main.py
M scap/tasks.py
A scap/template.py
M scap/utils.py
7 files changed, 227 insertions(+), 22 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/85/235385/1
diff --git a/bin/deploy-local-config b/bin/deploy-local-config
new file mode 100755
index 0000000..f555f4b
--- /dev/null
+++ b/bin/deploy-local-config
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Deploy service to localhost.
+#
+# Copyright © 2015 Wikimedia Foundation and contributors
+
+import os
+import sys
+
+# Add scap package to search path
+script = os.path.realpath(sys.argv[0])
+scap_src = os.path.dirname(os.path.dirname(script))
+sys.path.append(scap_src)
+
+import scap
+scap.DeployLocalConfig.run()
diff --git a/scap/__init__.py b/scap/__init__.py
index 70baa76..4d68bc9 100644
--- a/scap/__init__.py
+++ b/scap/__init__.py
@@ -9,6 +9,7 @@
from .main import (
CompileWikiversions,
DeployLocal,
+ DeployLocalConfig,
Deploy,
HHVMGracefulAll,
MWVersionsInUse,
@@ -28,6 +29,7 @@
__all__ = (
'CompileWikiversions',
'DeployLocal',
+ 'DeployLocalConfig',
'Deploy',
'HHVMGracefulAll',
'MWVersionsInUse',
@@ -47,6 +49,7 @@
any((
CompileWikiversions,
DeployLocal,
+ DeployLocalConfig,
Deploy,
HHVMGracefulAll,
MWVersionsInUse,
diff --git a/scap/config.py b/scap/config.py
index 3ef9463..340e7bb 100644
--- a/scap/config.py
+++ b/scap/config.py
@@ -9,6 +9,9 @@
import getpass
import os
import socket
+import yaml
+
+from . import utils
DEFAULT_CONFIG = {
@@ -35,6 +38,7 @@
'git_server': 'tin.eqiad.wmnet',
'git_scheme': 'http',
'git_submodules': False,
+ 'config_deploy': False,
}
@@ -63,6 +67,8 @@
:param overrides: Dict of configuration values
:returns: dict of configuration values
"""
+ local_cfg = os.path.join(os.getcwd(), 'scap')
+
parser = ConfigParser.SafeConfigParser()
if cfg_file:
try:
@@ -77,7 +83,7 @@
os.path.join(os.path.dirname(__file__), '..', 'scap.cfg'),
'/srv/scap/scap.cfg',
'/etc/scap.cfg',
- os.path.join(os.getcwd(), 'scap.cfg'),
+ os.path.join(local_cfg, 'scap.cfg'),
])
fqdn = socket.getfqdn().split('.')
@@ -95,4 +101,23 @@
if overrides:
config.update(overrides)
+ # Grab the config file
+ # config file section example:
+ # [config_files]
+ # /path/to/dest_config.file:
+ # template: file_in_the_template_dir.yaml.j2
+ # vars_file: /path/to/vars_file.yaml
+ if parser.has_section('config_files'):
+ config['config_files'] = []
+ for config_file in parser.items('config_files', True):
+ name = config_file[0]
+ info = yaml.load(config_file[1])
+ template = utils.get_realm_specific_filename(
+ os.path.join(local_cfg, 'templates', info['template']),
+ config['wmf_realm'],
+ config['datacenter']
+ )
+ info['template'] = template
+ config['config_files'].append({'name': name, 'info': info})
+
return config
diff --git a/scap/main.py b/scap/main.py
index fb6e37d..e6f4f86 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -11,12 +11,14 @@
import netifaces
import os
import psutil
+import requests
import subprocess
from . import cli
from . import log
from . import ssh
from . import tasks
+from . import template
from . import utils
@@ -575,8 +577,51 @@
return exit_code
+class DeployLocalConfig(cli.Application):
+ """Creates config files
+ #. Pull config_files.json (generated during deploy on tin)
+ #. Loop through config files and render template from variables in
+ final location
+ """
+
+ def main(self, *extra_args):
+ """Looks only in the scap directory for the repo being deployed
+ """
+ repo = self.config['git_repo']
+ server = self.config['git_server']
+ deploy_dir = self.config['git_deploy_dir']
+
+ location = os.path.normpath("{0}/{1}/scap".format(server, repo))
+ http_fmt = 'http://{}'
+ scap_config_url = http_fmt.format(location)
+
+ r = requests.get('{}/config_files.json'.format(scap_config_url))
+ config_files = r.json()
+ for config_file in config_files:
+ info = config_file['info']
+
+ name = info['template']
+ if os.path.commonprefix([name, deploy_dir]) == deploy_dir:
+ name = os.path.relpath(name, deploy_dir)
+
+ tmpl = template.Template(
+ name=name,
+ location=http_fmt.format(server),
+ var_file=info['vars_file']
+ )
+
+ with open(config_file['name'], 'w') as f:
+ f.write(tmpl.render())
+
+
class DeployLocal(cli.Application):
- """Deploy service code via git"""
+ """Deploy service locally
+
+ #. Deploy service configuration if specified
+ #. Deploy service code
+ #. Restart service
+ #. Check service port
+ """
def main(self, *extra_args):
repo = self.config['git_repo']
@@ -584,6 +629,7 @@
server = self.config['git_server']
rev = self.config['git_rev']
has_submodules = self.config['git_submodules']
+ config_deploy = self.config['config_deploy']
# service_name/service_port only set for deploys requiring restart
service_name = self.config.get('service_name', None)
@@ -595,9 +641,17 @@
# only supports http from tin for the moment
scheme = 'http'
- url = os.path.normpath('{0}/{1}'.format(server, repo))
- url = "{0}://{1}/.git".format(scheme, url)
+ base_url = os.path.normpath('{0}/{1}'.format(server, repo))
+ url = "{0}://{1}/.git".format(scheme, base_url)
self.get_logger().debug('Fetching from: {}'.format(url))
+
+ deploy_local_config = ' '.join(utils.pass_config(
+ self.get_script_path('deploy-local-config'),
+ self.config,
+ ))
+
+ if config_deploy:
+ utils.sudo_check_call(repo_user, deploy_local_config)
tasks.git_fetch(location, url, repo_user)
tasks.git_checkout(location, rev, has_submodules, repo_user)
@@ -655,25 +709,20 @@
# apache server
tasks.git_update_server_info(self.config['git_submodules'])
- deploy_conf = [
- 'git_deploy_dir',
- 'git_repo_user',
- 'git_server',
- 'git_scheme',
- 'git_repo',
- 'git_rev',
- 'git_submodules',
- 'service_name',
- 'service_port',
- ]
+ if self.config['config_deploy']:
+ config_file_info = os.path.join(os.getcwd(),
+ 'scap',
+ 'config_files.json')
- deploy_local_cmd = [self.get_script_path('deploy-local')]
+ tasks.config_deploy_setup(
+ cfg=self.config,
+ tmp_file=config_file_info
+ )
- deploy_local_cmd.extend([
- "-D '{}:{}'".format(x, self.config.get(x))
- for x in deploy_conf
- if self.config.get(x) is not None
- ])
+ deploy_local_cmd = utils.pass_config(
+ self.get_script_path('deploy-local'),
+ self.config
+ )
logger.debug('Running cmd {}'.format(deploy_local_cmd))
deploy_rev = ssh.Job(
diff --git a/scap/tasks.py b/scap/tasks.py
index a7fee9d..3c9c741 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -17,6 +17,8 @@
import subprocess
import time
+import requests
+
from . import cdblib
from . import log
from . import ssh
@@ -678,6 +680,19 @@
subprocess.check_call(cmd, shell=True)
+def config_deploy_setup(cfg, tmp_file):
+ """Creates json files that are used by deploy targets to compile and
+ write config files
+ """
+ logger = logging.getLogger('config_deploy_setup')
+ if not cfg['config_deploy'] or not cfg.get('config_files'):
+ return
+
+ logger.debug('Building config files')
+ with open(tmp_file, 'w') as f:
+ json.dump(cfg['config_files'], f)
+
+
def restart_service(service, user='mwdeploy'):
logger = logging.getLogger('service_restart')
diff --git a/scap/template.py b/scap/template.py
new file mode 100644
index 0000000..e2169e4
--- /dev/null
+++ b/scap/template.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+ scap.tmpl
+ ~~~~~~~~~~
+ Module for working with file templates
+"""
+
+import jinja2
+import requests
+import yaml
+
+
+class ScapTemplateLoader(jinja2.BaseLoader):
+ """Overrides the jinja2 baseloader with one that will fetch template
+ files from tin
+ """
+ def __init__(self, location):
+ self.location = location
+
+ def get_source(self, environment, template):
+ """Grab a template file from the staging server"""
+ uri = '{}/{}'.format(self.location, template)
+ template = self._get_remote_file(uri)
+ if not template:
+ raise jinja2.TemplateNotFound(template)
+
+ source, etag = template
+ uptodate = self._uptodate(uri, etag)
+ return source, uri, uptodate
+
+ def _get_remote_file(self, uri):
+ """Helper method to return etag and response body"""
+ r = requests.get(uri)
+ if r.status_code != requests.codes.ok:
+ return None
+ return r.text, r.headers.get('etag', '')
+
+ def _uptodate(self, uri, etag):
+ """Returns a functions that takes no arguments that will check if
+ a given template is up-to-date
+ """
+ def uptodate():
+ import requests
+ r = requests.get(uri)
+ if r.status_code == requests.codes.ok:
+ return etag == r.headers.get('etag', '')
+ # If we got a non-200 response, assume the template we have is fine
+ return True
+ return uptodate
+
+
+class Template(object):
+ """Adapter class that wraps jinja2 templates
+ """
+ def __init__(self, name, location, var_file):
+ loader = ScapTemplateLoader(location)
+ self._env = jinja2.Environment(loader=loader)
+ self._template = self._env.get_template(name)
+
+ self.var_file = var_file
+
+ def render(self):
+ """Renders the templates specified by `self.name` using the
+ variables sourced from the import yaml file specified by
+ `self.var_file`
+ """
+ with open(self.var_file, 'r') as variables:
+ template_vars = yaml.load(variables.read())
+ return self._template.render(template_vars)
diff --git a/scap/utils.py b/scap/utils.py
index 6da0376..644d98b 100644
--- a/scap/utils.py
+++ b/scap/utils.py
@@ -543,7 +543,7 @@
return subprocess.check_output(cmd, shell=True).strip()
-def generate_json_tag(user=get_real_username(), location=os.getcwd()):
+def generate_json_tag(location, user=get_real_username()):
"""Generates a json object"""
timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
tag = '{0}-{1}-{2}'.format(location, 'sync', timestamp)
@@ -554,3 +554,30 @@
}
return (tag, tag_info)
+
+
+def pass_config(cmd, cfg={}):
+ """Takes command and returns array with service_deploy config vars set
+ """
+ cmd = [cmd]
+
+ deploy_conf = [
+ 'git_deploy_dir',
+ 'git_repo_user',
+ 'git_server',
+ 'git_scheme',
+ 'git_repo',
+ 'git_rev',
+ 'git_submodules',
+ 'service_name',
+ 'service_port',
+ 'config_deploy',
+ ]
+
+ cmd.extend([
+ "-D '{}:{}'".format(x, cfg.get(x))
+ for x in deploy_conf
+ if cfg.get(x)
+ ])
+
+ return cmd
--
To view, visit https://gerrit.wikimedia.org/r/235385
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I64aa464039c0471f156795a3042ea44d48c8622a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
