Thcipriani has uploaded a new change for review.
https://gerrit.wikimedia.org/r/241066
Change subject: [WIP] Add optional canary deploy group and check
......................................................................
[WIP] Add optional canary deploy group and check
Adds optional `canary_dsh_targets` config variable that behaves the same
as `dsh_targets` (if it's an abolute path, use it, otherwise look for
`/etc/dsh/groups/[file]`).
Adds a `canary_check` stage to `DeployLocal` that is yet to be
implemented, but happens post canary-fetch, -promote, and -check and can be
used for extended canary checks.
Currently, after canary deploy, it waits for input from user; however,
idealy, `cannary_check` could continue to run while waiting for user input.
Bug: T113073
Change-Id: I7d1235197356201626a88056fefa344ddf92d2dd
---
M scap/main.py
1 file changed, 72 insertions(+), 15 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/66/241066/1
diff --git a/scap/main.py b/scap/main.py
index e707533..548a536 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -579,7 +579,7 @@
class DeployLocal(cli.Application):
"""Deploy service code via git"""
STAGES = ['fetch', 'promote', 'check']
- EX_STAGES = ['rollback']
+ EX_STAGES = ['rollback', 'canary_check']
rev = None
cache_dir = None
@@ -724,6 +724,10 @@
self.promote()
self._remove_progress_link()
+ def canary_check(self):
+ """Extended checks, only preformed on canary deployment machines"""
+ pass
+
def _link_rev_dir(self, symlink_path):
tasks.move_symlink(self.rev_dir, symlink_path, user=self.user)
@@ -788,11 +792,29 @@
utils.read_hosts_file(self.config['dsh_targets'])
)
- logger.info(
- 'Deploy will run on the following targets: \n\t- {}'.format(
- '\n\t- '.join(self.targets)
- )
- )
+ self.canary_targets = utils.read_hosts_file(
+ self.config.get('canary_dsh_targets', '/dev/null'))
+
+ self.all_targets = self.canary_targets + self.targets
+
+ # Remove any canary targets from other targets
+ self.targets = list(set(self.targets) - set(self.canary_targets))
+
+ if not len(self.all_targets):
+ logger.warn('No targets selected, check limits and dsh_targets')
+ return 1
+
+ canary_target_msg = ''
+ if len(self.canary_targets):
+ canary_target_msg = '\n== CANARIES ==\n:*{}'.format(
+ '\n:* '.join(self.canary_targets))
+
+ target_msg = ''
+ if len(self.targets):
+ target_msg = '\n\n== TARGETS ==\n:* {}\n'.format(
+ '\n:* '.join(self.targets))
+
+ logger.info('{}{}'.format(canary_target_msg, target_msg))
with utils.lock(self.config['lock_file']):
with log.Timer('deploy_' + self.repo):
@@ -817,26 +839,63 @@
# apache server
tasks.git_update_server_info(self.config['git_submodules'])
- for stage in stages:
- ret = self.execute_stage(stage)
- if ret > 0:
- self.execute_rollback(stage)
- return ret
+ deploy_groups = [
+ {'name': 'Canary',
+ 'targets': self.canary_targets,
+ 'stages': stages + ['canary_check'],
+ 'check': True},
+ {'name': 'Normal',
+ 'targets': self.targets,
+ 'stages': stages,
+ 'check': False},
+ ]
+
+ return self._execute_for_groups(deploy_groups)
+ return 0
+
+ def _execute_for_groups(self, deploy_groups):
+ logger = self.get_logger()
+ for group in deploy_groups:
+ if not len(group['targets']):
+ continue
+
+ logger.info('== {} deploy group =='.format(group['name']))
+
+ ret = self._execute_stages(group['stages'], group['targets'])
+ if ret != 0:
+ return ret
+
+ prompt = '{} deploy successful. Continue?'.format(
+ group['name'])
+
+ if group['check'] and utils.ask(prompt, 'y') != 'y':
+ break
return 0
+
+ def _execute_stages(self, stages, targets):
+ for stage in stages:
+ job = ssh.Job(hosts=targets, user=self.config['ssh_user'])
+ ret = self.execute_stage(stage, job)
+ if ret > 0:
+ self.execute_rollback(stage)
+ return ret
+ return ret
def execute_rollback(self, stage):
prompt = "Stage '{}' failed. Perform rollback?".format(stage)
+ job = ssh.Job(hosts=self.all_targets, user=self.config['ssh_user'])
if utils.ask(prompt, 'y') == 'y':
- return self.execute_stage('rollback')
+ return self.execute_stage('rollback', job)
return 0
- def execute_stage(self, stage):
+ def execute_stage(self, stage, job):
logger = self.get_logger()
deploy_local_cmd = [self.get_script_path('deploy-local')]
batch_size = self._get_batch_size(stage)
+ deploy_stage = job
deploy_local_cmd.extend([
"-D '{}:{}'".format(x, self.config.get(x))
@@ -846,8 +905,6 @@
deploy_stage_cmd = deploy_local_cmd + [stage]
logger.debug('Running cmd {}'.format(deploy_stage_cmd))
- deploy_stage = ssh.Job(
- hosts=self.targets, user=self.config['ssh_user'])
deploy_stage.max_failure = self.MAX_FAILURES
deploy_stage.command(deploy_stage_cmd)
deploy_stage.progress('deploy_{}_{}'.format(self.repo, stage))
--
To view, visit https://gerrit.wikimedia.org/r/241066
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7d1235197356201626a88056fefa344ddf92d2dd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <tcipri...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
