Ryan Lane has uploaded a new change for review.
https://gerrit.wikimedia.org/r/94680
Change subject: Add shadow_reference support to trebuchet
......................................................................
Add shadow_reference support to trebuchet
This change adds a new boolean configuration option for repos
named shadow_reference. When true, this option will create and
maintain a git reference clone of this repo in a dot directory.
For instance, if a repo has a name test/testrepo, this option will
create a git reference repo at test/.testrepo.
Though this change uses a reference repo for the primary repo, all
submodules will be full clones from the repo on the filesystem.
Change-Id: I2737ce42527c84c05167b3dcc96b5638c8732852
---
M modules/deployment/files/modules/deploy.py
1 file changed, 127 insertions(+), 64 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/80/94680/1
diff --git a/modules/deployment/files/modules/deploy.py
b/modules/deployment/files/modules/deploy.py
index f814568..72fa6a4 100644
--- a/modules/deployment/files/modules/deploy.py
+++ b/modules/deployment/files/modules/deploy.py
@@ -50,9 +50,19 @@
config = __pillar__.get('repo_config')
config = config[repo]
config.setdefault('type', 'git-http')
- if 'location' not in config:
- repoloc = '{0}/{1}'.format(deployment_config['parent_dir'], repo)
- config['location'] = repoloc
+ if 'location' in config:
+ location = config['location']
+ shadow_location = '{0}/.{1}'.format(os.path.dirname(location),
+ os.path.basename(location))
+ config['shadow_location'] = shadow_location
+ else:
+ location = '{0}/{1}'.format(deployment_config['parent_dir'], repo)
+ config['location'] = location
+ shadow_repo = '{0}/.{1}'.format(os.path.dirname(repo),
+ os.path.basename(repo))
+ shadow_location = '{0}/{1}'.format(deployment_config['parent_dir'],
+ shadow_repo)
+ config['shadow_location'] = shadow_location
site = __grains__.get('site')
server = deployment_config['servers'][site]
#TODO: fetch scheme/url from implementation
@@ -74,6 +84,7 @@
config.setdefault('fetch_module_calls', {})
config.setdefault('sync_script', 'shared.py')
config.setdefault('upstream', None)
+ config.setdefault('shadow_reference', False)
return config
@@ -144,6 +155,27 @@
return {'status': status, 'stats': stats}
+def _update_gitmodules(config, location, shadow=False)
+ gitmodules = '{0}/.gitmodules'.format(location)
+
+ cmd = '/usr/bin/git checkout .gitmodules'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # Transform .gitmodules file based on defined seds
+ for before, after in config['submodule_sed_regex'].items():
+ if shadow:
+ # When fetching a shadow reference, we want to use the repo's
+ # filesystem location rather than the deployment url.
+ repo_url = config['location']
+ else:
+ repo_url = config['url']
+ after = after.replace('__REPO_URL__', repo_url)
+ __salt__['file.sed'](gitmodules, before, after)
+ return 0
+
+
def fetch(repo):
'''
Call a fetch for the specified repo
@@ -153,7 +185,6 @@
salt -G 'cluster:appservers' deploy.fetch 'slot0'
'''
config = get_config(repo)
- gitmodules = config['location'] + '/.gitmodules'
depstats = []
for dependency in config['dependencies']:
@@ -173,47 +204,22 @@
cmd = '/usr/bin/git remote set-url origin %s' % config['url'] + "/.git"
__salt__['cmd.retcode'](cmd, config['location'])
- cmd = '/usr/bin/git fetch'
- status = __salt__['cmd.retcode'](cmd, config['location'])
+ status = _fetch_location(config, config['location'])
if status != 0:
- return {'status': 10, 'repo': repo, 'dependencies': depstats}
-
- cmd = '/usr/bin/git fetch --tags'
- status = __salt__['cmd.retcode'](cmd, config['location'])
- if status != 0:
- return {'status': 20, 'repo': repo, 'dependencies': depstats}
-
- if config['checkout_submodules']:
- cmd = '/usr/bin/git checkout .gitmodules'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 30, 'repo': repo, 'dependencies': depstats}
- # Transform .gitmodules file based on defined seds
- for before, after in config['submodule_sed_regex'].items():
- after = after.replace('__REPO_URL__', config['url'])
- __salt__['file.sed'](gitmodules, before, after)
-
- # Sync the .gitmodules config
- cmd = '/usr/bin/git submodule sync'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 40, 'repo': repo, 'dependencies': depstats}
-
- # fetch all submodules and tag for submodules
- cmd = '/usr/bin/git submodule foreach git fetch'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 50, 'repo': repo, 'dependencies': depstats}
-
- # fetch all submodules and tag for submodules
- cmd = '/usr/bin/git submodule foreach git fetch --tags'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 60, 'repo': repo, 'dependencies': depstats}
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
cmd = '/usr/bin/git describe --always --tag origin'
origin_tag = __salt__['cmd.run'](cmd, config['location'])
origin_tag = origin_tag.strip()
+
+ if config['shadow_reference']:
+ status = _fetch_shadow(config)
+ if status != 0:
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
+ status = _checkout_location(config, config['shadow_location'],
+ shadow=True)
+ if status != 0:
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
# Call modules on the repo's behalf ignore the return on these
for call, args in config['fetch_module_calls'].items():
@@ -221,6 +227,57 @@
__salt__[call](*mapped_args)
return {'status': status, 'repo': repo,
'dependencies': depstats, 'tag': origin_tag}
+
+
+def _fetch_shadow(config):
+ # Clone the repo if it doesn't exist yet
+ gitdir = '{0}/.git'.format(config['shadow_location'])
+ if not __salt__['file.directory_exists'](gitdir):
+ cmd = '/usr/bin/git clone --reference {0} {1} {2}'
+ cmd = cmd.format(config['location'], config['url'] + '/.git',
+ config['shadow_location'])
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ return _fetch_location(config, config['shadow_location'], shadow=True)
+
+
+def _fetch_location(config, location, shadow=False):
+ cmd = '/usr/bin/git fetch'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ cmd = '/usr/bin/git fetch --tags'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # TODO: update .gitmodules recursively, then run submodule commands
+ # recursively.
+ if config['checkout_submodules']:
+ ret = _update_gitmodules(config, location, shadow)
+ if ret != 0:
+ return ret
+
+ # Sync the .gitmodules config
+ cmd = '/usr/bin/git submodule sync'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # fetch all submodules and tag for submodules
+ cmd = '/usr/bin/git submodule foreach git fetch'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # fetch all submodules and tag for submodules
+ cmd = '/usr/bin/git submodule foreach git fetch --tags'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+ return 0
def checkout(repo, reset=False):
@@ -234,12 +291,25 @@
#TODO: replace the cmd.retcode calls with git module calls,
# where appropriate
config = get_config(repo)
- gitmodules = config['location'] + '/.gitmodules'
depstats = []
# Notify the deployment system we started
_check_in('deploy.checkout', repo)
+ status = _checkout_location(config, config['location'])
+ if status != 0:
+ return {'status': status, 'repo': repo, 'tag': tag,
+ 'dependencies': depstats}
+
+ # Call modules on the repo's behalf ignore the return on these
+ for call, args in config['checkout_module_calls'].items():
+ mapped_args = _map_args(repo, args)
+ __salt__[call](*mapped_args)
+ return {'status': status, 'repo': repo, 'tag': tag,
+ 'dependencies': depstats}
+
+
+def _checkout_location(config, location, shadow=False):
# Fetch the .deploy file from the server and get the current tag
deployfile = config['url'] + '/.deploy'
f = urllib.urlopen(deployfile)
@@ -250,11 +320,11 @@
tag = info[5:]
tag = tag.strip()
if not tag:
- return {'status': 10, 'repo': repo, 'dependencies': depstats}
+ return 10
# tags are user-input and are used in shell commands, ensure they are
# only passing alphanumeric.
if re.match('\W+', tag):
- return {'status': 1, 'repo': repo, 'dependencies': depstats}
+ return 1
for dependency in config['dependencies']:
depstats.append(__salt__['deploy.checkout'](dependency, reset))
@@ -262,43 +332,36 @@
if reset:
# User requested we hard reset the repo to the tag
cmd = '/usr/bin/git reset --hard tags/%s' % (tag)
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 20, 'repo': repo, 'dependencies': depstats}
+ return 20
else:
cmd = '/usr/bin/git describe --always --tag'
- current_tag = __salt__['cmd.run'](cmd, config['location'])
+ current_tag = __salt__['cmd.run'](cmd, location)
current_tag = current_tag.strip()
if current_tag == tag:
- return {'status': 0, 'repo': repo,
- 'tag': tag, 'dependencies': depstats}
+ return 0
# Switch to the tag defined in the server's .deploy file
cmd = '/usr/bin/git checkout --force --quiet tags/%s' % (tag)
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 30, 'repo': repo, 'dependencies': depstats}
+ return 30
if config['checkout_submodules']:
- # Transform .gitmodules file based on defined seds
- for before, after in config['submodule_sed_regex'].items():
- after = after.replace('__REPO_URL__', config['url'])
- __salt__['file.sed'](gitmodules, before, after)
+ ret = _update_gitmodules(config, location, shadow)
+ if ret != 0:
+ return ret
# Sync the .gitmodules config
cmd = '/usr/bin/git submodule sync'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 40, 'repo': repo, 'dependencies': depstats}
+ return 40
# Update the submodules to match this tag
cmd = '/usr/bin/git submodule update --init'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 50, 'repo': repo, 'dependencies': depstats}
-
- # Call modules on the repo's behalf ignore the return on these
- for call, args in config['checkout_module_calls'].items():
- mapped_args = _map_args(repo, args)
- __salt__[call](*mapped_args)
- return {'status': 0, 'repo': repo, 'tag': tag, 'dependencies': depstats}
+ return 50
+ return 0
--
To view, visit https://gerrit.wikimedia.org/r/94680
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2737ce42527c84c05167b3dcc96b5638c8732852
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
