Ryan Lane has submitted this change and it was merged.
Change subject: Add shadow_reference support to trebuchet
......................................................................
Add shadow_reference support to trebuchet
This change adds a new boolean configuration option for repos
named shadow_reference. When true, this option will create and
maintain a git reference clone of this repo in a dot directory.
For instance, if a repo has a name test/testrepo, this option will
create a git reference repo at test/.testrepo.
Though this change uses a reference repo for the primary repo, all
submodules will be full clones from the repo on the filesystem.
Change-Id: I2737ce42527c84c05167b3dcc96b5638c8732852
---
M modules/deployment/files/modules/deploy.py
1 file changed, 178 insertions(+), 89 deletions(-)
Approvals:
Ryan Lane: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/deployment/files/modules/deploy.py
b/modules/deployment/files/modules/deploy.py
index f814568..aff9441 100644
--- a/modules/deployment/files/modules/deploy.py
+++ b/modules/deployment/files/modules/deploy.py
@@ -50,9 +50,19 @@
config = __pillar__.get('repo_config')
config = config[repo]
config.setdefault('type', 'git-http')
- if 'location' not in config:
- repoloc = '{0}/{1}'.format(deployment_config['parent_dir'], repo)
- config['location'] = repoloc
+ if 'location' in config:
+ location = config['location']
+ shadow_location = '{0}/.{1}'.format(os.path.dirname(location),
+ os.path.basename(location))
+ config['shadow_location'] = shadow_location
+ else:
+ location = '{0}/{1}'.format(deployment_config['parent_dir'], repo)
+ config['location'] = location
+ shadow_repo = '{0}/.{1}'.format(os.path.dirname(repo),
+ os.path.basename(repo))
+ shadow_location = '{0}/{1}'.format(deployment_config['parent_dir'],
+ shadow_repo)
+ config['shadow_location'] = shadow_location
site = __grains__.get('site')
server = deployment_config['servers'][site]
#TODO: fetch scheme/url from implementation
@@ -74,6 +84,7 @@
config.setdefault('fetch_module_calls', {})
config.setdefault('sync_script', 'shared.py')
config.setdefault('upstream', None)
+ config.setdefault('shadow_reference', False)
return config
@@ -144,6 +155,46 @@
return {'status': status, 'stats': stats}
+def _update_gitmodules(config, location, shadow=False):
+ gitmodules = '{0}/.gitmodules'.format(location)
+
+ cmd = '/usr/bin/git checkout .gitmodules'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # Transform .gitmodules file based on defined seds
+ for before, after in config['submodule_sed_regex'].items():
+ if shadow:
+ # When fetching a shadow reference, we want to use the repo's
+ # filesystem location rather than the deployment url.
+ repo_url = config['location']
+ else:
+ repo_url = config['url']
+ after = after.replace('__REPO_URL__', repo_url)
+ __salt__['file.sed'](gitmodules, before, after)
+ return 0
+
+
+def _clone(config, location, tag, shadow=False):
+ if shadow:
+ cmd = '/usr/bin/git clone --reference {0} {1}/.git {2}'
+ cmd = cmd.format(config['location'], config['url'], location)
+ else:
+ cmd = '/usr/bin/git clone {0}/.git {1}'.format(config['url'], location)
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ status = _fetch_location(config, location, shadow=shadow)
+ if status != 0:
+ return status
+ status = _checkout_location(config, location, tag,
+ reset=True, shadow=shadow)
+ if status != 0:
+ return status
+ return 0
+
+
def fetch(repo):
'''
Call a fetch for the specified repo
@@ -153,7 +204,6 @@
salt -G 'cluster:appservers' deploy.fetch 'slot0'
'''
config = get_config(repo)
- gitmodules = config['location'] + '/.gitmodules'
depstats = []
for dependency in config['dependencies']:
@@ -162,65 +212,109 @@
# Notify the deployment system we started
_check_in('deploy.fetch', repo)
- # Clone the repo if it doesn't exist yet
- if not __salt__['file.directory_exists'](config['location'] + '/.git'):
- cmd = '/usr/bin/git clone %s %s' % (config['url'] + '/.git',
- config['location'])
- status = __salt__['cmd.retcode'](cmd)
- if status != 0:
- return {'status': 5, 'repo': repo, 'dependencies': depstats}
-
- cmd = '/usr/bin/git remote set-url origin %s' % config['url'] + "/.git"
- __salt__['cmd.retcode'](cmd, config['location'])
-
- cmd = '/usr/bin/git fetch'
- status = __salt__['cmd.retcode'](cmd, config['location'])
- if status != 0:
+ tag = _get_tag(config)
+ if not tag:
return {'status': 10, 'repo': repo, 'dependencies': depstats}
- cmd = '/usr/bin/git fetch --tags'
- status = __salt__['cmd.retcode'](cmd, config['location'])
+ # Clone the repo if it doesn't exist yet
+ if not __salt__['file.directory_exists'](config['location'] + '/.git'):
+ status = _clone(config, config['location'], tag)
+ if status != 0:
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
+ else:
+ status = _fetch_location(config, config['location'])
+ if status != 0:
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
+
+ cmd = '/usr/bin/git show-ref refs/tags/{0}'.format(tag)
+ status = __salt__['cmd.retcode'](cmd, cwd=config['location'])
if status != 0:
- return {'status': 20, 'repo': repo, 'dependencies': depstats}
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
- if config['checkout_submodules']:
- cmd = '/usr/bin/git checkout .gitmodules'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 30, 'repo': repo, 'dependencies': depstats}
- # Transform .gitmodules file based on defined seds
- for before, after in config['submodule_sed_regex'].items():
- after = after.replace('__REPO_URL__', config['url'])
- __salt__['file.sed'](gitmodules, before, after)
-
- # Sync the .gitmodules config
- cmd = '/usr/bin/git submodule sync'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 40, 'repo': repo, 'dependencies': depstats}
-
- # fetch all submodules and tag for submodules
- cmd = '/usr/bin/git submodule foreach git fetch'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 50, 'repo': repo, 'dependencies': depstats}
-
- # fetch all submodules and tag for submodules
- cmd = '/usr/bin/git submodule foreach git fetch --tags'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
- if ret != 0:
- return {'status': 60, 'repo': repo, 'dependencies': depstats}
-
- cmd = '/usr/bin/git describe --always --tag origin'
- origin_tag = __salt__['cmd.run'](cmd, config['location'])
- origin_tag = origin_tag.strip()
+ if config['shadow_reference']:
+ shadow_gitdir = config['shadow_location'] + '/.git'
+ if not __salt__['file.directory_exists'](shadow_gitdir):
+ status = _clone(config, config['shadow_location'], tag,
+ shadow=True)
+ if status != 0:
+ return {'status': status, 'repo': repo,
+ 'dependencies': depstats}
+ else:
+ status = _fetch_location(config, config['shadow_location'],
+ shadow=True)
+ if status != 0:
+ return {'status': status, 'repo': repo,
+ 'dependencies': depstats}
+ status = _checkout_location(config, config['shadow_location'], tag,
+ reset=False, shadow=True)
+ if status != 0:
+ return {'status': status, 'repo': repo,
+ 'dependencies': depstats}
# Call modules on the repo's behalf ignore the return on these
for call, args in config['fetch_module_calls'].items():
mapped_args = _map_args(repo, args)
__salt__[call](*mapped_args)
return {'status': status, 'repo': repo,
- 'dependencies': depstats, 'tag': origin_tag}
+ 'dependencies': depstats, 'tag': tag}
+
+
+def _fetch_location(config, location, shadow=False):
+ cmd = '/usr/bin/git fetch'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ cmd = '/usr/bin/git fetch --tags'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # TODO: update .gitmodules recursively, then run submodule commands
+ # recursively.
+ if config['checkout_submodules']:
+ ret = _update_gitmodules(config, location, shadow)
+ if ret != 0:
+ return ret
+
+ # Sync the .gitmodules config
+ cmd = '/usr/bin/git submodule sync'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # fetch all submodules and tag for submodules
+ cmd = '/usr/bin/git submodule foreach git fetch'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+
+ # fetch all submodules and tag for submodules
+ cmd = '/usr/bin/git submodule foreach git fetch --tags'
+ status = __salt__['cmd.retcode'](cmd, location)
+ if status != 0:
+ return status
+ return 0
+
+
+def _get_tag(config):
+ # Fetch the .deploy file from the server and get the current tag
+ deployfile = config['url'] + '/.deploy'
+ f = urllib.urlopen(deployfile)
+ deployinfo = f.readlines()
+
+ tag = ''
+ for info in deployinfo:
+ if info.startswith('tag: '):
+ tag = info[5:]
+ tag = tag.strip()
+ if not tag:
+ return None
+ # tags are user-input and are used in shell commands, ensure they are
+ # only passing alphanumeric.
+ if re.match('\W+', tag):
+ return None
+ return tag
def checkout(repo, reset=False):
@@ -234,71 +328,66 @@
#TODO: replace the cmd.retcode calls with git module calls,
# where appropriate
config = get_config(repo)
- gitmodules = config['location'] + '/.gitmodules'
depstats = []
# Notify the deployment system we started
_check_in('deploy.checkout', repo)
- # Fetch the .deploy file from the server and get the current tag
- deployfile = config['url'] + '/.deploy'
- f = urllib.urlopen(deployfile)
- deployinfo = f.readlines()
- tag = ''
- for info in deployinfo:
- if info.startswith('tag: '):
- tag = info[5:]
- tag = tag.strip()
+ tag = _get_tag(config)
if not tag:
- return {'status': 10, 'repo': repo, 'dependencies': depstats}
- # tags are user-input and are used in shell commands, ensure they are
- # only passing alphanumeric.
- if re.match('\W+', tag):
- return {'status': 1, 'repo': repo, 'dependencies': depstats}
+ return {'status': status, 'repo': repo, 'dependencies': depstats}
+ status = _checkout_location(config, config['location'], tag, reset)
+
+ if status != 0:
+ return {'status': status, 'repo': repo, 'tag': tag,
+ 'dependencies': depstats}
+
+ # Call modules on the repo's behalf ignore the return on these
+ for call, args in config['checkout_module_calls'].items():
+ mapped_args = _map_args(repo, args)
+ __salt__[call](*mapped_args)
+ return {'status': status, 'repo': repo, 'tag': tag,
+ 'dependencies': depstats}
+
+
+def _checkout_location(config, location, tag, reset=False, shadow=False):
for dependency in config['dependencies']:
depstats.append(__salt__['deploy.checkout'](dependency, reset))
if reset:
# User requested we hard reset the repo to the tag
cmd = '/usr/bin/git reset --hard tags/%s' % (tag)
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 20, 'repo': repo, 'dependencies': depstats}
+ return 20
else:
cmd = '/usr/bin/git describe --always --tag'
- current_tag = __salt__['cmd.run'](cmd, config['location'])
+ current_tag = __salt__['cmd.run'](cmd, location)
current_tag = current_tag.strip()
if current_tag == tag:
- return {'status': 0, 'repo': repo,
- 'tag': tag, 'dependencies': depstats}
+ return 0
# Switch to the tag defined in the server's .deploy file
cmd = '/usr/bin/git checkout --force --quiet tags/%s' % (tag)
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 30, 'repo': repo, 'dependencies': depstats}
+ return 30
if config['checkout_submodules']:
- # Transform .gitmodules file based on defined seds
- for before, after in config['submodule_sed_regex'].items():
- after = after.replace('__REPO_URL__', config['url'])
- __salt__['file.sed'](gitmodules, before, after)
+ ret = _update_gitmodules(config, location, shadow)
+ if ret != 0:
+ return ret
# Sync the .gitmodules config
cmd = '/usr/bin/git submodule sync'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 40, 'repo': repo, 'dependencies': depstats}
+ return 40
# Update the submodules to match this tag
cmd = '/usr/bin/git submodule update --init'
- ret = __salt__['cmd.retcode'](cmd, config['location'])
+ ret = __salt__['cmd.retcode'](cmd, location)
if ret != 0:
- return {'status': 50, 'repo': repo, 'dependencies': depstats}
-
- # Call modules on the repo's behalf ignore the return on these
- for call, args in config['checkout_module_calls'].items():
- mapped_args = _map_args(repo, args)
- __salt__[call](*mapped_args)
- return {'status': 0, 'repo': repo, 'tag': tag, 'dependencies': depstats}
+ return 50
+ return 0
--
To view, visit https://gerrit.wikimedia.org/r/94680
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2737ce42527c84c05167b3dcc96b5638c8732852
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Ryan Lane <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
