Ryan Lane has uploaded a new change for review.
https://gerrit.wikimedia.org/r/91319
Change subject: Maintain repositories on deployment server
......................................................................
Maintain repositories on deployment server
To avoid any manual steps in the deployment system it's necessary
to maintain the repositories on the deployment server for initial
clones and inits. This change adds an upstream config option for
repositories and does an initial clone/init and configuration
of all repositories configured.
Change-Id: I504f2f5e5936652c0783c38eaff080e18324a9be
---
M manifests/role/deployment.pp
M modules/deployment/files/modules/deploy.py
M modules/deployment/manifests/deployment_server.pp
M modules/deployment/manifests/salt_master.pp
4 files changed, 92 insertions(+), 37 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/19/91319/1
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 44b4470..f157070 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -2,13 +2,15 @@
class role::deployment::config {
$repo_config = {
'common' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/mediawiki-config',
},
'private' => {
'grain' => 'mediawiki',
},
'slot0' => {
'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -19,7 +21,8 @@
},
},
'slot1' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -30,7 +33,8 @@
},
},
'beta0' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -53,10 +57,12 @@
'grain' => 'mediawiki',
},
'gdash/gdash' => {
- 'grain' => 'gdash',
+ 'grain' => 'gdash',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/software/gdash',
},
'parsoid/Parsoid' => {
'grain' => 'parsoid',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/extensions/Parsoid',
'checkout_module_calls' => {
'parsoid.config_symlink' => ['__REPO__'],
'parsoid.restart_parsoid' => ['__REPO__'],
@@ -69,19 +75,23 @@
},
},
'eventlogging/EventLogging' => {
- 'grain' => 'eventlogging',
+ 'grain' => 'eventlogging',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/extensions/EventLogging',
},
'fluoride/fluoride' => {
- 'grain' => 'fluoride',
+ 'grain' => 'fluoride',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/tools/fluoride',
},
'test/testrepo' => {
'grain' => 'testrepo',
},
'elasticsearch/plugins' => {
- 'grain' => 'elasticsearchplugins',
+ 'grain' => 'elasticsearchplugins',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/software/elasticsearch/plugins',
},
'analytics/kraken' => {
- 'grain' => 'analytics-kraken',
+ 'grain' => 'analytics-kraken',
+ 'upstream' => 'https://gerrit.wikimedia.org/r/p/analytics/kraken',
},
}
}
@@ -235,6 +245,7 @@
vhost_name => "10.4.1.19",
port => 80,
docroot => "/srv/deployment",
+ docroot_owner => "sartoris",
docroot_group => "project-sartoris",
docroot_dir_allows => ["10.4.0.0/16"],
serveradmin => "[email protected]",
diff --git a/modules/deployment/files/modules/deploy.py
b/modules/deployment/files/modules/deploy.py
index c2a7ea3..bbed974 100644
--- a/modules/deployment/files/modules/deploy.py
+++ b/modules/deployment/files/modules/deploy.py
@@ -73,27 +73,50 @@
config.setdefault('checkout_module_calls', {})
config.setdefault('fetch_module_calls', {})
config.setdefault('sync_script', 'shared.py')
+ config.setdefault('upstream', None)
return config
-def init_sync_links():
+def deployment_server_init():
serv = _get_redis_serv()
is_deployment_server = __grains__.get('deployment_server')
hook_dir = __grains__.get('deployment_global_hook_dir')
if not is_deployment_server:
return 0
+ deploy_user = __grains__.get('deployment_repo_user')
repo_config = __pillar__.get('repo_config')
for repo in repo_config:
config = get_config(repo)
repo_sync_dir = '{0}/sync/{1}'.format(hook_dir, os.path.dirname(repo))
sync_link = '{0}/{1}.sync'.format(repo_sync_dir,
os.path.basename(repo))
+ # Create repo sync dir
if not __salt__['file.directory_exists'](repo_sync_dir):
__salt__['file.mkdir'](repo_sync_dir)
+ # Create repo sync script link
if not __salt__['file.file_exists'](sync_link):
sync_script = '{0}/sync/{1}'.format(hook_dir,
config['sync_script'])
__salt__['file.symlink'](sync_script, sync_link)
+ # Clone repo from upstream or init repo with no upstream
+ if not __salt__['file.directory_exists'](config['location'] + '/.git'):
+ if config['upstream']:
+ cmd = '/usr/bin/git clone %s/.git %s' % (config['upstream'],
+ config['location'])
+ else:
+ cmd = '/usr/bin/git init %s' % (config['location'])
+ status = __salt__['cmd.retcode'](cmd, runas=deploy_user,
+ umask=002)
+ if status != 0:
+ return status
+ # git clone does ignores umask and does explicit mkdir with 755
+ __salt__['file.set_mode'](config['location'], 2775)
+ # Set the repo name in the repo's config
+ cmd = 'git config deploy.tag-prefix %s' % repo
+ status = __salt__['cmd.retcode'](cmd, cwd=config['location'],
+ runas=deploy_user, umask=002)
+ if status != 0:
+ return status
return 0
diff --git a/modules/deployment/manifests/deployment_server.pp
b/modules/deployment/manifests/deployment_server.pp
index 21ac7ef..1b1fec6 100644
--- a/modules/deployment/manifests/deployment_server.pp
+++ b/modules/deployment/manifests/deployment_server.pp
@@ -1,16 +1,32 @@
-class
deployment::deployment_server($deployment_conffile="/etc/git-deploy/git-deploy.conf",
$deployment_ignorefile="/etc/git-deploy/gitignore",
$deployment_ignores=['.deploy'], $deployment_restrict_umask="002",
$deployment_block_file="/etc/ROLLOUTS_BLOCKED", $deployment_support_email="",
$deployment_repo_name_detection="dot-git-parent-dir",
$deployment_announce_email="", $deployment_send_mail_on_sync="false",
$deployment_send_mail_on_revert="false",
$deployment_log_directory="/var/log/git-deploy",
$deployment_log_timing_data="false",
$deployment_git_deploy_dir="/var/lib/git-deploy",
$deployment_per_repo_config={}, $deployer_groups=[]) {
- if ! defined(Package["git-deploy"]){
- package { "git-deploy":
+class deployment::deployment_server(
+ $deployment_conffile='/etc/git-deploy/git-deploy.conf',
+ $deployment_ignorefile='/etc/git-deploy/gitignore',
+ $deployment_ignores=['.deploy'],
+ $deployment_restrict_umask='002',
+ $deployment_block_file='/etc/ROLLOUTS_BLOCKED',
+ $deployment_support_email='',
+ $deployment_repo_name_detection='dot-git-parent-dir',
+ $deployment_announce_email='',
+ $deployment_send_mail_on_sync='false',
+ $deployment_send_mail_on_revert='false',
+ $deployment_log_directory='/var/log/git-deploy',
+ $deployment_log_timing_data='false',
+ $deployment_git_deploy_dir='/var/lib/git-deploy',
+ $deployment_per_repo_config={},
+ $deployer_groups=[]
+ ) {
+ if ! defined(Package['git-deploy']){
+ package { 'git-deploy':
ensure => present;
}
}
- if ! defined(Package["git-core"]){
- package { "git-core":
+ if ! defined(Package['git-core']){
+ package { 'git-core':
ensure => present;
}
}
- if ! defined(Package["python-redis"]){
- package { "python-redis":
+ if ! defined(Package['python-redis']){
+ package { 'python-redis':
ensure => present;
}
}
@@ -35,70 +51,75 @@
group => root,
require => [File["${$deployment_global_hook_dir}"]];
"${$deployment_global_hook_dir}/sync/deploylib.py":
- source => "puppet:///deployment/git-deploy/hooks/deploylib.py",
+ source => 'puppet:///deployment/git-deploy/hooks/deploylib.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_global_hook_dir}/sync/shared.py":
- source => "puppet:///deployment/git-deploy/hooks/shared.py",
+ source => 'puppet:///deployment/git-deploy/hooks/shared.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_global_hook_dir}/sync/depends.py":
- source => "puppet:///deployment/git-deploy/hooks/depends.py",
+ source => 'puppet:///deployment/git-deploy/hooks/depends.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_dependencies_dir}/l10n":
- source =>
"puppet:///deployment/git-deploy/dependencies/l10nupdate-quick",
+ source =>
'puppet:///deployment/git-deploy/dependencies/l10nupdate-quick',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_dependencies_dir}"]];
}
file {
- "/etc/gitconfig":
- content => template("deployment/git-deploy/gitconfig.erb"),
+ '/etc/gitconfig':
+ content => template('deployment/git-deploy/gitconfig.erb'),
mode => 0444,
owner => root,
group => root,
- require => [Package["git-core"]];
+ require => [Package['git-core']];
"${deployment_conffile}":
- content => template("deployment/git-deploy/git-deploy.conf.erb"),
+ content => template('deployment/git-deploy/git-deploy.conf.erb'),
mode => 0444,
owner => root,
group => root;
"${deployment_ignorefile}":
- content => template("deployment/git-deploy/gitignore.erb"),
+ content => template('deployment/git-deploy/gitignore.erb'),
mode => 0444,
owner => root,
group => root;
- "/usr/local/bin/deploy-info":
+ '/usr/local/bin/deploy-info':
owner => root,
group => root,
mode => 0555,
- source => "puppet:///deployment/git-deploy/utils/deploy-info",
- require => [Package["python-redis"]];
- "/usr/local/bin/submodule-update-server-info":
+ source => 'puppet:///deployment/git-deploy/utils/deploy-info',
+ require => [Package['python-redis']];
+ '/usr/local/bin/submodule-update-server-info':
owner => root,
group => root,
mode => 0555,
- source =>
"puppet:///deployment/git-deploy/utils/submodule-update-server-info",
+ source =>
'puppet:///deployment/git-deploy/utils/submodule-update-server-info',
}
- salt::grain { "deployment_server":
- grain => "deployment_server",
+ salt::grain { 'deployment_server':
+ grain => 'deployment_server',
value => true,
replace => true;
}
- salt::grain { "deployment_global_hook_dir":
- grain => "deployment_global_hook_dir",
+ salt::grain { 'deployment_global_hook_dir':
+ grain => 'deployment_global_hook_dir',
value => $deployment_global_hook_dir,
replace => true;
}
+ salt::grain { 'deployment_repo_user':
+ grain => 'deployment_repo_user',
+ value => 'sartoris',
+ replace => true;
+ }
systemuser {
- "sartoris": name => "sartoris", shell => "/bin/false", home =>
"/nonexistent", groups => $deployer_groups
+ 'sartoris': name => 'sartoris', shell => '/bin/false', home =>
'/nonexistent', groups => $deployer_groups
}
}
diff --git a/modules/deployment/manifests/salt_master.pp
b/modules/deployment/manifests/salt_master.pp
index 6afd6b7..2ded2af 100644
--- a/modules/deployment/manifests/salt_master.pp
+++ b/modules/deployment/manifests/salt_master.pp
@@ -89,8 +89,8 @@
subscribe => [File["${pillar_dir}/deployment/deployment_config.sls"],
File["${pillar_dir}/deployment/repo_config.sls"], File["${pillar_dir}"]],
refreshonly => true,
require => [Package["salt-master"]];
- "update_sync_links":
- command => "/usr/bin/salt -G 'deployment_server:true'
deploy.init_sync_links",
+ "deployment_server_init":
+ command => "/usr/bin/salt -G 'deployment_server:true'
deploy.deployment_server_init",
subscribe => [Exec['refresh_deployment_pillars']],
refreshonly => true,
require => [File["${module_dir}/deploy.py"]];
--
To view, visit https://gerrit.wikimedia.org/r/91319
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I504f2f5e5936652c0783c38eaff080e18324a9be
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
