Ryan Lane has submitted this change and it was merged.
Change subject: Maintain repositories on deployment server
......................................................................
Maintain repositories on deployment server
To avoid any manual steps in the deployment system it's necessary
to maintain the repositories on the deployment server for initial
clones and inits. This change adds an upstream config option for
repositories and does an initial clone/init and configuration
of all repositories configured.
Change-Id: I504f2f5e5936652c0783c38eaff080e18324a9be
---
M manifests/role/deployment.pp
M modules/deployment/files/modules/deploy.py
M modules/deployment/manifests/deployment_server.pp
M modules/deployment/manifests/salt_master.pp
4 files changed, 95 insertions(+), 37 deletions(-)
Approvals:
Ryan Lane: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 44b4470..3bbf30a 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -2,13 +2,15 @@
class role::deployment::config {
$repo_config = {
'common' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/mediawiki-config',
},
'private' => {
'grain' => 'mediawiki',
},
'slot0' => {
'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -19,7 +21,8 @@
},
},
'slot1' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -30,7 +33,8 @@
},
},
'beta0' => {
- 'grain' => 'mediawiki',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
'submodule_sed_regex' => {
'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
'.git' => '',
@@ -53,10 +57,12 @@
'grain' => 'mediawiki',
},
'gdash/gdash' => {
- 'grain' => 'gdash',
+ 'grain' => 'gdash',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/software/gdash',
},
'parsoid/Parsoid' => {
'grain' => 'parsoid',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/extensions/Parsoid',
'checkout_module_calls' => {
'parsoid.config_symlink' => ['__REPO__'],
'parsoid.restart_parsoid' => ['__REPO__'],
@@ -69,19 +75,23 @@
},
},
'eventlogging/EventLogging' => {
- 'grain' => 'eventlogging',
+ 'grain' => 'eventlogging',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/extensions/EventLogging',
},
'fluoride/fluoride' => {
- 'grain' => 'fluoride',
+ 'grain' => 'fluoride',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/tools/fluoride',
},
'test/testrepo' => {
'grain' => 'testrepo',
},
'elasticsearch/plugins' => {
- 'grain' => 'elasticsearchplugins',
+ 'grain' => 'elasticsearchplugins',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/software/elasticsearch/plugins',
},
'analytics/kraken' => {
- 'grain' => 'analytics-kraken',
+ 'grain' => 'analytics-kraken',
+ 'upstream' => 'https://gerrit.wikimedia.org/r/p/analytics/kraken',
},
}
}
@@ -177,6 +187,7 @@
vhost_name => "10.64.0.196",
port => 80,
docroot => "/srv/deployment",
+ docroot_owner => "sartoris",
docroot_group => "wikidev",
docroot_dir_allows => ["10.0.0.0/16","10.64.0.0/16","208.80.152.0/22"],
serveradmin => "[email protected]",
@@ -208,6 +219,8 @@
vhost_name => "10.4.0.58",
port => 80,
docroot => "/srv/deployment",
+ docroot_owner => "sartoris",
+ docroot_group => "project-deployment-prep",
docroot_dir_allows => ["10.4.0.0/16"],
serveradmin => "[email protected]",
configure_firewall => false,
@@ -235,6 +248,7 @@
vhost_name => "10.4.1.19",
port => 80,
docroot => "/srv/deployment",
+ docroot_owner => "sartoris",
docroot_group => "project-sartoris",
docroot_dir_allows => ["10.4.0.0/16"],
serveradmin => "[email protected]",
diff --git a/modules/deployment/files/modules/deploy.py
b/modules/deployment/files/modules/deploy.py
index c2a7ea3..f814568 100644
--- a/modules/deployment/files/modules/deploy.py
+++ b/modules/deployment/files/modules/deploy.py
@@ -73,27 +73,50 @@
config.setdefault('checkout_module_calls', {})
config.setdefault('fetch_module_calls', {})
config.setdefault('sync_script', 'shared.py')
+ config.setdefault('upstream', None)
return config
-def init_sync_links():
+def deployment_server_init():
serv = _get_redis_serv()
is_deployment_server = __grains__.get('deployment_server')
hook_dir = __grains__.get('deployment_global_hook_dir')
if not is_deployment_server:
return 0
+ deploy_user = __grains__.get('deployment_repo_user')
repo_config = __pillar__.get('repo_config')
for repo in repo_config:
config = get_config(repo)
repo_sync_dir = '{0}/sync/{1}'.format(hook_dir, os.path.dirname(repo))
sync_link = '{0}/{1}.sync'.format(repo_sync_dir,
os.path.basename(repo))
+ # Create repo sync dir
if not __salt__['file.directory_exists'](repo_sync_dir):
__salt__['file.mkdir'](repo_sync_dir)
+ # Create repo sync script link
if not __salt__['file.file_exists'](sync_link):
sync_script = '{0}/sync/{1}'.format(hook_dir,
config['sync_script'])
__salt__['file.symlink'](sync_script, sync_link)
+ # Clone repo from upstream or init repo with no upstream
+ if not __salt__['file.directory_exists'](config['location'] + '/.git'):
+ if config['upstream']:
+ cmd = '/usr/bin/git clone %s/.git %s' % (config['upstream'],
+ config['location'])
+ else:
+ cmd = '/usr/bin/git init %s' % (config['location'])
+ status = __salt__['cmd.retcode'](cmd, runas=deploy_user,
+ umask=002)
+ if status != 0:
+ return status
+ # git clone does ignores umask and does explicit mkdir with 755
+ __salt__['file.set_mode'](config['location'], 2775)
+ # Set the repo name in the repo's config
+ cmd = 'git config deploy.tag-prefix %s' % repo
+ status = __salt__['cmd.retcode'](cmd, cwd=config['location'],
+ runas=deploy_user, umask=002)
+ if status != 0:
+ return status
return 0
diff --git a/modules/deployment/manifests/deployment_server.pp
b/modules/deployment/manifests/deployment_server.pp
index 21ac7ef..1b1fec6 100644
--- a/modules/deployment/manifests/deployment_server.pp
+++ b/modules/deployment/manifests/deployment_server.pp
@@ -1,16 +1,32 @@
-class
deployment::deployment_server($deployment_conffile="/etc/git-deploy/git-deploy.conf",
$deployment_ignorefile="/etc/git-deploy/gitignore",
$deployment_ignores=['.deploy'], $deployment_restrict_umask="002",
$deployment_block_file="/etc/ROLLOUTS_BLOCKED", $deployment_support_email="",
$deployment_repo_name_detection="dot-git-parent-dir",
$deployment_announce_email="", $deployment_send_mail_on_sync="false",
$deployment_send_mail_on_revert="false",
$deployment_log_directory="/var/log/git-deploy",
$deployment_log_timing_data="false",
$deployment_git_deploy_dir="/var/lib/git-deploy",
$deployment_per_repo_config={}, $deployer_groups=[]) {
- if ! defined(Package["git-deploy"]){
- package { "git-deploy":
+class deployment::deployment_server(
+ $deployment_conffile='/etc/git-deploy/git-deploy.conf',
+ $deployment_ignorefile='/etc/git-deploy/gitignore',
+ $deployment_ignores=['.deploy'],
+ $deployment_restrict_umask='002',
+ $deployment_block_file='/etc/ROLLOUTS_BLOCKED',
+ $deployment_support_email='',
+ $deployment_repo_name_detection='dot-git-parent-dir',
+ $deployment_announce_email='',
+ $deployment_send_mail_on_sync='false',
+ $deployment_send_mail_on_revert='false',
+ $deployment_log_directory='/var/log/git-deploy',
+ $deployment_log_timing_data='false',
+ $deployment_git_deploy_dir='/var/lib/git-deploy',
+ $deployment_per_repo_config={},
+ $deployer_groups=[]
+ ) {
+ if ! defined(Package['git-deploy']){
+ package { 'git-deploy':
ensure => present;
}
}
- if ! defined(Package["git-core"]){
- package { "git-core":
+ if ! defined(Package['git-core']){
+ package { 'git-core':
ensure => present;
}
}
- if ! defined(Package["python-redis"]){
- package { "python-redis":
+ if ! defined(Package['python-redis']){
+ package { 'python-redis':
ensure => present;
}
}
@@ -35,70 +51,75 @@
group => root,
require => [File["${$deployment_global_hook_dir}"]];
"${$deployment_global_hook_dir}/sync/deploylib.py":
- source => "puppet:///deployment/git-deploy/hooks/deploylib.py",
+ source => 'puppet:///deployment/git-deploy/hooks/deploylib.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_global_hook_dir}/sync/shared.py":
- source => "puppet:///deployment/git-deploy/hooks/shared.py",
+ source => 'puppet:///deployment/git-deploy/hooks/shared.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_global_hook_dir}/sync/depends.py":
- source => "puppet:///deployment/git-deploy/hooks/depends.py",
+ source => 'puppet:///deployment/git-deploy/hooks/depends.py',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_global_hook_dir}/sync"]];
"${$deployment_dependencies_dir}/l10n":
- source =>
"puppet:///deployment/git-deploy/dependencies/l10nupdate-quick",
+ source =>
'puppet:///deployment/git-deploy/dependencies/l10nupdate-quick',
mode => 0555,
owner => root,
group => root,
require => [File["${$deployment_dependencies_dir}"]];
}
file {
- "/etc/gitconfig":
- content => template("deployment/git-deploy/gitconfig.erb"),
+ '/etc/gitconfig':
+ content => template('deployment/git-deploy/gitconfig.erb'),
mode => 0444,
owner => root,
group => root,
- require => [Package["git-core"]];
+ require => [Package['git-core']];
"${deployment_conffile}":
- content => template("deployment/git-deploy/git-deploy.conf.erb"),
+ content => template('deployment/git-deploy/git-deploy.conf.erb'),
mode => 0444,
owner => root,
group => root;
"${deployment_ignorefile}":
- content => template("deployment/git-deploy/gitignore.erb"),
+ content => template('deployment/git-deploy/gitignore.erb'),
mode => 0444,
owner => root,
group => root;
- "/usr/local/bin/deploy-info":
+ '/usr/local/bin/deploy-info':
owner => root,
group => root,
mode => 0555,
- source => "puppet:///deployment/git-deploy/utils/deploy-info",
- require => [Package["python-redis"]];
- "/usr/local/bin/submodule-update-server-info":
+ source => 'puppet:///deployment/git-deploy/utils/deploy-info',
+ require => [Package['python-redis']];
+ '/usr/local/bin/submodule-update-server-info':
owner => root,
group => root,
mode => 0555,
- source =>
"puppet:///deployment/git-deploy/utils/submodule-update-server-info",
+ source =>
'puppet:///deployment/git-deploy/utils/submodule-update-server-info',
}
- salt::grain { "deployment_server":
- grain => "deployment_server",
+ salt::grain { 'deployment_server':
+ grain => 'deployment_server',
value => true,
replace => true;
}
- salt::grain { "deployment_global_hook_dir":
- grain => "deployment_global_hook_dir",
+ salt::grain { 'deployment_global_hook_dir':
+ grain => 'deployment_global_hook_dir',
value => $deployment_global_hook_dir,
replace => true;
}
+ salt::grain { 'deployment_repo_user':
+ grain => 'deployment_repo_user',
+ value => 'sartoris',
+ replace => true;
+ }
systemuser {
- "sartoris": name => "sartoris", shell => "/bin/false", home =>
"/nonexistent", groups => $deployer_groups
+ 'sartoris': name => 'sartoris', shell => '/bin/false', home =>
'/nonexistent', groups => $deployer_groups
}
}
diff --git a/modules/deployment/manifests/salt_master.pp
b/modules/deployment/manifests/salt_master.pp
index 6afd6b7..2ded2af 100644
--- a/modules/deployment/manifests/salt_master.pp
+++ b/modules/deployment/manifests/salt_master.pp
@@ -89,8 +89,8 @@
subscribe => [File["${pillar_dir}/deployment/deployment_config.sls"],
File["${pillar_dir}/deployment/repo_config.sls"], File["${pillar_dir}"]],
refreshonly => true,
require => [Package["salt-master"]];
- "update_sync_links":
- command => "/usr/bin/salt -G 'deployment_server:true'
deploy.init_sync_links",
+ "deployment_server_init":
+ command => "/usr/bin/salt -G 'deployment_server:true'
deploy.deployment_server_init",
subscribe => [Exec['refresh_deployment_pillars']],
refreshonly => true,
require => [File["${module_dir}/deploy.py"]];
--
To view, visit https://gerrit.wikimedia.org/r/91319
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I504f2f5e5936652c0783c38eaff080e18324a9be
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
Gerrit-Reviewer: Ryan Lane <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
