Ryan Lane has uploaded a new change for review.
https://gerrit.wikimedia.org/r/94682
Change subject: Add mediawiki module for fetch and checkout hooks
......................................................................
Add mediawiki module for fetch and checkout hooks
To generate the localization cache, this change adds a fetch hook
and a checkout hook that will utilize the shadow reference
repository to generate the localization cache during the fetch
stage, then will rsync the cache during the checkout stage.
Assuming no files need to be changed, this approach should work
similarly to how the cache in generated from the deployment server.
Change-Id: I3fc95902b1c7668f55448a873933db58fc8e0a0a
---
M manifests/role/deployment.pp
A modules/deployment/files/modules/mediawiki.py
M modules/deployment/manifests/salt_master.pp
3 files changed, 72 insertions(+), 19 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/82/94682/1
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 1313880..bbffaa4 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -16,8 +16,11 @@
'.git' => '',
},
'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-slot0' => 'l10n',
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
+ },
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
},
},
'mediawiki/slot1' => {
@@ -28,8 +31,11 @@
'.git' => '',
},
'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-slot1' => 'l10n',
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
+ },
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
},
},
'mediawiki/beta0' => {
@@ -40,21 +46,12 @@
'.git' => '',
},
'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-beta' => 'l10n',
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
},
- },
- 'mediawiki/l10n-slot0' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
- },
- 'mediawiki/l10n-slot1' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
- },
- 'mediawiki/l10n-beta0' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
+ },
},
'gdash/gdash' => {
'grain' => 'gdash',
diff --git a/modules/deployment/files/modules/mediawiki.py
b/modules/deployment/files/modules/mediawiki.py
new file mode 100644
index 0000000..3f40eec
--- /dev/null
+++ b/modules/deployment/files/modules/mediawiki.py
@@ -0,0 +1,50 @@
+'''
+MediaWiki deployment module for trebuchet
+'''
+
+import os
+import subprocess
+
+
+def generate_localization_cache(repo):
+ '''
+ Generate the localization cache in the shadow_reference repo.
+ '''
+ config = __salt__['deploy.get_config'](repo)
+ common_config = __salt__['deploy.get_config']('mediawiki/common')
+ cmd = 'php {0}/maintenance/mergeMessageFileList.php'
+ '--list-file={1]/wmf-config/extension-list'
+ '--output={1}/l10n-{2}/ExtensionMessages.php'
+ cmd.format(config['shadow_location'],
+ common_config['shadow_location'],
+ os.path.basename(repo))
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ cmd = 'php {0}/maintenance/rebuildLocalisationCache.php'
+ '--outdir={0}/cache --threads=12'
+ cmd = cmd.format(config['shadow_location'])
+ status = __salt__['cmd.retcode'](cmd)
+ return status
+
+
+def update_localization_cache(repo):
+ '''
+ Update the repo's localization cache, based on the reference repo's cache.
+ '''
+ config = __salt__['deploy.get_config'](repo)
+ common_config = __salt__['deploy.get_config']('mediawiki/common')
+ shadow_cache = '{0}/cache'.format(config['shadow_location'])
+ current_cache = '{0}/cache'.format(config['location'])
+ cmd = 'rsync -a {0} {1}'.format(shadow_cache, current_cache)
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ messages = '{0}/l10n-{1}/ExtensionMessages.php'
+ shadow_messages = messages.format(common_config['shadow_location'],
+ os.path.basename(repo))
+ shadow_messages = messages.format(common_config['location'],
+ os.path.basename(repo))
+ cmd = 'rsync -a {0} {1}'.format(shadow_messages, current_messages)
+ status = __salt__['cmd.retcode'](cmd)
+ return status
diff --git a/modules/deployment/manifests/salt_master.pp
b/modules/deployment/manifests/salt_master.pp
index 2ded2af..77102e1 100644
--- a/modules/deployment/manifests/salt_master.pp
+++ b/modules/deployment/manifests/salt_master.pp
@@ -80,6 +80,12 @@
owner => root,
group => root,
require => [File["${module_dir}"]];
+ "${module_dir}/mediawiki.py":
+ source => "puppet:///deployment/modules/mediawiki.py",
+ mode => 0555,
+ owner => root,
+ group => root,
+ require => [File["${module_dir}"]];
}
# If pillars or modules change, we need to sync them with the minions
@@ -96,7 +102,7 @@
require => [File["${module_dir}/deploy.py"]];
"refresh_deployment_modules":
command => "/usr/bin/salt -G 'deployment_target:*'
saltutil.sync_modules",
- subscribe =>
[File["${module_dir}/deploy.py"],File["${module_dir}/parsoid.py"]],
+ subscribe => [File["${module_dir}/deploy.py"],
File["${module_dir}/parsoid.py"], File["${module_dir}/mediawiki.py"]],
refreshonly => true,
require => [Package["salt-master"]];
"refresh_deployment_returners":
--
To view, visit https://gerrit.wikimedia.org/r/94682
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3fc95902b1c7668f55448a873933db58fc8e0a0a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
