Ryan Lane has submitted this change and it was merged.
Change subject: Add mediawiki module for fetch and checkout hooks
......................................................................
Add mediawiki module for fetch and checkout hooks
To generate the localization cache, this change adds a fetch hook
and a checkout hook that will utilize the shadow reference
repository to generate the localization cache during the fetch
stage, then will rsync the cache during the checkout stage.
Assuming no files need to be changed, this approach should work
similarly to how the cache in generated from the deployment server.
Change-Id: I3fc95902b1c7668f55448a873933db58fc8e0a0a
---
M manifests/role/deployment.pp
A modules/deployment/files/modules/mediawiki.py
M modules/deployment/manifests/salt_master.pp
3 files changed, 84 insertions(+), 39 deletions(-)
Approvals:
Ryan Lane: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index 34db7d7..6f9212b 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -10,59 +10,48 @@
'upstream' => 'https://gerrit.wikimedia.org/r/integration/phpcs',
},
'mediawiki/common' => {
- 'grain' => 'mediawiki',
- 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/mediawiki-config',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/operations/mediawiki-config',
+ 'shadow_reference' => true,
},
'mediawiki/private' => {
'grain' => 'mediawiki',
},
'mediawiki/slot0' => {
- 'grain' => 'mediawiki',
- 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
- 'submodule_sed_regex' => {
- 'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
- '.git' => '',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
+ 'checkout_submodules' => true,
+ 'shadow_reference' => true,
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
},
- 'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-slot0' => 'l10n',
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
},
},
'mediawiki/slot1' => {
- 'grain' => 'mediawiki',
- 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
- 'submodule_sed_regex' => {
- 'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
- '.git' => '',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
+ 'checkout_submodules' => true,
+ 'shadow_reference' => true,
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
},
- 'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-slot1' => 'l10n',
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
},
},
'mediawiki/beta0' => {
- 'grain' => 'mediawiki',
- 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
- 'submodule_sed_regex' => {
- 'https://gerrit.wikimedia.org/r/p/mediawiki' =>
'__REPO_URL__/.git/modules',
- '.git' => '',
+ 'grain' => 'mediawiki',
+ 'upstream' =>
'https://gerrit.wikimedia.org/r/mediawiki/core',
+ 'checkout_submodules' => true,
+ 'shadow_reference' => true,
+ 'fetch_module_calls' => {
+ 'mediawiki.generate_localization_cache' => ['__REPO__'],
},
- 'checkout_submodules' => true,
- 'dependencies' => {
- 'mediawiki/l10n-beta' => 'l10n',
+ 'checkout_module_calls' => {
+ 'mediawiki.update_localization_cache' => ['__REPO__'],
},
- },
- 'mediawiki/l10n-slot0' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
- },
- 'mediawiki/l10n-slot1' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
- },
- 'mediawiki/l10n-beta0' => {
- 'automated' => true,
- 'grain' => 'mediawiki',
},
'gdash/gdash' => {
'grain' => 'gdash',
diff --git a/modules/deployment/files/modules/mediawiki.py
b/modules/deployment/files/modules/mediawiki.py
new file mode 100644
index 0000000..37fcf61
--- /dev/null
+++ b/modules/deployment/files/modules/mediawiki.py
@@ -0,0 +1,50 @@
+'''
+MediaWiki deployment module for trebuchet
+'''
+
+import os
+import subprocess
+
+
+def generate_localization_cache(repo):
+ '''
+ Generate the localization cache in the shadow_reference repo.
+ '''
+ config = __salt__['deploy.get_config'](repo)
+ common_config = __salt__['deploy.get_config']('mediawiki/common')
+ cmd = 'php {0}/maintenance/mergeMessageFileList.php' \
+ '--list-file={1}/wmf-config/extension-list' \
+ '--output={1}/wmf-config/ExtensionMessages-{2}.php'
+ cmd.format(config['shadow_location'],
+ common_config['shadow_location'],
+ os.path.basename(repo))
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ cmd = 'php {0}/maintenance/rebuildLocalisationCache.php' \
+ '--outdir={0}/cache/l10n --threads=12'
+ cmd = cmd.format(config['shadow_location'])
+ status = __salt__['cmd.retcode'](cmd)
+ return status
+
+
+def update_localization_cache(repo):
+ '''
+ Update the repo's localization cache, based on the reference repo's cache.
+ '''
+ config = __salt__['deploy.get_config'](repo)
+ common_config = __salt__['deploy.get_config']('mediawiki/common')
+ shadow_cache = '{0}/cache'.format(config['shadow_location'])
+ current_cache = '{0}/cache'.format(config['location'])
+ cmd = 'rsync -a {0}/l10n {1}'.format(shadow_cache, current_cache)
+ status = __salt__['cmd.retcode'](cmd)
+ if status != 0:
+ return status
+ messages = '{0}/wmf-config/ExtensionMessages-{1}.php'
+ shadow_messages = messages.format(common_config['shadow_location'],
+ os.path.basename(repo))
+ current_messages = messages.format(common_config['location'],
+ os.path.basename(repo))
+ cmd = 'rsync -a {0} {1}'.format(shadow_messages, current_messages)
+ status = __salt__['cmd.retcode'](cmd)
+ return status
diff --git a/modules/deployment/manifests/salt_master.pp
b/modules/deployment/manifests/salt_master.pp
index 2ded2af..77102e1 100644
--- a/modules/deployment/manifests/salt_master.pp
+++ b/modules/deployment/manifests/salt_master.pp
@@ -80,6 +80,12 @@
owner => root,
group => root,
require => [File["${module_dir}"]];
+ "${module_dir}/mediawiki.py":
+ source => "puppet:///deployment/modules/mediawiki.py",
+ mode => 0555,
+ owner => root,
+ group => root,
+ require => [File["${module_dir}"]];
}
# If pillars or modules change, we need to sync them with the minions
@@ -96,7 +102,7 @@
require => [File["${module_dir}/deploy.py"]];
"refresh_deployment_modules":
command => "/usr/bin/salt -G 'deployment_target:*'
saltutil.sync_modules",
- subscribe =>
[File["${module_dir}/deploy.py"],File["${module_dir}/parsoid.py"]],
+ subscribe => [File["${module_dir}/deploy.py"],
File["${module_dir}/parsoid.py"], File["${module_dir}/mediawiki.py"]],
refreshonly => true,
require => [Package["salt-master"]];
"refresh_deployment_returners":
--
To view, visit https://gerrit.wikimedia.org/r/94682
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3fc95902b1c7668f55448a873933db58fc8e0a0a
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ryan Lane <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Ryan Lane <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
