ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/400692 )
Change subject: make all misc dump jobs plus xml dumps config use hiera setting
for php
......................................................................
make all misc dump jobs plus xml dumps config use hiera setting for php
This lets us choose hhvm or some php version per host, and lets
us use different versions in labs as well
Change-Id: I4c8a768be76db30501790e5d31af8ca9cc1c4f52
---
M hieradata/hosts/snapshot1001.yaml
M hieradata/hosts/snapshot1005.yaml
M hieradata/hosts/snapshot1006.yaml
M hieradata/hosts/snapshot1007.yaml
M modules/profile/manifests/dumps/generation/worker/common.pp
M modules/snapshot/files/cron/dump-global-blocks.sh
M modules/snapshot/files/cron/dumpcategoriesrdf.sh
M modules/snapshot/files/cron/dumpcirrussearch.sh
M modules/snapshot/files/cron/dumpwikidatajson.sh
M modules/snapshot/files/cron/dumpwikidatardf.sh
M modules/snapshot/files/cron/wikidatadumps-shared.sh
M modules/snapshot/manifests/dumps.pp
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/templates/dumps/wikidump.conf.erb
14 files changed, 36 insertions(+), 21 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/92/400692/1
diff --git a/hieradata/hosts/snapshot1001.yaml
b/hieradata/hosts/snapshot1001.yaml
index d27cca6..9015ce4 100644
--- a/hieradata/hosts/snapshot1001.yaml
+++ b/hieradata/hosts/snapshot1001.yaml
@@ -1 +1,2 @@
snapshot::dumps::runtype: regular
+snapshot::dumps::php: php5
diff --git a/hieradata/hosts/snapshot1005.yaml
b/hieradata/hosts/snapshot1005.yaml
index c34955f..f51a605 100644
--- a/hieradata/hosts/snapshot1005.yaml
+++ b/hieradata/hosts/snapshot1005.yaml
@@ -1 +1,2 @@
snapshot::dumps::runtype: enwiki
+snapshot::dumps::php: php5
diff --git a/hieradata/hosts/snapshot1006.yaml
b/hieradata/hosts/snapshot1006.yaml
index ea234d3..0480659 100644
--- a/hieradata/hosts/snapshot1006.yaml
+++ b/hieradata/hosts/snapshot1006.yaml
@@ -1 +1,2 @@
snapshot::dumps::runtype: wikidatawiki
+snapshot::dumps::php: php5
diff --git a/hieradata/hosts/snapshot1007.yaml
b/hieradata/hosts/snapshot1007.yaml
index d27cca6..9015ce4 100644
--- a/hieradata/hosts/snapshot1007.yaml
+++ b/hieradata/hosts/snapshot1007.yaml
@@ -1 +1,2 @@
snapshot::dumps::runtype: regular
+snapshot::dumps::php: php5
diff --git a/modules/profile/manifests/dumps/generation/worker/common.pp
b/modules/profile/manifests/dumps/generation/worker/common.pp
index 940a017..b1d9150 100644
--- a/modules/profile/manifests/dumps/generation/worker/common.pp
+++ b/modules/profile/manifests/dumps/generation/worker/common.pp
@@ -1,6 +1,7 @@
class profile::dumps::generation::worker::common(
$nfs_server = hiera('dumps_nfs_server'),
$managed_subdirs = hiera('dumps_managed_subdirs'),
+ $php = hiera('snapshot::dumps::php'),
) {
# mw packages and dependencies
require ::profile::mediawiki::scap_proxy
@@ -29,7 +30,7 @@
cronsdir => "${xmldumpsmount}/otherdumps",
apachedir => '/srv/mediawiki',
}
- class { '::snapshot::dumps': }
+ class { '::snapshot::dumps': php => $php}
# scap3 deployment of dump scripts
scap::target { 'dumps/dumps':
diff --git a/modules/snapshot/files/cron/dump-global-blocks.sh
b/modules/snapshot/files/cron/dump-global-blocks.sh
index c3d2baf..16d2cef 100644
--- a/modules/snapshot/files/cron/dump-global-blocks.sh
+++ b/modules/snapshot/files/cron/dump-global-blocks.sh
@@ -6,12 +6,16 @@
source /usr/local/etc/dump_functions.sh
+args="tools:php"
+results=`python "${repodir}/getconfigvals.py" --configfile "$configfile"
--args "$args"`
+php=`getsetting "$results" "tools" "php"` || exit 1
+
get_db_host() {
apachedir=$1
multiversionscript="${apachedir}/multiversion/MWScript.php"
if [ -e "$multiversionscript" ]; then
- host=`php -q "$multiversionscript"
extensions/CentralAuth/maintenance/getCentralAuthDBInfo.php --wiki="aawiki"` ||
(echo $host >& 2; host="")
+ host=`$php -q "$multiversionscript"
extensions/CentralAuth/maintenance/getCentralAuthDBInfo.php --wiki="aawiki"` ||
(echo $host >& 2; host="")
fi
if [ -z "$host" ]; then
echo "can't locate db server for centralauth, exiting." >& 2
@@ -25,7 +29,7 @@
multiversionscript="${apachedir}/multiversion/MWScript.php"
if [ -e "$multiversionscript" ]; then
- db_user=`echo 'echo $wgDBadminuser;' | php "$multiversionscript"
eval.php aawiki`
+ db_user=`echo 'echo $wgDBadminuser;' | $php "$multiversionscript"
eval.php aawiki`
fi
if [ -z "$db_user" ]; then
echo "can't get db user name, exiting." >& 2
@@ -39,7 +43,7 @@
multiversionscript="${apachedir}/multiversion/MWScript.php"
if [ -e "$multiversionscript" ]; then
- db_pass=`echo 'echo $wgDBadminpassword;' | php "$multiversionscript"
eval.php aawiki`
+ db_pass=`echo 'echo $wgDBadminpassword;' | $php "$multiversionscript"
eval.php aawiki`
fi
if [ -z "$db_pass" ]; then
echo "can't get db password, exiting." >& 2
diff --git a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
index 898d088..665389a 100755
--- a/modules/snapshot/files/cron/dumpcategoriesrdf.sh
+++ b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
@@ -52,11 +52,12 @@
exit 1
fi
-args="wiki:privatelist;tools:gzip"
+args="wiki:privatelist;tools:gzip,php"
results=`python "${repodir}/getconfigvals.py" --configfile "$configFile"
--args "$args"`
privateList=`getsetting "$results" "wiki" "privatelist"` || exit 1
gzip=`getsetting "$results" "tools" "gzip"` || exit 1
+php=`getsetting "$results" "tools" "php"` || exit 1
for settingname in "gzip" "privateList"; do
checkval "$settingname" "${!settingname}"
@@ -93,9 +94,9 @@
targetFile="${targetDir}/${filename}.${dumpFormat}.gz"
tsFile="${timestampsDir}/${wiki}-categories.last"
if [ "$dryrun" == "true" ]; then
- echo "php $multiVersionScript
maintenance/dumpCategoriesAsRdf.php --wiki=$wiki --format=$dumpFormat 2>
/var/log/categoriesrdf/${filename}.log | $gzip > $targetFile"
+ echo "$php $multiVersionScript
maintenance/dumpCategoriesAsRdf.php --wiki=$wiki --format=$dumpFormat 2>
/var/log/categoriesrdf/${filename}.log | $gzip > $targetFile"
else
- php "$multiVersionScript"
maintenance/dumpCategoriesAsRdf.php --wiki="$wiki" --format="$dumpFormat" 2>
"/var/log/categoriesrdf/${filename}.log" | "$gzip" > "$targetFile"
+ $php "$multiVersionScript"
maintenance/dumpCategoriesAsRdf.php --wiki="$wiki" --format="$dumpFormat" 2>
"/var/log/categoriesrdf/${filename}.log" | "$gzip" > "$targetFile"
echo "$today" > "$tsFile"
fi
fi
diff --git a/modules/snapshot/files/cron/dumpcirrussearch.sh
b/modules/snapshot/files/cron/dumpcirrussearch.sh
index 9731e23..c913014 100644
--- a/modules/snapshot/files/cron/dumpcirrussearch.sh
+++ b/modules/snapshot/files/cron/dumpcirrussearch.sh
@@ -40,12 +40,13 @@
exit 1
fi
-args="wiki:dblist,privatelist;tools:gzip"
+args="wiki:dblist,privatelist;tools:gzip,php"
results=`python "${repodir}/getconfigvals.py" --configfile "$configFile"
--args "$args"`
allList=`getsetting "$results" "wiki" "dblist"` || exit 1
privateList=`getsetting "$results" "wiki" "privatelist"` || exit 1
gzip=`getsetting "$results" "tools" "gzip"` || exit 1
+php=`getsetting "$results" "tools" "php"` || exit 1
for settingname in "allList" "privateList" "gzip"; do
checkval "$settingname" "${!settingname}"
@@ -82,9 +83,9 @@
filename="$wiki-$today-cirrussearch-$type"
targetFile="$targetDir/$filename.json.gz"
if [ "$dryrun" == "true" ]; then
- echo "php $multiVersionScript
extensions/CirrusSearch/maintenance/dumpIndex.php --wiki=$wiki
--indexType=$type 2> /var/log/cirrusdump/cirrusdump-$filename.log | $gzip >
$targetFile"
+ echo "$php $multiVersionScript
extensions/CirrusSearch/maintenance/dumpIndex.php --wiki=$wiki
--indexType=$type 2> /var/log/cirrusdump/cirrusdump-$filename.log | $gzip >
$targetFile"
else
- php $multiVersionScript \
+ $php $multiVersionScript \
extensions/CirrusSearch/maintenance/dumpIndex.php \
--wiki=$wiki \
--indexType=$type \
@@ -104,9 +105,9 @@
filename="cirrus-metastore-$cluster-$today"
targetFile="$targetDir/$filename.json.gz"
if [ "$dryrun" == "true" ]; then
- echo "php $multiVersionScript
extensions/CirrusSearch/maintenance/metastore.php --wiki=metawiki --dump
--cluster=$cluster 2>> /var/log/cirrusdump/cirrusdump-$filename.log | $gzip >
$targetFile"
+ echo "$php $multiVersionScript
extensions/CirrusSearch/maintenance/metastore.php --wiki=metawiki --dump
--cluster=$cluster 2>> /var/log/cirrusdump/cirrusdump-$filename.log | $gzip >
$targetFile"
else
- php $multiVersionScript \
+ $php $multiVersionScript \
extensions/CirrusSearch/maintenance/metastore.php \
--wiki=metawiki \
--dump \
diff --git a/modules/snapshot/files/cron/dumpwikidatajson.sh
b/modules/snapshot/files/cron/dumpwikidatajson.sh
index 859fd72..7841c25 100644
--- a/modules/snapshot/files/cron/dumpwikidatajson.sh
+++ b/modules/snapshot/files/cron/dumpwikidatajson.sh
@@ -30,7 +30,7 @@
set -o pipefail
errorLog=/var/log/wikidatadump/dumpwikidatajson-$filename-$i.log
# Remove --no-cache once this runs on hhvm (or
everything is back on Zend), see T180048.
- php5 $multiversionscript
extensions/Wikibase/repo/maintenance/dumpJson.php --wiki wikidatawiki --shard
$i --sharding-factor $shards --batch-size `expr $shards \* 250` --snippet 2
--no-cache 2>> $errorLog | gzip -9 > $tempDir/wikidataJson.$i.gz
+ $php $multiversionscript
extensions/Wikibase/repo/maintenance/dumpJson.php --wiki wikidatawiki --shard
$i --sharding-factor $shards --batch-size `expr $shards \* 250` --snippet 2
--no-cache 2>> $errorLog | gzip -9 > $tempDir/wikidataJson.$i.gz
exitCode=$?
if [ $exitCode -gt 0 ]; then
echo -e "\n\n(`date --iso-8601=minutes`)
Process for shard $i failed with exit code $exitCode" >> $errorLog
diff --git a/modules/snapshot/files/cron/dumpwikidatardf.sh
b/modules/snapshot/files/cron/dumpwikidatardf.sh
index 92313bd..9e18edd 100755
--- a/modules/snapshot/files/cron/dumpwikidatardf.sh
+++ b/modules/snapshot/files/cron/dumpwikidatardf.sh
@@ -57,7 +57,7 @@
set -o pipefail
errorLog=/var/log/wikidatadump/dumpwikidata$dumpFormat-$filename-$i.log
# Remove --no-cache once this runs on hhvm (or
everything is back on Zend), see T180048.
- php5 $multiversionscript
extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wikidatawiki --shard $i
--sharding-factor $shards --batch-size `expr $shards \* 250` --format
$dumpFormat --flavor $dumpFlavor --no-cache 2>> $errorLog | gzip -9 >
$tempDir/wikidata$dumpFormat-$dumpName.$i.gz
+ $php $multiversionscript
extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wikidatawiki --shard $i
--sharding-factor $shards --batch-size `expr $shards \* 250` --format
$dumpFormat --flavor $dumpFlavor --no-cache 2>> $errorLog | gzip -9 >
$tempDir/wikidata$dumpFormat-$dumpName.$i.gz
exitCode=$?
if [ $exitCode -gt 0 ]; then
echo -e "\n\n(`date --iso-8601=minutes`)
Process for shard $i failed with exit code $exitCode" >> $errorLog
diff --git a/modules/snapshot/files/cron/wikidatadumps-shared.sh
b/modules/snapshot/files/cron/wikidatadumps-shared.sh
index 1dac803..bbaf598 100644
--- a/modules/snapshot/files/cron/wikidatadumps-shared.sh
+++ b/modules/snapshot/files/cron/wikidatadumps-shared.sh
@@ -14,10 +14,11 @@
today=`date +'%Y%m%d'`
daysToKeep=70
-args="output:temp"
+args="output:temp;tools:php"
results=`python "${repodir}/getconfigvals.py" --configfile "$configfile"
--args "$args"`
tempDir=`getsetting "$results" "output" "temp"` || exit 1
+php=`getsetting "$results" "tools" "php"` || exit 1
for settingname in "tempDir"; do
checkval "$settingname" "${!settingname}"
@@ -37,5 +38,5 @@
}
function runDcat {
- php /usr/local/share/dcat/DCAT.php
--config=/usr/local/etc/dcatconfig.json --dumpDir=$targetDirBase
--outputDir=$targetDirBase
+ $php /usr/local/share/dcat/DCAT.php
--config=/usr/local/etc/dcatconfig.json --dumpDir=$targetDirBase
--outputDir=$targetDirBase
}
diff --git a/modules/snapshot/manifests/dumps.pp
b/modules/snapshot/manifests/dumps.pp
index c96b929..d7b1509 100644
--- a/modules/snapshot/manifests/dumps.pp
+++ b/modules/snapshot/manifests/dumps.pp
@@ -1,6 +1,8 @@
-class snapshot::dumps {
+class snapshot::dumps(
+ $php = undef,
+) {
include ::snapshot::dumps::packages
- include ::snapshot::dumps::configs
+ class {'::snapshot::dumps::configs': php => $php}
include ::snapshot::dumps::dblists
include ::snapshot::dumps::templates
include ::snapshot::dumps::stagesconfig
diff --git a/modules/snapshot/manifests/dumps/configs.pp
b/modules/snapshot/manifests/dumps/configs.pp
index 36ce427..9140a54 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -1,4 +1,6 @@
-class snapshot::dumps::configs {
+class snapshot::dumps::configs(
+ $php = undef,
+) {
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
$apachedir = $snapshot::dumps::dirs::apachedir
$confsdir = $snapshot::dumps::dirs::confsdir
diff --git a/modules/snapshot/templates/dumps/wikidump.conf.erb
b/modules/snapshot/templates/dumps/wikidump.conf.erb
index 2024be7..fe04686 100644
--- a/modules/snapshot/templates/dumps/wikidump.conf.erb
+++ b/modules/snapshot/templates/dumps/wikidump.conf.erb
@@ -34,8 +34,7 @@
max_allowed_packet=32M
[tools]
-php=/usr/bin/php5
-#php=/usr/bin/php
+php=<%= scope.lookupvar('snapshot::dumps::configs::php') %>
mysql=/usr/bin/mysql
mysqldump=/usr/bin/mysqldump
gzip=/bin/gzip
--
To view, visit https://gerrit.wikimedia.org/r/400692
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4c8a768be76db30501790e5d31af8ca9cc1c4f52
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
