Hoo man has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/347838 )
Change subject: Allow running two dumpwikidatattl dumps side by side
......................................................................
Allow running two dumpwikidatattl dumps side by side
Vary file names by dump name.
Bug: T155103
Change-Id: Ie4e9f34a384089080b7cacaac00b74eaf9c41c9f
---
M modules/snapshot/files/cron/dumpwikidatattl.sh
1 file changed, 8 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/38/347838/1
diff --git a/modules/snapshot/files/cron/dumpwikidatattl.sh
b/modules/snapshot/files/cron/dumpwikidatattl.sh
index db0ac07..59cc24e 100644
--- a/modules/snapshot/files/cron/dumpwikidatattl.sh
+++ b/modules/snapshot/files/cron/dumpwikidatattl.sh
@@ -18,7 +18,7 @@
filename=wikidata-$today-$dumpName-BETA
targetFileGzip=$targetDir/$filename.ttl.gz
targetFileBzip2=$targetDir/$filename.ttl.bz2
-failureFile=/tmp/dumpwikidatattl-failure
+failureFile=/tmp/dumpwikidatattl-$dumpName-failure
mainLogFile=/var/log/wikidatadump/dumpwikidatattl-$filename-main.log
if [ -z $dumpFlavor ]; then
@@ -39,7 +39,7 @@
(
set -o pipefail
errorLog=/var/log/wikidatadump/dumpwikidatattl-$filename-$i.log
- php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --format ttl --flavor
$dumpFlavor 2>> $errorLog | gzip > $tempDir/wikidataTTL.$i.gz
+ php5 $multiversionscript
extensions/Wikidata/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki
wikidatawiki --shard $i --sharding-factor $shards --format ttl --flavor
$dumpFlavor 2>> $errorLog | gzip > $tempDir/wikidataTTL-$dumpName.$i.gz
exitCode=$?
if [ $exitCode -gt 0 ]; then
echo -e "\n\n(`date --iso-8601=minutes`)
Process for shard $i failed with exit code $exitCode" >> $errorLog
@@ -57,7 +57,7 @@
if [ -f $failureFile ]; then
# Something went wrong, let's clean up and maybe retry. Leave
logs in place.
rm -f $failureFile
- rm -f $tempDir/wikidataTTL.*.gz
+ rm -f $tempDir/wikidataTTL-$dumpName.*.gz
let retries++
echo "(`date --iso-8601=minutes`) Dumping one or more shards
failed. Retrying." >> $mainLogFile
@@ -76,7 +76,7 @@
i=0
while [ $i -lt $shards ]; do
- tempFile=$tempDir/wikidataTTL.$i.gz
+ tempFile=$tempDir/wikidataTTL-$dumpName.$i.gz
if [ ! -f $tempFile ]; then
echo "$tempFile does not exist. Aborting." >> $mainLogFile
exit 1
@@ -86,15 +86,15 @@
echo "File size of $tempFile is only $fileSize. Aborting." >>
$mainLogFile
exit 1
fi
- cat $tempFile >> $tempDir/wikidataTtl.gz
+ cat $tempFile >> $tempDir/wikidataTtl-$dumpName.gz
rm $tempFile
let i++
done
-mv $tempDir/wikidataTtl.gz $targetFileGzip
+mv $tempDir/wikidataTtl-$dumpName.gz $targetFileGzip
-gzip -dc $targetFileGzip | bzip2 -c > $tempDir/wikidataTtl.bz2
-mv $tempDir/wikidataTtl.bz2 $targetFileBzip2
+gzip -dc $targetFileGzip | bzip2 -c > $tempDir/wikidataTtl-$dumpName.bz2
+mv $tempDir/wikidataTtl-$dumpName.bz2 $targetFileBzip2
pruneOldDirectories
pruneOldLogs
--
To view, visit https://gerrit.wikimedia.org/r/347838
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie4e9f34a384089080b7cacaac00b74eaf9c41c9f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Hoo man <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
