ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/386388 )
Change subject: generate one config fule for xml/sql dumps for wikis
......................................................................
generate one config fule for xml/sql dumps for wikis
We had separate files for: big wikis, enwiki, wiidatawiki, the rest.
This commit combines them all into one. No more wondering which
onfig file to use with a dumps run.
This will help to facilitate the move of xml/sql dumps to the
dumpsdata hosts.
Bug: T178893
Change-Id: I6fc98e3152693b073591da1c582aa9cb9c9b1411
---
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/manifests/dumps/wikiconf.pp
M modules/snapshot/templates/dumps/wikidump.conf.erb
3 files changed, 125 insertions(+), 156 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/88/386388/1
diff --git a/modules/snapshot/manifests/dumps/configs.pp
b/modules/snapshot/manifests/dumps/configs.pp
index a6ffb47..48d3608 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -12,131 +12,99 @@
$wikidatachunkhistory2 =
'2587436,951696,942913,837759,1568292,1293747,2018593,1461235,1797642,1487121,2012246,874850,1486799'
$config = {
- smallwikis => {
- dblist => "${apachedir}/dblists/all.dblist",
- skipdblist => "${dblistsdir}/skip.dblist",
- keep => '10',
- chunksEnabled => '0',
- },
- bigwikis => {
- dblist => "${dblistsdir}/bigwikis.dblist",
- skipdblist => "${dblistsdir}/skipnone.dblist",
- keep => '8',
- chunksEnabled => '1',
- recombineHistory => '0',
- revsPerJob => '1500000',
- retryWait => '30',
- maxRetries => '3',
- revsMargin => '100',
- wikis => {
- ruwiki => {
- pagesPerChunkHistory => '311181,1142420,1627923,3122803',
- pagesPerChunkAbstract => '1200000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- eswiki => {
- pagesPerChunkHistory => '229078,854374,2324057,3860337',
- pagesPerChunkAbstract => '1500000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- dewiki => {
- pagesPerChunkHistory => '425449,1451596,3087456,4274631',
- pagesPerChunkAbstract => '2000000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- ptwiki => {
- pagesPerChunkHistory => '158756,789502,1361027,2669897',
- pagesPerChunkAbstract => '1000000',
- chunksForAbstract => '4',
- },
- plwiki => {
- pagesPerChunkHistory => '235373,514490,957079,1923498',
- pagesPerChunkAbstract => '800000',
- chunksForAbstract => '4',
- },
- nlwiki => {
- pagesPerChunkHistory => '224145,552515,1098243,2570924',
- pagesPerChunkAbstract => '1000000',
- chunksForAbstract => '4',
- },
- frwiki => {
- pagesPerChunkHistory => '412303,1235589,2771967,5313378',
- pagesPerChunkAbstract => '1900000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- itwiki => {
- pagesPerChunkHistory => '442893,1049883,1381705,2929437',
- pagesPerChunkAbstract => '1200000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- jawiki => {
- pagesPerChunkHistory => '168815,480631,943865,1760565',
- pagesPerChunkAbstract => '800000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- commonswiki => {
- pagesPerChunkHistory =>
'6457504,9672260,12929298,17539944',
- pagesPerChunkAbstract => '11000000',
- chunksForAbstract => '4',
- checkpointTime => '720',
- },
- zhwiki => {
- pagesPerChunkHistory => '231819,564192,1300322,3112369',
- pagesPerChunkAbstract => '1300000',
- chunksForAbstract => '4',
- },
- metawiki => {
- pagesPerChunkHistory => '823386,2594392,3242670,3284506',
- pagesPerChunkAbstract => '2500000',
- chunksForAbstract => '4',
+ allwikis => {
+ global => {
+ dblist => "${apachedir}/dblists/all.dblist",
+ skipdblist => "${dblistsdir}/skip.dblist",
+ keep => '10',
+ chunksEnabled => '0',
+ },
+ big => {
+ dblist => "${dblistsdir}/bigwikis.dblist",
+ skipdblist => "${dblistsdir}/skipnone.dblist",
+ keep => '8',
+ chunksEnabled => '1',
+ chunksForAbstract => '4',
+ checkpointTime => '720',
+ recombineHistory => '0',
+ revsPerJob => '1500000',
+ retryWait => '30',
+ maxRetries => '3',
+ revsMargin => '100',
+ wikis => {
+ ruwiki => {
+ pagesPerChunkHistory =>
'311181,1142420,1627923,3122803',
+ pagesPerChunkAbstract => '1200000',
+ },
+ eswiki => {
+ pagesPerChunkHistory =>
'229078,854374,2324057,3860337',
+ pagesPerChunkAbstract => '1500000',
+ },
+ dewiki => {
+ pagesPerChunkHistory =>
'425449,1451596,3087456,4274631',
+ pagesPerChunkAbstract => '2000000',
+ },
+ ptwiki => {
+ pagesPerChunkHistory =>
'158756,789502,1361027,2669897',
+ pagesPerChunkAbstract => '1000000',
+ },
+ plwiki => {
+ pagesPerChunkHistory =>
'235373,514490,957079,1923498',
+ pagesPerChunkAbstract => '800000',
+ },
+ nlwiki => {
+ pagesPerChunkHistory =>
'224145,552515,1098243,2570924',
+ pagesPerChunkAbstract => '1000000',
+ },
+ frwiki => {
+ pagesPerChunkHistory =>
'412303,1235589,2771967,5313378',
+ pagesPerChunkAbstract => '1900000',
+ },
+ itwiki => {
+ pagesPerChunkHistory =>
'442893,1049883,1381705,2929437',
+ pagesPerChunkAbstract => '1200000',
+ },
+ jawiki => {
+ pagesPerChunkHistory =>
'168815,480631,943865,1760565',
+ pagesPerChunkAbstract => '800000',
+ },
+ commonswiki => {
+ pagesPerChunkHistory =>
'6457504,9672260,12929298,17539944',
+ pagesPerChunkAbstract => '11000000',
+ },
+ zhwiki => {
+ pagesPerChunkHistory =>
'231819,564192,1300322,3112369',
+ pagesPerChunkAbstract => '1300000',
+ },
+ metawiki => {
+ pagesPerChunkHistory =>
'823386,2594392,3242670,3284506',
+ pagesPerChunkAbstract => '2500000',
+ },
},
},
- },
- wikidatawiki => {
- dblist => "${dblistsdir}/wikidatawiki.dblist",
- skipdblist => "${dblistsdir}/skipnone.dblist",
- keep => '7',
- chunksEnabled => '1',
- recombineHistory => '0',
- checkpointTime => '720',
- revsPerJob => '1500000',
- retryWait => '30',
- maxRetries => '3',
- revsMargin => '100',
- wikis => {
+ huge => {
+ skipdblist => "${dblistsdir}/skipnone.dblist",
+ jobsperbatch => 'xmlstubsdump=9,abstractsdump=9',
+ keep => '7',
+ chunksEnabled => '1',
+ chunksForAbstract => 27,
+ recombineHistory => '0',
+ checkpointTime => '720',
+ revsPerJob => '1500000',
+ retryWait => '30',
+ maxRetries => '3',
+ revsMargin => '100',
+ maxrevs => '20000',
+ wikis => {
wikidatawiki => {
- jobsperbatch => 'xmlstubsdump=9,abstractsdump=9',
+ dblist =>
"${dblistsdir}/wikidatawiki.dblist",
pagesPerChunkHistory =>
"${wikidatachunkhistory1},${$wikidatachunkhistory2}",
- chunksForAbstract => 27,
orderrevs => '1',
minpages => '10',
- maxrevs => '20000',
},
- },
- },
- enwiki => {
- dblist => "${dblistsdir}/enwiki.dblist",
- skipdblist => "${dblistsdir}/skipnone.dblist",
- keep => '7',
- chunksEnabled => '1',
- recombineHistory => '0',
- checkpointTime => '720',
- revsPerJob => '1500000',
- retryWait => '30',
- maxRetries => '3',
- revsMargin => '100',
- wikis => {
enwiki => {
- jobsperbatch => 'xmlstubsdump=9,abstractsdump=9',
+ dblist => "${dblistsdir}/enwiki.dblist",
pagesPerChunkHistory =>
"${enchunkhistory1},${enchunkhistory2}",
- chunksForAbstract => 27,
- maxrevs => '20000',
},
},
},
@@ -154,20 +122,8 @@
},
}
- snapshot::dumps::wikiconf { 'wikidump.conf':
- configtype => 'smallwikis',
- config => $config,
- }
- snapshot::dumps::wikiconf { 'wikidump.conf.bigwikis':
- configtype => 'bigwikis',
- config => $config,
- }
- snapshot::dumps::wikiconf { 'wikidump.conf.enwiki':
- configtype => 'enwiki',
- config => $config,
- }
- snapshot::dumps::wikiconf { 'wikidump.conf.wikidatawiki':
- configtype => 'wikidatawiki',
+ snapshot::dumps::wikiconf { 'wikidump.conf.allwikis':
+ configtype => 'allwikis',
config => $config,
}
snapshot::dumps::wikiconf { 'wikidump.conf.monitor':
diff --git a/modules/snapshot/manifests/dumps/wikiconf.pp
b/modules/snapshot/manifests/dumps/wikiconf.pp
index 2e7af1d..ed8f894 100644
--- a/modules/snapshot/manifests/dumps/wikiconf.pp
+++ b/modules/snapshot/manifests/dumps/wikiconf.pp
@@ -1,5 +1,5 @@
define snapshot::dumps::wikiconf(
- $configtype = 'smallwikis',
+ $configtype = 'allwikis',
$config = undef,
) {
diff --git a/modules/snapshot/templates/dumps/wikidump.conf.erb
b/modules/snapshot/templates/dumps/wikidump.conf.erb
index 6e580d7..3aeef3d 100644
--- a/modules/snapshot/templates/dumps/wikidump.conf.erb
+++ b/modules/snapshot/templates/dumps/wikidump.conf.erb
@@ -4,10 +4,10 @@
#############################################################
[wiki]
-dblist=<%= @config[@configtype]['dblist'] %>
+dblist=<%= @config[@configtype]['global']['dblist'] %>
privatelist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir')
-%>/dblists/private.dblist
closedlist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir')
-%>/dblists/closed.dblist
-skipdblist=<%= @config[@configtype]['skipdblist'] %>
+skipdblist=<%= @config[@configtype]['global']['skipdblist'] %>
flowlist=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir')
-%>/dblists/flow.dblist
dir=<%= scope.lookupvar('snapshot::dumps::dirs::apachedir') %>
adminsettings=private/PrivateSettings.php
@@ -46,28 +46,28 @@
recompressxml=/usr/local/bin/recompressxml
[cleanup]
-keep=<%= @config[@configtype]['keep'] %>
+keep=<%= @config[@configtype]['global']['keep'] %>
[chunks]
-<% if @config[@configtype].has_key?('chunksEnabled') -%>
-chunksEnabled=<%= @config[@configtype]['chunksEnabled'] %>
+<% if @config[@configtype]['global'].has_key?('chunksEnabled') -%>
+chunksEnabled=<%= @config[@configtype]['global']['chunksEnabled'] %>
<% end -%>
-<% if @config[@configtype].has_key?('recombineHistory') -%>
+<% if @config[@configtype]['global'].has_key?('recombineHistory') -%>
recombineHistory=<%= @config[@configtype]['recombineHistory'] %>
<% end -%>
-<% if @config[@configtype].has_key?('checkpointTime') -%>
+<% if @config[@configtype]['global'].has_key?('checkpointTime') -%>
checkpointTime=<%= @config[@configtype]['checkpointTime'] %>
<% end -%>
-<% if @config[@configtype].has_key?('revsPerJob') -%>
+<% if @config[@configtype]['global'].has_key?('revsPerJob') -%>
revsPerJob=<%= @config[@configtype]['revsPerJob'] %>
<% end -%>
-<% if @config[@configtype].has_key?('retryWait') -%>
+<% if @config[@configtype]['global'].has_key?('retryWait') -%>
retryWait=<%= @config[@configtype]['retryWait'] %>
<% end -%>
-<% if @config[@configtype].has_key?('maxRetries') -%>
+<% if @config[@configtype]['global'].has_key?('maxRetries') -%>
maxRetries=<%= @config[@configtype]['maxRetries'] %>
<% end -%>
-<% if @config[@configtype].has_key?('revsMargin') -%>
+<% if @config[@configtype]['global'].has_key?('revsMargin') -%>
revsMargin=<%= @config[@configtype]['revsMargin'] %>
<% end -%>
@@ -79,27 +79,40 @@
minpages=1
maxrevs=100000
-<% if @config[@configtype].has_key?('wikis') %>
+
+########################
+# wiki-specific settings
+
+
+<% if @config[@configtype].has_key?('big') %>
+<% if @config[@configtype]['big'].has_key?('wikis') %>
<% @config[@configtype]['wikis'].keys.sort.each do |w| -%>
[<%= w %>]
-<% if @config[@configtype]['wikis'][w].has_key?('jobsperbatch') -%>
-jobsperbatch=<%= @config[@configtype]['wikis'][w]['jobsperbatch'] %>
+# generic settings for big wikis
+<% @config[@configtype]['big'].keys.sort.each do |bigsetting| -%>
+<%= bigsetting %>=<%= @config[@configtype]['big'][bigsetting] %>
<% end -%>
-pagesPerChunkHistory=<%=
@config[@configtype]['wikis'][w]['pagesPerChunkHistory'] %>
-pagesPerChunkAbstract=<%=
@config[@configtype]['wikis'][w]['pagesPerChunkAbstract'] %>
-chunksForAbstract=<%= @config[@configtype]['wikis'][w]['chunksForAbstract'] %>
-<% if @config[@configtype]['wikis'][w].has_key?('checkpointTime') -%>
-checkpointTime=<%= @config[@configtype]['wikis'][w]['checkpointTime'] %>
+# specific settings for wiki <%= w %>
+<% @config[@configtype]['big'][w].keys.sort.each do |bigwikisetting| -%>
+<%= bigwikisetting %>=<%= @config[@configtype]['big'][bigwikisetting] %>
<% end -%>
-<% if @config[@configtype]['wikis'][w].has_key?('orderrevs') -%>
-orderrevs=<%= @config[@configtype]['wikis'][w]['orderrevs'] %>
<% end -%>
-<% if @config[@configtype]['wikis'][w].has_key?('minpages') -%>
-minpages=<%= @config[@configtype]['wikis'][w]['minpages'] %>
<% end -%>
-<% if @config[@configtype]['wikis'][w].has_key?('maxrevs') -%>
-maxrevs=<%= @config[@configtype]['wikis'][w]['maxrevs'] %>
<% end -%>
+
+<% if @config[@configtype].has_key?('huge') %>
+<% if @config[@configtype]['huge'].has_key?('wikis') %>
+<% @config[@configtype]['wikis'].keys.sort.each do |w| -%>
+[<%= w %>]
+# generic settings for huge wikis
+<% @config[@configtype]['huge'].keys.sort.each do |hugesetting| -%>
+<%= hugesetting %>=<%= @config[@configtype]['big'][hugesetting] %>
+<% end -%>
+# specific settings for wiki <%= w %>
+<% @config[@configtype]['huge'][w].keys.sort.each do |hugewikisetting| -%>
+<%= hugewikisetting %>=<%= @config[@configtype]['big'][hugewikisetting] %>
+<% end -%>
+<% end -%>
<% end -%>
<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/386388
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6fc98e3152693b073591da1c582aa9cb9c9b1411
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
