ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/263661
Change subject: dumps: add nocreate versions of stages files for huge wikis
......................................................................
dumps: add nocreate versions of stages files for huge wikis
also run 8 createdir processes at once for small/big wikis instead of 1
and fix up name of createdirs stage file for huge wikis to be
consistent with the other stage files for huge wikis
Change-Id: I71161ba044c55990b75d37d550f2b6132c322581
---
M modules/snapshot/manifests/dumps/stagesconfig.pp
M modules/snapshot/templates/dumpstages.erb
2 files changed, 14 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/61/263661/1
diff --git a/modules/snapshot/manifests/dumps/stagesconfig.pp
b/modules/snapshot/manifests/dumps/stagesconfig.pp
index 407e62a..86d8a92 100644
--- a/modules/snapshot/manifests/dumps/stagesconfig.pp
+++ b/modules/snapshot/manifests/dumps/stagesconfig.pp
@@ -73,11 +73,19 @@
stagestype => 'partial_huge',
stages => $stages,
}
+ snapshot::dumps::stagesconf { 'stages_normal_nocreate_hugewikis':
+ stagestype => 'normal_nocreate_huge',
+ stages => $stages,
+ }
+ snapshot::dumps::stagesconf { 'stages_partial_nocreate_hugewikis':
+ stagestype => 'partial_nocreate_huge',
+ stages => $stages,
+ }
snapshot::dumps::stagesconf { 'stages_create':
stagestype => 'create',
stages => $stages,
}
- snapshot::dumps::stagesconf { 'stages_create_huge':
+ snapshot::dumps::stagesconf { 'stages_create_hugewikis':
stagestype => 'create_huge',
stages => $stages,
}
diff --git a/modules/snapshot/templates/dumpstages.erb
b/modules/snapshot/templates/dumpstages.erb
index 786c479..b6b0ab1 100644
--- a/modules/snapshot/templates/dumpstages.erb
+++ b/modules/snapshot/templates/dumpstages.erb
@@ -9,7 +9,7 @@
<% if @stagestype == 'normal' or @stagestype == 'partial' or @stagestype ==
'create' %>
# mark the start of the run for all small, big wikis
-8 1 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs
--sleep 5; <%= @stages['bigwikis']['firststage'] %> --job createdirs --sleep 5
+8 8 continue none <%= @stages['smallwikis']['firststage'] -%> --job createdirs
--sleep 5; <%= @stages['bigwikis']['firststage'] %> --job createdirs --sleep 5
<% end -%>
@@ -48,8 +48,8 @@
<% end -%>
-<% if @stagestype == 'normal_huge' or @stagestype == 'partial_huge' %>
-# stubs first
+<% if @stagestype == 'normal_huge' or @stagestype == 'partial_huge' or
@stagestype == 'normal_huge_nocreate' or @stagestype == 'partial_huge_nocreate'
%>
+# stubs
27 1 continue none <%= @stages['hugewikis']['rest'] -%> --job xmlstubsdump
# tables next so inconsistencies between stubs and tables aren't too huge
1 1 continue none <%= @stages['hugewikis']['rest'] -%> --job tables
@@ -69,12 +69,12 @@
# recombine articles plus meta pages for huge wikis
1 1 continue none <%= @stages['hugewikis']['rest'] -%> --job
metacurrentdumprecombine
-<% if @stagestype == 'normal_huge' %>
+<% if @stagestype == 'normal_huge' or @stagestype == 'normal_huge_nocreate' %>
# all remaining jobs
27 1 continue none <%= @stages['hugewikis']['rest'] %>
<% end -%>
-<% if @stagestype == 'partial_huge' %>
+<% if @stagestype == 'partial_huge' or @stagestype == 'partial_huge_nocreate'
%>
# all remaining jobs except for the history revs
27 1 continue none <%= @stages['hugewikis']['rest'] %> <%=
@stages['skipjob_args'] %>
<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/263661
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I71161ba044c55990b75d37d550f2b6132c322581
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
