[MediaWiki-commits] [Gerrit] operations/puppet[production]: add dumps config settings stanzas for labs testing

[ArielGlenn (Code Review)]

ArielGlenn has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402633 )

Change subject: add dumps config settings stanzas for labs testing
......................................................................

add dumps config settings stanzas for labs testing

Change-Id: Idf94b66efd1ceb3b3aace3fd56022b086626aaf0
---
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/manifests/dumps/dblists.pp
M modules/snapshot/templates/dumps/wikidump.conf.erb
3 files changed, 68 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/33/402633/1

diff --git a/modules/snapshot/manifests/dumps/configs.pp 
b/modules/snapshot/manifests/dumps/configs.pp
index bd510ec..cfad2ad 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -137,6 +137,45 @@
             media => {
                 skipdblist    => 
"${dblistsdir}/skipmonitor.dblist,${dblistsdir}/globalusage.dblist",
             }
+            labs_small => {
+                dblist        => "${apachedir}/dblists/all-labs.dblist",
+                closedlist    => "${apachedir}/dblists/closed-labs.dblist",
+                flowlist      => "${apachedir}/dblists/flow_only_labs.dblist",
+                skipdblist    => "${dblistsdir}/skip-labs.dblist",
+                keep          => '2',
+                chunksEnabled => '0',
+            },
+            labs_big => {
+                dblist            => "${dblistsdir}/labs_bigwikis.dblist",
+                closedlist        => "${apachedir}/dblists/closed-labs.dblist",
+                flowlist          => 
"${apachedir}/dblists/flow_only_labs.dblist",
+                skipdblist        => "${dblistsdir}/skipmonitor.dblist",
+                keep              => '2',
+                chunksEnabled     => '1',
+                chunksForAbstract => '4',
+                chunksForPagelogs => '4',
+                checkpointTime    => '720',
+                recombineHistory  => '0',
+                revsPerJob        => '70000',
+                retryWait         => '30',
+                maxRetries        => '3',
+                revsMargin        => '20',
+                fixeddumporder    => '1',
+                wikis => {
+                    enwiki => {
+                        pagesPerChunkHistory  => '20000,40000,70000,80000',
+                        pagesPerChunkAbstract => '4000',
+                    },
+                    simplewiki => {
+                        pagesPerChunkHistory  => '20000,50000,90000,140000',
+                        pagesPerChunkAbstract => '60000',
+                    },
+                    wikidatawikiwiki => {
+                        pagesPerChunkHistory  => '540000,30000,20000,30000',
+                        pagesPerChunkAbstract => '25000',
+                    },
+                },
+            },
         },
     }
 
diff --git a/modules/snapshot/manifests/dumps/dblists.pp 
b/modules/snapshot/manifests/dumps/dblists.pp
index d28843e..3dcb1b9 100644
--- a/modules/snapshot/manifests/dumps/dblists.pp
+++ b/modules/snapshot/manifests/dumps/dblists.pp
@@ -10,11 +10,16 @@
                 'zhwiki']
     $bigwikis_dblist = join($bigwikis, "\n")
 
+    # for testing in deployment-prep
+    $labs_bigwikis = ['dewiki', 'simplewiki', 'wikidatawiki']
+    $labs_bigwikis_dblist = join($bigwikis, "\n")
+
     # labswiki(s) can't be dumped from snapshot hosts
     $excludewikis = ['labswiki', 'labtestwiki']
     $excludewikis_dblist = join($excludewikis, "\n")
 
     $skip_dblist = 
"${enwiki_dblist}\n${wikidatawiki_dblist}\n${bigwikis_dblist}\n${excludewikis_dblist}"
+    $skip_labs_dblist = "${labs_bigwikis_dblist}"
 
     $skipnone_dblist = ''
 
@@ -54,6 +59,14 @@
         group   => 'root',
         content => "${skip_dblist}\n",
     }
+    file { "${dblistsdir}/skip-labs.dblist":
+        ensure  => 'present',
+        path    => "${dblistsdir}/skip-labs.dblist",
+        mode    => '0644',
+        owner   => 'root',
+        group   => 'root',
+        content => "${skip_labs_dblist}\n",
+    }
     file { "${dblistsdir}/skipmonitor.dblist":
         ensure  => 'present',
         path    => "${dblistsdir}/skipmonitor.dblist",
diff --git a/modules/snapshot/templates/dumps/wikidump.conf.erb 
b/modules/snapshot/templates/dumps/wikidump.conf.erb
index fe04686..d893c99 100644
--- a/modules/snapshot/templates/dumps/wikidump.conf.erb
+++ b/modules/snapshot/templates/dumps/wikidump.conf.erb
@@ -118,6 +118,22 @@
 <% end -%>
 <% end -%>
 
+[labssmall]
+# generic settings for small wikis in labs
+<% @config[@configtype]['labs_small'].keys.sort.each do |bigsetting| -%>
+<% if bigsetting != 'wikis' -%>
+<%= bigsetting %>=<%= @config[@configtype]['labs_small'][bigsetting] %>
+<% end -%>
+<% end -%>
+
+[labsbig]
+# generic settings for big wikis in labs
+<% @config[@configtype]['labs_big'].keys.sort.each do |bigsetting| -%>
+<% if bigsetting != 'wikis' -%>
+<%= bigsetting %>=<%= @config[@configtype]['labs_big'][bigsetting] %>
+<% end -%>
+<% end -%>
+
 ########################
 # wiki-specific settings
 

-- 
To view, visit https://gerrit.wikimedia.org/r/402633
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Idf94b66efd1ceb3b3aace3fd56022b086626aaf0
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits