ArielGlenn has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/324702

Change subject: allow dumps of private tables to be skipped via config setting
......................................................................

allow dumps of private tables to be skipped via config setting

Bug: T152021
Change-Id: If1a5d0c8b2fd8b57ffb434fd19904c0ed554427c
---
M xmldumps-backup/dumps/WikiDump.py
M xmldumps-backup/dumps/runner.py
2 files changed, 56 insertions(+), 51 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/dumps 
refs/changes/02/324702/1

diff --git a/xmldumps-backup/dumps/WikiDump.py 
b/xmldumps-backup/dumps/WikiDump.py
index 01a1cfb..08de958 100644
--- a/xmldumps-backup/dumps/WikiDump.py
+++ b/xmldumps-backup/dumps/WikiDump.py
@@ -56,6 +56,7 @@
             "mailfrom": "root@localhost",
             "smtpserver": "localhost",
             "staleage": "3600",
+            "skipprivatetables": "0",
             # "database": {
             # these are now set in get_db_user_and_password() if needed
             "user": "",
@@ -232,6 +233,7 @@
         self.mail_from = self.conf.get("reporting", "mailfrom")
         self.smtp_server = self.conf.get("reporting", "smtpserver")
         self.stale_age = self.conf.getint("reporting", "staleage")
+        self.skip_privatetables = self.conf.getint("reporting", 
"skipprivatetables")
 
         if not self.conf.has_section('tools'):
             self.conf.add_section('tools')
diff --git a/xmldumps-backup/dumps/runner.py b/xmldumps-backup/dumps/runner.py
index 50c9b90..1f8e689 100644
--- a/xmldumps-backup/dumps/runner.py
+++ b/xmldumps-backup/dumps/runner.py
@@ -148,59 +148,62 @@
                 raise BackupError("You cannot specify a checkpoint file with 
the job %s, exiting.\n"
                                   % self._single_job)
 
-        self.dump_items = [PrivateTable("user", "usertable", "User account 
data."),
-                           PrivateTable("watchlist", "watchlisttable",
-                                        "Users' watchlist settings."),
-                           PrivateTable("ipblocks", "ipblockstable",
-                                        "Data for blocks of IP addresses, 
ranges, and users."),
-                           PrivateTable("archive", "archivetable",
-                                        "Deleted page and revision data."),
-                           PrivateTable("logging", "loggingtable",
-                                        "Data for various events (deletions, 
uploads, etc)."),
-                           PrivateTable("oldimage", "oldimagetable",
-                                        "Metadata on prior versions of 
uploaded images."),
+        if self.wiki.config.skip_privatetables:
+            self.dump_items = []
+        else:
+            self.dump_items = [PrivateTable("user", "usertable", "User account 
data."),
+                               PrivateTable("watchlist", "watchlisttable",
+                                            "Users' watchlist settings."),
+                               PrivateTable("ipblocks", "ipblockstable",
+                                            "Data for blocks of IP addresses, 
ranges, and users."),
+                               PrivateTable("archive", "archivetable",
+                                            "Deleted page and revision data."),
+                               PrivateTable("logging", "loggingtable",
+                                            "Data for various events 
(deletions, uploads, etc)."),
+                               PrivateTable("oldimage", "oldimagetable",
+                                            "Metadata on prior versions of 
uploaded images.")]
 
-                           PublicTable("site_stats", "sitestatstable",
-                                       "A few statistics such as the page 
count."),
-                           PublicTable("image", "imagetable",
-                                       "Metadata on current versions of 
uploaded media/files."),
-                           PublicTable("pagelinks", "pagelinkstable",
-                                       "Wiki page-to-page link records."),
-                           PublicTable("categorylinks", "categorylinkstable",
-                                       "Wiki category membership link 
records."),
-                           PublicTable("imagelinks", "imagelinkstable",
-                                       "Wiki media/files usage records."),
-                           PublicTable("templatelinks", "templatelinkstable",
-                                       "Wiki template inclusion link 
records."),
-                           PublicTable("externallinks", "externallinkstable",
-                                       "Wiki external URL link records."),
-                           PublicTable("langlinks", "langlinkstable",
-                                       "Wiki interlanguage link records."),
-                           PublicTable("user_groups", "usergroupstable", "User 
group assignments."),
-                           PublicTable("category", "categorytable", "Category 
information."),
+        self.dump_items.extend([PublicTable("site_stats", "sitestatstable",
+                                            "A few statistics such as the page 
count."),
+                                PublicTable("image", "imagetable",
+                                            "Metadata on current versions of 
uploaded media/files."),
+                                PublicTable("pagelinks", "pagelinkstable",
+                                            "Wiki page-to-page link records."),
+                                PublicTable("categorylinks", 
"categorylinkstable",
+                                            "Wiki category membership link 
records."),
+                                PublicTable("imagelinks", "imagelinkstable",
+                                            "Wiki media/files usage records."),
+                                PublicTable("templatelinks", 
"templatelinkstable",
+                                            "Wiki template inclusion link 
records."),
+                                PublicTable("externallinks", 
"externallinkstable",
+                                            "Wiki external URL link records."),
+                                PublicTable("langlinks", "langlinkstable",
+                                            "Wiki interlanguage link 
records."),
+                                PublicTable("user_groups", "usergroupstable", 
"User group assignments."),
+                                PublicTable("category", "categorytable", 
"Category information."),
 
-                           PublicTable("page", "pagetable",
-                                       "Base per-page data (id, title, old 
restrictions, etc)."),
-                           PublicTable("page_restrictions", 
"pagerestrictionstable",
-                                       "Newer per-page restrictions table."),
-                           PublicTable("page_props", "pagepropstable",
-                                       "Name/value pairs for pages."),
-                           PublicTable("protected_titles", 
"protectedtitlestable",
-                                       "Nonexistent pages that have been 
protected."),
-                           PublicTable("redirect", "redirecttable", "Redirect 
list"),
-                           PublicTable("iwlinks", "iwlinkstable",
-                                       "Interwiki link tracking records"),
-                           PublicTable("geo_tags", "geotagstable",
-                                       "List of pages' geographical 
coordinates"),
-                           PublicTable("change_tag", "changetagstable",
-                                       "List of annotations (tags) for 
revisions and log entries"),
+                                PublicTable("page", "pagetable",
+                                            "Base per-page data (id, title, 
old restrictions, etc)."),
+                                PublicTable("page_restrictions", 
"pagerestrictionstable",
+                                            "Newer per-page restrictions 
table."),
+                                PublicTable("page_props", "pagepropstable",
+                                            "Name/value pairs for pages."),
+                                PublicTable("protected_titles", 
"protectedtitlestable",
+                                            "Nonexistent pages that have been 
protected."),
+                                PublicTable("redirect", "redirecttable", 
"Redirect list"),
+                                PublicTable("iwlinks", "iwlinkstable",
+                                            "Interwiki link tracking records"),
+                                PublicTable("geo_tags", "geotagstable",
+                                            "List of pages' geographical 
coordinates"),
+                                PublicTable("change_tag", "changetagstable",
+                                            "List of annotations (tags) for 
revisions and log entries"),
 
-                           TitleDump("pagetitlesdump", "List of page titles in 
main namespace"),
-                           AllTitleDump("allpagetitlesdump", "List of all page 
titles"),
+                                TitleDump("pagetitlesdump", "List of page 
titles in main namespace"),
+                                AllTitleDump("allpagetitlesdump", "List of all 
page titles"),
 
-                           AbstractDump("abstractsdump", "Extracted page 
abstracts for Yahoo",
-                                        
self._get_partnum_todo("abstractsdump"), self.wiki.db_name,
-                                        
self.filepart.get_pages_per_filepart_abstract())]
+                                AbstractDump("abstractsdump", "Extracted page 
abstracts for Yahoo",
+                                             
self._get_partnum_todo("abstractsdump"), self.wiki.db_name,
+                                             
self.filepart.get_pages_per_filepart_abstract())])
 
         self.append_job_if_needed(RecombineAbstractDump(
             "abstractsdumprecombine", "Recombine extracted page abstracts for 
Yahoo",
@@ -526,7 +529,7 @@
             if self._partnum_todo is None and fname.partnum_int:
                 self._partnum_todo = fname.partnum_int
             elif (self._partnum_todo is not None and fname.partnum_int and
-                    self._partnum_todo != fname.partnum_int):
+                  self._partnum_todo != fname.partnum_int):
                 raise BackupError("specifed partnum to do does not match part 
number "
                                   "of checkpoint file %s to redo", 
self.checkpoint_file)
             self.checkpoint_file = fname
@@ -537,7 +540,7 @@
                         RunInfoFile.NAME, SymLinks.NAME, RunSettings.NAME,
                         Feeds.NAME, NoticeFile.NAME, "makedir", 
"clean_old_dumps",
                         "cleanup_old_files", "check_trunc_files"]:
-                self.enabled[setting] = True
+            self.enabled[setting] = True
 
         if not self.cleanup_old_files:
             if "cleanup_old_files" in self.enabled:

-- 
To view, visit https://gerrit.wikimedia.org/r/324702
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If1a5d0c8b2fd8b57ffb434fd19904c0ed554427c
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to