ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/325947 )
Change subject: fix up silly handling of table job names
......................................................................
fix up silly handling of table job names
they should all end in 'table' but this can be done internally,
and now it is.
Bug: T152679
Change-Id: I24f7a41184add8b8aca44a691fd55c3e270af2ed
---
M xmldumps-backup/default_tables.yaml
M xmldumps-backup/doc/README.job_config
M xmldumps-backup/dumps/runner.py
3 files changed, 55 insertions(+), 42 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/47/325947/1
diff --git a/xmldumps-backup/default_tables.yaml
b/xmldumps-backup/default_tables.yaml
index c19da27..4e3d10a 100644
--- a/xmldumps-backup/default_tables.yaml
+++ b/xmldumps-backup/default_tables.yaml
@@ -1,137 +1,137 @@
# default list of public and private tables to dump
tables:
user:
- job: usertable
+ job: user
description: User account data.
type: private
watchlist:
- job: watchlisttable
+ job: watchlist
description: Users' watchlist settings.
type: private
ipblocks:
- job: ipblockstable
+ job: ipblocks
description: Data for blocks of IP addresses, ranges, and users.
type: private
archive:
- job: archivetable
+ job: archive
description: Deleted page and revision data.
type: private
logging:
- job: loggingtable
+ job: logging
description: Data for various events (deletions, uploads, etc).
type: private
oldimage:
- job: oldimagetable
+ job: oldimage
description: Metadata on prior versions of uploaded images.
type: private
site_stats:
- job: sitestatstable
+ job: sitestats
description: A few statistics such as the page count.
type: public
image:
- job: imagetable
+ job: image
description: Metadata on current versions of uploaded media/files.
type: public
pagelinks:
- job: pagelinkstable
+ job: pagelinks
description: Wiki page-to-page link records.
type: public
categorylinks:
- job: categorylinkstable
+ job: categorylinks
description: Wiki category membership link records.
type: public
imagelinks:
- job: imagelinkstable
+ job: imagelinks
description: Wiki media/files usage records.
type: public
templatelinks:
- job: templatelinkstable
+ job: templatelinks
description: Wiki template inclusion link records.
type: public
externallinks:
- job: externallinkstable
+ job: externallinks
description: Wiki external URL link records.
type: public
langlinks:
- job: langlinkstable
+ job: langlinks
description: Wiki interlanguage link records.
type: public
user_groups:
- job: usergroupstable
+ job: usergroups
description: User group assignments.
type: public
category:
- job: categorytable
+ job: category
description: Category information.
type: public
page:
- job: pagetable
+ job: page
description: Base per-page data (id, title, old restrictions, etc).
type: public
page_restrictions:
- job: pagerestrictionstable
+ job: pagerestrictions
description: Newer per-page restrictions table.
type: public
page_props:
- job: pagepropstable
+ job: pageprops
description: Name/value pairs for pages.
type: public
protected_titles:
- job: protectedtitlestable
+ job: protectedtitles
description: Nonexistent pages that have been protected.
type: public
redirect:
- job: redirecttable
+ job: redirect
description: Redirect list
type: public
iwlinks:
- job: iwlinkstable
+ job: iwlinks
description: Interwiki link tracking records
type: public
geo_tags:
- job: geotagstable
+ job: geotags
description: List of pages' geographical coordinates
type: public
change_tag:
- job: changetagstable
+ job: changetags
description: List of annotations (tags) for revisions and log entries
type: public
flaggedpages:
- job: flaggedpagestable
+ job: flaggedpages
description: This contains a row for each flagged article containing the
stable revision ID, if the lastest edit was flagged, and how long edits have
been pending.
flaggedrevs:
- job: flaggedrevstable
+ job: flaggedrevs
description: This contains a row for each flagged revision containing
who flagged it, when it was flagged, reviewer comments, the flag values, and
the quality tier those flags fall under.
wb_items_per_site:
- job: wbitemspersitetable
+ job: wbitemspersite
description: For each Wikidata item, this contains rows with the
corresponding page name on a given wiki project.
type: public
wb_terms:
- job: wbtermstable
+ job: wbterms
description: For each Wikidata item, this contains rows with a label, an
alias and a description of the item in a given language.
type: public
wb_entity_per_page:
- job: wbentityperpagetable
+ job: wbentityperpage
description: Contains a mapping of page ids and entity ids, with an
additional entity type column.
type: public
wb_property_info:
- job: wbpropertyinfotable
+ job: wbpropertyinfo
description: Contains a mapping of Wikidata property ids and data types.
type: public
wb_changes_subscription:
- job: wbchangessubscriptiontable
+ job: wbchangessubscription
description: Tracks which Wikibase Client wikis are using which items.
type: public
sites:
- job: sitestable
+ job: sites
description: This contains the SiteMatrix information from
meta.wikimedia.org provided as a table.
type: public
globalimagelinks:
- job: globalimagelinkstable
+ job: globalimagelinks
description: Global wiki media/files usage records.
type: public
wbc_entity_usage:
- job: wbcentityusagetable
+ job: wbcentityusage
description: Tracks which pages use which Wikidata items or properties
and what aspect (e.g. item label) is used.
type: public
diff --git a/xmldumps-backup/doc/README.job_config
b/xmldumps-backup/doc/README.job_config
index fa46d2c..3755e6b 100644
--- a/xmldumps-backup/doc/README.job_config
+++ b/xmldumps-backup/doc/README.job_config
@@ -21,9 +21,10 @@
----------------
table name: the name of the table as it appears in the sql database
job: the internal name of the job, used in status reports
- and in running specific jobs from the command line; must
- end in the string 'table' in order to be included in the
- pseudo-job 'tables'
+ and in running specific jobs from the command line;
+ note that the name will automatically have the string
+ 'table' appended to it for use by the pseudo-job
+ 'tables', you need not do this in the config
type: public or private; private tables are dumped to a location
outside of the web docroot and links to them are not
provided to downloaders
@@ -34,7 +35,7 @@
Sample stanza
-------------
watchlist:
- job: watchlisttable
+ job: watchlist
description: Users' watchlist settings.
type: private
diff --git a/xmldumps-backup/dumps/runner.py b/xmldumps-backup/dumps/runner.py
index 69c5637..7d2ef7e 100644
--- a/xmldumps-backup/dumps/runner.py
+++ b/xmldumps-backup/dumps/runner.py
@@ -101,6 +101,13 @@
return None
+def normalize_tablejob_name(jobname):
+ if jobname.endswith("table"):
+ return jobname
+ else:
+ return jobname + "table"
+
+
class DumpItemList(object):
def __init__(self, wiki, prefetch, prefetchdate, spawn, partnum_todo,
checkpoint_file,
singleJob, skip_jobs, filepart, page_id_range, dumpjobdata,
dump_dir,
@@ -153,12 +160,17 @@
continue
try:
+ # tables job names end in 'table' so stick that on
if tables_configured[table]['type'] == 'private':
- self.dump_items.append(PrivateTable(table,
tables_configured[table]['job'],
-
tables_configured[table]['description']))
+ self.dump_items.append(PrivateTable(
+ table,
+
normalize_tablejob_name(tables_configured[table]['job']),
+ tables_configured[table]['description']))
elif tables_configured[table]['type'] == 'public':
- self.dump_items.append(PublicTable(table,
tables_configured[table]['job'],
-
tables_configured[table]['description']))
+ self.dump_items.append(PublicTable(
+ table,
+
normalize_tablejob_name(tables_configured[table]['job']),
+ tables_configured[table]['description']))
else:
raise BackupError("Unknown table type in table jobs
config: " +
tables_configured[table][type])
--
To view, visit https://gerrit.wikimedia.org/r/325947
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I24f7a41184add8b8aca44a691fd55c3e270af2ed
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
