Daniel Kinzler has submitted this change and it was merged.
Change subject: Added code docs, now using docstrings. Fixed a few syntax
errors. Removed trailing whitespace.
......................................................................
Added code docs, now using docstrings. Fixed a few syntax errors. Removed
trailing whitespace.
Change-Id: I0658eb7a5fdd8f873b5dc052a8332cdee3aa4417
---
M wikiparser/top-global-properties/topproperties.py
M wikiparser/top-global-properties/topproperties_r.py
2 files changed, 46 insertions(+), 36 deletions(-)
Approvals:
Daniel Kinzler: Verified; Looks good to me, approved
diff --git a/wikiparser/top-global-properties/topproperties.py
b/wikiparser/top-global-properties/topproperties.py
index beabbdc..fff8646 100644
--- a/wikiparser/top-global-properties/topproperties.py
+++ b/wikiparser/top-global-properties/topproperties.py
@@ -1,17 +1,19 @@
#!/usr/bin/python
-# The mapper script to count global property frequencies from the wikidata
dump using Hadoop.
-# Sample command to run this on Hadoop:
-# bin/hadoop jar contrib/streaming/hadoop*streaming*jar -libjars
/tmp/entity-suggester-client.jar \
-# -inputformat
org.wikimedia.wikibase.entitysuggester.wikiparser.WikiPageInputFormat \
-# -input /input/dump.xml -output /output \
-# -file /tmp/topproperties.py -mapper /tmp/topproperties.py \
-# -file /tmp/topproperties_r.py -reducer '/tmp/topproperties_r.py'
-#
-# bin/hadoop dfs -cat /output/part-* | sort -t, -k1,1nr > sortedoutput
-# Please see topproperties_r.py too
+""" The mapper script to count global property frequencies from the wikidata
dump using Hadoop.
+Sample command to run this on Hadoop:
+bin/hadoop jar contrib/streaming/hadoop*streaming*jar -libjars
/tmp/entity-suggester-client.jar \
+ -inputformat
org.wikimedia.wikibase.entitysuggester.wikiparser.WikiPageInputFormat \
+ -input /input/dump.xml -output /output \
+ -file /tmp/topproperties.py -mapper /tmp/topproperties.py \
+ -file /tmp/topproperties_r.py -reducer '/tmp/topproperties_r.py'
+bin/hadoop dfs -cat /output/part-* | sort -t, -k1,1nr > sortedoutput
+
+Please see topproperties_r.py too
+"""
from lxml import etree
+from itertools import izip
from StringIO import StringIO
import json
import sys
@@ -19,6 +21,7 @@
page = ''
def main():
+ """Iterates through the input and sends each page to parsePage"""
while True:
page = ''
i = sys.stdin.readline()
@@ -26,33 +29,35 @@
if '<page>' in i:
i = sys.stdin.readline()
while '</page>' not in i:
- page += i#.strip()
+ page += i
i = sys.stdin.readline()
page = '<page>' + page + '</page>'
parsePage(page)
def pairwise(iterable):
- from itertools import izip
a = iter(iterable)
return izip(a, a)
def parsePage(page):
+ """Parses a page and its JSON text to extract the properties from the
statements and sends them to stdout
+ with the property id as the key and "1" as the value. This is similar to
MapReduce wordcount.
+ """
tree = etree.parse(StringIO(page))
page = {child.tag:child.text for child in tree.iter()}
try:
- if page['ns'] == '0':
- title = page['title'][1:]
- text = json.loads(page['text'])
- statement = None
- if 'claims' in text:
- for a in text['claims']:
- statement = {i:j for i, j in pairwise(a['m'])}
- if statement != None:
- try:
- prop = str(statement['value']).encode("utf-8",
'ignore').strip() + "\n")
- sys.stdout.write(prop + "\t" + "1" + "\n")
- except KeyError:
- pass
+ if page['ns'] != '0': return
+ title = page['title'][1:]
+ text = json.loads(page['text'])
+ statement = None
+ if 'claims' not in text: return
+ for a in text['claims']:
+ statement = {i:j for i, j in pairwise(a['m'])}
+ if statement != None:
+ try:
+ prop = str(statement['value']).encode("utf-8",
'ignore').strip()
+ sys.stdout.write(prop + "\t" + "1" + "\n")
+ except KeyError:
+ pass
except (KeyError, ValueError, TypeError) as e:
sys.stderr.write("Error occurred for page : " + str(title) + ", ns = "
+ str(page['ns']) + "\n")
sys.stderr.write(traceback.format_exc() + "\n")
diff --git a/wikiparser/top-global-properties/topproperties_r.py
b/wikiparser/top-global-properties/topproperties_r.py
index 6f553e1..f66a1aa 100644
--- a/wikiparser/top-global-properties/topproperties_r.py
+++ b/wikiparser/top-global-properties/topproperties_r.py
@@ -1,12 +1,17 @@
#!/usr/bin/python
-# The reducer script to count global property frequencies from the wikidata
dump using Hadoop.
-# If the reducer output is set to "/output", run: bin/hadoop dfs -cat
/output/part-* | sort -t, -k1,1nr > sortedoutput
-# to get the properties sorted in descending order of their frequencies.
+""" The reducer script to count global property frequencies from the wikidata
dump using Hadoop.
+
+If the reducer output is set to "/output", run: bin/hadoop dfs -cat
/output/part-* | sort -t, -k1,1nr > sortedoutput
+to get the properties sorted in descending order of their frequencies.
+"""
import sys
def main():
+ """Counts the number of occurrences for each property and outputs the
property id as key
+ and the occurrence frequency as the value
+ """
current_word = None
current_count = 0
key = None
@@ -17,18 +22,18 @@
count = int(value.strip())
except ValueError:
continue
-
+
if current_word == key:
current_count += count
-
+
else:
if current_word:
- sys.stdout.write("%s\t%s\n" % current_word, str(current_count))
+ sys.stdout.write("%s\t%s\n" % (current_word,
str(current_count)))
current_count = count
- current_word = word
-
- if current_word == word:
- sys.stdout.write("%s\t%s\n" % current_word, str(current_count))
-
+ current_word = key
+
+ if current_word == key:
+ sys.stdout.write("%s\t%s\n" % (current_word, str(current_count)))
+
if __name__ == '__main__':
main()
--
To view, visit https://gerrit.wikimedia.org/r/77146
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0658eb7a5fdd8f873b5dc052a8332cdee3aa4417
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/extensions/WikidataEntitySuggester
Gerrit-Branch: master
Gerrit-Owner: Nilesh <[email protected]>
Gerrit-Reviewer: Daniel Kinzler <[email protected]>
Gerrit-Reviewer: Nilesh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
