Daniel Kinzler has submitted this change and it was merged.
Change subject: Added MapReduce scripts to count global property frequencies
......................................................................
Added MapReduce scripts to count global property frequencies
Change-Id: I6022ee0e3a59ac4a7d75dfac3eab2f7f4158f0fd
---
A wikiparser/top-global-properties/topproperties.py
A wikiparser/top-global-properties/topproperties_r.py
2 files changed, 95 insertions(+), 0 deletions(-)
Approvals:
Daniel Kinzler: Verified; Looks good to me, approved
diff --git a/wikiparser/top-global-properties/topproperties.py
b/wikiparser/top-global-properties/topproperties.py
new file mode 100644
index 0000000..beabbdc
--- /dev/null
+++ b/wikiparser/top-global-properties/topproperties.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+# The mapper script to count global property frequencies from the wikidata
dump using Hadoop.
+# Sample command to run this on Hadoop:
+# bin/hadoop jar contrib/streaming/hadoop*streaming*jar -libjars
/tmp/entity-suggester-client.jar \
+# -inputformat
org.wikimedia.wikibase.entitysuggester.wikiparser.WikiPageInputFormat \
+# -input /input/dump.xml -output /output \
+# -file /tmp/topproperties.py -mapper /tmp/topproperties.py \
+# -file /tmp/topproperties_r.py -reducer '/tmp/topproperties_r.py'
+#
+# bin/hadoop dfs -cat /output/part-* | sort -t, -k1,1nr > sortedoutput
+# Please see topproperties_r.py too
+
+from lxml import etree
+from StringIO import StringIO
+import json
+import sys
+
+page = ''
+
+def main():
+ while True:
+ page = ''
+ i = sys.stdin.readline()
+ if not i: break
+ if '<page>' in i:
+ i = sys.stdin.readline()
+ while '</page>' not in i:
+ page += i#.strip()
+ i = sys.stdin.readline()
+ page = '<page>' + page + '</page>'
+ parsePage(page)
+
+def pairwise(iterable):
+ from itertools import izip
+ a = iter(iterable)
+ return izip(a, a)
+
+def parsePage(page):
+ tree = etree.parse(StringIO(page))
+ page = {child.tag:child.text for child in tree.iter()}
+ try:
+ if page['ns'] == '0':
+ title = page['title'][1:]
+ text = json.loads(page['text'])
+ statement = None
+ if 'claims' in text:
+ for a in text['claims']:
+ statement = {i:j for i, j in pairwise(a['m'])}
+ if statement != None:
+ try:
+ prop = str(statement['value']).encode("utf-8",
'ignore').strip() + "\n")
+ sys.stdout.write(prop + "\t" + "1" + "\n")
+ except KeyError:
+ pass
+ except (KeyError, ValueError, TypeError) as e:
+ sys.stderr.write("Error occurred for page : " + str(title) + ", ns = "
+ str(page['ns']) + "\n")
+ sys.stderr.write(traceback.format_exc() + "\n")
+
+if __name__ == '__main__':
+ main()
diff --git a/wikiparser/top-global-properties/topproperties_r.py
b/wikiparser/top-global-properties/topproperties_r.py
new file mode 100644
index 0000000..6f553e1
--- /dev/null
+++ b/wikiparser/top-global-properties/topproperties_r.py
@@ -0,0 +1,34 @@
+#!/usr/bin/python
+
+# The reducer script to count global property frequencies from the wikidata
dump using Hadoop.
+# If the reducer output is set to "/output", run: bin/hadoop dfs -cat
/output/part-* | sort -t, -k1,1nr > sortedoutput
+# to get the properties sorted in descending order of their frequencies.
+
+import sys
+
+def main():
+ current_word = None
+ current_count = 0
+ key = None
+
+ for i in sys.stdin:
+ (key, value) = i.split("\t")
+ try:
+ count = int(value.strip())
+ except ValueError:
+ continue
+
+ if current_word == key:
+ current_count += count
+
+ else:
+ if current_word:
+ sys.stdout.write("%s\t%s\n" % current_word, str(current_count))
+ current_count = count
+ current_word = word
+
+ if current_word == word:
+ sys.stdout.write("%s\t%s\n" % current_word, str(current_count))
+
+if __name__ == '__main__':
+ main()
--
To view, visit https://gerrit.wikimedia.org/r/77070
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6022ee0e3a59ac4a7d75dfac3eab2f7f4158f0fd
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/extensions/WikidataEntitySuggester
Gerrit-Branch: master
Gerrit-Owner: Nilesh <[email protected]>
Gerrit-Reviewer: Daniel Kinzler <[email protected]>
Gerrit-Reviewer: Nilesh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
