Dbrant has submitted this change and it was merged.
Change subject: Implement PageTitle#isSpecial + Tests
......................................................................
Implement PageTitle#isSpecial + Tests
- Checks if a page is a special page or not
- Uses a static data store, which is a Java file that is
generated by a scripts/make-templates.py
- This script gathers the special page aliases via API
calls and generates Java code via Jinja templates
- The script is designed as a pipeline of functions. This
lets us add more such generations in the future easily
Change-Id: Ifad790990b7c2d48fc319c511369ed0ca227c69d
---
A scripts/make-templates.py
A scripts/templates/basichash.java.jinja
M wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java
M wikipedia/src/main/java/org/wikipedia/PageTitle.java
A wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java
5 files changed, 455 insertions(+), 0 deletions(-)
Approvals:
Dbrant: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/make-templates.py b/scripts/make-templates.py
new file mode 100644
index 0000000..165094d
--- /dev/null
+++ b/scripts/make-templates.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+import os
+import json
+import unicodecsv as csv
+import codecs
+from urllib2 import urlopen
+from jinja2 import Environment, FileSystemLoader
+
+
+# Wikis that cause problems and hence we pretend
+# do not exist.
+# - 'got' -> Gothic runes wiki. The name of got in got
+# contains characters outside the Unicode BMP. Android
+# hard crashes on these. Let's ignore these fellas
+# for now.
+OSTRITCH_WIKIS = [u'got']
+
+
+# Represents a single wiki, along with arbitrary properites of that wiki
+# Simple data container object
+class Wiki(object):
+ def __init__(self, lang):
+ self.lang = lang
+ self.props = {}
+
+
+# Represents a list of wikis plus their properties.
+# Encapsulates rendering code as well
+class WikiList(object):
+ def __init__(self, wikis):
+ self.wikis = wikis
+ self.template_env = Environment(loader=FileSystemLoader(
+ os.path.join(os.path.dirname(os.path.realpath(__file__)),
u'templates')
+ ))
+
+ def get_filtered_wiki_list(self):
+ return [wiki for wiki in self.wikis if wiki.lang not in OSTRITCH_WIKIS]
+
+ def render(self, template, class_name, **kwargs):
+ data = {
+ u"class_name": class_name,
+ u"wikis": self.get_filtered_wiki_list()
+ }
+ data.update(kwargs)
+ rendered = self.template_env.get_template(template).render(**data)
+ out = codecs.open(class_name + ".java", u'w', u'utf-8')
+ out.write(rendered)
+ out.close()
+
+
+def list_from_wikistats():
+ URL =
u"https://wikistats.wmflabs.org/api.php?action=dump&table=wikipedias&format=csv&s=good";
+
+ print "Fetching languages"
+ data = csv.reader(urlopen(URL))
+ wikis = []
+
+ is_first = True
+ for row in data:
+ if is_first:
+ is_first = False
+ continue # skip headers
+ wiki = Wiki(row[2])
+ wiki.props[u'english_name'] = row[1]
+ wiki.props[u'local_name'] = row[10]
+ wiki.props[u'total_pages'] = row[3]
+ wikis.append(wiki)
+
+ return WikiList(wikis)
+
+
+# Populate the aliases for 'Special:' in all wikis
+def populate_special_alias(wikis):
+ for wiki in wikis.wikis:
+ print "Fetching Special Page alias for %s" % wiki.lang
+ url =
u"https://%s.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&siprop=namespaces";
% wiki.lang
+ data = json.load(urlopen(url))
+ # -1 seems to be the ID for Special Pages
+ wiki.props[u'special_alias'] = data['query']['namespaces']['-1']['*']
+ return wikis
+
+
+# Returns a function that renders a particular template when passed
+# a WikiList object
+def render_template(template, filename, **kwargs):
+ def _actual_render(wikis):
+ wikis.render(template, filename, **kwargs)
+ return wikis
+ return _actual_render
+
+
+# Kinda like reduce(), but special cases first function
+def chain(*funcs):
+ res = funcs[0]()
+ for func in funcs[1:]:
+ res = func(res)
+
+
+chain(
+ list_from_wikistats,
+ populate_special_alias,
+ render_template(u'basichash.java.jinja', u'SpecialAliasData',
key=u'special_alias')
+)
diff --git a/scripts/templates/basichash.java.jinja
b/scripts/templates/basichash.java.jinja
new file mode 100644
index 0000000..0fbd9ee
--- /dev/null
+++ b/scripts/templates/basichash.java.jinja
@@ -0,0 +1,26 @@
+/* THIS IS AN AUTOMATICALLY GENERATED FILE
+ IF YOU MODIFY THIS BY HAND BE PREPARED!
+ TO HAVE YOUR CHANGES OVERWRITTEN */
+package org.wikipedia.staticdata;
+
+import java.util.*;
+
+public class {{ class_name }} {
+
+ private static HashMap<String, String> dataMap;
+ private static void setupData() {
+ dataMap = new HashMap<String, String>({{wikis|length}});
+
+ {%- for wiki in wikis %}
+ dataMap.put("{{wiki.lang}}", "{{wiki.props[key]}}");
+ {%- endfor %}
+ }
+
+ public static String valueFor(String key) {
+ if (dataMap == null) {
+ setupData();
+ }
+
+ return dataMap.get(key);
+ }
+}
diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java
b/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java
index d4fd523..b868663 100644
--- a/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java
+++ b/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java
@@ -75,4 +75,11 @@
assertEquals("Talk", new PageTitle("Talk:Hello",
enwiki).getNamespace());
assertEquals("Wikipedia talk:Hello world", new
PageTitle("Wikipedia_talk:Hello world", enwiki).getDisplayText());
}
+
+ public void testSpecial() throws Exception {
+ assertTrue(new PageTitle("Special:Version", new
Site("en.wikipedia.org")).isSpecial());
+ assertTrue(new PageTitle("特別:Version", new
Site("ja.wikipedia.org")).isSpecial());
+ assertFalse(new PageTitle("Special:Version", new
Site("ja.wikipedia.org")).isSpecial());
+ assertFalse(new PageTitle("特別:Version", new
Site("en.wikipedia.org")).isSpecial());
+ }
}
diff --git a/wikipedia/src/main/java/org/wikipedia/PageTitle.java
b/wikipedia/src/main/java/org/wikipedia/PageTitle.java
index 872c762..85912ff 100644
--- a/wikipedia/src/main/java/org/wikipedia/PageTitle.java
+++ b/wikipedia/src/main/java/org/wikipedia/PageTitle.java
@@ -3,6 +3,7 @@
import android.os.*;
import android.text.*;
import org.json.*;
+import org.wikipedia.staticdata.*;
import java.io.*;
import java.net.*;
@@ -135,6 +136,16 @@
return namespace == null ? text : namespace + ":" + text;
}
+ private Boolean isSpecial = null;
+ public boolean isSpecial() {
+ if (isSpecial == null) {
+ isSpecial = getNamespace() != null
+ &&
SpecialAliasData.valueFor(getSite().getLanguage()).equals(getNamespace());
+ }
+
+ return isSpecial;
+ }
+
@Override
public int describeContents() {
return 0;
diff --git
a/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java
b/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java
new file mode 100644
index 0000000..d1edc43
--- /dev/null
+++ b/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java
@@ -0,0 +1,308 @@
+/* THIS IS AN AUTOMATICALLY GENERATED FILE
+ IF YOU MODIFY THIS BY HAND BE PREPARED!
+ TO HAVE YOUR CHANGES OVERWRITTEN */
+package org.wikipedia.staticdata;
+
+import java.util.*;
+
+public class SpecialAliasData {
+
+ private static HashMap<String, String> dataMap;
+ private static void setupData() {
+ dataMap = new HashMap<String, String>(286);
+ dataMap.put("en", "Special");
+ dataMap.put("nl", "Speciaal");
+ dataMap.put("de", "Spezial");
+ dataMap.put("sv", "Special");
+ dataMap.put("fr", "Spécial");
+ dataMap.put("it", "Speciale");
+ dataMap.put("ru", "Служебная");
+ dataMap.put("es", "Especial");
+ dataMap.put("pl", "Specjalna");
+ dataMap.put("war", "Pinaurog");
+ dataMap.put("ja", "特別");
+ dataMap.put("ceb", "Espesyal");
+ dataMap.put("vi", "Đặc biệt");
+ dataMap.put("pt", "Especial");
+ dataMap.put("zh", "Special");
+ dataMap.put("uk", "Спеціальна");
+ dataMap.put("ca", "Especial");
+ dataMap.put("no", "Spesial");
+ dataMap.put("fa", "ویژه");
+ dataMap.put("id", "Istimewa");
+ dataMap.put("fi", "Toiminnot");
+ dataMap.put("cs", "Speciální");
+ dataMap.put("ar", "خاص");
+ dataMap.put("ko", "특수");
+ dataMap.put("hu", "Speciális");
+ dataMap.put("ms", "Khas");
+ dataMap.put("sr", "Посебно");
+ dataMap.put("ro", "Special");
+ dataMap.put("tr", "Özel");
+ dataMap.put("min", "Istimewa");
+ dataMap.put("kk", "Арнайы");
+ dataMap.put("eo", "Specialaĵo");
+ dataMap.put("sk", "Špeciálne");
+ dataMap.put("da", "Speciel");
+ dataMap.put("eu", "Berezi");
+ dataMap.put("lt", "Specialus");
+ dataMap.put("bg", "Специални");
+ dataMap.put("sh", "Posebno");
+ dataMap.put("he", "מיוחד");
+ dataMap.put("hr", "Posebno");
+ dataMap.put("sl", "Posebno");
+ dataMap.put("uz", "Maxsus");
+ dataMap.put("et", "Eri");
+ dataMap.put("vo", "Patikos");
+ dataMap.put("hy", "Սպասարկող");
+ dataMap.put("nn", "Spesial");
+ dataMap.put("gl", "Especial");
+ dataMap.put("simple", "Special");
+ dataMap.put("hi", "विशेष");
+ dataMap.put("la", "Specialis");
+ dataMap.put("az", "Xüsusi");
+ dataMap.put("el", "Ειδικό");
+ dataMap.put("th", "พิเศษ");
+ dataMap.put("oc", "Especial");
+ dataMap.put("ka", "სპეციალური");
+ dataMap.put("mk", "Специјална");
+ dataMap.put("be", "Адмысловае");
+ dataMap.put("new", "विशेष");
+ dataMap.put("pms", "Special");
+ dataMap.put("tl", "Natatangi");
+ dataMap.put("ta", "சிறப்பு");
+ dataMap.put("te", "ప్రత్యేక");
+ dataMap.put("tt", "Махсус");
+ dataMap.put("cy", "Arbennig");
+ dataMap.put("lv", "Special");
+ dataMap.put("ht", "Espesyal");
+ dataMap.put("be-x-old", "Спэцыяльныя");
+ dataMap.put("ur", "خاص");
+ dataMap.put("sq", "Speciale");
+ dataMap.put("bs", "Posebno");
+ dataMap.put("br", "Dibar");
+ dataMap.put("jv", "Astamiwa");
+ dataMap.put("mg", "Manokana");
+ dataMap.put("lb", "Spezial");
+ dataMap.put("mr", "विशेष");
+ dataMap.put("is", "Kerfissíða");
+ dataMap.put("ml", "പ്രത്യേകം");
+ dataMap.put("pnb", "Special");
+ dataMap.put("ba", "Махсус");
+ dataMap.put("my", "Special");
+ dataMap.put("af", "Spesiaal");
+ dataMap.put("yo", "Pàtàkì");
+ dataMap.put("ga", "Speisialta");
+ dataMap.put("an", "Especial");
+ dataMap.put("zh-yue", "Special");
+ dataMap.put("bn", "বিশেষ");
+ dataMap.put("tg", "Вижа");
+ dataMap.put("lmo", "Special");
+ dataMap.put("fy", "Wiki");
+ dataMap.put("cv", "Ятарлă");
+ dataMap.put("ky", "Атайын");
+ dataMap.put("sw", "Maalum");
+ dataMap.put("io", "Specala");
+ dataMap.put("ce", "Белхан");
+ dataMap.put("ne", "विशेष");
+ dataMap.put("gu", "વિશેષ");
+ dataMap.put("bpy", "বিশেষ");
+ dataMap.put("scn", "Spiciali");
+ dataMap.put("sco", "Special");
+ dataMap.put("nds", "Spezial");
+ dataMap.put("ku", "Taybet");
+ dataMap.put("ast", "Especial");
+ dataMap.put("qu", "Sapaq");
+ dataMap.put("su", "Husus");
+ dataMap.put("als", "Spezial");
+ dataMap.put("kn", "ವಿಶೇಷ");
+ dataMap.put("am", "ልዩ");
+ dataMap.put("ia", "Special");
+ dataMap.put("nap", "Speciàle");
+ dataMap.put("bug", "Istimewa");
+ dataMap.put("ckb", "تایبەت");
+ dataMap.put("bat-smg", "Specēlos");
+ dataMap.put("gd", "Sònraichte");
+ dataMap.put("wa", "Sipeciås");
+ dataMap.put("map-bms", "Astamiwa");
+ dataMap.put("mn", "Тусгай");
+ dataMap.put("arz", "خاص");
+ dataMap.put("mzn", "شا");
+ dataMap.put("zh-min-nan", "特殊");
+ dataMap.put("yi", "באַזונדער");
+ dataMap.put("si", "විශේෂ");
+ dataMap.put("pa", "ਖ਼ਾਸ");
+ dataMap.put("sah", "Аналлаах");
+ dataMap.put("vec", "Speciale");
+ dataMap.put("sa", "विशेषम्");
+ dataMap.put("nah", "Nōncuahquīzqui");
+ dataMap.put("bar", "Spezial");
+ dataMap.put("os", "Сæрмагонд");
+ dataMap.put("fo", "Serstakt");
+ dataMap.put("roa-tara", "Special");
+ dataMap.put("pam", "Special");
+ dataMap.put("hsb", "Specialnje");
+ dataMap.put("or", "ବିଶେଷ");
+ dataMap.put("se", "Erenoamáš");
+ dataMap.put("li", "Speciaal");
+ dataMap.put("mi", "Special");
+ dataMap.put("ilo", "Espesial");
+ dataMap.put("co", "Speciale");
+ dataMap.put("hif", "khaas");
+ dataMap.put("gan", "特別");
+ dataMap.put("frr", "Spezial");
+ dataMap.put("bo", "Special");
+ dataMap.put("glk", "ویژه");
+ dataMap.put("rue", "Шпеціална");
+ dataMap.put("bcl", "Espesyal");
+ dataMap.put("nds-nl", "Spesiaal");
+ dataMap.put("fiu-vro", "Tallituslehekülg");
+ dataMap.put("mrj", "Спецӹлӹштӓш");
+ dataMap.put("ps", "ځانګړی");
+ dataMap.put("tk", "Ýörite");
+ dataMap.put("vls", "Specioal");
+ dataMap.put("gv", "Er lheh");
+ dataMap.put("xmf", "სპეციალური");
+ dataMap.put("pag", "Special");
+ dataMap.put("diq", "Bağse");
+ dataMap.put("km", "ពិសេស");
+ dataMap.put("zea", "Speciaol");
+ dataMap.put("kv", "Отсасян");
+ dataMap.put("mhr", "Лӱмын ыштыме");
+ dataMap.put("csb", "Specjalnô");
+ dataMap.put("vep", "Specialine");
+ dataMap.put("ay", "Especial");
+ dataMap.put("hak", "特殊");
+ dataMap.put("dv", "ޚާއްސަ");
+ dataMap.put("so", "Special");
+ dataMap.put("nrm", "Special");
+ dataMap.put("rm", "Spezial");
+ dataMap.put("udm", "Панель");
+ dataMap.put("zh-classical", "Special");
+ dataMap.put("sc", "Ispetziale");
+ dataMap.put("koi", "Служебная");
+ dataMap.put("ug", "ئالاھىدە");
+ dataMap.put("lad", "Especial");
+ dataMap.put("stq", "Spezial");
+ dataMap.put("wuu", "特殊");
+ dataMap.put("lij", "Speçiale");
+ dataMap.put("fur", "Speciâl");
+ dataMap.put("mt", "Speċjali");
+ dataMap.put("eml", "Speciale");
+ dataMap.put("as", "বিশেষ");
+ dataMap.put("pi", "विसेस");
+ dataMap.put("nov", "Special");
+ dataMap.put("bh", "विशेष");
+ dataMap.put("gn", "Mba'echĩchĩ");
+ dataMap.put("ksh", "Extra");
+ dataMap.put("pcd", "Spécial");
+ dataMap.put("kw", "Arbennek");
+ dataMap.put("ang", "Syndrig");
+ dataMap.put("gag", "Maasus");
+ dataMap.put("szl", "Szpecyjalna");
+ dataMap.put("ace", "Kusuih");
+ dataMap.put("nv", "Special");
+ dataMap.put("ext", "Special");
+ dataMap.put("ie", "Special");
+ dataMap.put("frp", "Spèciâl");
+ dataMap.put("mwl", "Special");
+ dataMap.put("sn", "Special");
+ dataMap.put("ln", "Spécial");
+ dataMap.put("crh", "Mahsus");
+ dataMap.put("dsb", "Specialne");
+ dataMap.put("pfl", "Schbezial");
+ dataMap.put("lez", "Служебная");
+ dataMap.put("krc", "Къуллукъ");
+ dataMap.put("haw", "Papa nui");
+ dataMap.put("pdc", "Spezial");
+ dataMap.put("xal", "Көдлхнә");
+ dataMap.put("kab", "Uslig");
+ dataMap.put("rw", "Special");
+ dataMap.put("to", "Special");
+ dataMap.put("myv", "Башка тевень");
+ dataMap.put("arc", "ܕܝܠܢܝܐ");
+ dataMap.put("kl", "Immikkut");
+ dataMap.put("bjn", "Istimiwa");
+ dataMap.put("pap", "Special");
+ dataMap.put("kbd", "Служебная");
+ dataMap.put("lo", "ພິເສດ");
+ dataMap.put("tpi", "Sipesol");
+ dataMap.put("lbe", "Къуллугъирал лажин");
+ dataMap.put("wo", "Jagleel");
+ dataMap.put("mdf", "Башка");
+ dataMap.put("jbo", "Special");
+ dataMap.put("cbk-zam", "Especial");
+ dataMap.put("av", "Служебная");
+ dataMap.put("ty", "Spécial");
+ dataMap.put("srn", "Spesyal");
+ dataMap.put("bxr", "Тусхай");
+ dataMap.put("ig", "Ihü kárírí");
+ dataMap.put("kg", "Special");
+ dataMap.put("tet", "Espesiál");
+ dataMap.put("na", "Special");
+ dataMap.put("ab", "Цастәи");
+ dataMap.put("ltg", "Seviškuo");
+ dataMap.put("nso", "Special");
+ dataMap.put("za", "特殊");
+ dataMap.put("kaa", "Arnawlı");
+ dataMap.put("zu", "Special");
+ dataMap.put("ha", "Special");
+ dataMap.put("chy", "Special");
+ dataMap.put("rmy", "Uzalutno");
+ dataMap.put("cu", "Нарочьна");
+ dataMap.put("chr", "Special");
+ dataMap.put("tn", "Special");
+ dataMap.put("cdo", "特殊");
+ dataMap.put("roa-rup", "Special");
+ dataMap.put("bi", "Special");
+ dataMap.put("tyv", "Тускай");
+ dataMap.put("pih", "Special");
+ dataMap.put("tw", "Special");
+ dataMap.put("sm", "Special");
+ dataMap.put("mo", "Special");
+ dataMap.put("bm", "Spécial");
+ dataMap.put("iu", "Special");
+ dataMap.put("ss", "Special");
+ dataMap.put("sd", "خاص");
+ dataMap.put("pnt", "Ειδικόν");
+ dataMap.put("ki", "Special");
+ dataMap.put("rn", "Special");
+ dataMap.put("xh", "Special");
+ dataMap.put("om", "Special");
+ dataMap.put("ee", "Special");
+ dataMap.put("ts", "Special");
+ dataMap.put("ak", "Soronko");
+ dataMap.put("fj", "Special");
+ dataMap.put("ti", "Special");
+ dataMap.put("ks", "خاص");
+ dataMap.put("ff", "Spécial");
+ dataMap.put("sg", "Spécial");
+ dataMap.put("ve", "Special");
+ dataMap.put("cr", "Special");
+ dataMap.put("lg", "Special");
+ dataMap.put("st", "Special");
+ dataMap.put("dz", "Special");
+ dataMap.put("tum", "Special");
+ dataMap.put("ik", "Special");
+ dataMap.put("ny", "Special");
+ dataMap.put("ch", "Espesiat");
+ dataMap.put("ng", "Special");
+ dataMap.put("ii", "特殊");
+ dataMap.put("cho", "Special");
+ dataMap.put("mh", "Special");
+ dataMap.put("aa", "Special");
+ dataMap.put("kj", "Special");
+ dataMap.put("ho", "Special");
+ dataMap.put("mus", "Special");
+ dataMap.put("kr", "Special");
+ dataMap.put("hz", "Special");
+ }
+
+ public static String valueFor(String key) {
+ if (dataMap == null) {
+ setupData();
+ }
+
+ return dataMap.get(key);
+ }
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/130575
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ifad790990b7c2d48fc319c511369ed0ca227c69d
Gerrit-PatchSet: 5
Gerrit-Project: apps/android/wikipedia
Gerrit-Branch: master
Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com>
Gerrit-Reviewer: Dbrant <dbr...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
