Dbrant has submitted this change and it was merged. Change subject: Implement PageTitle#isSpecial + Tests ......................................................................
Implement PageTitle#isSpecial + Tests - Checks if a page is a special page or not - Uses a static data store, which is a Java file that is generated by a scripts/make-templates.py - This script gathers the special page aliases via API calls and generates Java code via Jinja templates - The script is designed as a pipeline of functions. This lets us add more such generations in the future easily Change-Id: Ifad790990b7c2d48fc319c511369ed0ca227c69d --- A scripts/make-templates.py A scripts/templates/basichash.java.jinja M wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java M wikipedia/src/main/java/org/wikipedia/PageTitle.java A wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java 5 files changed, 455 insertions(+), 0 deletions(-) Approvals: Dbrant: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/make-templates.py b/scripts/make-templates.py new file mode 100644 index 0000000..165094d --- /dev/null +++ b/scripts/make-templates.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import os +import json +import unicodecsv as csv +import codecs +from urllib2 import urlopen +from jinja2 import Environment, FileSystemLoader + + +# Wikis that cause problems and hence we pretend +# do not exist. +# - 'got' -> Gothic runes wiki. The name of got in got +# contains characters outside the Unicode BMP. Android +# hard crashes on these. Let's ignore these fellas +# for now. +OSTRITCH_WIKIS = [u'got'] + + +# Represents a single wiki, along with arbitrary properites of that wiki +# Simple data container object +class Wiki(object): + def __init__(self, lang): + self.lang = lang + self.props = {} + + +# Represents a list of wikis plus their properties. +# Encapsulates rendering code as well +class WikiList(object): + def __init__(self, wikis): + self.wikis = wikis + self.template_env = Environment(loader=FileSystemLoader( + os.path.join(os.path.dirname(os.path.realpath(__file__)), u'templates') + )) + + def get_filtered_wiki_list(self): + return [wiki for wiki in self.wikis if wiki.lang not in OSTRITCH_WIKIS] + + def render(self, template, class_name, **kwargs): + data = { + u"class_name": class_name, + u"wikis": self.get_filtered_wiki_list() + } + data.update(kwargs) + rendered = self.template_env.get_template(template).render(**data) + out = codecs.open(class_name + ".java", u'w', u'utf-8') + out.write(rendered) + out.close() + + +def list_from_wikistats(): + URL = u"https://wikistats.wmflabs.org/api.php?action=dump&table=wikipedias&format=csv&s=good" + + print "Fetching languages" + data = csv.reader(urlopen(URL)) + wikis = [] + + is_first = True + for row in data: + if is_first: + is_first = False + continue # skip headers + wiki = Wiki(row[2]) + wiki.props[u'english_name'] = row[1] + wiki.props[u'local_name'] = row[10] + wiki.props[u'total_pages'] = row[3] + wikis.append(wiki) + + return WikiList(wikis) + + +# Populate the aliases for 'Special:' in all wikis +def populate_special_alias(wikis): + for wiki in wikis.wikis: + print "Fetching Special Page alias for %s" % wiki.lang + url = u"https://%s.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&siprop=namespaces" % wiki.lang + data = json.load(urlopen(url)) + # -1 seems to be the ID for Special Pages + wiki.props[u'special_alias'] = data['query']['namespaces']['-1']['*'] + return wikis + + +# Returns a function that renders a particular template when passed +# a WikiList object +def render_template(template, filename, **kwargs): + def _actual_render(wikis): + wikis.render(template, filename, **kwargs) + return wikis + return _actual_render + + +# Kinda like reduce(), but special cases first function +def chain(*funcs): + res = funcs[0]() + for func in funcs[1:]: + res = func(res) + + +chain( + list_from_wikistats, + populate_special_alias, + render_template(u'basichash.java.jinja', u'SpecialAliasData', key=u'special_alias') +) diff --git a/scripts/templates/basichash.java.jinja b/scripts/templates/basichash.java.jinja new file mode 100644 index 0000000..0fbd9ee --- /dev/null +++ b/scripts/templates/basichash.java.jinja @@ -0,0 +1,26 @@ +/* THIS IS AN AUTOMATICALLY GENERATED FILE + IF YOU MODIFY THIS BY HAND BE PREPARED! + TO HAVE YOUR CHANGES OVERWRITTEN */ +package org.wikipedia.staticdata; + +import java.util.*; + +public class {{ class_name }} { + + private static HashMap<String, String> dataMap; + private static void setupData() { + dataMap = new HashMap<String, String>({{wikis|length}}); + + {%- for wiki in wikis %} + dataMap.put("{{wiki.lang}}", "{{wiki.props[key]}}"); + {%- endfor %} + } + + public static String valueFor(String key) { + if (dataMap == null) { + setupData(); + } + + return dataMap.get(key); + } +} diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java index d4fd523..b868663 100644 --- a/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java +++ b/wikipedia-it/src/main/java/org/wikipedia/test/PageTitleTests.java @@ -75,4 +75,11 @@ assertEquals("Talk", new PageTitle("Talk:Hello", enwiki).getNamespace()); assertEquals("Wikipedia talk:Hello world", new PageTitle("Wikipedia_talk:Hello world", enwiki).getDisplayText()); } + + public void testSpecial() throws Exception { + assertTrue(new PageTitle("Special:Version", new Site("en.wikipedia.org")).isSpecial()); + assertTrue(new PageTitle("特別:Version", new Site("ja.wikipedia.org")).isSpecial()); + assertFalse(new PageTitle("Special:Version", new Site("ja.wikipedia.org")).isSpecial()); + assertFalse(new PageTitle("特別:Version", new Site("en.wikipedia.org")).isSpecial()); + } } diff --git a/wikipedia/src/main/java/org/wikipedia/PageTitle.java b/wikipedia/src/main/java/org/wikipedia/PageTitle.java index 872c762..85912ff 100644 --- a/wikipedia/src/main/java/org/wikipedia/PageTitle.java +++ b/wikipedia/src/main/java/org/wikipedia/PageTitle.java @@ -3,6 +3,7 @@ import android.os.*; import android.text.*; import org.json.*; +import org.wikipedia.staticdata.*; import java.io.*; import java.net.*; @@ -135,6 +136,16 @@ return namespace == null ? text : namespace + ":" + text; } + private Boolean isSpecial = null; + public boolean isSpecial() { + if (isSpecial == null) { + isSpecial = getNamespace() != null + && SpecialAliasData.valueFor(getSite().getLanguage()).equals(getNamespace()); + } + + return isSpecial; + } + @Override public int describeContents() { return 0; diff --git a/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java b/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java new file mode 100644 index 0000000..d1edc43 --- /dev/null +++ b/wikipedia/src/main/java/org/wikipedia/staticdata/SpecialAliasData.java @@ -0,0 +1,308 @@ +/* THIS IS AN AUTOMATICALLY GENERATED FILE + IF YOU MODIFY THIS BY HAND BE PREPARED! + TO HAVE YOUR CHANGES OVERWRITTEN */ +package org.wikipedia.staticdata; + +import java.util.*; + +public class SpecialAliasData { + + private static HashMap<String, String> dataMap; + private static void setupData() { + dataMap = new HashMap<String, String>(286); + dataMap.put("en", "Special"); + dataMap.put("nl", "Speciaal"); + dataMap.put("de", "Spezial"); + dataMap.put("sv", "Special"); + dataMap.put("fr", "Spécial"); + dataMap.put("it", "Speciale"); + dataMap.put("ru", "Служебная"); + dataMap.put("es", "Especial"); + dataMap.put("pl", "Specjalna"); + dataMap.put("war", "Pinaurog"); + dataMap.put("ja", "特別"); + dataMap.put("ceb", "Espesyal"); + dataMap.put("vi", "Đặc biệt"); + dataMap.put("pt", "Especial"); + dataMap.put("zh", "Special"); + dataMap.put("uk", "Спеціальна"); + dataMap.put("ca", "Especial"); + dataMap.put("no", "Spesial"); + dataMap.put("fa", "ویژه"); + dataMap.put("id", "Istimewa"); + dataMap.put("fi", "Toiminnot"); + dataMap.put("cs", "Speciální"); + dataMap.put("ar", "خاص"); + dataMap.put("ko", "특수"); + dataMap.put("hu", "Speciális"); + dataMap.put("ms", "Khas"); + dataMap.put("sr", "Посебно"); + dataMap.put("ro", "Special"); + dataMap.put("tr", "Özel"); + dataMap.put("min", "Istimewa"); + dataMap.put("kk", "Арнайы"); + dataMap.put("eo", "Specialaĵo"); + dataMap.put("sk", "Špeciálne"); + dataMap.put("da", "Speciel"); + dataMap.put("eu", "Berezi"); + dataMap.put("lt", "Specialus"); + dataMap.put("bg", "Специални"); + dataMap.put("sh", "Posebno"); + dataMap.put("he", "מיוחד"); + dataMap.put("hr", "Posebno"); + dataMap.put("sl", "Posebno"); + dataMap.put("uz", "Maxsus"); + dataMap.put("et", "Eri"); + dataMap.put("vo", "Patikos"); + dataMap.put("hy", "Սպասարկող"); + dataMap.put("nn", "Spesial"); + dataMap.put("gl", "Especial"); + dataMap.put("simple", "Special"); + dataMap.put("hi", "विशेष"); + dataMap.put("la", "Specialis"); + dataMap.put("az", "Xüsusi"); + dataMap.put("el", "Ειδικό"); + dataMap.put("th", "พิเศษ"); + dataMap.put("oc", "Especial"); + dataMap.put("ka", "სპეციალური"); + dataMap.put("mk", "Специјална"); + dataMap.put("be", "Адмысловае"); + dataMap.put("new", "विशेष"); + dataMap.put("pms", "Special"); + dataMap.put("tl", "Natatangi"); + dataMap.put("ta", "சிறப்பு"); + dataMap.put("te", "ప్రత్యేక"); + dataMap.put("tt", "Махсус"); + dataMap.put("cy", "Arbennig"); + dataMap.put("lv", "Special"); + dataMap.put("ht", "Espesyal"); + dataMap.put("be-x-old", "Спэцыяльныя"); + dataMap.put("ur", "خاص"); + dataMap.put("sq", "Speciale"); + dataMap.put("bs", "Posebno"); + dataMap.put("br", "Dibar"); + dataMap.put("jv", "Astamiwa"); + dataMap.put("mg", "Manokana"); + dataMap.put("lb", "Spezial"); + dataMap.put("mr", "विशेष"); + dataMap.put("is", "Kerfissíða"); + dataMap.put("ml", "പ്രത്യേകം"); + dataMap.put("pnb", "Special"); + dataMap.put("ba", "Махсус"); + dataMap.put("my", "Special"); + dataMap.put("af", "Spesiaal"); + dataMap.put("yo", "Pàtàkì"); + dataMap.put("ga", "Speisialta"); + dataMap.put("an", "Especial"); + dataMap.put("zh-yue", "Special"); + dataMap.put("bn", "বিশেষ"); + dataMap.put("tg", "Вижа"); + dataMap.put("lmo", "Special"); + dataMap.put("fy", "Wiki"); + dataMap.put("cv", "Ятарлă"); + dataMap.put("ky", "Атайын"); + dataMap.put("sw", "Maalum"); + dataMap.put("io", "Specala"); + dataMap.put("ce", "Белхан"); + dataMap.put("ne", "विशेष"); + dataMap.put("gu", "વિશેષ"); + dataMap.put("bpy", "বিশেষ"); + dataMap.put("scn", "Spiciali"); + dataMap.put("sco", "Special"); + dataMap.put("nds", "Spezial"); + dataMap.put("ku", "Taybet"); + dataMap.put("ast", "Especial"); + dataMap.put("qu", "Sapaq"); + dataMap.put("su", "Husus"); + dataMap.put("als", "Spezial"); + dataMap.put("kn", "ವಿಶೇಷ"); + dataMap.put("am", "ልዩ"); + dataMap.put("ia", "Special"); + dataMap.put("nap", "Speciàle"); + dataMap.put("bug", "Istimewa"); + dataMap.put("ckb", "تایبەت"); + dataMap.put("bat-smg", "Specēlos"); + dataMap.put("gd", "Sònraichte"); + dataMap.put("wa", "Sipeciås"); + dataMap.put("map-bms", "Astamiwa"); + dataMap.put("mn", "Тусгай"); + dataMap.put("arz", "خاص"); + dataMap.put("mzn", "شا"); + dataMap.put("zh-min-nan", "特殊"); + dataMap.put("yi", "באַזונדער"); + dataMap.put("si", "විශේෂ"); + dataMap.put("pa", "ਖ਼ਾਸ"); + dataMap.put("sah", "Аналлаах"); + dataMap.put("vec", "Speciale"); + dataMap.put("sa", "विशेषम्"); + dataMap.put("nah", "Nōncuahquīzqui"); + dataMap.put("bar", "Spezial"); + dataMap.put("os", "Сæрмагонд"); + dataMap.put("fo", "Serstakt"); + dataMap.put("roa-tara", "Special"); + dataMap.put("pam", "Special"); + dataMap.put("hsb", "Specialnje"); + dataMap.put("or", "ବିଶେଷ"); + dataMap.put("se", "Erenoamáš"); + dataMap.put("li", "Speciaal"); + dataMap.put("mi", "Special"); + dataMap.put("ilo", "Espesial"); + dataMap.put("co", "Speciale"); + dataMap.put("hif", "khaas"); + dataMap.put("gan", "特別"); + dataMap.put("frr", "Spezial"); + dataMap.put("bo", "Special"); + dataMap.put("glk", "ویژه"); + dataMap.put("rue", "Шпеціална"); + dataMap.put("bcl", "Espesyal"); + dataMap.put("nds-nl", "Spesiaal"); + dataMap.put("fiu-vro", "Tallituslehekülg"); + dataMap.put("mrj", "Спецӹлӹштӓш"); + dataMap.put("ps", "ځانګړی"); + dataMap.put("tk", "Ýörite"); + dataMap.put("vls", "Specioal"); + dataMap.put("gv", "Er lheh"); + dataMap.put("xmf", "სპეციალური"); + dataMap.put("pag", "Special"); + dataMap.put("diq", "Bağse"); + dataMap.put("km", "ពិសេស"); + dataMap.put("zea", "Speciaol"); + dataMap.put("kv", "Отсасян"); + dataMap.put("mhr", "Лӱмын ыштыме"); + dataMap.put("csb", "Specjalnô"); + dataMap.put("vep", "Specialine"); + dataMap.put("ay", "Especial"); + dataMap.put("hak", "特殊"); + dataMap.put("dv", "ޚާއްސަ"); + dataMap.put("so", "Special"); + dataMap.put("nrm", "Special"); + dataMap.put("rm", "Spezial"); + dataMap.put("udm", "Панель"); + dataMap.put("zh-classical", "Special"); + dataMap.put("sc", "Ispetziale"); + dataMap.put("koi", "Служебная"); + dataMap.put("ug", "ئالاھىدە"); + dataMap.put("lad", "Especial"); + dataMap.put("stq", "Spezial"); + dataMap.put("wuu", "特殊"); + dataMap.put("lij", "Speçiale"); + dataMap.put("fur", "Speciâl"); + dataMap.put("mt", "Speċjali"); + dataMap.put("eml", "Speciale"); + dataMap.put("as", "বিশেষ"); + dataMap.put("pi", "विसेस"); + dataMap.put("nov", "Special"); + dataMap.put("bh", "विशेष"); + dataMap.put("gn", "Mba'echĩchĩ"); + dataMap.put("ksh", "Extra"); + dataMap.put("pcd", "Spécial"); + dataMap.put("kw", "Arbennek"); + dataMap.put("ang", "Syndrig"); + dataMap.put("gag", "Maasus"); + dataMap.put("szl", "Szpecyjalna"); + dataMap.put("ace", "Kusuih"); + dataMap.put("nv", "Special"); + dataMap.put("ext", "Special"); + dataMap.put("ie", "Special"); + dataMap.put("frp", "Spèciâl"); + dataMap.put("mwl", "Special"); + dataMap.put("sn", "Special"); + dataMap.put("ln", "Spécial"); + dataMap.put("crh", "Mahsus"); + dataMap.put("dsb", "Specialne"); + dataMap.put("pfl", "Schbezial"); + dataMap.put("lez", "Служебная"); + dataMap.put("krc", "Къуллукъ"); + dataMap.put("haw", "Papa nui"); + dataMap.put("pdc", "Spezial"); + dataMap.put("xal", "Көдлхнә"); + dataMap.put("kab", "Uslig"); + dataMap.put("rw", "Special"); + dataMap.put("to", "Special"); + dataMap.put("myv", "Башка тевень"); + dataMap.put("arc", "ܕܝܠܢܝܐ"); + dataMap.put("kl", "Immikkut"); + dataMap.put("bjn", "Istimiwa"); + dataMap.put("pap", "Special"); + dataMap.put("kbd", "Служебная"); + dataMap.put("lo", "ພິເສດ"); + dataMap.put("tpi", "Sipesol"); + dataMap.put("lbe", "Къуллугъирал лажин"); + dataMap.put("wo", "Jagleel"); + dataMap.put("mdf", "Башка"); + dataMap.put("jbo", "Special"); + dataMap.put("cbk-zam", "Especial"); + dataMap.put("av", "Служебная"); + dataMap.put("ty", "Spécial"); + dataMap.put("srn", "Spesyal"); + dataMap.put("bxr", "Тусхай"); + dataMap.put("ig", "Ihü kárírí"); + dataMap.put("kg", "Special"); + dataMap.put("tet", "Espesiál"); + dataMap.put("na", "Special"); + dataMap.put("ab", "Цастәи"); + dataMap.put("ltg", "Seviškuo"); + dataMap.put("nso", "Special"); + dataMap.put("za", "特殊"); + dataMap.put("kaa", "Arnawlı"); + dataMap.put("zu", "Special"); + dataMap.put("ha", "Special"); + dataMap.put("chy", "Special"); + dataMap.put("rmy", "Uzalutno"); + dataMap.put("cu", "Нарочьна"); + dataMap.put("chr", "Special"); + dataMap.put("tn", "Special"); + dataMap.put("cdo", "特殊"); + dataMap.put("roa-rup", "Special"); + dataMap.put("bi", "Special"); + dataMap.put("tyv", "Тускай"); + dataMap.put("pih", "Special"); + dataMap.put("tw", "Special"); + dataMap.put("sm", "Special"); + dataMap.put("mo", "Special"); + dataMap.put("bm", "Spécial"); + dataMap.put("iu", "Special"); + dataMap.put("ss", "Special"); + dataMap.put("sd", "خاص"); + dataMap.put("pnt", "Ειδικόν"); + dataMap.put("ki", "Special"); + dataMap.put("rn", "Special"); + dataMap.put("xh", "Special"); + dataMap.put("om", "Special"); + dataMap.put("ee", "Special"); + dataMap.put("ts", "Special"); + dataMap.put("ak", "Soronko"); + dataMap.put("fj", "Special"); + dataMap.put("ti", "Special"); + dataMap.put("ks", "خاص"); + dataMap.put("ff", "Spécial"); + dataMap.put("sg", "Spécial"); + dataMap.put("ve", "Special"); + dataMap.put("cr", "Special"); + dataMap.put("lg", "Special"); + dataMap.put("st", "Special"); + dataMap.put("dz", "Special"); + dataMap.put("tum", "Special"); + dataMap.put("ik", "Special"); + dataMap.put("ny", "Special"); + dataMap.put("ch", "Espesiat"); + dataMap.put("ng", "Special"); + dataMap.put("ii", "特殊"); + dataMap.put("cho", "Special"); + dataMap.put("mh", "Special"); + dataMap.put("aa", "Special"); + dataMap.put("kj", "Special"); + dataMap.put("ho", "Special"); + dataMap.put("mus", "Special"); + dataMap.put("kr", "Special"); + dataMap.put("hz", "Special"); + } + + public static String valueFor(String key) { + if (dataMap == null) { + setupData(); + } + + return dataMap.get(key); + } +} \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/130575 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ifad790990b7c2d48fc319c511369ed0ca227c69d Gerrit-PatchSet: 5 Gerrit-Project: apps/android/wikipedia Gerrit-Branch: master Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com> Gerrit-Reviewer: Dbrant <dbr...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits