Yuvipanda has uploaded a new change for review.
https://gerrit.wikimedia.org/r/93015
Change subject: Add code to fetch and parse a Page
......................................................................
Add code to fetch and parse a Page
Tests connect to english wikipedia directly, can be considered to
be slightly flaky.
Dependent on Iafdcc85d8c8ac030873bfe4e7f3675e8c868c45f
Change-Id: I83b393d6b37f5e47e05913f111d04e8cdb750f85
---
A
wikipedia-it/src/main/java/org/wikimedia/wikipedia/test/PageFetchTaskTests.java
A wikipedia/src/main/java/org/wikimedia/wikipedia/Page.java
A wikipedia/src/main/java/org/wikimedia/wikipedia/PageFetchTask.java
A wikipedia/src/main/java/org/wikimedia/wikipedia/PageTitle.java
A wikipedia/src/main/java/org/wikimedia/wikipedia/Section.java
A
wikipedia/src/main/java/org/wikimedia/wikipedia/concurrency/ExecutorService.java
6 files changed, 267 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/apps/android/wikipedia
refs/changes/15/93015/1
diff --git
a/wikipedia-it/src/main/java/org/wikimedia/wikipedia/test/PageFetchTaskTests.java
b/wikipedia-it/src/main/java/org/wikimedia/wikipedia/test/PageFetchTaskTests.java
new file mode 100644
index 0000000..e5f149d
--- /dev/null
+++
b/wikipedia-it/src/main/java/org/wikimedia/wikipedia/test/PageFetchTaskTests.java
@@ -0,0 +1,49 @@
+package org.wikimedia.wikipedia.test;
+
+import android.content.Intent;
+import android.test.ActivityUnitTestCase;
+import android.util.Log;
+import org.mediawiki.api.json.Api;
+import org.wikimedia.wikipedia.Page;
+import org.wikimedia.wikipedia.PageFetchTask;
+import org.wikimedia.wikipedia.PageTitle;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class PageFetchTaskTests extends
ActivityUnitTestCase<TestDummyActivity> {
+ private static final int TASK_COMPLETION_TIMEOUT = 20000;
+
+ private Api enwiki;
+
+ public PageFetchTaskTests() {
+ super(TestDummyActivity.class);
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ enwiki = new Api("en.wikipedia.org");
+ startActivity(new Intent(), null, null);
+ }
+
+ public void testPageFetch() throws Throwable {
+ final CountDownLatch completionLatch = new CountDownLatch(1);
+ runTestOnUiThread(new Runnable() {
+ @Override
+ public void run() {
+ new PageFetchTask(enwiki, new PageTitle(null, "India")) {
+ @Override
+ public void onFinish(Page result) {
+ assertNotNull(result);
+ // FIXME: SUPER FLAKY TEST BELOW! 15 is count of first
level sections + 1 in en:India article!
+ assertEquals(15, result.getSections().size());
+ assertEquals(4,
result.getSections().get(2).getSubSections().size());
+ completionLatch.countDown();
+ }
+ }.execute();
+ }
+ });
+ assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT,
TimeUnit.MILLISECONDS));
+ }
+}
diff --git a/wikipedia/src/main/java/org/wikimedia/wikipedia/Page.java
b/wikipedia/src/main/java/org/wikimedia/wikipedia/Page.java
new file mode 100644
index 0000000..b072fef
--- /dev/null
+++ b/wikipedia/src/main/java/org/wikimedia/wikipedia/Page.java
@@ -0,0 +1,33 @@
+package org.wikimedia.wikipedia;
+
+import java.util.ArrayList;
+
+/**
+ * Represents a particular page along with its full contents.
+ */
+public class Page {
+ private final PageTitle title;
+ private final ArrayList<Section> sections;
+
+ public Page(PageTitle title, ArrayList<Section> sections) {
+ this.title = title;
+ this.sections = sections;
+ }
+
+ public PageTitle getTitle() {
+ return title;
+ }
+
+ public ArrayList<Section> getSections() {
+ return sections;
+ }
+
+ public String getHTML() {
+ String html = "";
+ for (Section s : sections) {
+ html += s.toHTML();
+ }
+ return html;
+ }
+
+}
diff --git a/wikipedia/src/main/java/org/wikimedia/wikipedia/PageFetchTask.java
b/wikipedia/src/main/java/org/wikimedia/wikipedia/PageFetchTask.java
new file mode 100644
index 0000000..0930a8c
--- /dev/null
+++ b/wikipedia/src/main/java/org/wikimedia/wikipedia/PageFetchTask.java
@@ -0,0 +1,54 @@
+package org.wikimedia.wikipedia;
+
+import android.content.Context;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.mediawiki.api.json.Api;
+import org.mediawiki.api.json.ApiResult;
+import org.wikimedia.wikipedia.concurrency.ExecutorService;
+import org.wikimedia.wikipedia.concurrency.SaneAsyncTask;
+
+import java.util.ArrayList;
+
+public class PageFetchTask extends SaneAsyncTask<Page> {
+ PageTitle title;
+ Api api;
+
+ public PageFetchTask(Api api, PageTitle title) {
+ super(ExecutorService.getSingleton().getExecutor(PageFetchTask.class));
+ this.title = title;
+ this.api = api;
+ }
+
+ @Override
+ public Page performTask() throws Throwable {
+ ApiResult result = api.action("mobileview")
+ .param("page", title.getTitle()) //TODO: Support non main NS!
+ .param("sections", "all")
+ .param("sectionprop", "toclevel|line|anchor")
+ .param("noheadings", "true")
+ .get();
+ JSONArray sectionsJSON =
result.asObject().optJSONObject("mobileview").optJSONArray("sections");
+
+ ArrayList<Section> sections = new ArrayList<Section>();
+ Section curSection = null;
+
+ for (int i=0; i < sectionsJSON.length(); i++) {
+ JSONObject sectionJSON = sectionsJSON.getJSONObject(i);
+ Section newSection = new Section(sectionJSON.optInt("id"),
+ sectionJSON.optInt("toclevel", 1), // Assume all sections
are 1st level if they don't have that info
+ sectionJSON.optString("line", null),
+ sectionJSON.optString("anchor", null),
+ sectionJSON.optString("text")
+ );
+ if (curSection == null || newSection.getLevel() <=
curSection.getLevel()) {
+ curSection = newSection;
+ sections.add(newSection);
+ } else {
+ curSection.insertSection(newSection);
+ }
+ }
+
+ return new Page(title, sections);
+ }
+}
diff --git a/wikipedia/src/main/java/org/wikimedia/wikipedia/PageTitle.java
b/wikipedia/src/main/java/org/wikimedia/wikipedia/PageTitle.java
new file mode 100644
index 0000000..0ec9ca1
--- /dev/null
+++ b/wikipedia/src/main/java/org/wikimedia/wikipedia/PageTitle.java
@@ -0,0 +1,25 @@
+package org.wikimedia.wikipedia;
+
+/**
+ * Value object representing the title of a page.
+ *
+ * Points to a specific page in a specific namespace on a specific site.
+ * Is immutable.
+ */
+public class PageTitle {
+ private final String namespace;
+ private final String title;
+
+ public PageTitle(final String namesapce, final String title) {
+ this.namespace = namesapce;
+ this.title = title; //FIXME: Actually normalize this!
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+}
diff --git a/wikipedia/src/main/java/org/wikimedia/wikipedia/Section.java
b/wikipedia/src/main/java/org/wikimedia/wikipedia/Section.java
new file mode 100644
index 0000000..ee481d8
--- /dev/null
+++ b/wikipedia/src/main/java/org/wikimedia/wikipedia/Section.java
@@ -0,0 +1,74 @@
+package org.wikimedia.wikipedia;
+
+import java.util.ArrayList;
+
+/**
+ * Represents a particular section of an article.
+ */
+public class Section {
+ private int id;
+ private int level;
+ private String heading;
+ private String anchor;
+ private String content;
+
+ private ArrayList<Section> subSections = new ArrayList<Section>();
+
+ public Section(int id, int level, String heading, String anchor, String
content) {
+ this.id = id;
+ this.level = level;
+ this.heading = heading;
+ this.anchor = anchor;
+ this.content = content;
+ }
+
+ public boolean isLead() {
+ return level == 0;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public int getLevel() {
+ return level;
+ }
+
+ public String getHeading() {
+ return heading;
+ }
+
+ public String getAnchor() {
+ return anchor;
+ }
+
+ public String getContent() {
+ return content;
+ }
+
+ public void insertSection(Section subSection) {
+ subSections.add(subSection);
+ }
+
+ public ArrayList<Section> getSubSections() {
+ return subSections;
+ }
+
+ public String toHTML() {
+ String html = "";
+
+ if (!isLead()) {
+ int headingLevel = getLevel() + 1;
+ html = "<h" + headingLevel + ">" + getHeading() + "</h" +
headingLevel + ">";
+ }
+
+ html += "<div>";
+ html += getContent();
+ for (Section s : subSections) {
+ html += s.toHTML();
+ }
+ html += "</div>";
+
+ return html;
+ }
+}
diff --git
a/wikipedia/src/main/java/org/wikimedia/wikipedia/concurrency/ExecutorService.java
b/wikipedia/src/main/java/org/wikimedia/wikipedia/concurrency/ExecutorService.java
new file mode 100644
index 0000000..f1ea571
--- /dev/null
+++
b/wikipedia/src/main/java/org/wikimedia/wikipedia/concurrency/ExecutorService.java
@@ -0,0 +1,32 @@
+package org.wikimedia.wikipedia.concurrency;
+
+import java.util.HashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+
+/**
+ * Service registry for providing Executors for various purposes.
+ *
+ * TODO: Make number of threads per executor configurable.
+ */
+public class ExecutorService {
+ private static final ExecutorService singleton = new ExecutorService();
+
+ public static ExecutorService getSingleton() {
+ return singleton;
+ }
+
+ private final HashMap<Class, Executor> executors;
+
+ public ExecutorService() {
+ executors = new HashMap<Class, Executor>();
+ }
+
+ public Executor getExecutor(Class cls) {
+ if (!executors.containsKey(cls)) {
+ // TODO: Custom number of threads for services
+ executors.put(cls, new ScheduledThreadPoolExecutor(1));
+ }
+ return executors.get(cls);
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/93015
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I83b393d6b37f5e47e05913f111d04e8cdb750f85
Gerrit-PatchSet: 1
Gerrit-Project: apps/android/wikipedia
Gerrit-Branch: master
Gerrit-Owner: Yuvipanda <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
