Dbrant has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/364440 )
Change subject: ZIM Compilations: the beginning.
......................................................................
ZIM Compilations: the beginning.
This patch forms the kernel of how the rest of the app will interface with
ZIM compilations. We introduce the OfflineManager class, which is a
singleton that will provide content from a compilation. Whether it's using
one or more compilations simultaneously is completely transparent.
This does not yet introduce any actual usage of compilations in the app,
but simply brings the foundation for upcoming patches.
Also, tests.
Bug: T163584
Bug: T166562
Change-Id: If3eac3ce42412d605d51ea26c6232d8225d6a8c7
---
M app/build.gradle
A app/src/main/java/org/wikipedia/offline/Compilation.java
A app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
A app/src/main/java/org/wikipedia/offline/OfflineManager.java
A app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
M app/src/test/java/org/wikipedia/test/TestFileUtil.java
A app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
7 files changed, 367 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/apps/android/wikipedia
refs/changes/40/364440/1
diff --git a/app/build.gradle b/app/build.gradle
index 7f92afc..0471860 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -141,6 +141,10 @@
exclude 'META-INF/maven/com.google.guava/guava/pom.xml'
exclude 'META-INF/maven/com.google.guava/guava/pom.properties'
}
+
+ testOptions {
+ unitTests.returnDefaultValues = true
+ }
}
apply from: '../gradle/src/test.gradle'
@@ -204,6 +208,7 @@
compile 'net.hockeyapp.android:HockeySDK:4.1.3'
compile 'org.apache.commons:commons-lang3:3.5'
compile 'org.jsoup:jsoup:1.10.2'
+ compile 'com.dmitrybrant:zimdroid:0.0.7'
annotationProcessor
"com.jakewharton:butterknife-compiler:$butterKnifeVersion"
diff --git a/app/src/main/java/org/wikipedia/offline/Compilation.java
b/app/src/main/java/org/wikipedia/offline/Compilation.java
new file mode 100644
index 0000000..08317d6
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/Compilation.java
@@ -0,0 +1,96 @@
+package org.wikipedia.offline;
+
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.text.TextUtils;
+
+import com.dmitrybrant.zimdroid.ZIMFile;
+import com.dmitrybrant.zimdroid.ZIMReader;
+
+import org.wikipedia.util.log.L;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.List;
+
+public class Compilation {
+ @NonNull private ZIMFile file;
+ @NonNull private ZIMReader reader;
+
+ public Compilation(@NonNull File file) throws IOException {
+ this.file = new ZIMFile(file.getAbsolutePath());
+ reader = new ZIMReader(this.file);
+ }
+
+ public void close() {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ // close silently
+ }
+ }
+
+ @NonNull public String path() {
+ return file.getAbsolutePath();
+ }
+
+ public long size() {
+ return file.length();
+ }
+
+ @NonNull public String name() {
+ try {
+ return reader.getZimTitle();
+ } catch (IOException e) {
+ L.e(e);
+ }
+ return "";
+ }
+
+ @NonNull public String description() {
+ try {
+ return reader.getZimDescription();
+ } catch (IOException e) {
+ L.e(e);
+ }
+ return "";
+ }
+
+ @NonNull public List<String> searchByPrefix(@NonNull String prefix, int
maxResults) throws IOException {
+ return reader.searchByPrefix(prefix, maxResults);
+ }
+
+ public boolean titleExists(@NonNull String title) {
+ return !TextUtils.isEmpty(getNormalizedTitle(title));
+ }
+
+ @Nullable public String getNormalizedTitle(@NonNull String title) {
+ try {
+ return reader.getNormalizedTitle(title);
+ } catch (Exception e) {
+ L.e(e);
+ }
+ return null;
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForTitle(@NonNull String
title) throws IOException {
+ return reader.getDataForTitle(title);
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForUrl(@NonNull String url)
throws IOException {
+ if (url.startsWith("A/") || url.startsWith("I/")) {
+ url = url.substring(2);
+ }
+ return reader.getDataForUrl(URLDecoder.decode(url, "utf-8"));
+ }
+
+ @NonNull public String getRandomTitle() throws IOException {
+ return reader.getRandomTitle();
+ }
+
+ @NonNull public String getMainPageTitle() throws IOException {
+ return reader.getMainPageTitle();
+ }
+}
diff --git a/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
b/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
new file mode 100644
index 0000000..92074ce
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
@@ -0,0 +1,76 @@
+package org.wikipedia.offline;
+
+import android.content.Context;
+import android.os.Environment;
+import android.os.storage.StorageManager;
+import android.support.annotation.NonNull;
+
+import org.wikipedia.WikipediaApp;
+import org.wikipedia.concurrency.SaneAsyncTask;
+import org.wikipedia.util.log.L;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+class CompilationSearchTask extends SaneAsyncTask<List<Compilation>> {
+ private List<Compilation> zimFiles = new ArrayList<>();
+
+ @Override
+ public List<Compilation> performTask() throws Throwable {
+ List<String> pathList = new ArrayList<>();
+ StorageManager sm = (StorageManager)
WikipediaApp.getInstance().getSystemService(Context.STORAGE_SERVICE);
+ try {
+ String[] volumes = (String[])
sm.getClass().getMethod("getVolumePaths").invoke(sm);
+ if (volumes != null && volumes.length > 0) {
+ pathList.addAll(Arrays.asList(volumes));
+ }
+ } catch (Exception e) {
+ L.e(e);
+ }
+ if (pathList.size() == 0 && Environment.getExternalStorageDirectory()
!= null) {
+
pathList.add(Environment.getExternalStorageDirectory().getAbsolutePath());
+ }
+ for (String path : pathList) {
+ getZimFiles(new File(path), 0);
+ if (isCancelled()) {
+ return zimFiles;
+ }
+ }
+ return zimFiles;
+ }
+
+ private void getZimFiles(@NonNull File parentDir, int level) {
+ if (level > 10) {
+ return;
+ }
+ File[] files = parentDir.listFiles();
+ if (files == null) {
+ return;
+ }
+ for (File file : files) {
+ if (isCancelled()) {
+ return;
+ }
+ if (file.isDirectory()) {
+ getZimFiles(file, level + 1);
+ } else {
+ if (file.getName().toLowerCase(Locale.ROOT).endsWith(".zim")) {
+ addZimFile(file);
+ }
+ }
+ }
+ }
+
+ private void addZimFile(File file) {
+ try {
+ zimFiles.add(new Compilation(file));
+ L.d("Found Zim file: " + file.getAbsolutePath());
+ } catch (IOException e) {
+ L.d("Error opening Zim file: " + file.getAbsolutePath());
+ }
+ }
+}
diff --git a/app/src/main/java/org/wikipedia/offline/OfflineManager.java
b/app/src/main/java/org/wikipedia/offline/OfflineManager.java
new file mode 100644
index 0000000..98bb3ee
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/OfflineManager.java
@@ -0,0 +1,135 @@
+package org.wikipedia.offline;
+
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.support.annotation.VisibleForTesting;
+import android.text.TextUtils;
+
+import org.wikipedia.util.log.L;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public final class OfflineManager {
+ private static OfflineManager INSTANCE = new OfflineManager();
+ @Nullable private CompilationSearchTask searchTask;
+ @NonNull private List<Compilation> compilations = new ArrayList<>();
+
+ public interface Callback {
+ void onCompilationsFound(@NonNull List<Compilation> compilations);
+ void onError(@NonNull Throwable t);
+ }
+
+ public static OfflineManager instance() {
+ return INSTANCE;
+ }
+
+ public static boolean hasCompilation() {
+ return instance().compilations().size() > 0;
+ }
+
+ @NonNull
+ public List<Compilation> compilations() {
+ return compilations;
+ }
+
+ public void searchForCompilations(@NonNull final Callback callback) {
+ if (searchTask != null) {
+ searchTask.cancel();
+ }
+ searchTask = new CompilationSearchTask() {
+ @Override public void onFinish(List<Compilation> result) {
+ if (isCancelled()) {
+ return;
+ }
+ for (Compilation c : compilations) {
+ c.close();
+ }
+ compilations.clear();
+ compilations.addAll(result);
+ callback.onCompilationsFound(result);
+ }
+
+ @Override public void onCatch(Throwable caught) {
+ L.e("Error while searching for compilations.", caught);
+ callback.onError(caught);
+ }
+ };
+ searchTask.execute();
+ }
+
+ public boolean titleExists(@NonNull String title) {
+ for (Compilation c : compilations) {
+ if (c.titleExists(title)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @NonNull public List<String> searchByPrefix(@NonNull String prefix, int
maxResults) throws IOException {
+ List<String> results = new ArrayList<>();
+ for (Compilation c : compilations) {
+ results.addAll(c.searchByPrefix(prefix, maxResults));
+ }
+ return results;
+ }
+
+ @Nullable public String getNormalizedTitle(@NonNull String title) {
+ try {
+ for (Compilation c : compilations) {
+ String result = c.getNormalizedTitle(title);
+ if (!TextUtils.isEmpty(result)) {
+ return result;
+ }
+ }
+ } catch (Exception e) {
+ L.e(e);
+ }
+ return null;
+ }
+
+ @NonNull public String getHtmlForTitle(@NonNull String title) throws
IOException {
+ for (Compilation c : compilations) {
+ ByteArrayOutputStream stream = c.getDataForTitle(title);
+ if (stream != null) {
+ return stream.toString("utf-8");
+ }
+ }
+ throw new IOException("Content not found in any compilation for " +
title);
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForUrl(@NonNull String url)
throws IOException {
+ if (url.startsWith("A/") || url.startsWith("I/")) {
+ url = url.substring(2);
+ }
+ for (Compilation c : compilations) {
+ ByteArrayOutputStream stream = c.getDataForUrl(url);
+ if (stream != null) {
+ return stream;
+ }
+ }
+ return null;
+ }
+
+ @NonNull public String getRandomTitle() throws IOException {
+ int compIndex = new Random().nextInt(compilations.size());
+ return compilations.get(compIndex).getRandomTitle();
+ }
+
+ @NonNull public String getMainPageTitle() throws IOException {
+ int compIndex = new Random().nextInt(compilations.size());
+ return compilations.get(compIndex).getMainPageTitle();
+ }
+
+ @VisibleForTesting void setCompilation(@NonNull Compilation compilation) {
+ compilations.clear();
+ compilations.add(compilation);
+ }
+
+ private OfflineManager() {
+ }
+}
diff --git a/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
b/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
new file mode 100644
index 0000000..6876304
--- /dev/null
+++ b/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
@@ -0,0 +1,47 @@
+package org.wikipedia.offline;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.wikipedia.test.TestFileUtil;
+
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+
+public class OfflineManagerTest {
+ private static final String TEST_ZIM_FILE =
"wikipedia_en_ray_charles_2015-06.zim";
+
+ @Test
+ public void testOfflineManagerRandom() throws Exception {
+ String randomTitle = OfflineManager.instance().getRandomTitle();
+ assertThat(randomTitle.length(), greaterThan(0));
+ }
+
+ @Test
+ public void testOfflineManagerSearch() throws Exception {
+ List<String> results = OfflineManager.instance().searchByPrefix("R",
2);
+ assertThat(results.size(), is(2));
+ assertThat(results.get(0), is("Raelette"));
+ }
+
+ @Test
+ public void testOfflineManagerNormalizedTitle() throws Exception {
+ String normalizedTitle =
OfflineManager.instance().getNormalizedTitle("ray charles");
+ assertThat(normalizedTitle, is("Ray Charles"));
+ }
+
+ @Test
+ public void testOfflineManagerGetDataForTitle() throws Exception {
+ String html = OfflineManager.instance().getHtmlForTitle("Ray Charles");
+ assertThat(html.startsWith("<html>"), is(true));
+ assertThat(html.endsWith("</html>"), is(true));
+ }
+
+ @Before
+ public void setup() throws Exception {
+ Compilation compilation = new
Compilation(TestFileUtil.getRawFile(TEST_ZIM_FILE));
+ OfflineManager.instance().setCompilation(compilation);
+ }
+}
diff --git a/app/src/test/java/org/wikipedia/test/TestFileUtil.java
b/app/src/test/java/org/wikipedia/test/TestFileUtil.java
index 53fb61f..6d91a61 100644
--- a/app/src/test/java/org/wikipedia/test/TestFileUtil.java
+++ b/app/src/test/java/org/wikipedia/test/TestFileUtil.java
@@ -1,5 +1,7 @@
package org.wikipedia.test;
+import android.support.annotation.NonNull;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
@@ -9,15 +11,15 @@
private static final String MULTILINE_START_ANCHOR_REGEX = "\\A";
private static final String RAW_DIR = "src/test/res/raw/";
+ public static File getRawFile(@NonNull String rawFileName) {
+ return new File(RAW_DIR + rawFileName);
+ }
+
public static String readRawFile(String basename) throws
FileNotFoundException {
- return readFile(RAW_DIR + basename);
+ return readFile(getRawFile(basename));
}
- public static String readFile(String filename) throws
FileNotFoundException {
- return readFile(new File(filename));
- }
-
- public static String readFile(File file) throws FileNotFoundException {
+ private static String readFile(File file) throws FileNotFoundException {
Scanner scanner = new Scanner(file);
String ret = scanner.useDelimiter(MULTILINE_START_ANCHOR_REGEX).next();
scanner.close();
diff --git a/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
b/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
new file mode 100644
index 0000000..8015373
--- /dev/null
+++ b/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
Binary files differ
--
To view, visit https://gerrit.wikimedia.org/r/364440
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If3eac3ce42412d605d51ea26c6232d8225d6a8c7
Gerrit-PatchSet: 1
Gerrit-Project: apps/android/wikipedia
Gerrit-Branch: master
Gerrit-Owner: Dbrant <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
