jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/364440 )
Change subject: ZIM Compilations: the beginning.
......................................................................
ZIM Compilations: the beginning.
This patch forms the kernel of how the rest of the app will interface with
ZIM compilations. We introduce the OfflineManager class, which is a
singleton that will provide content from a compilation. Whether it's using
one or more compilations simultaneously is completely transparent.
This does not yet introduce any actual usage of compilations in the app,
but simply brings the foundation for upcoming patches.
Also, tests.
Bug: T163584
Bug: T166562
Change-Id: If3eac3ce42412d605d51ea26c6232d8225d6a8c7
---
M app/build.gradle
A app/src/main/java/org/wikipedia/offline/Compilation.java
A app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
A app/src/main/java/org/wikipedia/offline/OfflineManager.java
A app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
M app/src/test/java/org/wikipedia/test/TestFileUtil.java
A app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
7 files changed, 391 insertions(+), 6 deletions(-)
Approvals:
jenkins-bot: Verified
Mholloway: Looks good to me, approved
diff --git a/app/build.gradle b/app/build.gradle
index 7f92afc..cfcec27 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -204,6 +204,7 @@
compile 'net.hockeyapp.android:HockeySDK:4.1.3'
compile 'org.apache.commons:commons-lang3:3.5'
compile 'org.jsoup:jsoup:1.10.2'
+ compile 'com.dmitrybrant:zimdroid:0.0.9'
annotationProcessor
"com.jakewharton:butterknife-compiler:$butterKnifeVersion"
diff --git a/app/src/main/java/org/wikipedia/offline/Compilation.java
b/app/src/main/java/org/wikipedia/offline/Compilation.java
new file mode 100644
index 0000000..03d7c9a
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/Compilation.java
@@ -0,0 +1,104 @@
+package org.wikipedia.offline;
+
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.support.annotation.VisibleForTesting;
+import android.text.TextUtils;
+import android.util.LruCache;
+
+import com.dmitrybrant.zimdroid.ZimFile;
+import com.dmitrybrant.zimdroid.ZimReader;
+
+import org.wikipedia.util.log.L;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.List;
+
+public class Compilation {
+ @NonNull private ZimFile file;
+ @NonNull private ZimReader reader;
+
+ public Compilation(@NonNull File file) throws IOException {
+ this.file = new ZimFile(file.getAbsolutePath());
+ reader = new ZimReader(this.file);
+ }
+
+ @VisibleForTesting
+ Compilation(@NonNull File file, LruCache titleCache, LruCache urlCache)
throws Exception {
+ this.file = new ZimFile(file.getAbsolutePath());
+ reader = new ZimReader(this.file, titleCache, urlCache);
+ }
+
+ public void close() {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ // close silently
+ }
+ }
+
+ @NonNull public String path() {
+ return file.getAbsolutePath();
+ }
+
+ public long size() {
+ return file.length();
+ }
+
+ @NonNull public String name() {
+ try {
+ return reader.getZimTitle();
+ } catch (IOException e) {
+ L.e(e);
+ }
+ return "";
+ }
+
+ @NonNull public String description() {
+ try {
+ return reader.getZimDescription();
+ } catch (IOException e) {
+ L.e(e);
+ }
+ return "";
+ }
+
+ @NonNull public List<String> searchByPrefix(@NonNull String prefix, int
maxResults) throws IOException {
+ return reader.searchByPrefix(prefix, maxResults);
+ }
+
+ public boolean titleExists(@NonNull String title) {
+ return !TextUtils.isEmpty(getNormalizedTitle(title));
+ }
+
+ @Nullable public String getNormalizedTitle(@NonNull String title) {
+ try {
+ return reader.getNormalizedTitle(title);
+ } catch (Exception e) {
+ L.e(e);
+ }
+ return null;
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForTitle(@NonNull String
title) throws IOException {
+ return reader.getDataForTitle(title);
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForUrl(@NonNull String url)
throws IOException {
+ if (url.startsWith("A/") || url.startsWith("I/")) {
+ url = url.substring(2);
+ }
+ return reader.getDataForUrl(URLDecoder.decode(url, "utf-8"));
+ }
+
+ @NonNull public String getRandomTitle() throws IOException {
+ return reader.getRandomTitle();
+ }
+
+ @NonNull public String getMainPageTitle() throws IOException {
+ return reader.getMainPageTitle();
+ }
+}
diff --git a/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
b/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
new file mode 100644
index 0000000..0fe86d6
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/CompilationSearchTask.java
@@ -0,0 +1,81 @@
+package org.wikipedia.offline;
+
+import android.content.Context;
+import android.os.Environment;
+import android.os.storage.StorageManager;
+import android.support.annotation.NonNull;
+
+import org.wikipedia.WikipediaApp;
+import org.wikipedia.concurrency.SaneAsyncTask;
+import org.wikipedia.util.log.L;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+class CompilationSearchTask extends SaneAsyncTask<List<Compilation>> {
+ private List<Compilation> compilations = new ArrayList<>();
+
+ @Override
+ public List<Compilation> performTask() throws Throwable {
+ List<String> pathList = new ArrayList<>();
+ StorageManager sm = (StorageManager)
WikipediaApp.getInstance().getSystemService(Context.STORAGE_SERVICE);
+ try {
+ String[] volumes = (String[])
sm.getClass().getMethod("getVolumePaths").invoke(sm);
+ if (volumes != null && volumes.length > 0) {
+ pathList.addAll(Arrays.asList(volumes));
+ }
+ } catch (Exception e) {
+ L.e(e);
+ }
+ if (pathList.size() == 0 && Environment.getExternalStorageDirectory()
!= null) {
+
pathList.add(Environment.getExternalStorageDirectory().getAbsolutePath());
+ }
+ for (String path : pathList) {
+ findCompilations(new File(path), 0);
+ if (isCancelled()) {
+ break;
+ }
+ }
+ return compilations;
+ }
+
+ private void findCompilations(@NonNull File parentDir, int level) {
+ if (level > 10) {
+ return;
+ }
+ File[] files = parentDir.listFiles();
+ if (files == null) {
+ return;
+ }
+ for (File file : files) {
+ if (isCancelled()) {
+ return;
+ }
+ if (file.isDirectory()) {
+ findCompilations(file, level + 1);
+ } else {
+ if (isCompilation(file)) {
+ add(file);
+ }
+ }
+ }
+ }
+
+ private void add(@NonNull File file) {
+ try {
+ compilations.add(new Compilation(file));
+ L.d("Found compilation: " + file.getAbsolutePath());
+ } catch (IOException e) {
+ L.e("Error opening compilation: " + file.getAbsolutePath());
+ e.printStackTrace();
+ }
+ }
+
+ private boolean isCompilation(@NonNull File f) {
+ return f.getName().toLowerCase(Locale.ROOT).endsWith(".zim");
+ }
+}
diff --git a/app/src/main/java/org/wikipedia/offline/OfflineManager.java
b/app/src/main/java/org/wikipedia/offline/OfflineManager.java
new file mode 100644
index 0000000..b6e40d6
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/offline/OfflineManager.java
@@ -0,0 +1,133 @@
+package org.wikipedia.offline;
+
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.support.annotation.VisibleForTesting;
+
+import org.wikipedia.util.log.L;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public final class OfflineManager {
+ private static OfflineManager INSTANCE = new OfflineManager();
+ @Nullable private CompilationSearchTask searchTask;
+ @NonNull private List<Compilation> compilations = new ArrayList<>();
+
+ public interface Callback {
+ void onCompilationsFound(@NonNull List<Compilation> compilations);
+ void onError(@NonNull Throwable t);
+ }
+
+ public static OfflineManager instance() {
+ return INSTANCE;
+ }
+
+ public static boolean hasCompilation() {
+ return instance().compilations().size() > 0;
+ }
+
+ @NonNull
+ public List<Compilation> compilations() {
+ return compilations;
+ }
+
+ public void searchForCompilations(@NonNull final Callback callback) {
+ if (searchTask != null) {
+ searchTask.cancel();
+ }
+ searchTask = new CompilationSearchTask() {
+ @Override public void onFinish(List<Compilation> result) {
+ if (isCancelled()) {
+ return;
+ }
+ for (Compilation c : compilations) {
+ c.close();
+ }
+ compilations.clear();
+ compilations.addAll(result);
+ callback.onCompilationsFound(result);
+ }
+
+ @Override public void onCatch(Throwable caught) {
+ L.e("Error while searching for compilations.", caught);
+ callback.onError(caught);
+ }
+ };
+ searchTask.execute();
+ }
+
+ public boolean titleExists(@NonNull String title) {
+ for (Compilation c : compilations) {
+ if (c.titleExists(title)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @NonNull public List<String> searchByPrefix(@NonNull String prefix, int
maxResults) throws IOException {
+ List<String> results = new ArrayList<>();
+ for (Compilation c : compilations) {
+ results.addAll(c.searchByPrefix(prefix, maxResults));
+ }
+ return results;
+ }
+
+ @Nullable public String getNormalizedTitle(@NonNull String title) {
+ try {
+ for (Compilation c : compilations) {
+ String result = c.getNormalizedTitle(title);
+ if (result != null && result.length() > 0) {
+ return result;
+ }
+ }
+ } catch (Exception e) {
+ L.e(e);
+ }
+ return null;
+ }
+
+ @NonNull public String getHtmlForTitle(@NonNull String title) throws
IOException {
+ for (Compilation c : compilations) {
+ ByteArrayOutputStream stream = c.getDataForTitle(title);
+ if (stream != null) {
+ return stream.toString("utf-8");
+ }
+ }
+ throw new IOException("Content not found in any compilation for " +
title);
+ }
+
+ @Nullable public ByteArrayOutputStream getDataForUrl(@NonNull String url)
throws IOException {
+ if (url.startsWith("A/") || url.startsWith("I/")) {
+ url = url.substring(2);
+ }
+ for (Compilation c : compilations) {
+ ByteArrayOutputStream stream = c.getDataForUrl(url);
+ if (stream != null) {
+ return stream;
+ }
+ }
+ return null;
+ }
+
+ @NonNull public String getRandomTitle() throws IOException {
+ int compIndex = new Random().nextInt(compilations.size());
+ return compilations.get(compIndex).getRandomTitle();
+ }
+
+ @NonNull public String getMainPageTitle() throws IOException {
+ int compIndex = new Random().nextInt(compilations.size());
+ return compilations.get(compIndex).getMainPageTitle();
+ }
+
+ @VisibleForTesting void setCompilations(@NonNull List<Compilation>
compilations) {
+ this.compilations = compilations;
+ }
+
+ private OfflineManager() {
+ }
+}
diff --git a/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
b/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
new file mode 100644
index 0000000..3e57a6e
--- /dev/null
+++ b/app/src/test/java/org/wikipedia/offline/OfflineManagerTest.java
@@ -0,0 +1,64 @@
+package org.wikipedia.offline;
+
+import android.util.LruCache;
+
+import com.dmitrybrant.zimdroid.DirectoryEntry;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.wikipedia.test.TestFileUtil;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class OfflineManagerTest {
+ private static final String TEST_ZIM_FILE =
"wikipedia_en_ray_charles_2015-06.zim";
+
+ @Mock private LruCache<Integer, DirectoryEntry> mockCache;
+
+ @Test
+ public void testOfflineManagerRandom() throws Exception {
+ String randomTitle = OfflineManager.instance().getRandomTitle();
+ assertThat(randomTitle.length(), greaterThan(0));
+ }
+
+ @Test
+ public void testOfflineManagerSearch() throws Exception {
+ List<String> results = OfflineManager.instance().searchByPrefix("R",
2);
+ assertThat(results.size(), is(2));
+ assertThat(results.get(0), is("Raelette"));
+ }
+
+ @Test
+ public void testOfflineManagerNormalizedTitle() throws Exception {
+ String normalizedTitle =
OfflineManager.instance().getNormalizedTitle("You got the right one baby");
+ assertThat(normalizedTitle, is("You Got the Right One, Baby"));
+ }
+
+ @Test
+ public void testOfflineManagerGetDataForTitle() throws Exception {
+ String html = OfflineManager.instance().getHtmlForTitle("Ray Charles");
+ assertThat(html.startsWith("<html>"), is(true));
+ assertThat(html.endsWith("</html>"), is(true));
+ }
+
+ @Before
+ public void setup() throws Exception {
+
+ when(mockCache.get(any(Integer.TYPE))).thenReturn(null);
+
+ Compilation compilation = new
Compilation(TestFileUtil.getRawFile(TEST_ZIM_FILE),
+ mockCache, mockCache);
+
OfflineManager.instance().setCompilations(Collections.singletonList(compilation));
+ }
+}
diff --git a/app/src/test/java/org/wikipedia/test/TestFileUtil.java
b/app/src/test/java/org/wikipedia/test/TestFileUtil.java
index 53fb61f..6d91a61 100644
--- a/app/src/test/java/org/wikipedia/test/TestFileUtil.java
+++ b/app/src/test/java/org/wikipedia/test/TestFileUtil.java
@@ -1,5 +1,7 @@
package org.wikipedia.test;
+import android.support.annotation.NonNull;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
@@ -9,15 +11,15 @@
private static final String MULTILINE_START_ANCHOR_REGEX = "\\A";
private static final String RAW_DIR = "src/test/res/raw/";
+ public static File getRawFile(@NonNull String rawFileName) {
+ return new File(RAW_DIR + rawFileName);
+ }
+
public static String readRawFile(String basename) throws
FileNotFoundException {
- return readFile(RAW_DIR + basename);
+ return readFile(getRawFile(basename));
}
- public static String readFile(String filename) throws
FileNotFoundException {
- return readFile(new File(filename));
- }
-
- public static String readFile(File file) throws FileNotFoundException {
+ private static String readFile(File file) throws FileNotFoundException {
Scanner scanner = new Scanner(file);
String ret = scanner.useDelimiter(MULTILINE_START_ANCHOR_REGEX).next();
scanner.close();
diff --git a/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
b/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
new file mode 100644
index 0000000..8015373
--- /dev/null
+++ b/app/src/test/res/raw/wikipedia_en_ray_charles_2015-06.zim
Binary files differ
--
To view, visit https://gerrit.wikimedia.org/r/364440
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If3eac3ce42412d605d51ea26c6232d8225d6a8c7
Gerrit-PatchSet: 4
Gerrit-Project: apps/android/wikipedia
Gerrit-Branch: master
Gerrit-Owner: Dbrant <[email protected]>
Gerrit-Reviewer: Dbrant <[email protected]>
Gerrit-Reviewer: Mholloway <[email protected]>
Gerrit-Reviewer: Niedzielski <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
