Repository: gora Updated Branches: refs/heads/master 903aeb016 -> bb09d8916
GORA-424 Cache cursor size to improve performance. - Cache cursor size to improve performance in MongoStore - Avoid integer division bug - Added result progress test This closes apache/gora#26 PR. Squashed commit of the following: commit 8d47c13096d15f1936d08b69757b1973ea781955 Author: Alexander Yastrebov <[email protected]> Date: Wed Jun 10 11:37:35 2015 +0300 Fixed integer division bug. Added result progress test commit 1ebf813a1bbb803aff6bbcd15bbd7e6ee83b788b Author: Alexander Yastrebov <[email protected]> Date: Tue Jun 9 19:51:57 2015 +0300 Cache cursor size Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/bb09d891 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/bb09d891 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/bb09d891 Branch: refs/heads/master Commit: bb09d8916736e3e572eef2cb5e79fe11c80ad1de Parents: 903aeb0 Author: Damien Raude-Morvan <[email protected]> Authored: Wed Jun 10 22:33:14 2015 +0200 Committer: Damien Raude-Morvan <[email protected]> Committed: Wed Jun 10 22:33:14 2015 +0200 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../gora/mongodb/query/MongoDBResult.java | 20 ++--- .../gora/mongodb/store/TestMongoStore.java | 78 ++++++++++++++++++-- 3 files changed, 84 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 12d036e..1ac61d5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,8 @@ Apache Gora 0.6.1 Release - 02/03/2015 (dd/mm/yyyy) Release Report - http://s.apache.org/l69 +* GORA-424 Cache cursor size to improve performance (Alexander Yastrebov via drazzib) + * GORA-423 BSONDecorator returns empty string for null field value (Alexander Yastrebov via drazzib) * GORA-262 Add support for HTTPClient authentication in gora-solr (Furkan KAMACI via lewismc) http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java index 5524345..a032ec0 100644 --- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java +++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java @@ -31,17 +31,18 @@ import com.mongodb.DBObject; /** * MongoDB specific implementation of the {@link org.apache.gora.query.Result} * interface. - * + * * @author Fabien Poulard <[email protected]> * @author Damien Raude-Morvan <[email protected]> */ public class MongoDBResult<K, T extends PersistentBase> extends - ResultBase<K, T> { + ResultBase<K, T> { /** * Reference to the cursor pointing to the results */ private DBCursor cursor; + private int size; public MongoDBResult(DataStore<K, T> dataStore, Query<K, T> query) { super(dataStore, query); @@ -49,12 +50,13 @@ public class MongoDBResult<K, T extends PersistentBase> extends @Override public float getProgress() throws IOException { - if (cursor == null) + if (cursor == null) { return 0; - else if (cursor.size() == 0) + } else if (size == 0) { return 1; - else - return offset / cursor.size(); + } else { + return offset / (float) size; + } } @Override @@ -73,19 +75,19 @@ public class MongoDBResult<K, T extends PersistentBase> extends DBObject obj = cursor.next(); key = (K) obj.get("_id"); persistent = ((MongoStore<K, T>) getDataStore()).newInstance(obj, - getQuery().getFields()); + getQuery().getFields()); return persistent != null; } /** * Save the reference to the cursor that holds the actual results. - * + * * @param cursor * {@link DBCursor} obtained from a query execution and that holds * the actual results */ public void setCursor(DBCursor cursor) { this.cursor = cursor; + this.size = cursor.size(); } - } http://git-wip-us.apache.org/repos/asf/gora/blob/bb09d891/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java ---------------------------------------------------------------------- diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java index e29d4db..f2cc6ce 100644 --- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java +++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/store/TestMongoStore.java @@ -17,7 +17,7 @@ */ package org.apache.gora.mongodb.store; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; import java.io.IOException; @@ -33,27 +33,36 @@ import org.junit.Test; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; +import java.util.Collections; +import org.apache.avro.util.Utf8; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import static org.junit.Assert.assertNotNull; public abstract class TestMongoStore extends DataStoreTestBase { + private int keySequence; + @Deprecated @Override protected DataStore<String, Employee> createEmployeeDataStore() - throws IOException { + throws IOException { throw new UnsupportedOperationException(); } @Deprecated @Override protected DataStore<String, WebPage> createWebPageDataStore() - throws IOException { + throws IOException { throw new UnsupportedOperationException(); } @Before + @Override public void setUp() throws Exception { super.setUp(); + keySequence = 1; } public GoraMongodbTestDriver getTestDriver() { @@ -84,7 +93,7 @@ public abstract class TestMongoStore extends DataStoreTestBase { BasicDBObject noField = new BasicDBObject(); String field = "myField"; Object item = store.fromMongoList(field, null, new BSONDecorator(noField), - null); + null); assertNotNull(item); } @@ -94,7 +103,7 @@ public abstract class TestMongoStore extends DataStoreTestBase { String field = "myField"; BasicDBObject emptyField = new BasicDBObject(field, new BasicDBList()); Object item = store.fromMongoList(field, null, - new BSONDecorator(emptyField), null); + new BSONDecorator(emptyField), null); assertNotNull(item); } @@ -104,7 +113,7 @@ public abstract class TestMongoStore extends DataStoreTestBase { BasicDBObject noField = new BasicDBObject(); String field = "myField"; Object item = store.fromMongoMap(field, null, new BSONDecorator(noField), - null); + null); assertNotNull(item); } @@ -114,7 +123,62 @@ public abstract class TestMongoStore extends DataStoreTestBase { String field = "myField"; BasicDBObject emptyField = new BasicDBObject(field, new BasicDBObject()); Object item = store.fromMongoMap(field, null, - new BSONDecorator(emptyField), null); + new BSONDecorator(emptyField), null); assertNotNull(item); } + + @Test + public void testResultProgress() throws Exception { + Query<String, WebPage> q; + + // empty + q = webPageStore.newQuery(); + assertProgress(q, 1); + + addWebPage(); + + // one + q = webPageStore.newQuery(); + assertProgress(q, 0, 1); + + addWebPage(); + + // two + q = webPageStore.newQuery(); + assertProgress(q, 0, 0.5f, 1); + + addWebPage(); + + // three + q = webPageStore.newQuery(); + assertProgress(q, 0, 0.33f, 0.66f, 1); + } + + @Test + public void testResultProgressWithLimit() throws Exception { + for (int i = 0; i < 5; i++) { + addWebPage(); + } + Query<String, WebPage> q = webPageStore.newQuery(); + q.setLimit(2); + + assertProgress(q, 0, 0.5f, 1); + } + + private void assertProgress(Query<String, WebPage> q, float... progress) throws Exception { + Result<String, WebPage> r = webPageStore.execute(q); + int i = 0; + do { + assertEquals(progress[i++], r.getProgress(), 0.01f); + } while (r.next()); + r.close(); + } + + private void addWebPage() { + String key = String.valueOf(keySequence++); + WebPage p1 = webPageStore.newPersistent(); + p1.setUrl(new Utf8(key)); + p1.setHeaders(Collections.singletonMap((CharSequence) "header", (CharSequence) "value")); + webPageStore.put(key, p1); + } }
