Repository: gora Updated Branches: refs/heads/master 93cbd8cc3 -> 97fc0d678
GORA-240 Tests for MemStore Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/11b1d7ac Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/11b1d7ac Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/11b1d7ac Branch: refs/heads/master Commit: 11b1d7accbe787010218152b6eb25b17ecbe5809 Parents: 93cbd8c Author: Lewis John McGibbney <[email protected]> Authored: Sat Oct 3 12:52:21 2015 -0700 Committer: Lewis John McGibbney <[email protected]> Committed: Sat Oct 3 12:52:21 2015 -0700 ---------------------------------------------------------------------- gora-core/pom.xml | 10 -- .../org/apache/gora/memory/store/MemStore.java | 55 +++++-- .../apache/gora/memory/store/MemStoreTest.java | 164 ++++++++++++++++++- .../apache/gora/hbase/store/TestHBaseStore.java | 6 +- 4 files changed, 206 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/11b1d7ac/gora-core/pom.xml ---------------------------------------------------------------------- diff --git a/gora-core/pom.xml b/gora-core/pom.xml index 8d8a29d..3b7c402 100644 --- a/gora-core/pom.xml +++ b/gora-core/pom.xml @@ -157,10 +157,6 @@ <!-- Logging Dependencies --> <dependency> <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </dependency> <dependency> @@ -187,12 +183,6 @@ <scope>test</scope> </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-simple</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> </project> http://git-wip-us.apache.org/repos/asf/gora/blob/11b1d7ac/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java b/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java index 162485c..e5f9aca 100644 --- a/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java +++ b/gora-core/src/main/java/org/apache/gora/memory/store/MemStore.java @@ -27,9 +27,7 @@ import java.util.NavigableMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; - import org.apache.avro.Schema.Field; - import org.apache.gora.persistency.Persistent; import org.apache.gora.persistency.impl.PersistentBase; import org.apache.gora.query.PartitionQuery; @@ -41,11 +39,15 @@ import org.apache.gora.query.impl.ResultBase; import org.apache.gora.store.DataStore; import org.apache.gora.store.impl.DataStoreBase; import org.apache.gora.util.AvroUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Memory based {@link DataStore} implementation for tests. */ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { + + private static final Logger LOG = LoggerFactory.getLogger(MemStore.class); public static class MemQuery<K, T extends PersistentBase> extends QueryBase<K, T> { public MemQuery() { @@ -67,7 +69,7 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { } //@Override public void close() { } - + @Override public float getProgress() throws IOException { return 0; @@ -95,7 +97,7 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { @Override public String getSchemaName() { - return "default"; + return "MemStore"; } @Override @@ -105,8 +107,8 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { @Override public long deleteByQuery(Query<K, T> query) { - try{ - long deletedRows = 0; + try{ + long deletedRows = 0; Result<K,T> result = query.execute(); while(result.next()) { @@ -118,27 +120,43 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { return 0; } } - + + /** + * An important feature of {@link MemStore#execute(Query)} is + * that when specifying the {@link MemQuery} one should be aware + * that when fromKey and toKey are equal, the returned map is empty + * unless fromInclusive and toInclusive are both true. On the other hand + * if either or both of fromKey and toKey are null we return no results. + */ @SuppressWarnings("unchecked") @Override public Result<K, T> execute(Query<K, T> query) { K startKey = query.getStartKey(); K endKey = query.getEndKey(); if(startKey == null) { - startKey = (K) map.firstKey(); + if (!map.isEmpty()) { + startKey = (K) map.firstKey(); + } } if(endKey == null) { - endKey = (K) map.lastKey(); + if (!map.isEmpty()) { + endKey = (K) map.lastKey(); + } } //check if query.fields is null query.setFields(getFieldsToQuery(query.getFields())); - - ConcurrentNavigableMap<K,T> submap = map.subMap(startKey, true, endKey, true); - + ConcurrentNavigableMap<K,T> submap = null; + try { + submap = map.subMap(startKey, true, endKey, true); + } catch (NullPointerException npe){ + LOG.info("Either startKey || endKey || startKey and endKey value(s) is null. " + + "No results will be returned for query to MemStore."); + return new MemResult<>(this, query, new ConcurrentSkipListMap<K, T>()); + } return new MemResult<>(this, query, submap); } - + @SuppressWarnings("unchecked") @Override public T get(K key, String[] fields) { @@ -173,7 +191,7 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { public Query<K, T> newQuery() { return new MemQuery<>(this); } - + @SuppressWarnings("unchecked") @Override public void put(K key, T obj) { @@ -196,12 +214,19 @@ public class MemStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { public void close() { } + /** + * As MemStore is basically an implementation of + * {@link java.util.concurrent.ConcurrentSkipListMap} + * it has no concept of a schema. + */ @Override public void createSchema() { } @Override public void deleteSchema() { - map.clear(); + if (!map.isEmpty()) { + map.clear(); + } } @Override http://git-wip-us.apache.org/repos/asf/gora/blob/11b1d7ac/gora-core/src/test/java/org/apache/gora/memory/store/MemStoreTest.java ---------------------------------------------------------------------- diff --git a/gora-core/src/test/java/org/apache/gora/memory/store/MemStoreTest.java b/gora-core/src/test/java/org/apache/gora/memory/store/MemStoreTest.java index 3007298..f2131df 100644 --- a/gora-core/src/test/java/org/apache/gora/memory/store/MemStoreTest.java +++ b/gora-core/src/test/java/org/apache/gora/memory/store/MemStoreTest.java @@ -1,14 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.gora.memory.store; +import java.io.IOException; + +import org.apache.gora.examples.WebPageDataCreator; +import org.apache.gora.examples.generated.Employee; import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.query.Query; import org.apache.gora.store.DataStore; +import org.apache.gora.store.DataStoreFactory; +import org.apache.gora.store.DataStoreTestBase; +import org.apache.gora.store.DataStoreTestUtil; +import org.apache.hadoop.conf.Configuration; +import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static org.apache.gora.examples.WebPageDataCreator.SORTED_URLS; +import static org.apache.gora.examples.WebPageDataCreator.URLS; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; -public class MemStoreTest { +/** + * Testing class for all standard gora-memory functionality. + * We extend {@link org.apache.gora.store.DataStoreTestBase} enabling us to run the entire base test + * suite for Gora. + */ +public class MemStoreTest extends DataStoreTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(MemStoreTest.class); + + private static final int NUM_KEYS = 4; + + private Configuration conf; + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @SuppressWarnings("unchecked") + @Override + protected DataStore<String, Employee> createEmployeeDataStore() throws IOException { + return DataStoreFactory.getDataStore(MemStore.class, String.class, Employee.class, conf); + } + + @SuppressWarnings("unchecked") + @Override + protected DataStore<String, WebPage> createWebPageDataStore() throws IOException { + return DataStoreFactory.getDataStore(MemStore.class, String.class, Employee.class, conf); + } @Test public void testGetMissingValue() { @@ -19,6 +81,18 @@ public class MemStoreTest { } @Test + @Ignore("MemStore has no concept of a schema") + public void testTruncateSchema() throws Exception {} + + @Test + @Ignore("MemStore has no concept of a schema") + public void testDeleteSchema() throws Exception {} + + @Test + @Ignore("MemStore has no concept of a schema") + public void testSchemaExists() throws Exception {} + + @Test public void testPutGet() throws Exception { String key = "org.apache.gora:http:/"; DataStore<String, WebPage> store = new MemStore<>(); @@ -28,4 +102,92 @@ public class MemStoreTest { store.close(); } + @Ignore("This test assumes that ") + @Test + public void testDeleteByQueryFields() {} + + @Test + public void testMemStoreDeleteByQueryFields() throws Exception { + + DataStore<String, WebPage> store = new MemStore<>(); + Query<String, WebPage> query; + + //test 5 - delete all with some fields + WebPageDataCreator.createWebPageData(store); + + query = store.newQuery(); + query.setFields("outlinks", "parsedContent", "content"); + + Query<String, WebPage> newQuery = store.newQuery(); + newQuery.setStartKey(URLS[0]); + newQuery.setEndKey(URLS[9]); + //newQuery.setFields("outlinks", "parsedContent", "content"); + + DataStoreTestUtil.assertNumResults(newQuery, URLS.length); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily + + store.flush(); + + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + + //assert that data is deleted + for (String SORTED_URL : SORTED_URLS) { + WebPage page = store.get(SORTED_URL); + assertNotNull(page); + + assertNotNull(page.getUrl()); + assertEquals(page.getUrl().toString(), SORTED_URL); + assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery " + + "not only removes the data but also the data structure.", 0, page.getOutlinks().size()); + assertEquals(0, page.getParsedContent().size()); + if (page.getContent() != null) { + LOG.info("url:" + page.getUrl().toString()); + LOG.info("limit:" + page.getContent().limit()); + } else { + assertNull(page.getContent()); + } + } + + //test 6 - delete some with some fields + WebPageDataCreator.createWebPageData(store); + + query = store.newQuery(); + query.setFields("url"); + String startKey = SORTED_URLS[NUM_KEYS]; + String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS]; + query.setStartKey(startKey); + query.setEndKey(endKey); + + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily + + store.flush(); + + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + + //assert that data is deleted + for (int i = 0; i < URLS.length; i++) { + WebPage page = store.get(URLS[i]); + assertNotNull(page); + if( URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) >= 0) { + //not deleted + DataStoreTestUtil.assertWebPage(page, i); + } else { + //deleted + assertNull(page.getUrl()); + assertNotNull(page.getOutlinks()); + assertNotNull(page.getParsedContent()); + assertNotNull(page.getContent()); + assertTrue(page.getOutlinks().size() > 0); + assertTrue(page.getParsedContent().size() > 0); + } + } + + } } + + http://git-wip-us.apache.org/repos/asf/gora/blob/11b1d7ac/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java index 23a3c94..41be132 100644 --- a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java +++ b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java @@ -239,19 +239,19 @@ public class TestHBaseStore extends DataStoreTestBase { @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive for HBase.") @Override public void testQueryEndKey() throws IOException { - //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclussive or exclusive. + //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclusive or exclusive. } @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive for HBase.") @Override public void testQueryKeyRange() throws IOException { - //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclussive or exclusive. + //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclusive or exclusive. } @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive for HBase.") @Override public void testDeleteByQuery() throws IOException { - //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclussive or exclusive. + //TODO: We should raise an issue for HBase to allow us to specify if the endRow will be inclusive or exclusive. } }
