Author: hsaputra
Date: Thu Apr 11 04:43:48 2013
New Revision: 1466766
URL: http://svn.apache.org/r1466766
Log:
GORA-89 GORA-159 To shorten build time by having HBase minicluster as singleton.
Added:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java
(with props)
Modified:
gora/trunk/CHANGES.txt
gora/trunk/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/QueryCounter.java
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/GoraHBaseTestDriver.java
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreCountQuery.java
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreWordCount.java
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
Modified: gora/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/gora/trunk/CHANGES.txt?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
--- gora/trunk/CHANGES.txt (original)
+++ gora/trunk/CHANGES.txt Thu Apr 11 04:43:48 2013
@@ -5,6 +5,9 @@
Gora Change Log
trunk (current development)
+* GORA-159 gora-hbase MR tests should use HBaseTestingUtility instead of
deprecated HBaseClusterTestCase via GORA-89
+
+* GORA-89 Avoid HBase MiniCluster restarts to shorten gora-hbase tests
(hsaputra, alfonso, Ioan eugen Stan)
* GORA-203 Bug in setting column field attribute "qualifier" in
CassandraMapping (rmarroquin, lewismc, kazk)
Modified:
gora/trunk/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/QueryCounter.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/QueryCounter.java?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
---
gora/trunk/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/QueryCounter.java
(original)
+++
gora/trunk/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/QueryCounter.java
Thu Apr 11 04:43:48 2013
@@ -47,8 +47,7 @@ public class QueryCounter<K, T extends P
}
public static class QueryCounterMapper<K, T extends Persistent>
- extends GoraMapper<K, T
- , NullWritable, NullWritable> {
+ extends GoraMapper<K, T, NullWritable, NullWritable> {
@Override
protected void map(K key, T value,
@@ -95,6 +94,7 @@ public class QueryCounter<K, T extends P
public long countQuery(DataStore<K,T> dataStore, Query<K,T> query) throws
Exception {
Job job = createJob(dataStore, query);
job.waitForCompletion(true);
+ assert(job.isComplete() == true);
return job.getCounters().findCounter(COUNTER_GROUP, ROWS).getValue();
}
@@ -108,6 +108,7 @@ public class QueryCounter<K, T extends P
Job job = createJob(dataStore, query);
job.waitForCompletion(true);
+ assert(job.isComplete() == true);
return job.getCounters().findCounter(COUNTER_GROUP, ROWS).getValue();
}
Modified:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/GoraHBaseTestDriver.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/GoraHBaseTestDriver.java?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
---
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/GoraHBaseTestDriver.java
(original)
+++
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/GoraHBaseTestDriver.java
Thu Apr 11 04:43:48 2013
@@ -20,6 +20,7 @@ package org.apache.gora.hbase;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.hbase.store.HBaseStore;
+import org.apache.gora.hbase.util.HBaseClusterSingleton;
import org.apache.hadoop.conf.Configuration;
//HBase imports
@@ -32,56 +33,45 @@ import org.apache.hadoop.hbase.client.HB
* @see GoraTestDriver
*/
public class GoraHBaseTestDriver extends GoraTestDriver {
+ private static final HBaseClusterSingleton cluster =
HBaseClusterSingleton.build(1);
- protected HBaseTestingUtility hbaseUtil;
- protected int numServers = 1;
-
public GoraHBaseTestDriver() {
super(HBaseStore.class);
- hbaseUtil = new HBaseTestingUtility();
- }
-
- public void setNumServers(int numServers) {
- this.numServers = numServers;
- }
-
- public int getNumServers() {
- return numServers;
}
@Override
public void setUpClass() throws Exception {
super.setUpClass();
- log.info("Starting HBase cluster");
- hbaseUtil.startMiniCluster(numServers);
+ log.info("Setting up HBase Test Driver");
}
@Override
public void tearDownClass() throws Exception {
super.tearDownClass();
- log.info("Stoping HBase cluster");
- hbaseUtil.shutdownMiniCluster();
+ log.info("Teardown HBase test driver");
}
@Override
public void setUp() throws Exception {
- super.setUp();
+ cluster.truncateAllTables();
+ // super.setUp() deletes all tables, but must only truncate in the right
way -HBaseClusterSingleton-
+ //super.setUp();
}
+ @Override
+ public void tearDown() throws Exception {
+ // Do nothing. setUp() must ensure the right data.
+ }
public void deleteAllTables() throws Exception {
- HBaseAdmin admin = hbaseUtil.getHBaseAdmin();
- for(HTableDescriptor table:admin.listTables()) {
- admin.disableTable(table.getName());
- admin.deleteTable(table.getName());
- }
+ cluster.deleteAllTables();
}
public Configuration getConf() {
- return hbaseUtil.getConfiguration();
+ return cluster.getHbaseTestingUtil().getConfiguration();
}
public HBaseTestingUtility getHbaseUtil() {
- return hbaseUtil;
+ return cluster.getHbaseTestingUtil();
}
}
Modified:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreCountQuery.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreCountQuery.java?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
---
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreCountQuery.java
(original)
+++
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreCountQuery.java
Thu Apr 11 04:43:48 2013
@@ -21,9 +21,10 @@ package org.apache.gora.hbase.mapreduce;
import org.apache.gora.examples.generated.TokenDatum;
import org.apache.gora.examples.generated.WebPage;
import org.apache.gora.hbase.store.HBaseStore;
+import org.apache.gora.hbase.util.HBaseClusterSingleton;
import org.apache.gora.mapreduce.MapReduceTestUtils;
import org.apache.gora.store.DataStoreFactory;
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -31,28 +32,26 @@ import org.junit.Test;
/**
* Tests related to {@link HBaseStore} using mapreduce.
*/
-public class TestHBaseStoreCountQuery extends HBaseClusterTestCase{
+public class TestHBaseStoreCountQuery {
+ private static final HBaseClusterSingleton cluster =
HBaseClusterSingleton.build(1);
private HBaseStore<String, WebPage> webPageStore;
-
+
@Before
- @Override
public void setUp() throws Exception {
- super.setUp();
+ cluster.deleteAllTables();
webPageStore = DataStoreFactory.getDataStore(
- HBaseStore.class, String.class, WebPage.class, conf);
+ HBaseStore.class, String.class, WebPage.class, cluster.getConf());
}
@After
- @Override
public void tearDown() throws Exception {
webPageStore.close();
- super.tearDown();
}
@Test
public void testCountQuery() throws Exception {
- MapReduceTestUtils.testCountQuery(webPageStore, conf);
+ MapReduceTestUtils.testCountQuery(webPageStore, cluster.getConf());
}
public static void main(String[] args) throws Exception {
Modified:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreWordCount.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreWordCount.java?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
---
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreWordCount.java
(original)
+++
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/mapreduce/TestHBaseStoreWordCount.java
Thu Apr 11 04:43:48 2013
@@ -21,9 +21,10 @@ package org.apache.gora.hbase.mapreduce;
import org.apache.gora.examples.generated.TokenDatum;
import org.apache.gora.examples.generated.WebPage;
import org.apache.gora.hbase.store.HBaseStore;
+import org.apache.gora.hbase.util.HBaseClusterSingleton;
import org.apache.gora.mapreduce.MapReduceTestUtils;
import org.apache.gora.store.DataStoreFactory;
-import org.apache.hadoop.hbase.HBaseClusterTestCase;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -31,32 +32,30 @@ import org.junit.Test;
/**
* Tests related to {@link org.apache.gora.hbase.store.HBaseStore} using
mapreduce.
*/
-public class TestHBaseStoreWordCount extends HBaseClusterTestCase{
+public class TestHBaseStoreWordCount {
+ private static final HBaseClusterSingleton cluster =
HBaseClusterSingleton.build(1);
private HBaseStore<String, WebPage> webPageStore;
private HBaseStore<String, TokenDatum> tokenStore;
@Before
- @Override
public void setUp() throws Exception {
- super.setUp();
+ cluster.deleteAllTables();
webPageStore = DataStoreFactory.getDataStore(
- HBaseStore.class, String.class, WebPage.class, conf);
+ HBaseStore.class, String.class, WebPage.class, cluster.getConf());
tokenStore = DataStoreFactory.getDataStore(HBaseStore.class,
- String.class, TokenDatum.class, conf);
+ String.class, TokenDatum.class, cluster.getConf());
}
@After
- @Override
public void tearDown() throws Exception {
webPageStore.close();
tokenStore.close();
- super.tearDown();
}
@Test
public void testWordCount() throws Exception {
- MapReduceTestUtils.testWordCount(conf, webPageStore, tokenStore);
+ MapReduceTestUtils.testWordCount(cluster.getConf(), webPageStore,
tokenStore);
}
public static void main(String[] args) throws Exception {
Modified:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java?rev=1466766&r1=1466765&r2=1466766&view=diff
==============================================================================
---
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
(original)
+++
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
Thu Apr 11 04:43:48 2013
@@ -26,7 +26,6 @@ import junit.framework.Assert;
import org.apache.gora.examples.generated.Employee;
import org.apache.gora.examples.generated.WebPage;
import org.apache.gora.hbase.GoraHBaseTestDriver;
-import org.apache.gora.hbase.store.HBaseStore;
import org.apache.gora.store.DataStore;
import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.DataStoreTestBase;
@@ -122,31 +121,31 @@ public class TestHBaseStore extends Data
}
- @Override
- public void testQueryEndKey() throws IOException {
- //We need to skip this test since gora considers endRow inclusive,
while its exclusinve for HBase.
- //TODO: We should raise an issue for HBase to allow us to specify if
the endRow will be inclussive or exclusive.
- }
-
- @Override
- public void testQueryKeyRange() throws IOException {
- //We need to skip this test since gora considers endRow inclusive,
while its exclusinve for HBase.
- //TODO: We should raise an issue for HBase to allow us to specify if
the endRow will be inclussive or exclusive.
- }
-
- @Override
- public void testDeleteByQuery() throws IOException {
- //We need to skip this test since gora considers endRow inclusive,
while its exclusinve for HBase.
- //TODO: We should raise an issue for HBase to allow us to specify if
the endRow will be inclussive or exclusive.
- }
+ @Override
+ public void testQueryEndKey() throws IOException {
+ //We need to skip this test since gora considers endRow inclusive, while
its exclusinve for HBase.
+ //TODO: We should raise an issue for HBase to allow us to specify if the
endRow will be inclussive or exclusive.
+ }
+
+ @Override
+ public void testQueryKeyRange() throws IOException {
+ //We need to skip this test since gora considers endRow inclusive, while
its exclusinve for HBase.
+ //TODO: We should raise an issue for HBase to allow us to specify if the
endRow will be inclussive or exclusive.
+ }
+
+ @Override
+ public void testDeleteByQuery() throws IOException {
+ //We need to skip this test since gora considers endRow inclusive, while
its exclusinve for HBase.
+ //TODO: We should raise an issue for HBase to allow us to specify if the
endRow will be inclussive or exclusive.
+ }
- public static void main(String[] args) throws Exception {
+ public static void main(String[] args) throws Exception {
TestHBaseStore test = new TestHBaseStore();
test.setUpClass();
test.setUp();
-
+
test.testQuery();
-
+
test.tearDown();
test.tearDownClass();
}
Added:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java
URL:
http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java?rev=1466766&view=auto
==============================================================================
---
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java
(added)
+++
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java
Thu Apr 11 04:43:48 2013
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.hbase.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that will creates a single instance of HBase MiniCluster.
+ * Copied from HBase support for Apache James:
+ *
http://svn.apache.org/repos/asf/james/mailbox/trunk/hbase/src/test/java/org/apache/james/mailbox/hbase/
+ */
+public final class HBaseClusterSingleton {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HBaseClusterSingleton.class);
+ private static final HBaseTestingUtility htu = new HBaseTestingUtility();
+ private static HBaseClusterSingleton cluster = null;
+ private MiniHBaseCluster hbaseCluster;
+ private Configuration conf;
+
+ /**
+ * Builds a MiniCluster instance.
+ * @return the {@link HBaseClusterSingleton} instance
+ * @throws RuntimeException
+ */
+ public static synchronized HBaseClusterSingleton build(int numServers)
throws RuntimeException {
+ LOG.info("Retrieving cluster instance.");
+ if (cluster == null) {
+ cluster = new HBaseClusterSingleton(numServers);
+ }
+ return cluster;
+ }
+
+ private HBaseClusterSingleton(int numServers) throws RuntimeException {
+
+ // Workaround for HBASE-5711, we need to set config value
dfs.datanode.data.dir.perm
+ // equal to the permissions of the temp dirs on the filesystem. These temp
dirs were
+ // probably created using this process' umask. So we guess the temp dir
permissions as
+ // 0777 & ~umask, and use that to set the config value.
+ try {
+ Process process = Runtime.getRuntime().exec("/bin/sh -c umask");
+ BufferedReader br = new BufferedReader(new
InputStreamReader(process.getInputStream()));
+ int rc = process.waitFor();
+ if(rc == 0) {
+ String umask = br.readLine();
+
+ int umaskBits = Integer.parseInt(umask, 8);
+ int permBits = 0777 & ~umaskBits;
+ String perms = Integer.toString(permBits, 8);
+
+ LOG.info("Setting dfs.datanode.data.dir.perm to " + perms);
+ htu.getConfiguration().set("dfs.datanode.data.dir.perm", perms);
+ } else {
+ LOG.warn("Failed running umask command in a shell, nonzero return
value");
+ }
+ } catch (Exception e) {
+ // ignore errors, we might not be running on POSIX, or "sh" might not be
on the path
+ LOG.warn("Couldn't get umask", e);
+ }
+
+ htu.getConfiguration().setBoolean("dfs.support.append", true);
+ htu.getConfiguration().setInt("zookeeper.session.timeout", 20000);
+ try {
+ LOG.info("Start HBase mini cluster.");
+ hbaseCluster = htu.startMiniCluster(numServers);
+ LOG.info("After cluster start-up.");
+ hbaseCluster.waitForActiveAndReadyMaster();
+ LOG.info("After active and ready.");
+ conf = hbaseCluster.getConfiguration();
+ LOG.info("Start mini mapreduce cluster.");
+ htu.startMiniMapReduceCluster();
+ LOG.info("After mini mapreduce cluster start-up.");
+ } catch (Exception ex) {
+ throw new RuntimeException("Minicluster not starting.");
+ } finally {
+ // add a shutdown hook for shuting down the minicluster.
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (hbaseCluster != null) {
+ hbaseCluster.shutdown();
+ }
+ htu.shutdownMiniMapReduceCluster();
+ } catch (IOException e) {
+ throw new RuntimeException("Exception shutting down cluster.");
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * Return a configuration for the runnning MiniCluster.
+ * @return
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Creates a table with the specified column families.
+ * @param tableName the table name
+ * @param columnFamilies the colum families
+ * @throws IOException
+ */
+ public void ensureTable(String tableName, String... columnFamilies) throws
IOException {
+ byte[][] cfs = new byte[columnFamilies.length][];
+ for (int i = 0; i < columnFamilies.length; i++) {
+ cfs[i] = Bytes.toBytes(columnFamilies[i]);
+ }
+ ensureTable(Bytes.toBytes(tableName), cfs);
+ }
+
+ /**
+ * Creates a table with the specified column families.
+ * @param tableName the table name
+ * @param cfs the column families
+ * @throws IOException
+ */
+ public void ensureTable(byte[] tableName, byte[][] cfs) throws IOException {
+ HBaseAdmin admin = htu.getHBaseAdmin();
+ if (!admin.tableExists(tableName)) {
+ htu.createTable(tableName, cfs);
+ }
+ }
+
+ /**
+ * Truncates all tables
+ * @throws Exception
+ */
+ public void truncateAllTables() throws Exception {
+ HBaseAdmin admin = htu.getHBaseAdmin();
+ for(HTableDescriptor table:admin.listTables()) {
+ htu.truncateTable(table.getName());
+ }
+ }
+
+
+ /**
+ * Delete all tables
+ * @throws Exception
+ */
+ public void deleteAllTables() throws Exception {
+ HBaseAdmin admin = htu.getHBaseAdmin();
+ for(HTableDescriptor table:admin.listTables()) {
+ admin.disableTable(table.getName());
+ admin.deleteTable(table.getName());
+ }
+ }
+
+ public void shutdownMiniCluster() throws IOException {
+ hbaseCluster.shutdown();
+ }
+
+ /**
+ * Returns the HBaseTestingUtility instance
+ * @return static instance of HBaseTestingUtility
+ */
+ public HBaseTestingUtility getHbaseTestingUtil() {
+ return htu;
+ }
+}
Propchange:
gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/util/HBaseClusterSingleton.java
------------------------------------------------------------------------------
svn:eol-style = native
