Updates to GORA-443 issue
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/f5503ef3 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/f5503ef3 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/f5503ef3 Branch: refs/heads/master Commit: f5503ef369f2193f6395e72cc1875a94c04b33a3 Parents: 163d91b Author: Lewis John McGibbney <lewis.mcgibb...@gmail.com> Authored: Wed Sep 14 18:28:50 2016 -0700 Committer: Lewis John McGibbney <lewis.mcgibb...@gmail.com> Committed: Wed Sep 14 18:28:50 2016 -0700 ---------------------------------------------------------------------- README.md | 7 +++ gora-couchdb/pom.xml | 8 --- .../org/apache/gora/hbase/store/HBaseStore.java | 34 +++++++++---- .../gora/hbase/store/HBaseTableConnection.java | 52 ++++++-------------- .../apache/gora/hbase/store/TestHBaseStore.java | 2 +- pom.xml | 4 +- 6 files changed, 49 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index e9021b3..4171456 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ #Apache Gora Project <img src="http://gora.apache.org/resources/img/powered-by-gora.png"; align="right" width="300" /> + +[](http://www.apache.org/licenses/LICENSE-2.0) +[](https://builds.apache.org/job/gora-trunk/) +[](https://builds.apache.org/job/gora-trunk) +[](http://search.maven.org/#search|ga|1|g%3A%22org.apache.gora%22) +[](https://analysis.apache.org/dashboard/index/76356) +[](https://twitter.com/apachegora) The Apache Gora open source framework provides an in-memory data model and persistence for big data. Gora supports persisting to column stores, http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-couchdb/pom.xml ---------------------------------------------------------------------- diff --git a/gora-couchdb/pom.xml b/gora-couchdb/pom.xml index ab050cc..6f991e0 100644 --- a/gora-couchdb/pom.xml +++ b/gora-couchdb/pom.xml @@ -139,14 +139,6 @@ <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-core</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <!-- END of Gora Internal Dependencies --> <dependency> http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java index 65c07d9..7d9f527 100644 --- a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java +++ b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java @@ -57,7 +57,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.jdom.Document; @@ -133,7 +140,7 @@ implements Configurable { SCANNER_CACHING_PROPERTIES_KEY, String.valueOf(SCANNER_CACHING_PROPERTIES_DEFAULT)))) ; }catch(Exception e){ - LOG.error("Can not load " + SCANNER_CACHING_PROPERTIES_KEY + " from gora.properties. Setting to default value: " + SCANNER_CACHING_PROPERTIES_DEFAULT, e) ; + LOG.error("Can not load {} from gora.properties. Setting to default value: {}.", SCANNER_CACHING_PROPERTIES_KEY, SCANNER_CACHING_PROPERTIES_DEFAULT); this.setScannerCaching(SCANNER_CACHING_PROPERTIES_DEFAULT) ; // Default value if something is wrong } @@ -146,6 +153,7 @@ implements Configurable { } catch(IOException ex2){ LOG.error(ex2.getMessage(), ex2); } + closeHBaseAdmin(); } @Override @@ -170,6 +178,7 @@ implements Configurable { } catch(IOException ex2){ LOG.error(ex2.getMessage(), ex2); } + closeHBaseAdmin(); } @Override @@ -183,6 +192,7 @@ implements Configurable { } catch(IOException ex2){ LOG.error(ex2.getMessage(), ex2); } + closeHBaseAdmin(); } @Override @@ -202,7 +212,7 @@ implements Configurable { Get get = new Get(toBytes(key)); addFields(get, fields); Result result = table.get(get); - return newInstance(result, fields); + return newInstance(result, fields); } catch(IOException ex2){ LOG.error(ex2.getMessage(), ex2); return null; @@ -572,10 +582,10 @@ implements Configurable { break; case MAP: case ARRAY: - delete.deleteFamily(col.family); + delete.addFamily(col.family); break; default: - delete.deleteColumn(col.family, col.qualifier); + delete.addColumn(col.family, col.qualifier); break; } } @@ -756,9 +766,7 @@ implements Configurable { //tableNameFromMapping could be null here if (!tableName.equals(tableNameFromMapping)) { //TODO this might not be the desired behavior as the user might have actually made a mistake. - LOG.warn("Mismatching schema's names. Mappingfile schema: '" + tableNameFromMapping - + "'. PersistentClass schema's name: '" + tableName + "'" - + "Assuming they are the same."); + LOG.warn("Mismatching schema's names. Mappingfile schema: '{}'. PersistentClass schema's name: '{}'. Assuming they are the same.", tableNameFromMapping, tableName); if (tableNameFromMapping != null) { mappingBuilder.renameTable(tableNameFromMapping, tableName); } @@ -836,10 +844,18 @@ implements Configurable { */ public HBaseStore<K, T> setScannerCaching(int numRows) { if (numRows < 0) { - LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: " + numRows + ".") ; + LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: {}.", numRows) ; return this ; } this.scannerCaching = numRows ; return this ; } + + private void closeHBaseAdmin(){ + try { + admin.close(); + } catch (IOException ioe) { + LOG.error("An error occured whilst closing HBase Admin", ioe); + } + } } http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java index 000a8b5..9f5b7c4 100644 --- a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java +++ b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java @@ -18,19 +18,26 @@ package org.apache.gora.hbase.store; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.client.BufferedMutator; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Pair; /** @@ -44,7 +51,6 @@ public class HBaseTableConnection { * the connection when it is closed. HBase itself provides a utility called * HTablePool for maintaining a tPool of tables, but there are still some * drawbacks that are only solved in later releases. - * */ private final Configuration conf; @@ -56,26 +62,10 @@ public class HBaseTableConnection { private final BlockingQueue<Table> tPool = new LinkedBlockingQueue<>(); private final BlockingQueue<ConcurrentLinkedQueue<Mutation>> bPool = new LinkedBlockingQueue<>(); + @SuppressWarnings("unused") private final boolean autoFlush; private final TableName tableName; -// public class MutationPair { -// private Mutation mutation; -// private boolean type; -// -// public void MutationPair(Mutation m, boolean t) { -// this.mutation = m; -// this.type = t; -// } -// -// public boolean isType() { -// return type; -// } -// -// public Mutation getMutation() { -// return mutation; -// } -// } /** * Instantiate new connection. * @@ -101,7 +91,6 @@ public class HBaseTableConnection { Table table = tables.get(); if (table == null) { table = connection.getTable(tableName); -// table.setAutoFlushTo(autoFlush); tPool.add(table); //keep track tables.set(table); } @@ -111,8 +100,6 @@ public class HBaseTableConnection { private ConcurrentLinkedQueue<Mutation> getBuffer() throws IOException { ConcurrentLinkedQueue<Mutation> buffer = buffers.get(); if (buffer == null) { -// BufferedMutatorParams params = new BufferedMutatorParams(this.tableName).listener(listener); -// buffer = connection.getBufferedMutator(this.tableName); buffer = new ConcurrentLinkedQueue<>(); bPool.add(buffer); buffers.set(buffer); @@ -125,9 +112,9 @@ public class HBaseTableConnection { for (ConcurrentLinkedQueue<Mutation> buffer : bPool) { for (Mutation m: buffer) { bufMutator.mutate(m); - bufMutator.flush(); } } + bufMutator.flush(); bufMutator.close(); } @@ -184,29 +171,18 @@ public class HBaseTableConnection { public void put(Put put) throws IOException { getBuffer().add(put); -// getBuffer().flush(); -// getTable().put(put); } -// @Override public void put(List<Put> puts) throws IOException { -// getTable().put(puts); getBuffer().addAll(puts); -// getBuffer().flush(); } -// @Override public void delete(Delete delete) throws IOException { getBuffer().add(delete); -// getBuffer().flush(); -// getTable().delete(delete); } -// @Override public void delete(List<Delete> deletes) throws IOException { -// getTable().delete(deletes); getBuffer().addAll(deletes); -// getBuffer().flush(); } public TableName getName() { http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java ---------------------------------------------------------------------- diff --git a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java index 87101ed..695dea1 100644 --- a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java +++ b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java @@ -82,7 +82,7 @@ public class TestHBaseStore extends DataStoreTestBase { @Override public void assertSchemaExists(String schemaName) throws Exception { HBaseAdmin admin = getTestDriver().getHbaseUtil().getHBaseAdmin(); - assertTrue(admin.tableExists(schemaName)); + assertTrue("Table should exist for...", admin.tableExists(schemaName)); } @Override http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 062c7d9..679afa0 100644 --- a/pom.xml +++ b/pom.xml @@ -685,8 +685,8 @@ <!-- Hadoop Dependencies --> <hadoop-2.version>2.5.2</hadoop-2.version> <hadoop-2.test.version>2.5.2</hadoop-2.test.version> - <hbase.version>1.2.2</hbase.version> - <hbase.test.version>1.2.2</hbase.test.version> + <hbase.version>1.2.3</hbase.version> + <hbase.test.version>1.2.3</hbase.test.version> <cxf-rt-frontend-jaxrs.version>2.5.2</cxf-rt-frontend-jaxrs.version> <!-- Amazon Dependencies --> <amazon.version>1.10.55</amazon.version>
![http://gora.apache.org/resources/img/powered-by-gora.png"](http://gora.apache.org/resources/img/powered-by-gora.png"){kind=link}
