Author: alfonsonishikawa Date: Mon Jan 6 21:53:13 2014 New Revision: 1556044
URL: http://svn.apache.org/r1556044 Log: GORA-117 gora hbase does not have a mechanism to set the caching on a scanner, which makes for poor performance on map/reduce jobs Added: gora/trunk/gora-hbase/src/test/conf/gora.properties Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1556044&r1=1556043&r2=1556044&view=diff ============================================================================== --- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original) +++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Mon Jan 6 21:53:13 2014 @@ -54,6 +54,7 @@ import org.apache.gora.persistency.impl. import org.apache.gora.query.PartitionQuery; import org.apache.gora.query.Query; import org.apache.gora.query.impl.PartitionQueryImpl; +import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.impl.DataStoreBase; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -88,6 +89,9 @@ implements Configurable { private static final String DEPRECATED_MAPPING_FILE = "hbase-mapping.xml"; public static final String DEFAULT_MAPPING_FILE = "gora-hbase-mapping.xml"; + private static final String SCANNER_CACHING_PROPERTIES_KEY = "scanner.caching" ; + private static final int SCANNER_CACHING_PROPERTIES_DEFAULT = 0 ; + private volatile HBaseAdmin admin; private volatile HBaseTableConnection table; @@ -96,6 +100,8 @@ implements Configurable { private volatile HBaseMapping mapping; + private int scannerCaching = SCANNER_CACHING_PROPERTIES_DEFAULT ; + public HBaseStore() { } @@ -127,6 +133,17 @@ implements Configurable { throw new RuntimeException(e); } + // Set scanner caching option + try { + this.setScannerCaching( + Integer.valueOf(DataStoreFactory.findProperty(this.properties, this, + SCANNER_CACHING_PROPERTIES_KEY, + String.valueOf(SCANNER_CACHING_PROPERTIES_DEFAULT)))) ; + }catch(Exception e){ + LOG.error("Can not load " + SCANNER_CACHING_PROPERTIES_KEY + " from gora.properties. Setting to default value: " + SCANNER_CACHING_PROPERTIES_DEFAULT, e) ; + this.setScannerCaching(SCANNER_CACHING_PROPERTIES_DEFAULT) ; // Default value if something is wrong + } + if(autoCreateSchema) { createSchema(); } @@ -465,6 +482,9 @@ implements Configurable { public ResultScanner createScanner(Query<K, T> query) throws IOException { final Scan scan = new Scan(); + + scan.setCaching(this.getScannerCaching()) ; + if (query.getStartKey() != null) { scan.setStartRow(toBytes(query.getStartKey())); } @@ -729,4 +749,29 @@ implements Configurable { this.conf = conf; } + /** + * Gets the Scanner Caching optimization value + * @return The value used internally in {@link Scan#setCaching(int)} + */ + public int getScannerCaching() { + return this.scannerCaching ; + } + + /** + * Sets the value for Scanner Caching optimization + * + * @see Scan#setCaching(int) + * + * @param numRows the number of rows for caching >= 0 + * @return <<Fluent interface>> + */ + public HBaseStore<K, T> setScannerCaching(int numRows) { + if (numRows < 0) { + LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: " + numRows + ".") ; + return this ; + } + this.scannerCaching = numRows ; + return this ; + } + } Added: gora/trunk/gora-hbase/src/test/conf/gora.properties URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/conf/gora.properties?rev=1556044&view=auto ============================================================================== --- gora/trunk/gora-hbase/src/test/conf/gora.properties (added) +++ gora/trunk/gora-hbase/src/test/conf/gora.properties Mon Jan 6 21:53:13 2014 @@ -0,0 +1,6 @@ + +#Datastore is configured programatically in the tests +#gora.datastore.default= + +#Caching option for HBaseStore +gora.hbasestore.scanner.caching=1000 \ No newline at end of file Modified: gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java?rev=1556044&r1=1556043&r2=1556044&view=diff ============================================================================== --- gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java (original) +++ gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java Mon Jan 6 21:53:13 2014 @@ -224,6 +224,11 @@ public class TestHBaseStore extends Data table.close(); } + @Test + public void assertScannerCachingValue() { + assertEquals(1000, ((HBaseStore<String,WebPage>)this.webPageStore).getScannerCaching()) ; + assertEquals(1000, ((HBaseStore<String,Employee>)this.employeeStore).getScannerCaching()) ; + } @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive for HBase.") @Override
