Author: liyin Date: Fri Mar 14 18:09:49 2014 New Revision: 1577640 URL: http://svn.apache.org/r1577640 Log: [HBASE-9930] Fixing conf dependencies of ClientSideScan and fixing TestClientLocalScanner unit tests.
Author: manukranthk Summary: This diff achieves to do the following : * Fix TestClientLocalScanner * Remove conf dependencies from ClientLocalScanner Test Plan: Run TestClientLocalScanner Reviewers: gauravm, fan Reviewed By: gauravm CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D1217856 Task ID: 3667016, 3274463 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1577640&r1=1577639&r2=1577640&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Fri Mar 14 18:09:49 2014 @@ -1058,6 +1058,9 @@ public final class HConstants { public static final String THRIFT_HEADER_FROM_CLIENT = "clientHeader"; public static final boolean DISABLE_THRIFT_REGION_INFO_QUALIFIER = false; + public static final String USE_CONF_FROM_SERVER = + "hbase.client.use.conf.from.server"; + public static final boolean DEFAULT_USE_CONF_FROM_SERVER = true; private HConstants() { // Can't be instantiated with this constructor. Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java?rev=1577640&r1=1577639&r2=1577640&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java Fri Mar 14 18:09:49 2014 @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HConstant import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ipc.thrift.HBaseToThriftAdapter; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionUtilities; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -123,6 +124,15 @@ public class ClientLocalScanner extends flushRegionAndWaitForFlush(this.currentRegion, table.getRegionsInfo().get(info)); } + + if (conf.getBoolean(HConstants.USE_CONF_FROM_SERVER, + HConstants.DEFAULT_USE_CONF_FROM_SERVER)) { + conf.set("fs.default.name", + this.htable.getServerConfProperty("fs.default.name")); + if (this.areHardlinksCreated) { + conf.set("hbase.rootdir", htable.getServerConfProperty("hbase.rootdir")); + } + } Path rootDir = FSUtils.getRootDir(conf); final Path tableDir = HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName()); @@ -210,7 +220,8 @@ public class ClientLocalScanner extends public Result[] nextOnRS(int nbRows) throws IOException { try { RegionScanner s = (RegionScanner)this.currentScanner; - return s.nextRows(nbRows, HRegion.METRIC_NEXTSIZE); + Result[] ret = s.nextRows(nbRows, HRegion.METRIC_NEXTSIZE); + return HBaseToThriftAdapter.validateResults(ret); } catch (IOException e) { close(); throw e; Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java?rev=1577640&r1=1577639&r2=1577640&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java Fri Mar 14 18:09:49 2014 @@ -839,7 +839,7 @@ public class HBaseToThriftAdapter implem * @param res * @return */ - private Result[] validateResults(Result[] res) { + public static Result[] validateResults(Result[] res) { if (res == null) return null; if (res.length == 1) { return res[0].isSentinelResult() ? null : res; Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java?rev=1577640&r1=1577639&r2=1577640&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java Fri Mar 14 18:09:49 2014 @@ -87,7 +87,7 @@ public class TestClientLocalScanner { TEST_UTIL.loadTable(t, FAMILY2); t.flushCommits(); TEST_UTIL.flush(name); - TEST_UTIL.assertRowCount(t, rowCount); + HBaseTestingUtility.assertRowCount(t, rowCount); Scan scan = getScan(100, 100, true, FAMILY); assertTrue(compareScanners(tmpTable.getScanner(scan), t.getLocalScanner(scan))); @@ -179,10 +179,10 @@ public class TestClientLocalScanner { t.flushCommits(); TEST_UTIL.flush(name); - TEST_UTIL.assertRowCount(t, rowCount); + HBaseTestingUtility.assertRowCount(t, rowCount); // Split the table. Should split on a reasonable key; 'lqj' Map<HRegionInfo, HServerAddress> regions = splitTable(t); - TEST_UTIL.assertRowCount(t, rowCount); + HBaseTestingUtility.assertRowCount(t, rowCount); // Get end key of first region. byte [] endKey = regions.keySet().iterator().next().getEndKey(); // Count rows with a filter that stops us before passed 'endKey'. @@ -199,6 +199,11 @@ public class TestClientLocalScanner { // New test. Make it so scan goes into next region by one and then two. // Make sure count comes out right. byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)}; + String defaultName = t.getConfiguration().get("fs.default.name"); + String rootdir = t.getConfiguration().get("hbase.rootdir"); + + t.getConfiguration().set("fs.default.name", ""); + t.getConfiguration().set("hbase.rootdir", ""); int plusOneCount = countRows(t, t.getLocalScanner(createScanWithRowFilter(key))); assertEquals(endKeyCount + 1, plusOneCount); key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)}; @@ -224,6 +229,10 @@ public class TestClientLocalScanner { countGreater = countRows(t, t.getLocalScanner(createScanWithRowFilter(endKey, endKey, CompareFilter.CompareOp.GREATER_OR_EQUAL))); assertEquals(rowCount - endKeyCount, countGreater); + + t.getConfiguration().set("fs.default.name", defaultName); + t.getConfiguration().set("hbase.rootdir", rootdir); + t.close(); } @Test @@ -237,14 +246,18 @@ public class TestClientLocalScanner { assertTrue(fs.listStatus(p).length == 0); TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath); HTable t = TEST_UTIL.createTable(tableName, FAMILY); + t.getConfiguration().setBoolean(HConstants.USE_CONF_FROM_SERVER, false); TEST_UTIL.loadTable(t, FAMILY); try { t.getLocalScanner(new Scan()); - } catch (IOException e) { + } catch (Exception e) { + LOG.debug("Exception", e); assertTrue(fs.listStatus(p).length == 0); return; } finally { TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir); + t.getConfiguration().setBoolean(HConstants.USE_CONF_FROM_SERVER, false); + t.close(); } assertTrue(false); } @@ -257,7 +270,7 @@ public class TestClientLocalScanner { int rowCount = TEST_UTIL.loadTable(t, FAMILY); t.flushCommits(); TEST_UTIL.flush(tableName); - TEST_UTIL.assertRowCount(t, rowCount); + HBaseTestingUtility.assertRowCount(t, rowCount); Scan scan = getScan(100, 100, true, FAMILY); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); @@ -289,6 +302,8 @@ public class TestClientLocalScanner { assertTrue(fs.exists(homedir)); assertTrue(fs.listStatus(homedir).length == files); + t.close(); + fs.close(); } public Scan getScan(int caching, int batching,
