HBASE-14489 postScannerFilterRow consumes a lot of CPU.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8ea34aea Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8ea34aea Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8ea34aea Branch: refs/heads/branch-1.0 Commit: 8ea34aea023952df21f6ede66b63536960400ec1 Parents: 0139873 Author: Lars Hofhansl <[email protected]> Authored: Sat Sep 26 16:25:17 2015 -0700 Committer: Lars Hofhansl <[email protected]> Committed: Sat Sep 26 16:31:05 2015 -0700 ---------------------------------------------------------------------- .../regionserver/RegionCoprocessorHost.java | 35 ++++++++++++++++++++ .../hbase/coprocessor/SimpleRegionObserver.java | 12 +++++++ .../TestRegionObserverInterface.java | 24 ++++++++++++++ 3 files changed, 71 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8ea34aea/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index afb3ec1..e20df42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.EndpointObserver; @@ -100,6 +101,9 @@ public class RegionCoprocessorHost private static ReferenceMap sharedDataMap = new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK); + // optimization: no need to call postScannerFilterRow, if no coprocessor implements it + private final boolean hasCustomPostScannerFilterRow; + /** * * Encapsulation of the environment of each coprocessor @@ -236,6 +240,35 @@ public class RegionCoprocessorHost // load Coprocessor From HDFS loadTableCoprocessors(conf); + + // now check whether any coprocessor implements postScannerFilterRow + boolean hasCustomPostScannerFilterRow = false; + out: for (RegionEnvironment env: coprocessors) { + if (env.getInstance() instanceof RegionObserver) { + Class<?> clazz = env.getInstance().getClass(); + for(;;) { + if (clazz == null) { + // we must have directly implemented RegionObserver + hasCustomPostScannerFilterRow = true; + break out; + } + if (clazz == BaseRegionObserver.class) { + // we reached BaseRegionObserver, try next coprocessor + break; + } + try { + clazz.getDeclaredMethod("postScannerFilterRow", ObserverContext.class, + InternalScanner.class, byte[].class, int.class, short.class, boolean.class); + // this coprocessor has a custom version of postScannerFilterRow + hasCustomPostScannerFilterRow = true; + break out; + } catch (NoSuchMethodException ignore) { + } + clazz = clazz.getSuperclass(); + } + } + } + this.hasCustomPostScannerFilterRow = hasCustomPostScannerFilterRow; } static List<TableCoprocessorAttribute> getTableCoprocessorAttrsFromSchema(Configuration conf, @@ -1371,6 +1404,8 @@ public class RegionCoprocessorHost */ public boolean postScannerFilterRow(final InternalScanner s, final byte[] currentRow, final int offset, final short length) throws IOException { + // short circuit for performance + if (!hasCustomPostScannerFilterRow) return true; return execOperationWithResult(true, coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() { @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/8ea34aea/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index 7100ae7..ff1dd50 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -118,6 +118,7 @@ public class SimpleRegionObserver extends BaseRegionObserver { final AtomicInteger ctPostCheckAndDelete = new AtomicInteger(0); final AtomicInteger ctPreScannerNext = new AtomicInteger(0); final AtomicInteger ctPostScannerNext = new AtomicInteger(0); + final AtomicInteger ctPostScannerFilterRow = new AtomicInteger(0); final AtomicInteger ctPreScannerClose = new AtomicInteger(0); final AtomicInteger ctPostScannerClose = new AtomicInteger(0); final AtomicInteger ctPreScannerOpen = new AtomicInteger(0); @@ -316,6 +317,14 @@ public class SimpleRegionObserver extends BaseRegionObserver { } @Override + public boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> e, + final InternalScanner s, final byte[] currentRow, final int offset, final short length, + final boolean hasMore) throws IOException { + ctPostScannerFilterRow.incrementAndGet(); + return hasMore; + } + + @Override public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c, final InternalScanner s) throws IOException { ctPreScannerClose.incrementAndGet(); @@ -820,6 +829,9 @@ public class SimpleRegionObserver extends BaseRegionObserver { public boolean wasScannerNextCalled() { return ctPreScannerNext.get() > 0 && ctPostScannerNext.get() > 0; } + public boolean wasScannerFilterRowCalled() { + return ctPostScannerFilterRow.get() > 0; + } public boolean wasScannerCloseCalled() { return ctPreScannerClose.get() > 0 && ctPostScannerClose.get() > 0; } http://git-wip-us.apache.org/repos/asf/hbase/blob/8ea34aea/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java index 031e173..89b5049 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.FilterAllFilter; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -376,6 +377,29 @@ public class TestRegionObserverInterface { table.close(); } + @Test(timeout = 300000) + public void testHBASE14489() throws IOException { + TableName tableName = TableName.valueOf("testHBASE14489"); + HTable table = util.createTable(tableName, new byte[][] { A }); + Put put = new Put(ROW); + put.addColumn(A, A, A); + table.put(put); + + Scan s = new Scan(); + s.setFilter(new FilterAllFilter()); + ResultScanner scanner = table.getScanner(s); + try { + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + } + } finally { + scanner.close(); + } + verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" }, + tableName, new Boolean[] { true }); + util.deleteTable(tableName); + table.close(); + } + @Test (timeout=300000) // HBase-3758 public void testHBase3758() throws IOException {
