Repository: hbase Updated Branches: refs/heads/master 02699fe96 -> 526520de0
HBASE-14489 postScannerFilterRow consumes a lot of CPU. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/526520de Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/526520de Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/526520de Branch: refs/heads/master Commit: 526520de0a9d7a29fcf1b4c521f017ca75a46cbc Parents: 02699fe Author: Lars Hofhansl <[email protected]> Authored: Sat Sep 26 16:17:18 2015 -0700 Committer: Lars Hofhansl <[email protected]> Committed: Sat Sep 26 16:17:18 2015 -0700 ---------------------------------------------------------------------- .../regionserver/RegionCoprocessorHost.java | 44 ++++++++++++++++++++ .../hbase/coprocessor/SimpleRegionObserver.java | 11 +++++ .../TestRegionObserverInterface.java | 25 +++++++++++ 3 files changed, 80 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/526520de/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 648b762..0c3d470 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.EndpointObserver; @@ -96,6 +97,9 @@ public class RegionCoprocessorHost private static ReferenceMap sharedDataMap = new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK); + // optimization: no need to call postScannerFilterRow, if no coprocessor implements it + private final boolean hasCustomPostScannerFilterRow; + /** * * Encapsulation of the environment of each coprocessor @@ -219,6 +223,44 @@ public class RegionCoprocessorHost // load Coprocessor From HDFS loadTableCoprocessors(conf); + + // now check whether any coprocessor implements postScannerFilterRow + boolean hasCustomPostScannerFilterRow = false; + out: for (RegionEnvironment env: coprocessors) { + if (env.getInstance() instanceof RegionObserver) { + Class<?> clazz = env.getInstance().getClass(); + for(;;) { + if (clazz == null) { + // we must have directly implemented RegionObserver + hasCustomPostScannerFilterRow = true; + break out; + } + if (clazz == BaseRegionObserver.class) { + // we reached BaseRegionObserver, try next coprocessor + break; + } + try { + clazz.getDeclaredMethod("postScannerFilterRow", ObserverContext.class, + InternalScanner.class, Cell.class, boolean.class); + // this coprocessor has a custom version of postScannerFilterRow + hasCustomPostScannerFilterRow = true; + break out; + } catch (NoSuchMethodException ignore) { + } + // the deprecated signature still exists + try { + clazz.getDeclaredMethod("postScannerFilterRow", ObserverContext.class, + InternalScanner.class, byte[].class, int.class, short.class, boolean.class); + // this coprocessor has a custom version of postScannerFilterRow + hasCustomPostScannerFilterRow = true; + break out; + } catch (NoSuchMethodException ignore) { + } + clazz = clazz.getSuperclass(); + } + } + } + this.hasCustomPostScannerFilterRow = hasCustomPostScannerFilterRow; } static List<TableCoprocessorAttribute> getTableCoprocessorAttrsFromSchema(Configuration conf, @@ -1319,6 +1361,8 @@ public class RegionCoprocessorHost */ public boolean postScannerFilterRow(final InternalScanner s, final Cell curRowCell) throws IOException { + // short circuit for performance + if (!hasCustomPostScannerFilterRow) return true; return execOperationWithResult(true, coprocessors.isEmpty() ? null : new RegionOperationWithResult<Boolean>() { @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/526520de/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index 14b43df..58a2820 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -116,6 +116,7 @@ public class SimpleRegionObserver extends BaseRegionObserver { final AtomicInteger ctPostCheckAndDelete = new AtomicInteger(0); final AtomicInteger ctPreScannerNext = new AtomicInteger(0); final AtomicInteger ctPostScannerNext = new AtomicInteger(0); + final AtomicInteger ctPostScannerFilterRow = new AtomicInteger(0); final AtomicInteger ctPreScannerClose = new AtomicInteger(0); final AtomicInteger ctPostScannerClose = new AtomicInteger(0); final AtomicInteger ctPreScannerOpen = new AtomicInteger(0); @@ -314,6 +315,13 @@ public class SimpleRegionObserver extends BaseRegionObserver { } @Override + public boolean postScannerFilterRow(final ObserverContext<RegionCoprocessorEnvironment> e, + final InternalScanner s, final Cell currentRow, final boolean hasMore) throws IOException { + ctPostScannerFilterRow.incrementAndGet(); + return hasMore; + } + + @Override public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c, final InternalScanner s) throws IOException { ctPreScannerClose.incrementAndGet(); @@ -801,6 +809,9 @@ public class SimpleRegionObserver extends BaseRegionObserver { public boolean wasScannerNextCalled() { return ctPreScannerNext.get() > 0 && ctPostScannerNext.get() > 0; } + public boolean wasScannerFilterRowCalled() { + return ctPostScannerFilterRow.get() > 0; + } public boolean wasScannerCloseCalled() { return ctPreScannerClose.get() > 0 && ctPostScannerClose.get() > 0; } http://git-wip-us.apache.org/repos/asf/hbase/blob/526520de/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java index c710d3c..3ddf601 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.FilterAllFilter; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -373,6 +374,30 @@ public class TestRegionObserverInterface { } @Test (timeout=300000) + public void testHBASE14489() throws IOException { + TableName tableName = TableName.valueOf("testHBASE14489"); + HTable table = util.createTable(tableName, new byte[][] { A }); + Put put = new Put(ROW); + put.addColumn(A, A, A); + table.put(put); + + Scan s = new Scan(); + s.setFilter(new FilterAllFilter()); + ResultScanner scanner = table.getScanner(s); + try { + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { + } + } finally { + scanner.close(); + } + verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" }, + tableName, new Boolean[] { true }); + util.deleteTable(tableName); + table.close(); + + } + + @Test (timeout=300000) // HBase-3758 public void testHBase3758() throws IOException { TableName tableName =
