KYLIN-2233 Support HBase 1.0.2 Signed-off-by: Yang Li <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83d8157b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83d8157b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83d8157b Branch: refs/heads/yang21-hbase102 Commit: 83d8157bbe13f894c6cb85380818f4ee73ff0121 Parents: c45ada5 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Wed Oct 12 14:39:24 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Tue Dec 13 14:52:38 2016 +0800 ---------------------------------------------------------------------- pom.xml | 2 +- .../storage/hbase/cube/v1/RegionScannerAdapter.java | 11 ++--------- .../coprocessor/observer/AggregateRegionObserver.java | 4 ++-- .../v1/coprocessor/observer/AggregationScanner.java | 14 ++++---------- .../observer/ObserverAggregationCache.java | 10 ++-------- .../observer/AggregateRegionObserverTest.java | 12 +++--------- .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +-- 7 files changed, 15 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1df14f4..ad70ee3 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ <hive-hcatalog.version>1.2.1</hive-hcatalog.version> <!-- HBase versions --> - <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version> + <hbase-hadoop2.version>1.0.2</hbase-hadoop2.version> <kafka.version>0.8.1</kafka.version> <!-- Hadoop deps, keep compatible with hadoop2.version --> http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java index 0ade920..40da772 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; /** * @author yangli9 @@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override - public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException { + public boolean next(List<Cell> result, int limit) throws IOException { return next(result); } @@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner { } @Override - public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException { + public boolean nextRaw(List<Cell> result, int limit) throws IOException { return next(result); } @@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner { public long getMvccReadPoint() { return Long.MAX_VALUE; } - - @Override - public int getBatch() { - return -1; - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java index 7e25e4c..7139ca7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.kylin.gridtable.StorageSideBehavior; @@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver { // start/end region operation & sync on scanner is suggested by the // javadoc of RegionScanner.nextRaw() // FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb - Region region = ctxt.getEnvironment().getRegion(); + HRegion region = ctxt.getEnvironment().getRegion(); region.startRegionOperation(); try { synchronized (innerScanner) { http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java index d64f48f..a900ea1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java @@ -25,7 +25,6 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.kylin.gridtable.StorageSideBehavior; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey; @@ -117,8 +116,8 @@ public class AggregationScanner implements RegionScanner { } @Override - public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException { - return outerScanner.next(result, scannerContext); + public boolean next(List<Cell> result, int limit) throws IOException { + return outerScanner.next(result, limit); } @Override @@ -127,8 +126,8 @@ public class AggregationScanner implements RegionScanner { } @Override - public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException { - return outerScanner.nextRaw(result, scannerContext); + public boolean nextRaw(List<Cell> result, int limit) throws IOException { + return outerScanner.nextRaw(result, limit); } @Override @@ -161,11 +160,6 @@ public class AggregationScanner implements RegionScanner { return outerScanner.getMvccReadPoint(); } - @Override - public int getBatch() { - return outerScanner.getBatch(); - } - private static class Stats { long inputRows = 0; long inputBytes = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java index 331e34d..8404262 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey; import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache; @@ -113,7 +112,7 @@ public class ObserverAggregationCache extends AggregationCache { } @Override - public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException { + public boolean next(List<Cell> result, int limit) throws IOException { return next(result); } @@ -123,7 +122,7 @@ public class ObserverAggregationCache extends AggregationCache { } @Override - public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException { + public boolean nextRaw(List<Cell> result, int limit) throws IOException { return next(result); } @@ -162,11 +161,6 @@ public class ObserverAggregationCache extends AggregationCache { // AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()"); return Long.MAX_VALUE; } - - @Override - public int getBatch() { - return innerScanner.getBatch(); - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java index 1196ed6..8865125 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java @@ -35,13 +35,12 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.hadoop.hbase.regionserver.ScannerContext; import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.gridtable.StorageSideBehavior; import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.gridtable.StorageSideBehavior; import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter; import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector; import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType; @@ -232,7 +231,7 @@ public class AggregateRegionObserverTest { } @Override - public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException { + public boolean next(List<Cell> result, int limit) throws IOException { return next(result); } @@ -301,11 +300,6 @@ public class AggregateRegionObserverTest { return 0; } - @Override - public int getBatch() { - return 0; - } - /* * (non-Javadoc) * @@ -323,7 +317,7 @@ public class AggregateRegionObserverTest { } @Override - public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException { + public boolean nextRaw(List<Cell> list, int limit) throws IOException { return false; } http://git-wip-us.apache.org/repos/asf/kylin/blob/83d8157b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java index 04e2e8b..4e87093 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -225,7 +224,7 @@ public class TestFuzzyRowFilterV2EndToEnd { scan.addFamily(cf.getBytes()); scan.setFilter(filter); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes()); - Region first = regions.get(0); + HRegion first = regions.get(0); first.getScanner(scan); RegionScanner scanner = first.getScanner(scan); List<Cell> results = new ArrayList<Cell>();
