http://git-wip-us.apache.org/repos/asf/hbase/blob/052a6f07/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java ---------------------------------------------------------------------- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java new file mode 100644 index 0000000..e60208b --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java @@ -0,0 +1,1000 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SkipFilter; +import org.apache.hadoop.hbase.filter.SubstringComparator; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScannersWithFilters { + + private static final Log LOG = LogFactory.getLog(TestScannersWithFilters.class); + + private static final TableName TABLE = TableName.valueOf("TestScannersWithFilters"); + + private static final byte [][] ROWS_ONE = { + Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-1"), + Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") + }; + + private static final byte [][] ROWS_TWO = { + Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-1"), + Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") + }; + + private static final byte [][] FAMILIES = { + Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") + }; + + private static final byte [][] QUALIFIERS_ONE = { + Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"), + Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") + }; + + private static final byte [][] QUALIFIERS_TWO = { + Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"), + Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") + }; + + private static final byte [][] VALUES = { + Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") + }; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static long numRows = ROWS_ONE.length + ROWS_TWO.length; + private static long colsPerRow = FAMILIES.length * QUALIFIERS_ONE.length; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + context = JAXBContext.newInstance( + CellModel.class, + CellSetModel.class, + RowModel.class, + ScannerModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + Admin admin = TEST_UTIL.getHBaseAdmin(); + if (!admin.tableExists(TABLE)) { + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(FAMILIES[0])); + htd.addFamily(new HColumnDescriptor(FAMILIES[1])); + admin.createTable(htd); + Table table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + // Insert first half + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setDurability(Durability.SKIP_WAL); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[0], QUALIFIER, VALUES[0]); + } + table.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setDurability(Durability.SKIP_WAL); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[1], QUALIFIER, VALUES[1]); + } + table.put(p); + } + + // Insert second half (reverse families) + for(byte [] ROW : ROWS_ONE) { + Put p = new Put(ROW); + p.setDurability(Durability.SKIP_WAL); + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + p.add(FAMILIES[1], QUALIFIER, VALUES[0]); + } + table.put(p); + } + for(byte [] ROW : ROWS_TWO) { + Put p = new Put(ROW); + p.setDurability(Durability.SKIP_WAL); + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + p.add(FAMILIES[0], QUALIFIER, VALUES[1]); + } + table.put(p); + } + + // Delete the second qualifier from all rows and families + for(byte [] ROW : ROWS_ONE) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_ONE[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_ONE[1]); + table.delete(d); + } + for(byte [] ROW : ROWS_TWO) { + Delete d = new Delete(ROW); + d.deleteColumns(FAMILIES[0], QUALIFIERS_TWO[1]); + d.deleteColumns(FAMILIES[1], QUALIFIERS_TWO[1]); + table.delete(d); + } + colsPerRow -= 2; + + // Delete the second rows from both groups, one column at a time + for(byte [] QUALIFIER : QUALIFIERS_ONE) { + Delete d = new Delete(ROWS_ONE[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + table.delete(d); + } + for(byte [] QUALIFIER : QUALIFIERS_TWO) { + Delete d = new Delete(ROWS_TWO[1]); + d.deleteColumns(FAMILIES[0], QUALIFIER); + d.deleteColumns(FAMILIES[1], QUALIFIER); + table.delete(d); + } + numRows -= 2; + table.close(); + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void verifyScan(Scan s, long expectedRows, long expectedKeys) + throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cells = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + int rows = cells.getRows().size(); + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but scanned " + rows, expectedRows == rows); + for (RowModel row: cells.getRows()) { + int count = row.getCells().size(); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + count, expectedKeys, count); + } + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + } + + private static void verifyScanFull(Scan s, KeyValue [] kvs) + throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + + int row = 0; + int idx = 0; + Iterator<RowModel> i = cellSet.getRows().iterator(); + for (boolean done = true; done; row++) { + done = i.hasNext(); + if (!done) break; + RowModel rowModel = i.next(); + List<CellModel> cells = rowModel.getCells(); + if (cells.isEmpty()) break; + assertTrue("Scanned too many keys! Only expected " + kvs.length + + " total but already scanned " + (cells.size() + idx), + kvs.length >= idx + cells.size()); + for (CellModel cell: cells) { + assertTrue("Row mismatch", + Bytes.equals(rowModel.getKey(), kvs[idx].getRow())); + byte[][] split = KeyValue.parseColumn(cell.getColumn()); + assertTrue("Family mismatch", + Bytes.equals(split[0], kvs[idx].getFamily())); + assertTrue("Qualifier mismatch", + Bytes.equals(split[1], kvs[idx].getQualifier())); + assertTrue("Value mismatch", + Bytes.equals(cell.getValue(), kvs[idx].getValue())); + idx++; + } + } + assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, + kvs.length, idx); + } + + private static void verifyScanNoEarlyOut(Scan s, long expectedRows, + long expectedKeys) throws Exception { + ScannerModel model = ScannerModel.fromScan(s); + model.setBatch(Integer.MAX_VALUE); // fetch it all at once + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + LOG.debug(writer.toString()); + byte[] body = Bytes.toBytes(writer.toString()); + Response response = client.put("/" + TABLE + "/scanner", + Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) + unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); + + // delete the scanner + response = client.delete(scannerURI); + assertEquals(response.getCode(), 200); + + Iterator<RowModel> i = cellSet.getRows().iterator(); + int j = 0; + for (boolean done = true; done; j++) { + done = i.hasNext(); + if (!done) break; + RowModel rowModel = i.next(); + List<CellModel> cells = rowModel.getCells(); + if (cells.isEmpty()) break; + assertTrue("Scanned too many rows! Only expected " + expectedRows + + " total but already scanned " + (j+1), expectedRows > j); + assertEquals("Expected " + expectedKeys + " keys per row but " + + "returned " + cells.size(), expectedKeys, cells.size()); + } + assertEquals("Expected " + expectedRows + " rows but scanned " + j + + " rows", expectedRows, j); + } + + @Test + public void testNoFilter() throws Exception { + // No filter + long expectedRows = numRows; + long expectedKeys = colsPerRow; + + // Both families + Scan s = new Scan(); + verifyScan(s, expectedRows, expectedKeys); + + // One family + s = new Scan(); + s.addFamily(FAMILIES[0]); + verifyScan(s, expectedRows, expectedKeys/2); + } + + @Test + public void testPrefixFilter() throws Exception { + // Grab rows from group one (half of total) + long expectedRows = numRows / 2; + long expectedKeys = colsPerRow; + Scan s = new Scan(); + s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); + verifyScan(s, expectedRows, expectedKeys); + } + + @Test + public void testPageFilter() throws Exception { + // KVs in first 6 rows + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + + // Grab all 6 rows + long expectedRows = 6; + long expectedKeys = colsPerRow; + Scan s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, expectedKVs); + + // Grab first 4 rows (6 cols per row) + expectedRows = 4; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 24)); + + // Grab first 2 rows + expectedRows = 2; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 12)); + + // Grab first row + expectedRows = 1; + expectedKeys = colsPerRow; + s = new Scan(); + s.setFilter(new PageFilter(expectedRows)); + verifyScan(s, expectedRows, expectedKeys); + s.setFilter(new PageFilter(expectedRows)); + verifyScanFull(s, Arrays.copyOf(expectedKVs, 6)); + } + + @Test + public void testInclusiveStopFilter() throws Exception { + // Grab rows from group one + + // If we just use start/stop row, we get total/2 - 1 rows + long expectedRows = (numRows / 2) - 1; + long expectedKeys = colsPerRow; + Scan s = new Scan(Bytes.toBytes("testRowOne-0"), + Bytes.toBytes("testRowOne-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = numRows / 2; + s = new Scan(Bytes.toBytes("testRowOne-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3"))); + verifyScan(s, expectedRows, expectedKeys); + + // Grab rows from group two + + // If we just use start/stop row, we get total/2 - 1 rows + expectedRows = (numRows / 2) - 1; + expectedKeys = colsPerRow; + s = new Scan(Bytes.toBytes("testRowTwo-0"), + Bytes.toBytes("testRowTwo-3")); + verifyScan(s, expectedRows, expectedKeys); + + // Now use start row with inclusive stop filter + expectedRows = numRows / 2; + s = new Scan(Bytes.toBytes("testRowTwo-0")); + s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3"))); + verifyScan(s, expectedRows, expectedKeys); + } + + @Test + public void testQualifierFilter() throws Exception { + // Match two keys (one from each family) in half the rows + long expectedRows = numRows / 2; + long expectedKeys = 2; + Filter f = new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than same qualifier + // Expect only two keys (one from each family) in half the rows + expectedRows = numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys less than or equal + // Expect four keys (two from each family) in half the rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect four keys (two from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 4; + f = new QualifierFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect two keys (one from each family) + // Only look in first group of rows + expectedRows = numRows / 2; + expectedKeys = 2; + f = new QualifierFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); + s = new Scan(HConstants.EMPTY_START_ROW, Bytes.toBytes("testRowTwo")); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys not equal to + // Look across rows and fully validate the keys and ordering + // Expect varied numbers of keys, 4 per row in group one, 6 per row in + // group two + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(QUALIFIERS_ONE[2])); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + // Test across rows and groups with a regex + // Filter out "test*-2" + // Expect 4 keys per row across both groups + f = new QualifierFilter(CompareOp.NOT_EQUAL, + new RegexStringComparator("test.+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testRowFilter() throws Exception { + // Match a single row, all keys + long expectedRows = 1; + long expectedKeys = colsPerRow; + Filter f = new RowFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match a two rows, one from each group, using regex + expectedRows = 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator("testRow.+-2")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than + // Expect all keys in one row + expectedRows = 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows less than or equal + // Expect all keys in two rows + expectedRows = 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal + // Expect all keys in all but one row + expectedRows = numRows - 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater or equal + // Expect all keys in all but one row + expectedRows = numRows - 1; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match keys greater + // Expect all keys in all but two rows + expectedRows = numRows - 2; + expectedKeys = colsPerRow; + f = new RowFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match rows not equal to testRowTwo-2 + // Look across rows and fully validate the keys and ordering + // Should see all keys in all rows but testRowTwo-2 + f = new RowFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testRowOne-2"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + + // Test across rows and groups with a regex + // Filter out everything that doesn't match "*-2" + // Expect all keys in two rows + f = new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2")); + s = new Scan(); + s.setFilter(f); + + kvs = new KeyValue [] { + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) + }; + verifyScanFull(s, kvs); + } + + @Test + public void testValueFilter() throws Exception { + // Match group one rows + long expectedRows = numRows / 2; + long expectedKeys = colsPerRow; + Filter f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + Scan s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match group two rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match all values using regex + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.EQUAL, + new RegexStringComparator("testValue((One)|(Two))")); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than + // Expect group one rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect all rows + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueTwo"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values less than or equal + // Expect group one rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.LESS_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal + // Expect half the rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater or equal + // Expect all rows + expectedRows = numRows; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.GREATER_OR_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values greater + // Expect half rows + expectedRows = numRows / 2; + expectedKeys = colsPerRow; + f = new ValueFilter(CompareOp.GREATER, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, expectedRows, expectedKeys); + + // Match values not equal to testValueOne + // Look across rows and fully validate the keys and ordering + // Should see all keys in all group two rows + f = new ValueFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testValueOne"))); + s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testSkipFilter() throws Exception { + // Test for qualifier regex: "testQualifierOne-2" + // Should only get rows from second group, and all keys + Filter f = new SkipFilter(new QualifierFilter(CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("testQualifierOne-2")))); + Scan s = new Scan(); + s.setFilter(f); + + KeyValue [] kvs = { + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), + }; + verifyScanFull(s, kvs); + } + + @Test + public void testFilterList() throws Exception { + // Test getting a single row, single key using Row, Qualifier, and Value + // regular expression and substring filters + // Use must pass all + List<Filter> filters = new ArrayList<Filter>(); + filters.add(new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, + new SubstringComparator("One"))); + Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); + Scan s = new Scan(); + s.addFamily(FAMILIES[0]); + s.setFilter(f); + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) + }; + verifyScanFull(s, kvs); + + // Test getting everything with a MUST_PASS_ONE filter including row, qf, + // val, regular expression and substring filters + filters.clear(); + filters.add(new RowFilter(CompareOp.EQUAL, + new RegexStringComparator(".+Two.+"))); + filters.add(new QualifierFilter(CompareOp.EQUAL, + new RegexStringComparator(".+-2"))); + filters.add(new ValueFilter(CompareOp.EQUAL, + new SubstringComparator("One"))); + f = new FilterList(Operator.MUST_PASS_ONE, filters); + s = new Scan(); + s.setFilter(f); + verifyScanNoEarlyOut(s, numRows, colsPerRow); + } + + @Test + public void testFirstKeyOnlyFilter() throws Exception { + Scan s = new Scan(); + s.setFilter(new FirstKeyOnlyFilter()); + // Expected KVs, the first KV from each of the remaining 6 rows + KeyValue [] kvs = { + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) + }; + verifyScanFull(s, kvs); + } + +} +
http://git-wip-us.apache.org/repos/asf/hbase/blob/052a6f07/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java ---------------------------------------------------------------------- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java new file mode 100644 index 0000000..e8ec7e3 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */package org.apache.hadoop.hbase.rest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.security.PrivilegedExceptionAction; +import java.util.Iterator; +import java.util.Random; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.CellModel; +import org.apache.hadoop.hbase.rest.model.CellSetModel; +import org.apache.hadoop.hbase.rest.model.RowModel; +import org.apache.hadoop.hbase.rest.model.ScannerModel; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.visibility.CellVisibility; +import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator; +import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator; +import org.apache.hadoop.hbase.security.visibility.VisibilityClient; +import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; +import org.apache.hadoop.hbase.security.visibility.VisibilityController; +import org.apache.hadoop.hbase.security.visibility.VisibilityUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestScannersWithLabels { + private static final TableName TABLE = TableName.valueOf("TestScannersWithLabels"); + private static final String CFA = "a"; + private static final String CFB = "b"; + private static final String COLUMN_1 = CFA + ":1"; + private static final String COLUMN_2 = CFB + ":2"; + private final static String TOPSECRET = "topsecret"; + private final static String PUBLIC = "public"; + private final static String PRIVATE = "private"; + private final static String CONFIDENTIAL = "confidential"; + private final static String SECRET = "secret"; + private static User SUPERUSER; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Marshaller marshaller; + private static Unmarshaller unmarshaller; + private static Configuration conf; + + private static int insertData(TableName tableName, String column, double prob) throws IOException { + Random rng = new Random(); + int count = 0; + Table table = new HTable(TEST_UTIL.getConfiguration(), tableName); + byte[] k = new byte[3]; + byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); + + for (int i = 0; i < 9; i++) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.setDurability(Durability.SKIP_WAL); + put.add(famAndQf[0], famAndQf[1], k); + put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + + TOPSECRET)); + table.put(put); + count++; + } + table.flushCommits(); + return count; + } + + private static int countCellSet(CellSetModel model) { + int count = 0; + Iterator<RowModel> rows = model.getRows().iterator(); + while (rows.hasNext()) { + RowModel row = rows.next(); + Iterator<CellModel> cells = row.getCells().iterator(); + while (cells.hasNext()) { + cells.next(); + count++; + } + } + return count; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + SUPERUSER = User.createUserForTesting(conf, "admin", + new String[] { "supergroup" }); + conf = TEST_UTIL.getConfiguration(); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, + SimpleScanLabelGenerator.class, ScanLabelGenerator.class); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.superuser", SUPERUSER.getShortName()); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + TEST_UTIL.startMiniCluster(1); + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000); + createLabels(); + setAuths(); + REST_TEST_UTIL.startServletContainer(conf); + client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class, + ScannerModel.class); + marshaller = context.createMarshaller(); + unmarshaller = context.createUnmarshaller(); + Admin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(CFA)); + htd.addFamily(new HColumnDescriptor(CFB)); + admin.createTable(htd); + insertData(TABLE, COLUMN_1, 1.0); + insertData(TABLE, COLUMN_2, 0.5); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void createLabels() throws IOException, InterruptedException { + PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() { + public VisibilityLabelsResponse run() throws Exception { + String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET }; + try { + VisibilityClient.addLabels(conf, labels); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(action); + } + private static void setAuths() throws Exception { + String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET }; + try { + VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName()); + } catch (Throwable t) { + throw new IOException(t); + } + } + @Test + public void testSimpleScannerXMLWithLabelsThatReceivesNoData() throws IOException, JAXBException { + final int BATCH_SIZE = 5; + // new scanner + ScannerModel model = new ScannerModel(); + model.setBatch(BATCH_SIZE); + model.addColumn(Bytes.toBytes(COLUMN_1)); + model.addLabel(PUBLIC); + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + byte[] body = Bytes.toBytes(writer.toString()); + // recall previous put operation with read-only off + conf.set("hbase.rest.readonly", "false"); + Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + // Respond with 204 as there are no cells to be retrieved + assertEquals(response.getCode(), 204); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + } + + @Test + public void testSimpleScannerXMLWithLabelsThatReceivesData() throws IOException, JAXBException { + // new scanner + ScannerModel model = new ScannerModel(); + model.setBatch(5); + model.addColumn(Bytes.toBytes(COLUMN_1)); + model.addLabel(SECRET); + StringWriter writer = new StringWriter(); + marshaller.marshal(model, writer); + byte[] body = Bytes.toBytes(writer.toString()); + + // recall previous put operation with read-only off + conf.set("hbase.rest.readonly", "false"); + Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); + assertEquals(response.getCode(), 201); + String scannerURI = response.getLocation(); + assertNotNull(scannerURI); + + // get a cell set + response = client.get(scannerURI, Constants.MIMETYPE_XML); + // Respond with 204 as there are no cells to be retrieved + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + CellSetModel cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response + .getBody())); + assertEquals(countCellSet(cellSet), 5); + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/052a6f07/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java ---------------------------------------------------------------------- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java new file mode 100644 index 0000000..36cd193 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestSchemaResource.java @@ -0,0 +1,193 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringWriter; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.ColumnSchemaModel; +import org.apache.hadoop.hbase.rest.model.TableSchemaModel; +import org.apache.hadoop.hbase.rest.model.TestTableSchemaModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestSchemaResource { + private static String TABLE1 = "TestSchemaResource1"; + private static String TABLE2 = "TestSchemaResource2"; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + private static Configuration conf; + private static TestTableSchemaModel testTableSchemaModel; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(conf); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + testTableSchemaModel = new TestTableSchemaModel(); + context = JAXBContext.newInstance( + ColumnSchemaModel.class, + TableSchemaModel.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static byte[] toXML(TableSchemaModel model) throws JAXBException { + StringWriter writer = new StringWriter(); + context.createMarshaller().marshal(model, writer); + return Bytes.toBytes(writer.toString()); + } + + private static TableSchemaModel fromXML(byte[] content) + throws JAXBException { + return (TableSchemaModel) context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(content)); + } + + @Test + public void testTableCreateAndDeleteXML() throws IOException, JAXBException { + String schemaPath = "/" + TABLE1 + "/schema"; + TableSchemaModel model; + Response response; + + Admin admin = TEST_UTIL.getHBaseAdmin(); + assertFalse(admin.tableExists(TableName.valueOf(TABLE1))); + + // create the table + model = testTableSchemaModel.buildTestModel(TABLE1); + testTableSchemaModel.checkModel(model, TABLE1); + response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model)); + assertEquals(response.getCode(), 201); + + // recall the same put operation but in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model)); + assertEquals(response.getCode(), 403); + + // retrieve the schema and validate it + response = client.get(schemaPath, Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + model = fromXML(response.getBody()); + testTableSchemaModel.checkModel(model, TABLE1); + + // with json retrieve the schema and validate it + response = client.get(schemaPath, Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + model = testTableSchemaModel.fromJSON(Bytes.toString(response.getBody())); + testTableSchemaModel.checkModel(model, TABLE1); + + // test delete schema operation is forbidden in read-only mode + response = client.delete(schemaPath); + assertEquals(response.getCode(), 403); + + // return read-only setting back to default + conf.set("hbase.rest.readonly", "false"); + + // delete the table and make sure HBase concurs + response = client.delete(schemaPath); + assertEquals(response.getCode(), 200); + assertFalse(admin.tableExists(TableName.valueOf(TABLE1))); + } + + @Test + public void testTableCreateAndDeletePB() throws IOException, JAXBException { + String schemaPath = "/" + TABLE2 + "/schema"; + TableSchemaModel model; + Response response; + + Admin admin = TEST_UTIL.getHBaseAdmin(); + assertFalse(admin.tableExists(TableName.valueOf(TABLE2))); + + // create the table + model = testTableSchemaModel.buildTestModel(TABLE2); + testTableSchemaModel.checkModel(model, TABLE2); + response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + assertEquals(response.getCode(), 201); + + // recall the same put operation but in read-only mode + conf.set("hbase.rest.readonly", "true"); + response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, + model.createProtobufOutput()); + assertEquals(response.getCode(), 403); + + // retrieve the schema and validate it + response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + model = new TableSchemaModel(); + model.getObjectFromMessage(response.getBody()); + testTableSchemaModel.checkModel(model, TABLE2); + + // retrieve the schema and validate it with alternate pbuf type + response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableSchemaModel(); + model.getObjectFromMessage(response.getBody()); + testTableSchemaModel.checkModel(model, TABLE2); + + // test delete schema operation is forbidden in read-only mode + response = client.delete(schemaPath); + assertEquals(response.getCode(), 403); + + // return read-only setting back to default + conf.set("hbase.rest.readonly", "false"); + + // delete the table and make sure HBase concurs + response = client.delete(schemaPath); + assertEquals(response.getCode(), 200); + assertFalse(admin.tableExists(TableName.valueOf(TABLE2))); + } + +} + http://git-wip-us.apache.org/repos/asf/hbase/blob/052a6f07/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java ---------------------------------------------------------------------- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java new file mode 100644 index 0000000..c9aa191 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestStatusResource.java @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.StorageClusterStatusModel; +import org.apache.hadoop.hbase.util.Bytes; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestStatusResource { + private static final byte[] META_REGION_NAME = Bytes.toBytes(TableName.META_TABLE_NAME+",,1"); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + + private static void validate(StorageClusterStatusModel model) { + assertNotNull(model); + assertTrue(model.getRegions() + ">= 1", model.getRegions() >= 1); + assertTrue(model.getRequests() >= 0); + assertTrue(model.getAverageLoad() >= 0.0); + assertNotNull(model.getLiveNodes()); + assertNotNull(model.getDeadNodes()); + assertFalse(model.getLiveNodes().isEmpty()); + boolean foundMeta = false; + for (StorageClusterStatusModel.Node node: model.getLiveNodes()) { + assertNotNull(node.getName()); + assertTrue(node.getStartCode() > 0L); + assertTrue(node.getRequests() >= 0); + for (StorageClusterStatusModel.Node.Region region: node.getRegions()) { + if (Bytes.equals(region.getName(), META_REGION_NAME)) { + foundMeta = true; + } + } + } + assertTrue(foundMeta); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance(StorageClusterStatusModel.class); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testGetClusterStatusXML() throws IOException, JAXBException { + Response response = client.get("/status/cluster", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + StorageClusterStatusModel model = (StorageClusterStatusModel) + context.createUnmarshaller().unmarshal( + new ByteArrayInputStream(response.getBody())); + validate(model); + } + + @Test + public void testGetClusterStatusPB() throws IOException { + Response response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + StorageClusterStatusModel model = new StorageClusterStatusModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + response = client.get("/status/cluster", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new StorageClusterStatusModel(); + model.getObjectFromMessage(response.getBody()); + validate(model); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/052a6f07/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java ---------------------------------------------------------------------- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java new file mode 100644 index 0000000..719f7c2 --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java @@ -0,0 +1,262 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Iterator; +import java.util.Map; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.rest.client.Client; +import org.apache.hadoop.hbase.rest.client.Cluster; +import org.apache.hadoop.hbase.rest.client.Response; +import org.apache.hadoop.hbase.rest.model.TableModel; +import org.apache.hadoop.hbase.rest.model.TableInfoModel; +import org.apache.hadoop.hbase.rest.model.TableListModel; +import org.apache.hadoop.hbase.rest.model.TableRegionModel; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.StringUtils; + +import static org.junit.Assert.*; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestTableResource { + private static final Log LOG = LogFactory.getLog(TestTableResource.class); + + private static TableName TABLE = TableName.valueOf("TestTableResource"); + private static String COLUMN_FAMILY = "test"; + private static String COLUMN = COLUMN_FAMILY + ":qualifier"; + private static Map<HRegionInfo, ServerName> regionMap; + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final HBaseRESTTestingUtility REST_TEST_UTIL = + new HBaseRESTTestingUtility(); + private static Client client; + private static JAXBContext context; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); + client = new Client(new Cluster().add("localhost", + REST_TEST_UTIL.getServletPort())); + context = JAXBContext.newInstance( + TableModel.class, + TableInfoModel.class, + TableListModel.class, + TableRegionModel.class); + Admin admin = TEST_UTIL.getHBaseAdmin(); + if (admin.tableExists(TABLE)) { + return; + } + HTableDescriptor htd = new HTableDescriptor(TABLE); + htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + admin.createTable(htd); + HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE); + byte[] k = new byte[3]; + byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(COLUMN)); + for (byte b1 = 'a'; b1 < 'z'; b1++) { + for (byte b2 = 'a'; b2 < 'z'; b2++) { + for (byte b3 = 'a'; b3 < 'z'; b3++) { + k[0] = b1; + k[1] = b2; + k[2] = b3; + Put put = new Put(k); + put.setDurability(Durability.SKIP_WAL); + put.add(famAndQf[0], famAndQf[1], k); + table.put(put); + } + } + } + table.flushCommits(); + // get the initial layout (should just be one region) + Map<HRegionInfo, ServerName> m = table.getRegionLocations(); + assertEquals(m.size(), 1); + // tell the master to split the table + admin.split(TABLE); + // give some time for the split to happen + + long timeout = System.currentTimeMillis() + (15 * 1000); + while (System.currentTimeMillis() < timeout && m.size()!=2){ + try { + Thread.sleep(250); + } catch (InterruptedException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + // check again + m = table.getRegionLocations(); + } + + // should have two regions now + assertEquals(m.size(), 2); + regionMap = m; + LOG.info("regions: " + regionMap); + table.close(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + REST_TEST_UTIL.shutdownServletContainer(); + TEST_UTIL.shutdownMiniCluster(); + } + + private static void checkTableList(TableListModel model) { + boolean found = false; + Iterator<TableModel> tables = model.getTables().iterator(); + assertTrue(tables.hasNext()); + while (tables.hasNext()) { + TableModel table = tables.next(); + if (table.getName().equals(TABLE.getNameAsString())) { + found = true; + break; + } + } + assertTrue(found); + } + + void checkTableInfo(TableInfoModel model) { + assertEquals(model.getName(), TABLE.getNameAsString()); + Iterator<TableRegionModel> regions = model.getRegions().iterator(); + assertTrue(regions.hasNext()); + while (regions.hasNext()) { + TableRegionModel region = regions.next(); + boolean found = false; + for (Map.Entry<HRegionInfo, ServerName> e: regionMap.entrySet()) { + HRegionInfo hri = e.getKey(); + String hriRegionName = hri.getRegionNameAsString(); + String regionName = region.getName(); + if (hriRegionName.equals(regionName)) { + found = true; + byte[] startKey = hri.getStartKey(); + byte[] endKey = hri.getEndKey(); + InetSocketAddress sa = new InetSocketAddress(e.getValue().getHostname(), e.getValue().getPort()); + String location = sa.getHostName() + ":" + + Integer.valueOf(sa.getPort()); + assertEquals(hri.getRegionId(), region.getId()); + assertTrue(Bytes.equals(startKey, region.getStartKey())); + assertTrue(Bytes.equals(endKey, region.getEndKey())); + assertEquals(location, region.getLocation()); + break; + } + } + assertTrue(found); + } + } + + @Test + public void testTableListText() throws IOException { + Response response = client.get("/", Constants.MIMETYPE_TEXT); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + } + + @Test + public void testTableListXML() throws IOException, JAXBException { + Response response = client.get("/", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + TableListModel model = (TableListModel) + context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(response.getBody())); + checkTableList(model); + } + + @Test + public void testTableListJSON() throws IOException { + Response response = client.get("/", Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @Test + public void testTableListPB() throws IOException, JAXBException { + Response response = client.get("/", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + TableListModel model = new TableListModel(); + model.getObjectFromMessage(response.getBody()); + checkTableList(model); + response = client.get("/", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableListModel(); + model.getObjectFromMessage(response.getBody()); + checkTableList(model); + } + + @Test + public void testTableInfoText() throws IOException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_TEXT); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_TEXT, response.getHeader("content-type")); + } + + @Test + public void testTableInfoXML() throws IOException, JAXBException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_XML); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); + TableInfoModel model = (TableInfoModel) + context.createUnmarshaller() + .unmarshal(new ByteArrayInputStream(response.getBody())); + checkTableInfo(model); + } + + @Test + public void testTableInfoJSON() throws IOException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_JSON); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + } + + @Test + public void testTableInfoPB() throws IOException, JAXBException { + Response response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type")); + TableInfoModel model = new TableInfoModel(); + model.getObjectFromMessage(response.getBody()); + checkTableInfo(model); + response = client.get("/" + TABLE + "/regions", Constants.MIMETYPE_PROTOBUF_IETF); + assertEquals(response.getCode(), 200); + assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type")); + model = new TableInfoModel(); + model.getObjectFromMessage(response.getBody()); + checkTableInfo(model); + } + +} +
