http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java deleted file mode 100644 index 9d8b8f0..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TsvImporterCustomTestMapperForOprAttr.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException; -import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.ParsedLine; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * - * Just shows a simple example of how the attributes can be extracted and added - * to the puts - */ -public class TsvImporterCustomTestMapperForOprAttr extends TsvImporterMapper { - @Override - protected void populatePut(byte[] lineBytes, ParsedLine parsed, Put put, int i) - throws BadTsvLineException, IOException { - KeyValue kv; - kv = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), - parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, - parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), - parsed.getColumnLength(i)); - if (parsed.getIndividualAttributes() != null) { - String[] attributes = parsed.getIndividualAttributes(); - for (String attr : attributes) { - String[] split = attr.split(ImportTsv.DEFAULT_ATTRIBUTES_SEPERATOR); - if (split == null || split.length <= 1) { - throw new BadTsvLineException("Invalid attributes seperator specified" + attributes); - } else { - if (split[0].length() <= 0 || split[1].length() <= 0) { - throw new BadTsvLineException("Invalid attributes seperator specified" + attributes); - } - put.setAttribute(split[0], Bytes.toBytes(split[1])); - } - } - } - put.add(kv); - } -}
http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index f641887..a81d268 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -65,7 +65,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionServerObserver; -import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.TableNamespaceManager; @@ -336,7 +335,7 @@ public class TestNamespaceAuditor { byte[] columnFamily = Bytes.toBytes("info"); HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo); tableDescOne.addFamily(new HColumnDescriptor(columnFamily)); - ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions); + ADMIN.createTable(tableDescOne, Bytes.toBytes("0"), Bytes.toBytes("9"), initialRegions); Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); try (Table table = connection.getTable(tableTwo)) { UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999); @@ -354,7 +353,7 @@ public class TestNamespaceAuditor { hris = ADMIN.getTableRegions(tableTwo); assertEquals(initialRegions - 1, hris.size()); Collections.sort(hris); - ADMIN.split(tableTwo, Bytes.toBytes("500")); + ADMIN.split(tableTwo, Bytes.toBytes("3")); // Not much we can do here until we have split return a Future. Threads.sleep(5000); hris = ADMIN.getTableRegions(tableTwo); @@ -383,8 +382,7 @@ public class TestNamespaceAuditor { Collections.sort(hris); // verify that we cannot split HRegionInfo hriToSplit2 = hris.get(1); - ADMIN.split(tableTwo, - TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true)); + ADMIN.split(tableTwo, Bytes.toBytes("6")); Thread.sleep(2000); assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size()); } http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java index 0aa39f6..477c870 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java @@ -42,7 +42,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.PerformanceEvaluation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HTable; @@ -62,9 +61,11 @@ import org.junit.rules.TestName; public class TestHRegionFileSystem { private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final Log LOG = LogFactory.getLog(TestHRegionFileSystem.class); + + public static final byte[] FAMILY_NAME = Bytes.toBytes("info"); private static final byte[][] FAMILIES = { - Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-A")), - Bytes.add(PerformanceEvaluation.FAMILY_NAME, Bytes.toBytes("-B")) }; + Bytes.add(FAMILY_NAME, Bytes.toBytes("-A")), + Bytes.add(FAMILY_NAME, Bytes.toBytes("-B")) }; private static final TableName TABLE_NAME = TableName.valueOf("TestTable"); @Rule http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java deleted file mode 100644 index e1cb8ba..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java +++ /dev/null @@ -1,1059 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.replication; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.NavigableMap; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.hbase.replication.regionserver.Replication; -import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource; -import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.ReplicationTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.mapreduce.Job; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; - -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; - -@Category({ReplicationTests.class, LargeTests.class}) -public class TestReplicationSmallTests extends TestReplicationBase { - - private static final Log LOG = LogFactory.getLog(TestReplicationSmallTests.class); - private static final String PEER_ID = "2"; - - @Rule - public TestName name = new TestName(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - // Starting and stopping replication can make us miss new logs, - // rolling like this makes sure the most recent one gets added to the queue - for ( JVMClusterUtil.RegionServerThread r : - utility1.getHBaseCluster().getRegionServerThreads()) { - utility1.getAdmin().rollWALWriter(r.getRegionServer().getServerName()); - } - int rowCount = utility1.countRows(tableName); - utility1.deleteTableData(tableName); - // truncating the table will send one Delete per row to the slave cluster - // in an async fashion, which is why we cannot just call deleteTableData on - // utility2 since late writes could make it to the slave in some way. - // Instead, we truncate the first table and wait for all the Deletes to - // make it to the slave. - Scan scan = new Scan(); - int lastCount = 0; - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for truncate"); - } - ResultScanner scanner = htable2.getScanner(scan); - Result[] res = scanner.next(rowCount); - scanner.close(); - if (res.length != 0) { - if (res.length < lastCount) { - i--; // Don't increment timeout if we make progress - } - lastCount = res.length; - LOG.info("Still got " + res.length + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - /** - * Verify that version and column delete marker types are replicated - * correctly. - * @throws Exception - */ - @Test(timeout=300000) - public void testDeleteTypes() throws Exception { - LOG.info("testDeleteTypes"); - final byte[] v1 = Bytes.toBytes("v1"); - final byte[] v2 = Bytes.toBytes("v2"); - final byte[] v3 = Bytes.toBytes("v3"); - htable1 = utility1.getConnection().getTable(tableName); - - long t = EnvironmentEdgeManager.currentTime(); - // create three versions for "row" - Put put = new Put(row); - put.addColumn(famName, row, t, v1); - htable1.put(put); - - put = new Put(row); - put.addColumn(famName, row, t + 1, v2); - htable1.put(put); - - put = new Put(row); - put.addColumn(famName, row, t + 2, v3); - htable1.put(put); - - Get get = new Get(row); - get.setMaxVersions(); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for put replication"); - } - Result res = htable2.get(get); - if (res.size() < 3) { - LOG.info("Rows not available"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3); - assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2); - assertArrayEquals(CellUtil.cloneValue(res.rawCells()[2]), v1); - break; - } - } - // place a version delete marker (delete last version) - Delete d = new Delete(row); - d.addColumn(famName, row, t); - htable1.delete(d); - - get = new Get(row); - get.setMaxVersions(); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for put replication"); - } - Result res = htable2.get(get); - if (res.size() > 2) { - LOG.info("Version not deleted"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3); - assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2); - break; - } - } - - // place a column delete marker - d = new Delete(row); - d.addColumns(famName, row, t+2); - htable1.delete(d); - - // now *both* of the remaining version should be deleted - // at the replica - get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for del replication"); - } - Result res = htable2.get(get); - if (res.size() >= 1) { - LOG.info("Rows not deleted"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - /** - * Add a row, check it's replicated, delete it, check's gone - * @throws Exception - */ - @Test(timeout=300000) - public void testSimplePutDelete() throws Exception { - LOG.info("testSimplePutDelete"); - Put put = new Put(row); - put.addColumn(famName, row, row); - - htable1 = utility1.getConnection().getTable(tableName); - htable1.put(put); - - Get get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for put replication"); - } - Result res = htable2.get(get); - if (res.isEmpty()) { - LOG.info("Row not available"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(res.value(), row); - break; - } - } - - Delete del = new Delete(row); - htable1.delete(del); - - get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for del replication"); - } - Result res = htable2.get(get); - if (res.size() >= 1) { - LOG.info("Row not deleted"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - /** - * Try a small batch upload using the write buffer, check it's replicated - * @throws Exception - */ - @Test(timeout=300000) - public void testSmallBatch() throws Exception { - LOG.info("testSmallBatch"); - // normal Batch tests - loadData("", row); - - Scan scan = new Scan(); - - ResultScanner scanner1 = htable1.getScanner(scan); - Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH); - scanner1.close(); - assertEquals(NB_ROWS_IN_BATCH, res1.length); - - waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES); - } - - private void waitForReplication(int expectedRows, int retries) throws IOException, InterruptedException { - Scan scan; - for (int i = 0; i < retries; i++) { - scan = new Scan(); - if (i== retries -1) { - fail("Waited too much time for normal batch replication"); - } - ResultScanner scanner = htable2.getScanner(scan); - Result[] res = scanner.next(expectedRows); - scanner.close(); - if (res.length != expectedRows) { - LOG.info("Only got " + res.length + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - } - - private void loadData(String prefix, byte[] row) throws IOException { - List<Put> puts = new ArrayList<>(NB_ROWS_IN_BATCH); - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i))); - put.addColumn(famName, row, row); - puts.add(put); - } - htable1.put(puts); - } - - /** - * Test disable/enable replication, trying to insert, make sure nothing's - * replicated, enable it, the insert should be replicated - * - * @throws Exception - */ - @Test(timeout = 300000) - public void testDisableEnable() throws Exception { - - // Test disabling replication - admin.disablePeer(PEER_ID); - - byte[] rowkey = Bytes.toBytes("disable enable"); - Put put = new Put(rowkey); - put.addColumn(famName, row, row); - htable1.put(put); - - Get get = new Get(rowkey); - for (int i = 0; i < NB_RETRIES; i++) { - Result res = htable2.get(get); - if (res.size() >= 1) { - fail("Replication wasn't disabled"); - } else { - LOG.info("Row not replicated, let's wait a bit more..."); - Thread.sleep(SLEEP_TIME); - } - } - - // Test enable replication - admin.enablePeer(PEER_ID); - - for (int i = 0; i < NB_RETRIES; i++) { - Result res = htable2.get(get); - if (res.isEmpty()) { - LOG.info("Row not available"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(res.value(), row); - return; - } - } - fail("Waited too much time for put replication"); - } - - /** - * Integration test for TestReplicationAdmin, removes and re-add a peer - * cluster - * - * @throws Exception - */ - @Test(timeout=300000) - public void testAddAndRemoveClusters() throws Exception { - LOG.info("testAddAndRemoveClusters"); - admin.removePeer(PEER_ID); - Thread.sleep(SLEEP_TIME); - byte[] rowKey = Bytes.toBytes("Won't be replicated"); - Put put = new Put(rowKey); - put.addColumn(famName, row, row); - htable1.put(put); - - Get get = new Get(rowKey); - for (int i = 0; i < NB_RETRIES; i++) { - if (i == NB_RETRIES-1) { - break; - } - Result res = htable2.get(get); - if (res.size() >= 1) { - fail("Not supposed to be replicated"); - } else { - LOG.info("Row not replicated, let's wait a bit more..."); - Thread.sleep(SLEEP_TIME); - } - } - ReplicationPeerConfig rpc = new ReplicationPeerConfig(); - rpc.setClusterKey(utility2.getClusterKey()); - admin.addPeer(PEER_ID, rpc, null); - Thread.sleep(SLEEP_TIME); - rowKey = Bytes.toBytes("do rep"); - put = new Put(rowKey); - put.addColumn(famName, row, row); - LOG.info("Adding new row"); - htable1.put(put); - - get = new Get(rowKey); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for put replication"); - } - Result res = htable2.get(get); - if (res.isEmpty()) { - LOG.info("Row not available"); - Thread.sleep(SLEEP_TIME*i); - } else { - assertArrayEquals(res.value(), row); - break; - } - } - } - - - /** - * Do a more intense version testSmallBatch, one that will trigger - * wal rolling and other non-trivial code paths - * @throws Exception - */ - @Test(timeout=300000) - public void testLoading() throws Exception { - LOG.info("Writing out rows to table1 in testLoading"); - List<Put> puts = new ArrayList<>(NB_ROWS_IN_BIG_BATCH); - for (int i = 0; i < NB_ROWS_IN_BIG_BATCH; i++) { - Put put = new Put(Bytes.toBytes(i)); - put.addColumn(famName, row, row); - puts.add(put); - } - // The puts will be iterated through and flushed only when the buffer - // size is reached. - htable1.put(puts); - - Scan scan = new Scan(); - - ResultScanner scanner = htable1.getScanner(scan); - Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH); - scanner.close(); - - assertEquals(NB_ROWS_IN_BIG_BATCH, res.length); - - LOG.info("Looking in table2 for replicated rows in testLoading"); - long start = System.currentTimeMillis(); - // Retry more than NB_RETRIES. As it was, retries were done in 5 seconds and we'd fail - // sometimes. - final long retries = NB_RETRIES * 10; - for (int i = 0; i < retries; i++) { - scan = new Scan(); - scanner = htable2.getScanner(scan); - res = scanner.next(NB_ROWS_IN_BIG_BATCH); - scanner.close(); - if (res.length != NB_ROWS_IN_BIG_BATCH) { - if (i == retries - 1) { - int lastRow = -1; - for (Result result : res) { - int currentRow = Bytes.toInt(result.getRow()); - for (int row = lastRow+1; row < currentRow; row++) { - LOG.error("Row missing: " + row); - } - lastRow = currentRow; - } - LOG.error("Last row: " + lastRow); - fail("Waited too much time for normal batch replication, " + - res.length + " instead of " + NB_ROWS_IN_BIG_BATCH + "; waited=" + - (System.currentTimeMillis() - start) + "ms"); - } else { - LOG.info("Only got " + res.length + " rows... retrying"); - Thread.sleep(SLEEP_TIME); - } - } else { - break; - } - } - } - - /** - * Do a small loading into a table, make sure the data is really the same, - * then run the VerifyReplication job to check the results. Do a second - * comparison where all the cells are different. - * @throws Exception - */ - @Test(timeout=300000) - public void testVerifyRepJob() throws Exception { - // Populate the tables, at the same time it guarantees that the tables are - // identical since it does the check - testSmallBatch(); - - String[] args = new String[] {PEER_ID, tableName.getNameAsString()}; - runVerifyReplication(args, NB_ROWS_IN_BATCH, 0); - - Scan scan = new Scan(); - ResultScanner rs = htable2.getScanner(scan); - Put put = null; - for (Result result : rs) { - put = new Put(result.getRow()); - Cell firstVal = result.rawCells()[0]; - put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal), - Bytes.toBytes("diff data")); - htable2.put(put); - } - Delete delete = new Delete(put.getRow()); - htable2.delete(delete); - runVerifyReplication(args, 0, NB_ROWS_IN_BATCH); - } - - /** - * Load a row into a table, make sure the data is really the same, - * delete the row, make sure the delete marker is replicated, - * run verify replication with and without raw to check the results. - * @throws Exception - */ - @Test(timeout=300000) - public void testVerifyRepJobWithRawOptions() throws Exception { - LOG.info(name.getMethodName()); - - final TableName tableName = TableName.valueOf(name.getMethodName()); - byte[] familyname = Bytes.toBytes("fam_raw"); - byte[] row = Bytes.toBytes("row_raw"); - - Table lHtable1 = null; - Table lHtable2 = null; - - try { - HTableDescriptor table = new HTableDescriptor(tableName); - HColumnDescriptor fam = new HColumnDescriptor(familyname); - fam.setMaxVersions(100); - fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); - table.addFamily(fam); - scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for (HColumnDescriptor f : table.getColumnFamilies()) { - scopes.put(f.getName(), f.getScope()); - } - - Connection connection1 = ConnectionFactory.createConnection(conf1); - Connection connection2 = ConnectionFactory.createConnection(conf2); - try (Admin admin1 = connection1.getAdmin()) { - admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - } - try (Admin admin2 = connection2.getAdmin()) { - admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - } - utility1.waitUntilAllRegionsAssigned(tableName); - utility2.waitUntilAllRegionsAssigned(tableName); - - lHtable1 = utility1.getConnection().getTable(tableName); - lHtable2 = utility2.getConnection().getTable(tableName); - - Put put = new Put(row); - put.addColumn(familyname, row, row); - lHtable1.put(put); - - Get get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for put replication"); - } - Result res = lHtable2.get(get); - if (res.isEmpty()) { - LOG.info("Row not available"); - Thread.sleep(SLEEP_TIME); - } else { - assertArrayEquals(res.value(), row); - break; - } - } - - Delete del = new Delete(row); - lHtable1.delete(del); - - get = new Get(row); - for (int i = 0; i < NB_RETRIES; i++) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for del replication"); - } - Result res = lHtable2.get(get); - if (res.size() >= 1) { - LOG.info("Row not deleted"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - - // Checking verifyReplication for the default behavior. - String[] argsWithoutRaw = new String[] {PEER_ID, tableName.getNameAsString()}; - runVerifyReplication(argsWithoutRaw, 0, 0); - - // Checking verifyReplication with raw - String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID, tableName.getNameAsString()}; - runVerifyReplication(argsWithRawAsTrue, 1, 0); - } finally { - if (lHtable1 != null) { - lHtable1.close(); - } - if (lHtable2 != null) { - lHtable2.close(); - } - } - } - - private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows) - throws IOException, InterruptedException, ClassNotFoundException { - Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args); - if (job == null) { - fail("Job wasn't created, see the log"); - } - if (!job.waitForCompletion(true)) { - fail("Job failed, see the log"); - } - assertEquals(expectedGoodRows, job.getCounters(). - findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); - assertEquals(expectedBadRows, job.getCounters(). - findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); - } - - @Test(timeout=300000) - // VerifyReplication should honor versions option - public void testHBase14905() throws Exception { - // normal Batch tests - byte[] qualifierName = Bytes.toBytes("f1"); - Put put = new Put(Bytes.toBytes("r1")); - put.addColumn(famName, qualifierName, Bytes.toBytes("v1002")); - htable1.put(put); - put.addColumn(famName, qualifierName, Bytes.toBytes("v1001")); - htable1.put(put); - put.addColumn(famName, qualifierName, Bytes.toBytes("v1112")); - htable1.put(put); - - Scan scan = new Scan(); - scan.setMaxVersions(100); - ResultScanner scanner1 = htable1.getScanner(scan); - Result[] res1 = scanner1.next(1); - scanner1.close(); - - assertEquals(1, res1.length); - assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size()); - - for (int i = 0; i < NB_RETRIES; i++) { - scan = new Scan(); - scan.setMaxVersions(100); - scanner1 = htable2.getScanner(scan); - res1 = scanner1.next(1); - scanner1.close(); - if (res1.length != 1) { - LOG.info("Only got " + res1.length + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size(); - if (cellNumber != 3) { - LOG.info("Only got " + cellNumber + " cells"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - if (i == NB_RETRIES-1) { - fail("Waited too much time for normal batch replication"); - } - } - - put.addColumn(famName, qualifierName, Bytes.toBytes("v1111")); - htable2.put(put); - put.addColumn(famName, qualifierName, Bytes.toBytes("v1112")); - htable2.put(put); - - scan = new Scan(); - scan.setMaxVersions(100); - scanner1 = htable2.getScanner(scan); - res1 = scanner1.next(NB_ROWS_IN_BATCH); - scanner1.close(); - - assertEquals(1, res1.length); - assertEquals(5, res1[0].getColumnCells(famName, qualifierName).size()); - - String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()}; - runVerifyReplication(args, 0, 1); - } - - @Test(timeout=300000) - // VerifyReplication should honor versions option - public void testVersionMismatchHBase14905() throws Exception { - // normal Batch tests - byte[] qualifierName = Bytes.toBytes("f1"); - Put put = new Put(Bytes.toBytes("r1")); - long ts = System.currentTimeMillis(); - put.addColumn(famName, qualifierName, ts + 1, Bytes.toBytes("v1")); - htable1.put(put); - put.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v2")); - htable1.put(put); - put.addColumn(famName, qualifierName, ts + 3, Bytes.toBytes("v3")); - htable1.put(put); - - Scan scan = new Scan(); - scan.setMaxVersions(100); - ResultScanner scanner1 = htable1.getScanner(scan); - Result[] res1 = scanner1.next(1); - scanner1.close(); - - assertEquals(1, res1.length); - assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size()); - - for (int i = 0; i < NB_RETRIES; i++) { - scan = new Scan(); - scan.setMaxVersions(100); - scanner1 = htable2.getScanner(scan); - res1 = scanner1.next(1); - scanner1.close(); - if (res1.length != 1) { - LOG.info("Only got " + res1.length + " rows"); - Thread.sleep(SLEEP_TIME); - } else { - int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size(); - if (cellNumber != 3) { - LOG.info("Only got " + cellNumber + " cells"); - Thread.sleep(SLEEP_TIME); - } else { - break; - } - } - if (i == NB_RETRIES-1) { - fail("Waited too much time for normal batch replication"); - } - } - - try { - // Disabling replication and modifying the particular version of the cell to validate the feature. - admin.disablePeer(PEER_ID); - Put put2 = new Put(Bytes.toBytes("r1")); - put2.addColumn(famName, qualifierName, ts +2, Bytes.toBytes("v99")); - htable2.put(put2); - - scan = new Scan(); - scan.setMaxVersions(100); - scanner1 = htable2.getScanner(scan); - res1 = scanner1.next(NB_ROWS_IN_BATCH); - scanner1.close(); - assertEquals(1, res1.length); - assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size()); - - String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()}; - runVerifyReplication(args, 0, 1); - } - finally { - admin.enablePeer(PEER_ID); - } - } - - /** - * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out - * the compaction WALEdit - * @throws Exception - */ - @Test(timeout=300000) - public void testCompactionWALEdits() throws Exception { - WALProtos.CompactionDescriptor compactionDescriptor = - WALProtos.CompactionDescriptor.getDefaultInstance(); - HRegionInfo hri = new HRegionInfo(htable1.getName(), - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); - WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor); - Replication.scopeWALEdits(new WALKey(), edit, - htable1.getConfiguration(), null); - } - - /** - * Test for HBASE-8663 - * Create two new Tables with colfamilies enabled for replication then run - * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note: - * TestReplicationAdmin is a better place for this testing but it would need mocks. - * @throws Exception - */ - @Test(timeout = 300000) - public void testVerifyListReplicatedTable() throws Exception { - LOG.info("testVerifyListReplicatedTable"); - - final String tName = "VerifyListReplicated_"; - final String colFam = "cf1"; - final int numOfTables = 3; - - Admin hadmin = utility1.getAdmin(); - - // Create Tables - for (int i = 0; i < numOfTables; i++) { - HTableDescriptor ht = new HTableDescriptor(TableName.valueOf(tName + i)); - HColumnDescriptor cfd = new HColumnDescriptor(colFam); - cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); - ht.addFamily(cfd); - hadmin.createTable(ht); - } - - // verify the result - List<HashMap<String, String>> replicationColFams = admin.listReplicated(); - int[] match = new int[numOfTables]; // array of 3 with init value of zero - - for (int i = 0; i < replicationColFams.size(); i++) { - HashMap<String, String> replicationEntry = replicationColFams.get(i); - String tn = replicationEntry.get(ReplicationAdmin.TNAME); - if ((tn.startsWith(tName)) && replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) { - int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit - match[m]++; // should only increase once - } - } - - // check the matching result - for (int i = 0; i < match.length; i++) { - assertTrue("listReplicated() does not match table " + i, (match[i] == 1)); - } - - // drop tables - for (int i = 0; i < numOfTables; i++) { - TableName tableName = TableName.valueOf(tName + i); - hadmin.disableTable(tableName); - hadmin.deleteTable(tableName); - } - - hadmin.close(); - } - - /** - * Test for HBase-15259 WALEdits under replay will also be replicated - * */ - @Test - public void testReplicationInReplay() throws Exception { - final TableName tableName = htable1.getName(); - - HRegion region = utility1.getMiniHBaseCluster().getRegions(tableName).get(0); - HRegionInfo hri = region.getRegionInfo(); - NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for (byte[] fam : htable1.getTableDescriptor().getFamiliesKeys()) { - scopes.put(fam, 1); - } - final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); - int index = utility1.getMiniHBaseCluster().getServerWith(hri.getRegionName()); - WAL wal = utility1.getMiniHBaseCluster().getRegionServer(index).getWAL(region.getRegionInfo()); - final byte[] rowName = Bytes.toBytes("testReplicationInReplay"); - final byte[] qualifier = Bytes.toBytes("q"); - final byte[] value = Bytes.toBytes("v"); - WALEdit edit = new WALEdit(true); - long now = EnvironmentEdgeManager.currentTime(); - edit.add(new KeyValue(rowName, famName, qualifier, - now, value)); - WALKey walKey = new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes); - wal.append(hri, walKey, edit, true); - wal.sync(); - - Get get = new Get(rowName); - for (int i = 0; i < NB_RETRIES; i++) { - if (i == NB_RETRIES-1) { - break; - } - Result res = htable2.get(get); - if (res.size() >= 1) { - fail("Not supposed to be replicated for " + Bytes.toString(res.getRow())); - } else { - LOG.info("Row not replicated, let's wait a bit more..."); - Thread.sleep(SLEEP_TIME); - } - } - } - - @Test(timeout=300000) - public void testVerifyReplicationPrefixFiltering() throws Exception { - final byte[] prefixRow = Bytes.toBytes("prefixrow"); - final byte[] prefixRow2 = Bytes.toBytes("secondrow"); - loadData("prefixrow", prefixRow); - loadData("secondrow", prefixRow2); - loadData("aaa", row); - loadData("zzz", row); - waitForReplication(NB_ROWS_IN_BATCH * 4, NB_RETRIES * 4); - String[] args = new String[] {"--row-prefixes=prefixrow,secondrow", PEER_ID, - tableName.getNameAsString()}; - runVerifyReplication(args, NB_ROWS_IN_BATCH *2, 0); - } - - @Test(timeout = 300000) - public void testVerifyReplicationSnapshotArguments() { - String[] args = - new String[] { "--sourceSnapshotName=snapshot1", "2", tableName.getNameAsString() }; - assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--sourceSnapshotTmpDir=tmp", "2", tableName.getNameAsString() }; - assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=tmp", "2", - tableName.getNameAsString() }; - assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--peerSnapshotName=snapshot1", "2", tableName.getNameAsString() }; - assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--peerSnapshotTmpDir=/tmp/", "2", tableName.getNameAsString() }; - assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--peerSnapshotName=snapshot1", "--peerSnapshotTmpDir=/tmp/", - "--peerFSAddress=tempfs", "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", - tableName.getNameAsString() }; - assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - - args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=/tmp/", - "--peerSnapshotName=snapshot2", "--peerSnapshotTmpDir=/tmp/", "--peerFSAddress=tempfs", - "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", tableName.getNameAsString() }; - - assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args)); - } - - @Test(timeout = 300000) - public void testVerifyReplicationWithSnapshotSupport() throws Exception { - // Populate the tables, at the same time it guarantees that the tables are - // identical since it does the check - testSmallBatch(); - - // Take source and target tables snapshot - Path rootDir = FSUtils.getRootDir(conf1); - FileSystem fs = rootDir.getFileSystem(conf1); - String sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis(); - SnapshotTestingUtils.createSnapshotAndValidate(utility1.getHBaseAdmin(), tableName, - new String(famName), sourceSnapshotName, rootDir, fs, true); - - // Take target snapshot - Path peerRootDir = FSUtils.getRootDir(conf2); - FileSystem peerFs = peerRootDir.getFileSystem(conf2); - String peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis(); - SnapshotTestingUtils.createSnapshotAndValidate(utility2.getHBaseAdmin(), tableName, - new String(famName), peerSnapshotName, peerRootDir, peerFs, true); - - String peerFSAddress = peerFs.getUri().toString(); - String temPath1 = utility1.getRandomDir().toString(); - String temPath2 = "/tmp2"; - - String[] args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName, - "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName, - "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress, - "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() }; - - Job job = new VerifyReplication().createSubmittableJob(conf1, args); - if (job == null) { - fail("Job wasn't created, see the log"); - } - if (!job.waitForCompletion(true)) { - fail("Job failed, see the log"); - } - assertEquals(NB_ROWS_IN_BATCH, - job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); - assertEquals(0, - job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); - - Scan scan = new Scan(); - ResultScanner rs = htable2.getScanner(scan); - Put put = null; - for (Result result : rs) { - put = new Put(result.getRow()); - Cell firstVal = result.rawCells()[0]; - put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal), - Bytes.toBytes("diff data")); - htable2.put(put); - } - Delete delete = new Delete(put.getRow()); - htable2.delete(delete); - - sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis(); - SnapshotTestingUtils.createSnapshotAndValidate(utility1.getHBaseAdmin(), tableName, - new String(famName), sourceSnapshotName, rootDir, fs, true); - - peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis(); - SnapshotTestingUtils.createSnapshotAndValidate(utility2.getHBaseAdmin(), tableName, - new String(famName), peerSnapshotName, peerRootDir, peerFs, true); - - args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName, - "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName, - "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress, - "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() }; - - job = new VerifyReplication().createSubmittableJob(conf1, args); - if (job == null) { - fail("Job wasn't created, see the log"); - } - if (!job.waitForCompletion(true)) { - fail("Job failed, see the log"); - } - assertEquals(0, - job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); - assertEquals(NB_ROWS_IN_BATCH, - job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); - } - - @Test - public void testEmptyWALRecovery() throws Exception { - final int numRs = utility1.getHBaseCluster().getRegionServerThreads().size(); - - // for each RS, create an empty wal with same walGroupId - final List<Path> emptyWalPaths = new ArrayList<>(); - long ts = System.currentTimeMillis(); - for (int i = 0; i < numRs; i++) { - HRegionInfo regionInfo = - utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo(); - WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo); - Path currentWalPath = AbstractFSWALProvider.getCurrentFileName(wal); - String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(currentWalPath.getName()); - Path emptyWalPath = new Path(utility1.getDataTestDir(), walGroupId + "." + ts); - utility1.getTestFileSystem().create(emptyWalPath).close(); - emptyWalPaths.add(emptyWalPath); - } - - // inject our empty wal into the replication queue - for (int i = 0; i < numRs; i++) { - Replication replicationService = - (Replication) utility1.getHBaseCluster().getRegionServer(i).getReplicationSourceService(); - replicationService.preLogRoll(null, emptyWalPaths.get(i)); - replicationService.postLogRoll(null, emptyWalPaths.get(i)); - } - - // wait for ReplicationSource to start reading from our empty wal - waitForLogAdvance(numRs, emptyWalPaths, false); - - // roll the original wal, which enqueues a new wal behind our empty wal - for (int i = 0; i < numRs; i++) { - HRegionInfo regionInfo = - utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo(); - WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo); - wal.rollWriter(true); - } - - // ReplicationSource should advance past the empty wal, or else the test will fail - waitForLogAdvance(numRs, emptyWalPaths, true); - - // we're now writing to the new wal - // if everything works, the source should've stopped reading from the empty wal, and start - // replicating from the new wal - testSimplePutDelete(); - } - - /** - * Waits for the ReplicationSource to start reading from the given paths - * @param numRs number of regionservers - * @param emptyWalPaths path for each regionserver - * @param invert if true, waits until ReplicationSource is NOT reading from the given paths - */ - private void waitForLogAdvance(final int numRs, final List<Path> emptyWalPaths, - final boolean invert) throws Exception { - Waiter.waitFor(conf1, 10000, new Waiter.Predicate<Exception>() { - @Override - public boolean evaluate() throws Exception { - for (int i = 0; i < numRs; i++) { - Replication replicationService = (Replication) utility1.getHBaseCluster() - .getRegionServer(i).getReplicationSourceService(); - for (ReplicationSourceInterface rsi : replicationService.getReplicationManager() - .getSources()) { - ReplicationSource source = (ReplicationSource) rsi; - if (!invert && !emptyWalPaths.get(i).equals(source.getCurrentPath())) { - return false; - } - if (invert && emptyWalPaths.get(i).equals(source.getCurrentPath())) { - return false; - } - } - } - return true; - } - }); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java index 07bb2b7..94991e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java @@ -17,15 +17,22 @@ */ package org.apache.hadoop.hbase.security; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.classification.InterfaceAudience; - import org.apache.hadoop.hbase.shaded.com.google.common.base.Strings; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.net.InetAddress; @InterfaceAudience.Private public class HBaseKerberosUtils { + private static final Log LOG = LogFactory.getLog(HBaseKerberosUtils.class); + public static final String KRB_PRINCIPAL = "hbase.regionserver.kerberos.principal"; public static final String MASTER_KRB_PRINCIPAL = "hbase.master.kerberos.principal"; public static final String KRB_KEYTAB_FILE = "hbase.regionserver.keytab.file"; @@ -81,4 +88,21 @@ public class HBaseKerberosUtils { conf.set(KRB_PRINCIPAL, System.getProperty(KRB_PRINCIPAL)); conf.set(MASTER_KRB_PRINCIPAL, System.getProperty(KRB_PRINCIPAL)); } + + public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username) + throws IOException { + String hostname = InetAddress.getLocalHost().getHostName(); + String keyTabFileConfKey = "hbase." + username + ".keytab.file"; + String keyTabFileLocation = conf.get(keyTabFileConfKey); + String principalConfKey = "hbase." + username + ".kerberos.principal"; + String principal = org.apache.hadoop.security.SecurityUtil + .getServerPrincipal(conf.get(principalConfKey), hostname); + if (keyTabFileLocation == null || principal == null) { + LOG.warn("Principal or key tab file null for : " + principalConfKey + ", " + + keyTabFileConfKey); + } + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation); + return ugi; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java deleted file mode 100644 index 2e3cb5e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ /dev/null @@ -1,381 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import static org.apache.hadoop.util.ToolRunner.run; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.junit.rules.TestRule; - -/** - * Test Export Snapshot Tool - */ -@Ignore -@Category({VerySlowMapReduceTests.class, LargeTests.class}) -public class TestExportSnapshot { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - private static final Log LOG = LogFactory.getLog(TestExportSnapshot.class); - - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - protected final static byte[] FAMILY = Bytes.toBytes("cf"); - - @Rule - public final TestName testName = new TestName(); - - protected TableName tableName; - private byte[] emptySnapshotName; - private byte[] snapshotName; - private int tableNumFiles; - private Admin admin; - - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean("hbase.master.enabletable.roundrobin", true); - conf.setInt("mapreduce.map.maxattempts", 10); - // If a single node has enough failures (default 3), resource manager will blacklist it. - // With only 2 nodes and tests injecting faults, we don't want that. - conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(1, 3); - TEST_UTIL.startMiniMapReduceCluster(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * Create a table and take a snapshot of the table used by the export test. - */ - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getAdmin(); - - tableName = TableName.valueOf("testtb-" + testName.getMethodName()); - snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); - emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); - - // create Table - createTable(); - - // Take an empty snapshot - admin.snapshot(emptySnapshotName, tableName); - - // Add some rows - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - tableNumFiles = admin.getTableRegions(tableName).size(); - - // take a snapshot - admin.snapshot(snapshotName, tableName); - } - - protected void createTable() throws Exception { - SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); - } - - protected interface RegionPredicate { - boolean evaluate(final HRegionInfo regionInfo); - } - - protected RegionPredicate getBypassRegionPredicate() { - return null; - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(tableName); - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - } - - /** - * Verify if exported snapshot and copied files matches the original one. - */ - @Test - public void testExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } - - @Test - public void testExportFileSystemStateWithSkipTmp() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); - try { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } finally { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); - } - } - - @Test - public void testEmptyExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); - } - - @Test - public void testConsecutiveExports() throws Exception { - Path copyDir = getLocalDestinationDir(); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); - removeExportDir(copyDir); - } - - @Test - public void testExportWithTargetName() throws Exception { - final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); - testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); - } - - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected) throws Exception { - testExportFileSystemState(tableName, snapshotName, targetName, - filesExpected, getHdfsDestinationDir(), false); - } - - protected void testExportFileSystemState(final TableName tableName, - final byte[] snapshotName, final byte[] targetName, int filesExpected, - Path copyDir, boolean overwrite) throws Exception { - testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, - filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, - overwrite, getBypassRegionPredicate(), true); - } - - /** - * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. - */ - protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, - final byte[] snapshotName, final byte[] targetName, final int filesExpected, - final Path sourceDir, Path copyDir, final boolean overwrite, - final RegionPredicate bypassregionPredicate, boolean success) throws Exception { - URI hdfsUri = FileSystem.get(conf).getUri(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - - List<String> opts = new ArrayList<>(); - opts.add("--snapshot"); - opts.add(Bytes.toString(snapshotName)); - opts.add("--copy-to"); - opts.add(copyDir.toString()); - if (targetName != snapshotName) { - opts.add("--target"); - opts.add(Bytes.toString(targetName)); - } - if (overwrite) opts.add("--overwrite"); - - // Export Snapshot - int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); - assertEquals(success ? 0 : 1, res); - if (!success) { - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - assertFalse(fs.exists(new Path(copyDir, targetDir))); - return; - } - - // Verify File-System state - FileStatus[] rootFiles = fs.listStatus(copyDir); - assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); - for (FileStatus fileStatus: rootFiles) { - String name = fileStatus.getPath().getName(); - assertTrue(fileStatus.isDirectory()); - assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || - name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); - } - - // compare the snapshot metadata and verify the hfiles - final FileSystem hdfs = FileSystem.get(hdfsUri, conf); - final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), - fs, new Path(copyDir, targetDir)); - Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, - Bytes.toString(targetName), bypassregionPredicate); - assertEquals(filesExpected, snapshotFiles.size()); - } - - /** - * Check that ExportSnapshot will succeed if something fails but the retry succeed. - */ - @Test - public void testExportRetry() throws Exception { - Path copyDir = getLocalDestinationDir(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); - conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); - conf.setInt("mapreduce.map.maxattempts", 3); - testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, - TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), true); - } - - /** - * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. - */ - @Test - public void testExportFailure() throws Exception { - Path copyDir = getLocalDestinationDir(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); - conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); - conf.setInt("mapreduce.map.maxattempts", 3); - testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, - TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), false); - } - - /* - * verify if the snapshot folder on file-system 1 match the one on file-system 2 - */ - protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, - final FileSystem fs2, final Path root2) throws IOException { - assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); - } - - protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, - final TableName tableName, final String snapshotName) throws IOException { - return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, - snapshotName, getBypassRegionPredicate()); - } - - /* - * Verify if the files exists - */ - protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, - final Path rootDir, final TableName tableName, final String snapshotName, - final RegionPredicate bypassregionPredicate) throws IOException { - final Path exportedSnapshot = new Path(rootDir, - new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); - final Set<String> snapshotFiles = new HashSet<>(); - final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override - public void storeFile(final HRegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) - return; - - String hfile = storeFile.getName(); - snapshotFiles.add(hfile); - if (storeFile.hasReference()) { - // Nothing to do here, we have already the reference embedded - } else { - verifyNonEmptyFile(new Path(exportedArchive, - new Path(FSUtils.getTableDir(new Path("./"), tableName), - new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); - } - } - - private void verifyNonEmptyFile(final Path path) throws IOException { - assertTrue(path + " should exists", fs.exists(path)); - assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); - } - }); - - // Verify Snapshot description - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); - assertTrue(desc.getName().equals(snapshotName)); - assertTrue(desc.getTable().equals(tableName.getNameAsString())); - return snapshotFiles; - } - - private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) - throws IOException { - Set<String> files = new HashSet<>(); - int rootPrefix = root.makeQualified(fs).toString().length(); - FileStatus[] list = FSUtils.listStatus(fs, dir); - if (list != null) { - for (FileStatus fstat: list) { - LOG.debug(fstat.getPath()); - if (fstat.isDirectory()) { - files.addAll(listFiles(fs, root, fstat.getPath())); - } else { - files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); - } - } - } - return files; - } - - private Path getHdfsDestinationDir() { - Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis()); - LOG.info("HDFS export destination path: " + path); - return path; - } - - private Path getLocalDestinationDir() { - Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis()); - LOG.info("Local export destination path: " + path); - return path; - } - - private static void removeExportDir(final Path path) throws IOException { - FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); - fs.delete(path, true); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java deleted file mode 100644 index e31e81e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test Export Snapshot Tool helpers - */ -@Category({RegionServerTests.class, SmallTests.class}) -public class TestExportSnapshotHelpers { - /** - * Verfy the result of getBalanceSplits() method. - * The result are groups of files, used as input list for the "export" mappers. - * All the groups should have similar amount of data. - * - * The input list is a pair of file path and length. - * The getBalanceSplits() function sort it by length, - * and assign to each group a file, going back and forth through the groups. - */ - @Test - public void testBalanceSplit() throws Exception { - // Create a list of files - List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<>(21); - for (long i = 0; i <= 20; i++) { - SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() - .setType(SnapshotFileInfo.Type.HFILE) - .setHfile("file-" + i) - .build(); - files.add(new Pair<>(fileInfo, i)); - } - - // Create 5 groups (total size 210) - // group 0: 20, 11, 10, 1 (total size: 42) - // group 1: 19, 12, 9, 2 (total size: 42) - // group 2: 18, 13, 8, 3 (total size: 42) - // group 3: 17, 12, 7, 4 (total size: 42) - // group 4: 16, 11, 6, 5 (total size: 42) - List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5); - assertEquals(5, splits.size()); - - String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; - verifyBalanceSplit(splits.get(0), split0, 42); - String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; - verifyBalanceSplit(splits.get(1), split1, 42); - String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; - verifyBalanceSplit(splits.get(2), split2, 42); - String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; - verifyBalanceSplit(splits.get(3), split3, 42); - String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; - verifyBalanceSplit(splits.get(4), split4, 42); - } - - private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split, - final String[] expected, final long expectedSize) { - assertEquals(expected.length, split.size()); - long totalSize = 0; - for (int i = 0; i < expected.length; ++i) { - Pair<SnapshotFileInfo, Long> fileInfo = split.get(i); - assertEquals(expected[i], fileInfo.getFirst().getHfile()); - totalSize += fileInfo.getSecond(); - } - assertEquals(expectedSize, totalSize); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java deleted file mode 100644 index 00778502..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Test Export Snapshot Tool - */ -@Category({MapReduceTests.class, MediumTests.class}) -public class TestExportSnapshotNoCluster { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); - - protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); - - private static FileSystem fs; - private static Path testDir; - - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean("hbase.master.enabletable.roundrobin", true); - conf.setInt("mapreduce.map.maxattempts", 10); - conf.set(HConstants.HBASE_DIR, testDir.toString()); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - testDir = TEST_UTIL.getDataTestDir(); - fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); - - setUpBaseConf(TEST_UTIL.getConfiguration()); - } - - /** - * Mock a snapshot with files in the archive dir, - * two regions, and one reference file. - */ - @Test - public void testSnapshotWithRefsExportFileSystemState() throws Exception { - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", - "tableWithRefsV1"); - testSnapshotWithRefsExportFileSystemState(builder); - - snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); - builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); - testSnapshotWithRefsExportFileSystemState(builder); - } - - /** - * Generates a couple of regions for the specified SnapshotMock, - * and then it will run the export and verification. - */ - private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) - throws Exception { - Path[] r1Files = builder.addRegion(); - Path[] r2Files = builder.addRegion(); - builder.commit(); - int snapshotFilesCount = r1Files.length + r2Files.length; - - byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); - TableName tableName = builder.getTableDescriptor().getTableName(); - TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), - tableName, snapshotName, snapshotName, snapshotFilesCount, - testDir, getDestinationDir(), false, null, true); - } - - private Path getDestinationDir() { - Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); - LOG.info("HDFS export destination path: " + path); - return path; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java deleted file mode 100644 index 7407a7d..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; - -/** - * Test Export Snapshot Tool - */ -@Ignore -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestMobExportSnapshot extends TestExportSnapshot { - - public static void setUpBaseConf(Configuration conf) { - TestExportSnapshot.setUpBaseConf(conf); - conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - } - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - TEST_UTIL.startMiniCluster(1, 3); - TEST_UTIL.startMiniMapReduceCluster(); - } - - @Override - protected void createTable() throws Exception { - MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); - } - - @Override - protected RegionPredicate getBypassRegionPredicate() { - return new RegionPredicate() { - @Override - public boolean evaluate(final HRegionInfo regionInfo) { - return MobUtils.isMobRegionInfo(regionInfo); - } - }; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java deleted file mode 100644 index 98d03c0..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; - -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; - -/** - * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode. - */ -@Ignore -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - // Setup separate test-data directory for MR cluster and set corresponding configurations. - // Otherwise, different test classes running MR cluster can step on each other. - TEST_UTIL.getDataTestDir(); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(1, 3); - TEST_UTIL.startMiniMapReduceCluster(); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java deleted file mode 100644 index 7d4832c..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.security.access.SecureTestUtil; - -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Reruns TestExportSnapshot using ExportSnapshot in secure mode. - */ -@Ignore -@Category({VerySlowRegionServerTests.class, LargeTests.class}) -public class TestSecureExportSnapshot extends TestExportSnapshot { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - setUpBaseConf(TEST_UTIL.getConfiguration()); - // Setup separate test-data directory for MR cluster and set corresponding configurations. - // Otherwise, different test classes running MR cluster can step on each other. - TEST_UTIL.getDataTestDir(); - - // set the always on security provider - UserProvider.setUserProviderForTesting(TEST_UTIL.getConfiguration(), - HadoopSecurityEnabledUserProviderForTesting.class); - - // setup configuration - SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); - - TEST_UTIL.startMiniCluster(1, 3); - TEST_UTIL.startMiniMapReduceCluster(); - - // Wait for the ACL table to become available - TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java index 236994a..0487bf4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -29,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; @@ -50,6 +53,17 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; */ public class HFileTestUtil { + public static final String OPT_DATA_BLOCK_ENCODING_USAGE = + "Encoding algorithm (e.g. prefix " + + "compression) to use for data blocks in the test column family, " + + "one of " + Arrays.toString(DataBlockEncoding.values()) + "."; + public static final String OPT_DATA_BLOCK_ENCODING = + HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT); + /** Column family used by the test */ + public static byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("test_cf"); + /** Column families used by the test */ + public static final byte[][] DEFAULT_COLUMN_FAMILIES = { DEFAULT_COLUMN_FAMILY }; + /** * Create an HFile with the given number of rows between a given * start key and end key @ family:qualifier. The value will be the key value. http://git-wip-us.apache.org/repos/asf/hbase/blob/664b6be0/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java index 2ea01bb..0b3c612 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestDataGeneratorWithTags.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.MultiThreadedAction.DefaultDataGenerator; +import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; @InterfaceAudience.Private public class LoadTestDataGeneratorWithTags extends DefaultDataGenerator { @@ -74,7 +75,7 @@ public class LoadTestDataGeneratorWithTags extends DefaultDataGenerator { List<Tag> tags; for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); - byte[] tag = LoadTestTool.generateData(random, + byte[] tag = LoadTestDataGenerator.generateData(random, minTagLength + random.nextInt(maxTagLength - minTagLength)); tags = new ArrayList<>(); for (int n = 0; n < numTags; n++) {
