HBASE-16946 Provide Raw scan as an option in VerifyReplication (Sreekar
Pallapothu)
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Amending-Author: Andrew Purtell <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4cdd0adf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4cdd0adf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4cdd0adf
Branch: refs/heads/0.98
Commit: 4cdd0adfbdfb8b4c28b7e226b88fb69d0e4e404f
Parents: e2df4b8
Author: tedyu <[email protected]>
Authored: Fri Nov 4 03:36:17 2016 -0700
Committer: Andrew Purtell <[email protected]>
Committed: Fri Dec 2 16:04:29 2016 -0800
----------------------------------------------------------------------
.../replication/VerifyReplication.java | 13 +++
.../replication/TestReplicationSmallTests.java | 102 ++++++++++++++++++-
2 files changed, 112 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 1ec31a1..b22b11e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -84,6 +84,7 @@ public class VerifyReplication extends Configured implements
Tool {
static String peerId = null;
static String rowPrefixes = null;
static boolean verbose = false;
+ static boolean includeDeletedCells = false;
/**
* Map-only comparator for 2 tables
@@ -126,6 +127,8 @@ public class VerifyReplication extends Configured
implements Tool {
scan.addFamily(Bytes.toBytes(fam));
}
}
+ boolean includeDeletedCells = conf.getBoolean(NAME +
".includeDeletedCells", false);
+ scan.setRaw(includeDeletedCells);
String rowPrefixes = conf.get(NAME + ".rowPrefixes", null);
setRowPrefixFilter(scan, rowPrefixes);
scan.setTimeRange(startTime, endTime);
@@ -267,6 +270,7 @@ public class VerifyReplication extends Configured
implements Tool {
conf.setLong(NAME+".startTime", startTime);
conf.setLong(NAME+".endTime", endTime);
conf.setBoolean(NAME +".verbose", verbose);
+ conf.setBoolean(NAME +".includeDeletedCells", includeDeletedCells);
if (families != null) {
conf.set(NAME+".families", families);
}
@@ -291,6 +295,7 @@ public class VerifyReplication extends Configured
implements Tool {
Scan scan = new Scan();
scan.setTimeRange(startTime, endTime);
+ scan.setRaw(includeDeletedCells);
if (versions >= 0) {
scan.setMaxVersions(versions);
LOG.info("Number of versions set to " + versions);
@@ -368,6 +373,12 @@ public class VerifyReplication extends Configured
implements Tool {
continue;
}
+ final String includeDeletedCellsArgKey = "--raw";
+ if (cmd.equals(includeDeletedCellsArgKey)) {
+ includeDeletedCells = true;
+ continue;
+ }
+
final String versionsArgKey = "--versions=";
if (cmd.startsWith(versionsArgKey)) {
versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
@@ -433,6 +444,7 @@ public class VerifyReplication extends Configured
implements Tool {
families = null;
peerId = null;
rowPrefixes = null;
+ includeDeletedCells = false;
}
/*
@@ -451,6 +463,7 @@ public class VerifyReplication extends Configured
implements Tool {
System.err.println(" without endtime means from starttime to
forever");
System.err.println(" endtime end of the time range");
System.err.println(" versions number of cell versions to verify");
+ System.err.println(" raw includes raw scan if given in options");
System.err.println(" families comma-separated list of families to
copy");
System.err.println(" row-prefixes comma-separated list of row key prefixes
to filter on ");
System.err.println(" delimiter the delimiter used in display around
rowkey");
http://git-wip-us.apache.org/repos/asf/hbase/blob/4cdd0adf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 5df28a5..6ddc209 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -33,18 +33,18 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -486,6 +486,102 @@ public class TestReplicationSmallTests extends
TestReplicationBase {
runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
}
+ /**
+ * Load a row into a table, make sure the data is really the same,
+ * delete the row, make sure the delete marker is replicated,
+ * run verify replication with and without raw to check the results.
+ * @throws Exception
+ */
+ @Test(timeout=300000)
+ public void testVerifyRepJobWithRawOptions() throws Exception {
+ LOG.info("testVerifyRepJobWithRawOptions");
+
+ TableName tablename = TableName.valueOf("test_raw");
+ byte[] familyname = Bytes.toBytes("fam_raw");
+ byte[] row = Bytes.toBytes("row_raw");
+
+ HTableInterface lHtable1 = null;
+ HTableInterface lHtable2 = null;
+
+ try {
+ HTableDescriptor table = new HTableDescriptor(tablename);
+ HColumnDescriptor fam = new HColumnDescriptor(familyname);
+ fam.setMaxVersions(100);
+ fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+ table.addFamily(fam);
+
+ HBaseAdmin admin1 = new HBaseAdmin(conf1);
+ try {
+ admin1.createTable(table,
HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ } finally {
+ admin1.close();
+ }
+ utility1.waitUntilAllRegionsAssigned(tablename);
+
+ HBaseAdmin admin2 = new HBaseAdmin(conf2);
+ try {
+ admin2.createTable(table,
HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ } finally {
+ admin2.close();
+ }
+ utility2.waitUntilAllRegionsAssigned(tablename);
+
+ lHtable1 =
HConnectionManager.createConnection(conf1).getTable(tablename);
+ lHtable2 =
HConnectionManager.createConnection(conf2).getTable(tablename);
+
+ Put put = new Put(row);
+ put.add(familyname, row, row);
+ lHtable1.put(put);
+
+ Get get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i==NB_RETRIES-1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.size() == 0) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(res.value(), row);
+ break;
+ }
+ }
+
+ Delete del = new Delete(row);
+ lHtable1.delete(del);
+
+ get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i==NB_RETRIES-1) {
+ fail("Waited too much time for del replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.size() >= 1) {
+ LOG.info("Row not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+
+ // Checking verifyReplication for the default behavior.
+ String[] argsWithoutRaw = new String[] {PEER_ID,
tablename.getNameAsString()};
+ runVerifyReplication(argsWithoutRaw, 0, 0);
+
+ // Checking verifyReplication with raw
+ String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID,
tablename.getNameAsString()};
+ runVerifyReplication(argsWithRawAsTrue, 1, 0);
+ } finally {
+ if (lHtable1 != null) {
+ lHtable1.close();
+ }
+ if (lHtable2 != null) {
+ lHtable2.close();
+ }
+ }
+ }
+
private void runVerifyReplication(String[] args, int expectedGoodRows, int
expectedBadRows)
throws IOException, InterruptedException, ClassNotFoundException {
Job job = VerifyReplication.createSubmittableJob(new
Configuration(CONF_WITH_LOCALFS), args);
