Repository: hbase Updated Branches: refs/heads/HBASE-7912 a402006de -> 486fc54c6
HBASE-15861 Add support for table sets in restore operation (Vlad) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/486fc54c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/486fc54c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/486fc54c Branch: refs/heads/HBASE-7912 Commit: 486fc54c6c36eb488248c7e41bd5235f825c83dd Parents: a402006 Author: tedyu <yuzhih...@gmail.com> Authored: Tue May 31 21:04:33 2016 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Tue May 31 21:04:33 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/backup/RestoreDriver.java | 62 +++++++-- .../hadoop/hbase/backup/TestFullBackupSet.java | 22 +++- .../backup/TestFullBackupSetRestoreSet.java | 127 +++++++++++++++++++ 3 files changed, 198 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java index 83c8297..0dba079 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java @@ -18,14 +18,20 @@ package org.apache.hadoop.hbase.backup; import java.io.IOException; +import java.util.List; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.backup.util.BackupServerUtil; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.LogUtils; import org.apache.hadoop.util.ToolRunner; @@ -40,9 +46,12 @@ public class RestoreDriver extends AbstractHBaseTool { private static final String OPTION_OVERWRITE = "overwrite"; private static final String OPTION_CHECK = "check"; private static final String OPTION_AUTOMATIC = "automatic"; + private static final String OPTION_SET = "set"; + private static final String OPTION_DEBUG = "debug"; + private static final String USAGE = - "Usage: hbase restore <backup_root_path> <backup_id> <tables> [tableMapping] \n" + "Usage: hbase restore [-set set_name] <backup_root_path> <backup_id> <tables> [tableMapping] \n" + " [-overwrite] [-check] [-automatic]\n" + " backup_root_path The parent location where the backup images are stored\n" + " backup_id The id identifying the backup image\n" @@ -61,7 +70,8 @@ public class RestoreDriver extends AbstractHBaseTool { + " The restore dependencies can be checked by using \"-check\" " + "option,\n" + " or using \"hbase backup describe\" command. Without this option, " - + "only\n" + " this backup image is restored\n"; + + "only\n" + " this backup image is restored\n" + + " -set set_name Backup set to restore, mutually exclusive with table list <tables>."; protected RestoreDriver() throws IOException @@ -75,7 +85,8 @@ public class RestoreDriver extends AbstractHBaseTool { "Overwrite the data if any of the restore target tables exists"); addOptNoArg(OPTION_CHECK, "Check restore sequence and dependencies"); addOptNoArg(OPTION_AUTOMATIC, "Restore all dependencies"); - addOptNoArg("debug", "Enable debug logging"); + addOptNoArg(OPTION_DEBUG, "Enable debug logging"); + addOptWithArg(OPTION_SET, "Backup set name"); // disable irrelevant loggers to avoid it mess up command output LogUtils.disableUselessLoggers(LOG); @@ -85,7 +96,7 @@ public class RestoreDriver extends AbstractHBaseTool { // enable debug logging Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup"); - if (cmd.hasOption("debug")) { + if (cmd.hasOption(OPTION_DEBUG)) { backupClientLogger.setLevel(Level.DEBUG); } @@ -112,16 +123,36 @@ public class RestoreDriver extends AbstractHBaseTool { // parse main restore command options String[] remainArgs = cmd.getArgs(); - if (remainArgs.length < 3) { + if (remainArgs.length < 3 && !cmd.hasOption(OPTION_SET) || + (cmd.hasOption(OPTION_SET) && remainArgs.length < 2)) { + System.out.println("ERROR: remain args length="+ remainArgs.length); System.out.println(USAGE); return -1; - } + } String backupRootDir = remainArgs[0]; String backupId = remainArgs[1]; - String tables = remainArgs[2]; - - String tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null; + String tables = null; + String tableMapping = null; + // Check backup set + if (cmd.hasOption(OPTION_SET)) { + String setName = cmd.getOptionValue(OPTION_SET); + try{ + tables = getTablesForSet(setName, conf); + } catch(IOException e){ + System.out.println("ERROR: "+ e.getMessage()+" for setName="+setName); + return -2; + } + if (tables == null) { + System.out.println("ERROR: Backup set '" + setName + + "' is either empty or does not exist"); + return -3; + } + tableMapping = (remainArgs.length > 2) ? remainArgs[2] : null; + } else { + tables = remainArgs[2]; + tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null; + } TableName[] sTableArray = BackupServerUtil.parseTableNames(tables); TableName[] tTableArray = BackupServerUtil.parseTableNames(tableMapping); @@ -129,7 +160,7 @@ public class RestoreDriver extends AbstractHBaseTool { if (sTableArray != null && tTableArray != null && (sTableArray.length != tTableArray.length)) { System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); System.out.println(USAGE); - return -2; + return -4; } @@ -139,11 +170,20 @@ public class RestoreDriver extends AbstractHBaseTool { tTableArray, isOverwrite); } catch (Exception e){ e.printStackTrace(); - return -3; + return -5; } return 0; } + private String getTablesForSet(String name, Configuration conf) throws IOException { + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable table = new BackupSystemTable(conn)) { + List<TableName> tables = table.describeBackupSet(name); + if (tables == null) return null; + return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + } + } + @Override protected void addOptions() { } http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java index a4c0fa1..a8fa7de 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; import org.junit.Test; @@ -45,7 +46,7 @@ public class TestFullBackupSet extends TestBackupBase { @Test public void testFullBackupSetExist() throws Exception { - LOG.info("TFBSE test full backup, backup set exists"); + LOG.info("Test full backup, backup set exists"); //Create set try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { @@ -65,7 +66,24 @@ public class TestFullBackupSet extends TestBackupBase { assertTrue(backups.size() == 1); String backupId = backups.get(0).getBackupId(); assertTrue(checkSucceeded(backupId)); - LOG.info("TFBSE backup complete"); + + LOG.info("backup complete"); + + // Restore from set into other table + args = new String[]{BACKUP_ROOT_DIR, backupId, + "-set", name, table1_restore.getNameAsString(), "-overwrite" }; + // Run backup + ret = ToolRunner.run(conf1, new RestoreDriver(), args); + assertTrue(ret == 0); + HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); + assertTrue(hba.tableExists(table1_restore)); + // Verify number of rows in both tables + assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore)); + TEST_UTIL.deleteTable(table1_restore); + LOG.info("restore into other table is complete"); + hba.close(); + + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java new file mode 100644 index 0000000..35e84b1 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.util.ToolRunner; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(LargeTests.class) +public class TestFullBackupSetRestoreSet extends TestBackupBase { + + private static final Log LOG = LogFactory.getLog(TestFullBackupSetRestoreSet.class); + + @Test + public void testFullRestoreSetToOtherTable() throws Exception { + + LOG.info("Test full restore set"); + + // Create set + try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { + String name = "name"; + table.addToBackupSet(name, new String[] { table1.getNameAsString() }); + List<TableName> names = table.describeBackupSet(name); + + assertNotNull(names); + assertTrue(names.size() == 1); + assertTrue(names.get(0).equals(table1)); + + String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-set", name }; + // Run backup + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + ArrayList<BackupInfo> backups = table.getBackupHistory(); + assertTrue(backups.size() == 1); + String backupId = backups.get(0).getBackupId(); + assertTrue(checkSucceeded(backupId)); + + LOG.info("backup complete"); + + // Restore from set into other table + args = + new String[] { BACKUP_ROOT_DIR, backupId, "-set", name, table1_restore.getNameAsString(), + "-overwrite" }; + // Run backup + ret = ToolRunner.run(conf1, new RestoreDriver(), args); + assertTrue(ret == 0); + HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); + assertTrue(hba.tableExists(table1_restore)); + // Verify number of rows in both tables + assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore)); + TEST_UTIL.deleteTable(table1_restore); + LOG.info("restore into other table is complete"); + hba.close(); + } + } + + @Test + public void testFullRestoreSetToSameTable() throws Exception { + + LOG.info("Test full restore set to same table"); + + // Create set + try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { + String name = "name1"; + table.addToBackupSet(name, new String[] { table1.getNameAsString() }); + List<TableName> names = table.describeBackupSet(name); + + assertNotNull(names); + assertTrue(names.size() == 1); + assertTrue(names.get(0).equals(table1)); + + String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-set", name }; + // Run backup + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + ArrayList<BackupInfo> backups = table.getBackupHistory(); + String backupId = backups.get(0).getBackupId(); + assertTrue(checkSucceeded(backupId)); + + LOG.info("backup complete"); + int count = TEST_UTIL.countRows(table1); + TEST_UTIL.deleteTable(table1); + + // Restore from set into other table + args = new String[] { BACKUP_ROOT_DIR, backupId, "-set", name, "-overwrite" }; + // Run backup + ret = ToolRunner.run(conf1, new RestoreDriver(), args); + assertTrue(ret == 0); + HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); + assertTrue(hba.tableExists(table1)); + // Verify number of rows in both tables + assertEquals(count, TEST_UTIL.countRows(table1)); + LOG.info("restore into same table is complete"); + hba.close(); + + } + + } + +} \ No newline at end of file
