Repository: hbase Updated Branches: refs/heads/HBASE-7912 c3a960f2e -> e35f7b920
HBASE-16673 Enhance history command: history per backup set (Vladimir Rodionov) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e35f7b92 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e35f7b92 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e35f7b92 Branch: refs/heads/HBASE-7912 Commit: e35f7b920e80daa79631aa3c8c9846405658be21 Parents: c3a960f Author: tedyu <[email protected]> Authored: Wed Sep 28 13:20:58 2016 -0700 Committer: tedyu <[email protected]> Committed: Wed Sep 28 13:20:58 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hbase/backup/BackupInfo.java | 9 ++++ .../hbase/backup/impl/BackupCommands.java | 48 +++++++++++++++----- .../hbase/backup/util/BackupClientUtil.java | 36 +++++++-------- .../apache/hadoop/hbase/client/BackupAdmin.java | 6 +-- .../hadoop/hbase/client/HBaseBackupAdmin.java | 23 ++++++++-- .../hbase/backup/TestBackupShowHistory.java | 36 +++++++++++++-- 6 files changed, 116 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index df8860a..be5ffea 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -51,6 +51,15 @@ import org.apache.hadoop.hbase.util.Bytes; public class BackupInfo implements Comparable<BackupInfo> { private static final Log LOG = LogFactory.getLog(BackupInfo.class); + public static interface Filter { + + /** + * Filter interface + * @param info: backup info + * @return true if info passes filter, false otherwise + */ + public boolean apply(BackupInfo info); + } // backup status flag public static enum BackupState { WAITING, RUNNING, COMPLETE, FAILED, ANY; http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 06ff419..2ff5756 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -463,25 +463,44 @@ public final class BackupCommands { public void execute() throws IOException { super.execute(); - + int n = parseHistoryLength(); - TableName tableName = getTableName(); + final TableName tableName = getTableName(); + final String setName = getTableSetName(); + BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + if (tableName == null) return true; + List<TableName> names = info.getTableNames(); + return names.contains(tableName); + } + }; + BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + if (setName == null) return true; + String backupId = info.getBackupId(); + return backupId.startsWith(setName); + } + }; Path backupRootPath = getBackupRootPath(); List<BackupInfo> history = null; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - if(backupRootPath == null) { + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + if (backupRootPath == null) { // Load from hbase:backup - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - history = admin.getHistory(n, tableName); - } + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { + + history = admin.getHistory(n, tableNameFilter, tableSetFilter); + } } else { // load from backup FS - history = BackupClientUtil.getHistory(conf, n, tableName, backupRootPath); - } - for(BackupInfo info: history){ + history = BackupClientUtil.getHistory(conf, n, backupRootPath, + tableNameFilter, tableSetFilter); + } + for (BackupInfo info : history) { System.out.println(info.getShortDescription()); - } + } } private Path getBackupRootPath() throws IOException { @@ -509,6 +528,11 @@ public final class BackupCommands { } } + private String getTableSetName() throws IOException { + String value = cmdline.getOptionValue("set"); + return value; + } + private int parseHistoryLength() throws IOException { String value = cmdline.getOptionValue("n"); try{ http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java index 4f22164..c22f51b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java @@ -359,7 +359,7 @@ public final class BackupClientUtil { + HConstants.HREGION_LOGDIR_NAME; } - public static List<BackupInfo> getHistory(Configuration conf, Path backupRootPath) + private static List<BackupInfo> getHistory(Configuration conf, Path backupRootPath) throws IOException { // Get all (n) history from backup root destination FileSystem fs = FileSystem.get(conf); @@ -396,28 +396,28 @@ public final class BackupClientUtil { return infos; } - public static List<BackupInfo> getHistory(Configuration conf, int n, TableName name, - Path backupRootPath) throws IOException { + public static List<BackupInfo> getHistory(Configuration conf, int n, Path backupRootPath, + BackupInfo.Filter... filters) throws IOException { List<BackupInfo> infos = getHistory(conf, backupRootPath); - if (name == null) { - if (infos.size() <= n) return infos; - return infos.subList(0, n); - } else { - List<BackupInfo> ret = new ArrayList<BackupInfo>(); - int count = 0; - for (BackupInfo info : infos) { - List<TableName> names = info.getTableNames(); - if (names.contains(name)) { - ret.add(info); - if (++count == n) { - break; - } + List<BackupInfo> ret = new ArrayList<BackupInfo>(); + for (BackupInfo info : infos) { + if (ret.size() == n) { + break; + } + boolean passed = true; + for (int i = 0; i < filters.length; i++) { + if (!filters[i].apply(info)) { + passed = false; + break; } } - return ret; + if (passed) { + ret.add(info); + } } + return ret; } - + public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) throws IOException { Path backupPath = new Path(backupRootPath, backupId); http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java index b19dea1..2e5ca2a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java @@ -122,13 +122,13 @@ public interface BackupAdmin extends Closeable{ /** - * Show backup history command for a table + * Show backup history command with filters * @param n - last n backup sessions - * @param name - table's name + * @param f - list of filters * @return list of backup infos * @throws IOException exception */ - public List<BackupInfo> getHistory(int n, TableName name) throws IOException; + public List<BackupInfo> getHistory(int n, BackupInfo.Filter ... f) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java index 8e08fd0..dfa2fb1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java @@ -333,12 +333,25 @@ public class HBaseBackupAdmin implements BackupAdmin { } @Override - public List<BackupInfo> getHistory(int n, TableName name) throws IOException { - if (name == null) return getHistory(n); + public List<BackupInfo> getHistory(int n, BackupInfo.Filter ... filters) throws IOException { + if (filters.length == 0) return getHistory(n); try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<BackupInfo> history = table.getBackupHistoryForTable(name); - n = Math.min(n, history.size()); - return history.subList(0, n); + List<BackupInfo> history = table.getBackupHistory(); + List<BackupInfo> result = new ArrayList<BackupInfo>(); + for(BackupInfo bi: history) { + if(result.size() == n) break; + boolean passed = true; + for(int i=0; i < filters.length; i++) { + if(!filters[i].apply(bi)) { + passed = false; + break; + } + } + if(passed) { + result.add(bi); + } + } + return result; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/e35f7b92/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index 4a94cb9..4594338 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -70,7 +70,13 @@ public class TestBackupShowHistory extends TestBackupBase { List<BackupInfo> history = getBackupAdmin().getHistory(10); assertTrue(findBackup(history, backupId)); - history = BackupClientUtil.getHistory(conf1, 10, null, new Path(BACKUP_ROOT_DIR)); + BackupInfo.Filter nullFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + return true; + } + }; + history = BackupClientUtil.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), nullFilter); assertTrue(findBackup(history, backupId)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -90,8 +96,23 @@ public class TestBackupShowHistory extends TestBackupBase { String backupId2 = fullTableBackup(tableList); assertTrue(checkSucceeded(backupId2)); LOG.info("backup complete: "+ table2); + BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo image) { + if (table1 == null) return true; + List<TableName> names = image.getTableNames(); + return names.contains(table1); + } + }; + BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + String backupId = info.getBackupId(); + return backupId.startsWith("backup"); + } + }; - history = getBackupAdmin().getHistory(10, table1); + history = getBackupAdmin().getHistory(10, tableNameFilter, tableSetFilter); assertTrue(history.size() > 0); boolean success = true; for (BackupInfo info: history){ @@ -101,7 +122,9 @@ public class TestBackupShowHistory extends TestBackupBase { } } assertTrue(success); - history = BackupClientUtil.getHistory(conf1, 10, table1, new Path(BACKUP_ROOT_DIR)); + + history = BackupClientUtil.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), + tableNameFilter, tableSetFilter); assertTrue(history.size() > 0); success = true; for (BackupInfo info: history){ @@ -111,7 +134,12 @@ public class TestBackupShowHistory extends TestBackupBase { } } assertTrue(success); - + + args = new String[]{"history", "-n", "10", "-path", BACKUP_ROOT_DIR, + "-t", "table1", "-set", "backup"}; + // Run backup + ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); LOG.info("show_history"); }
