Repository: hbase Updated Branches: refs/heads/branch-2 bbc13c9a1 -> 2cd58d83d
HBASE-17826 Backup: submit M/R job to a particular Yarn queue (Vladimir Rodionov) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2cd58d83 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2cd58d83 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2cd58d83 Branch: refs/heads/branch-2 Commit: 2cd58d83d3956eee558e94c3d32cd1857148dcd0 Parents: bbc13c9 Author: tedyu <yuzhih...@gmail.com> Authored: Wed Aug 30 16:12:21 2017 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Wed Aug 30 16:12:21 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/backup/BackupRequest.java | 14 ++++++++++++++ .../hadoop/hbase/backup/BackupRestoreConstants.java | 4 ++++ .../org/apache/hadoop/hbase/backup/RestoreDriver.java | 12 +++++++++++- .../hadoop/hbase/backup/impl/BackupCommands.java | 10 ++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/2cd58d83/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java index dadd861..8659868 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java @@ -67,6 +67,11 @@ public final class BackupRequest { return this; } + public Builder withYarnPoolName(String name) { + request.setYarnPoolName(name); + return this; + } + public BackupRequest build() { return request; } @@ -79,6 +84,7 @@ public final class BackupRequest { private int totalTasks = -1; private long bandwidth = -1L; private String backupSetName; + private String yarnPoolName; private BackupRequest() { } @@ -136,4 +142,12 @@ public final class BackupRequest { this.backupSetName = backupSetName; return this; } + + public String getYarnPoolName() { + return yarnPoolName; + } + + public void setYarnPoolName(String yarnPoolName) { + this.yarnPoolName = yarnPoolName; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/2cd58d83/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index 48e70a1..f511cc4 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -89,6 +89,10 @@ public interface BackupRestoreConstants { public static final String OPTION_TABLE_MAPPING_DESC = "A comma separated list of target tables. " + "If specified, each table in <tables> must have a mapping"; + public static final String OPTION_YARN_QUEUE_NAME = "q"; + public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on"; + public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on"; + public static final String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n" + "hbase.master.logcleaner.plugins=" http://git-wip-us.apache.org/repos/asf/hbase/blob/2cd58d83/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java index 82a1b56..2a1458c 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java @@ -29,6 +29,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_RESTORE_DESC; import java.io.IOException; import java.net.URI; @@ -125,6 +127,13 @@ public class RestoreDriver extends AbstractHBaseTool { printToolUsage(); return -1; } + + if (cmd.hasOption(OPTION_YARN_QUEUE_NAME)) { + String queueName = cmd.getOptionValue(OPTION_YARN_QUEUE_NAME); + // Set system property value for MR job + System.setProperty("mapreduce.job.queuename", queueName); + } + // parse main restore command options String[] remainArgs = cmd.getArgs(); if (remainArgs.length != 2) { @@ -195,8 +204,9 @@ public class RestoreDriver extends AbstractHBaseTool { addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC); addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC); - addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC); + addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_RESTORE_DESC); + } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/2cd58d83/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 650ba2e..eaba049 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -32,6 +32,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; import java.io.IOException; import java.net.URI; @@ -321,6 +323,12 @@ public final class BackupCommands { cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline .getOptionValue(OPTION_WORKERS)) : -1; + if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) { + String poolName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME); + // Set system property value for MR job + System.setProperty("mapreduce.job.queuename", poolName); + } + try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { BackupRequest.Builder builder = new BackupRequest.Builder(); @@ -368,6 +376,8 @@ public final class BackupCommands { options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC); options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC); + options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC); + HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.setLeftPadding(2);