http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java new file mode 100644 index 0000000..1c43e88 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java @@ -0,0 +1,146 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupManifest; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * View to an on-disk Backup Image FileSytem Provides the set of methods necessary to interact with + * the on-disk Backup Image data. + */ +@InterfaceAudience.Private +public class HBackupFileSystem { + public static final Log LOG = LogFactory.getLog(HBackupFileSystem.class); + + /** + * This is utility class. + */ + private HBackupFileSystem() { + } + + /** + * Given the backup root dir, backup id and the table name, return the backup image location, + * which is also where the backup manifest file is. return value look like: + * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where + * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory + * @param backupRootDir backup root directory + * @param backupId backup id + * @param tableName table name + * @return backupPath String for the particular table + */ + public static String + getTableBackupDir(String backupRootDir, String backupId, TableName tableName) { + return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR + + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString() + + Path.SEPARATOR; + } + + public static String getTableBackupDataDir(String backupRootDir, String backupId, + TableName tableName) { + return getTableBackupDir(backupRootDir, backupId, tableName) + Path.SEPARATOR + "data"; + } + + public static Path getBackupPath(String backupRootDir, String backupId) { + return new Path(backupRootDir + Path.SEPARATOR + backupId); + } + + /** + * Given the backup root dir, backup id and the table name, return the backup image location, + * which is also where the backup manifest file is. return value look like: + * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where + * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory + * @param backupRootPath backup root path + * @param tableName table name + * @param backupId backup Id + * @return backupPath for the particular table + */ + public static Path getTableBackupPath(TableName tableName, Path backupRootPath, String backupId) { + return new Path(getTableBackupDir(backupRootPath.toString(), backupId, tableName)); + } + + /** + * Given the backup root dir and the backup id, return the log file location for an incremental + * backup. + * @param backupRootDir backup root directory + * @param backupId backup id + * @return logBackupDir: ".../user/biadmin/backup/WALs/backup_1396650096738" + */ + public static String getLogBackupDir(String backupRootDir, String backupId) { + return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR + + HConstants.HREGION_LOGDIR_NAME; + } + + public static Path getLogBackupPath(String backupRootDir, String backupId) { + return new Path(getLogBackupDir(backupRootDir, backupId)); + } + + // TODO we do not keep WAL files anymore + // Move manifest file to other place + private static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId) + throws IOException { + Path manifestPath = null; + + FileSystem fs = backupRootPath.getFileSystem(conf); + manifestPath = + new Path(getBackupPath(backupRootPath.toString(), backupId) + Path.SEPARATOR + + BackupManifest.MANIFEST_FILE_NAME); + if (!fs.exists(manifestPath)) { + String errorMsg = + "Could not find backup manifest " + BackupManifest.MANIFEST_FILE_NAME + " for " + + backupId + ". File " + manifestPath + " does not exists. Did " + backupId + + " correspond to previously taken backup ?"; + throw new IOException(errorMsg); + } + return manifestPath; + } + + public static BackupManifest + getManifest(Configuration conf, Path backupRootPath, String backupId) throws IOException { + BackupManifest manifest = + new BackupManifest(conf, getManifestPath(conf, backupRootPath, backupId)); + return manifest; + } + + /** + * Check whether the backup image path and there is manifest file in the path. + * @param backupManifestMap If all the manifests are found, then they are put into this map + * @param tableArray the tables involved + * @throws IOException exception + */ + public static void checkImageManifestExist(HashMap<TableName, BackupManifest> backupManifestMap, + TableName[] tableArray, Configuration conf, Path backupRootPath, String backupId) + throws IOException { + for (TableName tableName : tableArray) { + BackupManifest manifest = getManifest(conf, backupRootPath, backupId); + backupManifestMap.put(tableName, manifest); + } + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java new file mode 100644 index 0000000..1becb75 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/LogUtils.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.commons.logging.Log; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * Utility class for disabling Zk and client logging + * + */ +@InterfaceAudience.Private +final class LogUtils { + + private LogUtils() { + } + + /** + * Disables Zk- and HBase client logging + * @param log + */ + static void disableZkAndClientLoggers(Log log) { + // disable zookeeper log to avoid it mess up command output + Logger zkLogger = Logger.getLogger("org.apache.zookeeper"); + zkLogger.setLevel(Level.OFF); + // disable hbase zookeeper tool log to avoid it mess up command output + Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper"); + hbaseZkLogger.setLevel(Level.OFF); + // disable hbase client log to avoid it mess up command output + Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client"); + hbaseClientLogger.setLevel(Level.OFF); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java new file mode 100644 index 0000000..82a1b56 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_CHECK; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_CHECK_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_OVERWRITE; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_OVERWRITE_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_RESTORE_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; +import org.apache.hadoop.hbase.backup.impl.BackupManager; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * Command-line entry point for restore operation + * + */ +@InterfaceAudience.Private +public class RestoreDriver extends AbstractHBaseTool { + + private static final Log LOG = LogFactory.getLog(RestoreDriver.class); + private CommandLine cmd; + + private static final String USAGE_STRING = + "Usage: hbase restore <backup_path> <backup_id> [options]\n" + + " backup_path Path to a backup destination root\n" + + " backup_id Backup image ID to restore\n" + + " table(s) Comma-separated list of tables to restore\n"; + + private static final String USAGE_FOOTER = ""; + + protected RestoreDriver() throws IOException { + init(); + } + + protected void init() throws IOException { + // disable irrelevant loggers to avoid it mess up command output + LogUtils.disableZkAndClientLoggers(LOG); + } + + private int parseAndRun(String[] args) throws IOException { + // Check if backup is enabled + if (!BackupManager.isBackupEnabled(getConf())) { + System.err.println(BackupRestoreConstants.ENABLE_BACKUP); + return -1; + } + + System.out.println(BackupRestoreConstants.VERIFY_BACKUP); + + // enable debug logging + Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup"); + if (cmd.hasOption(OPTION_DEBUG)) { + backupClientLogger.setLevel(Level.DEBUG); + } + + // whether to overwrite to existing table if any, false by default + boolean overwrite = cmd.hasOption(OPTION_OVERWRITE); + if (overwrite) { + LOG.debug("Found -overwrite option in restore command, " + + "will overwrite to existing table if any in the restore target"); + } + + // whether to only check the dependencies, false by default + boolean check = cmd.hasOption(OPTION_CHECK); + if (check) { + LOG.debug("Found -check option in restore command, " + + "will check and verify the dependencies"); + } + + if (cmd.hasOption(OPTION_SET) && cmd.hasOption(OPTION_TABLE)) { + System.err.println("Options -s and -t are mutaully exclusive,"+ + " you can not specify both of them."); + printToolUsage(); + return -1; + } + + if (!cmd.hasOption(OPTION_SET) && !cmd.hasOption(OPTION_TABLE)) { + System.err.println("You have to specify either set name or table list to restore"); + printToolUsage(); + return -1; + } + // parse main restore command options + String[] remainArgs = cmd.getArgs(); + if (remainArgs.length != 2) { + printToolUsage(); + return -1; + } + + String backupRootDir = remainArgs[0]; + String backupId = remainArgs[1]; + String tables = null; + String tableMapping = + cmd.hasOption(OPTION_TABLE_MAPPING) ? cmd.getOptionValue(OPTION_TABLE_MAPPING) : null; + try (final Connection conn = ConnectionFactory.createConnection(conf); + BackupAdmin client = new BackupAdminImpl(conn);) { + // Check backup set + if (cmd.hasOption(OPTION_SET)) { + String setName = cmd.getOptionValue(OPTION_SET); + try { + tables = getTablesForSet(conn, setName, conf); + } catch (IOException e) { + System.out.println("ERROR: " + e.getMessage() + " for setName=" + setName); + printToolUsage(); + return -2; + } + if (tables == null) { + System.out.println("ERROR: Backup set '" + setName + + "' is either empty or does not exist"); + printToolUsage(); + return -3; + } + } else { + tables = cmd.getOptionValue(OPTION_TABLE); + } + + TableName[] sTableArray = BackupUtils.parseTableNames(tables); + TableName[] tTableArray = BackupUtils.parseTableNames(tableMapping); + + if (sTableArray != null && tTableArray != null && + (sTableArray.length != tTableArray.length)) { + System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); + printToolUsage(); + return -4; + } + + client.restore(BackupUtils.createRestoreRequest(backupRootDir, backupId, check, + sTableArray, tTableArray, overwrite)); + } catch (Exception e) { + e.printStackTrace(); + return -5; + } + return 0; + } + + private String getTablesForSet(Connection conn, String name, Configuration conf) + throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List<TableName> tables = table.describeBackupSet(name); + if (tables == null) return null; + return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + } + } + + @Override + protected void addOptions() { + // define supported options + addOptNoArg(OPTION_OVERWRITE, OPTION_OVERWRITE_DESC); + addOptNoArg(OPTION_CHECK, OPTION_CHECK_DESC); + addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); + addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC); + addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC); + + addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC); + } + + @Override + protected void processOptions(CommandLine cmd) { + this.cmd = cmd; + } + + @Override + protected int doWork() throws Exception { + return parseAndRun(cmd.getArgs()); + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + Path hbasedir = FSUtils.getRootDir(conf); + URI defaultFs = hbasedir.getFileSystem(conf).getUri(); + FSUtils.setFsDefault(conf, new Path(defaultFs)); + int ret = ToolRunner.run(conf, new RestoreDriver(), args); + System.exit(ret); + } + + @Override + public int run(String[] args) throws IOException { + if (conf == null) { + LOG.error("Tool configuration is not initialized"); + throw new NullPointerException("conf"); + } + + CommandLine cmd; + try { + // parse the command line arguments + cmd = parseArgs(args); + cmdLineArgs = args; + } catch (Exception e) { + System.out.println("Error when parsing command-line arguments: " + e.getMessage()); + printToolUsage(); + return EXIT_FAILURE; + } + + if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION)) { + printToolUsage(); + return EXIT_FAILURE; + } + + processOptions(cmd); + + int ret = EXIT_FAILURE; + try { + ret = doWork(); + } catch (Exception e) { + LOG.error("Error running command-line tool", e); + return EXIT_FAILURE; + } + return ret; + } + + protected void printToolUsage() throws IOException { + System.out.println(USAGE_STRING); + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setLeftPadding(2); + helpFormatter.setDescPadding(8); + helpFormatter.setWidth(100); + helpFormatter.setSyntaxPrefix("Options:"); + helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java new file mode 100644 index 0000000..86fb963 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreJob.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Restore operation job interface Concrete implementation is provided by backup provider, see + * {@link BackupRestoreFactory} + */ + +@InterfaceAudience.Private +public interface RestoreJob extends Configurable { + + /** + * Run restore operation + * @param dirPaths path array of WAL log directories + * @param fromTables from tables + * @param toTables to tables + * @param fullBackupRestore full backup restore + * @throws IOException + */ + void run(Path[] dirPaths, TableName[] fromTables, TableName[] toTables, + boolean fullBackupRestore) throws IOException; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java new file mode 100644 index 0000000..de3ad5a --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * POJO class for restore request + */ +@InterfaceAudience.Private +public class RestoreRequest { + + public static class Builder { + RestoreRequest request; + + public Builder() { + request = new RestoreRequest(); + } + + public Builder withBackupRootDir(String backupRootDir) { + request.setBackupRootDir(backupRootDir); + return this; + } + + public Builder withBackupId(String backupId) { + request.setBackupId(backupId); + return this; + } + + public Builder withCheck(boolean check) { + request.setCheck(check); + return this; + } + + public Builder withFromTables(TableName[] fromTables) { + request.setFromTables(fromTables); + return this; + } + + public Builder withToTables(TableName[] toTables) { + request.setToTables(toTables); + return this; + } + + public Builder withOvewrite(boolean overwrite) { + request.setOverwrite(overwrite); + return this; + } + + + public RestoreRequest build() { + return request; + } + } + + private String backupRootDir; + private String backupId; + private boolean check = false; + private TableName[] fromTables; + private TableName[] toTables; + private boolean overwrite = false; + + private RestoreRequest() { + } + + public String getBackupRootDir() { + return backupRootDir; + } + + private RestoreRequest setBackupRootDir(String backupRootDir) { + this.backupRootDir = backupRootDir; + return this; + } + + public String getBackupId() { + return backupId; + } + + private RestoreRequest setBackupId(String backupId) { + this.backupId = backupId; + return this; + } + + public boolean isCheck() { + return check; + } + + private RestoreRequest setCheck(boolean check) { + this.check = check; + return this; + } + + public TableName[] getFromTables() { + return fromTables; + } + + private RestoreRequest setFromTables(TableName[] fromTables) { + this.fromTables = fromTables; + return this; + } + + public TableName[] getToTables() { + return toTables; + } + + private RestoreRequest setToTables(TableName[] toTables) { + this.toTables = toTables; + return this; + } + + public boolean isOverwrite() { + return overwrite; + } + + private RestoreRequest setOverwrite(boolean overwrite) { + this.overwrite = overwrite; + return this; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2dda3712/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java new file mode 100644 index 0000000..99fb06c --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -0,0 +1,743 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupAdmin; +import org.apache.hadoop.hbase.backup.BackupClientFactory; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.BackupMergeJob; +import org.apache.hadoop.hbase.backup.BackupRequest; +import org.apache.hadoop.hbase.backup.BackupRestoreConstants; +import org.apache.hadoop.hbase.backup.BackupRestoreFactory; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.HBackupFileSystem; +import org.apache.hadoop.hbase.backup.RestoreRequest; +import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +@InterfaceAudience.Private +public class BackupAdminImpl implements BackupAdmin { + public final static String CHECK_OK = "Checking backup images: OK"; + public final static String CHECK_FAILED = + "Checking backup images: Failed. Some dependencies are missing for restore"; + private static final Log LOG = LogFactory.getLog(BackupAdminImpl.class); + + private final Connection conn; + + public BackupAdminImpl(Connection conn) { + this.conn = conn; + } + + @Override + public void close() throws IOException { + } + + @Override + public BackupInfo getBackupInfo(String backupId) throws IOException { + BackupInfo backupInfo = null; + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + if (backupId == null) { + ArrayList<BackupInfo> recentSessions = table.getBackupInfos(BackupState.RUNNING); + if (recentSessions.isEmpty()) { + LOG.warn("No ongoing sessions found."); + return null; + } + // else show status for ongoing session + // must be one maximum + return recentSessions.get(0); + } else { + backupInfo = table.readBackupInfo(backupId); + return backupInfo; + } + } + } + + @Override + public int deleteBackups(String[] backupIds) throws IOException { + + int totalDeleted = 0; + Map<String, HashSet<TableName>> allTablesMap = new HashMap<String, HashSet<TableName>>(); + + boolean deleteSessionStarted = false; + boolean snapshotDone = false; + try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { + + // Step 1: Make sure there is no active session + // is running by using startBackupSession API + // If there is an active session in progress, exception will be thrown + try { + sysTable.startBackupExclusiveOperation(); + deleteSessionStarted = true; + } catch (IOException e) { + LOG.warn("You can not run delete command while active backup session is in progress. \n" + + "If there is no active backup session running, run backup repair utility to restore \n" + + "backup system integrity."); + return -1; + } + + // Step 2: Make sure there is no failed session + List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING); + if (list.size() != 0) { + // ailed sessions found + LOG.warn("Failed backup session found. Run backup repair tool first."); + return -1; + } + + // Step 3: Record delete session + sysTable.startDeleteOperation(backupIds); + // Step 4: Snapshot backup system table + if (!BackupSystemTable.snapshotExists(conn)) { + BackupSystemTable.snapshot(conn); + } else { + LOG.warn("Backup system table snapshot exists"); + } + snapshotDone = true; + try { + for (int i = 0; i < backupIds.length; i++) { + BackupInfo info = sysTable.readBackupInfo(backupIds[i]); + if (info != null) { + String rootDir = info.getBackupRootDir(); + HashSet<TableName> allTables = allTablesMap.get(rootDir); + if (allTables == null) { + allTables = new HashSet<TableName>(); + allTablesMap.put(rootDir, allTables); + } + allTables.addAll(info.getTableNames()); + totalDeleted += deleteBackup(backupIds[i], sysTable); + } + } + finalizeDelete(allTablesMap, sysTable); + // Finish + sysTable.finishDeleteOperation(); + // delete snapshot + BackupSystemTable.deleteSnapshot(conn); + } catch (IOException e) { + // Fail delete operation + // Step 1 + if (snapshotDone) { + if (BackupSystemTable.snapshotExists(conn)) { + BackupSystemTable.restoreFromSnapshot(conn); + // delete snapshot + BackupSystemTable.deleteSnapshot(conn); + // We still have record with unfinished delete operation + LOG.error("Delete operation failed, please run backup repair utility to restore " + + "backup system integrity", e); + throw e; + } else { + LOG.warn("Delete operation succeeded, there were some errors: ", e); + } + } + + } finally { + if (deleteSessionStarted) { + sysTable.finishBackupExclusiveOperation(); + } + } + } + return totalDeleted; + } + + /** + * Updates incremental backup set for every backupRoot + * @param tablesMap map [backupRoot: Set<TableName>] + * @param table backup system table + * @throws IOException + */ + + private void finalizeDelete(Map<String, HashSet<TableName>> tablesMap, BackupSystemTable table) + throws IOException { + for (String backupRoot : tablesMap.keySet()) { + Set<TableName> incrTableSet = table.getIncrementalBackupTableSet(backupRoot); + Map<TableName, ArrayList<BackupInfo>> tableMap = + table.getBackupHistoryForTableSet(incrTableSet, backupRoot); + for (Map.Entry<TableName, ArrayList<BackupInfo>> entry : tableMap.entrySet()) { + if (entry.getValue() == null) { + // No more backups for a table + incrTableSet.remove(entry.getKey()); + } + } + if (!incrTableSet.isEmpty()) { + table.addIncrementalBackupTableSet(incrTableSet, backupRoot); + } else { // empty + table.deleteIncrementalBackupTableSet(backupRoot); + } + } + } + + /** + * Delete single backup and all related backups <br> + * Algorithm:<br> + * Backup type: FULL or INCREMENTAL <br> + * Is this last backup session for table T: YES or NO <br> + * For every table T from table list 'tables':<br> + * if(FULL, YES) deletes only physical data (PD) <br> + * if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo,<br> + * until we either reach the most recent backup for T in the system or FULL backup<br> + * which includes T<br> + * if(INCREMENTAL, YES) deletes only physical data (PD) if(INCREMENTAL, NO) deletes physical data + * and for table T scans all backup images between last<br> + * FULL backup, which is older than the backup being deleted and the next FULL backup (if exists) <br> + * or last one for a particular table T and removes T from list of backup tables. + * @param backupId backup id + * @param sysTable backup system table + * @return total number of deleted backup images + * @throws IOException + */ + private int deleteBackup(String backupId, BackupSystemTable sysTable) throws IOException { + + BackupInfo backupInfo = sysTable.readBackupInfo(backupId); + + int totalDeleted = 0; + if (backupInfo != null) { + LOG.info("Deleting backup " + backupInfo.getBackupId() + " ..."); + // Step 1: clean up data for backup session (idempotent) + BackupUtils.cleanupBackupData(backupInfo, conn.getConfiguration()); + // List of tables in this backup; + List<TableName> tables = backupInfo.getTableNames(); + long startTime = backupInfo.getStartTs(); + for (TableName tn : tables) { + boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime); + if (isLastBackupSession) { + continue; + } + // else + List<BackupInfo> affectedBackups = getAffectedBackupSessions(backupInfo, tn, sysTable); + for (BackupInfo info : affectedBackups) { + if (info.equals(backupInfo)) { + continue; + } + removeTableFromBackupImage(info, tn, sysTable); + } + } + Map<byte[], String> map = sysTable.readBulkLoadedFiles(backupId); + FileSystem fs = FileSystem.get(conn.getConfiguration()); + boolean success = true; + int numDeleted = 0; + for (String f : map.values()) { + Path p = new Path(f); + try { + LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId()); + if (!fs.delete(p)) { + if (fs.exists(p)) { + LOG.warn(f + " was not deleted"); + success = false; + } + } else { + numDeleted++; + } + } catch (IOException ioe) { + LOG.warn(f + " was not deleted", ioe); + success = false; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(numDeleted + " bulk loaded files out of " + map.size() + " were deleted"); + } + if (success) { + sysTable.deleteBulkLoadedFiles(map); + } + + sysTable.deleteBackupInfo(backupInfo.getBackupId()); + LOG.info("Delete backup " + backupInfo.getBackupId() + " completed."); + totalDeleted++; + } else { + LOG.warn("Delete backup failed: no information found for backupID=" + backupId); + } + return totalDeleted; + } + + private void + removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSystemTable sysTable) + throws IOException { + List<TableName> tables = info.getTableNames(); + LOG.debug("Remove " + tn + " from " + info.getBackupId() + " tables=" + + info.getTableListAsString()); + if (tables.contains(tn)) { + tables.remove(tn); + + if (tables.isEmpty()) { + LOG.debug("Delete backup info " + info.getBackupId()); + + sysTable.deleteBackupInfo(info.getBackupId()); + // Idempotent operation + BackupUtils.cleanupBackupData(info, conn.getConfiguration()); + } else { + info.setTables(tables); + sysTable.updateBackupInfo(info); + // Now, clean up directory for table (idempotent) + cleanupBackupDir(info, tn, conn.getConfiguration()); + } + } + } + + private List<BackupInfo> getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, + BackupSystemTable table) throws IOException { + LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); + long ts = backupInfo.getStartTs(); + List<BackupInfo> list = new ArrayList<BackupInfo>(); + List<BackupInfo> history = table.getBackupHistory(backupInfo.getBackupRootDir()); + // Scan from most recent to backupInfo + // break when backupInfo reached + for (BackupInfo info : history) { + if (info.getStartTs() == ts) { + break; + } + List<TableName> tables = info.getTableNames(); + if (tables.contains(tn)) { + BackupType bt = info.getType(); + if (bt == BackupType.FULL) { + // Clear list if we encounter FULL backup + list.clear(); + } else { + LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn + + " added " + info.getBackupId() + " tables=" + info.getTableListAsString()); + list.add(info); + } + } + } + return list; + } + + /** + * Clean up the data at target directory + * @throws IOException + */ + private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf) + throws IOException { + try { + // clean up the data at target directory + String targetDir = backupInfo.getBackupRootDir(); + if (targetDir == null) { + LOG.warn("No target directory specified for " + backupInfo.getBackupId()); + return; + } + + FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); + + Path targetDirPath = + new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(), + backupInfo.getBackupId(), table)); + if (outputFs.delete(targetDirPath, true)) { + LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); + } else { + LOG.info("No data has been found in " + targetDirPath.toString() + "."); + } + + } catch (IOException e1) { + LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table + + "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); + throw e1; + } + } + + private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime) + throws IOException { + List<BackupInfo> history = table.getBackupHistory(); + for (BackupInfo info : history) { + List<TableName> tables = info.getTableNames(); + if (!tables.contains(tn)) { + continue; + } + if (info.getStartTs() <= startTime) { + return true; + } else { + return false; + } + } + return false; + } + + @Override + public List<BackupInfo> getHistory(int n) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List<BackupInfo> history = table.getBackupHistory(); + if (history.size() <= n) return history; + List<BackupInfo> list = new ArrayList<BackupInfo>(); + for (int i = 0; i < n; i++) { + list.add(history.get(i)); + } + return list; + } + } + + @Override + public List<BackupInfo> getHistory(int n, BackupInfo.Filter... filters) throws IOException { + if (filters.length == 0) return getHistory(n); + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List<BackupInfo> history = table.getBackupHistory(); + List<BackupInfo> result = new ArrayList<BackupInfo>(); + for (BackupInfo bi : history) { + if (result.size() == n) break; + boolean passed = true; + for (int i = 0; i < filters.length; i++) { + if (!filters[i].apply(bi)) { + passed = false; + break; + } + } + if (passed) { + result.add(bi); + } + } + return result; + } + } + + @Override + public List<BackupSet> listBackupSets() throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List<String> list = table.listBackupSets(); + List<BackupSet> bslist = new ArrayList<BackupSet>(); + for (String s : list) { + List<TableName> tables = table.describeBackupSet(s); + if (tables != null) { + bslist.add(new BackupSet(s, tables)); + } + } + return bslist; + } + } + + @Override + public BackupSet getBackupSet(String name) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List<TableName> list = table.describeBackupSet(name); + if (list == null) return null; + return new BackupSet(name, list); + } + } + + @Override + public boolean deleteBackupSet(String name) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + if (table.describeBackupSet(name) == null) { + return false; + } + table.deleteBackupSet(name); + return true; + } + } + + @Override + public void addToBackupSet(String name, TableName[] tables) throws IOException { + String[] tableNames = new String[tables.length]; + try (final BackupSystemTable table = new BackupSystemTable(conn); + final Admin admin = conn.getAdmin();) { + for (int i = 0; i < tables.length; i++) { + tableNames[i] = tables[i].getNameAsString(); + if (!admin.tableExists(TableName.valueOf(tableNames[i]))) { + throw new IOException("Cannot add " + tableNames[i] + " because it doesn't exist"); + } + } + table.addToBackupSet(name, tableNames); + LOG.info("Added tables [" + StringUtils.join(tableNames, " ") + "] to '" + name + + "' backup set"); + } + } + + @Override + public void removeFromBackupSet(String name, TableName[] tables) throws IOException { + LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + "'"); + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + table.removeFromBackupSet(name, toStringArray(tables)); + LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + + "' completed."); + } + } + + private String[] toStringArray(TableName[] list) { + String[] arr = new String[list.length]; + for (int i = 0; i < list.length; i++) { + arr[i] = list[i].toString(); + } + return arr; + } + + @Override + public void restore(RestoreRequest request) throws IOException { + if (request.isCheck()) { + HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>(); + // check and load backup image manifest for the tables + Path rootPath = new Path(request.getBackupRootDir()); + String backupId = request.getBackupId(); + TableName[] sTableArray = request.getFromTables(); + HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, + conn.getConfiguration(), rootPath, backupId); + + // Check and validate the backup image and its dependencies + + if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) { + LOG.info(CHECK_OK); + } else { + LOG.error(CHECK_FAILED); + } + return; + } + // Execute restore request + new RestoreTablesClient(conn, request).execute(); + } + + @Override + public String backupTables(BackupRequest request) throws IOException { + BackupType type = request.getBackupType(); + String targetRootDir = request.getTargetRootDir(); + List<TableName> tableList = request.getTableList(); + + String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime(); + if (type == BackupType.INCREMENTAL) { + Set<TableName> incrTableSet = null; + try (BackupSystemTable table = new BackupSystemTable(conn)) { + incrTableSet = table.getIncrementalBackupTableSet(targetRootDir); + } + + if (incrTableSet.isEmpty()) { + String msg = + "Incremental backup table set contains no tables. " + + "You need to run full backup first " + + (tableList != null ? "on " + StringUtils.join(tableList, ",") : ""); + + throw new IOException(msg); + } + if (tableList != null) { + tableList.removeAll(incrTableSet); + if (!tableList.isEmpty()) { + String extraTables = StringUtils.join(tableList, ","); + String msg = + "Some tables (" + extraTables + ") haven't gone through full backup. " + + "Perform full backup on " + extraTables + " first, " + "then retry the command"; + throw new IOException(msg); + } + } + tableList = Lists.newArrayList(incrTableSet); + } + if (tableList != null && !tableList.isEmpty()) { + for (TableName table : tableList) { + String targetTableBackupDir = + HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table); + Path targetTableBackupDirPath = new Path(targetTableBackupDir); + FileSystem outputFs = + FileSystem.get(targetTableBackupDirPath.toUri(), conn.getConfiguration()); + if (outputFs.exists(targetTableBackupDirPath)) { + throw new IOException("Target backup directory " + targetTableBackupDir + + " exists already."); + } + } + ArrayList<TableName> nonExistingTableList = null; + try (Admin admin = conn.getAdmin();) { + for (TableName tableName : tableList) { + if (!admin.tableExists(tableName)) { + if (nonExistingTableList == null) { + nonExistingTableList = new ArrayList<>(); + } + nonExistingTableList.add(tableName); + } + } + } + if (nonExistingTableList != null) { + if (type == BackupType.INCREMENTAL) { + // Update incremental backup set + tableList = excludeNonExistingTables(tableList, nonExistingTableList); + } else { + // Throw exception only in full mode - we try to backup non-existing table + throw new IOException("Non-existing tables found in the table list: " + + nonExistingTableList); + } + } + } + + // update table list + BackupRequest.Builder builder = new BackupRequest.Builder(); + request = + builder.withBackupType(request.getBackupType()).withTableList(tableList) + .withTargetRootDir(request.getTargetRootDir()) + .withBackupSetName(request.getBackupSetName()).withTotalTasks(request.getTotalTasks()) + .withBandwidthPerTasks((int) request.getBandwidth()).build(); + + TableBackupClient client = null; + try { + client = BackupClientFactory.create(conn, backupId, request); + } catch (IOException e) { + LOG.error("There is an active session already running"); + throw e; + } + + client.execute(); + + return backupId; + } + + private List<TableName> excludeNonExistingTables(List<TableName> tableList, + List<TableName> nonExistingTableList) { + + for (TableName table : nonExistingTableList) { + tableList.remove(table); + } + return tableList; + } + + @Override + public void mergeBackups(String[] backupIds) throws IOException { + try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + checkIfValidForMerge(backupIds, sysTable); + BackupMergeJob job = BackupRestoreFactory.getBackupMergeJob(conn.getConfiguration()); + job.run(backupIds); + } + } + + /** + * Verifies that backup images are valid for merge. + * + * <ul> + * <li>All backups MUST be in the same destination + * <li>No FULL backups are allowed - only INCREMENTAL + * <li>All backups must be in COMPLETE state + * <li>No holes in backup list are allowed + * </ul> + * <p> + * @param backupIds list of backup ids + * @param table backup system table + * @throws IOException + */ + private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) throws IOException { + String backupRoot = null; + + final Set<TableName> allTables = new HashSet<TableName>(); + final Set<String> allBackups = new HashSet<String>(); + long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE; + for (String backupId : backupIds) { + BackupInfo bInfo = table.readBackupInfo(backupId); + if (bInfo == null) { + String msg = "Backup session " + backupId + " not found"; + throw new IOException(msg); + } + if (backupRoot == null) { + backupRoot = bInfo.getBackupRootDir(); + } else if (!bInfo.getBackupRootDir().equals(backupRoot)) { + throw new IOException("Found different backup destinations in a list of a backup sessions \n" + + "1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir()); + } + if (bInfo.getType() == BackupType.FULL) { + throw new IOException("FULL backup image can not be merged for: \n" + bInfo); + } + + if (bInfo.getState() != BackupState.COMPLETE) { + throw new IOException("Backup image " + backupId + + " can not be merged becuase of its state: " + bInfo.getState()); + } + allBackups.add(backupId); + allTables.addAll(bInfo.getTableNames()); + long time = bInfo.getStartTs(); + if (time < minTime) { + minTime = time; + } + if (time > maxTime) { + maxTime = time; + } + } + + + final long startRangeTime = minTime; + final long endRangeTime = maxTime; + final String backupDest = backupRoot; + // Check we have no 'holes' in backup id list + // Filter 1 : backupRoot + // Filter 2 : time range filter + // Filter 3 : table filter + + BackupInfo.Filter destinationFilter = new BackupInfo.Filter() { + + @Override + public boolean apply(BackupInfo info) { + return info.getBackupRootDir().equals(backupDest); + } + }; + + BackupInfo.Filter timeRangeFilter = new BackupInfo.Filter() { + + @Override + public boolean apply(BackupInfo info) { + long time = info.getStartTs(); + return time >= startRangeTime && time <= endRangeTime ; + } + }; + + BackupInfo.Filter tableFilter = new BackupInfo.Filter() { + + @Override + public boolean apply(BackupInfo info) { + List<TableName> tables = info.getTableNames(); + return !Collections.disjoint(allTables, tables); + } + }; + + BackupInfo.Filter typeFilter = new BackupInfo.Filter() { + + @Override + public boolean apply(BackupInfo info) { + return info.getType() == BackupType.INCREMENTAL; + } + }; + + BackupInfo.Filter stateFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + return info.getState() == BackupState.COMPLETE; + } + }; + + List<BackupInfo> allInfos = + table.getBackupHistory( -1, destinationFilter, + timeRangeFilter, tableFilter, typeFilter, stateFilter); + if (allInfos.size() != allBackups.size()) { + // Yes we have at least one hole in backup image sequence + List<String> missingIds = new ArrayList<String>(); + for(BackupInfo info: allInfos) { + if(allBackups.contains(info.getBackupId())) { + continue; + } + missingIds.add(info.getBackupId()); + } + String errMsg = + "Sequence of backup ids has 'holes'. The following backup images must be added:" + + org.apache.hadoop.util.StringUtils.join(",", missingIds); + throw new IOException(errMsg); + } + } +}