http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java deleted file mode 100644 index 99fb06c..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ /dev/null @@ -1,743 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupAdmin; -import org.apache.hadoop.hbase.backup.BackupClientFactory; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupMergeJob; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.RestoreRequest; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -@InterfaceAudience.Private -public class BackupAdminImpl implements BackupAdmin { - public final static String CHECK_OK = "Checking backup images: OK"; - public final static String CHECK_FAILED = - "Checking backup images: Failed. Some dependencies are missing for restore"; - private static final Log LOG = LogFactory.getLog(BackupAdminImpl.class); - - private final Connection conn; - - public BackupAdminImpl(Connection conn) { - this.conn = conn; - } - - @Override - public void close() throws IOException { - } - - @Override - public BackupInfo getBackupInfo(String backupId) throws IOException { - BackupInfo backupInfo = null; - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - if (backupId == null) { - ArrayList<BackupInfo> recentSessions = table.getBackupInfos(BackupState.RUNNING); - if (recentSessions.isEmpty()) { - LOG.warn("No ongoing sessions found."); - return null; - } - // else show status for ongoing session - // must be one maximum - return recentSessions.get(0); - } else { - backupInfo = table.readBackupInfo(backupId); - return backupInfo; - } - } - } - - @Override - public int deleteBackups(String[] backupIds) throws IOException { - - int totalDeleted = 0; - Map<String, HashSet<TableName>> allTablesMap = new HashMap<String, HashSet<TableName>>(); - - boolean deleteSessionStarted = false; - boolean snapshotDone = false; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { - - // Step 1: Make sure there is no active session - // is running by using startBackupSession API - // If there is an active session in progress, exception will be thrown - try { - sysTable.startBackupExclusiveOperation(); - deleteSessionStarted = true; - } catch (IOException e) { - LOG.warn("You can not run delete command while active backup session is in progress. \n" - + "If there is no active backup session running, run backup repair utility to restore \n" - + "backup system integrity."); - return -1; - } - - // Step 2: Make sure there is no failed session - List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING); - if (list.size() != 0) { - // ailed sessions found - LOG.warn("Failed backup session found. Run backup repair tool first."); - return -1; - } - - // Step 3: Record delete session - sysTable.startDeleteOperation(backupIds); - // Step 4: Snapshot backup system table - if (!BackupSystemTable.snapshotExists(conn)) { - BackupSystemTable.snapshot(conn); - } else { - LOG.warn("Backup system table snapshot exists"); - } - snapshotDone = true; - try { - for (int i = 0; i < backupIds.length; i++) { - BackupInfo info = sysTable.readBackupInfo(backupIds[i]); - if (info != null) { - String rootDir = info.getBackupRootDir(); - HashSet<TableName> allTables = allTablesMap.get(rootDir); - if (allTables == null) { - allTables = new HashSet<TableName>(); - allTablesMap.put(rootDir, allTables); - } - allTables.addAll(info.getTableNames()); - totalDeleted += deleteBackup(backupIds[i], sysTable); - } - } - finalizeDelete(allTablesMap, sysTable); - // Finish - sysTable.finishDeleteOperation(); - // delete snapshot - BackupSystemTable.deleteSnapshot(conn); - } catch (IOException e) { - // Fail delete operation - // Step 1 - if (snapshotDone) { - if (BackupSystemTable.snapshotExists(conn)) { - BackupSystemTable.restoreFromSnapshot(conn); - // delete snapshot - BackupSystemTable.deleteSnapshot(conn); - // We still have record with unfinished delete operation - LOG.error("Delete operation failed, please run backup repair utility to restore " - + "backup system integrity", e); - throw e; - } else { - LOG.warn("Delete operation succeeded, there were some errors: ", e); - } - } - - } finally { - if (deleteSessionStarted) { - sysTable.finishBackupExclusiveOperation(); - } - } - } - return totalDeleted; - } - - /** - * Updates incremental backup set for every backupRoot - * @param tablesMap map [backupRoot: Set<TableName>] - * @param table backup system table - * @throws IOException - */ - - private void finalizeDelete(Map<String, HashSet<TableName>> tablesMap, BackupSystemTable table) - throws IOException { - for (String backupRoot : tablesMap.keySet()) { - Set<TableName> incrTableSet = table.getIncrementalBackupTableSet(backupRoot); - Map<TableName, ArrayList<BackupInfo>> tableMap = - table.getBackupHistoryForTableSet(incrTableSet, backupRoot); - for (Map.Entry<TableName, ArrayList<BackupInfo>> entry : tableMap.entrySet()) { - if (entry.getValue() == null) { - // No more backups for a table - incrTableSet.remove(entry.getKey()); - } - } - if (!incrTableSet.isEmpty()) { - table.addIncrementalBackupTableSet(incrTableSet, backupRoot); - } else { // empty - table.deleteIncrementalBackupTableSet(backupRoot); - } - } - } - - /** - * Delete single backup and all related backups <br> - * Algorithm:<br> - * Backup type: FULL or INCREMENTAL <br> - * Is this last backup session for table T: YES or NO <br> - * For every table T from table list 'tables':<br> - * if(FULL, YES) deletes only physical data (PD) <br> - * if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo,<br> - * until we either reach the most recent backup for T in the system or FULL backup<br> - * which includes T<br> - * if(INCREMENTAL, YES) deletes only physical data (PD) if(INCREMENTAL, NO) deletes physical data - * and for table T scans all backup images between last<br> - * FULL backup, which is older than the backup being deleted and the next FULL backup (if exists) <br> - * or last one for a particular table T and removes T from list of backup tables. - * @param backupId backup id - * @param sysTable backup system table - * @return total number of deleted backup images - * @throws IOException - */ - private int deleteBackup(String backupId, BackupSystemTable sysTable) throws IOException { - - BackupInfo backupInfo = sysTable.readBackupInfo(backupId); - - int totalDeleted = 0; - if (backupInfo != null) { - LOG.info("Deleting backup " + backupInfo.getBackupId() + " ..."); - // Step 1: clean up data for backup session (idempotent) - BackupUtils.cleanupBackupData(backupInfo, conn.getConfiguration()); - // List of tables in this backup; - List<TableName> tables = backupInfo.getTableNames(); - long startTime = backupInfo.getStartTs(); - for (TableName tn : tables) { - boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime); - if (isLastBackupSession) { - continue; - } - // else - List<BackupInfo> affectedBackups = getAffectedBackupSessions(backupInfo, tn, sysTable); - for (BackupInfo info : affectedBackups) { - if (info.equals(backupInfo)) { - continue; - } - removeTableFromBackupImage(info, tn, sysTable); - } - } - Map<byte[], String> map = sysTable.readBulkLoadedFiles(backupId); - FileSystem fs = FileSystem.get(conn.getConfiguration()); - boolean success = true; - int numDeleted = 0; - for (String f : map.values()) { - Path p = new Path(f); - try { - LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId()); - if (!fs.delete(p)) { - if (fs.exists(p)) { - LOG.warn(f + " was not deleted"); - success = false; - } - } else { - numDeleted++; - } - } catch (IOException ioe) { - LOG.warn(f + " was not deleted", ioe); - success = false; - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(numDeleted + " bulk loaded files out of " + map.size() + " were deleted"); - } - if (success) { - sysTable.deleteBulkLoadedFiles(map); - } - - sysTable.deleteBackupInfo(backupInfo.getBackupId()); - LOG.info("Delete backup " + backupInfo.getBackupId() + " completed."); - totalDeleted++; - } else { - LOG.warn("Delete backup failed: no information found for backupID=" + backupId); - } - return totalDeleted; - } - - private void - removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSystemTable sysTable) - throws IOException { - List<TableName> tables = info.getTableNames(); - LOG.debug("Remove " + tn + " from " + info.getBackupId() + " tables=" - + info.getTableListAsString()); - if (tables.contains(tn)) { - tables.remove(tn); - - if (tables.isEmpty()) { - LOG.debug("Delete backup info " + info.getBackupId()); - - sysTable.deleteBackupInfo(info.getBackupId()); - // Idempotent operation - BackupUtils.cleanupBackupData(info, conn.getConfiguration()); - } else { - info.setTables(tables); - sysTable.updateBackupInfo(info); - // Now, clean up directory for table (idempotent) - cleanupBackupDir(info, tn, conn.getConfiguration()); - } - } - } - - private List<BackupInfo> getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, - BackupSystemTable table) throws IOException { - LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); - long ts = backupInfo.getStartTs(); - List<BackupInfo> list = new ArrayList<BackupInfo>(); - List<BackupInfo> history = table.getBackupHistory(backupInfo.getBackupRootDir()); - // Scan from most recent to backupInfo - // break when backupInfo reached - for (BackupInfo info : history) { - if (info.getStartTs() == ts) { - break; - } - List<TableName> tables = info.getTableNames(); - if (tables.contains(tn)) { - BackupType bt = info.getType(); - if (bt == BackupType.FULL) { - // Clear list if we encounter FULL backup - list.clear(); - } else { - LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn - + " added " + info.getBackupId() + " tables=" + info.getTableListAsString()); - list.add(info); - } - } - } - return list; - } - - /** - * Clean up the data at target directory - * @throws IOException - */ - private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf) - throws IOException { - try { - // clean up the data at target directory - String targetDir = backupInfo.getBackupRootDir(); - if (targetDir == null) { - LOG.warn("No target directory specified for " + backupInfo.getBackupId()); - return; - } - - FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); - - Path targetDirPath = - new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(), - backupInfo.getBackupId(), table)); - if (outputFs.delete(targetDirPath, true)) { - LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); - } else { - LOG.info("No data has been found in " + targetDirPath.toString() + "."); - } - - } catch (IOException e1) { - LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table - + "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); - throw e1; - } - } - - private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime) - throws IOException { - List<BackupInfo> history = table.getBackupHistory(); - for (BackupInfo info : history) { - List<TableName> tables = info.getTableNames(); - if (!tables.contains(tn)) { - continue; - } - if (info.getStartTs() <= startTime) { - return true; - } else { - return false; - } - } - return false; - } - - @Override - public List<BackupInfo> getHistory(int n) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<BackupInfo> history = table.getBackupHistory(); - if (history.size() <= n) return history; - List<BackupInfo> list = new ArrayList<BackupInfo>(); - for (int i = 0; i < n; i++) { - list.add(history.get(i)); - } - return list; - } - } - - @Override - public List<BackupInfo> getHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) return getHistory(n); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<BackupInfo> history = table.getBackupHistory(); - List<BackupInfo> result = new ArrayList<BackupInfo>(); - for (BackupInfo bi : history) { - if (result.size() == n) break; - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; - } - } - - @Override - public List<BackupSet> listBackupSets() throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<String> list = table.listBackupSets(); - List<BackupSet> bslist = new ArrayList<BackupSet>(); - for (String s : list) { - List<TableName> tables = table.describeBackupSet(s); - if (tables != null) { - bslist.add(new BackupSet(s, tables)); - } - } - return bslist; - } - } - - @Override - public BackupSet getBackupSet(String name) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<TableName> list = table.describeBackupSet(name); - if (list == null) return null; - return new BackupSet(name, list); - } - } - - @Override - public boolean deleteBackupSet(String name) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - if (table.describeBackupSet(name) == null) { - return false; - } - table.deleteBackupSet(name); - return true; - } - } - - @Override - public void addToBackupSet(String name, TableName[] tables) throws IOException { - String[] tableNames = new String[tables.length]; - try (final BackupSystemTable table = new BackupSystemTable(conn); - final Admin admin = conn.getAdmin();) { - for (int i = 0; i < tables.length; i++) { - tableNames[i] = tables[i].getNameAsString(); - if (!admin.tableExists(TableName.valueOf(tableNames[i]))) { - throw new IOException("Cannot add " + tableNames[i] + " because it doesn't exist"); - } - } - table.addToBackupSet(name, tableNames); - LOG.info("Added tables [" + StringUtils.join(tableNames, " ") + "] to '" + name - + "' backup set"); - } - } - - @Override - public void removeFromBackupSet(String name, TableName[] tables) throws IOException { - LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + "'"); - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - table.removeFromBackupSet(name, toStringArray(tables)); - LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name - + "' completed."); - } - } - - private String[] toStringArray(TableName[] list) { - String[] arr = new String[list.length]; - for (int i = 0; i < list.length; i++) { - arr[i] = list[i].toString(); - } - return arr; - } - - @Override - public void restore(RestoreRequest request) throws IOException { - if (request.isCheck()) { - HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>(); - // check and load backup image manifest for the tables - Path rootPath = new Path(request.getBackupRootDir()); - String backupId = request.getBackupId(); - TableName[] sTableArray = request.getFromTables(); - HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, - conn.getConfiguration(), rootPath, backupId); - - // Check and validate the backup image and its dependencies - - if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) { - LOG.info(CHECK_OK); - } else { - LOG.error(CHECK_FAILED); - } - return; - } - // Execute restore request - new RestoreTablesClient(conn, request).execute(); - } - - @Override - public String backupTables(BackupRequest request) throws IOException { - BackupType type = request.getBackupType(); - String targetRootDir = request.getTargetRootDir(); - List<TableName> tableList = request.getTableList(); - - String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime(); - if (type == BackupType.INCREMENTAL) { - Set<TableName> incrTableSet = null; - try (BackupSystemTable table = new BackupSystemTable(conn)) { - incrTableSet = table.getIncrementalBackupTableSet(targetRootDir); - } - - if (incrTableSet.isEmpty()) { - String msg = - "Incremental backup table set contains no tables. " - + "You need to run full backup first " - + (tableList != null ? "on " + StringUtils.join(tableList, ",") : ""); - - throw new IOException(msg); - } - if (tableList != null) { - tableList.removeAll(incrTableSet); - if (!tableList.isEmpty()) { - String extraTables = StringUtils.join(tableList, ","); - String msg = - "Some tables (" + extraTables + ") haven't gone through full backup. " - + "Perform full backup on " + extraTables + " first, " + "then retry the command"; - throw new IOException(msg); - } - } - tableList = Lists.newArrayList(incrTableSet); - } - if (tableList != null && !tableList.isEmpty()) { - for (TableName table : tableList) { - String targetTableBackupDir = - HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table); - Path targetTableBackupDirPath = new Path(targetTableBackupDir); - FileSystem outputFs = - FileSystem.get(targetTableBackupDirPath.toUri(), conn.getConfiguration()); - if (outputFs.exists(targetTableBackupDirPath)) { - throw new IOException("Target backup directory " + targetTableBackupDir - + " exists already."); - } - } - ArrayList<TableName> nonExistingTableList = null; - try (Admin admin = conn.getAdmin();) { - for (TableName tableName : tableList) { - if (!admin.tableExists(tableName)) { - if (nonExistingTableList == null) { - nonExistingTableList = new ArrayList<>(); - } - nonExistingTableList.add(tableName); - } - } - } - if (nonExistingTableList != null) { - if (type == BackupType.INCREMENTAL) { - // Update incremental backup set - tableList = excludeNonExistingTables(tableList, nonExistingTableList); - } else { - // Throw exception only in full mode - we try to backup non-existing table - throw new IOException("Non-existing tables found in the table list: " - + nonExistingTableList); - } - } - } - - // update table list - BackupRequest.Builder builder = new BackupRequest.Builder(); - request = - builder.withBackupType(request.getBackupType()).withTableList(tableList) - .withTargetRootDir(request.getTargetRootDir()) - .withBackupSetName(request.getBackupSetName()).withTotalTasks(request.getTotalTasks()) - .withBandwidthPerTasks((int) request.getBandwidth()).build(); - - TableBackupClient client = null; - try { - client = BackupClientFactory.create(conn, backupId, request); - } catch (IOException e) { - LOG.error("There is an active session already running"); - throw e; - } - - client.execute(); - - return backupId; - } - - private List<TableName> excludeNonExistingTables(List<TableName> tableList, - List<TableName> nonExistingTableList) { - - for (TableName table : nonExistingTableList) { - tableList.remove(table); - } - return tableList; - } - - @Override - public void mergeBackups(String[] backupIds) throws IOException { - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - checkIfValidForMerge(backupIds, sysTable); - BackupMergeJob job = BackupRestoreFactory.getBackupMergeJob(conn.getConfiguration()); - job.run(backupIds); - } - } - - /** - * Verifies that backup images are valid for merge. - * - * <ul> - * <li>All backups MUST be in the same destination - * <li>No FULL backups are allowed - only INCREMENTAL - * <li>All backups must be in COMPLETE state - * <li>No holes in backup list are allowed - * </ul> - * <p> - * @param backupIds list of backup ids - * @param table backup system table - * @throws IOException - */ - private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) throws IOException { - String backupRoot = null; - - final Set<TableName> allTables = new HashSet<TableName>(); - final Set<String> allBackups = new HashSet<String>(); - long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE; - for (String backupId : backupIds) { - BackupInfo bInfo = table.readBackupInfo(backupId); - if (bInfo == null) { - String msg = "Backup session " + backupId + " not found"; - throw new IOException(msg); - } - if (backupRoot == null) { - backupRoot = bInfo.getBackupRootDir(); - } else if (!bInfo.getBackupRootDir().equals(backupRoot)) { - throw new IOException("Found different backup destinations in a list of a backup sessions \n" - + "1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir()); - } - if (bInfo.getType() == BackupType.FULL) { - throw new IOException("FULL backup image can not be merged for: \n" + bInfo); - } - - if (bInfo.getState() != BackupState.COMPLETE) { - throw new IOException("Backup image " + backupId - + " can not be merged becuase of its state: " + bInfo.getState()); - } - allBackups.add(backupId); - allTables.addAll(bInfo.getTableNames()); - long time = bInfo.getStartTs(); - if (time < minTime) { - minTime = time; - } - if (time > maxTime) { - maxTime = time; - } - } - - - final long startRangeTime = minTime; - final long endRangeTime = maxTime; - final String backupDest = backupRoot; - // Check we have no 'holes' in backup id list - // Filter 1 : backupRoot - // Filter 2 : time range filter - // Filter 3 : table filter - - BackupInfo.Filter destinationFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - return info.getBackupRootDir().equals(backupDest); - } - }; - - BackupInfo.Filter timeRangeFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - long time = info.getStartTs(); - return time >= startRangeTime && time <= endRangeTime ; - } - }; - - BackupInfo.Filter tableFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - List<TableName> tables = info.getTableNames(); - return !Collections.disjoint(allTables, tables); - } - }; - - BackupInfo.Filter typeFilter = new BackupInfo.Filter() { - - @Override - public boolean apply(BackupInfo info) { - return info.getType() == BackupType.INCREMENTAL; - } - }; - - BackupInfo.Filter stateFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - return info.getState() == BackupState.COMPLETE; - } - }; - - List<BackupInfo> allInfos = - table.getBackupHistory( -1, destinationFilter, - timeRangeFilter, tableFilter, typeFilter, stateFilter); - if (allInfos.size() != allBackups.size()) { - // Yes we have at least one hole in backup image sequence - List<String> missingIds = new ArrayList<String>(); - for(BackupInfo info: allInfos) { - if(allBackups.contains(info.getBackupId())) { - continue; - } - missingIds.add(info.getBackupId()); - } - String errMsg = - "Sequence of backup ids has 'holes'. The following backup images must be added:" + - org.apache.hadoop.util.StringUtils.join(",", missingIds); - throw new IOException(errMsg); - } - } -}
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java deleted file mode 100644 index 650ba2e..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ /dev/null @@ -1,1022 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; -import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; - -import java.io.IOException; -import java.net.URI; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupAdmin; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.backup.util.BackupUtils; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -/** - * General backup commands, options and usage messages - */ - -@InterfaceAudience.Private -public final class BackupCommands { - - public final static String INCORRECT_USAGE = "Incorrect usage"; - - public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" - + "where COMMAND is one of:\n" + " create create a new backup image\n" - + " delete delete an existing backup image\n" - + " describe show the detailed information of a backup image\n" - + " history show history of all successful backups\n" - + " progress show the progress of the latest backup request\n" - + " set backup set management\n" - + " repair repair backup system table\n" - + " merge merge backup images\n" - + "Run \'hbase backup COMMAND -h\' to see help message for each command\n"; - - public static final String CREATE_CMD_USAGE = - "Usage: hbase backup create <type> <backup_path> [options]\n" - + " type \"full\" to create a full backup image\n" - + " \"incremental\" to create an incremental backup image\n" - + " backup_path Full path to store the backup image\n"; - - public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n" - + " backup_id Backup image id (optional). If no id specified, the command will show\n" - + " progress for currently running backup session."; - public static final String NO_INFO_FOUND = "No info was found for backup id: "; - public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found."; - - public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup describe <backup_id>\n" - + " backup_id Backup image id\n"; - - public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; - - public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backup_id>\n" - + " backup_id Backup image id\n"; - - public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; - - public static final String CANCEL_CMD_USAGE = "Usage: hbase backup cancel <backup_id>\n" - + " backup_id Backup image id\n"; - - public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" - + " name Backup set name\n" - + " tables Comma separated list of tables.\n" + "COMMAND is one of:\n" - + " add add tables to a set, create a set if needed\n" - + " remove remove tables from a set\n" - + " list list all backup sets in the system\n" - + " describe describe set\n" + " delete delete backup set\n"; - public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n" - + " backup_ids Comma separated list of backup image ids.\n"; - - public static final String USAGE_FOOTER = ""; - - public static abstract class Command extends Configured { - CommandLine cmdline; - Connection conn; - - Command(Configuration conf) { - if (conf == null) { - conf = HBaseConfiguration.create(); - } - setConf(conf); - } - - public void execute() throws IOException { - if (cmdline.hasOption("h") || cmdline.hasOption("help")) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - // Create connection - conn = ConnectionFactory.createConnection(getConf()); - if (requiresNoActiveSession()) { - // Check active session - try (BackupSystemTable table = new BackupSystemTable(conn);) { - List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING); - - if (sessions.size() > 0) { - System.err.println("Found backup session in a RUNNING state: "); - System.err.println(sessions.get(0)); - System.err.println("This may indicate that a previous session has failed abnormally."); - System.err.println("In this case, backup recovery is recommended."); - throw new IOException("Active session found, aborted command execution"); - } - } - } - if (requiresConsistentState()) { - // Check failed delete - try (BackupSystemTable table = new BackupSystemTable(conn);) { - String[] ids = table.getListOfBackupIdsFromDeleteOperation(); - - if (ids != null && ids.length > 0) { - System.err.println("Found failed backup DELETE coommand. "); - System.err.println("Backup system recovery is required."); - throw new IOException("Failed backup DELETE found, aborted command execution"); - } - - ids = table.getListOfBackupIdsFromMergeOperation(); - if (ids != null && ids.length > 0) { - System.err.println("Found failed backup MERGE coommand. "); - System.err.println("Backup system recovery is required."); - throw new IOException("Failed backup MERGE found, aborted command execution"); - } - - } - } - } - - public void finish() throws IOException { - if (conn != null) { - conn.close(); - } - } - - protected abstract void printUsage(); - - /** - * The command can't be run if active backup session is in progress - * @return true if no active sessions are in progress - */ - protected boolean requiresNoActiveSession() { - return false; - } - - /** - * Command requires consistent state of a backup system Backup system may become inconsistent - * because of an abnormal termination of a backup session or delete command - * @return true, if yes - */ - protected boolean requiresConsistentState() { - return false; - } - } - - private BackupCommands() { - throw new AssertionError("Instantiating utility class..."); - } - - public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { - Command cmd = null; - switch (type) { - case CREATE: - cmd = new CreateCommand(conf, cmdline); - break; - case DESCRIBE: - cmd = new DescribeCommand(conf, cmdline); - break; - case PROGRESS: - cmd = new ProgressCommand(conf, cmdline); - break; - case DELETE: - cmd = new DeleteCommand(conf, cmdline); - break; - case CANCEL: - cmd = new CancelCommand(conf, cmdline); - break; - case HISTORY: - cmd = new HistoryCommand(conf, cmdline); - break; - case SET: - cmd = new BackupSetCommand(conf, cmdline); - break; - case REPAIR: - cmd = new RepairCommand(conf, cmdline); - break; - case MERGE: - cmd = new MergeCommand(conf, cmdline); - break; - case HELP: - default: - cmd = new HelpCommand(conf, cmdline); - break; - } - return cmd; - } - - static int numOfArgs(String[] args) { - if (args == null) return 0; - return args.length; - } - - public static class CreateCommand extends Command { - - CreateCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - protected boolean requiresConsistentState() { - return true; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) - && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { - System.out.println("ERROR: invalid backup type: " + args[1]); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - if (!verifyPath(args[2])) { - System.out.println("ERROR: invalid backup destination: " + args[2]); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String tables = null; - - // Check if we have both: backup set and list of tables - if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) { - System.out.println("ERROR: You can specify either backup set or list" - + " of tables, but not both"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - // Creates connection - super.execute(); - // Check backup set - String setName = null; - if (cmdline.hasOption(OPTION_SET)) { - setName = cmdline.getOptionValue(OPTION_SET); - tables = getTablesForSet(setName, getConf()); - - if (tables == null) { - System.out.println("ERROR: Backup set '" + setName - + "' is either empty or does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } else { - tables = cmdline.getOptionValue(OPTION_TABLE); - } - int bandwidth = - cmdline.hasOption(OPTION_BANDWIDTH) ? Integer.parseInt(cmdline - .getOptionValue(OPTION_BANDWIDTH)) : -1; - int workers = - cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline - .getOptionValue(OPTION_WORKERS)) : -1; - - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - - BackupRequest.Builder builder = new BackupRequest.Builder(); - BackupRequest request = - builder - .withBackupType(BackupType.valueOf(args[1].toUpperCase())) - .withTableList( - tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null) - .withTargetRootDir(args[2]).withTotalTasks(workers) - .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build(); - String backupId = admin.backupTables(request); - System.out.println("Backup session " + backupId + " finished. Status: SUCCESS"); - } catch (IOException e) { - System.out.println("Backup session finished. Status: FAILURE"); - throw e; - } - } - - private boolean verifyPath(String path) { - try { - Path p = new Path(path); - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - URI uri = p.toUri(); - if (uri.getScheme() == null) return false; - FileSystem.get(uri, conf); - return true; - } catch (Exception e) { - return false; - } - } - - private String getTablesForSet(String name, Configuration conf) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List<TableName> tables = table.describeBackupSet(name); - if (tables == null) return null; - return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - } - } - - @Override - protected void printUsage() { - System.out.println(CREATE_CMD_USAGE); - Options options = new Options(); - options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC); - options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); - options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC); - options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC); - - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setLeftPadding(2); - helpFormatter.setDescPadding(8); - helpFormatter.setWidth(100); - helpFormatter.setSyntaxPrefix("Options:"); - helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); - - } - } - - private static class HelpCommand extends Command { - - HelpCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - if (args == null || args.length == 0) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (args.length != 2) { - System.out.println("ERROR: Only supports help message of a single command type"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String type = args[1]; - - if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { - System.out.println(CREATE_CMD_USAGE); - } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { - System.out.println(DESCRIBE_CMD_USAGE); - } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { - System.out.println(HISTORY_CMD_USAGE); - } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { - System.out.println(PROGRESS_CMD_USAGE); - } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { - System.out.println(DELETE_CMD_USAGE); - } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) { - System.out.println(CANCEL_CMD_USAGE); - } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { - System.out.println(SET_CMD_USAGE); - } else { - System.out.println("Unknown command : " + type); - printUsage(); - } - } - - @Override - protected void printUsage() { - System.out.println(USAGE); - } - } - - private static class DescribeCommand extends Command { - - DescribeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length != 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String backupId = args[1]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - BackupInfo info = sysTable.readBackupInfo(backupId); - if (info == null) { - System.out.println("ERROR: " + backupId + " does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - System.out.println(info.getShortDescription()); - } - } - - @Override - protected void printUsage() { - System.out.println(DESCRIBE_CMD_USAGE); - } - } - - private static class ProgressCommand extends Command { - - ProgressCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) { - System.out.println("No backup id was specified, " - + "will retrieve the most recent (ongoing) session"); - } - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args != null && args.length > 2) { - System.err.println("ERROR: wrong number of arguments: " + args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String backupId = (args == null || args.length <= 1) ? null : args[1]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - BackupInfo info = null; - - if (backupId != null) { - info = sysTable.readBackupInfo(backupId); - } else { - List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING); - if (infos != null && infos.size() > 0) { - info = infos.get(0); - backupId = info.getBackupId(); - System.out.println("Found ongoing session with backupId=" + backupId); - } else { - } - } - int progress = info == null ? -1 : info.getProgress(); - if (progress < 0) { - if (backupId != null) { - System.out.println(NO_INFO_FOUND + backupId); - } else { - System.err.println(NO_ACTIVE_SESSION_FOUND); - } - } else { - System.out.println(backupId + " progress=" + progress + "%"); - } - } - } - - @Override - protected void printUsage() { - System.out.println(PROGRESS_CMD_USAGE); - } - } - - private static class DeleteCommand extends Command { - - DeleteCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - super.execute(); - - String[] args = cmdline.getArgs(); - String[] backupIds = new String[args.length - 1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - int deleted = admin.deleteBackups(backupIds); - System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length); - } catch (IOException e) { - System.err - .println("Delete command FAILED. Please run backup repair tool to restore backup system integrity"); - throw e; - } - - } - - @Override - protected void printUsage() { - System.out.println(DELETE_CMD_USAGE); - } - } - - private static class RepairCommand extends Command { - - RepairCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args != null && args.length > 1) { - System.err.println("ERROR: wrong number of arguments: " + args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - - // Failed backup - BackupInfo backupInfo; - List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING); - if (list.size() == 0) { - // No failed sessions found - System.out.println("REPAIR status: no failed sessions found." - + " Checking failed delete backup operation ..."); - repairFailedBackupDeletionIfAny(conn, sysTable); - repairFailedBackupMergeIfAny(conn, sysTable); - return; - } - backupInfo = list.get(0); - // If this is a cancel exception, then we've already cleaned. - // set the failure timestamp of the overall backup - backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); - // set failure message - backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo); - // set overall backup status: failed - backupInfo.setState(BackupState.FAILED); - // compose the backup failed data - String backupFailedData = - "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs() - + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" - + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); - System.out.println(backupFailedData); - TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf); - // If backup session is updated to FAILED state - means we - // processed recovery already. - sysTable.updateBackupInfo(backupInfo); - sysTable.finishBackupExclusiveOperation(); - System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo); - - } - } - - private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable) - throws IOException { - String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation(); - if (backupIds == null || backupIds.length == 0) { - System.out.println("No failed backup DELETE operation found"); - // Delete backup table snapshot if exists - BackupSystemTable.deleteSnapshot(conn); - return; - } - System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds)); - System.out.println("Running DELETE again ..."); - // Restore table from snapshot - BackupSystemTable.restoreFromSnapshot(conn); - // Finish previous failed session - sysTable.finishBackupExclusiveOperation(); - try (BackupAdmin admin = new BackupAdminImpl(conn);) { - admin.deleteBackups(backupIds); - } - System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds)); - - } - - private void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable) - throws IOException { - String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation(); - if (backupIds == null || backupIds.length == 0) { - System.out.println("No failed backup MERGE operation found"); - // Delete backup table snapshot if exists - BackupSystemTable.deleteSnapshot(conn); - return; - } - System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds)); - System.out.println("Running MERGE again ..."); - // Restore table from snapshot - BackupSystemTable.restoreFromSnapshot(conn); - // Unlock backupo system - sysTable.finishBackupExclusiveOperation(); - // Finish previous failed session - sysTable.finishMergeOperation(); - try (BackupAdmin admin = new BackupAdminImpl(conn);) { - admin.mergeBackups(backupIds); - } - System.out.println("MERGE operation finished OK: " + StringUtils.join(backupIds)); - - } - - @Override - protected void printUsage() { - System.out.println(REPAIR_CMD_USAGE); - } - } - - private static class MergeCommand extends Command { - - MergeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - protected boolean requiresNoActiveSession() { - return true; - } - - @Override - protected boolean requiresConsistentState() { - return true; - } - - @Override - public void execute() throws IOException { - super.execute(); - - String[] args = cmdline == null ? null : cmdline.getArgs(); - if (args == null || (args.length != 2)) { - System.err.println("ERROR: wrong number of arguments: " - + (args == null ? null : args.length)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] backupIds = args[1].split(","); - if (backupIds.length < 2) { - String msg = "ERROR: can not merge a single backup image. "+ - "Number of images must be greater than 1."; - System.err.println(msg); - throw new IOException(msg); - - } - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.mergeBackups(backupIds); - } - } - - @Override - protected void printUsage() { - System.out.println(MERGE_CMD_USAGE); - } - } - - // TODO Cancel command - - private static class CancelCommand extends Command { - - CancelCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - throw new UnsupportedOperationException("Cancel command is not supported yet."); - } - - @Override - protected void printUsage() { - } - } - - private static class HistoryCommand extends Command { - - private final static int DEFAULT_HISTORY_LENGTH = 10; - - HistoryCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - int n = parseHistoryLength(); - final TableName tableName = getTableName(); - final String setName = getTableSetName(); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (tableName == null) return true; - List<TableName> names = info.getTableNames(); - return names.contains(tableName); - } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (setName == null) return true; - String backupId = info.getBackupId(); - return backupId.startsWith(setName); - } - }; - Path backupRootPath = getBackupRootPath(); - List<BackupInfo> history = null; - if (backupRootPath == null) { - // Load from backup system table - super.execute(); - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); - } - } else { - // load from backup FS - history = - BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter); - } - for (BackupInfo info : history) { - System.out.println(info.getShortDescription()); - } - } - - private Path getBackupRootPath() throws IOException { - String value = null; - try { - value = cmdline.getOptionValue(OPTION_PATH); - if (value == null) return null; - return new Path(value); - } catch (IllegalArgumentException e) { - System.out.println("ERROR: Illegal argument for backup root path: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private TableName getTableName() throws IOException { - String value = cmdline.getOptionValue(OPTION_TABLE); - if (value == null) return null; - try { - return TableName.valueOf(value); - } catch (IllegalArgumentException e) { - System.out.println("Illegal argument for table name: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private String getTableSetName() throws IOException { - String value = cmdline.getOptionValue(OPTION_SET); - return value; - } - - private int parseHistoryLength() throws IOException { - String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER); - try { - if (value == null) return DEFAULT_HISTORY_LENGTH; - return Integer.parseInt(value); - } catch (NumberFormatException e) { - System.out.println("Illegal argument for history length: " + value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.out.println(HISTORY_CMD_USAGE); - Options options = new Options(); - options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC); - options.addOption(OPTION_PATH, true, OPTION_PATH_DESC); - options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC); - options.addOption(OPTION_SET, true, OPTION_SET_DESC); - - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setLeftPadding(2); - helpFormatter.setDescPadding(8); - helpFormatter.setWidth(100); - helpFormatter.setSyntaxPrefix("Options:"); - helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); - } - } - - private static class BackupSetCommand extends Command { - private final static String SET_ADD_CMD = "add"; - private final static String SET_REMOVE_CMD = "remove"; - private final static String SET_DELETE_CMD = "delete"; - private final static String SET_DESCRIBE_CMD = "describe"; - private final static String SET_LIST_CMD = "list"; - - BackupSetCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - // Command-line must have at least one element - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - String cmdStr = args[1]; - BackupCommand cmd = getCommand(cmdStr); - - switch (cmd) { - case SET_ADD: - processSetAdd(args); - break; - case SET_REMOVE: - processSetRemove(args); - break; - case SET_DELETE: - processSetDelete(args); - break; - case SET_DESCRIBE: - processSetDescribe(args); - break; - case SET_LIST: - processSetList(args); - break; - default: - break; - - } - } - - private void processSetList(String[] args) throws IOException { - super.execute(); - - // List all backup set names - // does not expect any args - try (BackupAdminImpl admin = new BackupAdminImpl(conn);) { - List<BackupSet> list = admin.listBackupSets(); - for (BackupSet bs : list) { - System.out.println(bs); - } - } - } - - private void processSetDescribe(String[] args) throws IOException { - if (args == null || args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) { - List<TableName> tables = sysTable.describeBackupSet(setName); - BackupSet set = tables == null ? null : new BackupSet(setName, tables); - if (set == null) { - System.out.println("Set '" + setName + "' does not exist."); - } else { - System.out.println(set); - } - } - } - - private void processSetDelete(String[] args) throws IOException { - if (args == null || args.length != 3) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - boolean result = admin.deleteBackupSet(setName); - if (result) { - System.out.println("Delete set " + setName + " OK."); - } else { - System.out.println("Set " + setName + " does not exist"); - } - } - } - - private void processSetRemove(String[] args) throws IOException { - if (args == null || args.length != 4) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - String[] tables = args[3].split(","); - TableName[] tableNames = toTableNames(tables); - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.removeFromBackupSet(setName, tableNames); - } - } - - private TableName[] toTableNames(String[] tables) { - TableName[] arr = new TableName[tables.length]; - for (int i = 0; i < tables.length; i++) { - arr[i] = TableName.valueOf(tables[i]); - } - return arr; - } - - private void processSetAdd(String[] args) throws IOException { - if (args == null || args.length != 4) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - super.execute(); - - String setName = args[2]; - String[] tables = args[3].split(","); - TableName[] tableNames = new TableName[tables.length]; - for (int i = 0; i < tables.length; i++) { - tableNames[i] = TableName.valueOf(tables[i]); - } - try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) { - admin.addToBackupSet(setName, tableNames); - } - - } - - private BackupCommand getCommand(String cmdStr) throws IOException { - if (cmdStr.equals(SET_ADD_CMD)) { - return BackupCommand.SET_ADD; - } else if (cmdStr.equals(SET_REMOVE_CMD)) { - return BackupCommand.SET_REMOVE; - } else if (cmdStr.equals(SET_DELETE_CMD)) { - return BackupCommand.SET_DELETE; - } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { - return BackupCommand.SET_DESCRIBE; - } else if (cmdStr.equals(SET_LIST_CMD)) { - return BackupCommand.SET_LIST; - } else { - System.out.println("ERROR: Unknown command for 'set' :" + cmdStr); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.out.println(SET_CMD_USAGE); - } - - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java deleted file mode 100644 index 2c7d35f..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * Backup exception - */ -@SuppressWarnings("serial") -@InterfaceAudience.Private -public class BackupException extends HBaseIOException { - private BackupInfo info; - - /** - * Some exception happened for a backup and don't even know the backup that it was about - * @param msg Full description of the failure - */ - public BackupException(String msg) { - super(msg); - } - - /** - * Some exception happened for a backup with a cause - * @param cause the cause - */ - public BackupException(Throwable cause) { - super(cause); - } - - /** - * Exception for the given backup that has no previous root cause - * @param msg reason why the backup failed - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, BackupInfo desc) { - super(msg); - this.info = desc; - } - - /** - * Exception for the given backup due to another exception - * @param msg reason why the backup failed - * @param cause root cause of the failure - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, Throwable cause, BackupInfo desc) { - super(msg, cause); - this.info = desc; - } - - /** - * Exception when the description of the backup cannot be determined, due to some other root - * cause - * @param message description of what caused the failure - * @param e root cause - */ - public BackupException(String message, Exception e) { - super(message, e); - } - - public BackupInfo getBackupInfo() { - return this.info; - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java deleted file mode 100644 index 8fe5eaf..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; -import org.apache.hadoop.hbase.backup.master.BackupLogCleaner; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.procedure.ProcedureManagerHost; -import org.apache.hadoop.hbase.util.Pair; - -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; - -/** - * Handles backup requests, creates backup info records in backup system table to - * keep track of backup sessions, dispatches backup request. - */ -@InterfaceAudience.Private -public class BackupManager implements Closeable { - private static final Log LOG = LogFactory.getLog(BackupManager.class); - - protected Configuration conf = null; - protected BackupInfo backupInfo = null; - protected BackupSystemTable systemTable; - protected final Connection conn; - - /** - * Backup manager constructor. - * @param conn connection - * @param conf configuration - * @throws IOException exception - */ - public BackupManager(Connection conn, Configuration conf) throws IOException { - if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, - BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) { - throw new BackupException("HBase backup is not enabled. Check your " - + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting."); - } - this.conf = conf; - this.conn = conn; - this.systemTable = new BackupSystemTable(conn); - - } - - /** - * Returns backup info - */ - protected BackupInfo getBackupInfo() { - return backupInfo; - } - - /** - * This method modifies the master's configuration in order to inject backup-related features - * (TESTs only) - * @param conf configuration - */ - @VisibleForTesting - public static void decorateMasterConfiguration(Configuration conf) { - if (!isBackupEnabled(conf)) { - return; - } - // Add WAL archive cleaner plug-in - String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS); - String cleanerClass = BackupLogCleaner.class.getCanonicalName(); - if (!plugins.contains(cleanerClass)) { - conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass); - } - - String classes = conf.get(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY); - String masterProcedureClass = LogRollMasterProcedureManager.class.getName(); - if (classes == null) { - conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, masterProcedureClass); - } else if (!classes.contains(masterProcedureClass)) { - conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, classes + "," + masterProcedureClass); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Added log cleaner: " + cleanerClass + "\n" + "Added master procedure manager: " - + masterProcedureClass); - } - - } - - /** - * This method modifies the Region Server configuration in order to inject backup-related features - * TESTs only. - * @param conf configuration - */ - @VisibleForTesting - public static void decorateRegionServerConfiguration(Configuration conf) { - if (!isBackupEnabled(conf)) { - return; - } - - String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY); - String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName(); - if (classes == null) { - conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass); - } else if (!classes.contains(regionProcedureClass)) { - conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, classes + "," - + regionProcedureClass); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Added region procedure manager: " + regionProcedureClass); - } - - } - - public static boolean isBackupEnabled(Configuration conf) { - return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, - BackupRestoreConstants.BACKUP_ENABLE_DEFAULT); - } - - /** - * Get configuration - * @return configuration - */ - Configuration getConf() { - return conf; - } - - /** - * Stop all the work of backup. - */ - @Override - public void close() { - - if (systemTable != null) { - try { - systemTable.close(); - } catch (Exception e) { - LOG.error(e); - } - } - } - - /** - * Creates a backup info based on input backup request. - * @param backupId backup id - * @param type type - * @param tableList table list - * @param targetRootDir root dir - * @param workers number of parallel workers - * @param bandwidth bandwidth per worker in MB per sec - * @return BackupInfo - * @throws BackupException exception - */ - public BackupInfo createBackupInfo(String backupId, BackupType type, List<TableName> tableList, - String targetRootDir, int workers, long bandwidth) throws BackupException { - if (targetRootDir == null) { - throw new BackupException("Wrong backup request parameter: target backup root directory"); - } - - if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) { - // If table list is null for full backup, which means backup all tables. Then fill the table - // list with all user tables from meta. It no table available, throw the request exception. - - HTableDescriptor[] htds = null; - try (Admin admin = conn.getAdmin()) { - htds = admin.listTables(); - } catch (Exception e) { - throw new BackupException(e); - } - - if (htds == null) { - throw new BackupException("No table exists for full backup of all tables."); - } else { - tableList = new ArrayList<>(); - for (HTableDescriptor hTableDescriptor : htds) { - TableName tn = hTableDescriptor.getTableName(); - if (tn.equals(BackupSystemTable.getTableName(conf))) { - // skip backup system table - continue; - } - tableList.add(hTableDescriptor.getTableName()); - } - - LOG.info("Full backup all the tables available in the cluster: " + tableList); - } - } - - // there are one or more tables in the table list - backupInfo = - new BackupInfo(backupId, type, tableList.toArray(new TableName[tableList.size()]), - targetRootDir); - backupInfo.setBandwidth(bandwidth); - backupInfo.setWorkers(workers); - return backupInfo; - } - - /** - * Check if any ongoing backup. Currently, we only reply on checking status in backup system - * table. We need to consider to handle the case of orphan records in the future. Otherwise, all - * the coming request will fail. - * @return the ongoing backup id if on going backup exists, otherwise null - * @throws IOException exception - */ - private String getOngoingBackupId() throws IOException { - - ArrayList<BackupInfo> sessions = systemTable.getBackupInfos(BackupState.RUNNING); - if (sessions.size() == 0) { - return null; - } - return sessions.get(0).getBackupId(); - } - - /** - * Start the backup manager service. - * @throws IOException exception - */ - public void initialize() throws IOException { - String ongoingBackupId = this.getOngoingBackupId(); - if (ongoingBackupId != null) { - LOG.info("There is a ongoing backup " + ongoingBackupId - + ". Can not launch new backup until no ongoing backup remains."); - throw new BackupException("There is ongoing backup."); - } - } - - public void setBackupInfo(BackupInfo backupInfo) { - this.backupInfo = backupInfo; - } - - /** - * Get direct ancestors of the current backup. - * @param backupInfo The backup info for the current backup - * @return The ancestors for the current backup - * @throws IOException exception - * @throws BackupException exception - */ - public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo) throws IOException, - BackupException { - LOG.debug("Getting the direct ancestors of the current backup " + backupInfo.getBackupId()); - - ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>(); - - // full backup does not have ancestor - if (backupInfo.getType() == BackupType.FULL) { - LOG.debug("Current backup is a full backup, no direct ancestor for it."); - return ancestors; - } - - // get all backup history list in descending order - - ArrayList<BackupInfo> allHistoryList = getBackupHistory(true); - for (BackupInfo backup : allHistoryList) { - - BackupImage.Builder builder = BackupImage.newBuilder(); - - BackupImage image = - builder.withBackupId(backup.getBackupId()).withType(backup.getType()) - .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) - .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); - - // add the full backup image as an ancestor until the last incremental backup - if (backup.getType().equals(BackupType.FULL)) { - // check the backup image coverage, if previous image could be covered by the newer ones, - // then no need to add - if (!BackupManifest.canCoverImage(ancestors, image)) { - ancestors.add(image); - } - } else { - // found last incremental backup, if previously added full backup ancestor images can cover - // it, then this incremental ancestor is not the dependent of the current incremental - // backup, that is to say, this is the backup scope boundary of current table set. - // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing - // incremental backup - if (BackupManifest.canCoverImage(ancestors, image)) { - LOG.debug("Met the backup boundary of the current table set:"); - for (BackupImage image1 : ancestors) { - LOG.debug(" BackupID=" + image1.getBackupId() + ", BackupDir=" + image1.getRootDir()); - } - } else { - Path logBackupPath = - HBackupFileSystem.getBackupPath(backup.getBackupRootDir(), backup.getBackupId()); - LOG.debug("Current backup has an incremental backup ancestor, " - + "touching its image manifest in " + logBackupPath.toString() - + " to construct the dependency."); - BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath); - BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage(); - ancestors.add(lastIncrImage); - - LOG.debug("Last dependent incremental backup image: " + "{BackupID=" - + lastIncrImage.getBackupId() + "," + "BackupDir=" + lastIncrImage.getRootDir() + "}"); - } - } - } - LOG.debug("Got " + ancestors.size() + " ancestors for the current backup."); - return ancestors; - } - - /** - * Get the direct ancestors of this backup for one table involved. - * @param backupInfo backup info - * @param table table - * @return backupImages on the dependency list - * @throws BackupException exception - * @throws IOException exception - */ - public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo, TableName table) - throws BackupException, IOException { - ArrayList<BackupImage> ancestors = getAncestors(backupInfo); - ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>(); - for (BackupImage image : ancestors) { - if (image.hasTable(table)) { - tableAncestors.add(image); - if (image.getType() == BackupType.FULL) { - break; - } - } - } - return tableAncestors; - } - - /* - * backup system table operations - */ - - /** - * Updates status (state) of a backup session in a persistent store - * @param context context - * @throws IOException exception - */ - public void updateBackupInfo(BackupInfo context) throws IOException { - systemTable.updateBackupInfo(context); - } - - /** - * Starts new backup session - * @throws IOException if active session already exists - */ - public void startBackupSession() throws IOException { - systemTable.startBackupExclusiveOperation(); - } - - /** - * Finishes active backup session - * @throws IOException if no active session - */ - public void finishBackupSession() throws IOException { - systemTable.finishBackupExclusiveOperation(); - } - - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no startcode stored in backup system table or the value is of length 0. These two - * cases indicate there is no successful backup completed so far. - * @return the timestamp of a last successful backup - * @throws IOException exception - */ - public String readBackupStartCode() throws IOException { - return systemTable.readBackupStartCode(backupInfo.getBackupRootDir()); - } - - /** - * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. - * @param startCode start code - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode) throws IOException { - systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir()); - } - - /** - * Get the RS log information after the last log roll from backup system table. - * @return RS log info - * @throws IOException exception - */ - public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException { - return systemTable.readRegionServerLastLogRollResult(backupInfo.getBackupRootDir()); - } - - public Pair<Map<TableName, Map<String, Map<String, List<Pair<String, Boolean>>>>>, List<byte[]>> - readBulkloadRows(List<TableName> tableList) throws IOException { - return systemTable.readBulkloadRows(tableList); - } - - public void removeBulkLoadedRows(List<TableName> lst, List<byte[]> rows) throws IOException { - systemTable.removeBulkLoadedRows(lst, rows); - } - - public void writeBulkLoadedFiles(List<TableName> sTableList, Map<byte[], List<Path>>[] maps) - throws IOException { - systemTable.writeBulkLoadedFiles(sTableList, maps, backupInfo.getBackupId()); - } - - /** - * Get all completed backup information (in desc order by time) - * @return history info of BackupCompleteData - * @throws IOException exception - */ - public List<BackupInfo> getBackupHistory() throws IOException { - return systemTable.getBackupHistory(); - } - - public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException { - return systemTable.getBackupHistory(completed); - } - - /** - * Write the current timestamps for each regionserver to backup system table after a successful - * full or incremental backup. Each table may have a different set of log timestamps. The saved - * timestamp is of the last log file that was backed up already. - * @param tables tables - * @throws IOException exception - */ - public void writeRegionServerLogTimestamp(Set<TableName> tables, - HashMap<String, Long> newTimestamps) throws IOException { - systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir()); - } - - /** - * Read the timestamp for each region server log after the last successful backup. Each table has - * its own set of the timestamps. - * @return the timestamp for each region server. key: tableName value: - * RegionServer,PreviousTimeStamp - * @throws IOException exception - */ - public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException { - return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir()); - } - - /** - * Return the current tables covered by incremental backup. - * @return set of tableNames - * @throws IOException exception - */ - public Set<TableName> getIncrementalBackupTableSet() throws IOException { - return systemTable.getIncrementalBackupTableSet(backupInfo.getBackupRootDir()); - } - - /** - * Adds set of tables to overall incremental backup table set - * @param tables tables - * @throws IOException exception - */ - public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException { - systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir()); - } - - /** - * Saves list of WAL files after incremental backup operation. These files will be stored until - * TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be - * safely purged. - */ - public void recordWALFiles(List<String> files) throws IOException { - systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir()); - } - - /** - * Get WAL files iterator - * @return WAL files iterator from backup system table - * @throws IOException - */ - public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException { - return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir()); - } - - public Connection getConnection() { - return conn; - } -}