This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a09c0c8 HBASE-22899 logging improvements for snapshot operations
w/large manifests (#547)
a09c0c8 is described below
commit a09c0c88b694af25575fb50812e14fb667f5ac5b
Author: ravowlga123 <[email protected]>
AuthorDate: Sat Sep 14 19:16:49 2019 +0200
HBASE-22899 logging improvements for snapshot operations w/large manifests
(#547)
Signed-off-by: Jan Hentschel <[email protected]>
---
.../hbase/snapshot/RestoreSnapshotHelper.java | 21 ++++++++++++++-------
.../hadoop/hbase/snapshot/SnapshotManifest.java | 9 ++++++---
2 files changed, 20 insertions(+), 10 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index f533af4..43d892f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -501,6 +501,7 @@ public class RestoreSnapshotHelper {
getRegionHFileReferences(regionManifest);
String tableName = tableDesc.getTableName().getNameAsString();
+ final String snapshotName = snapshotDesc.getName();
// Restore families present in the table
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
@@ -523,20 +524,21 @@ public class RestoreSnapshotHelper {
// Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName);
- LOG.trace("Removing hfile=" + hfileName +
+ LOG.trace("Removing HFile=" + hfileName + " not present in
snapshot=" + snapshotName+
" from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir,
family, hfile);
}
// Restore Missing files
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
- LOG.debug("Adding HFileLink " + storeFile.getName() +
- " to region=" + regionInfo.getEncodedName() + " table=" +
tableName);
+ LOG.debug("Restoring missing HFileLink " + storeFile.getName() +
+ " of snapshot=" + snapshotName+
+ " to region=" + regionInfo.getEncodedName() + " table=" +
tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
- LOG.trace("Removing family=" + Bytes.toString(family) +
+ LOG.trace("Removing family=" + Bytes.toString(family) + " in
snapshot=" + snapshotName +
" from region=" + regionInfo.getEncodedName() + " table=" +
tableName);
HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo,
familyDir, family);
fs.delete(familyDir, true);
@@ -552,7 +554,8 @@ public class RestoreSnapshotHelper {
}
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue())
{
- LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" +
tableName);
+ LOG.trace("Adding HFileLink (Not present in the table) " +
storeFile.getName()
+ + " of snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
@@ -584,6 +587,7 @@ public class RestoreSnapshotHelper {
if (regions == null || regions.isEmpty()) return null;
final Map<String, RegionInfo> snapshotRegions = new
HashMap<>(regions.size());
+ final String snapshotName = snapshotDesc.getName();
// clone region info (change embedded tableName with the new one)
RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
@@ -596,7 +600,8 @@ public class RestoreSnapshotHelper {
String snapshotRegionName = snapshotRegionInfo.getEncodedName();
String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
regionsMap.put(Bytes.toBytes(snapshotRegionName),
Bytes.toBytes(clonedRegionName));
- LOG.info("clone region=" + snapshotRegionName + " as " +
clonedRegionName);
+ LOG.info("clone region=" + snapshotRegionName + " as " +
clonedRegionName +
+ " in snapshot " + snapshotName);
// Add mapping between cloned region name and snapshot region info
snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
@@ -640,10 +645,12 @@ public class RestoreSnapshotHelper {
private void cloneRegion(final Path regionDir, final RegionInfo
snapshotRegionInfo,
final SnapshotRegionManifest manifest) throws IOException {
final String tableName = tableDesc.getTableName().getNameAsString();
+ final String snapshotName = snapshotDesc.getName();
for (SnapshotRegionManifest.FamilyFiles familyFiles:
manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir,
familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile:
familyFiles.getStoreFilesList()) {
- LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" +
tableName);
+ LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned
region "
+ + "in snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile,
createBackRefs);
}
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index e2236ee..bf6699d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -185,7 +185,8 @@ public final class SnapshotManifest {
@VisibleForTesting
protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor)
throws IOException {
// 1. dump region meta info into the snapshot directory
- LOG.debug("Storing mob region '" + regionInfo + "' region-info for
snapshot.");
+ final String snapshotName = desc.getName();
+ LOG.debug("Storing mob region '" + regionInfo + "' region-info for
snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
@@ -232,7 +233,8 @@ public final class SnapshotManifest {
@VisibleForTesting
protected void addRegion(final HRegion region, RegionVisitor visitor) throws
IOException {
// 1. dump region meta info into the snapshot directory
- LOG.debug("Storing '" + region + "' region-info for snapshot.");
+ final String snapshotName = desc.getName();
+ LOG.debug("Storing '" + region + "' region-info for snapshot=" +
snapshotName);
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();
@@ -256,7 +258,8 @@ public final class SnapshotManifest {
monitor.rethrowException();
// create "reference" to this store file.
- LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " +
storeFile.getPath());
+ LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " +
storeFile.getPath() +
+ " for snapshot=" + snapshotName);
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);
