[14/50] [abbrv] hbase git commit: HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar)
HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2278f95 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2278f95 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2278f95 Branch: refs/heads/branch-1 Commit: e2278f9544380ec7abc92c3592bbe2068e62cb45 Parents: a3c29da Author: tedyu Authored: Wed Oct 12 19:42:18 2016 -0700 Committer: tedyu Committed: Wed Oct 12 19:42:18 2016 -0700 -- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 21 .../util/hbck/TestOfflineMetaRebuildBase.java | 26 2 files changed, 47 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e2278f95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 745adbf..458230f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; @@ -1524,12 +1525,32 @@ public class HBaseFsck extends Configured implements Closeable { } meta.batchMutate(puts.toArray(new Put[puts.size()])); HRegion.closeHRegion(meta); +// clean up the temporary hbck meta recovery WAL directory +removeHBCKMetaRecoveryWALDir(meta); LOG.info("Success! hbase:meta table rebuilt."); LOG.info("Old hbase:meta is moved into " + backupDir); return true; } /** + * Removes the empty Meta recovery WAL directory. + * @param meta Meta region + */ + private void removeHBCKMetaRecoveryWALDir(HRegion meta) throws IOException { +// TODO Since HBASE-11983 not available in this branch, so we don't know the walFactoryId. +// Retrieving WAL directory +Path walLogDir = ((FSHLog) meta.getWAL()).getCurrentFileName().getParent(); +FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); +FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); +if (walFiles == null || walFiles.length == 0) { + LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); + if (!FSUtils.deleteDirectory(fs, walLogDir)) { +LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); + } +} + } + + /** * Log an appropriate message about whether or not overlapping merges are computed in parallel. */ private void logParallelMerge() { http://git-wip-us.apache.org/repos/asf/hbase/blob/e2278f95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index 43b1444..b31e20e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -20,17 +20,24 @@ package org.apache.hadoop.hbase.util.hbck; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.util.FSUtils; import o
hbase git commit: HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar)
Repository: hbase Updated Branches: refs/heads/branch-1 a3c29da0b -> e2278f954 HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2278f95 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2278f95 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2278f95 Branch: refs/heads/branch-1 Commit: e2278f9544380ec7abc92c3592bbe2068e62cb45 Parents: a3c29da Author: tedyu Authored: Wed Oct 12 19:42:18 2016 -0700 Committer: tedyu Committed: Wed Oct 12 19:42:18 2016 -0700 -- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 21 .../util/hbck/TestOfflineMetaRebuildBase.java | 26 2 files changed, 47 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e2278f95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 745adbf..458230f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; @@ -1524,12 +1525,32 @@ public class HBaseFsck extends Configured implements Closeable { } meta.batchMutate(puts.toArray(new Put[puts.size()])); HRegion.closeHRegion(meta); +// clean up the temporary hbck meta recovery WAL directory +removeHBCKMetaRecoveryWALDir(meta); LOG.info("Success! hbase:meta table rebuilt."); LOG.info("Old hbase:meta is moved into " + backupDir); return true; } /** + * Removes the empty Meta recovery WAL directory. + * @param meta Meta region + */ + private void removeHBCKMetaRecoveryWALDir(HRegion meta) throws IOException { +// TODO Since HBASE-11983 not available in this branch, so we don't know the walFactoryId. +// Retrieving WAL directory +Path walLogDir = ((FSHLog) meta.getWAL()).getCurrentFileName().getParent(); +FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); +FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); +if (walFiles == null || walFiles.length == 0) { + LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); + if (!FSUtils.deleteDirectory(fs, walLogDir)) { +LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); + } +} + } + + /** * Log an appropriate message about whether or not overlapping merges are computed in parallel. */ private void logParallelMerge() { http://git-wip-us.apache.org/repos/asf/hbase/blob/e2278f95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index 43b1444..b31e20e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -20,17 +20,24 @@ package org.apache.hadoop.hbase.util.hbck; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbas
hbase git commit: HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar)
Repository: hbase Updated Branches: refs/heads/master d9f18c7f0 -> 4d0911669 HBASE-16716 OfflineMetaRepair leaves empty directory inside /hbase/WALs which remains forever (Pankaj Kumar) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4d091166 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4d091166 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4d091166 Branch: refs/heads/master Commit: 4d091166951a0b53e5fdd9dc694b4e5a12dc527c Parents: d9f18c7 Author: tedyu Authored: Tue Oct 11 07:19:54 2016 -0700 Committer: tedyu Committed: Tue Oct 11 07:19:54 2016 -0700 -- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 33 .../util/hbck/TestOfflineMetaRebuildBase.java | 25 +++ 2 files changed, 52 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4d091166/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index efd75a6..a31b7d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -1388,10 +1388,11 @@ public class HBaseFsck extends Configured implements Closeable { /** * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be * sure to close it as well as the region when you're finished. - * + * @param walFactoryID A unique identifier for WAL factory. Filesystem implementations will use + * this ID to make a directory inside WAL directory path. * @return an open hbase:meta HRegion */ - private HRegion createNewMeta() throws IOException { + private HRegion createNewMeta(String walFactoryID) throws IOException { Path rootdir = FSUtils.getRootDir(getConf()); Configuration c = getConf(); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); @@ -1402,9 +1403,8 @@ public class HBaseFsck extends Configured implements Closeable { Configuration confForWAL = new Configuration(c); confForWAL.set(HConstants.HBASE_DIR, rootdir.toString()); WAL wal = (new WALFactory(confForWAL, -Collections.singletonList(new MetricsWAL()), -"hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))). -getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace()); +Collections. singletonList(new MetricsWAL()), walFactoryID)) +.getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace()); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); return meta; @@ -1513,7 +1513,8 @@ public class HBaseFsck extends Configured implements Closeable { Path backupDir = sidelineOldMeta(); LOG.info("Creating new hbase:meta"); -HRegion meta = createNewMeta(); +String walFactoryId = "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8); +HRegion meta = createNewMeta(walFactoryId); // populate meta List puts = generatePuts(tablesInfo); @@ -1527,12 +1528,32 @@ public class HBaseFsck extends Configured implements Closeable { if (meta.getWAL() != null) { meta.getWAL().close(); } +// clean up the temporary hbck meta recovery WAL directory +removeHBCKMetaRecoveryWALDir(walFactoryId); LOG.info("Success! hbase:meta table rebuilt."); LOG.info("Old hbase:meta is moved into " + backupDir); return true; } /** + * Removes the empty Meta recovery WAL directory. + * @param walFactoryID A unique identifier for WAL factory which was used by Filesystem to make a + * Meta recovery WAL directory inside WAL directory path. + */ + private void removeHBCKMetaRecoveryWALDir(String walFactoryId) throws IOException { +Path rootdir = FSUtils.getRootDir(getConf()); +Path walLogDir = new Path(new Path(rootdir, HConstants.HREGION_LOGDIR_NAME), walFactoryId); +FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); +FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); +if (walFiles == null || walFiles.length == 0) { + LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); + if (!FSUtils.deleteDirectory(fs, walLogDir)) { +LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); + } +} + } + + /** * Log an appropriate message about whether or not overlapping merges are computed in parallel. */ private void logParallelM