Repository: hadoop Updated Branches: refs/heads/branch-2 a5f3156b3 -> 52d0724a2
HDFS-7789. DFSck should resolve the path to support cross-FS symlinks. (gera) (cherry picked from commit cbb492578ef09300821b7199de54c6508f9d7fe8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52d0724a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52d0724a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52d0724a Branch: refs/heads/branch-2 Commit: 52d0724a20112e29698a8b175866a9e714ace652 Parents: a5f3156 Author: Gera Shegalov <[email protected]> Authored: Thu Feb 12 04:32:43 2015 -0800 Committer: Gera Shegalov <[email protected]> Committed: Mon Mar 2 01:05:15 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 31 +++++++++++++------- .../hadoop/hdfs/server/namenode/TestFsck.java | 14 ++++++--- .../namenode/TestFsckWithMultipleNameNodes.java | 20 +++++++++++++ 4 files changed, 53 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d0724a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 028521e..d9b96ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -396,6 +396,9 @@ Release 2.7.0 - UNRELEASED HDFS-7439. Add BlockOpResponseProto's message to the exception messages. (Takanobu Asanuma via szetszwo) + HDFS-7789. DFSck should resolve the path to support cross-FS symlinks. + (gera) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d0724a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index 5b43290..900d8ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -223,6 +223,14 @@ public class DFSck extends Configured implements Tool { return errCode; } + + private Path getResolvedPath(String dir) throws IOException { + Configuration conf = getConf(); + Path dirPath = new Path(dir); + FileSystem fs = dirPath.getFileSystem(conf); + return fs.resolvePath(dirPath); + } + /** * Derive the namenode http address from the current file system, * either default or as set by "-fs" in the generic options. @@ -234,19 +242,12 @@ public class DFSck extends Configured implements Tool { Configuration conf = getConf(); //get the filesystem object to verify it is an HDFS system - final FileSystem fs; - try { - fs = target.getFileSystem(conf); - } catch (IOException ioe) { - System.err.println("FileSystem is inaccessible due to:\n" - + StringUtils.stringifyException(ioe)); - return null; - } + final FileSystem fs = target.getFileSystem(conf); if (!(fs instanceof DistributedFileSystem)) { System.err.println("FileSystem is " + fs.getUri()); return null; } - + return DFSUtil.getInfoServer(HAUtil.getAddressOfActive(fs), conf, DFSUtil.getHttpClientScheme(conf)); } @@ -300,8 +301,16 @@ public class DFSck extends Configured implements Tool { dir = "/"; } - final Path dirpath = new Path(dir); - final URI namenodeAddress = getCurrentNamenodeAddress(dirpath); + Path dirpath = null; + URI namenodeAddress = null; + try { + dirpath = getResolvedPath(dir); + namenodeAddress = getCurrentNamenodeAddress(dirpath); + } catch (IOException ioe) { + System.err.println("FileSystem is inaccessible due to:\n" + + StringUtils.stringifyException(ioe)); + } + if (namenodeAddress == null) { //Error message already output in {@link #getCurrentNamenodeAddress()} System.err.println("DFSck exiting."); http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d0724a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 3c69f1d..b8024a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -211,10 +211,16 @@ public class TestFsck { try { // Audit log should contain one getfileinfo and one fsck reader = new BufferedReader(new FileReader(auditLogFile)); - String line = reader.readLine(); - assertNotNull(line); - assertTrue("Expected getfileinfo event not found in audit log", - getfileinfoPattern.matcher(line).matches()); + String line; + + // one extra getfileinfo stems from resolving the path + // + for (int i = 0; i < 2; i++) { + line = reader.readLine(); + assertNotNull(line); + assertTrue("Expected getfileinfo event not found in audit log", + getfileinfoPattern.matcher(line).matches()); + } line = reader.readLine(); assertNotNull(line); assertTrue("Expected fsck event not found in audit log", fsckPattern http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d0724a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java index f4cb624..124b301 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; +import java.net.URI; import java.util.Random; import java.util.concurrent.TimeoutException; @@ -26,6 +27,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -119,6 +122,23 @@ public class TestFsckWithMultipleNameNodes { LOG.info("result=" + result); Assert.assertTrue(result.contains("Status: HEALTHY")); } + + // Test viewfs + // + LOG.info("RUN_TEST 3"); + final String[] vurls = new String[nNameNodes]; + for (int i = 0; i < vurls.length; i++) { + String link = "/mount/nn_" + i + FILE_NAME; + ConfigUtil.addLink(conf, link, new URI(urls[i])); + vurls[i] = "viewfs:" + link; + } + + for(int i = 0; i < vurls.length; i++) { + LOG.info("vurls[" + i + "]=" + vurls[i]); + final String result = TestFsck.runFsck(conf, 0, false, vurls[i]); + LOG.info("result=" + result); + Assert.assertTrue(result.contains("Status: HEALTHY")); + } } finally { cluster.shutdown(); }
