HDFS-10980. Optimize check for existence of parent directory. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e57fa81d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e57fa81d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e57fa81d Branch: refs/heads/HADOOP-13037 Commit: e57fa81d9559a93d77fd724f7792326c31a490be Parents: f3f37e6 Author: Kihwal Lee <kih...@apache.org> Authored: Fri Oct 7 17:20:15 2016 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Fri Oct 7 17:20:15 2016 -0500 ---------------------------------------------------------------------- .../hdfs/server/namenode/FSDirMkdirOp.java | 2 +- .../hdfs/server/namenode/FSDirSymlinkOp.java | 2 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 11 ++--- .../hdfs/server/namenode/TestFSDirectory.java | 48 ++++++++++++++++++++ 5 files changed, 56 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57fa81d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java index 2d1914f..4d8d7d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java @@ -66,7 +66,7 @@ class FSDirMkdirOp { } if (!createParent) { - fsd.verifyParentDir(iip, src); + fsd.verifyParentDir(iip); } // validate that we have enough inodes. This is, at best, a http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57fa81d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java index 6938a84..71362f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java @@ -58,7 +58,7 @@ class FSDirSymlinkOp { iip = fsd.resolvePathForWrite(pc, link, false); link = iip.getPath(); if (!createParent) { - fsd.verifyParentDir(iip, link); + fsd.verifyParentDir(iip); } if (!fsd.isValidToCreate(link, iip)) { throw new IOException( http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57fa81d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 40be83b..aab0f76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -323,7 +323,7 @@ class FSDirWriteFileOp { } } else { if (!createParent) { - dir.verifyParentDir(iip, src); + dir.verifyParentDir(iip); } if (!flag.contains(CreateFlag.CREATE)) { throw new FileNotFoundException("Can't overwrite non-existent " + src); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57fa81d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 8456da6..a059ee5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1765,17 +1765,16 @@ public class FSDirectory implements Closeable { /** * Verify that parent directory of src exists. */ - void verifyParentDir(INodesInPath iip, String src) + void verifyParentDir(INodesInPath iip) throws FileNotFoundException, ParentNotDirectoryException { - Path parent = new Path(src).getParent(); - if (parent != null) { + if (iip.length() > 2) { final INode parentNode = iip.getINode(-2); if (parentNode == null) { throw new FileNotFoundException("Parent directory doesn't exist: " - + parent); - } else if (!parentNode.isDirectory() && !parentNode.isSymlink()) { + + iip.getParentPath()); + } else if (!parentNode.isDirectory()) { throw new ParentNotDirectoryException("Parent path is not a directory: " - + parent); + + iip.getParentPath()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e57fa81d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java index 2b43c0f..071bdf7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringReader; import java.util.EnumSet; @@ -30,6 +31,7 @@ import com.google.common.collect.ImmutableList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; @@ -386,4 +388,50 @@ public class TestFSDirectory { XAttrSetFlag.REPLACE)); verifyXAttrsPresent(newXAttrs, 4); } + + @Test + public void testVerifyParentDir() throws Exception { + hdfs.mkdirs(new Path("/dir1/dir2")); + hdfs.createNewFile(new Path("/dir1/file")); + hdfs.createNewFile(new Path("/dir1/dir2/file")); + + INodesInPath iip = fsdir.resolvePath(null, "/"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/file"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir-nonexist/file"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (FileNotFoundException fnf) { + // expected. + } + + iip = fsdir.resolvePath(null, "/dir1/dir2"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/dir2/file"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/dir-nonexist/file"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (FileNotFoundException fnf) { + // expected. + } + + iip = fsdir.resolvePath(null, "/dir1/file/fail"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (ParentNotDirectoryException pnd) { + // expected. + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org