Repository: hadoop Updated Branches: refs/heads/branch-2.6 2f546755c -> b9d4144c2
HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp. (cherry picked from commit bd3dcf46e263b6e6aa3fca6a5d9936cc49e3280f) (cherry picked from commit 6a7fe835f19141cc633824cd5d21f1e30f014bce) (cherry picked from commit 2a5b8e5b3a7ac716253f68ccd25f95ae189bc3ea) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9d4144c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9d4144c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9d4144c Branch: refs/heads/branch-2.6 Commit: b9d4144c2b40bf29663c21633fb4008a1c1aeb8b Parents: 2f54675 Author: Jing Zhao <[email protected]> Authored: Thu Jul 21 11:14:39 2016 -0700 Committer: Zhe Zhang <[email protected]> Committed: Fri Jul 22 08:47:17 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 9 ++++++ .../hdfs/server/namenode/FSNamesystem.java | 9 +++--- .../hadoop/hdfs/server/namenode/INode.java | 20 +++++-------- .../server/namenode/TestSnapshotPathINodes.java | 30 +++++++------------- 4 files changed, 31 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 031896a..67880cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -417,6 +417,15 @@ public class DFSUtil { } /** + * Convert a UTF8 string to an array of byte arrays. + */ + public static byte[][] getPathComponents(String path) { + // avoid intermediate split to String[] + final byte[] bytes = string2Bytes(path); + return bytes2byteArray(bytes, bytes.length, (byte)Path.SEPARATOR_CHAR); + } + + /** * Splits the array of bytes into array of arrays of bytes * on byte separator * @param bytes the array of bytes to split http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index bd4f555..4a70ac0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4327,8 +4327,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, UnresolvedLinkException, SnapshotAccessControlException, AclException { src = FSDirectory.normalizePath(src); - String[] names = INode.getPathNames(src); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(src); final int lastInodeIndex = components.length - 1; dir.writeLock(); @@ -4344,7 +4343,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, StringBuilder pathbuilder = new StringBuilder(); int i = 1; for(; i < inodes.length && inodes[i] != null; i++) { - pathbuilder.append(Path.SEPARATOR).append(names[i]); + pathbuilder.append(Path.SEPARATOR). + append(DFSUtil.bytes2String(components[i])); if (!inodes[i].isDirectory()) { throw new FileAlreadyExistsException( "Parent path is not a directory: " @@ -4386,7 +4386,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // create directories beginning from the first null index for(; i < inodes.length; i++) { - pathbuilder.append(Path.SEPARATOR).append(names[i]); + pathbuilder.append(Path.SEPARATOR). + append(DFSUtil.bytes2String(components[i])); dir.unprotectedMkdir(allocateNewInodeId(), iip, i, components[i], (i < lastInodeIndex) ? parentPermissions : permissions, null, now); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 4454930..2349740 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -707,18 +707,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> { */ @VisibleForTesting public static byte[][] getPathComponents(String path) { - return getPathComponents(getPathNames(path)); - } - - /** Convert strings to byte arrays for path components. */ - static byte[][] getPathComponents(String[] strings) { - if (strings.length == 0) { - return new byte[][]{null}; - } - byte[][] bytes = new byte[strings.length][]; - for (int i = 0; i < strings.length; i++) - bytes[i] = DFSUtil.string2Bytes(strings[i]); - return bytes; + checkAbsolutePath(path); + return DFSUtil.getPathComponents(path); } /** @@ -727,10 +717,14 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> { * @return array of path components. */ public static String[] getPathNames(String path) { + checkAbsolutePath(path); + return StringUtils.split(path, Path.SEPARATOR_CHAR); + } + + private static void checkAbsolutePath(final String path) { if (path == null || !path.startsWith(Path.SEPARATOR)) { throw new AssertionError("Absolute path required"); } - return StringUtils.split(path, Path.SEPARATOR_CHAR); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index 95524e8..1e9be70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -142,8 +142,7 @@ public class TestSnapshotPathINodes { @Test (timeout=15000) public void testNonSnapshotPathINodes() throws Exception { // Get the inodes by resolving the path of a normal file - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // The number of inodes should be equal to components.length @@ -192,8 +191,7 @@ public class TestSnapshotPathINodes { // The path when accessing the snapshot file of file1 is // /TestSnapshot/sub1/.snapshot/s1/file1 String snapshotPath = sub1.toString() + "/.snapshot/s1/file1"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // Length of inodes should be (components.length - 1), since we will ignore @@ -228,8 +226,7 @@ public class TestSnapshotPathINodes { // Resolve the path "/TestSnapshot/sub1/.snapshot" String dotSnapshotPath = sub1.toString() + "/.snapshot"; - names = INode.getPathNames(dotSnapshotPath); - components = INode.getPathComponents(names); + components = INode.getPathComponents(dotSnapshotPath); nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); inodes = nodesInPath.getINodes(); // The number of INodes returned should be components.length - 1 since we @@ -277,8 +274,7 @@ public class TestSnapshotPathINodes { // Resolve the path for the snapshot file // /TestSnapshot/sub1/.snapshot/s2/file1 String snapshotPath = sub1.toString() + "/.snapshot/s2/file1"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // Length of inodes should be (components.length - 1), since we will ignore @@ -295,8 +291,7 @@ public class TestSnapshotPathINodes { } // Check the INodes for path /TestSnapshot/sub1/file1 - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // The length of inodes should be equal to components.length @@ -337,8 +332,7 @@ public class TestSnapshotPathINodes { { // Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3 String snapshotPath = sub1.toString() + "/.snapshot/s4/file3"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // Length of inodes should be (components.length - 1), since we will ignore @@ -357,8 +351,7 @@ public class TestSnapshotPathINodes { } // Check the inodes for /TestSnapshot/sub1/file3 - String[] names = INode.getPathNames(file3.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file3.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // The number of inodes should be equal to components.length @@ -385,8 +378,7 @@ public class TestSnapshotPathINodes { @Test (timeout=15000) public void testSnapshotPathINodesAfterModification() throws Exception { // First check the INode for /TestSnapshot/sub1/file1 - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] inodes = nodesInPath.getINodes(); // The number of inodes should be equal to components.length @@ -408,8 +400,7 @@ public class TestSnapshotPathINodes { // Check the INodes for snapshot of file1 String snapshotPath = sub1.toString() + "/.snapshot/s3/file1"; - names = INode.getPathNames(snapshotPath); - components = INode.getPathComponents(names); + components = INode.getPathComponents(snapshotPath); INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir, components); INode[] ssInodes = ssNodesInPath.getINodes(); // Length of ssInodes should be (components.length - 1), since we will @@ -427,8 +418,7 @@ public class TestSnapshotPathINodes { snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshotId())); // Check the INode for /TestSnapshot/sub1/file1 again - names = INode.getPathNames(file1.toString()); - components = INode.getPathComponents(names); + components = INode.getPathComponents(file1.toString()); INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir, components); assertSnapshot(newNodesInPath, false, s3, -1); INode[] newInodes = newNodesInPath.getINodes(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
