Author: jing9
Date: Fri Apr 25 02:07:25 2014
New Revision: 1589921
URL: http://svn.apache.org/r1589921
Log:
HDFS-6266. Merge r1589920 from trunk.
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java
- copied unchanged from r1589920,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Apr 25 02:07:25 2014
@@ -72,6 +72,8 @@ Release 2.5.0 - UNRELEASED
HDFS-6282. Re-add testIncludeByRegistrationName. (cmccabe)
+ HDFS-6266. Identify full path for a given INode. (jing9)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Fri Apr 25 02:07:25 2014
@@ -2911,7 +2911,9 @@ public class FSDirectory implements Clos
/**
* Given an INode get all the path complents leading to it from the root.
* If an Inode corresponding to C is given in /A/B/C, the returned
- * patch components will be {root, A, B, C}
+ * patch components will be {root, A, B, C}.
+ * Note that this method cannot handle scenarios where the inode is in a
+ * snapshot.
*/
static byte[][] getPathComponents(INode inode) {
List<byte[]> components = new ArrayList<byte[]>();
@@ -2922,7 +2924,54 @@ public class FSDirectory implements Clos
}
return components.toArray(new byte[components.size()][]);
}
-
+
+ /**
+ * The same functionality with {@link #getPathComponents(INode)}, but can
+ * handle snapshots.
+ */
+ public static byte[][] getPathComponentsWithSnapshot(INode inode) {
+ List<byte[]> components = new ArrayList<byte[]>();
+ boolean inSnapshot = false;
+ int snapshotId = Snapshot.CURRENT_STATE_ID;
+ do {
+ if (inode instanceof INodeReference.WithCount) {
+ // identify the corresponding WithName or DstReference node
+ inode = ((WithCount) inode).getParentRef(snapshotId);
+ } else { // normal INode and WithName/DstReference
+ if (inode instanceof INodeDirectory
+ && inode.asDirectory().isSnapshottable() && inSnapshot
+ && snapshotId != Snapshot.CURRENT_STATE_ID) {
+ INodeDirectorySnapshottable sdir = (INodeDirectorySnapshottable)
inode
+ .asDirectory();
+ Snapshot snapshot = sdir.getSnapshotById(snapshotId);
+ if (snapshot != null) {
+ components.add(0, snapshot.getRoot().getLocalNameBytes());
+ components.add(0, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES);
+ // the snapshot has been found, thus no need to check snapshottable
+ // directory afterwards
+ inSnapshot = false;
+ }
+ }
+ INode parent = inode.getParentReference() != null ? inode
+ .getParentReference() : inode.getParent();
+ if (parent != null && parent instanceof INodeDirectory) {
+ int sid = parent.asDirectory().searchChild(inode);
+ Preconditions.checkState(sid != Snapshot.NO_SNAPSHOT_ID);
+ if (sid != Snapshot.CURRENT_STATE_ID
+ && snapshotId == Snapshot.CURRENT_STATE_ID) {
+ snapshotId = sid;
+ inSnapshot = true;
+ }
+ components.add(0, inode.getLocalNameBytes());
+ } else if (parent == null) { // root
+ components.add(0, inode.getLocalNameBytes());
+ }
+ inode = parent;
+ }
+ } while (inode != null);
+ return components.toArray(new byte[components.size()][]);
+ }
+
/**
* @return path components for reserved path, else null.
*/
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
Fri Apr 25 02:07:25 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.na
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -656,7 +655,8 @@ public abstract class INode implements I
* @return array of byte arrays each of which represents
* a single path component.
*/
- static byte[][] getPathComponents(String path) {
+ @VisibleForTesting
+ public static byte[][] getPathComponents(String path) {
return getPathComponents(getPathNames(path));
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
Fri Apr 25 02:07:25 2014
@@ -356,6 +356,29 @@ public class INodeDirectory extends INod
return sf.getChild(this, name, snapshotId);
}
+
+ /**
+ * Search for the given INode in the children list and the deleted lists of
+ * snapshots.
+ * @return {@link Snapshot#CURRENT_STATE_ID} if the inode is in the children
+ * list; {@link Snapshot#NO_SNAPSHOT_ID} if the inode is neither in the
+ * children list nor in any snapshot; otherwise the snapshot id of the
+ * corresponding snapshot diff list.
+ */
+ int searchChild(INode inode) {
+ INode child = getChild(inode.getLocalNameBytes(),
Snapshot.CURRENT_STATE_ID);
+ if (child != inode) {
+ // inode is not in parent's children list, thus inode must be in
+ // snapshot. identify the snapshot id and later add it into the path
+ DirectoryDiffList diffs = getDiffs();
+ if (diffs == null) {
+ return Snapshot.NO_SNAPSHOT_ID;
+ }
+ return diffs.findSnapshotDeleted(inode);
+ } else {
+ return Snapshot.CURRENT_STATE_ID;
+ }
+ }
/**
* @param snapshotId
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
Fri Apr 25 02:07:25 2014
@@ -418,6 +418,23 @@ public abstract class INodeReference ext
return withNameList.get(-i - 2);
}
}
+
+ public INodeReference getParentRef(int snapshotId) {
+ // when the given snapshotId is CURRENT_STATE_ID, it is possible that we
+ // do not know where the corresponding inode belongs, thus we simply
+ // return the last reference node
+ if (snapshotId == Snapshot.CURRENT_STATE_ID) {
+ return this.getParentReference() != null ? this.getParentReference()
+ : this.getLastWithName();
+ }
+ // otherwise we search the withNameList
+ for (int i = 0; i < withNameList.size(); i++) {
+ if (snapshotId <= withNameList.get(i).lastSnapshotId) {
+ return withNameList.get(i);
+ }
+ }
+ return this.getParentReference();
+ }
}
/** A reference with a fixed name. */
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
Fri Apr 25 02:07:25 2014
@@ -364,6 +364,25 @@ public class DirectoryWithSnapshotFeatur
}
return false;
}
+
+ /**
+ * Find the corresponding snapshot whose deleted list contains the given
+ * inode.
+ * @return the id of the snapshot. {@link Snapshot#NO_SNAPSHOT_ID} if the
+ * given inode is not in any of the snapshot.
+ */
+ public int findSnapshotDeleted(final INode child) {
+ final List<DirectoryDiff> diffList = asList();
+ for(int i = diffList.size() - 1; i >= 0; i--) {
+ final ChildrenDiff diff = diffList.get(i).diff;
+ final int d = diff.searchIndex(ListType.DELETED,
+ child.getLocalNameBytes());
+ if (d >= 0 && diff.getList(ListType.DELETED).get(d) == child) {
+ return diffList.get(i).getSnapshotId();
+ }
+ }
+ return Snapshot.NO_SNAPSHOT_ID;
+ }
}
private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java
Fri Apr 25 02:07:25 2014
@@ -206,7 +206,7 @@ public class INodeDirectorySnapshottable
return i < 0? null: snapshotsByNames.get(i);
}
- Snapshot getSnapshotById(int sid) {
+ public Snapshot getSnapshotById(int sid) {
for (Snapshot s : snapshotsByNames) {
if (s.getId() == sid) {
return s;
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Fri Apr 25 02:07:25 2014
@@ -82,6 +82,7 @@ import java.util.concurrent.TimeoutExcep
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/** Utilities for HDFS tests */
public class DFSTestUtil {
@@ -1212,7 +1213,20 @@ public class DFSTestUtil {
long c = (val + factor - 1) / factor;
return c * factor;
}
-
+
+ public static void checkComponentsEquals(byte[][] expected, byte[][] actual)
{
+ assertEquals("expected: " + DFSUtil.byteArray2PathString(expected)
+ + ", actual: " + DFSUtil.byteArray2PathString(actual), expected.length,
+ actual.length);
+ int i = 0;
+ for (byte[] e : expected) {
+ byte[] actualComponent = actual[i++];
+ assertTrue("expected: " + DFSUtil.bytes2String(e) + ", actual: "
+ + DFSUtil.bytes2String(actualComponent),
+ Arrays.equals(e, actualComponent));
+ }
+ }
+
/**
* A short-circuit test context which makes it easier to get a short-circuit
* configuration and set everything up.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Fri Apr 25 02:07:25 2014
@@ -26,7 +26,6 @@ import static org.junit.Assert.fail;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import org.junit.Assert;
@@ -792,14 +791,6 @@ public class TestINodeFile {
return dir; // Last Inode in the chain
}
- private static void checkEquals(byte[][] expected, byte[][] actual) {
- assertEquals(expected.length, actual.length);
- int i = 0;
- for (byte[] e : expected) {
- assertTrue(Arrays.equals(e, actual[i++]));
- }
- }
-
/**
* Test for {@link FSDirectory#getPathComponents(INode)}
*/
@@ -809,7 +800,7 @@ public class TestINodeFile {
INode inode = createTreeOfInodes(path);
byte[][] expected = INode.getPathComponents(path);
byte[][] actual = FSDirectory.getPathComponents(inode);
- checkEquals(expected, actual);
+ DFSTestUtil.checkComponentsEquals(expected, actual);
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1589921&r1=1589920&r2=1589921&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
Fri Apr 25 02:07:25 2014
@@ -1829,7 +1829,7 @@ public class TestRenameWithSnapshots {
}
/**
- * move a directory to its prior descedant
+ * move a directory to its prior descendant
*/
@Test
public void testRename2PreDescendant_2() throws Exception {
