Author: shv
Date: Tue May 6 11:13:48 2008
New Revision: 653866
URL: http://svn.apache.org/viewvc?rev=653866&view=rev
Log:
HADOOP-3248. Optimization of saveFSImage. Contributed by Dhruba.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May 6 11:13:48 2008
@@ -100,6 +100,8 @@
On Linux DataNode takes 5 times less CPU while serving data. Results may
vary on other platforms. (rangadi)
+ HADOOP-3248. Optimization of saveFSImage. (Dhruba via shv)
+
BUG FIXES
HADOOP-2905. 'fsck -move' triggers NPE in NameNode.
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSImage.java Tue May 6
11:13:48 2008
@@ -37,8 +37,10 @@
import java.util.Properties;
import java.util.Random;
import java.lang.Math;
+import java.nio.ByteBuffer;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.dfs.FSConstants.CheckpointStates;
import org.apache.hadoop.dfs.FSConstants.StartupOption;
@@ -85,6 +87,13 @@
volatile private CheckpointStates ckptState = CheckpointStates.START;
/**
+ * Used for saving the image to disk
+ */
+ static private final FsPermission fileperm = new FsPermission((short)0);
+ static private final byte[] separator = INode.string2Bytes("/");
+ static private byte[] byteStore = null;
+
+ /**
*/
FSImage() {
super(NodeType.NAME_NODE);
@@ -785,7 +794,7 @@
/**
* Save the contents of the FS image to the file.
*/
- void saveFSImage(File newFile ) throws IOException {
+ void saveFSImage(File newFile) throws IOException {
FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
FSDirectory fsDir = fsNamesys.dir;
//
@@ -799,8 +808,12 @@
out.writeInt(namespaceID);
out.writeInt(fsDir.rootDir.numItemsInTree() - 1);
out.writeLong(fsNamesys.getGenerationStamp());
- saveImage("", fsDir.rootDir, out);
+ byteStore = new byte[4*FSConstants.MAX_PATH_LENGTH];
+ ByteBuffer strbuf = ByteBuffer.wrap(byteStore);
+ saveImage(strbuf, 0, fsDir.rootDir, out);
fsNamesys.saveFilesUnderConstruction(out);
+ byteStore = null;
+ strbuf = null;
} finally {
out.close();
}
@@ -875,13 +888,16 @@
/**
* Save file tree image starting from the given root.
*/
- private static void saveImage(String parentPrefix,
+ private static void saveImage(ByteBuffer parentPrefix,
+ int prefixLength,
INode inode,
DataOutputStream out) throws IOException {
- String fullName = "";
+ int newPrefixLength = prefixLength;
if (inode.getParent() != null) {
- fullName = parentPrefix + "/" + inode.getLocalName();
- new UTF8(fullName).write(out);
+ parentPrefix.put(separator).put(inode.getLocalNameBytes());
+ newPrefixLength += separator.length + inode.getLocalNameBytes().length;
+ out.writeShort(newPrefixLength);
+ out.write(byteStore, 0, newPrefixLength);
if (!inode.isDirectory()) { // write file inode
INodeFile fileINode = (INodeFile)inode;
out.writeShort(fileINode.getReplication());
@@ -891,7 +907,11 @@
out.writeInt(blocks.length);
for (Block blk : blocks)
blk.write(out);
- fileINode.getPermissionStatus().write(out);
+ fileperm.fromShort(fileINode.getFsPermissionShort());
+ PermissionStatus.write(out, fileINode.getUserName(),
+ fileINode.getGroupName(),
+ fileperm);
+ parentPrefix.position(prefixLength);
return;
}
// write directory inode
@@ -899,11 +919,17 @@
out.writeLong(inode.getModificationTime());
out.writeLong(0); // preferred block size
out.writeInt(-1); // # of blocks
- inode.getPermissionStatus().write(out);
- }
- for(INode child : ((INodeDirectory)inode).getChildren()) {
- saveImage(fullName, child, out);
+ fileperm.fromShort(inode.getFsPermissionShort());
+ PermissionStatus.write(out, inode.getUserName(),
+ inode.getGroupName(),
+ fileperm);
+ }
+ if (((INodeDirectory)inode).getChildrenRaw() != null) {
+ for(INode child : ((INodeDirectory)inode).getChildren()) {
+ saveImage(parentPrefix, newPrefixLength, child, out);
+ }
}
+ parentPrefix.position(prefixLength);
}
void loadDatanodes(int version, DataInputStream in) throws IOException {
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/INode.java Tue May 6
11:13:48 2008
@@ -125,6 +125,9 @@
return new FsPermission(
(short)PermissionStatusFormat.MODE.retrieve(permission));
}
+ protected short getFsPermissionShort() {
+ return (short)PermissionStatusFormat.MODE.retrieve(permission);
+ }
/** Set the [EMAIL PROTECTED] FsPermission} of this [EMAIL PROTECTED] INode}
*/
protected void setPermission(FsPermission permission) {
updatePermissionStatus(PermissionStatusFormat.MODE, permission.toShort());
@@ -156,6 +159,14 @@
}
/**
+ * Get local file name
+ * @return local file name
+ */
+ byte[] getLocalNameBytes() {
+ return name;
+ }
+
+ /**
* Set local file name
*/
void setLocalName(String name) {
@@ -565,6 +576,9 @@
List<INode> getChildren() {
return children==null ? new ArrayList<INode>() : children;
}
+ List<INode> getChildrenRaw() {
+ return children;
+ }
/**
* Collect all the blocks in all children of this INode.
Modified:
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
---
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
(original)
+++
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/FsPermission.java
Tue May 6 11:13:48 2008
@@ -94,7 +94,7 @@
groupaction = g;
otheraction = o;
}
- private void fromShort(short n) {
+ public void fromShort(short n) {
FsAction[] v = FsAction.values();
set(v[(n >>> 6) & 7], v[(n >>> 3) & 7], v[n & 7]);
}
@@ -194,4 +194,4 @@
}
return new FsPermission((short)n);
}
-}
\ No newline at end of file
+}
Modified:
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java?rev=653866&r1=653865&r2=653866&view=diff
==============================================================================
---
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java
(original)
+++
hadoop/core/trunk/src/java/org/apache/hadoop/fs/permission/PermissionStatus.java
Tue May 6 11:13:48 2008
@@ -87,9 +87,7 @@
/** [EMAIL PROTECTED] */
public void write(DataOutput out) throws IOException {
- Text.writeString(out, username);
- Text.writeString(out, groupname);
- permission.write(out);
+ write(out, username, groupname, permission);
}
/**
@@ -101,8 +99,20 @@
return p;
}
+ /**
+ * Serialize a [EMAIL PROTECTED] PermissionStatus} from its base components.
+ */
+ public static void write(DataOutput out,
+ String username,
+ String groupname,
+ FsPermission permission) throws IOException {
+ Text.writeString(out, username);
+ Text.writeString(out, groupname);
+ permission.write(out);
+ }
+
/** [EMAIL PROTECTED] */
public String toString() {
return username + ":" + groupname + ":" + permission;
}
-}
\ No newline at end of file
+}
