Author: cutting
Date: Thu Aug 23 07:34:59 2007
New Revision: 569015
URL: http://svn.apache.org/viewvc?rev=569015&view=rev
Log:
HADOOP-1759. Change file name in INode from String to byte[], saving memory on
the namenode. Contributed by Konstantin.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/INode.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=569015&r1=569014&r2=569015&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Aug 23 07:34:59 2007
@@ -41,6 +41,9 @@
class, with specialized subclasses for directories and files, to
save memory on the namenode. (Konstantin Shvachko via cutting)
+ HADOOP-1759. Change file name in INode from String to byte[],
+ saving memory on the namenode. (Konstantin Shvachko via cutting)
+
BUG FIXES
HADOOP-1463. HDFS report correct usage statistics for disk space
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/INode.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/INode.java?rev=569015&r1=569014&r2=569015&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/INode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/INode.java Thu Aug 23
07:34:59 2007
@@ -18,8 +18,10 @@
package org.apache.hadoop.dfs;
import java.io.FileNotFoundException;
+import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.fs.Path;
@@ -29,17 +31,19 @@
* This is a base INode class containing common fields for file and
* directory inodes.
*/
-abstract class INode implements Comparable<String> {
- protected String name;
+abstract class INode implements Comparable<byte[]> {
+ protected byte[] name;
protected INodeDirectory parent;
protected long modificationTime;
protected INode(String name) {
this(0L);
- this.name = name;
+ setLocalName(name);
}
INode(long mTime) {
+ this.name = null;
+ this.parent = null;
this.modificationTime = mTime;
}
@@ -55,14 +59,14 @@
* @return local file name
*/
String getLocalName() {
- return name;
+ return bytes2String(name);
}
/**
* Set local file name
*/
void setLocalName(String name) {
- this.name = name;
+ this.name = string2Bytes(name);
}
/**
@@ -78,9 +82,9 @@
return Path.SEPARATOR; // root directory is "/"
}
if (this.parent.parent == null) {
- return Path.SEPARATOR + name;
+ return Path.SEPARATOR + getLocalName();
}
- return parent.getAbsoluteName() + Path.SEPARATOR + name;
+ return parent.getAbsoluteName() + Path.SEPARATOR + getLocalName();
}
/**
@@ -109,13 +113,24 @@
}
}
- static String[] getPathComponents(String path) {
+ /**
+ * Breaks file path into components.
+ * @param path
+ * @return array of byte arrays each of which represents
+ * a single path component.
+ */
+ static byte[][] getPathComponents(String path) {
if (path == null || !path.startsWith(Path.SEPARATOR)) {
return null;
}
if (Path.SEPARATOR.equals(path)) // root
- return new String[]{""};
- return path.split(Path.SEPARATOR, -1);
+ return new byte[][]{null};
+ String[] strings = path.split(Path.SEPARATOR, -1);
+ int size = strings.length;
+ byte[][] bytes = new byte[size][];
+ for (int i = 0; i < size; i++)
+ bytes[i] = string2Bytes(strings[i]);
+ return bytes;
}
/**
@@ -133,19 +148,68 @@
//
// Comparable interface
//
- public int compareTo(String o) {
- return getLocalName().compareTo(o);
+ public int compareTo(byte[] o) {
+ return compareBytes(name, o);
}
public boolean equals(Object o) {
if (!(o instanceof INode)) {
return false;
}
- return getLocalName().equals(((INode)o).getLocalName());
+ return Arrays.equals(this.name, ((INode)o).name);
}
-
+
public int hashCode() {
- return getLocalName().hashCode();
+ return Arrays.hashCode(this.name);
+ }
+
+ //
+ // static methods
+ //
+ /**
+ * Compare two byte arrays.
+ *
+ * @return a negative integer, zero, or a positive integer
+ * as defined by [EMAIL PROTECTED] #compareTo(byte[])}.
+ */
+ static int compareBytes(byte[] a1, byte[] a2) {
+ if (a1==a2)
+ return 0;
+ int len1 = (a1==null ? 0 : a1.length);
+ int len2 = (a2==null ? 0 : a2.length);
+ int n = Math.min(len1, len2);
+ byte b1, b2;
+ for (int i=0; i<n; i++) {
+ b1 = a1[i];
+ b2 = a2[i];
+ if (b1 != b2)
+ return b1 - b2;
+ }
+ return len1 - len2;
+ }
+
+ /**
+ * Converts a byte array to a string using UTF8 encoding.
+ */
+ static String bytes2String(byte[] bytes) {
+ try {
+ return new String(bytes, "UTF8");
+ } catch(UnsupportedEncodingException e) {
+ assert false : "UTF8 encoding is not supported ";
+ }
+ return null;
+ }
+
+ /**
+ * Converts a string to a byte array using UTF8 encoding.
+ */
+ static byte[] string2Bytes(String str) {
+ try {
+ return str.getBytes("UTF8");
+ } catch(UnsupportedEncodingException e) {
+ assert false : "UTF8 encoding is not supported ";
+ }
+ return null;
}
}
@@ -177,13 +241,17 @@
void removeChild(INode node) {
assert children != null;
- int low = Collections.binarySearch(children, node.getLocalName());
+ int low = Collections.binarySearch(children, node.name);
if (low >= 0) {
children.remove(low);
}
}
INode getChild(String name) {
+ return getChildINode(string2Bytes(name));
+ }
+
+ private INode getChildINode(byte[] name) {
if (children == null) {
return null;
}
@@ -196,7 +264,7 @@
/**
*/
- private INode getNode(String[] components) {
+ private INode getNode(byte[][] components) {
return getINode(components, components.length-1);
}
@@ -207,8 +275,8 @@
* @param end the end component of the path
* @return found INode or null otherwise
*/
- private INode getINode(String[] components, int end) {
- assert getLocalName().equals(components[0]) :
+ private INode getINode(byte[][] components, int end) {
+ assert compareBytes(this.name, components[0]) == 0 :
"Incorrect name " + getLocalName() + " expected " + components[0];
if (end >= components.length)
end = components.length-1;
@@ -219,7 +287,7 @@
if(!curNode.isDirectory()) // file is not expected here
return null; // because there is more components in the path
INodeDirectory parentDir = (INodeDirectory)curNode;
- curNode = parentDir.getChild(components[start+1]);
+ curNode = parentDir.getChildINode(components[start+1]);
if(curNode == null) // not found
return null;
}
@@ -244,7 +312,7 @@
if (children == null) {
children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
}
- int low = Collections.binarySearch(children, node.getLocalName());
+ int low = Collections.binarySearch(children, node.name);
if(low >= 0)
return null;
node.parent = this;
@@ -264,7 +332,7 @@
* @throws FileNotFoundException
*/
<T extends INode> T addNode(String path, T newNode) throws
FileNotFoundException {
- String[] pathComponents = getPathComponents(path);
+ byte[][] pathComponents = getPathComponents(path);
assert pathComponents != null : "Incorrect path " + path;
int pathLen = pathComponents.length;
if (pathLen < 2) // add root
@@ -278,7 +346,7 @@
}
INodeDirectory parentNode = (INodeDirectory)node;
// insert into the parent children list
- newNode.setLocalName(pathComponents[pathLen-1]);
+ newNode.name = pathComponents[pathLen-1];
return parentNode.addChild(newNode);
}
