Author: jghoman
Date: Tue May 11 18:17:14 2010
New Revision: 943214
URL: http://svn.apache.org/viewvc?rev=943214&view=rev
Log:
HDFS-1061. Memory footprint optimization for INodeFile object. (Bharath
Mundlapudi)
Added:
hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue May 11 18:17:14 2010
@@ -2,6 +2,11 @@ Hadoop HDFS Change Log
Trunk (unreleased changes)
+ IMPROVEMENTS
+
+ HDFS-1061. Memory footprint optimization for INodeFile object.
+ (Bharath Mundlapudi via jghoman)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue May 11 18:17:14 2010
@@ -870,7 +870,7 @@ public class FSNamesystem implements FSC
throw new IllegalArgumentException("concat: "+ target + " file is
empty");
}
- long blockSize = trgInode.preferredBlockSize;
+ long blockSize = trgInode.getPreferredBlockSize();
// check the end block to be full
if(blockSize != trgInode.blocks[trgInode.blocks.length-1].getNumBytes())
{
@@ -878,7 +878,7 @@ public class FSNamesystem implements FSC
}
si.add(trgInode);
- short repl = trgInode.blockReplication;
+ short repl = trgInode.getReplication();
// now check the srcs
boolean endSrc = false; // final src file doesn't have to have full end
block
@@ -898,10 +898,10 @@ public class FSNamesystem implements FSC
}
// check replication and blocks size
- if(repl != srcInode.blockReplication) {
+ if(repl != srcInode.getReplication()) {
throw new IllegalArgumentException(src + " and " + target + " " +
"should have same replication: "
- + repl + " vs. " + srcInode.blockReplication);
+ + repl + " vs. " + srcInode.getReplication());
}
//boolean endBlock=false;
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=943214&r1=943213&r2=943214&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Tue May 11 18:17:14 2010
@@ -28,9 +28,16 @@ import org.apache.hadoop.hdfs.protocol.B
class INodeFile extends INode {
static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+ //Number of bits for Block size
+ static final short BLOCKBITS = 48;
+
+ //Header mask 64-bit representation
+ //Format: [16 bits for replication][48 bits for PreferredBlockSize]
+ static final long HEADERMASK = 0xffffL << BLOCKBITS;
+
+ protected long header;
+
protected BlockInfo blocks[] = null;
- protected short blockReplication;
- protected long preferredBlockSize;
INodeFile(PermissionStatus permissions,
int nrBlocks, short replication, long modificationTime,
@@ -41,16 +48,15 @@ class INodeFile extends INode {
protected INodeFile() {
blocks = null;
- blockReplication = 0;
- preferredBlockSize = 0;
+ header = 0;
}
protected INodeFile(PermissionStatus permissions, BlockInfo[] blklist,
short replication, long modificationTime,
long atime, long preferredBlockSize) {
super(permissions, modificationTime, atime);
- this.blockReplication = replication;
- this.preferredBlockSize = preferredBlockSize;
+ this.setReplication(replication);
+ this.setPreferredBlockSize(preferredBlockSize);
blocks = blklist;
}
@@ -69,14 +75,31 @@ class INodeFile extends INode {
/**
* Get block replication for the file
- * @return block replication
+ * @return block replication value
*/
public short getReplication() {
- return this.blockReplication;
+ return (short) ((header & HEADERMASK) >> BLOCKBITS);
}
- void setReplication(short replication) {
- this.blockReplication = replication;
+ public void setReplication(short replication) {
+ if(replication <= 0)
+ throw new IllegalArgumentException("Unexpected value for the
replication");
+ header = ((long)replication << BLOCKBITS) | (header & ~HEADERMASK);
+ }
+
+ /**
+ * Get preferred block size for the file
+ * @return preferred block size in bytes
+ */
+ public long getPreferredBlockSize() {
+ return header & ~HEADERMASK;
+ }
+
+ public void setPreferredBlockSize(long preferredBlkSize)
+ {
+ if((preferredBlkSize < 0) || (preferredBlkSize > ~HEADERMASK ))
+ throw new IllegalArgumentException("Unexpected value for the block
size");
+ header = (header & HEADERMASK) | (preferredBlkSize & ~HEADERMASK);
}
/**
@@ -195,20 +218,12 @@ class INodeFile extends INode {
*/
if (blkArr.length > 0 && blkArr[blkArr.length-1] != null &&
isUnderConstruction()) {
- size += preferredBlockSize - blocks[blocks.length-1].getNumBytes();
+ size += getPreferredBlockSize() - blocks[blocks.length-1].getNumBytes();
}
- return size * blockReplication;
+ return size * getReplication();
}
/**
- * Get the preferred block size of the file.
- * @return the number of bytes
- */
- public long getPreferredBlockSize() {
- return preferredBlockSize;
- }
-
- /**
* Return the penultimate allocated block for this file.
*/
BlockInfo getPenultimateBlock() {
Added:
hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=943214&view=auto
==============================================================================
---
hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
(added)
+++
hadoop/hdfs/trunk/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
Tue May 11 18:17:14 2010
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+
+import org.junit.Test;
+
+public class TestINodeFile {
+
+ static final short BLOCKBITS = 48;
+ static final long BLKSIZE_MAXVALUE = ~(0xffffL << BLOCKBITS);
+
+ private String userName = "Test";
+ private short replication;
+ private long preferredBlockSize;
+
+ /**
+ * Test for the Replication value. Sets a value and checks if it was set
+ * correct.
+ */
+ @Test
+ public void testReplication () {
+ replication = 3;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", replication,
+ inf.getReplication());
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting below lower bound
+ * for Replication.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testReplicationBelowLowerBound ()
+ throws IllegalArgumentException {
+ replication = -1;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+ /**
+ * Test for the PreferredBlockSize value. Sets a value and checks if it was
+ * set correct.
+ */
+ @Test
+ public void testPreferredBlockSize () {
+ replication = 3;
+ preferredBlockSize = 128*1024*1024;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", preferredBlockSize,
+ inf.getPreferredBlockSize());
+ }
+
+ @Test
+ public void testPreferredBlockSizeUpperBound () {
+ replication = 3;
+ preferredBlockSize = BLKSIZE_MAXVALUE;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ assertEquals("True has to be returned in this case", BLKSIZE_MAXVALUE,
+ inf.getPreferredBlockSize());
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting below lower bound
+ * for PreferredBlockSize.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testPreferredBlockSizeBelowLowerBound ()
+ throws IllegalArgumentException {
+ replication = 3;
+ preferredBlockSize = -1;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+ /**
+ * IllegalArgumentException is expected for setting above upper bound
+ * for PreferredBlockSize.
+ * @throws IllegalArgumentException as the result
+ */
+ @Test(expected=IllegalArgumentException.class)
+ public void testPreferredBlockSizeAboveUpperBound ()
+ throws IllegalArgumentException {
+ replication = 3;
+ preferredBlockSize = BLKSIZE_MAXVALUE+1;
+ INodeFile inf = new INodeFile(new PermissionStatus(userName, null,
+ FsPermission.getDefault()), null,
replication,
+ 0L, 0L, preferredBlockSize);
+ }
+
+}
