Author: wang
Date: Thu Jun 12 20:58:46 2014
New Revision: 1602289
URL: http://svn.apache.org/r1602289
Log:
HDFS-6395. Skip checking xattr limits for non-user-visible namespaces.
Contributed by Yi Liu.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
(props changed)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1602288
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1602289&r1=1602288&r2=1602289&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Jun 12 20:58:46 2014
@@ -169,6 +169,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6471. Make moveFromLocal CLI testcases to be non-disruptive
(Dasha Boudnik via cos)
+ HDFS-6395. Skip checking xattr limits for non-user-visible namespaces.
+ (Yi Liu via wang).
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1602288
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1602289&r1=1602288&r2=1602289&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Thu Jun 12 20:58:46 2014
@@ -2843,6 +2843,7 @@ public class FSDirectory implements Clos
EnumSet<XAttrSetFlag> flag) throws QuotaExceededException, IOException {
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(
existingXAttrs != null ? existingXAttrs.size() + 1 : 1);
+ int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
boolean exist = false;
if (existingXAttrs != null) {
for (XAttr a: existingXAttrs) {
@@ -2851,6 +2852,10 @@ public class FSDirectory implements Clos
exist = true;
} else {
xAttrs.add(a);
+
+ if (isUserVisible(a)) {
+ userVisibleXAttrsNum++;
+ }
}
}
}
@@ -2858,7 +2863,11 @@ public class FSDirectory implements Clos
XAttrSetFlag.validate(xAttr.getName(), exist, flag);
xAttrs.add(xAttr);
- if (xAttrs.size() > inodeXAttrsLimit) {
+ if (isUserVisible(xAttr)) {
+ userVisibleXAttrsNum++;
+ }
+
+ if (userVisibleXAttrsNum > inodeXAttrsLimit) {
throw new IOException("Cannot add additional XAttr to inode, "
+ "would exceed limit of " + inodeXAttrsLimit);
}
@@ -2866,6 +2875,14 @@ public class FSDirectory implements Clos
return xAttrs;
}
+ private boolean isUserVisible(XAttr xAttr) {
+ if (xAttr.getNameSpace() == XAttr.NameSpace.USER ||
+ xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED) {
+ return true;
+ }
+ return false;
+ }
+
List<XAttr> getXAttrs(String src) throws IOException {
String srcs = normalizePath(src);
readLock();
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1602288
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1602288
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1602288
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1602288
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1602288
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java?rev=1602289&r1=1602288&r2=1602289&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
Thu Jun 12 20:58:46 2014
@@ -22,22 +22,29 @@ package org.apache.hadoop.hdfs.server.na
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
+import java.util.EnumSet;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.collect.Lists;
+
/**
* Test {@link FSDirectory}, the in-memory namespace tree.
*/
@@ -70,6 +77,7 @@ public class TestFSDirectory {
@Before
public void setUp() throws Exception {
conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2);
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(REPLICATION)
.build();
@@ -172,4 +180,36 @@ public class TestFSDirectory {
Assert.assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
|| classname.startsWith(INodeDirectory.class.getSimpleName()));
}
+
+ @Test
+ public void testINodeXAttrsLimit() throws Exception {
+ List<XAttr> existingXAttrs = Lists.newArrayListWithCapacity(2);
+ XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a1").setValue(new byte[]{0x31, 0x32, 0x33}).build();
+ XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a2").setValue(new byte[]{0x31, 0x31, 0x31}).build();
+ existingXAttrs.add(xAttr1);
+ existingXAttrs.add(xAttr2);
+
+ // Adding a system namespace xAttr, isn't affected by inode xAttrs limit.
+ XAttr newXAttr = (new
XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM).
+ setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build();
+ List<XAttr> xAttrs = fsdir.setINodeXAttr(existingXAttrs, newXAttr,
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ Assert.assertEquals(xAttrs.size(), 3);
+
+ // Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
+ XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace(
+ XAttr.NameSpace.TRUSTED).setName("a4").
+ setValue(new byte[]{0x34, 0x34, 0x34}).build();
+ try {
+ fsdir.setINodeXAttr(existingXAttrs, newXAttr1,
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ Assert.fail("Setting user visable xattr on inode should fail if " +
+ "reaching limit.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Cannot add additional XAttr " +
+ "to inode, would exceed limit", e);
+ }
+ }
}
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot:r1602288
