Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java Thu Aug 21 05:22:10 2014 @@ -47,15 +47,27 @@ import com.google.common.base.Preconditi * <br> * SYSTEM - extended system attributes: these are used by the HDFS * core and are not available through admin/user API. + * <br> + * RAW - extended system attributes: these are used for internal system + * attributes that sometimes need to be exposed. Like SYSTEM namespace + * attributes they are not visible to the user except when getXAttr/getXAttrs + * is called on a file or directory in the /.reserved/raw HDFS directory + * hierarchy. These attributes can only be accessed by the superuser. + * </br> */ @InterfaceAudience.Private public class XAttrPermissionFilter { - static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr) + static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr, + boolean isRawPath) throws AccessControlException { + final boolean isSuperUser = pc.isSuperUser(); if (xAttr.getNameSpace() == XAttr.NameSpace.USER || - (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && - pc.isSuperUser())) { + (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser)) { + return; + } + if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && + isRawPath && isSuperUser) { return; } throw new AccessControlException("User doesn't have permission for xattr: " @@ -63,30 +75,34 @@ public class XAttrPermissionFilter { } static void checkPermissionForApi(FSPermissionChecker pc, - List<XAttr> xAttrs) throws AccessControlException { + List<XAttr> xAttrs, boolean isRawPath) throws AccessControlException { Preconditions.checkArgument(xAttrs != null); if (xAttrs.isEmpty()) { return; } for (XAttr xAttr : xAttrs) { - checkPermissionForApi(pc, xAttr); + checkPermissionForApi(pc, xAttr, isRawPath); } } static List<XAttr> filterXAttrsForApi(FSPermissionChecker pc, - List<XAttr> xAttrs) { + List<XAttr> xAttrs, boolean isRawPath) { assert xAttrs != null : "xAttrs can not be null"; if (xAttrs == null || xAttrs.isEmpty()) { return xAttrs; } List<XAttr> filteredXAttrs = Lists.newArrayListWithCapacity(xAttrs.size()); + final boolean isSuperUser = pc.isSuperUser(); for (XAttr xAttr : xAttrs) { if (xAttr.getNameSpace() == XAttr.NameSpace.USER) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && - pc.isSuperUser()) { + isSuperUser) { + filteredXAttrs.add(xAttr); + } else if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && + isSuperUser && isRawPath) { filteredXAttrs.add(xAttr); } }
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Thu Aug 21 05:22:10 2014 @@ -257,7 +257,7 @@ public class JsonUtil { BlockStoragePolicy.ID_UNSPECIFIED; return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication, blockSize, mTime, aTime, permission, owner, group, symlink, - DFSUtil.string2Bytes(localName), fileId, childrenNum, storagePolicy); + DFSUtil.string2Bytes(localName), fileId, childrenNum, null, storagePolicy); } /** Convert an ExtendedBlock to a Json map. */ @@ -537,7 +537,7 @@ public class JsonUtil { (Map<?, ?>)m.get("lastLocatedBlock")); final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete"); return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks, - lastLocatedBlock, isLastBlockComplete); + lastLocatedBlock, isLastBlockComplete, null); } /** Convert a ContentSummary to a Json string. */ Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1616894-1619277 Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1594376-1619194 Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Thu Aug 21 05:22:10 2014 @@ -32,6 +32,7 @@ import "Security.proto"; import "hdfs.proto"; import "acl.proto"; import "xattr.proto"; +import "encryption.proto"; /** * The ClientNamenodeProtocol Service defines the interface between a client @@ -73,6 +74,7 @@ message CreateRequestProto { required bool createParent = 5; required uint32 replication = 6; // Short: Only 16 bits used required uint64 blockSize = 7; + repeated CipherSuite cipherSuites = 8; } message CreateResponseProto { @@ -803,4 +805,10 @@ service ClientNamenodeProtocol { returns(RemoveXAttrResponseProto); rpc checkAccess(CheckAccessRequestProto) returns(CheckAccessResponseProto); + rpc createEncryptionZone(CreateEncryptionZoneRequestProto) + returns(CreateEncryptionZoneResponseProto); + rpc listEncryptionZones(ListEncryptionZonesRequestProto) + returns(ListEncryptionZonesResponseProto); + rpc getEZForPath(GetEZForPathRequestProto) + returns(GetEZForPathResponseProto); } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto Thu Aug 21 05:22:10 2014 @@ -201,6 +201,23 @@ message DataEncryptionKeyProto { optional string encryptionAlgorithm = 6; } +/** + * Cipher suite. + */ +enum CipherSuite { + UNKNOWN = 1; + AES_CTR_NOPADDING = 2; +} + +/** + * Encryption information for a file. + */ +message FileEncryptionInfoProto { + required CipherSuite suite = 1; + required bytes key = 2; + required bytes iv = 3; + required string ezKeyVersionName = 4; +} /** * A set of file blocks and their locations. @@ -211,9 +228,9 @@ message LocatedBlocksProto { required bool underConstruction = 3; optional LocatedBlockProto lastBlock = 4; required bool isLastBlockComplete = 5; + optional FileEncryptionInfoProto fileEncryptionInfo = 6; } - /** * Status of a file, directory or symlink * Optionally includes a file's block locations if requested by client on the rpc call. @@ -244,8 +261,12 @@ message HdfsFileStatusProto { // Optional field for fileId optional uint64 fileId = 13 [default = 0]; // default as an invalid id optional int32 childrenNum = 14 [default = -1]; - optional uint32 storagePolicy = 15 [default = 0]; // block storage policy id -} + + // Optional field for file encryption + optional FileEncryptionInfoProto fileEncryptionInfo = 15; + + optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id +} /** * Checksum algorithms/types used in HDFS Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto Thu Aug 21 05:22:10 2014 @@ -27,6 +27,7 @@ message XAttrProto { TRUSTED = 1; SECURITY = 2; SYSTEM = 3; + RAW = 4; } required XAttrNamespaceProto namespace = 1; Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Thu Aug 21 05:22:10 2014 @@ -1475,11 +1475,8 @@ <value></value> <description> SaslPropertiesResolver used to resolve the QOP used for a connection to the - DataNode when reading or writing block data. If not specified, the full set - of values specified in dfs.data.transfer.protection is used while - determining the QOP used for the connection. If a class is specified, then - the QOP values returned by the class will be used while determining the QOP - used for the connection. + DataNode when reading or writing block data. If not specified, the value of + hadoop.security.saslproperties.resolver.class is used as the default value. </description> </property> @@ -2061,4 +2058,13 @@ block layout (see HDFS-6482 for details on the layout).</description> </property> +<property> + <name>dfs.namenode.list.encryption.zones.num.responses</name> + <value>100</value> + <description>When listing encryption zones, the maximum number of zones + that will be returned in a batch. Fetching the list incrementally in + batches improves namenode performance. + </description> +</property> + </configuration> Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1609845-1619277 Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1616894-1619277 Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1594376-1619194 Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1614232-1619277 Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm Thu Aug 21 05:22:10 2014 @@ -30,7 +30,7 @@ Extended Attributes in HDFS ** {Namespaces and Permissions} - In HDFS, as in Linux, there are four valid namespaces: <<<user>>>, <<<trusted>>>, <<<system>>>, and <<<security>>>. Each of these namespaces have different access restrictions. + In HDFS, there are five valid namespaces: <<<user>>>, <<<trusted>>>, <<<system>>>, <<<security>>>, and <<<raw>>>. Each of these namespaces have different access restrictions. The <<<user>>> namespace is the namespace that will commonly be used by client applications. Access to extended attributes in the user namespace is controlled by the corresponding file permissions. @@ -40,6 +40,8 @@ Extended Attributes in HDFS The <<<security>>> namespace is reserved for internal HDFS use. This namespace is not accessible through userspace methods. It is currently unused. + The <<<raw>>> namespace is reserved for internal system attributes that sometimes need to be exposed. Like <<<system>>> namespace attributes they are not visible to the user except when <<<getXAttr>>>/<<<getXAttrs>>> is called on a file or directory in the <<</.reserved/raw>>> HDFS directory hierarchy. These attributes can only be accessed by the superuser. An example of where <<<raw>>> namespace extended attributes are used is the <<<distcp>>> utility. Encryption zone meta data is stored in <<<raw.*>>> extended attributes, so as long as the administrator uses <<</.reserved/raw>>> pathnames in source and target, the encrypted files in the encryption zones are transparently copied. + * {Interacting with extended attributes} The Hadoop shell has support for interacting with extended attributes via <<<hadoop fs -getfattr>>> and <<<hadoop fs -setfattr>>>. These commands are styled after the Linux {{{http://www.bestbits.at/acl/man/man1/getfattr.txt}getfattr(1)}} and {{{http://www.bestbits.at/acl/man/man1/setfattr.txt}setfattr(1)}} commands. Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm Thu Aug 21 05:22:10 2014 @@ -209,7 +209,7 @@ HDFS NFS Gateway [[2]] Start package included portmap (needs root privileges): ------------------------- - hadoop portmap + hdfs portmap OR @@ -224,7 +224,7 @@ HDFS NFS Gateway as long as the user has read access to the Kerberos keytab defined in "nfs.keytab.file". ------------------------- - hadoop nfs3 + hdfs nfs3 OR Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1609845-1619277 Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java Thu Aug 21 05:22:10 2014 @@ -29,7 +29,7 @@ import org.junit.Test; * Tests for <code>XAttr</code> objects. */ public class TestXAttr { - private static XAttr XATTR, XATTR1, XATTR2, XATTR3, XATTR4; + private static XAttr XATTR, XATTR1, XATTR2, XATTR3, XATTR4, XATTR5; @BeforeClass public static void setUp() throws Exception { @@ -58,6 +58,11 @@ public class TestXAttr { .setName("name") .setValue(value) .build(); + XATTR5 = new XAttr.Builder() + .setNameSpace(XAttr.NameSpace.RAW) + .setName("name") + .setValue(value) + .build(); } @Test @@ -65,14 +70,17 @@ public class TestXAttr { assertNotSame(XATTR1, XATTR2); assertNotSame(XATTR2, XATTR3); assertNotSame(XATTR3, XATTR4); + assertNotSame(XATTR4, XATTR5); assertEquals(XATTR, XATTR1); assertEquals(XATTR1, XATTR1); assertEquals(XATTR2, XATTR2); assertEquals(XATTR3, XATTR3); assertEquals(XATTR4, XATTR4); + assertEquals(XATTR5, XATTR5); assertFalse(XATTR1.equals(XATTR2)); assertFalse(XATTR2.equals(XATTR3)); assertFalse(XATTR3.equals(XATTR4)); + assertFalse(XATTR4.equals(XATTR5)); } @Test @@ -81,5 +89,6 @@ public class TestXAttr { assertFalse(XATTR1.hashCode() == XATTR2.hashCode()); assertFalse(XATTR2.hashCode() == XATTR3.hashCode()); assertFalse(XATTR3.hashCode() == XATTR4.hashCode()); + assertFalse(XATTR4.hashCode() == XATTR5.hashCode()); } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Thu Aug 21 05:22:10 2014 @@ -27,6 +27,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; @@ -78,6 +79,7 @@ import org.junit.Assume; import java.io.*; import java.net.*; import java.nio.ByteBuffer; +import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; import java.util.*; import java.util.concurrent.TimeoutException; @@ -86,6 +88,7 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** Utilities for HDFS tests */ public class DFSTestUtil { @@ -1305,6 +1308,71 @@ public class DFSTestUtil { } /** + * Verify that two files have the same contents. + * + * @param fs The file system containing the two files. + * @param p1 The path of the first file. + * @param p2 The path of the second file. + * @param len The length of the two files. + * @throws IOException + */ + public static void verifyFilesEqual(FileSystem fs, Path p1, Path p2, int len) + throws IOException { + final FSDataInputStream in1 = fs.open(p1); + final FSDataInputStream in2 = fs.open(p2); + for (int i = 0; i < len; i++) { + assertEquals("Mismatch at byte " + i, in1.read(), in2.read()); + } + in1.close(); + in2.close(); + } + + /** + * Verify that two files have different contents. + * + * @param fs The file system containing the two files. + * @param p1 The path of the first file. + * @param p2 The path of the second file. + * @param len The length of the two files. + * @throws IOException + */ + public static void verifyFilesNotEqual(FileSystem fs, Path p1, Path p2, + int len) + throws IOException { + final FSDataInputStream in1 = fs.open(p1); + final FSDataInputStream in2 = fs.open(p2); + try { + for (int i = 0; i < len; i++) { + if (in1.read() != in2.read()) { + return; + } + } + fail("files are equal, but should not be"); + } finally { + in1.close(); + in2.close(); + } + } + + /** + * Helper function to create a key in the Key Provider. + * + * @param keyName The name of the key to create + * @param cluster The cluster to create it in + * @param conf Configuration to use + */ + public static void createKey(String keyName, MiniDFSCluster cluster, + Configuration conf) + throws NoSuchAlgorithmException, IOException { + KeyProvider provider = cluster.getNameNode().getNamesystem().getProvider(); + final KeyProvider.Options options = KeyProvider.options(conf); + options.setDescription(keyName); + options.setBitLength(128); + provider.createKey(keyName, options); + provider.flush(); + } + + /** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the * given NameNOde. Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Thu Aug 21 05:22:10 2014 @@ -144,7 +144,7 @@ public class MiniDFSCluster { private int nameNodeHttpPort = 0; private final Configuration conf; private int numDataNodes = 1; - private StorageType storageType = StorageType.DEFAULT; + private StorageType[][] storageTypes = null; private boolean format = true; private boolean manageNameDfsDirs = true; private boolean manageNameDfsSharedDirs = true; @@ -193,10 +193,26 @@ public class MiniDFSCluster { } /** - * Default: StorageType.DEFAULT + * Set the same storage type configuration for each datanode. + * If storageTypes is uninitialized or passed null then + * StorageType.DEFAULT is used. */ - public Builder storageType(StorageType type) { - this.storageType = type; + public Builder storageTypes(StorageType[] types) { + assert types.length == DIRS_PER_DATANODE; + this.storageTypes = new StorageType[numDataNodes][types.length]; + for (int i = 0; i < numDataNodes; ++i) { + this.storageTypes[i] = types; + } + return this; + } + + /** + * Set custom storage type configuration for each datanode. + * If storageTypes is uninitialized or passed null then + * StorageType.DEFAULT is used. + */ + public Builder storageTypes(StorageType[][] types) { + this.storageTypes = types; return this; } @@ -369,7 +385,8 @@ public class MiniDFSCluster { builder.nnTopology = MiniDFSNNTopology.simpleSingleNN( builder.nameNodePort, builder.nameNodeHttpPort); } - + assert builder.storageTypes == null || + builder.storageTypes.length == builder.numDataNodes; final int numNameNodes = builder.nnTopology.countNameNodes(); LOG.info("starting cluster: numNameNodes=" + numNameNodes + ", numDataNodes=" + builder.numDataNodes); @@ -377,7 +394,7 @@ public class MiniDFSCluster { initMiniDFSCluster(builder.conf, builder.numDataNodes, - builder.storageType, + builder.storageTypes, builder.format, builder.manageNameDfsDirs, builder.manageNameDfsSharedDirs, @@ -477,8 +494,8 @@ public class MiniDFSCluster { * Servers will be started on free ports. * <p> * The caller must manage the creation of NameNode and DataNode directories - * and have already set {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} in the given conf. + * and have already set {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} in the given conf. * * @param conf the base configuration to use in starting the servers. This * will be modified as necessary. @@ -554,8 +571,8 @@ public class MiniDFSCluster { * @param format if true, format the NameNode and DataNodes before starting * up * @param manageDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -586,8 +603,8 @@ public class MiniDFSCluster { * @param numDataNodes Number of DataNodes to start; may be zero * @param format if true, format the NameNode and DataNodes before starting up * @param manageDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -620,11 +637,11 @@ public class MiniDFSCluster { * @param numDataNodes Number of DataNodes to start; may be zero * @param format if true, format the NameNode and DataNodes before starting up * @param manageNameDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param manageDataDfsDirs if true, the data directories for datanodes will - * be created and {@link #DFS_DATANODE_DATA_DIR_KEY} + * be created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} * set to same in the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -643,7 +660,7 @@ public class MiniDFSCluster { String[] racks, String hosts[], long[] simulatedCapacities) throws IOException { this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster - initMiniDFSCluster(conf, numDataNodes, StorageType.DEFAULT, format, + initMiniDFSCluster(conf, numDataNodes, null, format, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, null, racks, hosts, simulatedCapacities, null, true, false, @@ -652,7 +669,7 @@ public class MiniDFSCluster { private void initMiniDFSCluster( Configuration conf, - int numDataNodes, StorageType storageType, boolean format, boolean manageNameDfsDirs, + int numDataNodes, StorageType[][] storageTypes, boolean format, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean manageDataDfsDirs, StartupOption startOpt, StartupOption dnStartOpt, String[] racks, @@ -725,7 +742,7 @@ public class MiniDFSCluster { } // Start the DataNodes - startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs, + startDataNodes(conf, numDataNodes, storageTypes, manageDataDfsDirs, dnStartOpt != null ? dnStartOpt : startOpt, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays); @@ -1100,15 +1117,18 @@ public class MiniDFSCluster { } } - String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException { + String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws IOException { StringBuilder sb = new StringBuilder(); + assert storageTypes == null || storageTypes.length == DIRS_PER_DATANODE; for (int j = 0; j < DIRS_PER_DATANODE; ++j) { File dir = getInstanceStorageDir(dnIndex, j); dir.mkdirs(); if (!dir.isDirectory()) { throw new IOException("Mkdirs failed to create directory for DataNode " + dir); } - sb.append((j > 0 ? "," : "") + "[" + storageType + "]" + fileAsURI(dir)); + sb.append((j > 0 ? "," : "") + "[" + + (storageTypes == null ? StorageType.DEFAULT : storageTypes[j]) + + "]" + fileAsURI(dir)); } return sb.toString(); } @@ -1127,7 +1147,7 @@ public class MiniDFSCluster { * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be set + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set * in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1159,7 +1179,7 @@ public class MiniDFSCluster { * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be * set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1175,21 +1195,17 @@ public class MiniDFSCluster { String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, false, false, null); } - /** - * @see MiniDFSCluster#startDataNodes(Configuration, int, boolean, StartupOption, - * String[], String[], long[], boolean, boolean, boolean) - */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null); } @@ -1207,7 +1223,7 @@ public class MiniDFSCluster { * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be * set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1222,13 +1238,15 @@ public class MiniDFSCluster { * @throws IllegalStateException if NameNode has been shutdown */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, Configuration[] dnConfOverlays) throws IOException { + assert storageTypes == null || storageTypes.length == numDataNodes; + if (operation == StartupOption.RECOVER) { return; } @@ -1289,7 +1307,7 @@ public class MiniDFSCluster { // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - String dirs = makeDataNodeDirs(i, storageType); + String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]); dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); } @@ -2173,7 +2191,7 @@ public class MiniDFSCluster { } /** - * Multiple-NameNode version of {@link #injectBlocks(Iterable[])}. + * Multiple-NameNode version of injectBlocks. */ public void injectBlocks(int nameNodeIndex, int dataNodeIndex, Iterable<Block> blocksToInject) throws IOException { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java Thu Aug 21 05:22:10 2014 @@ -50,12 +50,14 @@ public class MiniDFSClusterWithNodeGroup } public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig) throws IOException { + assert storageTypes == null || storageTypes.length == numDataNodes; + if (operation == StartupOption.RECOVER) { return; } @@ -112,7 +114,7 @@ public class MiniDFSClusterWithNodeGroup // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - String dirs = makeDataNodeDirs(i, storageType); + String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]); dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); } @@ -190,7 +192,7 @@ public class MiniDFSClusterWithNodeGroup String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, nodeGroups, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, nodeGroups, hosts, simulatedCapacities, setupHostsFile, false, false); } @@ -205,14 +207,14 @@ public class MiniDFSClusterWithNodeGroup // This is for initialize from parent class. @Override public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, Configuration[] dnConfOverlays) throws IOException { - startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation, racks, + startDataNodes(conf, numDataNodes, storageTypes, manageDfsDirs, operation, racks, NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig); } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Thu Aug 21 05:22:10 2014 @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTru import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyShort; @@ -51,6 +52,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -253,16 +255,16 @@ public class TestDFSClientRetries { Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)).when(mockNN).getFileInfo(anyString()); + 1010, 0, null, (byte) 0)).when(mockNN).getFileInfo(anyString()); Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)) + 1010, 0, null, (byte) 0)) .when(mockNN) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), - anyShort(), anyLong()); + anyShort(), anyLong(), (List<CipherSuite>) anyList()); final DFSClient client = new DFSClient(null, mockNN, conf, null); OutputStream os = client.create("testfile", true); @@ -494,7 +496,8 @@ public class TestDFSClientRetries { List<LocatedBlock> badBlocks = new ArrayList<LocatedBlock>(); badBlocks.add(badLocatedBlock); return new LocatedBlocks(goodBlockList.getFileLength(), false, - badBlocks, null, true); + badBlocks, null, true, + null); } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java Thu Aug 21 05:22:10 2014 @@ -27,6 +27,9 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.junit.Test; @@ -125,4 +128,45 @@ public class TestDFSRename { if (cluster != null) {cluster.shutdown();} } } + + /** + * Check the blocks of dst file are cleaned after rename with overwrite + */ + @Test(timeout = 120000) + public void testRenameWithOverwrite() throws Exception { + final short replFactor = 2; + final long blockSize = 512; + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(replFactor).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + try { + + long fileLen = blockSize*3; + String src = "/foo/src"; + String dst = "/foo/dst"; + Path srcPath = new Path(src); + Path dstPath = new Path(dst); + + DFSTestUtil.createFile(dfs, srcPath, fileLen, replFactor, 1); + DFSTestUtil.createFile(dfs, dstPath, fileLen, replFactor, 1); + + LocatedBlocks lbs = NameNodeAdapter.getBlockLocations( + cluster.getNameNode(), dst, 0, fileLen); + BlockManager bm = NameNodeAdapter.getNamesystem(cluster.getNameNode()). + getBlockManager(); + assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock(). + getLocalBlock()) != null); + dfs.rename(srcPath, dstPath, Rename.OVERWRITE); + assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock(). + getLocalBlock()) == null); + } finally { + if (dfs != null) { + dfs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Thu Aug 21 05:22:10 2014 @@ -77,6 +77,13 @@ public class TestDFSShell { static final String TEST_ROOT_DIR = PathUtils.getTestDirName(TestDFSShell.class); + private static final String RAW_A1 = "raw.a1"; + private static final String TRUSTED_A1 = "trusted.a1"; + private static final String USER_A1 = "user.a1"; + private static final byte[] RAW_A1_VALUE = new byte[]{0x32, 0x32, 0x32}; + private static final byte[] TRUSTED_A1_VALUE = new byte[]{0x31, 0x31, 0x31}; + private static final byte[] USER_A1_VALUE = new byte[]{0x31, 0x32, 0x33}; + static Path writeFile(FileSystem fs, Path f) throws IOException { DataOutputStream out = fs.create(f); out.writeBytes("dhruba: " + f); @@ -1664,8 +1671,8 @@ public class TestDFSShell { final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - fs.setXAttr(src, "user.a1", new byte[]{0x31, 0x32, 0x33}); - fs.setXAttr(src, "trusted.a1", new byte[]{0x31, 0x31, 0x31}); + fs.setXAttr(src, USER_A1, USER_A1_VALUE); + fs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE); shell = new FsShell(conf); @@ -1722,8 +1729,8 @@ public class TestDFSShell { assertTrue(perm.equals(targetPerm)); xattrs = fs.getXAttrs(target3); assertEquals(xattrs.size(), 2); - assertArrayEquals(new byte[]{0x31, 0x32, 0x33}, xattrs.get("user.a1")); - assertArrayEquals(new byte[]{0x31, 0x31, 0x31}, xattrs.get("trusted.a1")); + assertArrayEquals(USER_A1_VALUE, xattrs.get(USER_A1)); + assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1)); acls = fs.getAclStatus(target3).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -1780,6 +1787,160 @@ public class TestDFSShell { } } + @Test (timeout = 120000) + public void testCopyCommandsWithRawXAttrs() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(1).format(true).build(); + FsShell shell = null; + FileSystem fs = null; + final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithRawXAttrs-" + + counter.getAndIncrement(); + final Path hdfsTestDir = new Path(testdir); + final Path rawHdfsTestDir = new Path("/.reserved/raw" + testdir); + try { + fs = cluster.getFileSystem(); + fs.mkdirs(hdfsTestDir); + final Path src = new Path(hdfsTestDir, "srcfile"); + final String rawSrcBase = "/.reserved/raw" + testdir; + final Path rawSrc = new Path(rawSrcBase, "srcfile"); + fs.create(src).close(); + + final Path srcDir = new Path(hdfsTestDir, "srcdir"); + final Path rawSrcDir = new Path("/.reserved/raw" + testdir, "srcdir"); + fs.mkdirs(srcDir); + final Path srcDirFile = new Path(srcDir, "srcfile"); + final Path rawSrcDirFile = + new Path("/.reserved/raw" + srcDirFile); + fs.create(srcDirFile).close(); + + final Path[] paths = { rawSrc, rawSrcDir, rawSrcDirFile }; + final String[] xattrNames = { USER_A1, RAW_A1 }; + final byte[][] xattrVals = { USER_A1_VALUE, RAW_A1_VALUE }; + + for (int i = 0; i < paths.length; i++) { + for (int j = 0; j < xattrNames.length; j++) { + fs.setXAttr(paths[i], xattrNames[j], xattrVals[j]); + } + } + + shell = new FsShell(conf); + + /* Check that a file as the source path works ok. */ + doTestCopyCommandsWithRawXAttrs(shell, fs, src, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, src, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, rawHdfsTestDir, true); + + /* Use a relative /.reserved/raw path. */ + final Path savedWd = fs.getWorkingDirectory(); + try { + fs.setWorkingDirectory(new Path(rawSrcBase)); + final Path relRawSrc = new Path("../srcfile"); + final Path relRawHdfsTestDir = new Path(".."); + doTestCopyCommandsWithRawXAttrs(shell, fs, relRawSrc, relRawHdfsTestDir, + true); + } finally { + fs.setWorkingDirectory(savedWd); + } + + /* Check that a directory as the source path works ok. */ + doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, rawHdfsTestDir, + true); + + /* Use relative in an absolute path. */ + final String relRawSrcDir = "./.reserved/../.reserved/raw/../raw" + + testdir + "/srcdir"; + final String relRawDstDir = "./.reserved/../.reserved/raw/../raw" + + testdir; + doTestCopyCommandsWithRawXAttrs(shell, fs, new Path(relRawSrcDir), + new Path(relRawDstDir), true); + } finally { + if (null != shell) { + shell.close(); + } + + if (null != fs) { + fs.delete(hdfsTestDir, true); + fs.close(); + } + cluster.shutdown(); + } + } + + private void doTestCopyCommandsWithRawXAttrs(FsShell shell, FileSystem fs, + Path src, Path hdfsTestDir, boolean expectRaw) throws Exception { + Path target; + boolean srcIsRaw; + if (src.isAbsolute()) { + srcIsRaw = src.toString().contains("/.reserved/raw"); + } else { + srcIsRaw = new Path(fs.getWorkingDirectory(), src). + toString().contains("/.reserved/raw"); + } + final boolean destIsRaw = hdfsTestDir.toString().contains("/.reserved/raw"); + final boolean srcDestMismatch = srcIsRaw ^ destIsRaw; + + // -p (possibly preserve raw if src & dst are both /.r/r */ + if (srcDestMismatch) { + doCopyAndTest(shell, hdfsTestDir, src, "-p", ERROR); + } else { + target = doCopyAndTest(shell, hdfsTestDir, src, "-p", SUCCESS); + checkXAttrs(fs, target, expectRaw, false); + } + + // -px (possibly preserve raw, always preserve non-raw xattrs. */ + if (srcDestMismatch) { + doCopyAndTest(shell, hdfsTestDir, src, "-px", ERROR); + } else { + target = doCopyAndTest(shell, hdfsTestDir, src, "-px", SUCCESS); + checkXAttrs(fs, target, expectRaw, true); + } + + // no args (possibly preserve raw, never preserve non-raw xattrs. */ + if (srcDestMismatch) { + doCopyAndTest(shell, hdfsTestDir, src, null, ERROR); + } else { + target = doCopyAndTest(shell, hdfsTestDir, src, null, SUCCESS); + checkXAttrs(fs, target, expectRaw, false); + } + } + + private Path doCopyAndTest(FsShell shell, Path dest, Path src, + String cpArgs, int expectedExitCode) throws Exception { + final Path target = new Path(dest, "targetfile" + + counter.getAndIncrement()); + final String[] argv = cpArgs == null ? + new String[] { "-cp", src.toUri().toString(), + target.toUri().toString() } : + new String[] { "-cp", cpArgs, src.toUri().toString(), + target.toUri().toString() }; + final int ret = ToolRunner.run(shell, argv); + assertEquals("cp -p is not working", expectedExitCode, ret); + return target; + } + + private void checkXAttrs(FileSystem fs, Path target, boolean expectRaw, + boolean expectVanillaXAttrs) throws Exception { + final Map<String, byte[]> xattrs = fs.getXAttrs(target); + int expectedCount = 0; + if (expectRaw) { + assertArrayEquals("raw.a1 has incorrect value", + RAW_A1_VALUE, xattrs.get(RAW_A1)); + expectedCount++; + } + if (expectVanillaXAttrs) { + assertArrayEquals("user.a1 has incorrect value", + USER_A1_VALUE, xattrs.get(USER_A1)); + expectedCount++; + } + assertEquals("xattrs size mismatch", expectedCount, xattrs.size()); + } + // verify cp -ptopxa option will preserve directory attributes. @Test (timeout = 120000) public void testCopyCommandsToDirectoryWithPreserveOption() @@ -1825,8 +1986,8 @@ public class TestDFSShell { final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - fs.setXAttr(srcDir, "user.a1", new byte[]{0x31, 0x32, 0x33}); - fs.setXAttr(srcDir, "trusted.a1", new byte[]{0x31, 0x31, 0x31}); + fs.setXAttr(srcDir, USER_A1, USER_A1_VALUE); + fs.setXAttr(srcDir, TRUSTED_A1, TRUSTED_A1_VALUE); shell = new FsShell(conf); @@ -1883,8 +2044,8 @@ public class TestDFSShell { assertTrue(perm.equals(targetPerm)); xattrs = fs.getXAttrs(targetDir3); assertEquals(xattrs.size(), 2); - assertArrayEquals(new byte[]{0x31, 0x32, 0x33}, xattrs.get("user.a1")); - assertArrayEquals(new byte[]{0x31, 0x31, 0x31}, xattrs.get("trusted.a1")); + assertArrayEquals(USER_A1_VALUE, xattrs.get(USER_A1)); + assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1)); acls = fs.getAclStatus(targetDir3).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Thu Aug 21 05:22:10 2014 @@ -104,7 +104,7 @@ public class TestDFSUtil { LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true); List<LocatedBlock> ls = Arrays.asList(l1, l2); - LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true); + LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null); BlockLocation[] bs = DFSUtil.locatedBlocks2Locations(lbs); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Thu Aug 21 05:22:10 2014 @@ -38,7 +38,6 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Random; -import java.util.concurrent.CancellationException; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.impl.Log4JLogger; Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java Thu Aug 21 05:22:10 2014 @@ -1142,7 +1142,7 @@ public class TestFileCreation { try { nnrpc.create(pathStr, new FsPermission((short)0755), "client", new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)), - true, (short)1, 128*1024*1024L); + true, (short)1, 128*1024*1024L, null); fail("Should have thrown exception when creating '" + pathStr + "'" + " by " + method); } catch (InvalidPathException ipe) { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java Thu Aug 21 05:22:10 2014 @@ -18,6 +18,9 @@ package org.apache.hadoop.hdfs; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyList; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.anyShort; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyShort; @@ -29,10 +32,12 @@ import static org.mockito.Mockito.spy; import java.io.DataOutputStream; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -338,16 +343,16 @@ public class TestLease { Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)).when(mcp).getFileInfo(anyString()); + 1010, 0, null, (byte) 0)).when(mcp).getFileInfo(anyString()); Mockito .doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, (byte) 0)) + 1010, 0, null, (byte) 0)) .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), - anyShort(), anyLong()); + anyShort(), anyLong(), (List<CipherSuite>) anyList()); final Configuration conf = new Configuration(); final DFSClient c1 = createDFSClientAs(ugi[0], conf); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java Thu Aug 21 05:22:10 2014 @@ -58,7 +58,7 @@ public class TestStorageReport { conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(REPL_FACTOR) - .storageType(storageType) + .storageTypes(new StorageType[] { storageType, storageType } ) .build(); fs = cluster.getFileSystem(); bpid = cluster.getNamesystem().getBlockPoolId(); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java Thu Aug 21 05:22:10 2014 @@ -69,6 +69,7 @@ public class FSXAttrBaseTest { protected static Configuration conf; private static int pathCount = 0; protected static Path path; + protected static Path rawPath; // XAttrs protected static final String name1 = "user.a1"; @@ -78,6 +79,8 @@ public class FSXAttrBaseTest { protected static final byte[] value2 = {0x37, 0x38, 0x39}; protected static final String name3 = "user.a3"; protected static final String name4 = "user.a4"; + protected static final String raw1 = "raw.a1"; + protected static final String raw2 = "raw.a2"; protected FileSystem fs; @@ -107,6 +110,7 @@ public class FSXAttrBaseTest { public void setUp() throws Exception { pathCount += 1; path = new Path("/p" + pathCount); + rawPath = new Path("/.reserved/raw/p" + pathCount); initFileSystem(); } @@ -395,7 +399,8 @@ public class FSXAttrBaseTest { Assert.fail("expected IOException"); } catch (Exception e) { GenericTestUtils.assertExceptionContains - ("An XAttr name must be prefixed with user/trusted/security/system, " + + ("An XAttr name must be prefixed with " + + "user/trusted/security/system/raw, " + "followed by a '.'", e); } @@ -582,7 +587,7 @@ public class FSXAttrBaseTest { /* Unknown namespace should throw an exception. */ final String expectedExceptionString = "An XAttr name must be prefixed " + - "with user/trusted/security/system, followed by a '.'"; + "with user/trusted/security/system/raw, followed by a '.'"; try { fs.removeXAttr(path, "wackynamespace.foo"); Assert.fail("expected IOException"); @@ -918,6 +923,176 @@ public class FSXAttrBaseTest { fsAsDiana.removeXAttr(path, name2); } + @Test(timeout = 120000) + public void testRawXAttrs() throws Exception { + final UserGroupInformation user = UserGroupInformation. + createUserForTesting("user", new String[] {"mygroup"}); + + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 0750)); + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE, + XAttrSetFlag.REPLACE)); + + { + // getXAttr + final byte[] value = fs.getXAttr(rawPath, raw1); + Assert.assertArrayEquals(value, value1); + } + + { + // getXAttrs + final Map<String, byte[]> xattrs = fs.getXAttrs(rawPath); + Assert.assertEquals(xattrs.size(), 1); + Assert.assertArrayEquals(value1, xattrs.get(raw1)); + fs.removeXAttr(rawPath, raw1); + } + + { + // replace and re-get + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw1, newValue1, EnumSet.of(XAttrSetFlag.CREATE, + XAttrSetFlag.REPLACE)); + + final Map<String,byte[]> xattrs = fs.getXAttrs(rawPath); + Assert.assertEquals(xattrs.size(), 1); + Assert.assertArrayEquals(newValue1, xattrs.get(raw1)); + + fs.removeXAttr(rawPath, raw1); + } + + { + // listXAttrs on rawPath ensuring raw.* xattrs are returned + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + + final List<String> xattrNames = fs.listXAttrs(rawPath); + assertTrue(xattrNames.contains(raw1)); + assertTrue(xattrNames.contains(raw2)); + assertTrue(xattrNames.size() == 2); + fs.removeXAttr(rawPath, raw1); + fs.removeXAttr(rawPath, raw2); + } + + { + // listXAttrs on non-rawPath ensuring no raw.* xattrs returned + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + + final List<String> xattrNames = fs.listXAttrs(path); + assertTrue(xattrNames.size() == 0); + fs.removeXAttr(rawPath, raw1); + fs.removeXAttr(rawPath, raw2); + } + + { + /* + * Test non-root user operations in the "raw.*" namespace. + */ + user.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + // Test that non-root can not set xattrs in the "raw.*" namespace + try { + // non-raw path + userFs.setXAttr(path, raw1, value1); + fail("setXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.setXAttr(rawPath, raw1, value1); + fail("setXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + // Test that non-root can not do getXAttrs in the "raw.*" namespace + try { + // non-raw path + userFs.getXAttrs(rawPath); + fail("getXAttrs should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttrs(path); + fail("getXAttrs should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + // Test that non-root can not do getXAttr in the "raw.*" namespace + try { + // non-raw path + userFs.getXAttr(rawPath, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttr(path, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + return null; + } + }); + } + + { + /* + * Test that non-root can not do getXAttr in the "raw.*" namespace + */ + fs.setXAttr(rawPath, raw1, value1); + user.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + try { + // non-raw path + userFs.getXAttr(rawPath, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttr(path, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + /* + * Test that only root can see raw.* xattrs returned from listXAttr + * and non-root can't do listXAttrs on /.reserved/raw. + */ + // non-raw path + final List<String> xattrNames = userFs.listXAttrs(path); + assertTrue(xattrNames.size() == 0); + try { + // raw path + userFs.listXAttrs(rawPath); + fail("listXAttrs on raw path should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + return null; + } + }); + fs.removeXAttr(rawPath, raw1); + } + } + /** * Creates a FileSystem for the super-user. * Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Thu Aug 21 05:22:10 2014 @@ -587,7 +587,8 @@ public class NNThroughputBenchmark imple // dummyActionNoSynch(fileIdx); nameNodeProto.create(fileNames[daemonId][inputIdx], FsPermission.getDefault(), clientName, new EnumSetWritable<CreateFlag>(EnumSet - .of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, replication, BLOCK_SIZE); + .of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, + replication, BLOCK_SIZE, null); long end = Time.now(); for(boolean written = !closeUponCreate; !written; written = nameNodeProto.complete(fileNames[daemonId][inputIdx], @@ -1133,7 +1134,7 @@ public class NNThroughputBenchmark imple String fileName = nameGenerator.getNextFileName("ThroughputBench"); nameNodeProto.create(fileName, FsPermission.getDefault(), clientName, new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)), true, replication, - BLOCK_SIZE); + BLOCK_SIZE, null); ExtendedBlock lastBlock = addBlocks(fileName, clientName); nameNodeProto.complete(fileName, clientName, lastBlock, INodeId.GRANDFATHER_INODE_ID); } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Thu Aug 21 05:22:10 2014 @@ -128,7 +128,7 @@ public class TestAddBlockRetry { nn.create(src, FsPermission.getFileDefault(), "clientName", new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)), - true, (short)3, 1024); + true, (short)3, 1024, null); // start first addBlock() LOG.info("Starting first addBlock for " + src); @@ -155,7 +155,7 @@ public class TestAddBlockRetry { // create file nameNodeRpc.create(src, FsPermission.getFileDefault(), "clientName", new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)), true, - (short) 3, 1024); + (short) 3, 1024, null); // start first addBlock() LOG.info("Starting first addBlock for " + src); LocatedBlock lb1 = nameNodeRpc.addBlock(src, "clientName", null, null, Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java Thu Aug 21 05:22:10 2014 @@ -191,14 +191,19 @@ public class TestFSDirectory { existingXAttrs.add(xAttr1); existingXAttrs.add(xAttr2); - // Adding a system namespace xAttr, isn't affected by inode xAttrs limit. - XAttr newXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM). + // Adding system and raw namespace xAttrs aren't affected by inode + // xAttrs limit. + XAttr newSystemXAttr = (new XAttr.Builder()). + setNameSpace(XAttr.NameSpace.SYSTEM).setName("a3"). + setValue(new byte[]{0x33, 0x33, 0x33}).build(); + XAttr newRawXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.RAW). setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build(); - List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(1); - newXAttrs.add(newXAttr); + List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(2); + newXAttrs.add(newSystemXAttr); + newXAttrs.add(newRawXAttr); List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - assertEquals(xAttrs.size(), 3); + assertEquals(xAttrs.size(), 4); // Adding a trusted namespace xAttr, is affected by inode xAttrs limit. XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace( Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Thu Aug 21 05:22:10 2014 @@ -1019,7 +1019,7 @@ public class TestFsck { HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, path, - fileId, numChildren, storagePolicy); + fileId, numChildren, null, storagePolicy); Result res = new Result(conf); try { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java Thu Aug 21 05:22:10 2014 @@ -209,19 +209,20 @@ public class TestNamenodeRetryCache { // Two retried calls succeed newCall(); HdfsFileStatus status = namesystem.startFile(src, perm, "holder", - "clientmachine", EnumSet.of(CreateFlag.CREATE), true, (short) 1, BlockSize); + "clientmachine", EnumSet.of(CreateFlag.CREATE), true, (short) 1, + BlockSize, null); Assert.assertEquals(status, namesystem.startFile(src, perm, "holder", "clientmachine", EnumSet.of(CreateFlag.CREATE), - true, (short) 1, BlockSize)); + true, (short) 1, BlockSize, null)); Assert.assertEquals(status, namesystem.startFile(src, perm, "holder", "clientmachine", EnumSet.of(CreateFlag.CREATE), - true, (short) 1, BlockSize)); + true, (short) 1, BlockSize, null)); // A non-retried call fails newCall(); try { namesystem.startFile(src, perm, "holder", "clientmachine", - EnumSet.of(CreateFlag.CREATE), true, (short) 1, BlockSize); + EnumSet.of(CreateFlag.CREATE), true, (short) 1, BlockSize, null); Assert.fail("testCreate - expected exception is not thrown"); } catch (IOException e) { // expected Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java Thu Aug 21 05:22:10 2014 @@ -395,7 +395,7 @@ public class TestRetryCacheWithHA { this.status = client.getNamenode().create(fileName, FsPermission.getFileDefault(), client.getClientName(), new EnumSetWritable<CreateFlag>(createFlag), false, DataNodes, - BlockSize); + BlockSize, null); } @Override Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Thu Aug 21 05:22:10 2014 @@ -64,7 +64,7 @@ public class TestJsonUtil { final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26, now, now + 10, new FsPermission((short) 0644), "user", "group", DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"), - INodeId.GRANDFATHER_INODE_ID, 0, (byte) 0); + INodeId.GRANDFATHER_INODE_ID, 0, null, (byte) 0); final FileStatus fstatus = toFileStatus(status, parent); System.out.println("status = " + status); System.out.println("fstatus = " + fstatus); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml Thu Aug 21 05:22:10 2014 @@ -64,7 +64,7 @@ <comparators> <comparator> <type>SubstringComparator</type> - <expected-output>name must be prefixed with user/trusted/security/system, followed by a '.'</expected-output> + <expected-output>name must be prefixed with user/trusted/security/system/raw, followed by a '.'</expected-output> </comparator> </comparators> </test> @@ -126,6 +126,42 @@ </test> <test> + <description>setfattr : Add an xattr of raw namespace</description> + <test-commands> + <command>-fs NAMENODE -touchz /file1</command> + <command>-fs NAMENODE -setfattr -n raw.a1 -v 123456 /file1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /file1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>setfattr: User doesn't have permission for xattr: raw.a1</expected-output> + </comparator> + </comparators> + + </test> + + <test> + <description>setfattr : Add an xattr of raw namespace</description> + <test-commands> + <command>-fs NAMENODE -touchz /file1</command> + <command>-fs NAMENODE -setfattr -n raw.a1 -v 123456 /.reserved/raw/file1</command> + <command>-fs NAMENODE -getfattr -n raw.a1 /.reserved/raw/file1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /file1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>raw.a1="123456"</expected-output> + </comparator> + </comparators> + </test> + + <test> <description>setfattr : Add an xattr, and encode is text</description> <test-commands> <command>-fs NAMENODE -touchz /file1</command> @@ -256,6 +292,26 @@ </comparator> </comparators> </test> + + <test> + <description>setfattr : Remove an xattr of raw namespace</description> + <test-commands> + <command>-fs NAMENODE -touchz /file1</command> + <command>-fs NAMENODE -setfattr -n raw.a1 -v 123456 /.reserved/raw/file1</command> + <command>-fs NAMENODE -setfattr -n raw.a2 -v 123456 /.reserved/raw/file1</command> + <command>-fs NAMENODE -setfattr -x raw.a2 /.reserved/raw/file1</command> + <command>-fs NAMENODE -getfattr -d /.reserved/raw/file1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /file1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output># file: /.reserved/raw/file1#LF#raw.a1="123456"#LF#</expected-output> + </comparator> + </comparators> + </test> <test> <description>getfattr : Get an xattr</description>
