Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java Thu Aug 21 05:22:10 2014 @@ -102,7 +102,7 @@ public class FSDataOutputStream extends } /** - * Get a reference to the wrapped output stream. Used by unit tests. + * Get a reference to the wrapped output stream. * * @return the underlying output stream */
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Thu Aug 21 05:22:10 2014 @@ -58,6 +58,17 @@ abstract class CommandWithDestination ex private boolean writeChecksum = true; /** + * The name of the raw xattr namespace. It would be nice to use + * XAttr.RAW.name() but we can't reference the hadoop-hdfs project. + */ + private static final String RAW = "raw."; + + /** + * The name of the reserved raw directory. + */ + private static final String RESERVED_RAW = "/.reserved/raw"; + + /** * * This method is used to enable the force(-f) option while copying the files. * @@ -231,7 +242,7 @@ abstract class CommandWithDestination ex /** * Called with a source and target destination pair * @param src for the operation - * @param target for the operation + * @param dst for the operation * @throws IOException if anything goes wrong */ protected void processPath(PathData src, PathData dst) throws IOException { @@ -253,6 +264,8 @@ abstract class CommandWithDestination ex // modify dst as we descend to append the basename of the // current directory being processed dst = getTargetPath(src); + final boolean preserveRawXattrs = + checkPathsForReservedRaw(src.path, dst.path); if (dst.exists) { if (!dst.stat.isDirectory()) { throw new PathIsNotDirectoryException(dst.toString()); @@ -268,7 +281,7 @@ abstract class CommandWithDestination ex } super.recursePath(src); if (dst.stat.isDirectory()) { - preserveAttributes(src, dst); + preserveAttributes(src, dst, preserveRawXattrs); } } finally { dst = savedDst; @@ -295,19 +308,61 @@ abstract class CommandWithDestination ex * @param target where to copy the item * @throws IOException if copy fails */ - protected void copyFileToTarget(PathData src, PathData target) throws IOException { + protected void copyFileToTarget(PathData src, PathData target) + throws IOException { + final boolean preserveRawXattrs = + checkPathsForReservedRaw(src.path, target.path); src.fs.setVerifyChecksum(verifyChecksum); InputStream in = null; try { in = src.fs.open(src.path); copyStreamToTarget(in, target); - preserveAttributes(src, target); + preserveAttributes(src, target, preserveRawXattrs); } finally { IOUtils.closeStream(in); } } /** + * Check the source and target paths to ensure that they are either both in + * /.reserved/raw or neither in /.reserved/raw. If neither src nor target are + * in /.reserved/raw, then return false, indicating not to preserve raw.* + * xattrs. If both src/target are in /.reserved/raw, then return true, + * indicating raw.* xattrs should be preserved. If only one of src/target is + * in /.reserved/raw then throw an exception. + * + * @param src The source path to check. This should be a fully-qualified + * path, not relative. + * @param target The target path to check. This should be a fully-qualified + * path, not relative. + * @return true if raw.* xattrs should be preserved. + * @throws PathOperationException is only one of src/target are in + * /.reserved/raw. + */ + private boolean checkPathsForReservedRaw(Path src, Path target) + throws PathOperationException { + final boolean srcIsRR = Path.getPathWithoutSchemeAndAuthority(src). + toString().startsWith(RESERVED_RAW); + final boolean dstIsRR = Path.getPathWithoutSchemeAndAuthority(target). + toString().startsWith(RESERVED_RAW); + boolean preserveRawXattrs = false; + if (srcIsRR && !dstIsRR) { + final String s = "' copy from '" + RESERVED_RAW + "' to non '" + + RESERVED_RAW + "'. Either both source and target must be in '" + + RESERVED_RAW + "' or neither."; + throw new PathOperationException("'" + src.toString() + s); + } else if (!srcIsRR && dstIsRR) { + final String s = "' copy from non '" + RESERVED_RAW +"' to '" + + RESERVED_RAW + "'. Either both source and target must be in '" + + RESERVED_RAW + "' or neither."; + throw new PathOperationException("'" + dst.toString() + s); + } else if (srcIsRR && dstIsRR) { + preserveRawXattrs = true; + } + return preserveRawXattrs; + } + + /** * Copies the stream contents to a temporary file. If the copy is * successful, the temporary file will be renamed to the real path, * else the temporary file will be deleted. @@ -337,9 +392,11 @@ abstract class CommandWithDestination ex * attribute to preserve. * @param src source to preserve * @param target where to preserve attributes + * @param preserveRawXAttrs true if raw.* xattrs should be preserved * @throws IOException if fails to preserve attributes */ - protected void preserveAttributes(PathData src, PathData target) + protected void preserveAttributes(PathData src, PathData target, + boolean preserveRawXAttrs) throws IOException { if (shouldPreserve(FileAttribute.TIMESTAMPS)) { target.fs.setTimes( @@ -369,13 +426,17 @@ abstract class CommandWithDestination ex target.fs.setAcl(target.path, srcFullEntries); } } - if (shouldPreserve(FileAttribute.XATTR)) { + final boolean preserveXAttrs = shouldPreserve(FileAttribute.XATTR); + if (preserveXAttrs || preserveRawXAttrs) { Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path); if (srcXAttrs != null) { Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator(); while (iter.hasNext()) { Entry<String, byte[]> entry = iter.next(); - target.fs.setXAttr(target.path, entry.getKey(), entry.getValue()); + final String xattrName = entry.getKey(); + if (xattrName.startsWith(RAW) || preserveXAttrs) { + target.fs.setXAttr(target.path, entry.getKey(), entry.getValue()); + } } } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Thu Aug 21 05:22:10 2014 @@ -143,7 +143,11 @@ class CopyCommands { "timestamps, ownership, permission. If -pa is specified, " + "then preserves permission also because ACL is a super-set of " + "permission. Passing -f overwrites the destination if it " + - "already exists.\n"; + "already exists. raw namespace extended attributes are preserved " + + "if (1) they are supported (HDFS only) and, (2) all of the source and " + + "target pathnames are in the /.reserved/raw hierarchy. raw namespace " + + "xattr preservation is determined solely by the presence (or absence) " + + "of the /.reserved/raw prefix and not by the -p option.\n"; @Override protected void processOptions(LinkedList<String> args) throws IOException { Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java Thu Aug 21 05:22:10 2014 @@ -78,6 +78,11 @@ public class NativeCodeLoader { * Returns true only if this build was compiled with support for snappy. */ public static native boolean buildSupportsSnappy(); + + /** + * Returns true only if this build was compiled with support for openssl. + */ + public static native boolean buildSupportsOpenssl(); public static native String getLibraryName(); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java Thu Aug 21 05:22:10 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.util; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; @@ -60,6 +61,8 @@ public class NativeLibraryChecker { // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); + boolean openSslLoaded = false; + String openSslDetail = ""; String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; @@ -76,6 +79,13 @@ public class NativeLibraryChecker { if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { snappyLibraryName = SnappyCodec.getLibraryName(); } + if (OpensslCipher.getLoadingFailureReason() != null) { + openSslDetail = OpensslCipher.getLoadingFailureReason(); + openSslLoaded = false; + } else { + openSslDetail = OpensslCipher.getLibraryName(); + openSslLoaded = true; + } if (lz4Loaded) { lz4LibraryName = Lz4Codec.getLibraryName(); } @@ -84,11 +94,12 @@ public class NativeLibraryChecker { } } System.out.println("Native library checking:"); - System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); - System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); - System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); - System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); - System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); + System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); + System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); + System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); + System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); + System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); + System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail); if ((!nativeHadoopLoaded) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c Thu Aug 21 05:22:10 2014 @@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apac #endif } +JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl + (JNIEnv *env, jclass clazz) +{ +#ifdef HADOOP_OPENSSL_LIBRARY + return JNI_TRUE; +#else + return JNI_FALSE; +#endif +} + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName (JNIEnv *env, jclass clazz) { Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Thu Aug 21 05:22:10 2014 @@ -1445,6 +1445,74 @@ for ldap providers in the same way as ab true. </description> </property> + +<property> + <name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name> + <value></value> + <description> + The prefix for a given crypto codec, contains a comma-separated + list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE). + The first implementation will be used if available, others are fallbacks. + </description> +</property> + +<property> + <name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name> + <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value> + <description> + Comma-separated list of crypto codec implementations for AES/CTR/NoPadding. + The first implementation will be used if available, others are fallbacks. + </description> +</property> + +<property> + <name>hadoop.security.crypto.cipher.suite</name> + <value>AES/CTR/NoPadding</value> + <description> + Cipher suite for crypto codec. + </description> +</property> + +<property> + <name>hadoop.security.crypto.jce.provider</name> + <value></value> + <description> + The JCE provider name used in CryptoCodec. + </description> +</property> + +<property> + <name>hadoop.security.crypto.buffer.size</name> + <value>8192</value> + <description> + The buffer size used by CryptoInputStream and CryptoOutputStream. + </description> +</property> + +<property> + <name>hadoop.security.java.secure.random.algorithm</name> + <value>SHA1PRNG</value> + <description> + The java secure random algorithm. + </description> +</property> + +<property> + <name>hadoop.security.secure.random.impl</name> + <value></value> + <description> + Implementation of secure random. + </description> +</property> + +<property> + <name>hadoop.security.random.device.file.path</name> + <value>/dev/urandom</value> + <description> + OS security random device file path. + </description> +</property> + <property> <name>fs.har.impl.disable.cache</name> <value>true</value> @@ -1483,4 +1551,5 @@ for ldap providers in the same way as ab key will be dropped. Default = 12hrs </description> </property> + </configuration> Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Thu Aug 21 05:22:10 2014 @@ -168,15 +168,22 @@ cp Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory. + 'raw.*' namespace extended attributes are preserved if (1) the source and + destination filesystems support them (HDFS only), and (2) all source and + destination pathnames are in the /.reserved/raw hierarchy. Determination of + whether raw.* namespace xattrs are preserved is independent of the + -p (preserve) flag. + Options: * The -f option will overwrite the destination if it already exists. - * The -p option will preserve file attributes [topx] (timestamps, + * The -p option will preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no <arg>, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of - permission. + permission. Determination of whether raw namespace extended attributes + are preserved is independent of the -p flag. Example: Propchange: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/core:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1609845-1619277 Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java Thu Aug 21 05:22:10 2014 @@ -22,6 +22,7 @@ import static org.junit.Assert.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.zlib.ZlibFactory; @@ -54,6 +55,9 @@ public class TestNativeCodeLoader { if (NativeCodeLoader.buildSupportsSnappy()) { assertFalse(SnappyCodec.getLibraryName().isEmpty()); } + if (NativeCodeLoader.buildSupportsOpenssl()) { + assertFalse(OpensslCipher.getLibraryName().isEmpty()); + } assertFalse(Lz4Codec.getLibraryName().isEmpty()); LOG.info("TestNativeCodeLoader: libhadoop.so is loaded."); } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml Thu Aug 21 05:22:10 2014 @@ -324,7 +324,23 @@ </comparator> <comparator> <type>RegexpComparator</type> - <expected-output>^\s*permission. Passing -f overwrites the destination if it already exists.( )*</expected-output> + <expected-output>^( |\t)*permission. Passing -f overwrites the destination if it already exists. raw( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*namespace extended attributes are preserved if \(1\) they are supported \(HDFS( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*only\) and, \(2\) all of the source and target pathnames are in the \/\.reserved\/raw( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*hierarchy. raw namespace xattr preservation is determined solely by the presence( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^\s*\(or absence\) of the \/\.reserved\/raw prefix and not by the -p option.( )*</expected-output> </comparator> </comparators> </test>
