Author: cnauroth
Date: Thu Jun 12 16:58:40 2014
New Revision: 1602229
URL: http://svn.apache.org/r1602229
Log:
HADOOP-10561. Merging change r1602225 from trunk to branch-2.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1602229&r1=1602228&r2=1602229&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Jun 12 16:58:40 2014
@@ -218,6 +218,9 @@ Release 2.5.0 - UNRELEASED
HADOOP-10575. Small fixes for XAttrCommands and test. (Yi Liu via
umamahesh)
+ HADOOP-10561. Copy command with preserve option should handle Xattrs.
+ (Yi Liu via cnauroth)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1602229&r1=1602228&r2=1602229&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
Thu Jun 12 16:58:40 2014
@@ -22,7 +22,12 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.EnumSet;
+import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -45,7 +50,6 @@ import org.apache.hadoop.io.IOUtils;
abstract class CommandWithDestination extends FsCommand {
protected PathData dst;
private boolean overwrite = false;
- private boolean preserve = false;
private boolean verifyChecksum = true;
private boolean writeChecksum = true;
@@ -74,7 +78,54 @@ abstract class CommandWithDestination ex
* implementation allows.
*/
protected void setPreserve(boolean preserve) {
- this.preserve = preserve;
+ if (preserve) {
+ preserve(FileAttribute.TIMESTAMPS);
+ preserve(FileAttribute.OWNERSHIP);
+ preserve(FileAttribute.PERMISSION);
+ } else {
+ preserveStatus.clear();
+ }
+ }
+
+ protected static enum FileAttribute {
+ TIMESTAMPS, OWNERSHIP, PERMISSION, XATTR;
+
+ public static FileAttribute getAttribute(char symbol) {
+ for (FileAttribute attribute : values()) {
+ if (attribute.name().charAt(0) == Character.toUpperCase(symbol)) {
+ return attribute;
+ }
+ }
+ throw new NoSuchElementException("No attribute for " + symbol);
+ }
+ }
+
+ private EnumSet<FileAttribute> preserveStatus =
+ EnumSet.noneOf(FileAttribute.class);
+
+ /**
+ * Checks if the input attribute should be preserved or not
+ *
+ * @param attribute - Attribute to check
+ * @return boolean true if attribute should be preserved, false otherwise
+ */
+ private boolean shouldPreserve(FileAttribute attribute) {
+ return preserveStatus.contains(attribute);
+ }
+
+ /**
+ * Add file attributes that need to be preserved. This method may be
+ * called multiple times to add attributes.
+ *
+ * @param fileAttribute - Attribute to add, one at a time
+ */
+ protected void preserve(FileAttribute fileAttribute) {
+ for (FileAttribute attribute : preserveStatus) {
+ if (attribute.equals(fileAttribute)) {
+ return;
+ }
+ }
+ preserveStatus.add(fileAttribute);
}
/**
@@ -243,22 +294,36 @@ abstract class CommandWithDestination ex
try {
in = src.fs.open(src.path);
copyStreamToTarget(in, target);
+ if (shouldPreserve(FileAttribute.TIMESTAMPS)) {
+ target.fs.setTimes(
+ target.path,
+ src.stat.getModificationTime(),
+ src.stat.getAccessTime());
+ }
+ if (shouldPreserve(FileAttribute.OWNERSHIP)) {
+ target.fs.setOwner(
+ target.path,
+ src.stat.getOwner(),
+ src.stat.getGroup());
+ }
+ if (shouldPreserve(FileAttribute.PERMISSION)) {
+ target.fs.setPermission(
+ target.path,
+ src.stat.getPermission());
+ }
+ if (shouldPreserve(FileAttribute.XATTR)) {
+ Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
+ if (srcXAttrs != null) {
+ Iterator<Entry<String, byte[]>> iter =
srcXAttrs.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<String, byte[]> entry = iter.next();
+ target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
+ }
+ }
+ }
} finally {
IOUtils.closeStream(in);
}
- if(preserve) {
- target.fs.setTimes(
- target.path,
- src.stat.getModificationTime(),
- src.stat.getAccessTime());
- target.fs.setOwner(
- target.path,
- src.stat.getOwner(),
- src.stat.getGroup());
- target.fs.setPermission(
- target.path,
- src.stat.getPermission());
- }
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1602229&r1=1602228&r2=1602229&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
Thu Jun 12 16:58:40 2014
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -132,24 +133,47 @@ class CopyCommands {
static class Cp extends CommandWithDestination {
public static final String NAME = "cp";
- public static final String USAGE = "[-f] [-p] <src> ... <dst>";
+ public static final String USAGE = "[-f] [-p | -p[topx]] <src> ... <dst>";
public static final String DESCRIPTION =
"Copy files that match the file pattern <src> to a\n" +
"destination. When copying multiple files, the destination\n" +
- "must be a directory. Passing -p preserves access and\n" +
- "modification times, ownership and the mode. Passing -f\n" +
+ "must be a directory. Passing -p preserves status\n" +
+ "[topx] (timestamps, ownership, permission, XAttr).\n" +
+ "If -p is specified with no <arg>, then preserves\n" +
+ "timestamps, ownership, permission. Passing -f\n" +
"overwrites the destination if it already exists.\n";
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "p");
+ popPreserveOption(args);
+ CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f");
cf.parse(args);
setOverwrite(cf.getOpt("f"));
- setPreserve(cf.getOpt("p"));
// should have a -r option
setRecursive(true);
getRemoteDestination(args);
}
+
+ private void popPreserveOption(List<String> args) {
+ for (Iterator<String> iter = args.iterator(); iter.hasNext(); ) {
+ String cur = iter.next();
+ if (cur.equals("--")) {
+ // stop parsing arguments when you see --
+ break;
+ } else if (cur.startsWith("-p")) {
+ iter.remove();
+ if (cur.length() == 2) {
+ setPreserve(true);
+ } else {
+ String attributes = cur.substring(2);
+ for (int index = 0; index < attributes.length(); index++) {
+ preserve(FileAttribute.getAttribute(attributes.charAt(index)));
+ }
+ }
+ return;
+ }
+ }
+ }
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1602229&r1=1602228&r2=1602229&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
Thu Jun 12 16:58:40 2014
@@ -159,7 +159,7 @@ count
cp
- Usage: <<<hdfs dfs -cp [-f] URI [URI ...] <dest> >>>
+ Usage: <<<hdfs dfs -cp [-f] [-p | -p[topx]] URI [URI ...] <dest> >>>
Copy files from source to destination. This command allows multiple sources
as well in which case the destination must be a directory.
@@ -167,6 +167,10 @@ cp
Options:
* The -f option will overwrite the destination if it already exists.
+
+ * The -p option will preserve file attributes [topx] (timestamps,
+ ownership, permission, XAttr). If -p is specified with no <arg>,
+ then preserves timestamps, ownership, permission.
Example:
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1602229&r1=1602228&r2=1602229&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
Thu Jun 12 16:58:40 2014
@@ -276,7 +276,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-cp \[-f\] \[-p\] <src> \.\.\. <dst>:(
|\t)*Copy files that match the file pattern <src> to a(
)*</expected-output>
+ <expected-output>^-cp \[-f\] \[-p \| -p\[topx\]\] <src> \.\.\.
<dst>:( |\t)*Copy files that match the file pattern <src> to a(
)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -284,11 +284,19 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*must be a directory.( )*Passing -p
preserves access and( )*</expected-output>
+ <expected-output>^( |\t)*must be a directory.( )*Passing -p
preserves status( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*modification times, ownership and the mode.
Passing -f( )*</expected-output>
+ <expected-output>^( |\t)*\[topx\] \(timestamps, ownership,
permission, XAttr\).( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*If -p is specified with no <arg>,
then preserves( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*timestamps, ownership, permission. Passing
-f( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
