HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cde37c9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cde37c9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cde37c9 Branch: refs/heads/HDFS-6581 Commit: 3cde37c991b18370cc1b383f920a9d5bd2d91adb Parents: cbf0ae7 Author: Allen Wittenauer <a...@apache.org> Authored: Wed Sep 24 15:38:18 2014 -0700 Committer: Allen Wittenauer <a...@apache.org> Committed: Wed Sep 24 15:38:30 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../hadoop/tools/CopyListingFileStatus.java | 2 +- .../apache/hadoop/tools/DistCpOptionSwitch.java | 21 +- .../org/apache/hadoop/tools/DistCpOptions.java | 2 +- .../apache/hadoop/tools/util/DistCpUtils.java | 58 +- .../apache/hadoop/tools/TestOptionsParser.java | 2 +- .../hadoop/tools/mapred/TestCopyMapper.java | 1 + .../hadoop/tools/util/TestDistCpUtils.java | 1024 ++++++++++++++++-- 8 files changed, 996 insertions(+), 116 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b2a119f..a4b3314 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -543,6 +543,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11017. KMS delegation token secret manager should be able to use zookeeper as store. (asuresh via tucu) + HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 04437d7..9b31dfb 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -101,7 +101,7 @@ public final class CopyListingFileStatus extends FileStatus { * @return Map<String, byte[]> containing all xAttrs */ public Map<String, byte[]> getXAttrs() { - return xAttrs; + return xAttrs != null ? xAttrs : Collections.<String, byte[]>emptyMap(); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index c544813..9b85997 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -37,18 +37,21 @@ public enum DistCpOptionSwitch { /** * Preserves status of file/path in the target. * Default behavior with -p, is to preserve replication, - * block size, user, group, permission and checksum type on the target file. - * Note that when preserving checksum type, block size is also preserved. + * block size, user, group, permission, checksum type and timestamps on the + * target file. Note that when preserving checksum type, block size is also + * preserved. * - * If any of the optional switches are present among rbugpc, then - * only the corresponding file attribute is preserved. + * @see PRESERVE_STATUS_DEFAULT * + * If any of the optional switches are present among rbugpcaxt, then + * only the corresponding file attribute is preserved. */ PRESERVE_STATUS(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, - new Option("p", true, "preserve status (rbugpcax)(replication, " + - "block-size, user, group, permission, checksum-type, ACL, XATTR). " + - "If -p is specified with no <arg>, then preserves replication, " + - "block size, user, group, permission and checksum type." + + new Option("p", true, "preserve status (rbugpcaxt)(replication, " + + "block-size, user, group, permission, checksum-type, ACL, XATTR, " + + "timestamps). If -p is specified with no <arg>, then preserves " + + "replication, block size, user, group, permission, checksum type " + + "and timestamps. " + "raw.* xattrs are preserved when both the source and destination " + "paths are in the /.reserved/raw hierarchy (HDFS only). raw.* xattr" + "preservation is independent of the -p flag." + @@ -166,7 +169,7 @@ public enum DistCpOptionSwitch { BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, new Option("bandwidth", true, "Specify bandwidth per map in MB")); - static final String PRESERVE_STATUS_DEFAULT = "-prbugpc"; + public static final String PRESERVE_STATUS_DEFAULT = "-prbugpct"; private final String confLabel; private final Option option; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index a2a5be3..57d2fb7 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -68,7 +68,7 @@ public class DistCpOptions { private boolean targetPathExists = true; public static enum FileAttribute{ - REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR; + REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; public static FileAttribute getAttribute(char symbol) { for (FileAttribute attribute : values()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index abd30ee..71e84a1 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -18,39 +18,39 @@ package org.apache.hadoop.tools.util; -import com.google.common.collect.Maps; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.text.DecimalFormat; +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException; import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException; import org.apache.hadoop.tools.CopyListingFileStatus; +import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.mapred.UniformSizeInputFormat; -import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException; -import org.apache.hadoop.tools.DistCpOptions; -import org.apache.hadoop.mapreduce.InputFormat; -import java.io.IOException; -import java.util.EnumSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.text.DecimalFormat; -import java.net.URI; -import java.net.InetAddress; -import java.net.UnknownHostException; +import com.google.common.collect.Maps; /** * Utility functions used in DistCp. @@ -163,7 +163,7 @@ public class DistCpUtils { } /** - * Un packs preservation attribute string containing the first character of + * Unpacks preservation attribute string containing the first character of * each preservation attribute back to a set of attributes to preserve * @param attributes - Attribute string * @return - Attribute set @@ -209,7 +209,7 @@ public class DistCpUtils { if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } - // setAcl can't preserve sticky bit, so also call setPermission if needed. + // setAcl doesn't preserve sticky bit, so also call setPermission if needed. if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); @@ -225,30 +225,28 @@ public class DistCpUtils { Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { - Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator(); - while (iter.hasNext()) { - Entry<String, byte[]> entry = iter.next(); - final String xattrName = entry.getKey(); + for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { + String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { - targetFS.setXAttr(path, entry.getKey(), entry.getValue()); + targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } - if (attributes.contains(FileAttribute.REPLICATION) && ! targetFileStatus.isDirectory() && - srcFileStatus.getReplication() != targetFileStatus.getReplication()) { + if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && + (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && - !group.equals(srcFileStatus.getGroup())) { + !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && - !user.equals(srcFileStatus.getOwner())) { + !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } @@ -256,6 +254,12 @@ public class DistCpUtils { if (chown) { targetFS.setOwner(path, user, group); } + + if (attributes.contains(FileAttribute.TIMES)) { + targetFS.setTimes(path, + srcFileStatus.getModificationTime(), + srcFileStatus.getAccessTime()); + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index 2c42269..30fb25b 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -497,7 +497,7 @@ public class TestOptionsParser { attribIterator.next(); i++; } - Assert.assertEquals(i, 6); + Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2); try { OptionsParser.parse(new String[] { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java index 3f84772..ec60fa8 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java @@ -590,6 +590,7 @@ public class TestCopyMapper { EnumSet.allOf(DistCpOptions.FileAttribute.class); preserveStatus.remove(DistCpOptions.FileAttribute.ACL); preserveStatus.remove(DistCpOptions.FileAttribute.XATTR); + preserveStatus.remove(DistCpOptions.FileAttribute.TIMES); context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde37c9/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java index c4b64b9..200d786 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java @@ -18,38 +18,44 @@ package org.apache.hadoop.tools.util; -import org.apache.hadoop.tools.DistCpOptions.FileAttribute; +import java.io.IOException; +import java.io.OutputStream; +import java.util.EnumSet; +import java.util.Random; +import java.util.Stack; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.tools.CopyListingFileStatus; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.tools.DistCpOptionSwitch; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Test; import org.junit.BeforeClass; -import org.junit.AfterClass; - -import java.util.EnumSet; -import java.util.Random; -import java.util.Stack; -import java.io.IOException; -import java.io.OutputStream; +import org.junit.Test; public class TestDistCpUtils { private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class); private static final Configuration config = new Configuration(); private static MiniDFSCluster cluster; - + private static final FsPermission fullPerm = new FsPermission((short) 777); + private static final FsPermission almostFullPerm = new FsPermission((short) 666); + private static final FsPermission noPerm = new FsPermission((short) 0); + @BeforeClass public static void create() throws IOException { - cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).format(true) - .build(); + cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(1) + .format(true) + .build(); } @AfterClass @@ -61,13 +67,16 @@ public class TestDistCpUtils { @Test public void testGetRelativePathRoot() { + Path root = new Path("/"); + Path child = new Path("/a"); + Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a"); + } + + @Test + public void testGetRelativePath() { Path root = new Path("/tmp/abc"); Path child = new Path("/tmp/abc/xyz/file"); Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/xyz/file"); - - root = new Path("/"); - child = new Path("/a"); - Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a"); } @Test @@ -77,70 +86,916 @@ public class TestDistCpUtils { attributes.add(FileAttribute.REPLICATION); Assert.assertEquals(DistCpUtils.packAttributes(attributes), "R"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("R")); attributes.add(FileAttribute.BLOCKSIZE); Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RB"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RB")); attributes.add(FileAttribute.USER); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBU"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBU")); + attributes.add(FileAttribute.CHECKSUMTYPE); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUC"); attributes.add(FileAttribute.GROUP); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUG"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUG")); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGC"); attributes.add(FileAttribute.PERMISSION); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGP"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUGP")); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPC"); + + attributes.add(FileAttribute.TIMES); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPCT"); + } + + public void testUnpackAttributes() { + EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RCBUGPAXT")); + + attributes.remove(FileAttribute.REPLICATION); + attributes.remove(FileAttribute.CHECKSUMTYPE); + attributes.remove(FileAttribute.ACL); + attributes.remove(FileAttribute.XATTR); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGPT")); + + attributes.remove(FileAttribute.TIMES); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGP")); + + attributes.remove(FileAttribute.BLOCKSIZE); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UGP")); + + attributes.remove(FileAttribute.GROUP); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UP")); + + attributes.remove(FileAttribute.USER); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("P")); + + attributes.remove(FileAttribute.PERMISSION); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("")); } @Test - public void testPreserve() { - try { - FileSystem fs = FileSystem.get(config); - EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); - - - Path path = new Path("/tmp/abc"); - Path src = new Path("/tmp/src"); - fs.mkdirs(path); - fs.mkdirs(src); - CopyListingFileStatus srcStatus = new CopyListingFileStatus( - fs.getFileStatus(src)); - - FsPermission noPerm = new FsPermission((short) 0); - fs.setPermission(path, noPerm); - fs.setOwner(path, "nobody", "nobody"); - - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - FileStatus target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), noPerm); - Assert.assertEquals(target.getOwner(), "nobody"); - Assert.assertEquals(target.getGroup(), "nobody"); - - attributes.add(FileAttribute.PERMISSION); - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); - Assert.assertEquals(target.getOwner(), "nobody"); - Assert.assertEquals(target.getGroup(), "nobody"); - - attributes.add(FileAttribute.GROUP); - attributes.add(FileAttribute.USER); - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); - Assert.assertEquals(target.getOwner(), srcStatus.getOwner()); - Assert.assertEquals(target.getGroup(), srcStatus.getGroup()); - - fs.delete(path, true); - fs.delete(src, true); - } catch (IOException e) { - LOG.error("Exception encountered ", e); - Assert.fail("Preserve test failure"); - } + public void testPreserveDefaults() throws IOException { + FileSystem fs = FileSystem.get(config); + + // preserve replication, block size, user, group, permission, + // checksum type and timestamps + EnumSet<FileAttribute> attributes = + DistCpUtils.unpackAttributes( + DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.substring(1)); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveNothingOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(dstStatus.getAccessTime() == 100); + Assert.assertTrue(dstStatus.getModificationTime() == 100); + Assert.assertTrue(dstStatus.getReplication() == 0); + } + + @Test + public void testPreservePermissionOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveGroupOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveUserOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveReplicationOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + // Replication shouldn't apply to dirs so this should still be 0 == 0 + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveTimestampOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + } + + @Test + public void testPreserveNothingOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreservePermissionOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveGroupOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveUserOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveReplicationOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveTimestampOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveOnFileUpwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, f2, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> f2 ? should be yes + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertTrue(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(d2Status.getAccessTime() == 300); + Assert.assertTrue(d2Status.getModificationTime() == 300); + Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertTrue(d1Status.getAccessTime() == 400); + Assert.assertTrue(d1Status.getModificationTime() == 400); + Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); + } + + @Test + public void testPreserveOnDirectoryUpwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); + + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, d2, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> d2 ? should be yes + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertTrue(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == d2Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == d2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + } + + @Test + public void testPreserveOnFileDownwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, f0, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> f0 ? should be yes + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertTrue(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertTrue(d1Status.getAccessTime() == 400); + Assert.assertTrue(d1Status.getModificationTime() == 400); + Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(d2Status.getAccessTime() == 300); + Assert.assertTrue(d2Status.getModificationTime() == 300); + Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); + } + + @Test + public void testPreserveOnDirectoryDownwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + Path root = new Path("/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(root, fullPerm); + fs.setOwner(root, "anybody", "anybody-group"); + fs.setTimes(root, 400, 400); + fs.setReplication(root, (short) 3); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, root, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> root ? should be yes + CopyListingFileStatus rootStatus = new CopyListingFileStatus(fs.getFileStatus(root)); + Assert.assertTrue(srcStatus.getPermission().equals(rootStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(rootStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(rootStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == rootStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == rootStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != rootStatus.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); } private static Random rand = new Random(); @@ -168,10 +1023,10 @@ public class TestDistCpUtils { fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm); fs.setPermission(new Path(base + "/newTest/hello/world3"), perm); fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm); - createFile(fs, base + "/newTest/1"); - createFile(fs, base + "/newTest/hello/2"); - createFile(fs, base + "/newTest/hello/world3/oldworld/3"); - createFile(fs, base + "/newTest/hello/world2/4"); + createFile(fs, new Path(base, "/newTest/1")); + createFile(fs, new Path(base, "/newTest/hello/2")); + createFile(fs, new Path(base, "/newTest/hello/world3/oldworld/3")); + createFile(fs, new Path(base, "/newTest/hello/world2/4")); return base; } @@ -191,12 +1046,27 @@ public class TestDistCpUtils { LOG.warn("Exception encountered ", e); } } - + public static void createFile(FileSystem fs, String filePath) throws IOException { - OutputStream out = fs.create(new Path(filePath)); + Path path = new Path(filePath); + createFile(fs, path); + } + + /** Creates a new, empty file at filePath and always overwrites */ + public static void createFile(FileSystem fs, Path filePath) throws IOException { + OutputStream out = fs.create(filePath, true); IOUtils.closeStream(out); } + /** Creates a new, empty directory at dirPath and always overwrites */ + public static void createDirectory(FileSystem fs, Path dirPath) throws IOException { + fs.delete(dirPath, true); + boolean created = fs.mkdirs(dirPath); + if (!created) { + LOG.warn("Could not create directory " + dirPath + " this might cause test failures."); + } + } + public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase) throws IOException { Path base = new Path(targetBase);
