(hadoop) branch trunk updated: HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. (#6540)

[sodonnell]

This is an automated email from the ASF dual-hosted git repository.

sodonnell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 0bf439c0f9c3 HDFS-17376. Distcp creates Factor 1 replication file on 
target if Source is EC. (#6540)
0bf439c0f9c3 is described below

commit 0bf439c0f9c336a535ab8fbe97142a2016cc8168
Author: Sadanand Shenoy <sadanand.shenoy4...@gmail.com>
AuthorDate: Fri Feb 9 22:56:32 2024 +0530

    HDFS-17376. Distcp creates Factor 1 replication file on target if Source is 
EC. (#6540)
---
 .../tools/mapred/RetriableFileCopyCommand.java     |  3 +++
 .../hadoop/tools/TestDistCpWithRawXAttrs.java      | 24 +++++++++++++++++++---
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index ae6b734f4860..84bb00100863 100644
--- 
a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ 
b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -380,6 +380,9 @@ public class RetriableFileCopyCommand extends 
RetriableCommand {
   private static short getReplicationFactor(
           EnumSet<FileAttribute> fileAttributes, CopyListingFileStatus source,
           FileSystem targetFS, Path tmpTargetPath) {
+    if (source.isErasureCoded()) {
+      return targetFS.getDefaultReplication(tmpTargetPath);
+    }
     return fileAttributes.contains(FileAttribute.REPLICATION)
         ? source.getReplication()
         : targetFS.getDefaultReplication(tmpTargetPath);
diff --git 
a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
 
b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
index b16f6b233405..841501869b5e 100644
--- 
a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
+++ 
b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
@@ -62,6 +62,7 @@ public class TestDistCpWithRawXAttrs {
   private static final Path dir1 = new Path("/src/dir1");
   private static final Path subDir1 = new Path(dir1, "subdir1");
   private static final Path file1 = new Path("/src/file1");
+  private static final Path FILE_2 = new Path("/src/dir1/file2");
   private static final String rawRootName = "/.reserved/raw";
   private static final String rootedDestName = "/dest";
   private static final String rootedSrcName = "/src";
@@ -73,7 +74,7 @@ public class TestDistCpWithRawXAttrs {
     conf = new Configuration();
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
     conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true)
             .build();
     cluster.waitActive();
     fs = cluster.getFileSystem();
@@ -178,7 +179,7 @@ public class TestDistCpWithRawXAttrs {
   }
 
   @Test
-  public void testPreserveEC() throws Exception {
+  public void testPreserveAndNoPreserveEC() throws Exception {
     final String src = "/src";
     final String dest = "/dest";
 
@@ -190,9 +191,11 @@ public class TestDistCpWithRawXAttrs {
 
     fs.delete(new Path("/dest"), true);
     fs.mkdirs(subDir1);
-    fs.create(file1).close();
     DistributedFileSystem dfs = (DistributedFileSystem) fs;
     dfs.enableErasureCodingPolicy("XOR-2-1-1024k");
+    dfs.setErasureCodingPolicy(dir1, "XOR-2-1-1024k");
+    fs.create(file1).close();
+    fs.create(FILE_2).close();
     int res = ToolRunner.run(conf, new ECAdmin(conf), args);
     assertEquals("Unable to set EC policy on " + subDir1.toString(), res, 0);
 
@@ -203,6 +206,7 @@ public class TestDistCpWithRawXAttrs {
     FileStatus srcStatus = fs.getFileStatus(new Path(src));
     FileStatus srcDir1Status = fs.getFileStatus(dir1);
     FileStatus srcSubDir1Status = fs.getFileStatus(subDir1);
+    FileStatus srcFile2Status = fs.getFileStatus(FILE_2);
 
     FileStatus destStatus = fs.getFileStatus(new Path(dest));
     FileStatus destDir1Status = fs.getFileStatus(destDir1);
@@ -214,12 +218,26 @@ public class TestDistCpWithRawXAttrs {
         destStatus.isErasureCoded());
     assertTrue("/src/dir1 is not erasure coded!",
         srcDir1Status.isErasureCoded());
+    assertTrue("/src/dir1/file2 is not erasure coded",
+        srcFile2Status.isErasureCoded());
     assertTrue("/dest/dir1 is not erasure coded!",
         destDir1Status.isErasureCoded());
     assertTrue("/src/dir1/subdir1 is not erasure coded!",
         srcSubDir1Status.isErasureCoded());
     assertTrue("/dest/dir1/subdir1 is not erasure coded!",
         destSubDir1Status.isErasureCoded());
+
+    // test without -p to check if src is EC then target FS default replication
+    // is obeyed on the target file.
+
+    fs.delete(new Path(dest), true);
+    DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, src, dest, null,
+        conf);
+    FileStatus destFileStatus = fs.getFileStatus(new Path(destDir1, "file2"));
+    assertFalse(destFileStatus.isErasureCoded());
+    assertEquals(fs.getDefaultReplication(new Path(destDir1, "file2")),
+        destFileStatus.getReplication());
+    dfs.unsetErasureCodingPolicy(dir1);
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org