[2/2] hadoop git commit: MAPREDUCE-6954. Disable erasure coding for files that are uploaded to the MR staging area (pbacsko via rkanter)

2017-09-18 Thread rkanter 
MAPREDUCE-6954. Disable erasure coding for files that are uploaded to the MR 
staging area (pbacsko via rkanter)

(cherry picked from commit 0adc0471d0c06f66a31060f270dcb50a7b4ffafa)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/917e875e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/917e875e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/917e875e

Branch: refs/heads/branch-3.0
Commit: 917e875e511589c297971f6a14fd85de7925409f
Parents: 3847b8c
Author: Robert Kanter 
Authored: Mon Sep 18 10:40:06 2017 -0700
Committer: Robert Kanter 
Committed: Mon Sep 18 10:42:01 2017 -0700

--
 .../hadoop-mapreduce-client-core/pom.xml|  4 ++
 .../hadoop/mapreduce/JobResourceUploader.java   | 17 
 .../apache/hadoop/mapreduce/MRJobConfig.java|  5 +++
 .../src/main/resources/mapred-default.xml   |  9 
 .../mapreduce/TestJobResourceUploader.java  | 46 
 5 files changed, 81 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/917e875e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index a23827d..ff88bcc 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -44,6 +44,10 @@
 
 
   org.apache.hadoop
+  hadoop-hdfs-client
+
+
+  org.apache.hadoop
   hadoop-hdfs
   test
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/917e875e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index f1cad57..d9bf988 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 
@@ -94,6 +96,11 @@ class JobResourceUploader {
 new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
 mkdirs(jtFs, submitJobDir, mapredSysPerms);
 
+if (!conf.getBoolean(MRJobConfig.MR_AM_STAGING_DIR_ERASURECODING_ENABLED,
+MRJobConfig.DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED)) {
+  disableErasureCodingForPath(jtFs, submitJobDir);
+}
+
 Collection files = conf.getStringCollection("tmpfiles");
 Collection libjars = conf.getStringCollection("tmpjars");
 Collection archives = conf.getStringCollection("tmparchives");
@@ -575,4 +582,14 @@ class JobResourceUploader {
 }
 return finalPath;
   }
+
+  private void disableErasureCodingForPath(FileSystem fs, Path path)
+  throws IOException {
+if (jtFs instanceof DistributedFileSystem) {
+  LOG.info("Disabling Erasure Coding for path: " + path);
+  DistributedFileSystem dfs = (DistributedFileSystem) jtFs;
+  dfs.setErasureCodingPolicy(path,
+  SystemErasureCodingPolicies.getReplicationPolicy().getName());
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/917e875e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index

[2/2] hadoop git commit: MAPREDUCE-6954. Disable erasure coding for files that are uploaded to the MR staging area (pbacsko via rkanter)

2017-09-18 Thread rkanter 
MAPREDUCE-6954. Disable erasure coding for files that are uploaded to the MR 
staging area (pbacsko via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0adc0471
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0adc0471
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0adc0471

Branch: refs/heads/trunk
Commit: 0adc0471d0c06f66a31060f270dcb50a7b4ffafa
Parents: 5f49668
Author: Robert Kanter 
Authored: Mon Sep 18 10:40:06 2017 -0700
Committer: Robert Kanter 
Committed: Mon Sep 18 10:40:06 2017 -0700

--
 .../hadoop-mapreduce-client-core/pom.xml|  4 ++
 .../hadoop/mapreduce/JobResourceUploader.java   | 17 
 .../apache/hadoop/mapreduce/MRJobConfig.java|  5 +++
 .../src/main/resources/mapred-default.xml   |  9 
 .../mapreduce/TestJobResourceUploader.java  | 46 
 5 files changed, 81 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adc0471/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index c34f7bd..ce5fdc8 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -44,6 +44,10 @@
 
 
   org.apache.hadoop
+  hadoop-hdfs-client
+
+
+  org.apache.hadoop
   hadoop-hdfs
   test
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adc0471/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index f1cad57..d9bf988 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 
@@ -94,6 +96,11 @@ class JobResourceUploader {
 new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
 mkdirs(jtFs, submitJobDir, mapredSysPerms);
 
+if (!conf.getBoolean(MRJobConfig.MR_AM_STAGING_DIR_ERASURECODING_ENABLED,
+MRJobConfig.DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED)) {
+  disableErasureCodingForPath(jtFs, submitJobDir);
+}
+
 Collection files = conf.getStringCollection("tmpfiles");
 Collection libjars = conf.getStringCollection("tmpjars");
 Collection archives = conf.getStringCollection("tmparchives");
@@ -575,4 +582,14 @@ class JobResourceUploader {
 }
 return finalPath;
   }
+
+  private void disableErasureCodingForPath(FileSystem fs, Path path)
+  throws IOException {
+if (jtFs instanceof DistributedFileSystem) {
+  LOG.info("Disabling Erasure Coding for path: " + path);
+  DistributedFileSystem dfs = (DistributedFileSystem) jtFs;
+  dfs.setErasureCodingPolicy(path,
+  SystemErasureCodingPolicies.getReplicationPolicy().getName());
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adc0471/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 2023ba3..86abb42 100644
---