[18/50] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-21 Thread asuresh 
MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3d61304
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3d61304
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3d61304

Branch: refs/heads/YARN-6592
Commit: b3d61304f2fa4a99526f7a60ccaac9f262083079
Parents: 1ee2527
Author: Jason Lowe 
Authored: Mon Sep 18 17:04:43 2017 -0500
Committer: Jason Lowe 
Committed: Mon Sep 18 17:04:43 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d61304/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..06a3e42 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ").append(jobId);
+sb.append(" mappers: ").append(mapIds);
+sb.append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[27/50] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-21 Thread asuresh 
MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a20debd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a20debd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a20debd

Branch: refs/heads/YARN-6592
Commit: 3a20debddeac69596ceb5b36f8413529ea8570e6
Parents: ea845ba
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:13:17 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a20debd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..b7f2c6d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[26/50] [abbrv] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-20 Thread jhung 
MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a20debd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a20debd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a20debd

Branch: refs/heads/YARN-5734
Commit: 3a20debddeac69596ceb5b36f8413529ea8570e6
Parents: ea845ba
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:13:17 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a20debd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..b7f2c6d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[17/50] [abbrv] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-20 Thread jhung 
MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3d61304
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3d61304
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3d61304

Branch: refs/heads/YARN-5734
Commit: b3d61304f2fa4a99526f7a60ccaac9f262083079
Parents: 1ee2527
Author: Jason Lowe 
Authored: Mon Sep 18 17:04:43 2017 -0500
Committer: Jason Lowe 
Committed: Mon Sep 18 17:04:43 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d61304/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..06a3e42 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ").append(jobId);
+sb.append(" mappers: ").append(mapIds);
+sb.append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[10/18] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread xyao 
MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a20debd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a20debd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a20debd

Branch: refs/heads/HDFS-7240
Commit: 3a20debddeac69596ceb5b36f8413529ea8570e6
Parents: ea845ba
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:13:17 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a20debd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..b7f2c6d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[01/18] hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread xyao 
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 14e184a28 -> d77c8107f


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3d61304
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3d61304
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3d61304

Branch: refs/heads/HDFS-7240
Commit: b3d61304f2fa4a99526f7a60ccaac9f262083079
Parents: 1ee2527
Author: Jason Lowe 
Authored: Mon Sep 18 17:04:43 2017 -0500
Committer: Jason Lowe 
Committed: Mon Sep 18 17:04:43 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d61304/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..06a3e42 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ").append(jobId);
+sb.append(" mappers: ").append(mapIds);
+sb.append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 b1e1f7546 -> 03a380dae


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03a380da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03a380da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03a380da

Branch: refs/heads/branch-2.8
Commit: 03a380daec9e6ca4254bf10a2d8028f2b0854b84
Parents: b1e1f75
Author: Jason Lowe 
Authored: Tue Sep 19 10:27:04 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 10:27:04 2017 -0500

--
 .../apache/hadoop/mapred/ShuffleHandler.java| 30 +++-
 .../hadoop/mapred/TestShuffleHandler.java   | 10 +++
 2 files changed, 22 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a380da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 21afd8f..1ffa150 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -923,13 +923,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -973,8 +966,8 @@ public class ShuffleHandler extends AuxiliaryService {
   String outputBasePathStr = getBaseLocation(jobId, user);
 
   try {
-populateHeaders(mapIds, outputBasePathStr, user, reduceId, request,
-  response, keepAliveParam, mapOutputInfoMap);
+populateHeaders(mapIds, jobId, outputBasePathStr, user, reduceId,
+request, response, keepAliveParam, mapOutputInfoMap);
   } catch(IOException e) {
 ch.write(response);
 LOG.error("Shuffle error in populating headers :", e);
@@ -1080,10 +1073,10 @@ public class ShuffleHandler extends AuxiliaryService {
   return outputInfo;
 }
 
-protected void populateHeaders(List mapIds, String outputBaseStr,
-String user, int reduce, HttpRequest request, HttpResponse response,
-boolean keepAliveParam, Map mapOutputInfoMap)
-throws IOException {
+protected void populateHeaders(List mapIds, String jobId,
+String outputBaseStr, String user, int reduce, HttpRequest request,
+HttpResponse response, boolean keepAliveParam,
+Map mapOutputInfoMap) throws IOException {
 
   long contentLength = 0;
   for (String mapId : mapIds) {
@@ -1108,6 +1101,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a380da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1944b0b28 -> 41c2a73c5


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe

(cherry picked from commit 3a20debddeac69596ceb5b36f8413529ea8570e6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41c2a73c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41c2a73c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41c2a73c

Branch: refs/heads/branch-2
Commit: 41c2a73c52886a63f7716a85f65c0d6461eb76b6
Parents: 1944b0b
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:14:41 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/41c2a73c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index f5a1171..e670236 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -990,13 +990,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1181,6 +1174,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 db88b6179 -> 731a62262


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe

(cherry picked from commit 3a20debddeac69596ceb5b36f8413529ea8570e6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/731a6226
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/731a6226
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/731a6226

Branch: refs/heads/branch-3.0
Commit: 731a62262774139b88b8eae08b219f1adb1fdc3f
Parents: db88b61
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:14:12 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/731a6226/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..b7f2c6d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-19 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ea845ba58 -> 3a20debdd


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a20debd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a20debd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a20debd

Branch: refs/heads/trunk
Commit: 3a20debddeac69596ceb5b36f8413529ea8570e6
Parents: ea845ba
Author: Jason Lowe 
Authored: Tue Sep 19 09:13:17 2017 -0500
Committer: Jason Lowe 
Committed: Tue Sep 19 09:13:17 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a20debd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..b7f2c6d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ");
+sb.append(jobId).append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+sb.append(" mappers: ").append(mapIds);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-18 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 5aa2c6baa -> 8894654fd


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe

(cherry picked from commit b3d61304f2fa4a99526f7a60ccaac9f262083079)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8894654f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8894654f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8894654f

Branch: refs/heads/branch-3.0
Commit: 8894654fde96754d9b094fe7b329bae014cc7cf3
Parents: 5aa2c6b
Author: Jason Lowe 
Authored: Mon Sep 18 17:04:43 2017 -0500
Committer: Jason Lowe 
Committed: Mon Sep 18 17:06:02 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8894654f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..06a3e42 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ").append(jobId);
+sb.append(" mappers: ").append(mapIds);
+sb.append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. Contributed by Jason Lowe

2017-09-18 Thread jlowe 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 1ee25278c -> b3d61304f


MAPREDUCE-6958. Shuffle audit logger should log size of shuffle transfer. 
Contributed by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3d61304
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3d61304
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3d61304

Branch: refs/heads/trunk
Commit: b3d61304f2fa4a99526f7a60ccaac9f262083079
Parents: 1ee2527
Author: Jason Lowe 
Authored: Mon Sep 18 17:04:43 2017 -0500
Committer: Jason Lowe 
Committed: Mon Sep 18 17:04:43 2017 -0500

--
 .../org/apache/hadoop/mapred/ShuffleHandler.java  | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d61304/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 863da7e..06a3e42 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -992,13 +992,6 @@ public class ShuffleHandler extends AuxiliaryService {
 return;
   }
 
-  // this audit log is disabled by default,
-  // to turn it on please enable this audit log
-  // on log4j.properties by uncommenting the setting
-  if (AUDITLOG.isDebugEnabled()) {
-AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mappers: " + mapIds +
- " reducer " + reduceQ.get(0));
-  }
   int reduceId;
   String jobId;
   try {
@@ -1183,6 +1176,17 @@ public class ShuffleHandler extends AuxiliaryService {
 
   // Now set the response headers.
   setResponseHeaders(response, keepAliveParam, contentLength);
+
+  // this audit log is disabled by default,
+  // to turn it on please enable this audit log
+  // on log4j.properties by uncommenting the setting
+  if (AUDITLOG.isDebugEnabled()) {
+StringBuilder sb = new StringBuilder("shuffle for ").append(jobId);
+sb.append(" mappers: ").append(mapIds);
+sb.append(" reducer ").append(reduce);
+sb.append(" length ").append(contentLength);
+AUDITLOG.debug(sb.toString());
+  }
 }
 
 protected void setResponseHeaders(HttpResponse response,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org