[10/40] hadoop git commit: YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong.

2017-03-20 Thread liuml07 
YARN-6313. YARN logs cli should provide logs for a completed container even 
when application is still running. Contributed by Xuan Gong.

(cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/871dc420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/871dc420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/871dc420

Branch: refs/heads/HADOOP-13345
Commit: 871dc420f8a4f151189c0925e062c64859a8f275
Parents: 0a3aa40
Author: Junping Du 
Authored: Tue Mar 14 12:56:54 2017 -0700
Committer: Junping Du 
Committed: Tue Mar 14 12:58:12 2017 -0700

--
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  | 172 +--
 .../hadoop/yarn/client/cli/TestLogsCLI.java |  31 
 .../yarn/logaggregation/LogCLIHelpers.java  |  11 +-
 3 files changed, 160 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 3cb1c7d..8407b19 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -44,6 +44,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -65,6 +66,7 @@ import 
org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
 import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool {
 return false;
   }
 
-  private List getContainerLogFiles(
+  private List> getContainerLogFiles(
   Configuration conf, String containerIdStr, String nodeHttpAddress)
   throws IOException {
-List logFileInfos = new ArrayList<>();
+List> logFileInfos
+= new ArrayList<>();
 Client webServiceClient = Client.create();
 try {
   WebResource webResource = webServiceClient
@@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool {
   }
   for (int i = 0; i < array.length(); i++) {
 JSONObject log = array.getJSONObject(i);
+String aggregateType = log.has("logAggregationType") ?
+log.getString("logAggregationType") : "N/A";
 Object ob = log.get("containerLogInfo");
 if (ob instanceof JSONArray) {
   JSONArray obArray = (JSONArray)ob;
   for (int j = 0; j < obArray.length(); j++) {
-logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-obArray.getJSONObject(j)));
+logFileInfos.add(new Pair(
+generatePerContainerLogFileInfoFromJSON(
+obArray.getJSONObject(j)), aggregateType));
   }
 } else if (ob instanceof JSONObject) {
-  logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-  (JSONObject)ob));
+  logFileInfos.add(new Pair(
+  generatePerContainerLogFileInfoFromJSON(
+  (JSONObject)ob), aggregateType));
 }
   }
 } catch (Exception e) {
@@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool {
   IOUtils.closeQuietly(is);
 }
   }
-  // for the case, we have already uploaded partial logs in HDFS
-  int result = logCliHelper.dumpAContainerLogsForLogType(
-  newOptions, false);
-  if (result == 0

[31/50] [abbrv] hadoop git commit: YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong.

2017-03-16 Thread inigoiri 
YARN-6313. YARN logs cli should provide logs for a completed container even 
when application is still running. Contributed by Xuan Gong.

(cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/871dc420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/871dc420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/871dc420

Branch: refs/heads/HDFS-10467
Commit: 871dc420f8a4f151189c0925e062c64859a8f275
Parents: 0a3aa40
Author: Junping Du 
Authored: Tue Mar 14 12:56:54 2017 -0700
Committer: Junping Du 
Committed: Tue Mar 14 12:58:12 2017 -0700

--
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  | 172 +--
 .../hadoop/yarn/client/cli/TestLogsCLI.java |  31 
 .../yarn/logaggregation/LogCLIHelpers.java  |  11 +-
 3 files changed, 160 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 3cb1c7d..8407b19 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -44,6 +44,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -65,6 +66,7 @@ import 
org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
 import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool {
 return false;
   }
 
-  private List getContainerLogFiles(
+  private List> getContainerLogFiles(
   Configuration conf, String containerIdStr, String nodeHttpAddress)
   throws IOException {
-List logFileInfos = new ArrayList<>();
+List> logFileInfos
+= new ArrayList<>();
 Client webServiceClient = Client.create();
 try {
   WebResource webResource = webServiceClient
@@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool {
   }
   for (int i = 0; i < array.length(); i++) {
 JSONObject log = array.getJSONObject(i);
+String aggregateType = log.has("logAggregationType") ?
+log.getString("logAggregationType") : "N/A";
 Object ob = log.get("containerLogInfo");
 if (ob instanceof JSONArray) {
   JSONArray obArray = (JSONArray)ob;
   for (int j = 0; j < obArray.length(); j++) {
-logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-obArray.getJSONObject(j)));
+logFileInfos.add(new Pair(
+generatePerContainerLogFileInfoFromJSON(
+obArray.getJSONObject(j)), aggregateType));
   }
 } else if (ob instanceof JSONObject) {
-  logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-  (JSONObject)ob));
+  logFileInfos.add(new Pair(
+  generatePerContainerLogFileInfoFromJSON(
+  (JSONObject)ob), aggregateType));
 }
   }
 } catch (Exception e) {
@@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool {
   IOUtils.closeQuietly(is);
 }
   }
-  // for the case, we have already uploaded partial logs in HDFS
-  int result = logCliHelper.dumpAContainerLogsForLogType(
-  newOptions, false);
-  if (result == 0

[27/33] hadoop git commit: YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong.

2017-03-14 Thread jhung 
YARN-6313. YARN logs cli should provide logs for a completed container even 
when application is still running. Contributed by Xuan Gong.

(cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/871dc420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/871dc420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/871dc420

Branch: refs/heads/YARN-5734
Commit: 871dc420f8a4f151189c0925e062c64859a8f275
Parents: 0a3aa40
Author: Junping Du 
Authored: Tue Mar 14 12:56:54 2017 -0700
Committer: Junping Du 
Committed: Tue Mar 14 12:58:12 2017 -0700

--
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  | 172 +--
 .../hadoop/yarn/client/cli/TestLogsCLI.java |  31 
 .../yarn/logaggregation/LogCLIHelpers.java  |  11 +-
 3 files changed, 160 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 3cb1c7d..8407b19 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -44,6 +44,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -65,6 +66,7 @@ import 
org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
 import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool {
 return false;
   }
 
-  private List getContainerLogFiles(
+  private List> getContainerLogFiles(
   Configuration conf, String containerIdStr, String nodeHttpAddress)
   throws IOException {
-List logFileInfos = new ArrayList<>();
+List> logFileInfos
+= new ArrayList<>();
 Client webServiceClient = Client.create();
 try {
   WebResource webResource = webServiceClient
@@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool {
   }
   for (int i = 0; i < array.length(); i++) {
 JSONObject log = array.getJSONObject(i);
+String aggregateType = log.has("logAggregationType") ?
+log.getString("logAggregationType") : "N/A";
 Object ob = log.get("containerLogInfo");
 if (ob instanceof JSONArray) {
   JSONArray obArray = (JSONArray)ob;
   for (int j = 0; j < obArray.length(); j++) {
-logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-obArray.getJSONObject(j)));
+logFileInfos.add(new Pair(
+generatePerContainerLogFileInfoFromJSON(
+obArray.getJSONObject(j)), aggregateType));
   }
 } else if (ob instanceof JSONObject) {
-  logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-  (JSONObject)ob));
+  logFileInfos.add(new Pair(
+  generatePerContainerLogFileInfoFromJSON(
+  (JSONObject)ob), aggregateType));
 }
   }
 } catch (Exception e) {
@@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool {
   IOUtils.closeQuietly(is);
 }
   }
-  // for the case, we have already uploaded partial logs in HDFS
-  int result = logCliHelper.dumpAContainerLogsForLogType(
-  newOptions, false);
-  if (result == 0 ||

hadoop git commit: YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong.

2017-03-14 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f254002f1 -> 0e7879052


YARN-6313. YARN logs cli should provide logs for a completed container even 
when application is still running. Contributed by Xuan Gong.

(cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e787905
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e787905
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e787905

Branch: refs/heads/branch-2
Commit: 0e7879052ac76d5efd9e716a521b3bba6319010b
Parents: f254002
Author: Junping Du 
Authored: Tue Mar 14 12:56:54 2017 -0700
Committer: Junping Du 
Committed: Tue Mar 14 12:58:41 2017 -0700

--
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  | 172 +--
 .../hadoop/yarn/client/cli/TestLogsCLI.java |  31 
 .../yarn/logaggregation/LogCLIHelpers.java  |  11 +-
 3 files changed, 160 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e787905/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index e04143f..de0b64b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -44,6 +44,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -65,6 +66,7 @@ import 
org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
 import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool {
 return false;
   }
 
-  private List getContainerLogFiles(
+  private List> getContainerLogFiles(
   Configuration conf, String containerIdStr, String nodeHttpAddress)
   throws IOException {
-List logFileInfos = new ArrayList<>();
+List> logFileInfos
+= new ArrayList<>();
 Client webServiceClient = Client.create();
 try {
   WebResource webResource = webServiceClient
@@ -439,16 +442,20 @@ public class LogsCLI extends Configured implements Tool {
   }
   for (int i = 0; i < array.length(); i++) {
 JSONObject log = array.getJSONObject(i);
+String aggregateType = log.has("logAggregationType") ?
+log.getString("logAggregationType") : "N/A";
 Object ob = log.get("containerLogInfo");
 if (ob instanceof JSONArray) {
   JSONArray obArray = (JSONArray)ob;
   for (int j = 0; j < obArray.length(); j++) {
-logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-obArray.getJSONObject(j)));
+logFileInfos.add(new Pair(
+generatePerContainerLogFileInfoFromJSON(
+obArray.getJSONObject(j)), aggregateType));
   }
 } else if (ob instanceof JSONObject) {
-  logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-  (JSONObject)ob));
+  logFileInfos.add(new Pair(
+  generatePerContainerLogFileInfoFromJSON(
+  (JSONObject)ob), aggregateType));
 }
   }
 } catch (Exception e) {
@@ -543,10 +550,8 @@ public class LogsCLI extends Configured implements Tool {
   IOUtils.closeQuietly(is);
 }
   }
-  // for the case, we have already uploaded partial logs in HDFS
-  int result =

hadoop git commit: YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong.

2017-03-14 Thread junping_du 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 0a3aa40fe -> 871dc420f


YARN-6313. YARN logs cli should provide logs for a completed container even 
when application is still running. Contributed by Xuan Gong.

(cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/871dc420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/871dc420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/871dc420

Branch: refs/heads/trunk
Commit: 871dc420f8a4f151189c0925e062c64859a8f275
Parents: 0a3aa40
Author: Junping Du 
Authored: Tue Mar 14 12:56:54 2017 -0700
Committer: Junping Du 
Committed: Tue Mar 14 12:58:12 2017 -0700

--
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  | 172 +--
 .../hadoop/yarn/client/cli/TestLogsCLI.java |  31 
 .../yarn/logaggregation/LogCLIHelpers.java  |  11 +-
 3 files changed, 160 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/871dc420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 3cb1c7d..8407b19 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -44,6 +44,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -65,6 +66,7 @@ import 
org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
 import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool {
 return false;
   }
 
-  private List getContainerLogFiles(
+  private List> getContainerLogFiles(
   Configuration conf, String containerIdStr, String nodeHttpAddress)
   throws IOException {
-List logFileInfos = new ArrayList<>();
+List> logFileInfos
+= new ArrayList<>();
 Client webServiceClient = Client.create();
 try {
   WebResource webResource = webServiceClient
@@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool {
   }
   for (int i = 0; i < array.length(); i++) {
 JSONObject log = array.getJSONObject(i);
+String aggregateType = log.has("logAggregationType") ?
+log.getString("logAggregationType") : "N/A";
 Object ob = log.get("containerLogInfo");
 if (ob instanceof JSONArray) {
   JSONArray obArray = (JSONArray)ob;
   for (int j = 0; j < obArray.length(); j++) {
-logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-obArray.getJSONObject(j)));
+logFileInfos.add(new Pair(
+generatePerContainerLogFileInfoFromJSON(
+obArray.getJSONObject(j)), aggregateType));
   }
 } else if (ob instanceof JSONObject) {
-  logFileInfos.add(generatePerContainerLogFileInfoFromJSON(
-  (JSONObject)ob));
+  logFileInfos.add(new Pair(
+  generatePerContainerLogFileInfoFromJSON(
+  (JSONObject)ob), aggregateType));
 }
   }
 } catch (Exception e) {
@@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool {
   IOUtils.closeQuietly(is);
 }
   }
-  // for the case, we have already uploaded partial logs in HDFS
-  int result =