This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 54ac801 Logging fileSize of log files under NM Local Dir. Contributed
by Prabhu Joseph
54ac801 is described below
commit 54ac80176e8487b7a18cd9e16a11efa289d0b7df
Author: Szilard Nemeth <snem...@apache.org>
AuthorDate: Fri Aug 2 13:38:06 2019 +0200
Logging fileSize of log files under NM Local Dir. Contributed by Prabhu
Joseph
---
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++
.../src/main/resources/yarn-default.xml | 8 ++++++++
.../logaggregation/AppLogAggregatorImpl.java | 18 +++++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 548d868..917d32b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1370,6 +1370,11 @@ public class YarnConfiguration extends Configuration {
public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
+ "log-aggregation.retain-seconds";
public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
+
+ public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX
+ + "log-aggregation.debug.filesize";
+ public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE
+ = 100 * 1024 * 1024;
/**
* How long to wait between aggregated log retention checks. If set to
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8582522..f379844 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1292,6 +1292,14 @@
</property>
<property>
+ <description>The log files created under NM Local Directories
+ will be logged if it exceeds the configured bytes. This
+ only takes effect if log4j level is at least Debug.</description>
+ <name>yarn.log-aggregation.debug.filesize</name>
+ <value>104857600</value>
+ </property>
+
+ <property>
<description>Specify which log file controllers we will support. The first
file controller we add will be used to write the aggregated logs.
This comma separated configuration will work with the configuration:
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index fdac2e4..ef14d2a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -109,7 +109,7 @@ public class AppLogAggregatorImpl implements
AppLogAggregator {
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
private int cleanupOldLogTimes = 0;
-
+ private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
private final Map<ContainerId, ContainerLogAggregator>
containerLogAggregators =
@@ -176,6 +176,9 @@ public class AppLogAggregatorImpl implements
AppLogAggregator {
this.nodeId = nodeId;
this.logAggPolicy = getLogAggPolicy(conf);
this.recoveredLogInitedTime = recoveredLogInitedTime;
+ this.logFileSizeThreshold =
+ conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE);
if (logAggregationFileController == null) {
// by default, use T-File Controller
this.logAggregationFileController = new LogAggregationTFileController();
@@ -330,6 +333,19 @@ public class AppLogAggregatorImpl implements
AppLogAggregator {
uploadedLogsInThisCycle = true;
List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
+ if (LOG.isDebugEnabled()) {
+ for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
+ try {
+ long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
+ if (fileSize >= logFileSizeThreshold) {
+ LOG.debug("Log File " + uploadedFilePath
+ + " size is " + fileSize + " bytes");
+ }
+ } catch (Exception e1) {
+ LOG.error("Failed to get log file size " + e1);
+ }
+ }
+ }
deletionTask = new FileDeletionTask(delService,
this.userUgi.getShortUserName(), null,
uploadedFilePathsInThisCycleList);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
