This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new be9ac8a Logging fileSize of log files under NM Local Dir. Contributed by Prabhu Joseph be9ac8a is described below commit be9ac8adf9d0e2327bc5d0871bd153bef9ec7694 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Fri Aug 2 13:38:06 2019 +0200 Logging fileSize of log files under NM Local Dir. Contributed by Prabhu Joseph (cherry picked from commit 54ac80176e8487b7a18cd9e16a11efa289d0b7df) --- .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++ .../src/main/resources/yarn-default.xml | 8 ++++++++ .../logaggregation/AppLogAggregatorImpl.java | 18 +++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9f3136d..abf6fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1322,6 +1322,11 @@ public class YarnConfiguration extends Configuration { public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX + + "log-aggregation.debug.filesize"; + public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE + = 100 * 1024 * 1024; /** * How long to wait between aggregated log retention checks. If set to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index ebeecdb..dfb6cbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1263,6 +1263,14 @@ </property> <property> + <description>The log files created under NM Local Directories + will be logged if it exceeds the configured bytes. This + only takes effect if log4j level is at least Debug.</description> + <name>yarn.log-aggregation.debug.filesize</name> + <value>104857600</value> + </property> + + <property> <description>Specify which log file controllers we will support. The first file controller we add will be used to write the aggregated logs. This comma separated configuration will work with the configuration: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index c290eb4..5dfe6d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -108,7 +108,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; private int cleanupOldLogTimes = 0; - + private long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; private final Map<ContainerId, ContainerLogAggregator> containerLogAggregators = @@ -175,6 +175,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { this.nodeId = nodeId; this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; + this.logFileSizeThreshold = + conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE); if (logAggregationFileController == null) { // by default, use T-File Controller this.logAggregationFileController = new LogAggregationTFileController(); @@ -327,6 +330,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator { uploadedLogsInThisCycle = true; List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + try { + long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if (fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + + " size is " + fileSize + " bytes"); + } + } catch (Exception e1) { + LOG.error("Failed to get log file size " + e1); + } + } + } DeletionTask deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, uploadedFilePathsInThisCycleList); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org