Repository: hadoop Updated Branches: refs/heads/branch-2 c3dfc7119 -> e51ae6476
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong. (cherry picked from commit 4aed2d8e91c7dccc78fbaffc409d3076c3316289) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e51ae647 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e51ae647 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e51ae647 Branch: refs/heads/branch-2 Commit: e51ae6476181c0675546854da31966c7d021c48d Parents: c3dfc71 Author: Zhijie Shen <[email protected]> Authored: Mon Oct 13 10:53:37 2014 -0700 Committer: Zhijie Shen <[email protected]> Committed: Mon Oct 13 10:55:09 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/api/records/LogAggregationContext.java | 28 +------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 7 +++++ .../src/main/proto/yarn_protos.proto | 1 - .../impl/pb/LogAggregationContextPBImpl.java | 15 ----------- .../src/main/resources/yarn-default.xml | 10 +++++++ .../logaggregation/AppLogAggregatorImpl.java | 8 +++--- .../TestContainerManagerRecovery.java | 5 +--- .../TestLogAggregationService.java | 4 ++- .../capacity/TestContainerAllocation.java | 4 +-- 10 files changed, 31 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2cf5a93..81de1d3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -600,6 +600,9 @@ Release 2.6.0 - UNRELEASED YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry (Yi Liu via jlowe) + YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong + via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java index 9a0a157..46c1809 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java @@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records; * which match the defined exclude pattern and those log files * will not be uploaded. If the log file name matches both the * include and the exclude pattern, this file will be excluded eventually</li> - * <li>rollingIntervalSeconds. The default value is -1. By default, - * the logAggregationService only uploads container logs when - * the application is finished. This configure defines - * how often the logAggregationSerivce uploads container logs in seconds. - * By setting this configure, the logAggregationSerivce can upload container - * logs periodically when the application is running. - * </li> * </ul> * </p> * @@ -57,11 +50,10 @@ public abstract class LogAggregationContext { @Public @Unstable public static LogAggregationContext newInstance(String includePattern, - String excludePattern, long rollingIntervalSeconds) { + String excludePattern) { LogAggregationContext context = Records.newRecord(LogAggregationContext.class); context.setIncludePattern(includePattern); context.setExcludePattern(excludePattern); - context.setRollingIntervalSeconds(rollingIntervalSeconds); return context; } @@ -100,22 +92,4 @@ public abstract class LogAggregationContext { @Public @Unstable public abstract void setExcludePattern(String excludePattern); - - /** - * Get rollingIntervalSeconds - * - * @return the rollingIntervalSeconds - */ - @Public - @Unstable - public abstract long getRollingIntervalSeconds(); - - /** - * Set rollingIntervalSeconds - * - * @param rollingIntervalSeconds - */ - @Public - @Unstable - public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e2fd8dd..dd7fa89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -712,6 +712,13 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; /** + * Define how often NMs wake up and upload log files + */ + public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = + NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; + public static final long + DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; + /** * Number of threads used in log cleanup. Only applicable if Log aggregation * is disabled */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 8db451d..136192d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto { message LogAggregationContextProto { optional string include_pattern = 1 [default = ".*"]; optional string exclude_pattern = 2 [default = ""]; - optional int64 rolling_interval_seconds = 3 [default = -1]; } enum ApplicationAccessTypeProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java index 4406ef9..dc7a21d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java @@ -116,19 +116,4 @@ public class LogAggregationContextPBImpl extends LogAggregationContext{ } builder.setExcludePattern(excludePattern); } - - @Override - public long getRollingIntervalSeconds() { - LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder; - if (! p.hasRollingIntervalSeconds()) { - return -1; - } - return p.getRollingIntervalSeconds(); - } - - @Override - public void setRollingIntervalSeconds(long rollingIntervalSeconds) { - maybeInitBuilder(); - builder.setRollingIntervalSeconds(rollingIntervalSeconds); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1a66a6f..13059f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1522,4 +1522,14 @@ <value>Client</value> </property> + <property> + <description>Defines how often NMs wake up to upload log files. + The default value is -1. By default, the logs will be uploaded when + the application is finished. By setting this configure, logs can be uploaded + periodically when the application is running. The minimum rolling-interval-seconds + can be set is 3600. + </description> + <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name> + <value>-1</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 63f7c66..98e3caf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -148,9 +148,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator { } else { this.retentionSize = configuredRentionSize; } - long configuredRollingMonitorInterval = - this.logAggregationContext == null ? -1 : this.logAggregationContext - .getRollingIntervalSeconds(); + long configuredRollingMonitorInterval = conf.getLong( + YarnConfiguration + .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); boolean debug_mode = conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index 2c69843..7850a1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -130,8 +130,7 @@ public class TestContainerManagerRecovery { containerTokens, acls); // create the logAggregationContext LogAggregationContext logAggregationContext = - LogAggregationContext.newInstance("includePattern", "excludePattern", - 1000); + LogAggregationContext.newInstance("includePattern", "excludePattern"); StartContainersResponse startResponse = startContainer(context, cm, cid, clc, logAggregationContext); assertTrue(startResponse.getFailedRequests().isEmpty()); @@ -168,8 +167,6 @@ public class TestContainerManagerRecovery { LogAggregationContext recovered = ((ApplicationImpl) app).getLogAggregationContext(); assertNotNull(recovered); - assertEquals(logAggregationContext.getRollingIntervalSeconds(), - recovered.getRollingIntervalSeconds()); assertEquals(logAggregationContext.getIncludePattern(), recovered.getIncludePattern()); assertEquals(logAggregationContext.getExcludePattern(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 2c0f349..ab86a18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -1235,10 +1235,12 @@ public class TestLogAggregationService extends BaseContainerManagerTest { throws Exception { LogAggregationContext logAggregationContextWithInterval = Records.newRecord(LogAggregationContext.class); - logAggregationContextWithInterval.setRollingIntervalSeconds(5000); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath()); + this.conf.setLong( + YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, + 3600); if (retentionSizeLimitation) { // set the retention size as 1. The number of logs for one application // in one NM should be 1. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 85ef381..365c754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -212,16 +212,14 @@ public class TestContainerAllocation { .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null)); // create a not-null LogAggregationContext - final int interval = 2000; LogAggregationContext logAggregationContext = LogAggregationContext.newInstance( - "includePattern", "excludePattern", interval); + "includePattern", "excludePattern"); LogAggregationContext returned = getLogAggregationContextFromContainerToken(rm1, nm2, logAggregationContext); Assert.assertEquals("includePattern", returned.getIncludePattern()); Assert.assertEquals("excludePattern", returned.getExcludePattern()); - Assert.assertEquals(interval, returned.getRollingIntervalSeconds()); rm1.stop(); }
