Repository: apex-core Updated Branches: refs/heads/master 5e23fb44c -> f9c1701a2
APEXCORE-744 Add setting of predefined static logger appender properties Added setting of the static logger appender properties: application name, container id, user name, service name, node name. Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/f9c1701a Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/f9c1701a Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/f9c1701a Branch: refs/heads/master Commit: f9c1701a2f22bf7405496c3ee4d6a71185762070 Parents: 5e23fb4 Author: Sergey Golovko <[email protected]> Authored: Wed Jun 7 11:05:22 2017 -0700 Committer: Sergey Golovko <[email protected]> Committed: Fri Jun 23 21:18:47 2017 -0700 ---------------------------------------------------------------------- .../datatorrent/stram/StreamingAppMaster.java | 2 ++ .../java/com/datatorrent/stram/cli/ApexCli.java | 2 +- .../stram/engine/StreamingContainer.java | 1 + .../com/datatorrent/stram/util/LoggerUtil.java | 37 ++++++++++++++++++++ .../datatorrent/stram/util/LoggerUtilTest.java | 32 +++++++++++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-core/blob/f9c1701a/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java b/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java index 7598b4f..de8ffa7 100644 --- a/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java +++ b/engine/src/main/java/com/datatorrent/stram/StreamingAppMaster.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import com.datatorrent.stram.debug.StdOutErrLog; +import com.datatorrent.stram.util.LoggerUtil; import com.datatorrent.stram.util.VersionInfo; /** @@ -56,6 +57,7 @@ public class StreamingAppMaster extends StramUtils.YarnContainerMain */ public static void main(final String[] args) throws Throwable { + LoggerUtil.setupMDC("master"); StdOutErrLog.tieSystemOutAndErrToLog(); LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path")); http://git-wip-us.apache.org/repos/asf/apex-core/blob/f9c1701a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java index 787f20b..152e6f1 100644 --- a/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java +++ b/engine/src/main/java/com/datatorrent/stram/cli/ApexCli.java @@ -4129,7 +4129,7 @@ public class ApexCli public static void main(final String[] args) throws Exception { - LoggerUtil.addAppenders(); + LoggerUtil.setupMDC("client"); final ApexCli shell = new ApexCli(); shell.preImpersonationInit(args); String hadoopUserName = System.getenv("HADOOP_USER_NAME"); http://git-wip-us.apache.org/repos/asf/apex-core/blob/f9c1701a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java index dd215a9..f5aaf35 100644 --- a/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java +++ b/engine/src/main/java/com/datatorrent/stram/engine/StreamingContainer.java @@ -286,6 +286,7 @@ public class StreamingContainer extends YarnContainerMain */ public static void main(String[] args) throws Throwable { + LoggerUtil.setupMDC("worker"); StdOutErrLog.tieSystemOutAndErrToLog(); logger.debug("PID: " + System.getenv().get("JVM_PID")); logger.info("Child starting with classpath: {}", System.getProperty("java.class.path")); http://git-wip-us.apache.org/repos/asf/apex-core/blob/f9c1701a/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java b/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java index 14662e1..c862634 100644 --- a/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java +++ b/engine/src/main/java/com/datatorrent/stram/util/LoggerUtil.java @@ -35,12 +35,17 @@ import javax.annotation.Nullable; import org.apache.apex.log.LogFileInformation; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Appender; import org.apache.log4j.Category; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.apache.log4j.MDC; import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.spi.DefaultRepositorySelector; import org.apache.log4j.spi.HierarchyEventListener; @@ -52,6 +57,9 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.datatorrent.stram.client.StramClientUtils; + +import static com.datatorrent.api.Context.DAGContext.APPLICATION_NAME; import static com.datatorrent.api.Context.DAGContext.LOGGER_APPENDER; /** @@ -464,4 +472,33 @@ public class LoggerUtil } return names; } + + /** + * Makes MDC properties + */ + public static void setupMDC(String service) + { + MDC.put("apex.service", service); + + String value = StramClientUtils.getHostName(); + MDC.put("apex.node", value == null ? "unknown" : value); + + value = System.getenv(Environment.USER.key()); + if (value != null) { + MDC.put("apex.user", value); + } + + value = System.getenv(Environment.CONTAINER_ID.name()); + if (value != null) { + ContainerId containerId = ConverterUtils.toContainerId(value); + ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); + MDC.put("apex.containerId", containerId.toString()); + MDC.put("apex.applicationId", applicationId.toString()); + } + + value = System.getProperty(APPLICATION_NAME.getLongName()); + if (value != null) { + MDC.put("apex.application", value); + } + } } http://git-wip-us.apache.org/repos/asf/apex-core/blob/f9c1701a/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java b/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java index bc73ca8..88aec3b 100644 --- a/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java +++ b/engine/src/test/java/com/datatorrent/stram/util/LoggerUtilTest.java @@ -36,7 +36,9 @@ import org.apache.log4j.spi.LoggingEvent; import com.google.common.collect.Maps; import com.datatorrent.api.Context; +import com.datatorrent.stram.client.StramClientUtils; +import static com.datatorrent.api.Context.DAGContext.APPLICATION_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -220,14 +222,44 @@ public class LoggerUtilTest assertNull(org.apache.log4j.Logger.getRootLogger().getAppender(name)); } + @Test + public void testSetupMDC() + { + // The test does not test MDC properties that are passed via environment variables + + String appenderName = "mdcTestAppender"; + String service = "test"; + String application = "my application"; + String args = "log4j.appender.mdcTestAppender=com.datatorrent.stram.util.LoggerUtilTest$TestAppender" + + ",log4j.appender.mdcTestAppender.layout=org.apache.log4j.PatternLayout" + + ",log4j.appender.mdcTestAppender.layout.ConversionPattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n"; + + System.setProperty(APPLICATION_NAME.getLongName(), application); + LoggerUtil.setupMDC(service); + + LoggerUtil.addAppenders(new String[] {appenderName }, args, ","); + TestAppender appender = (TestAppender)org.apache.log4j.Logger.getRootLogger().getAppender(appenderName); + + logger.info(args); + assertEquals(service, appender.mdcProperties.get("apex.service")); + String node = StramClientUtils.getHostName(); + assertEquals(node == null ? "unknown" : node, appender.mdcProperties.get("apex.node")); + assertEquals(application, appender.mdcProperties.get("apex.application")); + + LoggerUtil.removeAppender(appenderName); + } + public static class TestAppender extends ConsoleAppender { private String lastMessage = null; private Level level; + private Map mdcProperties; @Override public void append(LoggingEvent event) { + event.getMDCCopy(); + mdcProperties = event.getProperties(); lastMessage = event.getRenderedMessage(); level = event.getLevel(); super.append(event);
