Repository: incubator-gobblin Updated Branches: refs/heads/master a66205a7e -> 88383014c
[GOBBLIN-499] Log the job name with the tracking URL for easier debugging Closes #2369 from htran1/log_job_name_with_tracking_url Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/88383014 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/88383014 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/88383014 Branch: refs/heads/master Commit: 88383014c483013e3035bc9ead8b15f998275155 Parents: a66205a Author: Hung Tran <[email protected]> Authored: Wed May 23 11:58:42 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Wed May 23 11:58:42 2018 -0700 ---------------------------------------------------------------------- .../gobblin/runtime/mapreduce/MRTask.java | 3 +++ .../runtime/mapreduce/MRTaskFactoryTest.java | 22 ++++++++++++++++++++ 2 files changed, 25 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/88383014/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRTask.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRTask.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRTask.java index bae259c..af12004 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRTask.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRTask.java @@ -103,6 +103,9 @@ public class MRTask extends BaseAbstractTask { } job.submit(); + + log.info("MR tracking URL {} for job {}", job.getTrackingURL(), job.getJobName()); + this.eventSubmitter.submit(Events.MR_JOB_STARTED_EVENT, Events.JOB_URL, job.getTrackingURL()); job.waitForCompletion(false); this.mrJob = job; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/88383014/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRTaskFactoryTest.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRTaskFactoryTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRTaskFactoryTest.java index a1e91f1..4a54259 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRTaskFactoryTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRTaskFactoryTest.java @@ -33,6 +33,7 @@ import org.apache.gobblin.source.extractor.Extractor; import org.apache.gobblin.source.workunit.WorkUnit; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -46,6 +47,10 @@ import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.testng.Assert; import org.testng.annotations.Test; @@ -70,13 +75,21 @@ public class MRTaskFactoryTest { Assert.assertTrue(job2Dir.mkdir()); writeFileWithContent(job2Dir, "file1", "word1 word2 word2"); + TestAppender testAppender = new TestAppender(); + Logger logger = LogManager.getLogger(MRTask.class.getName()); + logger.addAppender(testAppender); + EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("WordCounter") .setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MRWordCountSource.class.getName()) .setConfiguration(MRWordCountSource.INPUT_DIRECTORIES_KEY, job1Dir.getAbsolutePath() + "," + job2Dir.getAbsolutePath()) .setConfiguration(MRWordCountSource.OUTPUT_LOCATION, outputSuperPath.getAbsolutePath()); JobExecutionResult result = embeddedGobblin.run(); + logger.removeAppender(testAppender); + Assert.assertTrue(result.isSuccessful()); + Assert.assertTrue(testAppender.events.stream().anyMatch(e -> e.getRenderedMessage() + .startsWith("MR tracking URL http://localhost:8080/ for job WordCount_job1"))); File output1 = new File(new File(outputSuperPath, "job1"), "part-r-00000"); Assert.assertTrue(output1.exists()); @@ -193,4 +206,13 @@ public class MRTaskFactoryTest { } } + private class TestAppender extends AppenderSkeleton { + List<LoggingEvent> events = new ArrayList<>(); + public void close() {} + public boolean requiresLayout() {return false;} + @Override + protected void append(LoggingEvent event) { + events.add(event); + } + } }
