Repository: eagle Updated Branches: refs/heads/master eae6e8f11 -> 9381c2d17
[MINOR] fix small bugs in MR job performance monitoring Author: Zhao, Qingwen <[email protected]> Closes #789 from qingwen220/minor. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/9381c2d1 Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/9381c2d1 Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/9381c2d1 Branch: refs/heads/master Commit: 9381c2d17e3e10a02e9f0f359a05d0728ee17752 Parents: eae6e8f Author: Zhao, Qingwen <[email protected]> Authored: Mon Jan 23 14:39:52 2017 +0800 Committer: Zhao, Qingwen <[email protected]> Committed: Mon Jan 23 14:39:52 2017 +0800 ---------------------------------------------------------------------- .../apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +- .../jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java | 4 ++++ .../jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java | 3 +-- .../java/org/apache/eagle/jpm/analyzer/publisher/Result.java | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e32a37c..57e1765 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -48,7 +48,7 @@ public class MRJobPerformanceAnalyzer<T extends AnalyzerEntity> implements JobAn evaluators.add(new JobSuggestionEvaluator(config)); publishers.add(new EagleStorePublisher(config)); - publishers.add(new EmailPublisher(config)); + //publishers.add(new EmailPublisher(config)); } @Override http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index 103de7a..4007747 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -20,6 +20,7 @@ package org.apache.eagle.jpm.analyzer.mr.suggestion; import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.jpm.util.jobcounter.JobCounters; import java.util.ArrayList; @@ -45,9 +46,11 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + String message = "%s GC_MILLISECONDS is: %s, and CPU_MILLISECONDS is: %s. "; if (mapGCTime > mapCPUTime * 0.1) { setting = String.format("-D%s", MAP_JAVA_OPTS); optSettings.add(setting); + sb.append(String.format(message, Constants.TaskType.MAP, mapGCTime, mapCPUTime)); sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via ").append(setting); sb.append(", or optimize your mapper class.\n"); } @@ -58,6 +61,7 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti if (reduceGCTime > reduceCPUTime * 0.1) { setting = String.format("-D%s", REDUCE_JAVA_OPTS); optSettings.add(setting); + sb.append(String.format(message, Constants.TaskType.REDUCE, reduceGCTime, reduceCPUTime)); sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via ").append(setting); sb.append(", or optimize your reducer class.\n"); } http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 835b382..96be2d5 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -82,7 +82,7 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); if (heapSize < 3 * minMapSpillMemMB) { long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 * 1024; - setting = String.format(" -D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB); + setting = String.format("-D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB); sb.append(" ").append(setting); optSettings.add(setting); } @@ -90,7 +90,6 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit sb.append(" to avoid spilled records.\n"); } - long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); if (reduceInputRecords < spillRecords) { http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java index a9f5132..7d7442b 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java @@ -63,9 +63,11 @@ public class Result { } private void normalizeResult(ProcessorResult processorResult) { + String settingList = ""; if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty()) { - processorResult.setSettingList(StringUtils.join(processorResult.getSettings(), "\n")); + settingList = StringUtils.join(processorResult.getSettings(), "\n"); } + processorResult.setSettingList(settingList); } /**
