Author: amarrk
Date: Thu Sep 8 11:29:55 2011
New Revision: 1166636
URL: http://svn.apache.org/viewvc?rev=1166636&view=rev
Log:
MAPREDUCE-2784. [Gridmix] Bug fixes in ExecutionSummarizer and
ResourceUsageMatcher. (amarrk)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1166636&r1=1166635&r2=1166636&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Sep 8
11:29:55 2011
@@ -6,6 +6,10 @@ Trunk (unreleased changes)
MAPREDUCE-2887 due to HADOOP-7524 Change RPC to allow multiple protocols
including multuple versions of the same protocol (sanjay Radia)
+ BUG FIXES
+ MAPREDUCE-2784. [Gridmix] Bug fixes in ExecutionSummarizer and
+ ResourceUsageMatcher. (amarrk)
+
Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java?rev=1166636&r1=1166635&r2=1166636&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
Thu Sep 8 11:29:55 2011
@@ -149,10 +149,15 @@ class ExecutionSummarizer implements Sta
throws IOException {
numJobsInInputTrace = factory.numJobsInTrace;
endTime = System.currentTimeMillis();
- Path inputTracePath = new Path(inputPath);
- FileSystem fs = inputTracePath.getFileSystem(conf);
- inputTraceLocation = fs.makeQualified(inputTracePath).toString();
- inputTraceSignature = getTraceSignature(inputTraceLocation);
+ if ("-".equals(inputPath)) {
+ inputTraceLocation = Summarizer.NA;
+ inputTraceSignature = Summarizer.NA;
+ } else {
+ Path inputTracePath = new Path(inputPath);
+ FileSystem fs = inputTracePath.getFileSystem(conf);
+ inputTraceLocation = fs.makeQualified(inputTracePath).toString();
+ inputTraceSignature = getTraceSignature(inputPath);
+ }
jobSubmissionPolicy = Gridmix.getJobSubmissionPolicy(conf).name();
resolver = userResolver.getClass().getName();
if (dataSize > 0) {
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1166636&r1=1166635&r2=1166636&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
Thu Sep 8 11:29:55 2011
@@ -314,9 +314,13 @@ public class Gridmix extends Configured
}
});
- // print the run summary
- System.out.print("\n\n");
- System.out.println(summarizer.toString());
+ // print the gridmix summary if the run was successful
+ if (val == 0) {
+ // print the run summary
+ System.out.print("\n\n");
+ System.out.println(summarizer.toString());
+ }
+
return val;
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java?rev=1166636&r1=1166635&r2=1166636&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
Thu Sep 8 11:29:55 2011
@@ -52,15 +52,23 @@ public class ResourceUsageMatcher {
@SuppressWarnings("unchecked")
public void configure(Configuration conf, ResourceCalculatorPlugin monitor,
ResourceUsageMetrics metrics, Progressive progress) {
- Class[] plugins =
- conf.getClasses(RESOURCE_USAGE_EMULATION_PLUGINS,
- ResourceUsageEmulatorPlugin.class);
+ Class[] plugins = conf.getClasses(RESOURCE_USAGE_EMULATION_PLUGINS);
if (plugins == null) {
System.out.println("No resource usage emulator plugins configured.");
} else {
- for (Class<? extends ResourceUsageEmulatorPlugin> plugin : plugins) {
- if (plugin != null) {
- emulationPlugins.add(ReflectionUtils.newInstance(plugin, conf));
+ for (Class clazz : plugins) {
+ if (clazz != null) {
+ if (ResourceUsageEmulatorPlugin.class.isAssignableFrom(clazz)) {
+ ResourceUsageEmulatorPlugin plugin =
+ (ResourceUsageEmulatorPlugin) ReflectionUtils.newInstance(clazz,
+ conf);
+ emulationPlugins.add(plugin);
+ } else {
+ throw new RuntimeException("Misconfigured resource usage plugins.
"
+ + "Class " + clazz.getClass().getName() + " is not a resource "
+ + "usage plugin as it does not extend "
+ + ResourceUsageEmulatorPlugin.class.getName());
+ }
}
}
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java?rev=1166636&r1=1166635&r2=1166636&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
Thu Sep 8 11:29:55 2011
@@ -159,7 +159,7 @@ public class TestGridmixSummary {
@Override
protected Thread createReaderThread() {
- return null;
+ return new Thread();
}
}
@@ -243,7 +243,7 @@ public class TestGridmixSummary {
tid, es.getInputTraceSignature());
// test trace location
Path qPath = fs.makeQualified(testTraceFile);
- assertEquals("Mismatch in trace signature",
+ assertEquals("Mismatch in trace filename",
qPath.toString(), es.getInputTraceLocation());
// test expected data size
assertEquals("Mismatch in expected data size",
@@ -275,7 +275,7 @@ public class TestGridmixSummary {
es.finalize(factory, testTraceFile.toString(), 0L, resolver, dataStats,
conf);
// test missing expected data size
- assertEquals("Mismatch in trace signature",
+ assertEquals("Mismatch in trace data size",
Summarizer.NA, es.getExpectedDataSize());
assertFalse("Mismatch in trace signature",
tid.equals(es.getInputTraceSignature()));
@@ -295,6 +295,12 @@ public class TestGridmixSummary {
assertEquals("Mismatch in trace signature",
tid, es.getInputTraceSignature());
+ // finalize trace identifier '-' input
+ es.finalize(factory, "-", 0L, resolver, dataStats, conf);
+ assertEquals("Mismatch in trace signature",
+ Summarizer.NA, es.getInputTraceSignature());
+ assertEquals("Mismatch in trace file location",
+ Summarizer.NA, es.getInputTraceLocation());
}
// test the ExecutionSummarizer
