Author: szetszwo
Date: Sat Mar 30 03:50:03 2013
New Revision: 1462698
URL: http://svn.apache.org/r1462698
Log:
Merge r1460409 through r1462697 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
- copied unchanged from r1462697,
hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
Modified:
hadoop/common/branches/HDFS-2802/ (props changed)
hadoop/common/branches/HDFS-2802/BUILDING.txt
hadoop/common/branches/HDFS-2802/dev-support/test-patch.sh
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
Propchange: hadoop/common/branches/HDFS-2802/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1460409-1462697
Modified: hadoop/common/branches/HDFS-2802/BUILDING.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/BUILDING.txt?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/BUILDING.txt (original)
+++ hadoop/common/branches/HDFS-2802/BUILDING.txt Sat Mar 30 03:50:03 2013
@@ -139,6 +139,15 @@ Create a local staging version of the we
----------------------------------------------------------------------------------
+Building on OS/X
+
+----------------------------------------------------------------------------------
+
+Hadoop does not build on OS/X with Java 7.
+see: https://issues.apache.org/jira/browse/HADOOP-9350
+
+----------------------------------------------------------------------------------
+
Building on Windows
----------------------------------------------------------------------------------
Modified: hadoop/common/branches/HDFS-2802/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/dev-support/test-patch.sh?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/HDFS-2802/dev-support/test-patch.sh Sat Mar 30
03:50:03 2013
@@ -323,7 +323,7 @@ checkAuthor () {
}
###############################################################################
-### Check for tests and their timeout in the patch
+### Check for tests in the patch
checkTests () {
echo ""
echo ""
@@ -357,25 +357,7 @@ checkTests () {
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 tests included{color}. The patch appears to include
$testReferences new or modified test files."
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking if the tests have timeout assigned in this patch."
- echo "======================================================================"
- echo "======================================================================"
-
- nontimeoutTests=`cat $PATCH_DIR/patch | $AWK '{ printf "%s ", $0 }' | $GREP
--extended-regex --count '[ ]*\+[ ]*((@Test[\+ ]*[A-Za-z]+)|([\+ ]*@Test[
\+]*\([ \+]*\)[\ ]*\+?[ ]*[A-Za-z]+)|([\+ ]*@Test[\+ ]*\(exception[ \+]*=[
\+]*[A-Z.a-z0-9A-Z ]*\)))'`
-
- if [[ $nontimeoutTests == 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 tests included appear to have a timeout.{color}"
- return 0
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 one of tests included doesn't have a timeout.{color}"
- return 1
+ return 0
}
cleanUpXml () {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
Sat Mar 30 03:50:03 2013
@@ -37,11 +37,11 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.rumen.JobStory;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
import org.apache.hadoop.tools.rumen.TaskInfo;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
Sat Mar 30 03:50:03 2013
@@ -22,8 +22,8 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* <p>A {@link ResourceUsageEmulatorPlugin} that emulates the cumulative CPU
@@ -166,7 +166,7 @@ implements ResourceUsageEmulatorPlugin {
*/
public void calibrate(ResourceCalculatorPlugin monitor,
long totalCpuUsage) {
- long initTime = monitor.getProcResourceValues().getCumulativeCpuTime();
+ long initTime = monitor.getCumulativeCpuTime();
long defaultLoopSize = 0;
long finalTime = initTime;
@@ -175,7 +175,7 @@ implements ResourceUsageEmulatorPlugin {
while (finalTime - initTime < 100) { // 100 ms
++defaultLoopSize;
performUnitComputation(); //perform unit computation
- finalTime = monitor.getProcResourceValues().getCumulativeCpuTime();
+ finalTime = monitor.getCumulativeCpuTime();
}
long referenceRuntime = finalTime - initTime;
@@ -230,7 +230,7 @@ implements ResourceUsageEmulatorPlugin {
}
private synchronized long getCurrentCPUUsage() {
- return monitor.getProcResourceValues().getCumulativeCpuTime();
+ return monitor.getCumulativeCpuTime();
}
@Override
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
Sat Mar 30 03:50:03 2013
@@ -20,7 +20,7 @@ package org.apache.hadoop.mapred.gridmix
import java.io.IOException;
import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
import org.apache.hadoop.conf.Configuration;
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
Sat Mar 30 03:50:03 2013
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* <p>This is the driver class for managing all the resource usage emulators.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
Sat Mar 30 03:50:03 2013
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* <p>A {@link ResourceUsageEmulatorPlugin} that emulates the total heap
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
Sat Mar 30 03:50:03 2013
@@ -23,7 +23,6 @@ import static org.junit.Assert.*;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob;
import org.apache.hadoop.mapred.gridmix.TestHighRamJob.DummyGridmixJob;
@@ -32,8 +31,8 @@ import org.apache.hadoop.mapred.gridmix.
import
org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin.DefaultHeapUsageEmulator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* Test Gridmix memory emulation.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
Sat Mar 30 03:50:03 2013
@@ -31,14 +31,13 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.task.MapContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
-import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin;
import org.apache.hadoop.mapred.gridmix.LoadJob.ResourceUsageMatcherRunner;
import
org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin;
import
org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageEmulatorPlugin;
import
org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageMatcher;
import
org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin.DefaultCpuUsageEmulator;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* Test Gridmix's resource emulator framework and supported plugins.
@@ -242,16 +241,6 @@ public class TestResourceUsageEmulators
public long getCumulativeCpuTime() {
return core.getCpuUsage();
}
-
- /**
- * Returns a {@link ProcResourceValues} with cumulative cpu usage
- * computed using {@link #getCumulativeCpuTime()}.
- */
- @Override
- public ProcResourceValues getProcResourceValues() {
- long usageValue = getCumulativeCpuTime();
- return new ProcResourceValues(usageValue, -1, -1);
- }
}
/**
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java
Sat Mar 30 03:50:03 2013
@@ -68,6 +68,7 @@ public class TestStreamReduceNone
"-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer",
"-numReduceTasks", "0",
"-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
+ "-jobconf", "mapreduce.job.maps=1",
"-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
};
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java?rev=1462698&r1=1462697&r2=1462698&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
Sat Mar 30 03:50:03 2013
@@ -54,6 +54,8 @@ public class TestStreamXmlRecordReader e
protected String[] genArgs() {
args.add("-inputreader");
args.add("StreamXmlRecordReader,begin=<xmltag>,end=</xmltag>");
+ args.add("-jobconf");
+ args.add("mapreduce.job.maps=1");
return super.genArgs();
}
}
