Author: acmurthy
Date: Thu Jul 5 07:05:24 2012
New Revision: 1357475
URL: http://svn.apache.org/viewvc?rev=1357475&view=rev
Log:
MAPREDUCE-4369. Fix streaming test failures caused by
WindowsResourceCalculatorPlugin. Contributed by Bikas Saha.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.txt
hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Thu Jul 5 07:05:24 2012
@@ -53,6 +53,9 @@ branch-hadoop-1-win - unreleased
MAPREDUCE-4332 Fix command-length abort issues on Windows. (Ivan Mitic via
acmurthy)
+ MAPREDUCE-4369. Fix streaming test failures caused by
+ WindowsResourceCalculatorPlugin. (Bikas Saha via acmurthy)
+
Release 1.1.0 - unreleased
NEW FEATURES
Modified:
hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
(original)
+++
hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
Thu Jul 5 07:05:24 2012
@@ -185,6 +185,10 @@ class LoadJob extends GridmixJob {
ResourceCalculatorPlugin plugin =
ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
+ if(plugin != null) {
+ plugin.setProcessPid(System.getenv("JVM_PID"));
+ }
+
// set the other parameters
this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
progress = new BoostingProgress(context);
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
Thu Jul 5 07:05:24 2012
@@ -396,10 +396,6 @@ public class LinuxResourceCalculatorPlug
@Override
public ProcResourceValues getProcResourceValues() {
if(pTree == null) {
- if(processPid == null) {
- // process pid not set. try to obtain on our own
- processPid = System.getenv().get("JVM_PID");
- }
pTree = new ProcfsBasedProcessTree(processPid);
}
pTree.getProcessTree();
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
Thu Jul 5 07:05:24 2012
@@ -100,6 +100,8 @@ public abstract class ResourceCalculator
/**
* Obtain resource status used by current process tree.
+ * Must call {@link #setProcessPid(String)} to set the process id before
+ * calling this method
*/
public abstract ProcResourceValues getProcResourceValues();
Modified:
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
(original)
+++
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
Thu Jul 5 07:05:24 2012
@@ -101,30 +101,33 @@ public class WindowsBasedProcessTree ext
@Override
public ResourceCalculatorProcessTree getProcessTree() {
- String processesInfoStr = getAllProcessInfoFromShell();
- if (processesInfoStr != null && processesInfoStr.length() > 0) {
- Map<String, ProcessInfo> allProcessInfo =
createProcessInfo(processesInfoStr);
-
- for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
- String pid = entry.getKey();
- ProcessInfo pInfo = entry.getValue();
- ProcessInfo oldInfo = processTree.get(pid);
- if (oldInfo != null) {
- // existing process, update age and replace value
- pInfo.age += oldInfo.age;
- // calculate the delta since the last refresh. totals are being kept
- // in the WindowsBasedProcessTree object
- pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
- } else {
- // new process. delta cpu == total cpu
- pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+ if(taskProcessId != null) {
+ // taskProcessId can be null in some tests
+ String processesInfoStr = getAllProcessInfoFromShell();
+ if (processesInfoStr != null && processesInfoStr.length() > 0) {
+ Map<String, ProcessInfo> allProcessInfo =
createProcessInfo(processesInfoStr);
+
+ for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet())
{
+ String pid = entry.getKey();
+ ProcessInfo pInfo = entry.getValue();
+ ProcessInfo oldInfo = processTree.get(pid);
+ if (oldInfo != null) {
+ // existing process, update age and replace value
+ pInfo.age += oldInfo.age;
+ // calculate the delta since the last refresh. totals are being
kept
+ // in the WindowsBasedProcessTree object
+ pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
+ } else {
+ // new process. delta cpu == total cpu
+ pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+ }
}
+ processTree.clear();
+ processTree = allProcessInfo;
+ } else {
+ // clearing process tree to mimic semantics of existing Procfs impl
+ processTree.clear();
}
- processTree.clear();
- processTree = allProcessInfo;
- }
- else {
- processTree.clear();
}
return this;
Modified:
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
---
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
(original)
+++
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
Thu Jul 5 07:05:24 2012
@@ -533,9 +533,14 @@ abstract public class Task implements Wr
.getResourceCalculatorPlugin(clazz, conf);
LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator);
if (resourceCalculator != null) {
- resourceCalculator.setProcessPid(jvmContext.pid);
- initCpuCumulativeTime =
- resourceCalculator.getProcResourceValues().getCumulativeCpuTime();
+ if (jvmContext != null) {
+ resourceCalculator.setProcessPid(jvmContext.pid);
+ initCpuCumulativeTime = resourceCalculator.getProcResourceValues()
+ .getCumulativeCpuTime();
+ } else {
+ // jvmContext is not set in local job mode
+ resourceCalculator = null;
+ }
}
}
