Author: tjungblut
Date: Mon Aug 13 16:47:59 2012
New Revision: 1372505
URL: http://svn.apache.org/viewvc?rev=1372505&view=rev
Log:
[HAMA-621]: Input Splits are not initialized for a job
Modified:
hama/trunk/CHANGES.txt
hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java
hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java
hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
hama/trunk/pom.xml
Modified: hama/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Mon Aug 13 16:47:59 2012
@@ -30,11 +30,12 @@ Release 0.5 - April 10, 2012
BUG FIXES
+ HAMA-621: Input Splits are not initialized for a job (surajmenon via
tjungblut)
HAMA-595: Fix NullPointerException in Task Scheduler (surajmenon)
IMPROVEMENTS
- HAMA-584: Change Pagerank IO format to human-readable text for easy debug
(tjungblut vai edwardyoon)
+ HAMA-584: Change Pagerank IO format to human-readable text for easy debug
(tjungblut via edwardyoon)
HAMA-590: Fix TestSubmitGraphJob tests (tjungblut)
HAMA-582: Task's error logs should be displayed on client-end when job is
failed (edwardyoon)
HAMA-580: Improve input of graph module (tjungblut)
Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java
(original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/JobInProgress.java Mon
Aug 13 16:47:59 2012
@@ -348,8 +348,12 @@ public class JobInProgress {
GroomServerStatus groomStatus = taskAllocationStrategy
.getGroomToAllocate(groomStatuses, selectedGrooms,
taskCountInGroomMap, resources, tasks[i]);
- if (groomStatus != null)
+ if (groomStatus != null){
result = tasks[i].constructTask(groomStatus);
+ }
+ else if (LOG.isDebugEnabled()){
+ LOG.debug("Could not find a groom to schedule task");
+ }
if (result != null) {
updateGroomTaskDetails(tasks[i].getGroomServerStatus(), result);
}
Modified: hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java
URL:
http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java
(original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/TaskInProgress.java Mon
Aug 13 16:47:59 2012
@@ -199,6 +199,10 @@ public class TaskInProgress {
} else {
String splitClass = null;
BytesWritable split = null;
+ if (rawSplit != null) {
+ splitClass = rawSplit.getClassName();
+ split = rawSplit.getBytes();
+ }
currentTaskId = taskId;
String groomName = groomStatus.getGroomHostName();
Task t = new BSPTask(jobId, jobFile, taskId, partition, splitClass,
split);
@@ -209,44 +213,16 @@ public class TaskInProgress {
}
- // /* Remove */
- // private Task getGroomForTask(TaskAttemptID taskid,
- // Map<String, GroomServerStatus> grooms,
- // Map<GroomServerStatus, Integer> tasksInGroomMap) {
- // String splitClass = null;
- // BytesWritable split = null;
- // Task t = null;
- // if (rawSplit != null) {
- // splitClass = rawSplit.getClassName();
- // split = rawSplit.getBytes();
- // String[] possibleLocations = rawSplit.getLocations();
- // String groomName = getGroomToSchedule(taskid, grooms, tasksInGroomMap,
- // possibleLocations);
- // if (groomName != null) {
- // t = new BSPTask(jobId, jobFile, taskid, partition, splitClass, split);
- // activeTasks.put(taskid, groomName);
- // myGroomStatus = grooms.get(groomName);
- // }
- // }
- //
- // if (t == null) {
- // String groomName = getAnyGroomToSchedule(grooms, tasksInGroomMap);
- // if (groomName != null) {
- // t = new BSPTask(jobId, jobFile, taskid, partition, splitClass, split);
- // activeTasks.put(taskid, groomName);
- // myGroomStatus = grooms.get(groomName);
- // }
- // }
- //
- // return t;
- // }
-
private Task getGroomForRecoverTaskInHosts(TaskAttemptID taskid,
Map<String, GroomServerStatus> grooms,
Map<GroomServerStatus, Integer> tasksInGroomMap,
String[] possibleLocations) {
String splitClass = null;
BytesWritable split = null;
+ if (rawSplit != null) {
+ splitClass = rawSplit.getClassName();
+ split = rawSplit.getBytes();
+ }
Task t = null;
String groomName = getGroomToSchedule(taskid, grooms, tasksInGroomMap,
possibleLocations);
Modified:
hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
URL:
http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
(original)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/sync/ZKSyncClient.java
Mon Aug 13 16:47:59 2012
@@ -464,7 +464,7 @@ public abstract class ZKSyncClient imple
} else {
for (String node : list) {
clearZKNodes(path + "/" + node);
- LOG.info("Deleting " + path + "/" + node);
+ LOG.debug("Deleting " + path + "/" + node);
zk.delete(path + "/" + node, -1); // delete any version of this
// node.
}
Modified:
hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
URL:
http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
---
hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
(original)
+++
hama/trunk/core/src/main/java/org/apache/hama/bsp/taskallocation/BestEffortDataLocalTaskAllocator.java
Mon Aug 13 16:47:59 2012
@@ -83,15 +83,26 @@ public class BestEffortDataLocalTaskAllo
for (int i = 0; i < possibleLocations.length; ++i) {
String location = possibleLocations[i];
GroomServerStatus groom = grooms.get(location);
- if (groom == null)
+ if (groom == null){
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Could not find groom for location " + location);
+ }
continue;
+ }
Integer taskInGroom = tasksInGroomMap.get(groom);
taskInGroom = (taskInGroom == null) ? 0 : taskInGroom;
+ if(LOG.isDebugEnabled()){
+ LOG.debug("taskInGroom = " + taskInGroom + " max tasks = " +
groom.getMaxTasks()
+ + " location = " + location + " groomhostname = " +
groom.getGroomHostName());
+ }
if (taskInGroom < groom.getMaxTasks()
&& location.equals(groom.getGroomHostName())) {
return groom.getGroomHostName();
}
}
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Returning null");
+ }
return null;
}
@@ -100,8 +111,12 @@ public class BestEffortDataLocalTaskAllo
Map<String, GroomServerStatus> groomStatuses, String[] selectedGrooms,
Map<GroomServerStatus, Integer> taskCountInGroomMap,
BSPResource[] resources, TaskInProgress taskInProgress) {
- if (!taskInProgress.canStartTask())
+ if (!taskInProgress.canStartTask()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exceeded allowed attempts.");
+ }
return null;
+ }
String groomName = null;
if (selectedGrooms != null) {
Modified: hama/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1372505&r1=1372504&r2=1372505&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Mon Aug 13 16:47:59 2012
@@ -269,7 +269,7 @@
<version>2.6</version>
<configuration>
<parallel>methods</parallel>
- <threadCount>10</threadCount>
+ <threadCount>1</threadCount>
<forkMode>pertest</forkMode>
</configuration>
</plugin>
