Author: challngr
Date: Wed May 29 18:05:59 2013
New Revision: 1487580
URL: http://svn.apache.org/r1487580
Log:
UIMA-2593
Fix RM overcounting a dead node with work in limbo. Also fix the system tests
to include the random number seed (found while working on this boog).
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment
Wed May 29 18:05:59 2013
@@ -3,6 +3,11 @@
# Compression of 96 and spread 900 gives a good 15-20 minute test on test
system with
# 136 15-G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir = jobs.in # where the jobs are
dest.dir = jobs # where to put prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed
Wed May 29 18:05:59 2013
@@ -3,6 +3,11 @@
# Compression of 96 and spread 900 gives a good 15-20 minute test on test
system with
# 136 15-G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir = jobs.in # where the jobs are
dest.dir = jobs # where to put prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare
Wed May 29 18:05:59 2013
@@ -1,6 +1,11 @@
# These jobs represent approximately 144 hours real time; adjust compression
accordingly
# Compresion of 12 produces an approximately 12 hour test on a system with 136
15G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir = jobs.in # where the jobs are
dest.dir = jobs # where to put prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment
Wed May 29 18:05:59 2013
@@ -1,6 +1,11 @@
# These jobs represent approximately 144 hours real time; adjust compression
accordingly
# Compresion of 12 produces an approximately 12 hour test on a system with 136
15G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir =
projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs.in # where the jobs are
dest.dir =
projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs # where to put
prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed
Wed May 29 18:05:59 2013
@@ -1,6 +1,11 @@
# These jobs represent approximately 144 hours real time; adjust compression
accordingly
# Compresion of 12 and spread of 42000 produces a good 12 hour test on a
system with 136 15G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir =
projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs.in # where the jobs are
dest.dir =
projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs # where to put
prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare
Wed May 29 18:05:59 2013
@@ -3,6 +3,11 @@
# Compression of 48 and spread 1800 gives a good 30 minute test on test system
with
# 136 15-G shares
+random.seed = 0 # a number, for determinate randoms
+ # or TOD, and the seed will use
+ # current time of day
+
+
src.dir = jobs.in # where the jobs are
dest.dir = jobs # where to put prepared jobs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
Wed May 29 18:05:59 2013
@@ -1219,6 +1219,8 @@ class NodePool
int order = j.getShareOrder();
int given = 0;
+ logger.debug(methodName, j.getId(), "counted", counted, "current",
current, "needed", needed, "order", order, "given", given);
+
if ( needed > 0 ) {
whatof: {
for ( int i = order; i < getArraySize(); i++ ) {
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1487580&r1=1487579&r2=1487580&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
Wed May 29 18:05:59 2013
@@ -980,9 +980,9 @@ public class RmJob
c++;
}
- c = Math.max(c, assignedShares.size()); // if job is ending we
could be fragmented and have to be
- // careful not to
underestimate, or we end up possibly
- // evicting something
that should be left alone.
+ c = Math.max(c, assignedShares.size() - pendingRemoves.size()); // if
job is ending we could be fragmented and have to be
+ //
careful not to underestimate, or we end up possibly
+ //
evicting something that should be left alone.
//
// If waiting for initialization, we have to cap as well on the
maximum number of shares
