svn commit: r1737721 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
Author: challngr
Date: Mon Apr 4 18:33:54 2016
New Revision: 1737721
URL: http://svn.apache.org/viewvc?rev=1737721=rev
Log:
UIMA-4882 Extend init_wait semantics to include "no recorded job time yet."
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1737721=1737720=1737721=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
Mon Apr 4 18:33:54 2016
@@ -569,7 +569,7 @@ public class RmJob
sharesByMachine.clear();
}
-/**
+ /**
* I've shrunk or this share has nothing left to do. Remove this specific
share.
*/
public void removeShare(Share share)
@@ -1000,7 +1000,25 @@ public class RmJob
{
String methodName = "getPrjCap"; // want this to
line up with getJobCap in logs
-if ( init_wait || Double.isNaN(time_per_item) || (time_per_item ==
0.0)) { // no cap if not initialized, or no per-itme time yet
+// UIMA-4882 jrc
+// Must enhance semantics of init_wait to mean "nothing initialized,
and have never seen any
+// execution time for the job." This accounts for the moment after a
job initializes, and before it
+// gets anything running and helps to throttle expansion until a job
starts to run.
+//
+// After initialization, the time_per_item will be quite small but
non-zero, so we'll tend to predict
+// a future cap as the moral equicalent of "not too many more needed".
For installations without
+// doubling, or where doubling is too fast, this leads to better
controlled expansion if the job
+// actually is going to compete soon.
+//
+// The other part of this update includes the OR updating its
"time_per_item" to account for
+// work items in progress as well as work items completed, so we're
guarantteed to get a
+// time_per_item != 0 shortly after first initialization.
+//
+// (We update init_wait here because it's used later and needs to be
used with the same
+// semantics as is used here.)
+init_wait = init_wait || Double.isNaN(time_per_item) || (time_per_item
== 0.0);
+
+if ( init_wait ) { // no cap if not initialized, or no per-itme time
yet
logger.info(methodName, getId(), username, "Cannot predict cap:
init_wait", init_wait, "|| time_per_item", time_per_item);
return Integer.MAX_VALUE;
}
svn commit: r1737719 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
Author: degenaro
Date: Mon Apr 4 18:27:51 2016
New Revision: 1737719
URL: http://svn.apache.org/viewvc?rev=1737719=rev
Log:
UIMA-4882 DUCC Resource Manager (RM) does not adequately expand Job Processes
(JPs)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1737719=1737718=1737719=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
Mon Apr 4 18:27:51 2016
@@ -360,17 +360,31 @@ public class JobManagerConverter
arith_mean = stats.getMean();
}
+// The skewed mean is the arithmetic mean of work items both
completed
+// (if any) and active (if any). All completed work items
contribute,
+// but only active work items whose time already exceeds the mean
of
+// the completed ones contribute.
+
+// To schedule, we always use the skewed_mean when it is > 0.
+
+double skewed_mean = si.getAvgTimeForWorkItemsSkewedByActive();
+
logger.info(methodName, job.getDuccId(),
-String.format("tot: %d %s -> %s compl: %s err: %s rem:
%d mean: %f",
+String.format("tot: %d %s -> %s compl: %s err: %s rem:
%d mean: %f skew: %f",
total_work,
state,
job.getStateObject(),
si.getWorkItemsCompleted(),// note
this comes in as string (!) from OR
si.getWorkItemsError(),// also
string
remaining_work,
- arith_mean
+ arith_mean,
+ skewed_mean
));
+if(skewed_mean > 0) {
+ arith_mean = skewed_mean;
+}
+
if ( max_shares != existing_max_shares ) {
j.setMaxShares(max_shares);
logger.info(methodName, job.getDuccId(), "Max shares adjusted
from", existing_max_shares, "to", max_shares, "(incoming)",
svn commit: r1737718 - /uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
Author: degenaro
Date: Mon Apr 4 18:25:41 2016
New Revision: 1737718
URL: http://svn.apache.org/viewvc?rev=1737718=rev
Log:
UIMA-4882 DUCC Resource Manager (RM) does not adequately expand Job Processes
(JPs)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java?rev=1737718=1737717=1737718=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
Mon Apr 4 18:25:41 2016
@@ -36,6 +36,7 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.IDuccState.JobState;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkExecutable;
@@ -796,9 +797,14 @@ public class ProcessAccounting {
duccWorkJob.getSchedulingInfo().setWorkItemsPreempt(jdPreemptWorkItems);
}
-
duccWorkJob.getSchedulingInfo().setMostRecentWorkItemStart(jdStatusReport.getMostRecentStart());
-
duccWorkJob.getSchedulingInfo().setPerWorkItemStatistics(jdStatusReport.getPerWorkItemStatistics());
+ IDuccSchedulingInfo si = duccWorkJob.getSchedulingInfo();
+
+
si.setMostRecentWorkItemStart(jdStatusReport.getMostRecentStart());
+
si.setPerWorkItemStatistics(jdStatusReport.getPerWorkItemStatistics());
+ double avgTimeForWorkItemsSkewedByActive =
jdStatusReport.getAvgTimeForWorkItemsSkewedByActive();
+
si.setAvgTimeForWorkItemsSkewedByActive(avgTimeForWorkItemsSkewedByActive);
+
logger.trace(methodName, null, messages.fetch("exit"));
return retVal;
}
svn commit: r1737716 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event: common/DuccSchedulingInfo.java common/IDuccSchedulingInfo.java jd/IDrive
Author: degenaro
Date: Mon Apr 4 18:22:03 2016
New Revision: 1737716
URL: http://svn.apache.org/viewvc?rev=1737716=rev
Log:
UIMA-4882 DUCC Resource Manager (RM) does not adequately expand Job Processes
(JPs)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/IDriverStatusReport.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java?rev=1737716=1737715=1737716=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
Mon Apr 4 18:22:03 2016
@@ -68,6 +68,7 @@ public class DuccSchedulingInfo implemen
@Deprecated
private String workItemsPending = defaultWorkItemsPending;
+ private double avgTimeForWorkItemsSkewedByActive = 0;
public String getSchedulingClass() {
return schedulingClass;
@@ -563,4 +564,16 @@ public class DuccSchedulingInfo implemen
return true;
}
+
+ @Override
+ public void setAvgTimeForWorkItemsSkewedByActive(double value) {
+ avgTimeForWorkItemsSkewedByActive = value;
+ }
+
+
+ @Override
+ public double getAvgTimeForWorkItemsSkewedByActive() {
+ return avgTimeForWorkItemsSkewedByActive;
+ }
+
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java?rev=1737716=1737715=1737716=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
Mon Apr 4 18:22:03 2016
@@ -52,7 +52,7 @@ public interface IDuccSchedulingInfo ext
public static final String defaultWorkItemsPreempt = "0";
@Deprecated
public static final String defaultWorkItemsPending = "unknown";
-
+
// common
public String getSchedulingClass();
@@ -148,4 +148,7 @@ public interface IDuccSchedulingInfo ext
public String getWorkItemsPending();
@Deprecated
public void setWorkItemsPending(String number);
+
+ public void setAvgTimeForWorkItemsSkewedByActive(double value);
+ public double getAvgTimeForWorkItemsSkewedByActive();
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/IDriverStatusReport.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/IDriverStatusReport.java?rev=1737716=1737715=1737716=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/IDriverStatusReport.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/IDriverStatusReport.java
Mon Apr 4 18:22:03 2016
@@ -96,6 +96,8 @@ public interface IDriverStatusReport {
public IDuccPerWorkItemStatistics getPerWorkItemStatistics();
+ public double getAvgTimeForWorkItemsSkewedByActive();
+
public IDuccProcessWorkItemsReport getDuccProcessWorkItemsMap();
public ConcurrentHashMap getOperatingMillisMap();
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java
URL:
