Author: degenaro Date: Wed Apr 10 20:17:38 2013 New Revision: 1466657 URL: http://svn.apache.org/r1466657 Log: UIMA-2804 DUCC orchestrator (OR) CGroups support for JD and JPs
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java?rev=1466657&r1=1466656&r2=1466657&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java Wed Apr 10 20:17:38 2013 @@ -18,15 +18,13 @@ */ package org.apache.uima.ducc.orchestrator; -import org.apache.uima.ducc.common.utils.id.IDuccId; import org.apache.uima.ducc.transport.event.common.CGroup; import org.apache.uima.ducc.transport.event.common.IDuccProcess; public class CGroupManager { - public static void assign(IDuccProcess process) { - IDuccId duccId = process.getDuccId(); - CGroup cgroup = new CGroup(duccId); + public static void assign(IDuccProcess process, long max_size_in_bytes) { + CGroup cgroup = new CGroup(max_size_in_bytes); process.setCGroup(cgroup); } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java?rev=1466657&r1=1466656&r2=1466657&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java Wed Apr 10 20:17:38 2013 @@ -69,6 +69,15 @@ public class JobFactory { return jobFactory; } + private long driver_max_size_in_bytes = 0; + + public JobFactory() { + String ducc_jd_share_quantum = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_jd_share_quantum); + long oneKB = 1024; + long oneMB = 1024*oneKB; + driver_max_size_in_bytes = Long.parseLong(ducc_jd_share_quantum) * oneMB; + } + private OrchestratorCommonArea orchestratorCommonArea = OrchestratorCommonArea.getInstance(); private IDuccIdFactory duccIdFactory = orchestratorCommonArea.getDuccIdFactory(); private JobDriverHostManager hostManager = orchestratorCommonArea.getHostManager(); @@ -332,7 +341,7 @@ public class JobFactory { DuccId duccId = jdIdFactory.next(); duccId.setFriendly(0); DuccProcess driverProcess = new DuccProcess(duccId,nodeIdentity,ProcessType.Pop); - CGroupManager.assign(driverProcess); + CGroupManager.assign(driverProcess, driver_max_size_in_bytes); driverProcess.setResourceState(ResourceState.Allocated); driverProcess.setNodeIdentity(nodeIdentity); driver.getProcessMap().put(driverProcess.getDuccId(), driverProcess); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1466657&r1=1466656&r2=1466657&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java Wed Apr 10 20:17:38 2013 @@ -30,6 +30,7 @@ import org.apache.uima.ducc.common.NodeI import org.apache.uima.ducc.common.internationalization.Messages; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccLoggerComponents; +import org.apache.uima.ducc.common.utils.DuccPropertiesResolver; import org.apache.uima.ducc.common.utils.TimeStamp; import org.apache.uima.ducc.common.utils.id.DuccId; import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent; @@ -78,6 +79,16 @@ public class StateManager { return stateManager; } + private long quantum_size_in_bytes = 0; + + public StateManager() { + String ducc_rm_share_quantum = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_rm_share_quantum); + long oneKB = 1024; + long oneMB = 1024*oneKB; + long oneGB = 1024*oneMB; + quantum_size_in_bytes = Long.parseLong(ducc_rm_share_quantum) * oneGB; + } + private OrchestratorCommonArea orchestratorCommonArea = OrchestratorCommonArea.getInstance(); private Messages messages = orchestratorCommonArea.getSystemMessages(); private DuccWorkMap workMap = orchestratorCommonArea.getWorkMap(); @@ -799,6 +810,7 @@ public class StateManager { Iterator<DuccId> resourceMapIterator = resourceMap.keySet().iterator(); while(resourceMapIterator.hasNext()) { DuccId duccId = resourceMapIterator.next(); + IResource resource = resourceMap.get(duccId); Node node = resourceMap.get(duccId).getNode(); NodeIdentity nodeId = node.getNodeIdentity(); if(!processMap.containsKey(duccId)) { @@ -814,7 +826,8 @@ public class StateManager { break; } DuccProcess process = new DuccProcess(duccId, node, processType); - CGroupManager.assign(process); + long process_max_size_in_bytes = quantum_size_in_bytes * resource.countShares(); + CGroupManager.assign(process, process_max_size_in_bytes); orchestratorCommonArea.getProcessAccounting().addProcess(duccId, duccWorkJob.getDuccId()); processMap.addProcess(process); process.setResourceState(ResourceState.Allocated); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java?rev=1466657&r1=1466656&r2=1466657&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java Wed Apr 10 20:17:38 2013 @@ -28,12 +28,12 @@ public class CGroup implements Serializa private static final long serialVersionUID = 1L; private IDuccId id; - private long maxMemoryLimit; + private long maxMemoryLimit; // in bytes private boolean reservation; private int shares; - public CGroup(IDuccId duccId) { - setId(id); + public CGroup(long max_size_in_bytes) { + setMaxMemoryLimit(max_size_in_bytes); } public int getShares() { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java?rev=1466657&r1=1466656&r2=1466657&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java Wed Apr 10 20:17:38 2013 @@ -25,7 +25,6 @@ import org.apache.uima.ducc.common.Node; import org.apache.uima.ducc.common.NodeIdentity; import org.apache.uima.ducc.common.node.metrics.ProcessGarbageCollectionStats; import org.apache.uima.ducc.common.utils.id.DuccId; -import org.apache.uima.ducc.common.utils.id.IDuccId; import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent; import org.apache.uima.ducc.transport.event.common.IDuccProcessType.ProcessType; import org.apache.uima.ducc.transport.event.common.IProcessState.ProcessState;