Repository: incubator-myriad Updated Branches: refs/heads/master dcccf2e3b -> df4cbc0e9
MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods which are only used in unit tests JIRA: [MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218 Pull Request: Closes #87 Author: ponnu <ponnulinga...@gmail.com> Date: Mon Jul 25 21:03:15 2016 -0400 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/df4cbc0e Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/df4cbc0e Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/df4cbc0e Branch: refs/heads/master Commit: df4cbc0e97024cff4064e58fa792b3cbf3eee4f3 Parents: dcccf2e Author: ponnu <ponnulinga...@gmail.com> Authored: Mon Jul 25 21:03:15 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Fri Jul 29 16:33:39 2016 -0400 ---------------------------------------------------------------------- .../org/apache/myriad/scheduler/TaskUtils.java | 162 +------------------ .../apache/myriad/scheduler/TestTaskUtils.java | 33 +--- 2 files changed, 13 insertions(+), 182 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java index c8e2a21..6ab6ee0 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java @@ -18,60 +18,28 @@ */ package org.apache.myriad.scheduler; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; - +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; import org.apache.mesos.Protos; -import org.apache.myriad.configuration.MyriadBadConfigurationException; import org.apache.myriad.configuration.MyriadConfiguration; import org.apache.myriad.configuration.MyriadContainerConfiguration; import org.apache.myriad.configuration.MyriadDockerConfiguration; -import org.apache.myriad.configuration.NodeManagerConfiguration; -import org.apache.myriad.configuration.ServiceConfiguration; import org.apache.myriad.executor.MyriadExecutorDefaults; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * utility class for working with tasks and node manager profiles */ public class TaskUtils { private static final Logger LOGGER = LoggerFactory.getLogger(TaskUtils.class); - private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES = "yarn.nodemanager.resource.cpu-vcores"; - private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB = "yarn.nodemanager.resource.memory-mb"; private static final String CONTAINER_PATH_KEY = "containerPath"; private static final String HOST_PATH_KEY = "hostPath"; private static final String RW_MODE = "mode"; @@ -79,90 +47,12 @@ public class TaskUtils { private static final String PARAMETER_VALUE_KEY = "value"; private MyriadConfiguration cfg; - Random random = new Random(); @Inject public TaskUtils(MyriadConfiguration cfg) { this.cfg = cfg; } - public static String getRevisedConfig(Double cpu, Double memory) { - String revisedConfig = ""; - try { - - // todo:(kgs) replace with more abstract xml parser - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - DocumentBuilder builder; - Document doc; - - builder = factory.newDocumentBuilder(); - InputStream resourceAsStream = TaskUtils.class.getClassLoader().getResourceAsStream("yarn-site.xml"); - - doc = builder.parse(new InputSource(resourceAsStream)); - resourceAsStream.close(); - - XPathFactory xFactory = XPathFactory.newInstance(); - - XPath xpath = xFactory.newXPath(); - XPathExpression cpuXpath = xpath.compile("//property/name"); - Object cpuNodeObj = cpuXpath.evaluate(doc, XPathConstants.NODESET); - - NodeList cpuNode = (NodeList) cpuNodeObj; - - for (int i = 0; i < cpuNode.getLength(); i++) { - Node item = cpuNode.item(i); - if (YARN_NODEMANAGER_RESOURCE_CPU_VCORES.equals(item.getTextContent())) { - Node propertyNode = item.getParentNode(); - NodeList childNodes = propertyNode.getChildNodes(); - for (int j = 0; j < childNodes.getLength(); j++) { - Node item2 = childNodes.item(j); - if ("value".equals(item2.getNodeName())) { - item2.setTextContent(cpu.intValue() + ""); - } - } - } else if (YARN_NODEMANAGER_RESOURCE_MEMORY_MB.equals(item.getTextContent())) { - Node propertyNode = item.getParentNode(); - NodeList childNodes = propertyNode.getChildNodes(); - for (int j = 0; j < childNodes.getLength(); j++) { - Node item2 = childNodes.item(j); - if ("value".equals(item2.getNodeName())) { - item2.setTextContent(memory.intValue() + ""); - } - } - } - } - - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - StringWriter writer = new StringWriter(); - transformer.transform(new DOMSource(doc), new StreamResult(writer)); - - revisedConfig = writer.getBuffer().toString().replaceAll("\n|\r", ""); - } catch (TransformerConfigurationException e) { - e.printStackTrace(); - } catch (TransformerException | SAXException | XPathExpressionException | ParserConfigurationException e) { - LOGGER.error("Error with xml operation", e); - } catch (IOException e) { - LOGGER.error("Error with xml operation", e); - } - return revisedConfig; - } - - public double getAggregateMemory(NMProfile profile) { - double totalTaskMemory; - double executorMemory = getExecutorMemory(); - double nmJvmMaxMemoryMB = getNodeManagerMemory(); - double advertisableMemory = profile.getMemory(); - totalTaskMemory = executorMemory + nmJvmMaxMemoryMB + advertisableMemory; - return totalTaskMemory; - } - - public double getAggregateCpus(NMProfile profile) { - return getNodeManagerCpus() + MyriadExecutorDefaults.DEFAULT_CPUS + profile.getCpus(); - } - public double getNodeManagerMemory() { return cfg.getNodeManagerConfiguration().getJvmMaxMemoryMB(); } @@ -180,42 +70,6 @@ public class TaskUtils { return cfg.getMyriadExecutorConfiguration().getJvmMaxMemoryMB(); } - public double getTaskCpus(NMProfile profile) { - - return getAggregateCpus(profile) - getExecutorCpus(); - } - - public double getTaskMemory(NMProfile profile) { - - return getAggregateMemory(profile) - getExecutorMemory(); - } - - public double getAuxTaskCpus(NMProfile profile, String taskName) throws MyriadBadConfigurationException { - if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) { - return getAggregateCpus(profile); - } - - Optional<ServiceConfiguration> auxConf = cfg.getServiceConfiguration(taskName); - if (!auxConf.isPresent()) { - throw new MyriadBadConfigurationException("Can not find profile for task name: " + taskName); - } - - return auxConf.get().getCpus(); - } - - public double getAuxTaskMemory(NMProfile profile, String taskName) throws MyriadBadConfigurationException { - if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) { - return getAggregateMemory(profile); - } - - Optional<ServiceConfiguration> auxConf = cfg.getServiceConfiguration(taskName); - if (!auxConf.isPresent()) { - throw new MyriadBadConfigurationException("Cannot find profile for task name: " + taskName); - } - - return auxConf.get().getJvmMaxMemoryMB(); - } - public TaskUtils() { super(); } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java index 29d7691..d543c29 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java @@ -17,18 +17,15 @@ */ package org.apache.myriad.scheduler; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.apache.mesos.Protos; -import org.apache.myriad.BaseConfigurableTest; -import org.apache.myriad.configuration.MyriadBadConfigurationException; -import org.junit.Test; - import com.google.common.collect.Range; import com.google.common.collect.Ranges; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.mesos.Protos; +import org.apache.myriad.BaseConfigurableTest; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; /** * Tests for TaskUtils @@ -37,26 +34,6 @@ public class TestTaskUtils extends BaseConfigurableTest { static double epsilon = .0001; @Test - public void testGetResource() { - TaskUtils taskUtils = new TaskUtils(cfg); - - NMProfile fooProfile = new NMProfile("abc", 1L, 1000L); - try { - taskUtils.getAuxTaskCpus(fooProfile, "foo"); - fail("Should not complete sucessfully for foo"); - } catch (MyriadBadConfigurationException e) { - // success - } - - try { - double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory"); - assertTrue(cpu > 0.0); - } catch (MyriadBadConfigurationException e) { - fail("cpu should be defined for jobhistory"); - } - } - - @Test public void testServiceResourceProfile() throws Exception { // testing custom deserializer