Repository: incubator-myriad
Updated Branches:
refs/heads/master dcccf2e3b -> df4cbc0e9
MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods
which are only used in unit tests
JIRA:
[MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218
Pull Request:
Closes #87
Author: ponnu <ponnulinga...@gmail.com>
Date: Mon Jul 25 21:03:15 2016 -0400
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/df4cbc0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/df4cbc0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/df4cbc0e
Branch: refs/heads/master
Commit: df4cbc0e97024cff4064e58fa792b3cbf3eee4f3
Parents: dcccf2e
Author: ponnu <ponnulinga...@gmail.com>
Authored: Mon Jul 25 21:03:15 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Fri Jul 29 16:33:39 2016 -0400
----------------------------------------------------------------------
.../org/apache/myriad/scheduler/TaskUtils.java | 162 +------------------
.../apache/myriad/scheduler/TestTaskUtils.java | 33 +---
2 files changed, 13 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
----------------------------------------------------------------------
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
index c8e2a21..6ab6ee0 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
@@ -18,60 +18,28 @@
*/
package org.apache.myriad.scheduler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import org.apache.mesos.Protos;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
import org.apache.myriad.configuration.MyriadConfiguration;
import org.apache.myriad.configuration.MyriadContainerConfiguration;
import org.apache.myriad.configuration.MyriadDockerConfiguration;
-import org.apache.myriad.configuration.NodeManagerConfiguration;
-import org.apache.myriad.configuration.ServiceConfiguration;
import org.apache.myriad.executor.MyriadExecutorDefaults;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
* utility class for working with tasks and node manager profiles
*/
public class TaskUtils {
private static final Logger LOGGER =
LoggerFactory.getLogger(TaskUtils.class);
- private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES =
"yarn.nodemanager.resource.cpu-vcores";
- private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB =
"yarn.nodemanager.resource.memory-mb";
private static final String CONTAINER_PATH_KEY = "containerPath";
private static final String HOST_PATH_KEY = "hostPath";
private static final String RW_MODE = "mode";
@@ -79,90 +47,12 @@ public class TaskUtils {
private static final String PARAMETER_VALUE_KEY = "value";
private MyriadConfiguration cfg;
- Random random = new Random();
@Inject
public TaskUtils(MyriadConfiguration cfg) {
this.cfg = cfg;
}
- public static String getRevisedConfig(Double cpu, Double memory) {
- String revisedConfig = "";
- try {
-
- // todo:(kgs) replace with more abstract xml parser
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- factory.setNamespaceAware(true);
- DocumentBuilder builder;
- Document doc;
-
- builder = factory.newDocumentBuilder();
- InputStream resourceAsStream =
TaskUtils.class.getClassLoader().getResourceAsStream("yarn-site.xml");
-
- doc = builder.parse(new InputSource(resourceAsStream));
- resourceAsStream.close();
-
- XPathFactory xFactory = XPathFactory.newInstance();
-
- XPath xpath = xFactory.newXPath();
- XPathExpression cpuXpath = xpath.compile("//property/name");
- Object cpuNodeObj = cpuXpath.evaluate(doc, XPathConstants.NODESET);
-
- NodeList cpuNode = (NodeList) cpuNodeObj;
-
- for (int i = 0; i < cpuNode.getLength(); i++) {
- Node item = cpuNode.item(i);
- if
(YARN_NODEMANAGER_RESOURCE_CPU_VCORES.equals(item.getTextContent())) {
- Node propertyNode = item.getParentNode();
- NodeList childNodes = propertyNode.getChildNodes();
- for (int j = 0; j < childNodes.getLength(); j++) {
- Node item2 = childNodes.item(j);
- if ("value".equals(item2.getNodeName())) {
- item2.setTextContent(cpu.intValue() + "");
- }
- }
- } else if
(YARN_NODEMANAGER_RESOURCE_MEMORY_MB.equals(item.getTextContent())) {
- Node propertyNode = item.getParentNode();
- NodeList childNodes = propertyNode.getChildNodes();
- for (int j = 0; j < childNodes.getLength(); j++) {
- Node item2 = childNodes.item(j);
- if ("value".equals(item2.getNodeName())) {
- item2.setTextContent(memory.intValue() + "");
- }
- }
- }
- }
-
- TransformerFactory tf = TransformerFactory.newInstance();
- Transformer transformer = tf.newTransformer();
- transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
- StringWriter writer = new StringWriter();
- transformer.transform(new DOMSource(doc), new StreamResult(writer));
-
- revisedConfig = writer.getBuffer().toString().replaceAll("\n|\r", "");
- } catch (TransformerConfigurationException e) {
- e.printStackTrace();
- } catch (TransformerException | SAXException | XPathExpressionException |
ParserConfigurationException e) {
- LOGGER.error("Error with xml operation", e);
- } catch (IOException e) {
- LOGGER.error("Error with xml operation", e);
- }
- return revisedConfig;
- }
-
- public double getAggregateMemory(NMProfile profile) {
- double totalTaskMemory;
- double executorMemory = getExecutorMemory();
- double nmJvmMaxMemoryMB = getNodeManagerMemory();
- double advertisableMemory = profile.getMemory();
- totalTaskMemory = executorMemory + nmJvmMaxMemoryMB + advertisableMemory;
- return totalTaskMemory;
- }
-
- public double getAggregateCpus(NMProfile profile) {
- return getNodeManagerCpus() + MyriadExecutorDefaults.DEFAULT_CPUS +
profile.getCpus();
- }
-
public double getNodeManagerMemory() {
return cfg.getNodeManagerConfiguration().getJvmMaxMemoryMB();
}
@@ -180,42 +70,6 @@ public class TaskUtils {
return cfg.getMyriadExecutorConfiguration().getJvmMaxMemoryMB();
}
- public double getTaskCpus(NMProfile profile) {
-
- return getAggregateCpus(profile) - getExecutorCpus();
- }
-
- public double getTaskMemory(NMProfile profile) {
-
- return getAggregateMemory(profile) - getExecutorMemory();
- }
-
- public double getAuxTaskCpus(NMProfile profile, String taskName) throws
MyriadBadConfigurationException {
- if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) {
- return getAggregateCpus(profile);
- }
-
- Optional<ServiceConfiguration> auxConf =
cfg.getServiceConfiguration(taskName);
- if (!auxConf.isPresent()) {
- throw new MyriadBadConfigurationException("Can not find profile for task
name: " + taskName);
- }
-
- return auxConf.get().getCpus();
- }
-
- public double getAuxTaskMemory(NMProfile profile, String taskName) throws
MyriadBadConfigurationException {
- if (taskName.startsWith(NodeManagerConfiguration.NM_TASK_PREFIX)) {
- return getAggregateMemory(profile);
- }
-
- Optional<ServiceConfiguration> auxConf =
cfg.getServiceConfiguration(taskName);
- if (!auxConf.isPresent()) {
- throw new MyriadBadConfigurationException("Cannot find profile for task
name: " + taskName);
- }
-
- return auxConf.get().getJvmMaxMemoryMB();
- }
-
public TaskUtils() {
super();
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index 29d7691..d543c29 100644
---
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -17,18 +17,15 @@
*/
package org.apache.myriad.scheduler;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.mesos.Protos;
-import org.apache.myriad.BaseConfigurableTest;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
-import org.junit.Test;
-
import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import org.apache.mesos.Protos;
+import org.apache.myriad.BaseConfigurableTest;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
/**
* Tests for TaskUtils
@@ -37,26 +34,6 @@ public class TestTaskUtils extends BaseConfigurableTest {
static double epsilon = .0001;
@Test
- public void testGetResource() {
- TaskUtils taskUtils = new TaskUtils(cfg);
-
- NMProfile fooProfile = new NMProfile("abc", 1L, 1000L);
- try {
- taskUtils.getAuxTaskCpus(fooProfile, "foo");
- fail("Should not complete sucessfully for foo");
- } catch (MyriadBadConfigurationException e) {
- // success
- }
-
- try {
- double cpu = taskUtils.getAuxTaskCpus(fooProfile, "jobhistory");
- assertTrue(cpu > 0.0);
- } catch (MyriadBadConfigurationException e) {
- fail("cpu should be defined for jobhistory");
- }
- }
-
- @Test
public void testServiceResourceProfile() throws Exception {
// testing custom deserializer
