Repository: hadoop Updated Branches: refs/heads/branch-2 a5511debd -> 2c218ca8a refs/heads/branch-2.8 606547dbd -> 5f68f640a refs/heads/trunk 27b77751c -> 140cb5d74
YARN-4709. NMWebServices produces incorrect JSON for containers. Contributed by Varun Saxena. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/140cb5d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/140cb5d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/140cb5d7 Branch: refs/heads/trunk Commit: 140cb5d74565aad848b40a251ec02b597a3a20d4 Parents: 27b7775 Author: Varun Vasudev <[email protected]> Authored: Tue Feb 23 12:29:25 2016 +0530 Committer: Varun Vasudev <[email protected]> Committed: Tue Feb 23 12:29:25 2016 +0530 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/webapp/WebServicesTestUtils.java | 28 ++++++++++++++ .../nodemanager/webapp/dao/ContainerInfo.java | 1 - .../webapp/TestNMWebServicesContainers.java | 39 +++++++++++++++++--- 4 files changed, 65 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9b57666..058ea87 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1436,6 +1436,9 @@ Release 2.8.0 - UNRELEASED YARN-4386. refreshNodesGracefully() should send recommission event to active RMNodes only. (Kuhu Shukla via junping_du) + YARN-4709. NMWebServices produces incorrect JSON for containers. + (Varun Saxena via vvasudev) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java index aa211a6..0454ce6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.webapp; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.List; + import org.w3c.dom.Attr; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -47,6 +50,31 @@ public class WebServicesTestUtils { return Float.parseFloat(val); } + public static List<String> getXmlStrings(Element element, String name) { + NodeList id = element.getElementsByTagName(name); + List<String> strings = new ArrayList<>(); + int len = id.getLength(); + if (id.getLength() == 0) { + return strings; + } + for (int i = 0; i < len; i++) { + Element line = (Element) id.item(i); + if (line == null) { + continue; + } + Node first = line.getFirstChild(); + if (first == null) { + continue; + } + String val = first.getNodeValue(); + if (val == null) { + continue; + } + strings.add(val); + } + return strings; + } + public static String getXmlString(Element element, String name) { NodeList id = element.getElementsByTagName(name); Element line = (Element) id.item(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index e462eba..cf022b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -53,7 +53,6 @@ public class ContainerInfo { @XmlTransient protected String exitStatus; - @XmlElementWrapper protected List<String> containerLogFiles; public ContainerInfo() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 0ed56d3..3c4a660 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -20,13 +20,16 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.ujoin; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; @@ -48,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.ResourceView; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -203,12 +207,27 @@ public class TestNMWebServicesContainers extends JerseyTestBase { app.getAppId(), 1); Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 1); + ((MockContainer)container1).setState(ContainerState.RUNNING); Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2); + ((MockContainer)container2).setState(ContainerState.RUNNING); nmContext.getContainers() .put(container1.getContainerId(), container1); nmContext.getContainers() .put(container2.getContainerId(), container2); + File appDir = new File(testLogDir + "/" + app.getAppId().toString()); + appDir.mkdir(); + File container1Dir = + new File(appDir + "/" + container1.getContainerId().toString()); + container1Dir.mkdir(); + // Create log files for containers. + new File(container1Dir + "/" + "syslog").createNewFile(); + new File(container1Dir + "/" + "stdout").createNewFile(); + File container2Dir = + new File(appDir + "/" + container2.getContainerId().toString()); + container2Dir.mkdir(); + new File(container2Dir + "/" + "syslog").createNewFile(); + new File(container2Dir + "/" + "stdout").createNewFile(); app.getContainers().put(container1.getContainerId(), container1); app.getContainers().put(container2.getContainerId(), container2); @@ -475,9 +494,13 @@ public class TestNMWebServicesContainers extends JerseyTestBase { WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.getXmlString(element, "containerLogsLink")); // verify that the container log files element exists - assertTrue("containerLogFiles missing", - WebServicesTestUtils.getXmlString(element, "containerLogFiles") - != null); + List<String> containerLogFiles = + WebServicesTestUtils.getXmlStrings(element, "containerLogFiles"); + assertFalse("containerLogFiles missing",containerLogFiles.isEmpty()); + assertEquals(2, containerLogFiles.size()); + assertTrue("syslog and stdout expected", + containerLogFiles.contains("syslog") && + containerLogFiles.contains("stdout")); } } @@ -492,8 +515,14 @@ public class TestNMWebServicesContainers extends JerseyTestBase { info.getInt("totalVCoresNeeded"), info.getString("containerLogsLink")); // verify that the container log files element exists - assertTrue("containerLogFiles missing", - info.getJSONArray("containerLogFiles") != null); + JSONArray containerLogFilesArr = info.getJSONArray("containerLogFiles"); + assertTrue("containerLogFiles missing", containerLogFilesArr != null); + assertEquals(2, containerLogFilesArr.length()); + for (int i = 0; i < 2; i++) { + assertTrue("syslog and stdout expected", + containerLogFilesArr.get(i).equals("syslog") || + containerLogFilesArr.get(i).equals("stdout")); + } } public void verifyNodeContainerInfoGeneric(Container cont, String id,
