YARN-6314. Potential infinite redirection on YARN log redirection web service. Contributed by Xuan Gong.
(cherry picked from commit 5a9dda796f0e73060ada794ad5752cc6a237ab2e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34424e98 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34424e98 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34424e98 Branch: refs/heads/YARN-5734 Commit: 34424e98a618a9fefce800746168be2b72e17de9 Parents: 023b941 Author: Junping Du <junping...@apache.org> Authored: Tue Mar 14 02:56:18 2017 -0700 Committer: Junping Du <junping...@apache.org> Committed: Tue Mar 14 02:58:07 2017 -0700 ---------------------------------------------------------------------- .../webapp/AHSWebServices.java | 32 +++++++++++++++----- .../webapp/TestAHSWebServices.java | 17 +++++++++++ .../server/webapp/YarnWebServiceParams.java | 1 + .../nodemanager/webapp/NMWebServices.java | 6 +++- .../nodemanager/webapp/TestNMWebServices.java | 6 ++++ 5 files changed, 54 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/34424e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index c296aaa..6195199 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -28,6 +28,7 @@ import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -235,6 +236,8 @@ public class AHSWebServices extends WebServices { * The container ID * @param nmId * The Node Manager NodeId + * @param redirected_from_node + * Whether this is a redirected request from NM * @return * The log file's name and current file size */ @@ -245,7 +248,9 @@ public class AHSWebServices extends WebServices { @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { ContainerId containerId = null; init(res); try { @@ -253,6 +258,7 @@ public class AHSWebServices extends WebServices { } catch (IllegalArgumentException e) { throw new BadRequestException("invalid container id, " + containerIdStr); } + ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); AppInfo appInfo; @@ -297,9 +303,12 @@ public class AHSWebServices extends WebServices { // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get log meta for aggregated logs instead of // re-directing the request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirected_from_node) { // return log meta for the aggregated logs if exists. // It will also return empty log meta for the local logs. + // If this is the redirect request from NM, we should not + // re-direct the request back. Simply output the aggregated log meta. return getContainerLogMeta(appId, appOwner, null, containerIdStr, true); } @@ -338,6 +347,8 @@ public class AHSWebServices extends WebServices { * the size of the log file * @param nmId * The Node Manager NodeId + * @param redirected_from_node + * Whether this is the redirect request from NM * @return * The contents of the container's log file */ @@ -352,9 +363,11 @@ public class AHSWebServices extends WebServices { @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + boolean redirected_from_node) { return getLogs(req, res, containerIdStr, filename, format, - size, nmId); + size, nmId, redirected_from_node); } //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and @@ -371,7 +384,9 @@ public class AHSWebServices extends WebServices { @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { init(res); ContainerId containerId; try { @@ -426,8 +441,11 @@ public class AHSWebServices extends WebServices { nodeHttpAddress = containerInfo.getNodeHttpAddress(); // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get aggregated logs instead of re-directing the - // request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + // request. + // If this is the redirect request from NM, we should not re-direct the + // request back. Simply output the aggregated logs. + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirected_from_node) { // output the aggregated logs return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/34424e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index c2cfb3b..dc692a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -771,6 +771,23 @@ public class TestAHSWebServices extends JerseyTestBase { assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL)); assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning())); + + // If this is the redirect request, we would not re-direct the request + // back and get the aggregated logs. + String content1 = "Hello." + containerId1; + NodeId nodeId1 = NodeId.fromString(NM_ID); + TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, + rootLogDir, containerId1, nodeId1, fileName, user, content1, true); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs").path(fileName) + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE, "true") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains(content1)); + assertTrue(responseText.contains("LogAggregationType: " + + ContainerLogAggregationType.AGGREGATED)); } @Test(timeout = 10000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/34424e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 87e540f..479cc75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -34,4 +34,5 @@ public interface YarnWebServiceParams { String RESPONSE_CONTENT_FORMAT = "format"; String RESPONSE_CONTENT_SIZE = "size"; String NM_ID = "nm.id"; + String REDIRECTED_FROM_NODE = "redirected_from_node"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/34424e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 4e72746..04a889f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -499,7 +499,11 @@ public class NMWebServices { String requestParams = WebAppUtils.removeQueryParams(httpRequest, YarnWebServiceParams.NM_ID); if (requestParams != null && !requestParams.isEmpty()) { - redirectPath.append("?" + requestParams); + redirectPath.append("?" + requestParams + "&" + + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true"); + } else { + redirectPath.append("?" + YarnWebServiceParams.REDIRECTED_FROM_NODE + + "=true"); } ResponseBuilder res = Response.status( HttpServletResponse.SC_TEMPORARY_REDIRECT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/34424e98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index d9fd289..1641171 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -383,6 +383,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); // check the new api @@ -397,6 +399,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); requestURI = r.path("ws").path("v1").path("node") @@ -409,6 +413,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
