Repository: hadoop Updated Branches: refs/heads/branch-2 598329e6f -> 42e873d4e
YARN-6625. yarn application -list returns a tracking URL for AM that doesn't work in secured and HA environment. (Yufei Gu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42e873d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42e873d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42e873d4 Branch: refs/heads/branch-2 Commit: 42e873d4eaa48b1d2c9bf122d2fafdbe4b89a913 Parents: 598329e Author: Wangda Tan <[email protected]> Authored: Tue Oct 3 12:04:56 2017 -0700 Committer: Wangda Tan <[email protected]> Committed: Tue Oct 3 12:04:56 2017 -0700 ---------------------------------------------------------------------- .../server/webproxy/amfilter/AmIpFilter.java | 61 ++++++++++++---- .../server/webproxy/amfilter/TestAmFilter.java | 73 ++++++++++++++++---- 2 files changed, 108 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e873d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index 6579191..6febd2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -23,10 +23,12 @@ import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URL; import java.net.UnknownHostException; +import java.net.HttpURLConnection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.Collection; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -38,12 +40,12 @@ import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; -import org.apache.hadoop.yarn.util.RMHAUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +68,8 @@ public class AmIpFilter implements Filter { private String[] proxyHosts; private Set<String> proxyAddresses = null; private long lastUpdate; - private Map<String, String> proxyUriBases; + @VisibleForTesting + Map<String, String> proxyUriBases; @Override public void init(FilterConfig conf) throws ServletException { @@ -187,24 +190,56 @@ public class AmIpFilter implements Filter { } } - protected String findRedirectUrl() throws ServletException { - String addr; - if (proxyUriBases.size() == 1) { // external proxy or not RM HA + @VisibleForTesting + public String findRedirectUrl() throws ServletException { + String addr = null; + if (proxyUriBases.size() == 1) { + // external proxy or not RM HA addr = proxyUriBases.values().iterator().next(); - } else { // RM HA + } else { + // RM HA YarnConfiguration conf = new YarnConfiguration(); - String activeRMId = RMHAUtils.findActiveRMHAId(conf); - String addressPropertyPrefix = YarnConfiguration.useHttps(conf) - ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS - : YarnConfiguration.RM_WEBAPP_ADDRESS; - String host = conf.get( - HAUtil.addSuffix(addressPropertyPrefix, activeRMId)); - addr = proxyUriBases.get(host); + for (String rmId : getRmIds(conf)) { + String url = getUrlByRmId(conf, rmId); + if (isValidUrl(url)) { + addr = url; + break; + } + } } + if (addr == null) { throw new ServletException( "Could not determine the proxy server for redirection"); } return addr; } + + @VisibleForTesting + Collection<String> getRmIds(YarnConfiguration conf) { + return conf.getStringCollection(YarnConfiguration.RM_HA_IDS); + } + + @VisibleForTesting + String getUrlByRmId(YarnConfiguration conf, String rmId) { + String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ? + YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS : + YarnConfiguration.RM_WEBAPP_ADDRESS; + String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId)); + return proxyUriBases.get(host); + } + + @VisibleForTesting + public boolean isValidUrl(String url) { + boolean isValid = false; + try { + HttpURLConnection conn = + (HttpURLConnection) new URL(url).openConnection(); + conn.connect(); + isValid = conn.getResponseCode() == HttpURLConnection.HTTP_OK; + } catch (Exception e) { + LOG.debug("Failed to connect to " + url + ": " + e.toString()); + } + return isValid; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42e873d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java index b788f5d..12bda8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java @@ -18,25 +18,42 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.glassfish.grizzly.servlet.HttpServletResponseImpl; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.net.HttpURLConnection; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import javax.servlet.*; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import static org.junit.Assert.*; - -import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; -import org.glassfish.grizzly.servlet.HttpServletResponseImpl; -import org.junit.Test; -import org.mockito.Mockito; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Test AmIpFilter. Requests to a no declared hosts should has way through @@ -121,6 +138,36 @@ public class TestAmFilter { filter.destroy(); } + @Test + public void testFindRedirectUrl() throws Exception { + final String rm1 = "rm1"; + final String rm2 = "rm2"; + + final String rm1Url = "host1:8088"; + final String rm2Url = "host2:8088"; + + TestAmIpFilter filter = new TestAmIpFilter(); + TestAmIpFilter spy = Mockito.spy(filter); + // make sure findRedirectUrl() go to HA branch + spy.proxyUriBases = new HashMap<>(); + spy.proxyUriBases.put(rm1, rm1Url); + spy.proxyUriBases.put(rm2, rm2Url); + + Collection<String> rmIds = new ArrayList<>(Arrays.asList(rm1, rm2)); + Mockito.doReturn(rmIds).when(spy).getRmIds( + Mockito.any(YarnConfiguration.class)); + Mockito.doReturn(rm1Url).when(spy).getUrlByRmId( + Mockito.any(YarnConfiguration.class), Mockito.eq(rm2)); + Mockito.doReturn(rm2Url).when(spy) + .getUrlByRmId(Mockito.any(YarnConfiguration.class), Mockito.eq(rm1)); + + // Stub "isValidUrl" and returns false for rm1, returns true for rm2. + Mockito.doReturn(false).when(spy).isValidUrl(Mockito.eq(rm1Url)); + Mockito.doReturn(true).when(spy).isValidUrl(Mockito.eq(rm2Url)); + + assertEquals(spy.findRedirectUrl(), rm2Url); + } + /** * Test AmIpFilter */ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
