Repository: knox Updated Branches: refs/heads/master 08cf0cbca -> 2a6d72c34
KNOX-1335 - Spark service discovery when SSL is configured Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/2a6d72c3 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/2a6d72c3 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/2a6d72c3 Branch: refs/heads/master Commit: 2a6d72c34c90faf861b714c4ad3260571fb328a8 Parents: 08cf0cb Author: Phil Zampino <[email protected]> Authored: Fri May 25 15:32:11 2018 -0400 Committer: Phil Zampino <[email protected]> Committed: Fri May 25 15:32:11 2018 -0400 ---------------------------------------------------------------------- .../discovery/ambari/LivyServiceURLCreator.java | 6 + .../ambari/SparkCommonServiceURLCreator.java | 18 ++- .../ambari/SparkHistoryUIServiceURLCreator.java | 32 +++++ .../ambari/SparkServiceURLCreatorTest.java | 118 +++++++++++++++++++ 4 files changed, 171 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java index 6c12b18..c098693 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java @@ -33,4 +33,10 @@ public class LivyServiceURLCreator extends SparkCommonServiceURLCreator { return RESOURCE_ROLE; } + @Override + boolean isSSL(AmbariComponent comp) { + String keystore = comp.getConfigProperty("livy.keystore"); + return (keystore != null && !keystore.isEmpty()); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java index ae223d3..7f98872 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java @@ -22,7 +22,10 @@ import java.util.Map; public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator { - private static final String URL_TEMPLATE = "http://%s:%s";; + private static final String SCHEME_HTTP = "http"; + private static final String SCHEME_HTTPS = "https"; + + private static final String URL_TEMPLATE = "%s://%s:%s"; protected AmbariCluster cluster = null; @@ -37,6 +40,15 @@ public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator this.cluster = cluster; } + boolean isSSL(AmbariComponent comp) { + return false; + } + + String getPort(AmbariComponent comp) { + return comp.getConfigProperty(portConfigProperty); + } + + @Override public List<String> create(String service, Map<String, String> serviceParams) { List<String> urls = new ArrayList<>(); @@ -48,10 +60,10 @@ public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator } if (comp != null) { - String port = comp.getConfigProperty(portConfigProperty); + String port = getPort(comp); List<String> hostNames = comp.getHostNames(); for (String host : hostNames) { - urls.add(String.format(URL_TEMPLATE, host, port)); + urls.add(String.format(URL_TEMPLATE, (isSSL(comp) ? SCHEME_HTTPS : SCHEME_HTTP), host, port)); } } } http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java index a0de1d5..09506c9 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java @@ -20,6 +20,14 @@ public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreato private static final String RESOURCE_ROLE = "SPARKHISTORYUI"; + + private static final String SSL_FLAG_PRIMARY = "spark.ssl.historyServer.enabled"; + private static final String SSL_FLAG_SECONDARY = "spark.ssl.enabled"; + + private static final String SSL_PORT_PROPERTY = "spark.ssl.historyServer.port"; + + private static final int SSL_PORT_OFFSET = 400; + @Override public void init(AmbariCluster cluster) { super.init(cluster); @@ -33,4 +41,28 @@ public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreato return RESOURCE_ROLE; } + + @Override + String getPort(AmbariComponent comp) { + String port; + + if (isSSL(comp)) { + String sslPort = comp.getConfigProperty(SSL_PORT_PROPERTY); + if (sslPort == null || sslPort.isEmpty()) { + int p = Integer.valueOf(comp.getConfigProperty(portConfigProperty)) + SSL_PORT_OFFSET; + sslPort = String.valueOf(p); + } + port = sslPort; + } else { + port = comp.getConfigProperty(portConfigProperty); + } + + return port; + } + + @Override + boolean isSSL(AmbariComponent comp) { + return Boolean.valueOf(comp.getConfigProperty(SSL_FLAG_PRIMARY)) || Boolean.valueOf(comp.getConfigProperty(SSL_FLAG_SECONDARY)); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/2a6d72c3/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java index 9ee6dc8..e346166 100644 --- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java @@ -64,6 +64,90 @@ public class SparkServiceURLCreatorTest { @Test + public void testSparkHistoryUI_SSL() { + doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", true, false, "4321"); + } + + @Test + public void testSparkHistoryUI_SSL_FALLBACK_FLAG() { + doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", false, true, "4321"); + } + + @Test + public void testSparkHistoryUI_SSL_FALLBACK_PORT() { + doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", true, false, null); + } + + @Test + public void testSparkHistoryUI_SSL_FALLBACK_FLAG_AND_PORT() { + doTestSparkHistoryUI_SSL("SPARK_JOBHISTORYSERVER", false, true, null); + } + + @Test + public void testSpark2HistoryUI_SSL() { + doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", true, false, "4321"); + } + + @Test + public void testSparkwHistoryUI_SSL_FALLBACK_FLAG() { + doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", false, true, "4321"); + } + + @Test + public void testSpark2HistoryUI_SSL_FALLBACK_PORT() { + doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", true, false, null); + } + + @Test + public void testSpark2HistoryUI_SSL_FALLBACK_FLAG_AND_PORT() { + doTestSparkHistoryUI_SSL("SPARK2_JOBHISTORYSERVER", false, true, null); + } + + private void doTestSparkHistoryUI_SSL(String componentName, Boolean sslSHS, Boolean sslSpark, String sslSHSPort) { + final String PORT = "4545"; + + boolean isSSLConfigured = false; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + List<String> hostNames = Arrays.asList("host1", "host2"); + EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes(); + EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes(); + if (sslSHS != null) { + isSSLConfigured = true; + EasyMock.expect(ac.getConfigProperty("spark.ssl.historyServer.enabled")).andReturn(String.valueOf(sslSHS)).anyTimes(); + } + if (sslSpark != null) { + isSSLConfigured = true; + EasyMock.expect(ac.getConfigProperty("spark.ssl.enabled")).andReturn(String.valueOf(sslSpark)).anyTimes(); + } + if (sslSHSPort != null) { + EasyMock.expect(ac.getConfigProperty("spark.ssl.historyServer.port")).andReturn(sslSHSPort).anyTimes(); + } + EasyMock.replay(ac); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes(); + EasyMock.replay(cluster); + + SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("SPARKHISTORYUI", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + + if (isSSLConfigured) { + String expectedPort = sslSHSPort != null ? sslSHSPort : String.valueOf(Integer.valueOf(PORT) + 400); + assertEquals("https://host1:"; + expectedPort, urls.get(0)); + assertEquals("https://host2:"; + expectedPort, urls.get(1)); + } else { + assertEquals("http://host1:"; + PORT, urls.get(0)); + assertEquals("http://host2:"; + PORT, urls.get(1)); + } + } + + + @Test public void testSparkAndSpark2HistoryUI() { final String PORT = "4545"; final String PORT2 = "6767"; @@ -128,6 +212,40 @@ public class SparkServiceURLCreatorTest { } @Test + public void testLivyServer_SSL() { + doTestLivyServerSSL("LIVY_SERVER"); + } + + @Test + public void testLivy2Server_SSL() { + doTestLivyServerSSL("LIVY2_SERVER"); + } + + private void doTestLivyServerSSL(String componentName) { + final String PORT = "4545"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + List<String> hostNames = Arrays.asList("host1", "host2"); + EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes(); + EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes(); + EasyMock.expect(ac.getConfigProperty("livy.keystore")).andReturn("mykeystore").anyTimes(); + EasyMock.replay(ac); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes(); + EasyMock.replay(cluster); + + LivyServiceURLCreator c = new LivyServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("LIVYSERVER", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("https://host1:"; + PORT, urls.get(0)); + assertEquals("https://host2:"; + PORT, urls.get(1)); + } + + @Test public void testLivyAndLivy2Server() { final String PORT = "4545"; final String PORT2 = "2323";
