Repository: knox Updated Branches: refs/heads/master 691a21212 -> b4455c4c6
KNOX-1193 - Add service discovery support for Spark Thrift UI Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/b4455c4c Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/b4455c4c Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/b4455c4c Branch: refs/heads/master Commit: b4455c4c63b209d8e778d9e88984adf8786cd570 Parents: 691a212 Author: Phil Zampino <[email protected]> Authored: Mon May 7 15:45:55 2018 -0400 Committer: Phil Zampino <[email protected]> Committed: Mon May 7 15:45:55 2018 -0400 ---------------------------------------------------------------------- .../SparkThriftServerUIServiceURLCreator.java | 68 +++++++++++++++++++ ....topology.discovery.ambari.ServiceURLCreator | 3 +- ...iscovery-component-config-mapping.properties | 2 + .../ambari/SparkServiceURLCreatorTest.java | 69 ++++++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java new file mode 100644 index 0000000..3db0883 --- /dev/null +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkThriftServerUIServiceURLCreator.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SparkThriftServerUIServiceURLCreator extends SparkCommonServiceURLCreator { + + private static final String RESOURCE_ROLE = "THRIFTSERVERUI"; + + @Override + public void init(AmbariCluster cluster) { + super.init(cluster); + primaryComponentName = "SPARK_THRIFTSERVER"; + secondaryComponentName = "SPARK2_THRIFTSERVER"; + portConfigProperty = "hive.server2.thrift.http.port"; + } + + @Override + public String getTargetService() { + return RESOURCE_ROLE; + } + + @Override + public List<String> create(String service, Map<String, String> serviceParams) { + List<String> thriftURLs = new ArrayList<>(); + + // Discover the URLs using the common mechanism + List<String> urls = super.create(service, serviceParams); + + // If at least one URL was discovered, validate the transport mode, and append the endpoint path if configured + if (urls != null && !urls.isEmpty()) { + AmbariComponent comp = cluster.getComponent(primaryComponentName); + if (comp == null) { + comp = cluster.getComponent(secondaryComponentName); + } + + if (comp != null) { + String transportMode = comp.getConfigProperty("hive.server2.transport.mode"); + if (transportMode.equalsIgnoreCase("http")) { + String endpoint = comp.getConfigProperty("hive.server2.http.endpoint"); + for (String url : urls) { + thriftURLs.add(url + (endpoint != null ? "/" + endpoint : "")); + } + } + } + } + + return thriftURLs; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator index 89edbfc..f385ae6 100644 --- a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator +++ b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator @@ -23,4 +23,5 @@ org.apache.knox.gateway.topology.discovery.ambari.ResourceManagerURLCreator org.apache.knox.gateway.topology.discovery.ambari.YarnUIURLCreator org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator org.apache.knox.gateway.topology.discovery.ambari.SparkHistoryUIServiceURLCreator -org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator \ No newline at end of file +org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator +org.apache.knox.gateway.topology.discovery.ambari.SparkThriftServerUIServiceURLCreator http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties index 884e143..de3474a 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties @@ -40,6 +40,8 @@ RESOURCEMANAGER=yarn-site SPARK_JOBHISTORYSERVER=spark-defaults SPARK2_JOBHISTORYSERVER=spark2-defaults STORM_UI_SERVER=storm-site +SPARK_THRIFTSERVER=spark-hive-site-override +SPARK2_THRIFTSERVER=spark2-hive-site-override WEBHCAT_SERVER=webhcat-site ZEPPELIN_MASTER=zeppelin-config http://git-wip-us.apache.org/repos/asf/knox/blob/b4455c4c/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java index e12b653..9ee6dc8 100644 --- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java @@ -25,6 +25,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class SparkServiceURLCreatorTest { @@ -157,4 +158,72 @@ public class SparkServiceURLCreatorTest { } + @Test + public void testSparkThriftUI() { + doTestSparkThriftUI("SPARK_THRIFTSERVER", null); + } + + @Test + public void testSpark2ThriftUI() { + doTestSparkThriftUI("SPARK2_THRIFTSERVER", null); + } + + @Test + public void testSparkThriftUIWithEndpointPath() { doTestSparkThriftUIWithEndpointPath("SPARK_THRIFTSERVER"); } + + @Test + public void testSpark2ThriftUIWithEndpointPath() { doTestSparkThriftUIWithEndpointPath("SPARK2_THRIFTSERVER"); } + + @Test + public void testSparkThriftUIBinaryTransport() { doTestSparkThriftUIBinaryTransport("SPARK_THRIFTSERVER"); } + + @Test + public void testSpark2ThriftUIBinaryTransport() { doTestSparkThriftUIBinaryTransport("SPARK2_THRIFTSERVER"); } + + + private void doTestSparkThriftUIWithEndpointPath(String componentName) { + doTestSparkThriftUI(componentName, "http", "mypath"); + } + + private void doTestSparkThriftUIBinaryTransport(String componentName) { + doTestSparkThriftUI(componentName, "binary", null); + } + + + private void doTestSparkThriftUI(String componentName, String endpointPath) { + doTestSparkThriftUI(componentName, "http", endpointPath); + } + + + private void doTestSparkThriftUI(String componentName, String transportMode, String endpointPath) { + final String PORT = "4545"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + List<String> hostNames = Arrays.asList("host1", "host2"); + EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes(); + EasyMock.expect(ac.getConfigProperty("hive.server2.thrift.http.port")).andReturn(PORT).anyTimes(); + EasyMock.expect(ac.getConfigProperty("hive.server2.transport.mode")).andReturn(transportMode).anyTimes(); + EasyMock.expect(ac.getConfigProperty("hive.server2.http.endpoint")).andReturn(endpointPath).anyTimes(); + EasyMock.replay(ac); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes(); + EasyMock.replay(cluster); + + SparkThriftServerUIServiceURLCreator c = new SparkThriftServerUIServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("THRIFTSERVERUI", null); + assertNotNull(urls); + + if ("http".equalsIgnoreCase(transportMode)) { + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("http://host1:"; + PORT + (endpointPath != null ? "/" + endpointPath : ""), urls.get(0)); + assertEquals("http://host2:"; + PORT + (endpointPath != null ? "/" + endpointPath : ""), urls.get(1)); + } else { + assertTrue(urls.isEmpty()); + } + } + + }
