Repository: knox Updated Branches: refs/heads/master 11796c96b -> 691a21212
KNOX-1293 - Support for Ambari discovery of SPARK2 services Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/691a2121 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/691a2121 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/691a2121 Branch: refs/heads/master Commit: 691a212129fbf50a091d2b5c05e070d88929eaf8 Parents: 11796c9 Author: Phil Zampino <pzamp...@apache.org> Authored: Tue May 1 22:57:22 2018 -0400 Committer: Phil Zampino <pzamp...@apache.org> Committed: Mon May 7 09:35:16 2018 -0400 ---------------------------------------------------------------------- .../discovery/ambari/LivyServiceURLCreator.java | 36 +++++ .../ambari/SparkCommonServiceURLCreator.java | 64 ++++++++ .../ambari/SparkHistoryUIServiceURLCreator.java | 36 +++++ ....topology.discovery.ambari.ServiceURLCreator | 4 +- ...iscovery-component-config-mapping.properties | 2 + .../ambari-service-discovery-url-mappings.xml | 28 ---- .../ambari/SparkServiceURLCreatorTest.java | 160 +++++++++++++++++++ 7 files changed, 301 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java new file mode 100644 index 0000000..6c12b18 --- /dev/null +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/LivyServiceURLCreator.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +public class LivyServiceURLCreator extends SparkCommonServiceURLCreator { + + private static final String RESOURCE_ROLE = "LIVYSERVER"; + + @Override + public void init(AmbariCluster cluster) { + super.init(cluster); + primaryComponentName = "LIVY_SERVER"; + secondaryComponentName = "LIVY2_SERVER"; + portConfigProperty = "livy.server.port"; + } + + @Override + public String getTargetService() { + return RESOURCE_ROLE; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java new file mode 100644 index 0000000..ae223d3 --- /dev/null +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkCommonServiceURLCreator.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class SparkCommonServiceURLCreator implements ServiceURLCreator { + + private static final String URL_TEMPLATE = "http://%s:%s";; + + protected AmbariCluster cluster = null; + + String primaryComponentName = null; + + String secondaryComponentName = null; + + String portConfigProperty = null; + + @Override + public void init(AmbariCluster cluster) { + this.cluster = cluster; + } + + @Override + public List<String> create(String service, Map<String, String> serviceParams) { + List<String> urls = new ArrayList<>(); + + if (getTargetService().equalsIgnoreCase(service)) { + AmbariComponent comp = cluster.getComponent(primaryComponentName); + if (comp == null) { + comp = cluster.getComponent(secondaryComponentName); + } + + if (comp != null) { + String port = comp.getConfigProperty(portConfigProperty); + List<String> hostNames = comp.getHostNames(); + for (String host : hostNames) { + urls.add(String.format(URL_TEMPLATE, host, port)); + } + } + } + + return urls; + } + + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java new file mode 100644 index 0000000..a0de1d5 --- /dev/null +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/SparkHistoryUIServiceURLCreator.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +public class SparkHistoryUIServiceURLCreator extends SparkCommonServiceURLCreator { + + private static final String RESOURCE_ROLE = "SPARKHISTORYUI"; + + @Override + public void init(AmbariCluster cluster) { + super.init(cluster); + primaryComponentName = "SPARK_JOBHISTORYSERVER"; + secondaryComponentName = "SPARK2_JOBHISTORYSERVER"; + portConfigProperty = "spark.history.ui.port"; + } + + @Override + public String getTargetService() { + return RESOURCE_ROLE; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator index a6e2a7e..89edbfc 100644 --- a/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator +++ b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.ambari.ServiceURLCreator @@ -21,4 +21,6 @@ org.apache.knox.gateway.topology.discovery.ambari.WebHdfsUrlCreator org.apache.knox.gateway.topology.discovery.ambari.HdfsUIUrlCreator org.apache.knox.gateway.topology.discovery.ambari.ResourceManagerURLCreator org.apache.knox.gateway.topology.discovery.ambari.YarnUIURLCreator -org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator \ No newline at end of file +org.apache.knox.gateway.topology.discovery.ambari.YarnUIV2URLCreator +org.apache.knox.gateway.topology.discovery.ambari.SparkHistoryUIServiceURLCreator +org.apache.knox.gateway.topology.discovery.ambari.LivyServiceURLCreator \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties index bf53db0..884e143 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties @@ -32,11 +32,13 @@ HISTORYSERVER=mapred-site HIVE_SERVER=hive-site INFRA_SOLR=infra-solr-env LIVY_SERVER=livy-conf +LIVY2_SERVER=livy2-conf NAMENODE=hdfs-site OOZIE_SERVER=oozie-site RANGER_ADMIN=ranger-admin-site RESOURCEMANAGER=yarn-site SPARK_JOBHISTORYSERVER=spark-defaults +SPARK2_JOBHISTORYSERVER=spark2-defaults STORM_UI_SERVER=storm-site WEBHCAT_SERVER=webhcat-site ZEPPELIN_MASTER=zeppelin-config http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml index a1f1c88..a3e1def 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml @@ -485,34 +485,6 @@ </properties> </service> - <service name="SPARKHISTORYUI"> - <url-pattern>http://{HOST}:{PORT}</url-pattern> - <properties> - <property name="HOST"> - <component>SPARK_JOBHISTORYSERVER</component> - <hostname/> - </property> - <property name="PORT"> - <component>SPARK_JOBHISTORYSERVER</component> - <config-property>spark.history.ui.port</config-property> - </property> - </properties> - </service> - - <service name="LIVYSERVER"> - <url-pattern>http://{HOST}:{PORT}</url-pattern> - <properties> - <property name="HOST"> - <component>LIVY_SERVER</component> - <hostname/> - </property> - <property name="PORT"> - <component>LIVY_SERVER</component> - <config-property>livy.server.port</config-property> - </property> - </properties> - </service> - <service name="SOLR"> <url-pattern>http://{HOST}:{PORT}</url-pattern> <properties> http://git-wip-us.apache.org/repos/asf/knox/blob/691a2121/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java new file mode 100644 index 0000000..e12b653 --- /dev/null +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/SparkServiceURLCreatorTest.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import org.easymock.EasyMock; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +public class SparkServiceURLCreatorTest { + + @Test + public void testSparkHistoryUI() { + doTestSparkHistoryUI("SPARK_JOBHISTORYSERVER"); + } + + @Test + public void testSpark2HistoryUI() { + doTestSparkHistoryUI("SPARK2_JOBHISTORYSERVER"); + } + + private void doTestSparkHistoryUI(String componentName) { + final String PORT = "4545"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + List<String> hostNames = Arrays.asList("host1", "host2"); + EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes(); + EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes(); + EasyMock.replay(ac); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes(); + EasyMock.replay(cluster); + + SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("SPARKHISTORYUI", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("http://host1:"; + PORT, urls.get(0)); + assertEquals("http://host2:"; + PORT, urls.get(1)); + } + + + @Test + public void testSparkAndSpark2HistoryUI() { + final String PORT = "4545"; + final String PORT2 = "6767"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(ac.getHostNames()).andReturn(Arrays.asList("host1", "host2")).anyTimes(); + EasyMock.expect(ac.getConfigProperty("spark.history.ui.port")).andReturn(PORT).anyTimes(); + EasyMock.replay(ac); + + AmbariComponent ac2 = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(ac2.getHostNames()).andReturn(Arrays.asList("host3", "host4")).anyTimes(); + EasyMock.expect(ac2.getConfigProperty("spark.history.ui.port")).andReturn(PORT2).anyTimes(); + EasyMock.replay(ac2); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("SPARK_JOBHISTORYSERVER")).andReturn(ac).anyTimes(); + EasyMock.expect(cluster.getComponent("SPARK2_JOBHISTORYSERVER")).andReturn(ac2).anyTimes(); + EasyMock.replay(cluster); + + SparkHistoryUIServiceURLCreator c = new SparkHistoryUIServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("SPARKHISTORYUI", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("http://host1:"; + PORT, urls.get(0)); + assertEquals("http://host2:"; + PORT, urls.get(1)); + } + + + @Test + public void testLivyServer() { + doTestLivyServer("LIVY_SERVER"); + } + + @Test + public void testLivy2Server() { + doTestLivyServer("LIVY2_SERVER"); + } + + private void doTestLivyServer(String componentName) { + final String PORT = "4545"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + List<String> hostNames = Arrays.asList("host1", "host2"); + EasyMock.expect(ac.getHostNames()).andReturn(hostNames).anyTimes(); + EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes(); + EasyMock.replay(ac); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent(componentName)).andReturn(ac).anyTimes(); + EasyMock.replay(cluster); + + LivyServiceURLCreator c = new LivyServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("LIVYSERVER", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("http://host1:"; + PORT, urls.get(0)); + assertEquals("http://host2:"; + PORT, urls.get(1)); + } + + @Test + public void testLivyAndLivy2Server() { + final String PORT = "4545"; + final String PORT2 = "2323"; + + AmbariComponent ac = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(ac.getHostNames()).andReturn(Arrays.asList("host1", "host2")).anyTimes(); + EasyMock.expect(ac.getConfigProperty("livy.server.port")).andReturn(PORT).anyTimes(); + EasyMock.replay(ac); + + AmbariComponent ac2 = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(ac2.getHostNames()).andReturn(Arrays.asList("host3", "host4")).anyTimes(); + EasyMock.expect(ac2.getConfigProperty("livy.server.port")).andReturn(PORT2).anyTimes(); + EasyMock.replay(ac2); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("LIVY_SERVER")).andReturn(ac).anyTimes(); + EasyMock.expect(cluster.getComponent("LIVY2_SERVER")).andReturn(ac2).anyTimes(); + EasyMock.replay(cluster); + + LivyServiceURLCreator c = new LivyServiceURLCreator(); + c.init(cluster); + List<String> urls = c.create("LIVYSERVER", null); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + assertEquals(2, urls.size()); + assertEquals("http://host1:"; + PORT, urls.get(0)); + assertEquals("http://host2:"; + PORT, urls.get(1)); + } + + +}
