http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java ---------------------------------------------------------------------- diff --cc gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java index 21627ad,0000000..05fc4eb mode 100644,000000..100644 --- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java @@@ -1,858 -1,0 +1,870 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import net.minidev.json.JSONObject; +import net.minidev.json.JSONValue; +import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; +import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig; +import org.easymock.EasyMock; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + + +/** + * Test the Ambari ServiceDiscovery implementation. + * + * N.B. These tests do NOT verify Ambari API responses. They DO validate the Ambari ServiceDiscovery implementation's + * treatment of the responses as they were observed at the time the tests are developed. + */ +public class AmbariServiceDiscoveryTest { + + @Test + public void testSingleClusterDiscovery() throws Exception { + final String discoveryAddress = "http://ambarihost:8080";; + final String clusterName = "testCluster"; + ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName); + + ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class); + EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes(); + EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes(); + EasyMock.replay(sdc); + + ServiceDiscovery.Cluster cluster = sd.discover(sdc, clusterName); + assertNotNull(cluster); + assertEquals(clusterName, cluster.getName()); + assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass())); + assertEquals(6, ((AmbariCluster) cluster).getComponents().size()); + +// printServiceURLs(cluster); + } + + + @Test + public void testBulkClusterDiscovery() throws Exception { + final String discoveryAddress = "http://ambarihost:8080";; + final String clusterName = "anotherCluster"; + ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName); + + ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class); + EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes(); + EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes(); + EasyMock.replay(sdc); + + Map<String, ServiceDiscovery.Cluster> clusters = sd.discover(sdc); + assertNotNull(clusters); + assertEquals(1, clusters.size()); + ServiceDiscovery.Cluster cluster = clusters.get(clusterName); + assertNotNull(cluster); + assertEquals(clusterName, cluster.getName()); + assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass())); + assertEquals(6, ((AmbariCluster) cluster).getComponents().size()); + +// printServiceURLs(cluster, "NAMENODE", "WEBHCAT", "OOZIE", "RESOURCEMANAGER"); + } + + + private static void printServiceURLs(ServiceDiscovery.Cluster cluster) { + final String[] services = new String[]{"NAMENODE", + "JOBTRACKER", + "WEBHDFS", + "WEBHCAT", + "OOZIE", + "WEBHBASE", + "HIVE", + "RESOURCEMANAGER"}; + printServiceURLs(cluster, services); + } + + + private static void printServiceURLs(ServiceDiscovery.Cluster cluster, String...services) { + for (String name : services) { + StringBuilder sb = new StringBuilder(); + List<String> urls = cluster.getServiceURLs(name); + if (urls != null && !urls.isEmpty()) { + for (String url : urls) { + sb.append(url); + sb.append(" "); + } + } + System.out.println(String.format("%18s: %s", name, sb.toString())); + } + } + + + /** + * ServiceDiscovery implementation derived from AmbariServiceDiscovery, so the invokeREST method can be overridden + * to eliminate the need to perform actual HTTP interactions with a real Ambari endpoint. + */ + private static final class TestAmbariServiceDiscovery extends AmbariServiceDiscovery { + ++ final static String CLUSTER_PLACEHOLDER = TestRESTInvoker.CLUSTER_PLACEHOLDER; ++ ++ TestAmbariServiceDiscovery(String clusterName) { ++ super(new TestRESTInvoker(clusterName)); ++ } ++ ++ } ++ ++ private static final class TestRESTInvoker extends RESTInvoker { ++ + final static String CLUSTER_PLACEHOLDER = "CLUSTER_NAME"; + + private Map<String, JSONObject> cannedResponses = new HashMap<>(); + - TestAmbariServiceDiscovery(String clusterName) { - cannedResponses.put(AMBARI_CLUSTERS_URI, - (JSONObject) JSONValue.parse(CLUSTERS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, - clusterName))); ++ TestRESTInvoker(String clusterName) { ++ super(null); ++ ++ cannedResponses.put(AmbariServiceDiscovery.AMBARI_CLUSTERS_URI, ++ (JSONObject) JSONValue.parse(CLUSTERS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, ++ clusterName))); + - cannedResponses.put(String.format(AMBARI_HOSTROLES_URI, clusterName), - (JSONObject) JSONValue.parse(HOSTROLES_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, - clusterName))); ++ cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_HOSTROLES_URI, clusterName), ++ (JSONObject) JSONValue.parse(HOSTROLES_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, ++ clusterName))); + - cannedResponses.put(String.format(AMBARI_SERVICECONFIGS_URI, clusterName), - (JSONObject) JSONValue.parse(SERVICECONFIGS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, - clusterName))); ++ cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_SERVICECONFIGS_URI, clusterName), ++ (JSONObject) JSONValue.parse(SERVICECONFIGS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, ++ clusterName))); + } + + @Override - protected JSONObject invokeREST(String url, String username, String passwordAlias) { ++ JSONObject invoke(String url, String username, String passwordAlias) { + return cannedResponses.get(url.substring(url.indexOf("/api"))); + } + } + + + //////////////////////////////////////////////////////////////////////// + // JSON response templates, based on actual response content excerpts + //////////////////////////////////////////////////////////////////////// + + private static final String CLUSTERS_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters\",\n"; + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n"; + + " \"Clusters\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"version\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]" + + "}"; + + + private static final String HOSTROLES_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services?fields=components/host_components/HostRoles\",\n"; + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"AMBARI_METRICS\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS/components/METRICS_COLLECTOR\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"METRICS_COLLECTOR\",\n" + + " \"service_name\" : \"AMBARI_METRICS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/METRICS_COLLECTOR\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"METRICS_COLLECTOR\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"AMBARI_METRICS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HBASE/components/HBASE_MASTER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HBASE_MASTER\",\n" + + " \"service_name\" : \"HBASE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/HBASE_MASTER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HBASE_MASTER\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"HBASE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/NAMENODE\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NAMENODE\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/NAMENODE\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NAMENODE\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/SECONDARY_NAMENODE\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"SECONDARY_NAMENODE\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/SECONDARY_NAMENODE\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"SECONDARY_NAMENODE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HCAT\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HCAT\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/HCAT\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HCAT\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_METASTORE\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_METASTORE\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_METASTORE\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_METASTORE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_SERVER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_SERVER\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/WEBHCAT_SERVER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"WEBHCAT_SERVER\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/WEBHCAT_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"WEBHCAT_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"OOZIE\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE/components/OOZIE_SERVER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"OOZIE_SERVER\",\n" + + " \"service_name\" : \"OOZIE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/OOZIE_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"OOZIE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"OOZIE\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/APP_TIMELINE_SERVER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/APP_TIMELINE_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/NODEMANAGER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NODEMANAGER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/NODEMANAGER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NODEMANAGER\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/RESOURCEMANAGER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"RESOURCEMANAGER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/RESOURCEMANAGER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"RESOURCEMANAGER\",\n" + + " \"ha_state\" : \"ACTIVE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER\",\n"; + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"ZOOKEEPER\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER/components/ZOOKEEPER_SERVER\",\n"; + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"service_name\" : \"ZOOKEEPER\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n"; + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}\n"; + + + private static final String SERVICECONFIGS_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?is_current=true\",\n"; + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HBASE&service_config_version=1\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hbase-site\",\n" + + " \"tag\" : \"version1503410563715\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hbase.master.info.bindAddress\" : \"0.0.0.0\",\n" + + " \"hbase.master.info.port\" : \"16010\",\n" + + " \"hbase.master.port\" : \"16000\",\n" + + " \"hbase.regionserver.info.port\" : \"16030\",\n" + + " \"hbase.regionserver.port\" : \"16020\",\n" + + " \"hbase.zookeeper.property.clientPort\" : \"2181\",\n" + + " \"hbase.zookeeper.quorum\" : \"c6403.ambari.apache.org,c6402.ambari.apache.org,c6401.ambari.apache.org\",\n" + + " \"hbase.zookeeper.useMulti\" : \"true\",\n" + + " \"zookeeper.znode.parent\" : \"/hbase-unsecure\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_config_version_note\" : \"Initial configurations for HBase\",\n" + + " \"service_name\" : \"HBASE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HDFS&service_config_version=2\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hdfs-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"dfs.cluster.administrators\" : \" hdfs\",\n" + + " \"dfs.datanode.address\" : \"0.0.0.0:50010\",\n" + + " \"dfs.datanode.http.address\" : \"0.0.0.0:50075\",\n" + + " \"dfs.datanode.https.address\" : \"0.0.0.0:50475\",\n" + + " \"dfs.datanode.ipc.address\" : \"0.0.0.0:8010\",\n" + + " \"dfs.http.policy\" : \"HTTP_ONLY\",\n" + + " \"dfs.https.port\" : \"50470\",\n" + + " \"dfs.journalnode.http-address\" : \"0.0.0.0:8480\",\n" + + " \"dfs.journalnode.https-address\" : \"0.0.0.0:8481\",\n" + + " \"dfs.namenode.http-address\" : \"c6401.ambari.apache.org:50070\",\n" + + " \"dfs.namenode.https-address\" : \"c6401.ambari.apache.org:50470\",\n" + + " \"dfs.namenode.rpc-address\" : \"c6401.ambari.apache.org:8020\",\n" + + " \"dfs.namenode.secondary.http-address\" : \"c6402.ambari.apache.org:50090\",\n" + + " \"dfs.webhdfs.enabled\" : \"true\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"final\" : {\n" + + " \"dfs.webhdfs.enabled\" : \"true\",\n" + + " \"dfs.namenode.http-address\" : \"true\",\n" + + " \"dfs.support.append\" : \"true\",\n" + + " \"dfs.namenode.name.dir\" : \"true\",\n" + + " \"dfs.datanode.failed.volumes.tolerated\" : \"true\",\n" + + " \"dfs.datanode.data.dir\" : \"true\"\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"core-site\",\n" + + " \"tag\" : \"version1502131215159\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"hadoop.http.authentication.simple.anonymous.allowed\" : \"true\",\n" + + " \"net.topology.script.file.name\" : \"/etc/hadoop/conf/topology_script.py\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"final\" : {\n" + + " \"fs.defaultFS\" : \"true\"\n" + + " }\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 2,\n" + + " \"service_config_version_note\" : \"knox trusted proxy support\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HIVE&service_config_version=3\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-env\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive_security_authorization\" : \"None\",\n" + + " \"webhcat_user\" : \"hcat\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hiveserver2-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive.metastore.metrics.enabled\" : \"true\",\n" + + " \"hive.security.authorization.enabled\" : \"false\",\n" + + " \"hive.service.metrics.hadoop2.component\" : \"hiveserver2\",\n" + + " \"hive.service.metrics.reporter\" : \"HADOOP2\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-interactive-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive.server2.enable.doAs\" : \"false\",\n" + + " \"hive.server2.tez.default.queues\" : \"default\",\n" + + " \"hive.server2.tez.initialize.default.sessions\" : \"true\",\n" + + " \"hive.server2.tez.sessions.custom.queue.allowed\" : \"ignore\",\n" + + " \"hive.server2.tez.sessions.per.default.queue\" : \"1\",\n" + + " \"hive.server2.tez.sessions.restricted.configs\" : \"hive.execution.mode,hive.execution.engine\",\n" + + " \"hive.server2.thrift.http.port\" : \"10501\",\n" + + " \"hive.server2.thrift.port\" : \"10500\",\n" + + " \"hive.server2.webui.port\" : \"10502\",\n" + + " \"hive.server2.webui.use.ssl\" : \"false\",\n" + + " \"hive.server2.zookeeper.namespace\" : \"hiveserver2-hive2\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"tez-interactive-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"tez.am.am-rm.heartbeat.interval-ms.max\" : \"10000\",\n" + + " \"tez.am.client.heartbeat.poll.interval.millis\" : \"6000\",\n" + + " \"tez.am.client.heartbeat.timeout.secs\" : \"90\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-site\",\n" + + " \"tag\" : \"version1502130841736\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"hive.metastore.sasl.enabled\" : \"false\",\n" + + " \"hive.metastore.server.max.threads\" : \"100000\",\n" + + " \"hive.metastore.uris\" : \"thrift://c6402.ambari.apache.org:9083\",\n" + + " \"hive.server2.allow.user.substitution\" : \"true\",\n" + + " \"hive.server2.authentication\" : \"NONE\",\n" + + " \"hive.server2.authentication.spnego.keytab\" : \"HTTP/[email protected]\",\n" + + " \"hive.server2.authentication.spnego.principal\" : \"/etc/security/keytabs/spnego.service.keytab\",\n" + + " \"hive.server2.enable.doAs\" : \"true\",\n" + + " \"hive.server2.support.dynamic.service.discovery\" : \"true\",\n" + + " \"hive.server2.thrift.http.path\" : \"cliservice\",\n" + + " \"hive.server2.thrift.http.port\" : \"10001\",\n" + + " \"hive.server2.thrift.max.worker.threads\" : \"500\",\n" + + " \"hive.server2.thrift.port\" : \"10000\",\n" + + " \"hive.server2.thrift.sasl.qop\" : \"auth\",\n" + + " \"hive.server2.transport.mode\" : \"http\",\n" + + " \"hive.server2.use.SSL\" : \"false\",\n" + + " \"hive.server2.zookeeper.namespace\" : \"hiveserver2\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"hidden\" : {\n" + + " \"javax.jdo.option.ConnectionPassword\" : \"HIVE_CLIENT,WEBHCAT_SERVER,HCAT,CONFIG_DOWNLOAD\"\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"webhcat-site\",\n" + + " \"tag\" : \"version1502131111746\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"templeton.port\" : \"50111\",\n" + + " \"templeton.zookeeper.hosts\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" + + " \"webhcat.proxyuser.knox.groups\" : \"users\",\n" + + " \"webhcat.proxyuser.knox.hosts\" : \"*\",\n" + + " \"webhcat.proxyuser.root.groups\" : \"*\",\n" + + " \"webhcat.proxyuser.root.hosts\" : \"c6401.ambari.apache.org\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"createtime\" : 1502131110745,\n" + + " \"group_id\" : -1,\n" + + " \"group_name\" : \"Default\",\n" + + " \"hosts\" : [ ],\n" + + " \"is_cluster_compatible\" : true,\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 3,\n" + + " \"service_config_version_note\" : \"knox trusted proxy support\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=OOZIE&service_config_version=3\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"oozie-site\",\n" + + " \"tag\" : \"version1502131137103\",\n" + + " \"version\" : 3,\n" + + " \"properties\" : {\n" + + " \"oozie.base.url\" : \"http://c6402.ambari.apache.org:11000/oozie\",\n"; + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 3,\n" + + " \"service_name\" : \"OOZIE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=TEZ&service_config_version=1\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"tez-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"tez.use.cluster.hadoop-libs\" : \"false\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"createtime\" : 1502122253525,\n" + + " \"group_id\" : -1,\n" + + " \"group_name\" : \"Default\",\n" + + " \"hosts\" : [ ],\n" + + " \"is_cluster_compatible\" : true,\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_config_version_note\" : \"Initial configurations for Tez\",\n" + + " \"service_name\" : \"TEZ\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=YARN&service_config_version=1\",\n"; + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"yarn-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hadoop.registry.rm.enabled\" : \"true\",\n" + + " \"hadoop.registry.zk.quorum\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" + + " \"yarn.acl.enable\" : \"false\",\n" + + " \"yarn.http.policy\" : \"HTTP_ONLY\",\n" + + " \"yarn.nodemanager.address\" : \"0.0.0.0:45454\",\n" + + " \"yarn.nodemanager.bind-host\" : \"0.0.0.0\",\n" + + " \"yarn.resourcemanager.address\" : \"c6402.ambari.apache.org:8050\",\n" + + " \"yarn.resourcemanager.admin.address\" : \"c6402.ambari.apache.org:8141\",\n" + + " \"yarn.resourcemanager.ha.enabled\" : \"false\",\n" + + " \"yarn.resourcemanager.hostname\" : \"c6402.ambari.apache.org\",\n" + + " \"yarn.resourcemanager.resource-tracker.address\" : \"c6402.ambari.apache.org:8025\",\n" + + " \"yarn.resourcemanager.scheduler.address\" : \"c6402.ambari.apache.org:8030\",\n" + + " \"yarn.resourcemanager.webapp.address\" : \"c6402.ambari.apache.org:8088\",\n" + + " \"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\" : \"false\",\n" + + " \"yarn.resourcemanager.webapp.https.address\" : \"c6402.ambari.apache.org:8090\",\n" + + " \"yarn.resourcemanager.zk-address\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " }\n" + + " ]\n" + + "}"; + +}
http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jDispatcherFilter.java ---------------------------------------------------------------------- diff --cc gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jDispatcherFilter.java index fe39f25,0000000..6e04932 mode 100644,000000..100644 --- a/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jDispatcherFilter.java +++ b/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jDispatcherFilter.java @@@ -1,214 -1,0 +1,215 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.pac4j.filter; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.pac4j.Pac4jMessages; +import org.apache.knox.gateway.pac4j.session.KnoxSessionStore; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.security.KeystoreService; +import org.apache.knox.gateway.services.security.MasterService; +import org.apache.knox.gateway.services.security.AliasService; +import org.apache.knox.gateway.services.security.AliasServiceException; +import org.apache.knox.gateway.services.security.CryptoService; +import org.pac4j.config.client.PropertiesConfigFactory; +import org.pac4j.core.client.Client; +import org.pac4j.core.config.Config; - import org.pac4j.core.config.ConfigSingleton; - import org.pac4j.core.context.J2EContext; +import org.pac4j.core.util.CommonHelper; +import org.pac4j.http.client.indirect.IndirectBasicAuthClient; +import org.pac4j.http.credentials.authenticator.test.SimpleTestUsernamePasswordAuthenticator; +import org.pac4j.j2e.filter.CallbackFilter; +import org.pac4j.j2e.filter.SecurityFilter; + +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * <p>This is the main filter for the pac4j provider. The pac4j provider module heavily relies on the j2e-pac4j library (https://github.com/pac4j/j2e-pac4j).</p> + * <p>This filter dispatches the HTTP calls between the j2e-pac4j filters:</p> + * <ul> + * <li>to the {@link CallbackFilter} if the <code>client_name</code> parameter exists: it finishes the authentication process</li> + * <li>to the {@link RequiresAuthenticationFilter} otherwise: it starts the authentication process (redirection to the identity provider) if the user is not authenticated</li> + * </ul> + * <p>It uses the {@link KnoxSessionStore} to manage session data. The generated cookies are defined on a domain name + * which can be configured via the domain suffix parameter: <code>pac4j.cookie.domain.suffix</code>.</p> + * <p>The callback url must be defined to the current protected url (KnoxSSO service for example) via the parameter: <code>pac4j.callbackUrl</code>.</p> + * + * @since 0.8.0 + */ +public class Pac4jDispatcherFilter implements Filter { + + private static Pac4jMessages log = MessagesFactory.get(Pac4jMessages.class); + + public static final String TEST_BASIC_AUTH = "testBasicAuth"; + + public static final String PAC4J_CALLBACK_URL = "pac4j.callbackUrl"; + + public static final String PAC4J_CALLBACK_PARAMETER = "pac4jCallback"; + + private static final String PAC4J_COOKIE_DOMAIN_SUFFIX_PARAM = "pac4j.cookie.domain.suffix"; + ++ private static final String PAC4J_CONFIG = "pac4j.config"; ++ + private CallbackFilter callbackFilter; + + private SecurityFilter securityFilter; + private MasterService masterService = null; + private KeystoreService keystoreService = null; + private AliasService aliasService = null; + + @Override + public void init( FilterConfig filterConfig ) throws ServletException { + // JWT service + final ServletContext context = filterConfig.getServletContext(); + CryptoService cryptoService = null; + String clusterName = null; + if (context != null) { + GatewayServices services = (GatewayServices) context.getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + clusterName = (String) context.getAttribute(GatewayServices.GATEWAY_CLUSTER_ATTRIBUTE); + if (services != null) { + keystoreService = (KeystoreService) services.getService(GatewayServices.KEYSTORE_SERVICE); + cryptoService = (CryptoService) services.getService(GatewayServices.CRYPTO_SERVICE); + aliasService = (AliasService) services.getService(GatewayServices.ALIAS_SERVICE); + masterService = (MasterService) services.getService("MasterService"); + } + } + // crypto service, alias service and cluster name are mandatory + if (cryptoService == null || aliasService == null || clusterName == null) { + log.cryptoServiceAndAliasServiceAndClusterNameRequired(); + throw new ServletException("The crypto service, alias service and cluster name are required."); + } + try { + aliasService.getPasswordFromAliasForCluster(clusterName, KnoxSessionStore.PAC4J_PASSWORD, true); + } catch (AliasServiceException e) { + log.unableToGenerateAPasswordForEncryption(e); + throw new ServletException("Unable to generate a password for encryption."); + } + + // url to SSO authentication provider + String pac4jCallbackUrl = filterConfig.getInitParameter(PAC4J_CALLBACK_URL); + if (pac4jCallbackUrl == null) { + log.ssoAuthenticationProviderUrlRequired(); + throw new ServletException("Required pac4j callback URL is missing."); + } + // add the callback parameter to know it's a callback + pac4jCallbackUrl = CommonHelper.addParameter(pac4jCallbackUrl, PAC4J_CALLBACK_PARAMETER, "true"); + + final Config config; + final String clientName; + // client name from servlet parameter (mandatory) + final String clientNameParameter = filterConfig.getInitParameter("clientName"); + if (clientNameParameter == null) { + log.clientNameParameterRequired(); + throw new ServletException("Required pac4j clientName parameter is missing."); + } + if (TEST_BASIC_AUTH.equalsIgnoreCase(clientNameParameter)) { + // test configuration + final IndirectBasicAuthClient indirectBasicAuthClient = new IndirectBasicAuthClient(new SimpleTestUsernamePasswordAuthenticator()); + indirectBasicAuthClient.setRealmName("Knox TEST"); + config = new Config(pac4jCallbackUrl, indirectBasicAuthClient); + clientName = "IndirectBasicAuthClient"; + } else { + // get clients from the init parameters + final Map<String, String> properties = new HashMap<>(); + final Enumeration<String> names = filterConfig.getInitParameterNames(); + addDefaultConfig(clientNameParameter, properties); + while (names.hasMoreElements()) { + final String key = names.nextElement(); + properties.put(key, filterConfig.getInitParameter(key)); + } + final PropertiesConfigFactory propertiesConfigFactory = new PropertiesConfigFactory(pac4jCallbackUrl, properties); + config = propertiesConfigFactory.build(); + final List<Client> clients = config.getClients().getClients(); + if (clients == null || clients.size() == 0) { + log.atLeastOnePac4jClientMustBeDefined(); + throw new ServletException("At least one pac4j client must be defined."); + } + if (CommonHelper.isBlank(clientNameParameter)) { + clientName = clients.get(0).getName(); + } else { + clientName = clientNameParameter; + } + } + + callbackFilter = new CallbackFilter(); ++ callbackFilter.setConfigOnly(config); + securityFilter = new SecurityFilter(); + securityFilter.setClients(clientName); - securityFilter.setConfig(config); ++ securityFilter.setConfigOnly(config); + + final String domainSuffix = filterConfig.getInitParameter(PAC4J_COOKIE_DOMAIN_SUFFIX_PARAM); + config.setSessionStore(new KnoxSessionStore(cryptoService, clusterName, domainSuffix)); - ConfigSingleton.setConfig(config); + } + + private void addDefaultConfig(String clientNameParameter, Map<String, String> properties) { + // add default saml params + if (clientNameParameter.contains("SAML2Client")) { + properties.put(PropertiesConfigFactory.SAML_KEYSTORE_PATH, + keystoreService.getKeystorePath()); + + properties.put(PropertiesConfigFactory.SAML_KEYSTORE_PASSWORD, + new String(masterService.getMasterSecret())); + + // check for provisioned alias for private key + char[] gip = null; + try { + gip = aliasService.getGatewayIdentityPassphrase(); + } + catch(AliasServiceException ase) { + log.noPrivateKeyPasshraseProvisioned(ase); + } + if (gip != null) { + properties.put(PropertiesConfigFactory.SAML_PRIVATE_KEY_PASSWORD, + new String(gip)); + } + else { + // no alias provisioned then use the master + properties.put(PropertiesConfigFactory.SAML_PRIVATE_KEY_PASSWORD, + new String(masterService.getMasterSecret())); + } + } + } + + @Override + public void doFilter( ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + + final HttpServletRequest request = (HttpServletRequest) servletRequest; + final HttpServletResponse response = (HttpServletResponse) servletResponse; - final J2EContext context = new J2EContext(request, response, ConfigSingleton.getConfig().getSessionStore()); ++ request.setAttribute(PAC4J_CONFIG, securityFilter.getConfig()); ++// final J2EContext context = new J2EContext(request, response, securityFilter.getConfig().getSessionStore()); + + // it's a callback from an identity provider + if (request.getParameter(PAC4J_CALLBACK_PARAMETER) != null) { + // apply CallbackFilter + callbackFilter.doFilter(servletRequest, servletResponse, filterChain); + } else { + // otherwise just apply security and requires authentication + // apply RequiresAuthenticationFilter + securityFilter.doFilter(servletRequest, servletResponse, filterChain); + } + } + + @Override + public void destroy() { } +} http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jIdentityAdapter.java ---------------------------------------------------------------------- diff --cc gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jIdentityAdapter.java index 6387a0b,0000000..bc66003 mode 100644,000000..100644 --- a/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jIdentityAdapter.java +++ b/gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jIdentityAdapter.java @@@ -1,146 -1,0 +1,161 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.pac4j.filter; + - import org.apache.knox.gateway.audit.api.Action; - import org.apache.knox.gateway.audit.api.ActionOutcome; - import org.apache.knox.gateway.audit.api.AuditService; - import org.apache.knox.gateway.audit.api.AuditServiceFactory; - import org.apache.knox.gateway.audit.api.Auditor; - import org.apache.knox.gateway.audit.api.ResourceType; ++import org.apache.knox.gateway.audit.api.*; +import org.apache.knox.gateway.audit.log4j.audit.AuditConstants; +import org.apache.knox.gateway.filter.AbstractGatewayFilter; +import org.apache.knox.gateway.security.PrimaryPrincipal; - import org.pac4j.core.config.ConfigSingleton; ++import org.pac4j.core.config.Config; +import org.pac4j.core.context.J2EContext; +import org.pac4j.core.profile.CommonProfile; +import org.pac4j.core.profile.ProfileManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.Optional; + +/** + * <p>This filter retrieves the authenticated user saved by the pac4j provider and injects it into the J2E HTTP request.</p> + * + * @since 0.8.0 + */ +public class Pac4jIdentityAdapter implements Filter { + + private static final Logger logger = LoggerFactory.getLogger(Pac4jIdentityAdapter.class); + ++ public static final String PAC4J_ID_ATTRIBUTE = "pac4j.id_attribute"; ++ private static final String PAC4J_CONFIG = "pac4j.config"; ++ + private static AuditService auditService = AuditServiceFactory.getAuditService(); + private static Auditor auditor = auditService.getAuditor( + AuditConstants.DEFAULT_AUDITOR_NAME, AuditConstants.KNOX_SERVICE_NAME, + AuditConstants.KNOX_COMPONENT_NAME ); + + private String testIdentifier; + ++ private String idAttribute; ++ + @Override + public void init( FilterConfig filterConfig ) throws ServletException { ++ idAttribute = filterConfig.getInitParameter(PAC4J_ID_ATTRIBUTE); + } + + public void destroy() { + } + + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain chain) + throws IOException, ServletException { + + final HttpServletRequest request = (HttpServletRequest) servletRequest; + final HttpServletResponse response = (HttpServletResponse) servletResponse; - final J2EContext context = new J2EContext(request, response, ConfigSingleton.getConfig().getSessionStore()); ++ final J2EContext context = new J2EContext(request, response, ++ ((Config)request.getAttribute(PAC4J_CONFIG)).getSessionStore()); + final ProfileManager<CommonProfile> manager = new ProfileManager<CommonProfile>(context); + final Optional<CommonProfile> optional = manager.get(true); + if (optional.isPresent()) { + CommonProfile profile = optional.get(); + logger.debug("User authenticated as: {}", profile); + manager.remove(true); - final String id = profile.getId(); ++ String id = null; ++ if (idAttribute != null) { ++ Object attribute = profile.getAttribute(idAttribute); ++ if (attribute != null) { ++ id = attribute.toString(); ++ } ++ if (id == null) { ++ logger.error("Invalid attribute_id: {} configured to be used as principal" ++ + " falling back to default id", idAttribute); ++ } ++ } ++ if (id == null) { ++ id = profile.getId(); ++ } + testIdentifier = id; + PrimaryPrincipal pp = new PrimaryPrincipal(id); + Subject subject = new Subject(); + subject.getPrincipals().add(pp); + auditService.getContext().setUsername(id); + String sourceUri = (String)request.getAttribute( AbstractGatewayFilter.SOURCE_REQUEST_CONTEXT_URL_ATTRIBUTE_NAME ); + auditor.audit(Action.AUTHENTICATION, sourceUri, ResourceType.URI, ActionOutcome.SUCCESS); + + doAs(request, response, chain, subject); + } + } + + private void doAs(final ServletRequest request, + final ServletResponse response, final FilterChain chain, Subject subject) + throws IOException, ServletException { + try { + Subject.doAs( + subject, + new PrivilegedExceptionAction<Object>() { + public Object run() throws Exception { + chain.doFilter(request, response); + return null; + } + } + ); + } + catch (PrivilegedActionException e) { + Throwable t = e.getCause(); + if (t instanceof IOException) { + throw (IOException) t; + } + else if (t instanceof ServletException) { + throw (ServletException) t; + } + else { + throw new ServletException(t); + } + } + } + + /** + * For tests only. + */ + public static void setAuditService(AuditService auditService) { + Pac4jIdentityAdapter.auditService = auditService; + } + + /** + * For tests only. + */ + public static void setAuditor(Auditor auditor) { + Pac4jIdentityAdapter.auditor = auditor; + } + + /** + * For tests only. + */ + public String getTestIdentifier() { + return testIdentifier; + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-provider-security-pac4j/src/test/java/org/apache/knox/gateway/pac4j/MockHttpServletRequest.java ---------------------------------------------------------------------- diff --cc gateway-provider-security-pac4j/src/test/java/org/apache/knox/gateway/pac4j/MockHttpServletRequest.java index 7a3a833,0000000..18f4913 mode 100644,000000..100644 --- a/gateway-provider-security-pac4j/src/test/java/org/apache/knox/gateway/pac4j/MockHttpServletRequest.java +++ b/gateway-provider-security-pac4j/src/test/java/org/apache/knox/gateway/pac4j/MockHttpServletRequest.java @@@ -1,88 -1,0 +1,94 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.pac4j; + +import javax.servlet.http.*; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.*; + +public class MockHttpServletRequest extends HttpServletRequestWrapper { + + private String requestUrl; + private Cookie[] cookies; + private String serverName; + private Map<String, String> parameters = new HashMap<>(); + private Map<String, String> headers = new HashMap<>(); ++ private Map<String, Object> attributes = new HashMap<>(); + + public MockHttpServletRequest() { + super(mock(HttpServletRequest.class)); + } + + @Override + public Cookie[] getCookies() { + return cookies; + } + + public void setCookies(final Cookie[] cookies) { + this.cookies = cookies; + } + + @Override + public StringBuffer getRequestURL() { + return new StringBuffer(requestUrl); + } + + public void setRequestURL(final String requestUrl) { + this.requestUrl = requestUrl; + } + + @Override + public String getServerName() { + return serverName; + } + + public void setServerName(final String serverName) { + this.serverName = serverName; + } + + @Override + public String getParameter(String name) { + return parameters.get(name); + } + + public void addParameter(String key, String value) { + parameters.put(key, value); + } + + @Override + public String getHeader(String name) { + return headers.get(name); + } + + public void addHeader(String key, String value) { + headers.put(key, value); + } + + @Override ++ public void setAttribute(String name, Object value) { ++ attributes.put(name, value); ++ } ++ ++ @Override + public Object getAttribute(String name) { - return null; ++ return attributes.get(name); + } +}
