Repository: ambari Updated Branches: refs/heads/trunk a3f7a1fb2 -> e63063f6c
AMBARI-9917. Kerberos: Add Host did not generate keytabs (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e63063f6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e63063f6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e63063f6 Branch: refs/heads/trunk Commit: e63063f6cc30fb065ad92cd90ee9efc4819c9e10 Parents: a3f7a1f Author: Robert Levas <rle...@hortonworks.com> Authored: Wed Mar 4 16:47:13 2015 -0500 Committer: Robert Levas <rle...@hortonworks.com> Committed: Wed Mar 4 16:47:13 2015 -0500 ---------------------------------------------------------------------- .../AmbariManagementControllerImpl.java | 13 +- .../server/controller/KerberosHelper.java | 58 ++++++--- .../server/controller/KerberosHelperTest.java | 120 +++++++++++++------ 3 files changed, 136 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e63063f6/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 731227d..98390fd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -1915,6 +1915,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle clusterHostInfoJson, "{}", HostParamsJson); Collection<ServiceComponentHost> componentsToEnableKerberos = new ArrayList<ServiceComponentHost>(); + Set<String> hostsToForceKerberosOperations = new HashSet<String>(); //HACK String jobtrackerHost = getJobTrackerHost(cluster); @@ -1968,6 +1969,16 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } componentsToEnableKerberos.add(scHost); + + if(Service.Type.KERBEROS.name().equalsIgnoreCase(scHost.getServiceName()) && + Role.KERBEROS_CLIENT.name().equalsIgnoreCase(scHost.getServiceComponentName())) { + // Since the KERBEROS/KERBEROS_CLIENT is about to be moved from the INIT to the + // INSTALLED state (and it should be by the time the stages (in this request) + // that need to be execute), collect the relevant hostname to make sure the + // Kerberos logic doest not skip operations for it. + hostsToForceKerberosOperations.add(scHost.getHostName()); + } + } } else if (oldSchState == State.STARTED // TODO: oldSchState == State.INSTALLED is always false, looks like a bug @@ -2168,7 +2179,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } try { - kerberosHelper.ensureIdentities(cluster, serviceFilter, null, requestStages); + kerberosHelper.ensureIdentities(cluster, serviceFilter, null, hostsToForceKerberosOperations, requestStages); } catch (KerberosOperationException e) { throw new IllegalArgumentException(e.getMessage(), e); } http://git-wip-us.apache.org/repos/asf/ambari/blob/e63063f6/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java index c4a5f4f..e01d38d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java @@ -205,10 +205,10 @@ public class KerberosHelper { if (securityType == SecurityType.KERBEROS) { LOG.info("Configuring Kerberos for realm {} on cluster, {}", kerberosDetails.getDefaultRealm(), cluster.getClusterName()); - requestStageContainer = handle(cluster, kerberosDetails, null, null, requestStageContainer, new EnableKerberosHandler()); + requestStageContainer = handle(cluster, kerberosDetails, null, null, null, requestStageContainer, new EnableKerberosHandler()); } else if (securityType == SecurityType.NONE) { LOG.info("Disabling Kerberos from cluster, {}", cluster.getClusterName()); - requestStageContainer = handle(cluster, kerberosDetails, null, null, requestStageContainer, new DisableKerberosHandler()); + requestStageContainer = handle(cluster, kerberosDetails, null, null, null, requestStageContainer, new DisableKerberosHandler()); } else { throw new AmbariException(String.format("Unexpected security type value: %s", securityType.name())); } @@ -248,10 +248,10 @@ public class KerberosHelper { } if ("true".equalsIgnoreCase(value) || "all".equalsIgnoreCase(value)) { - requestStageContainer = handle(cluster, getKerberosDetails(cluster), null, null, + requestStageContainer = handle(cluster, getKerberosDetails(cluster), null, null, null, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(true)); } else if ("missing".equalsIgnoreCase(value)) { - requestStageContainer = handle(cluster, getKerberosDetails(cluster), null, null, + requestStageContainer = handle(cluster, getKerberosDetails(cluster), null, null, null, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(false)); } else { throw new AmbariException(String.format("Unexpected directive value: %s", value)); @@ -269,6 +269,7 @@ public class KerberosHelper { return requestStageContainer; } + /** * Ensures the set of filtered principals and keytabs exist on the cluster. * <p/> @@ -283,27 +284,35 @@ public class KerberosHelper { * It is expected tht the "kerberos-env" configuration type is available. It is used to obtain * information about the Kerberos configuration, generally specific to the KDC being used. * - * @param cluster the relevant Cluster - * @param serviceComponentFilter a Map of service names to component names indicating the relevant - * set of services and components - if null, no filter is relevant; - * if empty, the filter indicates no relevant services or components - * @param identityFilter a Collection of identity names indicating the relevant identities - - * if null, no filter is relevant; if empty, the filter indicates no - * relevant identities - * @param requestStageContainer a RequestStageContainer to place generated stages, if needed - - * if null a new RequestStageContainer will be created. + * @param cluster the relevant Cluster + * @param serviceComponentFilter a Map of service names to component names indicating the + * relevant set of services and components - if null, no + * filter is relevant; if empty, the filter indicates no + * relevant services or components + * @param identityFilter a Collection of identity names indicating the relevant + * identities - if null, no filter is relevant; if empty, + * the filter indicates no relevant identities + * @param hostsToForceKerberosOperations a set of host names on which it is expected that the + * Kerberos client is or will be in the INSTALLED state by + * the time the operations targeted for them are to be + * executed - if empty or null, this no hosts will be + * "forced" + * @param requestStageContainer a RequestStageContainer to place generated stages, if + * needed - if null a new RequestStageContainer will be + * created. * @return the updated or a new RequestStageContainer containing the stages that need to be * executed to complete this task; or null if no stages need to be executed. * @throws AmbariException * @throws KerberosOperationException * @throws KerberosInvalidConfigurationException if an issue occurs trying to get the - * Kerberos-specific configuration details + * Kerberos-specific configuration details */ public RequestStageContainer ensureIdentities(Cluster cluster, Map<String, ? extends Collection<String>> serviceComponentFilter, - Collection<String> identityFilter, RequestStageContainer requestStageContainer) + Collection<String> identityFilter, Set<String> hostsToForceKerberosOperations, + RequestStageContainer requestStageContainer) throws AmbariException, KerberosOperationException { return handle(cluster, getKerberosDetails(cluster), serviceComponentFilter, identityFilter, - requestStageContainer, new CreatePrincipalsAndKeytabsHandler(false)); + hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(false)); } /** @@ -339,7 +348,7 @@ public class KerberosHelper { public RequestStageContainer deleteIdentities(Cluster cluster, Map<String, ? extends Collection<String>> serviceComponentFilter, Collection<String> identityFilter, RequestStageContainer requestStageContainer) throws AmbariException, KerberosOperationException { - return handle(cluster, getKerberosDetails(cluster), serviceComponentFilter, identityFilter, + return handle(cluster, getKerberosDetails(cluster), serviceComponentFilter, identityFilter, null, requestStageContainer, new DeletePrincipalsAndKeytabsHandler()); } @@ -642,6 +651,11 @@ public class KerberosHelper { * relevant identities * @param requestStageContainer a RequestStageContainer to place generated stages, if needed - * if null a new RequestStageContainer will be created. + * @param hostsToForceKerberosOperations a set of host names on which it is expected that the + * Kerberos client is or will be in the INSTALLED state by + * the time the operations targeted for them are to be + * executed - if empty or null, this no hosts will be + * "forced" * @param handler a Handler to use to provide guidance and set up stages * to perform the work needed to complete the relative action * @return the updated or a new RequestStageContainer containing the stages that need to be @@ -655,8 +669,10 @@ public class KerberosHelper { KerberosDetails kerberosDetails, Map<String, ? extends Collection<String>> serviceComponentFilter, Collection<String> identityFilter, + Set<String> hostsToForceKerberosOperations, RequestStageContainer requestStageContainer, - Handler handler) throws AmbariException, KerberosOperationException { + Handler handler) + throws AmbariException, KerberosOperationException { Map<String, Service> services = cluster.getServices(); @@ -677,6 +693,12 @@ public class KerberosHelper { // This is needed to help determine which hosts to perform actions for and create tasks for. Set<String> hostsWithValidKerberosClient = new HashSet<String>(); + // Ensure that that hosts that should be assumed to be in the correct state when needed are + // in the hostsWithValidKerberosClient collection. + if(hostsToForceKerberosOperations != null) { + hostsWithValidKerberosClient.addAll(hostsToForceKerberosOperations); + } + // Create a temporary directory to store metadata needed to complete this task. Information // such as which principals and keytabs files to create as well as what configurations need // to be update are stored in data files in this directory. Any keytab files are stored in http://git-wip-us.apache.org/repos/asf/ambari/blob/e63063f6/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java index 8e1c0e8..d766d8c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java @@ -1297,13 +1297,23 @@ public class KerberosHelperTest extends EasyMockSupport { final ServiceComponentHost schKerberosClient = createMock(ServiceComponentHost.class); expect(schKerberosClient.getServiceName()).andReturn(Service.Type.KERBEROS.name()).anyTimes(); expect(schKerberosClient.getServiceComponentName()).andReturn(Role.KERBEROS_CLIENT.name()).anyTimes(); - expect(schKerberosClient.getHostName()).andReturn("host1").anyTimes(); + expect(schKerberosClient.getHostName()).andReturn("hostA").anyTimes(); expect(schKerberosClient.getState()).andReturn(State.INSTALLED).anyTimes(); - final ServiceComponentHost sch1 = createMock(ServiceComponentHost.class); - expect(sch1.getServiceName()).andReturn("SERVICE1").anyTimes(); - expect(sch1.getServiceComponentName()).andReturn("COMPONENT1").anyTimes(); - expect(sch1.getHostName()).andReturn("host1").anyTimes(); + final ServiceComponentHost sch1A = createMock(ServiceComponentHost.class); + expect(sch1A.getServiceName()).andReturn("SERVICE1").anyTimes(); + expect(sch1A.getServiceComponentName()).andReturn("COMPONENT1").anyTimes(); + expect(sch1A.getHostName()).andReturn("hostA").anyTimes(); + + final ServiceComponentHost sch1B = createMock(ServiceComponentHost.class); + expect(sch1B.getServiceName()).andReturn("SERVICE1").anyTimes(); + expect(sch1B.getServiceComponentName()).andReturn("COMPONENT1").anyTimes(); + expect(sch1B.getHostName()).andReturn("hostB").anyTimes(); + + final ServiceComponentHost sch1C = createMock(ServiceComponentHost.class); + expect(sch1C.getServiceName()).andReturn("SERVICE1").anyTimes(); + expect(sch1C.getServiceComponentName()).andReturn("COMPONENT1").anyTimes(); + expect(sch1C.getHostName()).andReturn("hostC").anyTimes(); final ServiceComponentHost sch2 = createStrictMock(ServiceComponentHost.class); expect(sch2.getServiceName()).andReturn("SERVICE2").anyTimes(); @@ -1312,33 +1322,41 @@ public class KerberosHelperTest extends EasyMockSupport { final ServiceComponentHost sch3 = createStrictMock(ServiceComponentHost.class); expect(sch3.getServiceName()).andReturn("SERVICE3").anyTimes(); expect(sch3.getServiceComponentName()).andReturn("COMPONENT3").anyTimes(); - expect(sch3.getHostName()).andReturn("host1").anyTimes(); + expect(sch3.getHostName()).andReturn("hostA").anyTimes(); - final Host host = createNiceMock(Host.class); - expect(host.getHostName()).andReturn("host1").anyTimes(); - expect(host.getState()).andReturn(HostState.HEALTHY).anyTimes(); + final Host hostA = createNiceMock(Host.class); + expect(hostA.getHostName()).andReturn("hostA").anyTimes(); + expect(hostA.getState()).andReturn(HostState.HEALTHY).anyTimes(); + + final Host hostB = createNiceMock(Host.class); + expect(hostB.getHostName()).andReturn("hostB").anyTimes(); + expect(hostB.getState()).andReturn(HostState.HEALTHY).anyTimes(); + + final Host hostC = createNiceMock(Host.class); + expect(hostC.getHostName()).andReturn("hostC").anyTimes(); + expect(hostC.getState()).andReturn(HostState.HEALTHY).anyTimes(); final ServiceComponent serviceComponentKerberosClient = createNiceMock(ServiceComponent.class); expect(serviceComponentKerberosClient.getName()).andReturn(Role.KERBEROS_CLIENT.name()).anyTimes(); - expect(serviceComponentKerberosClient.getServiceComponentHosts()).andReturn(Collections.singletonMap("host1", schKerberosClient)).anyTimes(); + expect(serviceComponentKerberosClient.getServiceComponentHosts()).andReturn(Collections.singletonMap("hostA", schKerberosClient)).anyTimes(); final Service serviceKerberos = createStrictMock(Service.class); expect(serviceKerberos.getName()).andReturn(Service.Type.KERBEROS.name()).anyTimes(); expect(serviceKerberos.getServiceComponents()) .andReturn(Collections.singletonMap(Role.KERBEROS_CLIENT.name(), serviceComponentKerberosClient)) - .times(3); + .times(5); final Service service1 = createStrictMock(Service.class); expect(service1.getName()).andReturn("SERVICE1").anyTimes(); expect(service1.getServiceComponents()) .andReturn(Collections.<String, ServiceComponent>emptyMap()) - .times(3); + .times(5); final Service service2 = createStrictMock(Service.class); expect(service2.getName()).andReturn("SERVICE2").anyTimes(); expect(service2.getServiceComponents()) .andReturn(Collections.<String, ServiceComponent>emptyMap()) - .times(3); + .times(5); final Map<String, String> kerberosEnvProperties = createNiceMock(Map.class); expect(kerberosEnvProperties.get("kdc_type")).andReturn("mit-kdc").anyTimes(); @@ -1365,16 +1383,31 @@ public class KerberosHelperTest extends EasyMockSupport { } }) .anyTimes(); - expect(cluster.getServiceComponentHosts("host1")) + expect(cluster.getServiceComponentHosts("hostA")) .andReturn(new ArrayList<ServiceComponentHost>() { { - add(sch1); + add(sch1A); add(sch2); add(sch3); add(schKerberosClient); } }) .once(); + expect(cluster.getServiceComponentHosts("hostB")) + .andReturn(new ArrayList<ServiceComponentHost>() { + { + add(sch1B); + add(schKerberosClient); + } + }) + .once(); + expect(cluster.getServiceComponentHosts("hostC")) + .andReturn(new ArrayList<ServiceComponentHost>() { + { + add(sch1C); + } + }) + .once(); expect(cluster.getCurrentStackVersion()) .andReturn(new StackId("HDP", "2.2")) .anyTimes(); @@ -1390,21 +1423,32 @@ public class KerberosHelperTest extends EasyMockSupport { expect(clusters.getHostsForCluster("c1")) .andReturn(new HashMap<String, Host>() { { - put("host1", host); + put("hostA", hostA); + put("hostB", hostB); + put("hostC", hostC); } }) .once(); - expect(clusters.getHost("host1")) - .andReturn(host) - .once(); + expect(clusters.getHost("hostA")) + .andReturn(hostA) + .times(1); + expect(clusters.getHost("hostB")) + .andReturn(hostB) + .times(1); final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1")) + expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "hostA")) .andReturn(Collections.<String, Map<String, String>>emptyMap()) .once(); - expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null)) + expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "hostB")) + .andReturn(Collections.<String, Map<String, String>>emptyMap()) + .once(); + expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "hostC")) .andReturn(Collections.<String, Map<String, String>>emptyMap()) .once(); + expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null)) + .andReturn(Collections.<String, Map<String, String>>emptyMap()) + .times(3); expect(ambariManagementController.getRoleCommandOrder(cluster)) .andReturn(createNiceMock(RoleCommandOrder.class)) .once(); @@ -1418,7 +1462,7 @@ public class KerberosHelperTest extends EasyMockSupport { }}); } }) - .times(2); + .times(4); final KerberosPrincipalDescriptor principalDescriptor1a = createMock(KerberosPrincipalDescriptor.class); expect(principalDescriptor1a.getValue()).andReturn("component1a/_HOST@${realm}").anyTimes(); @@ -1439,12 +1483,12 @@ public class KerberosHelperTest extends EasyMockSupport { expect(principalDescriptor3.getConfiguration()).andReturn("service3-site/component3.kerberos.principal").anyTimes(); final KerberosKeytabDescriptor keytabDescriptor1 = createMock(KerberosKeytabDescriptor.class); - expect(keytabDescriptor1.getFile()).andReturn("${keytab_dir}/service1.keytab").once(); - expect(keytabDescriptor1.getOwnerName()).andReturn("service1").once(); - expect(keytabDescriptor1.getOwnerAccess()).andReturn("rw").once(); - expect(keytabDescriptor1.getGroupName()).andReturn("hadoop").once(); - expect(keytabDescriptor1.getGroupAccess()).andReturn("").once(); - expect(keytabDescriptor1.getConfiguration()).andReturn("service1-site/component1.keytab.file").once(); + expect(keytabDescriptor1.getFile()).andReturn("${keytab_dir}/service1.keytab").times(3); + expect(keytabDescriptor1.getOwnerName()).andReturn("service1").times(3); + expect(keytabDescriptor1.getOwnerAccess()).andReturn("rw").times(3); + expect(keytabDescriptor1.getGroupName()).andReturn("hadoop").times(3); + expect(keytabDescriptor1.getGroupAccess()).andReturn("").times(3); + expect(keytabDescriptor1.getConfiguration()).andReturn("service1-site/component1.keytab.file").times(3); final KerberosKeytabDescriptor keytabDescriptor3 = createMock(KerberosKeytabDescriptor.class); expect(keytabDescriptor3.getFile()).andReturn("${keytab_dir}/service3.keytab").once(); @@ -1476,6 +1520,10 @@ public class KerberosHelperTest extends EasyMockSupport { expect(componentDescriptor1.getIdentities(true)).andReturn(identityDescriptors1).times(1); expect(componentDescriptor1.getConfigurations(true)).andReturn(null).times(1); expect(componentDescriptor1.getIdentities(true)).andReturn(identityDescriptors1).times(1); + expect(componentDescriptor1.getConfigurations(true)).andReturn(null).times(1); + expect(componentDescriptor1.getIdentities(true)).andReturn(identityDescriptors1).times(1); + expect(componentDescriptor1.getConfigurations(true)).andReturn(null).times(1); + expect(componentDescriptor1.getIdentities(true)).andReturn(identityDescriptors1).times(1); expect(componentDescriptor1.getAuthToLocalProperties()).andReturn(null).times(1); final ArrayList<KerberosIdentityDescriptor> identityDescriptors3 = new ArrayList<KerberosIdentityDescriptor>() {{ @@ -1486,13 +1534,12 @@ public class KerberosHelperTest extends EasyMockSupport { expect(componentDescriptor3.getConfigurations(true)).andReturn(null).times(1); final KerberosServiceDescriptor serviceDescriptor1 = createMock(KerberosServiceDescriptor.class); - expect(serviceDescriptor1.getIdentities(true)).andReturn(null).times(1); + expect(serviceDescriptor1.getIdentities(true)).andReturn(null).times(4); expect(serviceDescriptor1.getName()).andReturn("SERVICE1").times(1); - expect(serviceDescriptor1.getIdentities(true)).andReturn(null).times(1); expect(serviceDescriptor1.getComponents()).andReturn(new HashMap<String, KerberosComponentDescriptor>(){{ put("COMPONENT1", componentDescriptor1); }}).times(1); - expect(serviceDescriptor1.getComponent("COMPONENT1")).andReturn(componentDescriptor1).once(); + expect(serviceDescriptor1.getComponent("COMPONENT1")).andReturn(componentDescriptor1).times(3); expect(serviceDescriptor1.getAuthToLocalProperties()).andReturn(null).once(); final KerberosServiceDescriptor serviceDescriptor3 = createMock(KerberosServiceDescriptor.class); @@ -1502,14 +1549,15 @@ public class KerberosHelperTest extends EasyMockSupport { final KerberosDescriptor kerberosDescriptor = createStrictMock(KerberosDescriptor.class); expect(kerberosDescriptor.getProperties()).andReturn(null).once(); - expect(kerberosDescriptor.getService("SERVICE1")).andReturn(serviceDescriptor1).once(); - expect(kerberosDescriptor.getService("SERVICE3")).andReturn(serviceDescriptor3).once(); + expect(kerberosDescriptor.getService("SERVICE1")).andReturn(serviceDescriptor1).times(3); + expect(kerberosDescriptor.getService("SERVICE3")).andReturn(serviceDescriptor3).times(1); expect(kerberosDescriptor.getIdentities()).andReturn(null).once(); expect(kerberosDescriptor.getAuthToLocalProperties()).andReturn(null).once(); + expect(kerberosDescriptor.getServices()).andReturn(new HashMap<String, KerberosServiceDescriptor>() {{ - put("SERVCE1", serviceDescriptor1); - put("SERVCE2", serviceDescriptor3); + put("SERVICE1", serviceDescriptor1); + put("SERVICE3", serviceDescriptor3); }}).once(); setupGetDescriptorFromCluster(kerberosDescriptor); @@ -1566,7 +1614,7 @@ public class KerberosHelperTest extends EasyMockSupport { serviceComponentFilter.put("SERVICE3", Collections.singleton("COMPONENT3")); serviceComponentFilter.put("SERVICE1", null); - kerberosHelper.ensureIdentities(cluster, serviceComponentFilter, identityFilter, requestStageContainer); + kerberosHelper.ensureIdentities(cluster, serviceComponentFilter, identityFilter, null, requestStageContainer); verifyAll(); }