http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java index 6d7498b..18c7bee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java @@ -17,6 +17,11 @@ */ package org.apache.ambari.server.orm.entities; +import java.util.Collections; +import java.util.Set; + +import org.apache.ambari.server.state.stack.RepoTag; + /** * Emulates entity to provide a quick way to change it to real entity in future. */ @@ -30,6 +35,8 @@ public class RepositoryEntity { private String mirrorsList; private boolean unique; + private Set<RepoTag> tags; + public String getName() { return name; } @@ -70,6 +77,36 @@ public class RepositoryEntity { this.repositoryId = repositoryId; } + public String getMirrorsList() { + return mirrorsList; + } + + public void setMirrorsList(String mirrorsList) { + this.mirrorsList = mirrorsList; + } + + public boolean isUnique() { + return unique; + } + + public void setUnique(boolean unique) { + this.unique = unique; + } + + /** + * @return the repo tags + */ + public Set<RepoTag> getTags() { + return tags == null ? Collections.<RepoTag>emptySet() : tags; + } + + /** + * @param repoTags the tags to set + */ + public void setTags(Set<RepoTag> repoTags) { + tags = repoTags; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -95,20 +132,4 @@ public class RepositoryEntity { result = 31 * result + (repositoryId != null ? repositoryId.hashCode() : 0); return result; } - - public String getMirrorsList() { - return mirrorsList; - } - - public void setMirrorsList(String mirrorsList) { - this.mirrorsList = mirrorsList; - } - - public boolean isUnique() { - return unique; - } - - public void setUnique(boolean unique) { - this.unique = unique; - } }
http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariLdapAuthenticationProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariLdapAuthenticationProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariLdapAuthenticationProvider.java index b7ff297..b2cdb1d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariLdapAuthenticationProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/AmbariLdapAuthenticationProvider.java @@ -42,7 +42,7 @@ import com.google.inject.Inject; * Provides LDAP user authorization logic for Ambari Server */ public class AmbariLdapAuthenticationProvider implements AuthenticationProvider { - Logger LOG = LoggerFactory.getLogger(AmbariLdapAuthenticationProvider.class); + static Logger LOG = LoggerFactory.getLogger(AmbariLdapAuthenticationProvider.class); // exposed and mutable for "test" Configuration configuration; http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/security/encryption/CredentialProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/encryption/CredentialProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/security/encryption/CredentialProvider.java index 1f2f6db..ce0e843 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/security/encryption/CredentialProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/encryption/CredentialProvider.java @@ -42,7 +42,7 @@ public class CredentialProvider { '2', '3', '4', '5', '6', '7', '8', '9'}; private CredentialStore keystoreService; - static final Logger LOG = LoggerFactory.getLogger(CredentialProvider.class); + private static final Logger LOG = LoggerFactory.getLogger(CredentialProvider.class); public CredentialProvider(String masterKey, File masterKeyLocation, boolean isMasterKeyPersisted, File masterKeyStoreLocation) throws AmbariException { http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java index 50e3cfe..e219dc3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java @@ -18,10 +18,17 @@ package org.apache.ambari.server.serveraction; +import java.io.File; +import java.io.FilenameFilter; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; @@ -38,11 +45,17 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.UpgradeContext.UpgradeServiceSummary; +import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.util.ClassUtils; import com.google.inject.Inject; import com.google.inject.Injector; @@ -495,7 +508,6 @@ public class ServerActionExecutor { throw new AmbariException("Missing ExecutionCommand data"); } else { Map<String, String> roleParams = executionCommand.getRoleParams(); - if (roleParams == null) { throw new AmbariException("Missing RoleParams data"); } else { @@ -504,8 +516,30 @@ public class ServerActionExecutor { if (actionClassname == null) { throw new AmbariException("Missing action classname for server action"); } else { - ServerAction action = createServerAction(actionClassname); - + Map<String, ServiceInfo> services = new HashMap<String, ServiceInfo>(); + UpgradeSummary upgradeSummary = executionCommand.getUpgradeSummary(); + if (upgradeSummary != null) { + Map<String, UpgradeServiceSummary> upgradeServiceSummaries = upgradeSummary.services; + LOG.debug("UpgradeServiceSummary: " + upgradeServiceSummaries); + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + AmbariMetaInfo ambariMetaInfo = ambariManagementController.getAmbariMetaInfo(); + String serviceName = executionCommand.getServiceName(); + if (serviceName != null && !serviceName.isEmpty()){ + LOG.info(String.format("Server action %s is associated with service %s", actionClassname, serviceName)); + //Execution stage of a given service, only need to examine stack information for this one service + UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(serviceName); + addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, serviceName); + } else { + LOG.info(String.format("Server action %s is not associated with a service", actionClassname)); + //Load all Jars + for(String key: upgradeServiceSummaries.keySet()){ + UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(key); + addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, key); + } + } + LOG.info(String.format("Attempt to load server action classes from %s", services.keySet().toString())); + } + ServerAction action = createServerAction(actionClassname, services); if (action == null) { throw new AmbariException("Failed to create server action: " + actionClassname); } else { @@ -520,6 +554,30 @@ public class ServerActionExecutor { } } + private void addServiceInfo(Map<String, ServiceInfo> services, AmbariMetaInfo ambariMetaInfo, String stackId, String serviceName) { + List<String> stackInfo = getStackInfo(stackId); + LOG.debug(String.format("Stack info list: %s", stackInfo)); + if (stackInfo.size() > 1) { + try { + ServiceInfo service = ambariMetaInfo.getService(stackInfo.get(0), stackInfo.get(1), serviceName); + LOG.debug(String.format("Adding %s to the list of services for loading external Jars...", service.getName())); + services.put(serviceName, service); + } catch (AmbariException e) { + LOG.error(String.format("Failed to obtain service info for stack %s, service name %s", stackId, serviceName), e); + } + } + } + + private List<String> getStackInfo(String stackId) { + LOG.debug(String.format("Stack id: %s", stackId)); + StringTokenizer tokens = new StringTokenizer(stackId, "-"); + List<String> info = new ArrayList<String>(); + while (tokens.hasMoreElements()) { + info.add((String)tokens.nextElement()); + } + return info; + } + /** * Attempts to create an instance of the ServerAction class implementation specified in * classname. @@ -528,24 +586,85 @@ public class ServerActionExecutor { * @return the instantiated ServerAction implementation * @throws AmbariException */ - private ServerAction createServerAction(String classname) throws AmbariException { - try { - Class<?> actionClass = Class.forName(classname); + private ServerAction createServerAction(String classname, Map<String, ServiceInfo> services) throws AmbariException { + Class<?> actionClass = null; + actionClass = getServerActionClass(classname); + if (actionClass == null) { + LOG.debug(String.format("Did not find %s in Ambari, try to load it from external directories", classname)); + actionClass = getServiceLevelServerActionClass(classname, services); + } - if (actionClass == null) { - throw new AmbariException("Unable to load server action class: " + classname); + if (actionClass == null) { + throw new AmbariException("Unable to load server action class: " + classname); + } else { + LOG.debug(String.format("Ready to init server action %s", classname)); + Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class); + if (serverActionClass == null) { + throw new AmbariException("Unable to execute server action class, invalid type: " + classname); } else { - Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class); + return injector.getInstance(serverActionClass); + } + } + } - if (serverActionClass == null) { - throw new AmbariException("Unable to execute server action class, invalid type: " + classname); - } else { - return injector.getInstance(serverActionClass); + /** + * Load server action classes defined in the service level Jar files + * */ + private Class<?> getServiceLevelServerActionClass(String classname, Map<String, ServiceInfo> services) { + List<URL> urls = new ArrayList<>(); + for (ServiceInfo service : services.values()) { + LOG.debug(String.format("Checking service %s", service)); + File dir = service.getServerActionsFolder(); + if ( dir != null) { + LOG.debug(String.format("Service %s, external dir %s",service.getName(), dir.getAbsolutePath())); + File[] jars = dir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + LOG.debug(String.format("Checking folder %s", name)); + return name.endsWith(".jar"); + } + }); + for (File jar : jars) { + try { + URL url = jar.toURI().toURL(); + urls.add(url); + LOG.info("Adding server action jar to classpath: {}", url); + } + catch (Exception e) { + LOG.error("Failed to add server action jar to classpath: {}", jar.getAbsolutePath(), e); + } } + } else { + LOG.error(String.format("%s service server actions folder returned null", service)); + } + } + + ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), ClassUtils.getDefaultClassLoader()); + Class<?> actionClass = null; + try { + actionClass = ClassUtils.resolveClassName(classname, classLoader); + LOG.debug(String.format("Found external server action %s", classname)); + } catch(IllegalArgumentException illegalArgumentException) { + LOG.error(String.format("Unable to find server action %s in external server action directories", classname), illegalArgumentException); + } + + return actionClass; + } + + /** + * Load server action classes defined in Ambari source code + * */ + private Class<?> getServerActionClass(String classname) throws AmbariException{ + Class<?> actionClass = null; + try { + actionClass = Class.forName(classname); + if (actionClass == null) { + LOG.warn(String.format("Unable to load server action class: %s from Ambari", classname)); } } catch (ClassNotFoundException e) { - throw new AmbariException("Unable to load server action class: " + classname, e); + LOG.error(String.format("Unable to load server action class: %s", classname), e); } + return actionClass; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java index f7d6060..d7b91b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java @@ -175,9 +175,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { throw new KerberosLDAPContainerException("principalContainerDn is not a valid LDAP name", e); } - setAdministratorCredential(administratorCredential); - setDefaultRealm(realm); - setKeyEncryptionTypes(translateEncryptionTypes(kerberosConfiguration.get(KERBEROS_ENV_ENCRYPTION_TYPES), "\\s+")); + super.open(administratorCredential, realm, kerberosConfiguration); this.ldapContext = createLdapContext(); this.searchControls = createSearchControls(); @@ -215,11 +213,12 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { * The implementation is specific to a particular type of KDC. * * @param principal a String containing the principal to test + * @param service a boolean value indicating whether the principal is for a service or not * @return true if the principal exists; false otherwise * @throws KerberosOperationException */ @Override - public boolean principalExists(String principal) throws KerberosOperationException { + public boolean principalExists(String principal, boolean service) throws KerberosOperationException { if (!isOpen()) { throw new KerberosOperationException("This operation handler has not been opened"); } @@ -260,7 +259,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { if (password == null) { throw new KerberosOperationException("principal password is null"); } - if (principalExists(principal)) { + if (principalExists(principal, service)) { throw new KerberosPrincipalAlreadyExistsException(principal); } @@ -347,12 +346,13 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { * * @param principal a String containing the principal to update * @param password a String containing the password to set + * @param service a boolean value indicating whether the principal is for a service or not * @return an Integer declaring the new key number * @throws KerberosPrincipalDoesNotExistException if the principal does not exist * @throws KerberosOperationException */ @Override - public Integer setPrincipalPassword(String principal, String password) throws KerberosOperationException { + public Integer setPrincipalPassword(String principal, String password, boolean service) throws KerberosOperationException { if (!isOpen()) { throw new KerberosOperationException("This operation handler has not been opened"); } @@ -362,7 +362,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { if (password == null) { throw new KerberosOperationException("principal password is null"); } - if(!principalExists(principal)) { + if (!principalExists(principal, service)) { throw new KerberosPrincipalDoesNotExistException(principal); } @@ -396,11 +396,12 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { * The implementation is specific to a particular type of KDC. * * @param principal a String containing the principal to remove + * @param service a boolean value indicating whether the principal is for a service or not * @return true if the principal was successfully removed; otherwise false * @throws KerberosOperationException */ @Override - public boolean removePrincipal(String principal) throws KerberosOperationException { + public boolean removePrincipal(String principal, boolean service) throws KerberosOperationException { if (!isOpen()) { throw new KerberosOperationException("This operation handler has not been opened"); } @@ -469,10 +470,9 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler { String message = String.format("Failed to communicate with the Active Directory at %s: %s", ldapUrl, e.getMessage()); LOG.warn(message, e); - if(rootCause instanceof SSLHandshakeException) { + if (rootCause instanceof SSLHandshakeException) { throw new KerberosKDCSSLConnectionException(message, e); - } - else { + } else { throw new KerberosKDCConnectionException(message, e); } } catch (AuthenticationException e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java index 7948a60..b8affb4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java @@ -33,6 +33,9 @@ import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.controller.RootComponent; +import org.apache.ambari.server.controller.RootService; +import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; @@ -77,13 +80,12 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer Map<String, Map<String, String>> currentConfigurations, Map<String, Map<String, String>> kerberosConfigurations, boolean includeAmbariIdentity, - Map<String, Set<String>> propertiesToBeIgnored, - boolean excludeHeadless) throws AmbariException { + Map<String, Set<String>> propertiesToBeIgnored) throws AmbariException { List<Component> components = new ArrayList<>(); for (ServiceComponentHost each : schToProcess) { components.add(Component.fromServiceComponentHost(each)); } - processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored, excludeHeadless); + processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored); } protected void processServiceComponents(Cluster cluster, KerberosDescriptor kerberosDescriptor, @@ -92,8 +94,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer Map<String, Map<String, String>> currentConfigurations, Map<String, Map<String, String>> kerberosConfigurations, boolean includeAmbariIdentity, - Map<String, Set<String>> propertiesToBeIgnored, - boolean excludeHeadless) throws AmbariException { + Map<String, Set<String>> propertiesToBeIgnored) throws AmbariException { actionLog.writeStdOut("Processing Kerberos identities and configurations"); @@ -125,15 +126,17 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer throw new AmbariException(message, e); } + HashMap<String, ResolvedKerberosKeytab> resolvedKeytabs = new HashMap<>(); + String realm = getDefaultRealm(getCommandParameters()); + try { Map<String, Set<String>> propertiesToIgnore = null; - // Iterate over the components installed on the current host to get the service and // component-level Kerberos descriptors in order to determine which principals, // keytab files, and configurations need to be created or updated. for (Component sch : schToProcess) { String hostName = sch.getHostName(); - + Long hostId = sch.getHostId(); String serviceName = sch.getServiceName(); String componentName = sch.getServiceComponentName(); @@ -157,7 +160,8 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer // Add service-level principals (and keytabs) kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, serviceIdentities, - identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); + identityFilter, hostName, hostId, serviceName, componentName, kerberosConfigurations, currentConfigurations, + resolvedKeytabs, realm); propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore); KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName); @@ -172,7 +176,8 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer // Add component-level principals (and keytabs) kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities, - identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); + identityFilter, hostName, hostId, serviceName, componentName, kerberosConfigurations, currentConfigurations, + resolvedKeytabs, realm); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } @@ -189,11 +194,12 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer // component. String componentName = KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME.equals(identity.getName()) ? "AMBARI_SERVER_SELF" - : "AMBARI_SERVER"; + : RootComponent.AMBARI_SERVER.name(); List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity); kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities, - identityFilter, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); + identityFilter, StageUtils.getHostName(), ambariServerHostID(), RootService.AMBARI.name(), componentName, kerberosConfigurations, currentConfigurations, + resolvedKeytabs, realm); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } @@ -202,6 +208,11 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer if ((propertiesToBeIgnored != null) && (propertiesToIgnore != null)) { propertiesToBeIgnored.putAll(propertiesToIgnore); } + + // create database records for keytabs that must be presented on cluster + for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) { + kerberosHelper.processResolvedKeytab(keytab); + } } catch (IOException e) { String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath()); LOG.error(message, e); @@ -227,19 +238,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer protected Map<String, ? extends Collection<String>> getServiceComponentFilter() { String serializedValue = getCommandParameterValue(SERVICE_COMPONENT_FILTER); - if(serializedValue != null) { - Type type = new TypeToken<Map<String, ? extends Collection<String>>>() {}.getType(); - return StageUtils.getGson().fromJson(serializedValue, type); - } else { - return null; - } - } - - protected Set<String> getHostFilter() { - String serializedValue = getCommandParameterValue(HOST_FILTER); - - if(serializedValue != null) { - Type type = new TypeToken<Set<String>>() {}.getType(); + if (serializedValue != null) { + Type type = new TypeToken<Map<String, ? extends Collection<String>>>() { + }.getType(); return StageUtils.getGson().fromJson(serializedValue, type); } else { return null; @@ -249,8 +250,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer protected Collection<String> getIdentityFilter() { String serializedValue = getCommandParameterValue(IDENTITY_FILTER); - if(serializedValue != null) { - Type type = new TypeToken<Collection<String>>() {}.getType(); + if (serializedValue != null) { + Type type = new TypeToken<Collection<String>>() { + }.getType(); return StageUtils.getGson().fromJson(serializedValue, type); } else { return null; http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java index dae8254..002076d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java @@ -32,15 +32,24 @@ import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.ResourceProvider; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.controller.utilities.PredicateBuilder; +import org.apache.ambari.server.orm.dao.KerberosKeytabDAO; +import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.inject.Inject; + /** * Used to perform Kerberos Cleanup Operations as part of the Unkerberization process */ public class CleanupServerAction extends KerberosServerAction { + @Inject + KerberosKeytabDAO kerberosKeytabDAO; + + @Inject + KerberosPrincipalDAO kerberosPrincipalDAO; private final static Logger LOG = LoggerFactory.getLogger(CleanupServerAction.class); @@ -102,11 +111,12 @@ public class CleanupServerAction extends KerberosServerAction { ClusterController clusterController = ClusterControllerHelper.getClusterController(); - ResourceProvider artifactProvider = - clusterController.ensureResourceProvider(Resource.Type.Artifact); + ResourceProvider artifactProvider = clusterController.ensureResourceProvider(Resource.Type.Artifact); try { artifactProvider.deleteResources(new RequestImpl(null, null, null, null), predicate); + kerberosPrincipalDAO.remove(kerberosPrincipalDAO.findAll()); + kerberosKeytabDAO.remove(kerberosKeytabDAO.findAll()); LOG.info("Kerberos descriptor removed successfully."); actionLog.writeStdOut("Kerberos descriptor removed successfully."); } catch (NoSuchResourceException e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java index 4f1ee52..ed7642c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java @@ -25,18 +25,21 @@ public class Component { private final String hostName; private final String serviceName; private final String serviceComponentName; + private final Long hostId; public static Component fromServiceComponentHost(ServiceComponentHost serviceComponentHost) { return new Component( serviceComponentHost.getHostName(), serviceComponentHost.getServiceName(), - serviceComponentHost.getServiceComponentName()); + serviceComponentHost.getServiceComponentName(), + serviceComponentHost.getHost().getHostId()); } - public Component(String hostName, String serviceName, String serviceComponentName) { + public Component(String hostName, String serviceName, String serviceComponentName, Long hostId) { this.hostName = hostName; this.serviceName = serviceName; this.serviceComponentName = serviceComponentName; + this.hostId = hostId; } public String getHostName() { @@ -51,6 +54,10 @@ public class Component { return serviceComponentName; } + public Long getHostId() { + return hostId; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -60,6 +67,7 @@ public class Component { .append(hostName, component.hostName) .append(serviceName, component.serviceName) .append(serviceComponentName, component.serviceComponentName) + .append(hostId, component.hostId) .isEquals(); } @@ -69,6 +77,7 @@ public class Component { .append(hostName) .append(serviceName) .append(serviceComponentName) + .append(hostId) .toHashCode(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java index fca1b6f..3384152 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java @@ -26,11 +26,10 @@ import java.util.concurrent.ConcurrentMap; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.controller.utilities.KerberosChecker; -import org.apache.ambari.server.orm.dao.HostDAO; +import org.apache.ambari.server.orm.dao.KerberosKeytabDAO; import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO; -import org.apache.ambari.server.orm.entities.HostEntity; +import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity; import org.apache.ambari.server.serveraction.ActionLog; import org.apache.ambari.server.utils.ShellCommandUtil; import org.apache.ambari.server.utils.StageUtils; @@ -63,7 +62,7 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction private KerberosPrincipalHostDAO kerberosPrincipalHostDAO; @Inject - private HostDAO hostDAO; + private KerberosKeytabDAO kerberosKeytabDAO; /** * Called to execute this action. Upon invocation, calls @@ -121,7 +120,8 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction } else { String hostName = identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME); - if (hostName != null && hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) { + String serviceName = identityRecord.get(KerberosIdentityDataFileReader.SERVICE); + if (hostName != null && serviceName.equals("AMBARI")) { String destKeytabFilePath = identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH); File hostDirectory = new File(dataDirectory, hostName); File srcKeytabFile = new File(hostDirectory, DigestUtils.sha1Hex(destKeytabFilePath)); @@ -182,11 +182,7 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction groupName, groupReadable, groupWritable); String ambariServerHostName = StageUtils.getHostName(); - HostEntity ambariServerHostEntity = hostDAO.findByName(ambariServerHostName); - Long ambariServerHostID = (ambariServerHostEntity == null) - ? null - : ambariServerHostEntity.getHostId(); - + Long ambariServerHostID = ambariServerHostID(); if (ambariServerHostID == null) { String message = String.format("Failed to add the kerberos_principal_host record for %s on " + "the Ambari server host since the host id for Ambari server host, %s, was not found." + @@ -196,8 +192,19 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction if (actionLog != null) { actionLog.writeStdErr(message); } - } else if (!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID)) { - kerberosPrincipalHostDAO.create(principal, ambariServerHostID); + } else if (!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID, destKeytabFilePath)) { + if (!kerberosKeytabDAO.exists(destKeytabFilePath)) { + kerberosKeytabDAO.create(destKeytabFilePath); + } + if(!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID, destKeytabFilePath)) { + kerberosPrincipalHostDAO.create( + new KerberosPrincipalHostEntity(principal, ambariServerHostID, destKeytabFilePath, true) + ); + } else { + KerberosPrincipalHostEntity kphe = kerberosPrincipalHostDAO.find(principal, ambariServerHostID, destKeytabFilePath); + kphe.setDistributed(true); + kerberosPrincipalHostDAO.merge(kphe); + } } if (actionLog != null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java index 355f515..5ec4c10 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java @@ -205,34 +205,31 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { ensureAmbariOnlyAccess(hostDirectory); } - if (hostDirectory.exists()) { - File destinationKeytabFile = new File(hostDirectory, DigestUtils.sha1Hex(keytabFilePath)); - HostEntity hostEntity = hostDAO.findByName(hostName); - // in case of ambari-server identity there's no host entity for ambari_server host - if (hostEntity == null && !hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) { - message = "Failed to find HostEntity for hostname = " + hostName; - actionLog.writeStdErr(message); - LOG.error(message); - commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr()); - return commandReport; - } + if (hostDirectory.exists()) { + File destinationKeytabFile = new File(hostDirectory, DigestUtils.sha1Hex(keytabFilePath)); + HostEntity hostEntity = hostDAO.findByName(hostName); + // in case of ambari-server identity there's no host entity for ambari_server host + if (hostEntity == null && !hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) { + message = "Failed to find HostEntity for hostname = " + hostName; + actionLog.writeStdErr(message); + LOG.error(message); + commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr()); + return commandReport; + } - boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; - boolean onlyKeytabWrite = "true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE)); - boolean grabKeytabFromCache = regenerateKeytabs && onlyKeytabWrite; - // if grabKeytabFromCache=true we will try to get keytab from cache and send to agent, it will be true for - // headless cached keytabs - if (password == null) { - if (!grabKeytabFromCache && (hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || kerberosPrincipalHostDAO - .exists(evaluatedPrincipal, hostEntity.getHostId()))) { - // There is nothing to do for this since it must already exist and we don't want to - // regenerate the keytab - message = String.format("Skipping keytab file for %s, missing password indicates nothing to do", evaluatedPrincipal); - LOG.debug(message); - } else { - KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); - String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath(); + boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; + + KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); + String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath(); + if (password == null) { + if (!regenerateKeytabs && (hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || kerberosPrincipalHostDAO + .exists(evaluatedPrincipal, hostEntity.getHostId(), keytabFilePath)) && cachedKeytabPath == null) { + // There is nothing to do for this since it must already exist and we don't want to + // regenerate the keytab + message = String.format("Skipping keytab file for %s, missing password indicates nothing to do", evaluatedPrincipal); + LOG.debug(message); + } else { if (cachedKeytabPath == null) { message = String.format("Failed to create keytab for %s, missing cached file", evaluatedPrincipal); actionLog.writeStdErr(message); @@ -250,9 +247,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { } } } else { - boolean canCache = ("true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_IS_CACHABLE))); - - Keytab keytab = createKeytab(evaluatedPrincipal, password, keyNumber, operationHandler, visitedPrincipalKeys != null, canCache, actionLog); + Keytab keytab = createKeytab(evaluatedPrincipal, password, keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog); if (keytab != null) { try { @@ -287,7 +282,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { } } else { message = String.format("Failed to create keytab file for %s, the container directory does not exist: %s", - evaluatedPrincipal, hostDirectory.getAbsolutePath()); + evaluatedPrincipal, hostDirectory.getAbsolutePath()); actionLog.writeStdErr(message); LOG.error(message); commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr()); @@ -299,10 +294,10 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { } } } finally { - if(commandReport != null && HostRoleStatus.FAILED.toString().equals(commandReport.getStatus())) { + if (commandReport != null && HostRoleStatus.FAILED.toString().equals(commandReport.getStatus())) { auditEventBuilder.withReasonOfFailure(message == null ? "Unknown error" : message); } - if(commandReport != null || auditEventBuilder.hasPrincipal()) { + if (commandReport != null || auditEventBuilder.hasPrincipal()) { auditLog(auditEventBuilder.build()); } } @@ -359,12 +354,12 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { // and store that location so it can be reused rather than recreate it. KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal); if (principalEntity != null) { - if (!principalEntity.isService() && canCache) { + if (canCache) { File cachedKeytabFile = cacheKeytab(principal, keytab); String previousCachedFilePath = principalEntity.getCachedKeytabPath(); String cachedKeytabFilePath = ((cachedKeytabFile == null) || !cachedKeytabFile.exists()) - ? null - : cachedKeytabFile.getAbsolutePath(); + ? null + : cachedKeytabFile.getAbsolutePath(); principalEntity.setCachedKeytabPath(cachedKeytabFilePath); kerberosPrincipalDAO.merge(principalEntity); http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java index 1c0853b9..0c90659 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.serveraction.kerberos; +import java.io.File; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -30,6 +31,7 @@ import org.apache.ambari.server.audit.event.kerberos.CreatePrincipalKerberosAudi import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO; import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO; import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity; +import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity; import org.apache.ambari.server.security.SecurePasswordHelper; import org.apache.ambari.server.serveraction.ActionLog; import org.apache.commons.lang.StringUtils; @@ -128,27 +130,25 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { seenPrincipals.add(evaluatedPrincipal); boolean processPrincipal; - boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; + // TODO add invalidate_principals option to make keytabs invalid all over the cluster. + KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); + + boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; + boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE)); if (regenerateKeytabs) { - // do not process cached identities that can be passed as is(headless identities) - processPrincipal = "false".equals(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE).toLowerCase()); + // force recreation of principal due to keytab regeneration + // regenerate only service principals if request filtered by hosts + processPrincipal = !hasHostFilters() || servicePrincipal; + } else if (kerberosPrincipalEntity == null) { + // This principal has not been processed before, process it. + processPrincipal = true; + } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) { + // This principal has been processed and a keytab file has been cached for it... do not process it. + processPrincipal = false; } else { - KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); - - if (kerberosPrincipalEntity == null) { - // This principal has not been processed before, process it. - processPrincipal = true; - } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) { - // This principal has been processed and a keytab file has been cached for it... do not process it. - processPrincipal = false; - } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) { - // This principal has been processed and a keytab file has been distributed... do not process it. - processPrincipal = false; - } else { - // This principal has been processed but a keytab file for it has not been distributed... process it. - processPrincipal = true; - } + // This principal has been processed but a keytab file for it has not been distributed... process it. + processPrincipal = true; } if (processPrincipal) { @@ -157,9 +157,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { String password = principalPasswordMap.get(evaluatedPrincipal); if (password == null) { - boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE)); CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, actionLog); - if (result == null) { commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr()); } else { @@ -167,6 +165,20 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { principalPasswordMap.put(evaluatedPrincipal, result.getPassword()); principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber()); + // invalidate given principal for all keytabs to make them redistributed again + for (KerberosPrincipalHostEntity kphe: kerberosPrincipalHostDAO.findByPrincipal(evaluatedPrincipal)) { + kphe.setDistributed(false); + kerberosPrincipalHostDAO.merge(kphe); + } + // invalidate principal cache + KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); + try { + new File(principalEntity.getCachedKeytabPath()).delete(); + } catch (Exception e) { + LOG.debug("Failed to delete cache file '{}'", principalEntity.getCachedKeytabPath()); + } + principalEntity.setCachedKeytabPath(null); + kerberosPrincipalDAO.merge(principalEntity); } } } @@ -241,7 +253,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { if (regenerateKeytabs) { try { - keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password); + keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password, isServicePrincipal); created = false; } catch (KerberosPrincipalDoesNotExistException e) { message = String.format("Principal, %s, does not exist, creating new principal", principal); @@ -264,7 +276,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { actionLog.writeStdOut(message); } - keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password); + keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password, isServicePrincipal); created = false; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e83bf1bd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java index 2b3a0ca..4c80bd4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java @@ -114,7 +114,8 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction { try { try { - operationHandler.removePrincipal(evaluatedPrincipal); + boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE)); + operationHandler.removePrincipal(evaluatedPrincipal, servicePrincipal); } catch (KerberosOperationException e) { message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage()); LOG.warn(message);
