SLIDER-517 enable certificate and key localization for containers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46e131df Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46e131df Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46e131df Branch: refs/heads/feature/SLIDER-460-stderr Commit: 46e131df617c2461ef88905d083a2802468f0494 Parents: 0fa1141 Author: Jon Maron <[email protected]> Authored: Tue Oct 21 14:24:47 2014 -0400 Committer: Jon Maron <[email protected]> Committed: Tue Oct 21 14:24:47 2014 -0400 ---------------------------------------------------------------------- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 155 +++++++++++++++---- .../server/appmaster/SliderAppMaster.java | 30 ++++ .../appmaster/web/rest/agent/AgentWebApp.java | 4 +- .../services/security/CertificateManager.java | 74 ++++++++- 5 files changed, 229 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 2612865..9d5eb3d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -97,6 +97,8 @@ public interface AgentKeys { String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval"; String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data"; String AGENT_OUT_FILE = "slider-agent.out"; + String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth"; + String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 19ec015..d0ae5be 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -20,18 +20,22 @@ package org.apache.slider.providers.agent; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.registry.client.types.Endpoint; +import org.apache.hadoop.registry.client.types.ProtocolTypes; +import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ProtocolTypes; -import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterNode; @@ -39,6 +43,7 @@ import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; +import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; @@ -88,6 +93,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.Register; import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse; import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus; import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand; +import org.apache.slider.server.services.security.CertificateManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +109,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -378,33 +383,15 @@ public class AgentProviderService extends AbstractProviderService implements } if (SliderUtils.isHadoopClusterSecure(getConfig())) { - String keytabPathOnHost = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - // we need to localize the keytab files in the directory - Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - clusterName); - FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath); - LocalResource keytabRes; - for (FileStatus keytab : keytabs) { - if (!amKeytabName.equals(keytab.getPath().getName()) - && keytab.getPath().getName().endsWith(".keytab")) { - log.info("Localizing keytab {}", keytab.getPath().getName()); - keytabRes = fileSystem.createAmResource(keytab.getPath(), - LocalResourceType.FILE); - launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" + - keytab.getPath().getName(), - keytabRes); - } - } - } + localizeServiceKeytabs(launcher, instanceDefinition, fileSystem); + } + + MapOperations amComponent = instanceDefinition. + getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM); + boolean twoWayEnabled = amComponent != null ? Boolean.valueOf(amComponent. + getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) : false; + if (twoWayEnabled) { + localizeContainerSSLResources(launcher, container, fileSystem); } //add the configuration resources @@ -442,6 +429,112 @@ public class AgentProviderService extends AbstractProviderService implements getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME))); } + private void localizeContainerSSLResources(ContainerLauncher launcher, + Container container, + SliderFileSystem fileSystem) + throws SliderException { + try { + // localize server cert + Path certsDir = new Path(fileSystem.buildClusterDirPath( + getClusterName()), "certs"); + LocalResource certResource = fileSystem.createAmResource( + new Path(certsDir, SliderKeys.CRT_FILE_NAME), + LocalResourceType.FILE); + launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH, + certResource); + + // generate and localize agent cert + CertificateManager certMgr = new CertificateManager(); + String hostname = container.getNodeId().getHost(); + String containerId = container.getId().toString(); + certMgr.generateAgentCertificate(hostname, containerId); + LocalResource agentCertResource = fileSystem.createAmResource( + uploadSecurityResource( + CertificateManager.getAgentCertficateFilePath(containerId), + fileSystem), LocalResourceType.FILE); + // still using hostname as file name on the agent side, but the files + // do end up under the specific container's file space + launcher.addLocalResource("certs/" + hostname + ".crt", + agentCertResource); + LocalResource agentKeyResource = fileSystem.createAmResource( + uploadSecurityResource( + CertificateManager.getAgentKeyFilePath(containerId), fileSystem), + LocalResourceType.FILE); + launcher.addLocalResource("certs/" + hostname + ".key", + agentKeyResource); + + } catch (Exception e) { + throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e, + "Unable to localize certificates. Two-way SSL cannot be enabled"); + } + } + + private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem) + throws IOException { + Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()), + "certs"); + if (!fileSystem.getFileSystem().exists(certsDir)) { + fileSystem.getFileSystem().mkdirs(certsDir, + new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); + } + Path destPath = new Path(certsDir, resource.getName()); + if (!fileSystem.getFileSystem().exists(destPath)) { + FSDataOutputStream os = fileSystem.getFileSystem().create(destPath); + byte[] contents = FileUtils.readFileToByteArray(resource); + os.write(contents, 0, contents.length); + + os.flush(); + os.close(); + log.info("Uploaded {} to localization path {}", resource, destPath); + } + + while (!fileSystem.getFileSystem().exists(destPath)) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + // ignore + } + } + + fileSystem.getFileSystem().setPermission(destPath, + new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); + + return destPath; + } + + private void localizeServiceKeytabs(ContainerLauncher launcher, + AggregateConf instanceDefinition, + SliderFileSystem fileSystem) + throws IOException { + String keytabPathOnHost = instanceDefinition.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + if (SliderUtils.isUnset(keytabPathOnHost)) { + String amKeytabName = instanceDefinition.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = instanceDefinition.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + // we need to localize the keytab files in the directory + Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, + getClusterName()); + FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath); + LocalResource keytabRes; + for (FileStatus keytab : keytabs) { + if (!amKeytabName.equals(keytab.getPath().getName()) + && keytab.getPath().getName().endsWith(".keytab")) { + log.info("Localizing keytab {}", keytab.getPath().getName()); + keytabRes = fileSystem.createAmResource(keytab.getPath(), + LocalResourceType.FILE); + launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" + + keytab.getPath().getName(), + keytabRes); + } + } + } + } + /** * build the zookeeper registry path. * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 0a7fe33..d696f45 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -25,6 +25,8 @@ import com.google.protobuf.BlockingService; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -108,6 +110,7 @@ import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; +import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.providers.slideram.SliderAMProviderService; import org.apache.slider.server.appmaster.actions.ActionKillContainer; @@ -679,6 +682,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM)); + if (Boolean.valueOf(instanceDefinition. + getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM). + getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false))) { + uploadServerCertForLocalization(clustername, fs); + } + startAgentWebApp(appInformation, serviceConf); webApp = new SliderAMWebApp(registryOperations); @@ -891,6 +900,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService return finish(); } + private void uploadServerCertForLocalization(String clustername, + SliderFileSystem fs) + throws IOException { + Path certsDir = new Path(fs.buildClusterDirPath(clustername), + "certs"); + if (!fs.getFileSystem().exists(certsDir)) { + fs.getFileSystem().mkdirs(certsDir, + new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); + } + Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME); + if (!fs.getFileSystem().exists(destPath)) { + fs.getFileSystem().copyFromLocalFile( + new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()), + destPath); + log.info("Uploaded server cert to localization path {}", destPath); + } + + fs.getFileSystem().setPermission(destPath, + new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); + } + protected void login(String principal, File localKeytabFile) throws IOException, SliderException { UserGroupInformation.loginUserFromKeytab(principal, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java index 8aac490..f8d7b88 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java @@ -23,6 +23,7 @@ import com.sun.jersey.spi.container.servlet.ServletContainer; import com.sun.jersey.spi.container.servlet.WebConfig; import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider; import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.services.security.SecurityUtils; @@ -82,7 +83,8 @@ public class AgentWebApp implements Closeable { SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false); SslSelectChannelConnector ssl2WayConnector = createSSLConnector(Boolean.valueOf( - configsMap.getOption("ssl.server.client.auth","false"))); + configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, + "false"))); agentServer.setConnectors(new Connector[]{ssl1WayConnector, ssl2WayConnector}); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java index d200033..8934892 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java @@ -55,7 +55,11 @@ public class CertificateManager { "-passin pass:{3} -cert {0}/{5}"; private static final String SIGN_AGENT_CRT = "openssl ca -config " + "{0}/ca.config -in {0}/{1} -out {0}/{2} -batch -passin pass:{3} " + - "-keyfile {0}/{4} -cert {0}/{5}"; /** + "-keyfile {0}/{4} -cert {0}/{5}"; + private static final String GEN_AGENT_KEY="openssl req -new -newkey " + + "rsa:1024 -nodes -keyout {0}/{2}.key -subj /OU={1}/CN={2} -out {0}/{2}.csr"; + + /** * Verify that root certificate exists, generate it otherwise. */ public void initRootCert(MapOperations compOperations) { @@ -148,7 +152,7 @@ public class CertificateManager { SecurityUtils.logOpenSslExitCode(command, process.exitValue()); exitCode = process.exitValue(); if (exitCode != 0) { - throw new SliderException(exitCode, "Error running command {}", command); + throw new SliderException(exitCode, "Error running command %s", command); } } catch (InterruptedException e) { e.printStackTrace(); @@ -169,6 +173,23 @@ public class CertificateManager { } + public synchronized void generateAgentCertificate(String agentHostname, String containerId) { + LOG.info("Generation of agent certificate for {}", agentHostname); + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + Object[] scriptArgs = {srvrKstrDir, agentHostname, containerId}; + + try { + String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs); + runCommand(command); + + signAgentCertificate(containerId); + + } catch (SliderException e) { + LOG.error("Error generating the agent certificate", e); + } + } + private void generateServerCertificate(){ LOG.info("Generation of server certificate"); @@ -205,8 +226,7 @@ public class CertificateManager { * @return string with server certificate content */ public String getServerCert() { - File certFile = new File(SecurityUtils.getSecurityDir() + - File.separator + SliderKeys.CRT_FILE_NAME); + File certFile = getServerCertficateFilePath(); String srvrCrtContent = null; try { srvrCrtContent = FileUtils.readFileToString(certFile); @@ -216,6 +236,21 @@ public class CertificateManager { return srvrCrtContent; } + public static File getServerCertficateFilePath() { + return new File(SecurityUtils.getSecurityDir() + + File.separator + SliderKeys.CRT_FILE_NAME); + } + + public static File getAgentCertficateFilePath(String containerId) { + return new File(SecurityUtils.getSecurityDir() + + File.separator + containerId + ".crt"); + } + + public static File getAgentKeyFilePath(String containerId) { + return new File(SecurityUtils.getSecurityDir() + + File.separator + containerId + ".key"); + } + /** * Signs agent certificate * Adds agent certificate to server keystore @@ -302,4 +337,35 @@ public class CertificateManager { //LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode)); return response; } + + private String signAgentCertificate (String containerId) + throws SliderException { + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtPass = SecurityUtils.getKeystorePass(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String agentCrtReqName = containerId + ".csr"; + String agentCrtName = containerId + ".crt"; + + Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName, + srvrCrtPass, srvrKeyName, srvrCrtName}; + + //Revoke previous agent certificate if exists + File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName); + + String command; + if (agentCrtFile.exists()) { + LOG.info("Revoking of " + containerId + " certificate."); + command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs); + runCommand(command); + } + + command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs); + + LOG.debug(SecurityUtils.hideOpenSslPassword(command)); + runCommand(command); + + return agentCrtName; + + } }
