Repository: incubator-slider Updated Branches: refs/heads/develop 3337b31d4 -> 0b0022185
SLIDER-585 enable security stores generation and localization Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b002218 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b002218 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b002218 Branch: refs/heads/develop Commit: 0b0022185b78a644bb86d5165676654e484d839a Parents: 3337b31 Author: Jon Maron <[email protected]> Authored: Fri Feb 13 12:48:32 2015 -0500 Committer: Jon Maron <[email protected]> Committed: Fri Feb 13 12:48:32 2015 -0500 ---------------------------------------------------------------------- .../org/apache/slider/common/SliderKeys.java | 15 + .../slider/common/tools/CoreFileSystem.java | 12 + .../providers/agent/AgentProviderService.java | 49 ++- .../server/appmaster/SliderAppMaster.java | 3 +- .../AbstractSecurityStoreGenerator.java | 89 +++++ .../services/security/CertificateManager.java | 81 ++++- .../services/security/KeystoreGenerator.java | 64 ++++ .../security/SecurityStoreGenerator.java | 38 +++ .../server/services/security/SecurityUtils.java | 11 +- .../services/security/StoresGenerator.java | 65 ++++ .../services/security/TruststoreGenerator.java | 63 ++++ .../security/TestCertificateManager.java | 342 ++++++++++++++++++- 12 files changed, 807 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 8437087..5cf7022 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -197,6 +197,21 @@ public interface SliderKeys extends SliderXmlConfKeys { String CRT_PASS_FILE_NAME = "pass.txt"; String PASS_LEN = "50"; + String COMP_STORES_REQUIRED_KEY = + "slider.component.security.stores.required"; + String COMP_KEYSTORE_PASSWORD_PROPERTY_KEY = + "slider.component.keystore.password.property"; + String COMP_KEYSTORE_PASSWORD_ALIAS_KEY = + "slider.component.keystore.credential.alias.property"; + String COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT = + "component.keystore.credential.alias"; + String COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY = + "slider.component.truststore.password.property"; + String COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY = + "slider.component.truststore.credential.alias.property"; + String COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT = + "component.truststore.credential.alias"; + /** * Python specific */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 3a26a1a..e835312 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -134,6 +134,18 @@ public class CoreFileSystem { } /** + * Build up the path string for package install location -no attempt to + * create the directory is made + * + * @return the path for persistent app package + */ + public Path buildClusterSecurityDirPath(String clusterName) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.SECURITY_DIR); + } + + /** * Build up the path string for keytab install location -no attempt to * create the directory is made * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 10804cf..0730beb 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; @@ -94,6 +95,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse; import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus; import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand; import org.apache.slider.server.services.security.CertificateManager; +import org.apache.slider.server.services.security.StoresGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -411,6 +413,13 @@ public class AgentProviderService extends AbstractProviderService implements localizeContainerSSLResources(launcher, container, fileSystem); } + MapOperations compOps = instanceDefinition. + getAppConfOperations().getComponent(role); + if (areStoresRequested(compOps)) { + localizeContainerSecurityStores(launcher, container, role, fileSystem, + instanceDefinition); + } + //add the configuration resources launcher.addLocalResources(fileSystem.submitDirectory( generatedConfPath, @@ -450,14 +459,47 @@ public class AgentProviderService extends AbstractProviderService implements getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME))); } + private void localizeContainerSecurityStores(ContainerLauncher launcher, + Container container, + String role, + SliderFileSystem fileSystem, + AggregateConf instanceDefinition) + throws SliderException, IOException { + MapOperations compOps = instanceDefinition.getAppConfOperations() + .getComponent(role); + // generate and localize security stores + File[] stores = generateSecurityStores(container, role, + instanceDefinition, compOps); + for (File store : stores) { + LocalResource keystoreResource = fileSystem.createAmResource( + uploadSecurityResource(store, fileSystem), LocalResourceType.FILE); + launcher.addLocalResource(String.format("secstores/%s.p12", role), + keystoreResource); + } + } + + private File[] generateSecurityStores(Container container, + String role, + AggregateConf instanceDefinition, + MapOperations compOps) + throws SliderException, IOException { + return StoresGenerator.generateSecurityStores(container.getNodeId().getHost(), + container.getId().toString(), role, + instanceDefinition, compOps); + } + + private boolean areStoresRequested(MapOperations compOps) { + return compOps != null ? Boolean.valueOf(compOps. + getOptionBool(SliderKeys.COMP_STORES_REQUIRED_KEY, false)) : false; + } + private void localizeContainerSSLResources(ContainerLauncher launcher, Container container, SliderFileSystem fileSystem) throws SliderException { try { // localize server cert - Path certsDir = new Path(fileSystem.buildClusterDirPath( - getClusterName()), "certs"); + Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName()); LocalResource certResource = fileSystem.createAmResource( new Path(certsDir, SliderKeys.CRT_FILE_NAME), LocalResourceType.FILE); @@ -492,8 +534,7 @@ public class AgentProviderService extends AbstractProviderService implements private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem) throws IOException { - Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()), - "certs"); + Path certsDir = fileSystem.buildClusterSecurityDirPath(getClusterName()); if (!fileSystem.getFileSystem().exists(certsDir)) { fileSystem.getFileSystem().mkdirs(certsDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index f1e3a84..571c53f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1029,8 +1029,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService private void uploadServerCertForLocalization(String clustername, SliderFileSystem fs) throws IOException { - Path certsDir = new Path(fs.buildClusterDirPath(clustername), - "certs"); + Path certsDir = fs.buildClusterSecurityDirPath(clustername); if (!fs.getFileSystem().exists(certsDir)) { fs.getFileSystem().mkdirs(certsDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java b/slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java new file mode 100644 index 0000000..04daaf4 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * + */ +public abstract class AbstractSecurityStoreGenerator implements + SecurityStoreGenerator { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractSecurityStoreGenerator.class); + + protected CertificateManager certificateMgr; + + public AbstractSecurityStoreGenerator(CertificateManager certificateMgr) { + this.certificateMgr = certificateMgr; + } + + protected String getStorePassword(Map<String, List<String>> credentials, + MapOperations compOps, String role) + throws SliderException, IOException { + String password = getPassword(compOps); + if (password == null) { + // need to leverage credential provider + String alias = getAlias(compOps); + if (alias == null) { + throw new SliderException("No store password or credential provider " + + "alias found"); + } + if (credentials.isEmpty()) { + LOG.info("Credentials can not be retrieved for store generation since " + + "no CP paths are configured"); + } + for (Map.Entry<String, List<String>> cred : credentials.entrySet()) { + String provider = cred.getKey(); + Configuration c = new Configuration(); + c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider); + char[] credential = c.getPassword(alias); + if (credential != null) { + return String.valueOf(credential); + } + } + + if (password == null) { + LOG.info("No store credential found for alias {}. " + + "Generation of store for {} is not possible.", alias, role); + + } + } + + return password; + + } + + @Override + public boolean isStoreRequested(MapOperations compOps) { + return compOps.getOptionBool(SliderKeys.COMP_STORES_REQUIRED_KEY, false); + } + + abstract String getPassword(MapOperations compOps); + + abstract String getAlias(MapOperations compOps); +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java index 4473b74..74d1799 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java @@ -48,8 +48,8 @@ public class CertificateManager { "-extensions jdk7_ca -config {1}/ca.config -batch " + "-infiles {1}/{5}"; private static final String EXPRT_KSTR = "openssl pkcs12 -export" + - " -in {1}/{3} -inkey {1}/{2} -certfile {1}/{3} -out {1}/{4} " + - "-password pass:{0} -passin pass:{0} \n"; + " -in {2}/{4} -inkey {2}/{3} -certfile {2}/{4} -out {2}/{5} " + + "-password pass:{1} -passin pass:{0} \n"; private static final String REVOKE_AGENT_CRT = "openssl ca " + "-config {0}/ca.config -keyfile {0}/{4} -revoke {0}/{2} -batch " + "-passin pass:{3} -cert {0}/{5}"; @@ -196,8 +196,9 @@ public class CertificateManager { } } - public synchronized void generateContainerKeystore(String hostname, + public synchronized File generateContainerKeystore(String hostname, String containerId, + String role, String keystorePass) throws SliderException { LOG.info("Generation of container keystore for container {} on {}", @@ -206,16 +207,24 @@ public class CertificateManager { generateContainerCertificate(hostname, containerId); // come up with correct args to invoke keystore command + String srvrCrtPass = SecurityUtils.getKeystorePass(); String srvrKstrDir = SecurityUtils.getSecurityDir(); String containerCrtName = containerId + ".crt"; String containerKeyName = containerId + ".key"; - String kstrName = String.format("%s-%s.p12", hostname, containerId); + String kstrName = getKeystoreFileName(containerId, role); - Object[] scriptArgs = {keystorePass, srvrKstrDir, containerKeyName, - containerCrtName, kstrName}; + Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir, + containerKeyName, containerCrtName, kstrName}; String command = MessageFormat.format(EXPRT_KSTR, scriptArgs); runCommand(command); + + return new File(srvrKstrDir, kstrName); + } + + private static String getKeystoreFileName(String containerId, + String role) { + return String.format("keystore-%s-%s.p12", containerId, role); } private void generateAMKeystore() throws SliderException { @@ -240,8 +249,34 @@ public class CertificateManager { command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs); runCommand(command); - command = MessageFormat.format(EXPRT_KSTR, scriptArgs); + Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName, + srvrCrtName, kstrName, srvrCsrName}; + command = MessageFormat.format(EXPRT_KSTR, keystoreArgs); + runCommand(command); + } + + public File generateContainerTruststore(String containerId, String role, + String truststorePass) + throws SliderException { + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrCsrName = SliderKeys.CSR_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String kstrName = getTruststoreFileName(role, containerId); + String srvrCrtPass = SecurityUtils.getKeystorePass(); + + Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName, + srvrCrtName, kstrName, srvrCsrName}; + + String command = MessageFormat.format(EXPRT_KSTR, scriptArgs); runCommand(command); + + return new File(srvrKstrDir, kstrName); + } + + private static String getTruststoreFileName(String role, String containerId) { + return String.format("truststore-%s-%s.p12", containerId, role); } /** @@ -260,18 +295,38 @@ public class CertificateManager { } public static File getServerCertficateFilePath() { - return new File(SecurityUtils.getSecurityDir() + - File.separator + SliderKeys.CRT_FILE_NAME); + return new File(String.format("%s%s%s", + SecurityUtils.getSecurityDir(), + File.separator, + SliderKeys.CRT_FILE_NAME)); } public static File getAgentCertficateFilePath(String containerId) { - return new File(SecurityUtils.getSecurityDir() + - File.separator + containerId + ".crt"); + return new File(String.format("%s%s%s.crt", + SecurityUtils.getSecurityDir(), + File.separator, + containerId)); + } + + public static File getContainerKeystoreFilePath(String containerId, + String role) { + return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName( + containerId, + role + )); + } + + public static File getContainerTruststoreFilePath(String role, + String containerId) { + return new File(SecurityUtils.getSecurityDir(), + getTruststoreFileName(role, containerId)); } public static File getAgentKeyFilePath(String containerId) { - return new File(SecurityUtils.getSecurityDir() + - File.separator + containerId + ".key"); + return new File(String.format("%s%s%s.key", + SecurityUtils.getSecurityDir(), + File.separator, + containerId)); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java b/slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java new file mode 100644 index 0000000..8611024 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public class KeystoreGenerator extends AbstractSecurityStoreGenerator { + + + public KeystoreGenerator(CertificateManager certificateMgr) { + super(certificateMgr); + } + + @Override + public File generate(String hostname, String containerId, + AggregateConf instanceDefinition, + MapOperations compOps, String role) + throws SliderException, IOException { + File keystore = null; + String password = getStorePassword( + instanceDefinition.getAppConf().credentials, compOps, role); + if (password != null) { + keystore = + certificateMgr.generateContainerKeystore(hostname, containerId, role, + password); + } + return keystore; + } + + @Override + String getPassword(MapOperations compOps) { + return compOps.get( + compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY)); + } + + @Override + String getAlias(MapOperations compOps) { + return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY, + SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java new file mode 100644 index 0000000..5549a92 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public interface SecurityStoreGenerator { + + File generate(String hostname, String containerId, + AggregateConf instanceDefinition, MapOperations compOps, + String role) + throws SliderException, IOException; + + boolean isStoreRequested(MapOperations compOps); +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java index 107b56e..ce25c34 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java @@ -132,9 +132,14 @@ public class SecurityUtils { } public static String hideOpenSslPassword(String command){ - int start = command.indexOf(PASS_TOKEN)+PASS_TOKEN.length(); - CharSequence cs = command.subSequence(start, command.indexOf(" ", start)); - return command.replace(cs, "****"); + int start = command.indexOf(PASS_TOKEN); + while (start >= 0) { + start += PASS_TOKEN.length(); + CharSequence cs = command.subSequence(start, command.indexOf(" ", start)); + command = command.replace(cs, "****"); + start = command.indexOf(PASS_TOKEN, start + 1); + } + return command; } public static String getOpenSslCommandResult(String command, int exitCode) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java b/slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java new file mode 100644 index 0000000..98395b5 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class StoresGenerator { + + static CertificateManager certMgr = new CertificateManager(); + private static SecurityStoreGenerator[] GENERATORS = { + new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr) + }; + + public static File[] generateSecurityStores(String hostname, + String containerId, + String role, + AggregateConf instanceDefinition, + MapOperations compOps) + throws SliderException, IOException { + //discover which stores need generation based on the passwords configured + List<File> files = new ArrayList<File>(); + for (SecurityStoreGenerator generator : GENERATORS) { + if (generator.isStoreRequested(compOps)) { + File store = generator.generate(hostname, containerId, + instanceDefinition, compOps, role); + if (store != null) { + files.add(store); + } + } + } + + if (files.isEmpty()) { + throw new SliderException("Security stores were requested but none were " + + "generated. Check the AM logs and ensure " + + "passwords are configured for the components " + + "requiring the stores."); + } + return files.toArray(new File[files.size()]); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java b/slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java new file mode 100644 index 0000000..4d0371b --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public class TruststoreGenerator extends AbstractSecurityStoreGenerator { + + + public TruststoreGenerator(CertificateManager certificateMgr) { + super(certificateMgr); + } + + @Override + public File generate(String hostname, String containerId, + AggregateConf instanceDefinition, + MapOperations compOps, String role) + throws SliderException, IOException { + File truststore = null; + String password = getStorePassword( + instanceDefinition.getAppConf().credentials, compOps, role); + if (password != null) { + truststore = certificateMgr.generateContainerTruststore(containerId, + role, password); + } + return truststore; + } + + @Override + String getPassword(MapOperations compOps) { + return compOps.get( + compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY)); + } + + @Override + String getAlias(MapOperations compOps) { + return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY, + SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b002218/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java index 5416650..29d4fa0 100644 --- a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java +++ b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java @@ -16,24 +16,41 @@ */ package org.apache.slider.server.services.security; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.slider.Slider; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; import java.security.Principal; import java.security.cert.Certificate; +import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.util.Enumeration; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; /** * @@ -112,8 +129,15 @@ public class TestCertificateManager { @Test public void testContainerKeystoreGeneration() throws Exception { - certMan.generateContainerKeystore("localhost", "container1", "password"); - File keystoreFile = new File(secDir, "localhost-container1.p12"); + File keystoreFile = certMan.generateContainerKeystore("localhost", + "container1", + "component1", + "password"); + validateKeystore(keystoreFile); + } + + private void validateKeystore(File keystoreFile) + throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { Assert.assertTrue("container keystore not generated", keystoreFile.exists()); @@ -152,4 +176,316 @@ public class TestCertificateManager { } } + @Test + public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = new MapOperations(); + instanceDefinition.getAppConf().components.put("component1", compOps); + compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, + "app1.component1.password.property"); + compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); + instanceDefinition.getAppConf().global.put( + "app1.component1.password.property", "password"); + instanceDefinition.resolve(); + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 1, files.length); + validateKeystore(files[0]); + } + + @Test + public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = setupComponentOptions(true, null, + "app1.component1.password.property", + null, null); + instanceDefinition.getAppConf().components.put("component1", compOps); + instanceDefinition.getAppConf().global.put( + "app1.component1.password.property", "password"); + instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false"); + instanceDefinition.resolve(); + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 1, files.length); + validateKeystore(files[0]); + } + + @Test + public void testContainerTrusttoreGeneration() throws Exception { + File keystoreFile = + certMan.generateContainerKeystore("localhost", + "container1", + "component1", + "keypass"); + Assert.assertTrue("container keystore not generated", + keystoreFile.exists()); + File truststoreFile = + certMan.generateContainerTruststore("container1", + "component1", "trustpass" + ); + Assert.assertTrue("container truststore not generated", + truststoreFile.exists()); + + validateTruststore(keystoreFile, truststoreFile); + } + + @Test + public void testContainerGenerationUsingStoresGeneratorNoTruststore() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = new MapOperations(); + compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); + compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY, + "test.keystore.password"); + + setupCredentials(instanceDefinition, "test.keystore.password", null); + + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 1, files.length); + File keystoreFile = CertificateManager.getContainerKeystoreFilePath( + "container1", "component1"); + Assert.assertTrue("container keystore not generated", + keystoreFile.exists()); + Assert.assertTrue("keystore not in returned list", + Arrays.asList(files).contains(keystoreFile)); + File truststoreFile = + CertificateManager.getContainerTruststoreFilePath("component1", + "container1"); + Assert.assertFalse("container truststore generated", + truststoreFile.exists()); + Assert.assertFalse("truststore in returned list", + Arrays.asList(files).contains(truststoreFile)); + + } + + @Test + public void testContainerGenerationUsingStoresGeneratorJustTruststoreWithDefaultAlias() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = setupComponentOptions(true); + + setupCredentials(instanceDefinition, null, + SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT); + + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 1, files.length); + File keystoreFile = CertificateManager.getContainerKeystoreFilePath( + "container1", "component1"); + Assert.assertFalse("container keystore generated", + keystoreFile.exists()); + Assert.assertFalse("keystore in returned list", + Arrays.asList(files).contains(keystoreFile)); + File truststoreFile = + CertificateManager.getContainerTruststoreFilePath("component1", + "container1"); + Assert.assertTrue("container truststore not generated", + truststoreFile.exists()); + Assert.assertTrue("truststore not in returned list", + Arrays.asList(files).contains(truststoreFile)); + + } + + @Test + public void testContainerTrusttoreGenerationUsingStoresGenerator() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = setupComponentOptions(true, + "test.keystore.password", + null, + "test.truststore.password", + null); + + setupCredentials(instanceDefinition, "test.keystore.password", + "test.truststore.password"); + + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 2, files.length); + File keystoreFile = CertificateManager.getContainerKeystoreFilePath( + "container1", "component1"); + Assert.assertTrue("container keystore not generated", + keystoreFile.exists()); + Assert.assertTrue("keystore not in returned list", + Arrays.asList(files).contains(keystoreFile)); + File truststoreFile = + CertificateManager.getContainerTruststoreFilePath("component1", + "container1"); + Assert.assertTrue("container truststore not generated", + truststoreFile.exists()); + Assert.assertTrue("truststore not in returned list", + Arrays.asList(files).contains(truststoreFile)); + + validateTruststore(keystoreFile, truststoreFile); + } + + private void setupCredentials(AggregateConf instanceDefinition, + String keyAlias, String trustAlias) + throws Exception { + Configuration conf = new Configuration(); + final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks"); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); + + File file = new File(SecurityUtils.getSecurityDir(), "test.jks"); + file.delete(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + + instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>()); + + CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + + // create new aliases + try { + + if (keyAlias != null) { + char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + provider.createCredentialEntry( + keyAlias, storepass); + } + + if (trustAlias != null) { + char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'}; + provider.createCredentialEntry( + trustAlias, trustpass); + } + + // write out so that it can be found in checks + provider.flush(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + private MapOperations setupComponentOptions(boolean storesRequired) { + return this.setupComponentOptions(storesRequired, null, null, null, null); + } + + private MapOperations setupComponentOptions(boolean storesRequired, + String keyAlias, + String keyPwd, + String trustAlias, + String trustPwd) { + MapOperations compOps = new MapOperations(); + compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, + Boolean.toString(storesRequired)); + if (keyAlias != null) { + compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY, + "test.keystore.password"); + } + if (trustAlias != null) { + compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY, + "test.truststore.password"); + } + if (keyPwd != null) { + compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY, + keyPwd); + } + if (trustPwd != null) { + compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY, + trustPwd); + } + return compOps; + } + + @Test + public void testContainerStoresGenerationKeystoreOnly() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = new MapOperations(); + compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); + + setupCredentials(instanceDefinition, + SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null); + + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + assertEquals("wrong number of stores", 1, files.length); + File keystoreFile = CertificateManager.getContainerKeystoreFilePath( + "container1", "component1"); + Assert.assertTrue("container keystore not generated", + keystoreFile.exists()); + Assert.assertTrue("keystore not in returned list", + Arrays.asList(files).contains(keystoreFile)); + File truststoreFile = + CertificateManager.getContainerTruststoreFilePath("component1", + "container1"); + Assert.assertFalse("container truststore generated", + truststoreFile.exists()); + Assert.assertFalse("truststore in returned list", + Arrays.asList(files).contains(truststoreFile)); + + } + + @Test + public void testContainerStoresGenerationMisconfiguration() throws Exception { + AggregateConf instanceDefinition = new AggregateConf(); + MapOperations compOps = new MapOperations(); + compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true"); + + setupCredentials(instanceDefinition, "cant.be.found", null); + + try { + File[] files = StoresGenerator.generateSecurityStores("localhost", "container1", + "component1", instanceDefinition, + compOps); + Assert.fail("SliderException should have been generated"); + } catch (SliderException e) { + // ignore - should be thrown + } + } + + private void validateTruststore(File keystoreFile, File truststoreFile) + throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException { + InputStream keyis = null; + InputStream trustis = null; + try { + + // create keystore + keyis = new FileInputStream(keystoreFile); + KeyStore keystore = KeyStore.getInstance("pkcs12"); + String password = "keypass"; + keystore.load(keyis, password.toCharArray()); + + // obtain server cert + Certificate certificate = keystore.getCertificate( + keystore.aliases().nextElement()); + Assert.assertNotNull(certificate); + + // create trust store from generated trust store file + trustis = new FileInputStream(truststoreFile); + KeyStore truststore = KeyStore.getInstance("pkcs12"); + password = "trustpass"; + truststore.load(trustis, password.toCharArray()); + + // validate keystore cert using trust store + TrustManagerFactory + trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(truststore); + + for (TrustManager trustManager: trustManagerFactory.getTrustManagers()) { + if (trustManager instanceof X509TrustManager) { + X509TrustManager x509TrustManager = (X509TrustManager)trustManager; + x509TrustManager.checkServerTrusted( + new X509Certificate[] {(X509Certificate) certificate}, + "RSA_EXPORT"); + } + } + + } finally { + if(null != keyis) { + keyis.close(); + } + if(null != trustis) { + trustis.close(); + } + } + } + }
