Repository: incubator-slider Updated Branches: refs/heads/develop e39d99e19 -> 3b3886d4f
SLIDER-497 Secure_funtests_failing_no_keytab Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b3886d4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b3886d4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b3886d4 Branch: refs/heads/develop Commit: 3b3886d4ff5489daca9685f789038d53de16cfbf Parents: e39d99e Author: Steve Loughran <[email protected]> Authored: Thu Oct 9 18:40:27 2014 -0700 Committer: Steve Loughran <[email protected]> Committed: Thu Oct 9 18:40:27 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 10 ++- .../org/apache/slider/common/SliderKeys.java | 5 -- .../common/SliderXMLConfKeysForTesting.java | 5 ++ .../apache/slider/common/SliderXmlConfKeys.java | 5 ++ .../providers/agent/AgentProviderService.java | 5 +- .../server/appmaster/SliderAppMaster.java | 2 +- .../security/SecurityConfiguration.java | 27 +++---- .../server/services/security/SecurityUtils.java | 5 +- .../agent/TestAgentAMManagementWS.groovy | 4 +- .../security/SecurityConfigurationTest.groovy | 25 +++---- .../security/TestCertificateManager.java | 3 +- .../framework/AgentCommandTestBase.groovy | 15 ++-- .../funtest/framework/CommandTestBase.groovy | 75 +++++++++++++++++++- .../funtest/lifecycle/AMFailuresIT.groovy | 9 +-- .../lifecycle/AgentClusterLifecycleIT.groovy | 10 +-- .../funtest/lifecycle/AgentFailures2IT.groovy | 12 ++-- .../funtest/lifecycle/AgentFailuresIT.groovy | 9 +-- .../funtest/lifecycle/AppsThroughAgentIT.groovy | 9 +-- .../AppsThroughAgentQueueAndLabelsIT.groovy | 12 ++-- .../clusters/morzine/slider/slider-client.xml | 17 +++++ .../clusters/offline/slider/slider-client.xml | 16 +++++ .../clusters/sandbox/slider/slider-client.xml | 17 +++++ 22 files changed, 206 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 476f300..78f214e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.fs.permission.FsPermission; @@ -125,6 +124,7 @@ import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; +import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -1106,6 +1106,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // will be valid. propagatePrincipals(config, instanceDefinition); + // validate security data +/* + // turned off until tested + SecurityConfiguration securityConfiguration = + new SecurityConfiguration(config, + instanceDefinition, clustername); + +*/ Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 68cef45..e75ec73 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -170,11 +170,6 @@ public interface SliderKeys extends SliderXmlConfKeys { String CRT_PASS_FILE_NAME = "pass.txt"; String PASSPHRASE = "DEV"; String PASS_LEN = "50"; - String KEYSTORE_LOCATION = "ssl.server.keystore.location"; - String AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name"; - String AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path"; - String KEYTAB_PRINCIPAL = "slider.keytab.principal.name"; - String SECURITY_ENABLED = "site.global.security_enabled"; /** * Python specific http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java index 8886cb7..bc1eaba 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java @@ -83,4 +83,9 @@ public interface SliderXMLConfKeysForTesting { * security related keys */ String TEST_SECURITY_DIR = "/tmp/work/security"; + + /** + * Local path to AM keytab: {@value} + */ + String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index 243dd95..d82bbe8 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -142,4 +142,9 @@ public interface SliderXmlConfKeys { "ipc.client.fallback-to-simple-auth-allowed"; String HADOOP_HTTP_FILTER_INITIALIZERS = "hadoop.http.filter.initializers"; + String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location"; + String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name"; + String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path"; + String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name"; + String KEY_SECURITY_ENABLED = "site.global.security_enabled"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 4fe4b6a..058a838 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -40,6 +40,7 @@ import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.AggregateConf; @@ -358,10 +359,10 @@ public class AgentProviderService extends AbstractProviderService implements if (SliderUtils.isHadoopClusterSecure(getConfig())) { String keytabFullPath = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM).get( - SliderKeys.AM_KEYTAB_LOCAL_PATH); + SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); String amKeytabName = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM).get( - SliderKeys.AM_LOGIN_KEYTAB_NAME); + SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); if (SliderUtils.isUnset(keytabFullPath)) { // we need to localize the keytab files in the directory Path keytabDir = fileSystem.buildKeytabPath(null, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index fcc2802..3f52fd8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -560,7 +560,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService boolean securityEnabled = securityConfiguration.isSecurityEnabled(); // set the global security flag for the instance definition instanceDefinition.getAppConfOperations().set( - SECURITY_ENABLED, securityEnabled); + KEY_SECURITY_ENABLED, securityEnabled); // triggers resolution and snapshotting in agent appState.updateInstanceDefinition(instanceDefinition); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java index 448d02f..e5cdad2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.AggregateConf; @@ -63,7 +64,7 @@ public class SecurityConfiguration { private void validate() throws SliderException { if (isSecurityEnabled()) { String principal = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL); + .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); if(SliderUtils.isUnset(principal)) { // if no login identity is available, fail UserGroupInformation loginUser = null; @@ -71,13 +72,13 @@ public class SecurityConfiguration { loginUser = getLoginUser(); } catch (IOException e) { throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e, - "No principal configured for the application and" + "No principal configured for the application and " + "exception raised during retrieval of login user. " + "Unable to proceed with application " + "initialization. Please ensure a value " + "for %s exists in the application " + "configuration or the login issue is addressed", - SliderKeys.KEYTAB_PRINCIPAL); + SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); } if (loginUser == null) { throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION, @@ -87,25 +88,25 @@ public class SecurityConfiguration { + "initialization. Please ensure a value " + "for %s exists in the application " + "configuration or the login issue is addressed", - SliderKeys.KEYTAB_PRINCIPAL); + SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); } } // ensure that either local or distributed keytab mechanism is enabled, // but not both String keytabFullPath = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM) - .get(SliderKeys.AM_KEYTAB_LOCAL_PATH); + .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); String keytabName = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM) - .get(SliderKeys.AM_LOGIN_KEYTAB_NAME); + .get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); if (SliderUtils.isUnset(keytabFullPath) && SliderUtils.isUnset(keytabName)) { throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION, "Either a keytab path on the cluster host (%s) or a" + " keytab to be retrieved from HDFS (%s) are" + " required. Please configure one of the keytab" + " retrieval mechanisms.", - SliderKeys.AM_KEYTAB_LOCAL_PATH, - SliderKeys.AM_LOGIN_KEYTAB_NAME); + SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); } if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) { throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION, @@ -113,8 +114,8 @@ public class SecurityConfiguration { + " keytab to be retrieved from HDFS (%s) are" + " specified. Please configure only one keytab" + " retrieval mechanism.", - SliderKeys.AM_KEYTAB_LOCAL_PATH, - SliderKeys.AM_LOGIN_KEYTAB_NAME); + SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, + SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); } } @@ -130,7 +131,7 @@ public class SecurityConfiguration { public String getPrincipal () throws IOException { String principal = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL); + .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); if (SliderUtils.isUnset(principal)) { principal = UserGroupInformation.getLoginUser().getShortUserName(); log.info("No principal set in the slider configuration. Will use AM login" @@ -145,12 +146,12 @@ public class SecurityConfiguration { throws SliderException, IOException { String keytabFullPath = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM) - .get(SliderKeys.AM_KEYTAB_LOCAL_PATH); + .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); File localKeytabFile; if (SliderUtils.isUnset(keytabFullPath)) { // get the keytab String keytabName = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.AM_LOGIN_KEYTAB_NAME); + .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); log.info("No host keytab file path specified. Downloading keytab {}" + " from HDFS to perform login of using principal {}", keytabName, principal); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java index 56ee199..c7ad8dd 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.core.conf.MapOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +35,7 @@ import java.io.IOException; //import java.nio.file.Paths; //import java.nio.file.attribute.PosixFilePermission; //import java.nio.file.attribute.PosixFilePermissions; -import java.util.Set; + /** * @@ -140,7 +141,7 @@ public class SecurityUtils { public static void initializeSecurityParameters(MapOperations configMap) { String keyStoreLocation = configMap.getOption( - SliderKeys.KEYSTORE_LOCATION, getDefaultKeystoreLocation()); + SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation()); File secDirFile = new File(keyStoreLocation).getParentFile(); if (!secDirFile.exists()) { // create entire required directory structure http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy index 7a39035..da62792 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy @@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException import org.apache.slider.api.StatusKeys import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderKeys -import org.apache.slider.common.params.AbstractClusterBuildingActionArgs +import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.core.build.InstanceBuilder import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.MapOperations @@ -90,7 +90,7 @@ class TestAgentAMManagementWS extends AgentTestBase { void setup() { super.setup() MapOperations compOperations = new MapOperations(); - compOperations.put(SliderKeys.KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12"); + compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12"); SecurityUtils.initializeSecurityParameters(compOperations); CertificateManager certificateManager = new CertificateManager(); certificateManager.initRootCert(compOperations); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy index 1dcbd9c..4ef142a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.CommonConfigurationKeysPublic import org.apache.hadoop.security.UserGroupInformation import org.apache.slider.common.SliderKeys +import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.MapOperations import org.apache.slider.core.exceptions.SliderException; @@ -38,8 +39,8 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test") - compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path") + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test") + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path") SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, aggregateConf, "testCluster") @@ -52,8 +53,8 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test") - compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab") + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test") + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab") SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, aggregateConf, "testCluster") @@ -66,7 +67,7 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab") + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab") shouldFail(SliderException) { SecurityConfiguration securityConfiguration = @@ -86,7 +87,7 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path") + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path") shouldFail(SliderException) { SecurityConfiguration securityConfiguration = @@ -106,9 +107,9 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test") - compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path") - compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab") + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test") + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path") + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab") shouldFail(SliderException) { SecurityConfiguration securityConfiguration = @@ -123,7 +124,7 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test") + compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test") shouldFail(SliderException) { SecurityConfiguration securityConfiguration = @@ -138,7 +139,7 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab") + compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab") SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, aggregateConf, "testCluster") @@ -151,7 +152,7 @@ public class SecurityConfigurationTest { AggregateConf aggregateConf = new AggregateConf(); MapOperations compOps = aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM) - compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path") + compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path") SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, aggregateConf, "testCluster") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java index 6d2d051..710757d 100644 --- a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java +++ b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java @@ -17,6 +17,7 @@ package org.apache.slider.server.services.security; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.core.conf.MapOperations; import org.junit.Assert; import org.junit.Before; @@ -40,7 +41,7 @@ public class TestCertificateManager { MapOperations compOperations = new MapOperations(); secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR); File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME); - compOperations.put(SliderKeys.KEYSTORE_LOCATION, + compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, keystoreFile.getAbsolutePath()); certMan.initRootCert(compOperations); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 602fe2c..ec7d3e3 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -21,7 +21,6 @@ package org.apache.slider.funtest.framework import groovy.util.logging.Slf4j import org.apache.hadoop.fs.Path import org.apache.slider.common.SliderExitCodes -import org.apache.slider.common.SliderXMLConfKeysForTesting import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions import org.apache.slider.common.tools.SliderUtils @@ -111,7 +110,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { [ ACTION_INSTALL_PACKAGE, Arguments.ARG_NAME, TEST_APP_PKG_NAME, - Arguments.ARG_PACKAGE, zipFileName, + Arguments.ARG_PACKAGE, zipFileName.absolutePath, Arguments.ARG_REPLACE_PKG ]) logShell(shell) @@ -123,12 +122,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { } public static void logShell(SliderShell shell) { - for (String str in shell.out) { - log.info str - } - for (String str in shell.err) { - log.error str - } + shell.dumpOutput(); } public static void assertComponentCount(String component, int count, SliderShell shell) { @@ -185,13 +179,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { } public static void addDir(File dirObj, ZipOutputStream zipFile, String prefix) { - dirObj.eachFile() { file -> + dirObj.eachFile() {File file -> if (file.directory) { addDir(file, zipFile, prefix + file.name + File.separator) } else { log.info("Adding to zip - " + prefix + file.getName()) zipFile.putNextEntry(new ZipEntry(prefix + file.getName())) - file.eachByte(1024) { buffer, len -> zipFile.write(buffer, 0, len) } + file.eachByte(1024) { + byte[] buffer, int len -> zipFile.write(buffer, 0, len) } zipFile.closeEntry() } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index d780ac7..db1c562 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path import org.apache.hadoop.util.ExitUtil import org.apache.hadoop.util.Shell import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.core.main.LauncherExitCodes import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys @@ -71,7 +72,12 @@ abstract class CommandTestBase extends SliderTestUtils { public static final int SLIDER_TEST_TIMEOUT public static final String YARN_RAM_REQUEST - + + /** + * Keytab for secure cluster + */ + public static final String TEST_AM_KEYTAB + static File keytabFile static { @@ -86,10 +92,13 @@ abstract class CommandTestBase extends SliderTestUtils { KEY_TEST_TIMEOUT, 1000 * DEFAULT_TEST_TIMEOUT_SECONDS) - YARN_RAM_REQUEST = SLIDER_CONFIG.get( + YARN_RAM_REQUEST = SLIDER_CONFIG.getTrimmed( KEY_TEST_YARN_RAM_REQUEST, DEFAULT_YARN_RAM_REQUEST) - + + TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed( + KEY_TEST_AM_KEYTAB) + } @Rule @@ -102,6 +111,18 @@ abstract class CommandTestBase extends SliderTestUtils { if (SliderUtils.maybeInitSecurity(conf)) { log.debug("Security enabled") SliderUtils.forceLogin() + // now look for the security key +/* + if (!TEST_AM_KEYTAB) { + fail("Security keytab is not defined in $KEY_TEST_AM_KEYTAB") + } + keytabFile = new File(TEST_AM_KEYTAB) + if (!keytabFile.exists()) { + throw new FileNotFoundException("Security keytab ${keytabFile} " + + " defined in $KEY_TEST_AM_KEYTAB") + } +*/ + } else { log.info "Security is off" } @@ -465,6 +486,54 @@ abstract class CommandTestBase extends SliderTestUtils { clusterOps) } + /** + * Create a templated slider app + * @param name name + * @param appTemplate application template + * @param resourceTemplate resource template + * @return the shell + */ + public SliderShell createTemplatedSliderApplication( + String name, + String appTemplate, + String resourceTemplate, + List<String> extraArgs=[]) { + List<String> commands = [ + ACTION_CREATE, name, + ARG_TEMPLATE, appTemplate, + ARG_RESOURCES, resourceTemplate + ] + + maybeAddCommandOption(commands, + [ARG_COMP_OPT, SliderKeys.COMPONENT_AM], + SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME)); + maybeAddCommandOption(commands, + [ARG_COMP_OPT, SliderKeys.COMPONENT_AM], + SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH)); + maybeAddCommandOption(commands, + [ARG_COMP_OPT, SliderKeys.COMPONENT_AM], + SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL)); + commands.addAll(extraArgs) + SliderShell shell = slider(LauncherExitCodes.EXIT_SUCCESS, commands) + return shell + } + + /** + * If the option is not null/empty, add the command and the option + * @param args arg list being built up + * @param command command to add option + * @param option option to probe and use + * @return the (possibly extended) list + */ + public List<String> maybeAddCommandOption( + List<String> args, List<String> commands, String option) { + if ( SliderUtils.isSet(option)) { + args.addAll(commands) + args << option + } + return args + } + public Path buildClusterPath(String clustername) { return new Path( clusterFS.homeDirectory, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy index 9db0fec..988d34d 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy @@ -60,12 +60,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { @Test public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable { cleanup(APPLICATION_NAME) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, APPLICATION_NAME, - ARG_TEMPLATE, APP_TEMPLATE, - ARG_RESOURCES, APP_RESOURCE - ]) + SliderShell shell = createTemplatedSliderApplication( + APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE + ) logShell(shell) ensureApplicationIsUp(APPLICATION_NAME) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy index 48ae266..d048aca 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy @@ -64,13 +64,9 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase def clusterpath = buildClusterPath(CLUSTER) assert !clusterFS.exists(clusterpath) - - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, CLUSTER, - ARG_TEMPLATE, APP_TEMPLATE, - ARG_RESOURCES, APP_RESOURCE2 - ]) + SliderShell shell = createTemplatedSliderApplication(CLUSTER, + APP_TEMPLATE, + APP_RESOURCE2) logShell(shell) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy index 9359521..a02fc0e 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy @@ -53,12 +53,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { } cleanup(APPLICATION_NAME) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, APPLICATION_NAME, - ARG_TEMPLATE, APP_TEMPLATE3, - ARG_RESOURCES, APP_RESOURCE - ]) + SliderShell shell = createTemplatedSliderApplication(APP_TEMPLATE3, + ARG_RESOURCES, + APP_RESOURCE) logShell(shell) @@ -74,7 +71,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { APPLICATION_NAME]) assertComponentCount(COMMAND_LOGGER, 1, shell) - String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[]) + String requested = findLineEntryValue(shell, + ["statistics", COMMAND_LOGGER, "containers.requested"] as String[]) assert requested != null && requested.isInteger() && requested.toInteger() >= 3, 'At least 2 containers must be requested' http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy index fcbfb3c..7575fc6 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy @@ -53,12 +53,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { } cleanup(APPLICATION_NAME) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, APPLICATION_NAME, - ARG_TEMPLATE, APP_TEMPLATE2, - ARG_RESOURCES, APP_RESOURCE - ]) + def shell = createTemplatedSliderApplication( APPLICATION_NAME, + APP_TEMPLATE2, + APP_RESOURCE) logShell(shell) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy index 7e39791..234275a 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy @@ -47,12 +47,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assumeAgentTestsEnabled() cleanup(APPLICATION_NAME) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, APPLICATION_NAME, - ARG_TEMPLATE, APP_TEMPLATE, - ARG_RESOURCES, APP_RESOURCE - ]) + SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME, + APP_TEMPLATE, + APP_RESOURCE) logShell(shell) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy index a3b0ccb..6732691 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy @@ -78,13 +78,11 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assumeLabelsRedAndBlueAdded() cleanup(APPLICATION_NAME) - SliderShell shell = slider(EXIT_SUCCESS, - [ - ACTION_CREATE, APPLICATION_NAME, - ARG_QUEUE, TARGET_QUEUE, - ARG_TEMPLATE, APP_TEMPLATE, - ARG_RESOURCES, APP_RESOURCE4 - ]) + SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME, + APP_TEMPLATE, + APP_RESOURCE4, + [ARG_QUEUE, TARGET_QUEUE] + ) logShell(shell) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/morzine/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/morzine/slider/slider-client.xml b/src/test/clusters/morzine/slider/slider-client.xml index 4d7ab41..dbf25bd 100644 --- a/src/test/clusters/morzine/slider/slider-client.xml +++ b/src/test/clusters/morzine/slider/slider-client.xml @@ -73,4 +73,21 @@ </property> + <property> + <name>slider.am.login.keytab.name</name> + <value>Location of keytab in HDFS</value> + </property> + + <property> + <name>slider.am.keytab.local.path</name> + <description>absolute path to keytab</description> + <value></value> + </property> + + <property> + <name>slider.keytab.principal.name</name> + <value>Optional principal name in keytab</value> + </property> + + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/offline/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/offline/slider/slider-client.xml b/src/test/clusters/offline/slider/slider-client.xml index 25c01cf..0f780f0 100644 --- a/src/test/clusters/offline/slider/slider-client.xml +++ b/src/test/clusters/offline/slider/slider-client.xml @@ -90,6 +90,22 @@ <value>file://${user.dir}/src/test/configs/sandbox/accumulo</value> </property> + <property> + <name>slider.am.login.keytab.name</name> + <value>Location of keytab in HDFS</value> + </property> + + <property> + <name>slider.am.keytab.local.path</name> + <description>absolute path to keytab</description> + <value></value> + </property> + + <property> + <name>slider.keytab.principal.name</name> + <value>Optional principal name in keytab</value> + </property> + <property> <name>zk.home</name> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b3886d4/src/test/clusters/sandbox/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml index 5ac5d59..41629ce 100644 --- a/src/test/clusters/sandbox/slider/slider-client.xml +++ b/src/test/clusters/sandbox/slider/slider-client.xml @@ -107,6 +107,23 @@ <property> + <name>slider.am.login.keytab.name</name> + <value>Location of keytab in HDFS</value> + </property> + + <property> + <name>slider.am.keytab.local.path</name> + <description>absolute path to keytab</description> + <value></value> + </property> + + <property> + <name>slider.keytab.principal.name</name> + <value>Optional principal name in keytab</value> + </property> + + + <property> <name>zk.home</name> <value>/usr/lib/zookeeper</value> <description>Zookeeper home dir on target systems</description>
