SLIDER-1081 add a command, "slider tokens" to save/list tokens to a file
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/297e9319 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/297e9319 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/297e9319 Branch: refs/heads/feature/SLIDER-906_docker_support Commit: 297e9319f1058a8a5bf0af6582972b0af404a4fa Parents: 8004bc8 Author: Steve Loughran <ste...@apache.org> Authored: Wed Feb 3 17:35:05 2016 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Feb 3 18:00:37 2016 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 22 ++- .../apache/slider/client/TokensOperation.java | 109 ++++++++++++ .../slider/common/params/ActionTokensArgs.java | 78 +++++++++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 10 ++ .../slider/common/params/SliderActions.java | 16 +- .../slider/core/launch/CredentialUtils.java | 174 +++++++++++++++---- .../client/TestSliderTokensCommand.groovy | 129 ++++++++++++++ .../apache/slider/test/SliderTestUtils.groovy | 20 ++- 9 files changed, 514 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index c141d25..21a1cb6 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -102,6 +102,7 @@ import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; +import org.apache.slider.common.params.ActionTokensArgs; import org.apache.slider.common.params.ActionUpgradeArgs; import org.apache.slider.common.params.Arguments; import org.apache.slider.common.params.ClientArgs; @@ -448,6 +449,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs()); break; + case ACTION_TOKENS: + exitCode = actionTokens(serviceArgs.getActionTokenArgs()); + break; + case ACTION_UPDATE: exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs()); break; @@ -1916,7 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Credentials credentials = null; if (clusterSecure) { // pick up oozie credentials - credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config); + credentials = CredentialUtils.loadTokensFromEnvironment(System.getenv(), + config); if (credentials == null) { // nothing from oozie, so build up directly credentials = new Credentials( @@ -4373,6 +4379,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throws IOException, YarnException { return new SliderApplicationIpcClient(createClusterOperations()); } + + /** + * Save/list tokens. This is for testing oozie integration + * @param args commands + * @return status + */ + private int actionTokens(ActionTokensArgs args) + throws IOException, YarnException { + return new TokensOperation().actionTokens(args, + sliderFileSystem.getFileSystem(), + getConfig(), + yarnClient); + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java new file mode 100644 index 0000000..9b9c141 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/client/TokensOperation.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.common.params.ActionTokensArgs; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.NotFoundException; +import static org.apache.slider.core.launch.CredentialUtils.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +public class TokensOperation { + + private static final Logger log = LoggerFactory.getLogger(TokensOperation.class); + public static final String E_INSECURE + = "Cluster is not secure -tokens cannot be acquired"; + public static final String E_MISSING_SOURCE_FILE = "Missing source file: "; + public static final String E_NO_KEYTAB = "No keytab: "; + + public int actionTokens(ActionTokensArgs args, FileSystem fs, + Configuration conf, + YarnClientImpl yarnClient) + throws IOException, YarnException { + Credentials credentials; + String footnote = ""; + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + boolean isSecure = UserGroupInformation.isSecurityEnabled(); + if (args.keytab != null) { + File keytab = args.keytab; + if (!keytab.isFile()) { + throw new NotFoundException(E_NO_KEYTAB + keytab.getAbsolutePath()); + } + String principal = args.principal; + log.info("Logging in as {} from keytab {}", principal, keytab); + user = UserGroupInformation.loginUserFromKeytabAndReturnUGI( + principal, keytab.getCanonicalPath()); + } + Credentials userCredentials = user.getCredentials(); + File output = args.output; + if (output != null) { + if (!isSecure) { + throw new BadClusterStateException(E_INSECURE); + } + credentials = new Credentials(userCredentials); + // filesystem + addRMRenewableFSDelegationTokens(conf, fs, credentials); + addRMDelegationToken(yarnClient, credentials); + if (maybeAddTimelineToken(conf, credentials) != null) { + log.debug("Added timeline token"); + } + saveTokens(output, credentials); + String filename = output.getCanonicalPath(); + footnote = String.format("%d tokens saved to %s\n" + + "To use these in the environment:\n" + + "export %s=%s", + credentials.numberOfTokens(), + filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename); + } else if (args.source != null) { + File source = args.source; + log.info("Reading credentials from file {}", source); + if (!source.isFile()) { + throw new NotFoundException( E_MISSING_SOURCE_FILE + source.getAbsolutePath()); + } + credentials = Credentials.readTokenStorageFile(args.source, conf); + } else { + StringBuffer origin = new StringBuffer(); + File file = locateEnvCredentials(System.getenv(), conf, + origin); + if (file != null) { + log.info("Credential Source {}", origin); + } else { + log.info("Credential source: logged in user"); + } + credentials = userCredentials; + } + // list the tokens + log.info("\n{}", dumpTokens(credentials, "\n")); + if (!footnote.isEmpty()) { + log.info(footnote); + } + return 0; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java new file mode 100644 index 0000000..9f93c4e --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionTokensArgs.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.UsageException; + +import java.io.File; + +@Parameters(commandNames = {SliderActions.ACTION_TOKENS}, + commandDescription = "save tokens to a file or list tokens in a file") +public class ActionTokensArgs extends AbstractActionArgs { + + public static final String DUPLICATE_ARGS = "Only one of " + + ARG_SOURCE + " and " + ARG_OUTPUT + " allowed"; + + public static final String MISSING_KT_PROVIDER = + "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL + + " must be provided"; + + @Override + public String getActionName() { + return SliderActions.ACTION_TOKENS; + } + + @Parameter(names = {ARG_OUTPUT}, + description = "File to write") + public File output; + + @Parameter(names = {ARG_SOURCE}, + description = "source file") + public File source; + + @Parameter(names = {ARG_KEYTAB}, description = "keytab to use") + public File keytab; + + @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab") + public String principal=""; + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public void validate() throws BadCommandArgumentsException, UsageException { + super.validate(); + if (output != null && source != null) { + throw new BadCommandArgumentsException(DUPLICATE_ARGS); + } + + // this is actually a !xor + if (keytab != null ^ !principal.isEmpty()) { + throw new BadCommandArgumentsException(MISSING_KT_PROVIDER); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index d133f25..bac20d7 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -113,6 +113,7 @@ public interface Arguments { String ARG_SERVICETYPE = "--servicetype"; String ARG_SERVICES = "--services"; String ARG_SLIDER = "--slider"; + String ARG_SOURCE = "--source"; String ARG_STATE = "--state"; String ARG_SYSPROP = "-S"; String ARG_TEMPLATE = "--template"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index b441a2a..0a658ea 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -77,6 +77,7 @@ public class ClientArgs extends CommonArgs { private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs(); private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs(); private final ActionThawArgs actionThawArgs = new ActionThawArgs(); + private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs(); private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs(); private final ActionUpgradeArgs actionUpgradeArgs = new ActionUpgradeArgs(); private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs(); @@ -117,6 +118,7 @@ public class ClientArgs extends CommonArgs { actionResolveArgs, actionStatusArgs, actionThawArgs, + actionTokenArgs, actionUpdateArgs, actionUpgradeArgs, actionVersionArgs @@ -233,6 +235,10 @@ public class ClientArgs extends CommonArgs { return actionThawArgs; } + public ActionTokensArgs getActionTokenArgs() { + return actionTokenArgs; + } + /** * Look at the chosen action and bind it as the core action for the operation. * @throws SliderException bad argument or similar @@ -344,6 +350,10 @@ public class ClientArgs extends CommonArgs { bindCoreAction(actionStatusArgs); break; + case ACTION_TOKENS: + bindCoreAction(actionTokenArgs); + break; + case ACTION_UPDATE: bindCoreAction(actionUpdateArgs); break; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index 5849e5e..aab7c98 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -26,7 +26,9 @@ package org.apache.slider.common.params; public interface SliderActions { String ACTION_AM_SUICIDE = "am-suicide"; String ACTION_BUILD = "build"; + String ACTION_CLIENT = "client"; String ACTION_CREATE = "create"; + String ACTION_DIAGNOSTICS = "diagnostics"; String ACTION_DEPENDENCY = "dependency"; String ACTION_UPDATE = "update"; String ACTION_UPGRADE = "upgrade"; @@ -36,26 +38,26 @@ public interface SliderActions { String ACTION_FLEX = "flex"; String ACTION_FREEZE = "stop"; String ACTION_HELP = "help"; + String ACTION_INSTALL_KEYTAB = "install-keytab"; + String ACTION_INSTALL_PACKAGE = "install-package"; String ACTION_KDIAG = "kdiag"; + String ACTION_KEYTAB = "keytab"; String ACTION_KILL_CONTAINER = "kill-container"; String ACTION_LIST = "list"; String ACTION_LOOKUP = "lookup"; String ACTION_NODES = "nodes"; + String ACTION_PACKAGE = "package"; String ACTION_PREFLIGHT = "preflight"; String ACTION_RECONFIGURE = "reconfigure"; String ACTION_REGISTRY = "registry"; String ACTION_RESOLVE = "resolve"; String ACTION_STATUS = "status"; String ACTION_THAW = "start"; + String ACTION_TOKENS = "tokens"; + String ACTION_VERSION = "version"; - String ACTION_DIAGNOSTICS = "diagnostics"; - String ACTION_INSTALL_PACKAGE = "install-package"; - String ACTION_PACKAGE = "package"; - String ACTION_INSTALL_KEYTAB = "install-keytab"; - String ACTION_CLIENT = "client"; - String ACTION_KEYTAB = "keytab"; String DESCRIBE_ACTION_AM_SUICIDE = - "Tell the Slider Application Master to simulate a process failure by terminating itself"; + "Tell the Slider Application Master to simulate a process failure by terminating itself"; String DESCRIBE_ACTION_BUILD = "Build a Slider cluster specification, but do not start it"; String DESCRIBE_ACTION_CREATE = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java index 3245c13..0f4f534 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java @@ -29,18 +29,30 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.slider.common.SliderXmlConfKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataOutputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.text.DateFormat; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -83,47 +95,74 @@ public final class CredentialUtils { return buffer; } - /** - * Load the credentials from the environment. This looks at - * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION} - * and attempts to read in the value - * @param env environment to resolve the variable from - * @param conf configuration use when reading the tokens - * @return a set of credentials, or null if the environment did not - * specify any - * @throws IOException if a location for credentials was defined, but - * the credentials could not be loaded. - */ - public static Credentials loadFromEnvironment(Map<String, String> env, - Configuration conf) - throws IOException { + public static File locateEnvCredentials(Map<String, String> env, + Configuration conf, + StringBuffer sourceTextOut) throws FileNotFoundException { String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION); - String source = HADOOP_TOKEN_FILE_LOCATION; + String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION; if (tokenFilename == null) { tokenFilename = conf.get(JOB_CREDENTIALS_BINARY); - source = "Configuration option " + JOB_CREDENTIALS_BINARY; + source = "configuration option " + JOB_CREDENTIALS_BINARY; } if (tokenFilename != null) { // use delegation tokens, i.e. from Oozie File file = new File(tokenFilename.trim()); - String details = String.format("Token File %s from environment variable %s", + String details = String.format( + "Token File %s from %s", file, source); - LOG.debug("Using {}", details); if (!file.exists()) { throw new FileNotFoundException("No " + details); } if (!file.isFile() && !file.canRead()) { - throw new IOException("Cannot read " + details); + throw new FileNotFoundException("Cannot read " + details); } - Credentials creds = Credentials.readTokenStorageFile(file, conf); - return creds; + sourceTextOut.append(details); + return file; } else { return null; } } /** + * Load the credentials from the environment. This looks at + * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION} + * and attempts to read in the value + * @param env environment to resolve the variable from + * @param conf configuration use when reading the tokens + * @return a set of credentials, or null if the environment did not + * specify any + * @throws IOException if a location for credentials was defined, but + * the credentials could not be loaded. + */ + public static Credentials loadTokensFromEnvironment(Map<String, String> env, + Configuration conf) + throws IOException { + StringBuffer origin = new StringBuffer(); + File file = locateEnvCredentials(env, conf, origin); + if (file != null) { + LOG.debug("Using {}", origin); + return Credentials.readTokenStorageFile(file, conf); + } else { + return null; + } + } + + /** + * Save credentials to a file + * @param file file to save to (will be overwritten) + * @param credentials credentials to write + * @throws IOException + */ + public static void saveTokens(File file, + Credentials credentials) throws IOException { + try(DataOutputStream daos = new DataOutputStream( + new FileOutputStream(file))) { + credentials.writeTokenStorageToStream(daos); + } + } + + /** * Look up and return the resource manager's principal. This method * automatically does the <code>_HOST</code> replacement in the principal and * correctly handles HA resource manager configurations. @@ -179,8 +218,8 @@ public final class CredentialUtils { Preconditions.checkArgument(conf != null); Preconditions.checkArgument(credentials != null); if (UserGroupInformation.isSecurityEnabled()) { - String tokenRenewer = CredentialUtils.getRMPrincipal(conf); - return fs.addDelegationTokens(tokenRenewer, credentials); + return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf), + credentials); } return null; } @@ -197,10 +236,58 @@ public final class CredentialUtils { Preconditions.checkArgument(fs != null); Preconditions.checkArgument(credentials != null); fs.addDelegationTokens( - UserGroupInformation.getLoginUser().getShortUserName(), + getSelfRenewer(), credentials); } + public static String getSelfRenewer() throws IOException { + return UserGroupInformation.getLoginUser().getShortUserName(); + } + + /** + * Create and add an RM delegation token to the credentials + * @param yarnClient + * @param credentials to add token to + * @return the token which was added + * @throws IOException + * @throws YarnException + */ + public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient, + Credentials credentials) + throws IOException, YarnException { + Configuration conf = yarnClient.getConfig(); + Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf)); + Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf); + Token<TokenIdentifier> rmDelegationToken = + ConverterUtils.convertFromYarn( + yarnClient.getRMDelegationToken(rmPrincipal), + rmDTService); + credentials.addToken(rmDelegationToken.getService(), rmDelegationToken); + return rmDelegationToken; + } + + public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken( + Configuration conf, + Credentials credentials) + throws IOException, YarnException { + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { + LOG.debug("Timeline service enabled -fetching token"); + + try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) { + timelineClient.init(conf); + timelineClient.start(); + Token<TimelineDelegationTokenIdentifier> token = + timelineClient.getDelegationToken( + CredentialUtils.getRMPrincipal(conf)); + credentials.addToken(token.getService(), token); + return token; + } + } else { + LOG.debug("Timeline service is disabled"); + return null; + } + } + /** * Filter a list of tokens from a set of credentials * @param credentials credential source (a new credential set os re @@ -224,18 +311,22 @@ public final class CredentialUtils { } public static String dumpTokens(Credentials credentials, String separator) { - Collection<Token<? extends TokenIdentifier>> allTokens - = credentials.getAllTokens(); - StringBuilder buffer = new StringBuilder(allTokens.size()* 128); - DateFormat df = DateFormat.getDateTimeInstance( - DateFormat.SHORT, DateFormat.SHORT); - for (Token<? extends TokenIdentifier> token : allTokens) { - buffer.append(toString(token)).append(separator); + ArrayList<Token<? extends TokenIdentifier>> sorted = + new ArrayList<>(credentials.getAllTokens()); + Collections.sort(sorted, new TokenComparator()); + StringBuilder buffer = new StringBuilder(sorted.size()* 128); + for (Token<? extends TokenIdentifier> token : sorted) { + buffer.append(tokenToString(token)).append(separator); } return buffer.toString(); } - public static String toString(Token<? extends TokenIdentifier> token) { + /** + * Create a string for people to look at + * @param token token to convert to a string form + * @return a printable view of the token + */ + public static String tokenToString(Token<? extends TokenIdentifier> token) { DateFormat df = DateFormat.getDateTimeInstance( DateFormat.SHORT, DateFormat.SHORT); StringBuilder buffer = new StringBuilder(128); @@ -244,16 +335,27 @@ public final class CredentialUtils { TokenIdentifier ti = token.decodeIdentifier(); buffer.append("; ").append(ti); if (ti instanceof AbstractDelegationTokenIdentifier) { - AbstractDelegationTokenIdentifier dt - = (AbstractDelegationTokenIdentifier) ti; - buffer.append(" Issued: ") + // details in human readable form, and compensate for information HDFS DT omits + AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti; + buffer.append("; Renewer: ").append(dt.getRenewer()); + buffer.append("; Issued: ") .append(df.format(new Date(dt.getIssueDate()))); - buffer.append(" Max Date: ") + buffer.append("; Max Date: ") .append(df.format(new Date(dt.getMaxDate()))); } } catch (IOException e) { + //marshall problem; not ours LOG.debug("Failed to decode {}: {}", token, e, e); } return buffer.toString(); } + + private static class TokenComparator + implements Comparator<Token<? extends TokenIdentifier>>, Serializable { + @Override + public int compare(Token<? extends TokenIdentifier> left, + Token<? extends TokenIdentifier> right) { + return left.getKind().toString().compareTo(right.getKind().toString()); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy new file mode 100644 index 0000000..ee70979 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderTokensCommand.groovy @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.common.params.ActionTokensArgs +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.BadClusterStateException +import org.apache.slider.core.exceptions.NotFoundException +import org.apache.slider.core.main.ServiceLauncherBaseTest +import org.junit.Test + +/** + * Test the argument parsing/validation logic + */ +@CompileStatic +@Slf4j +class TestSliderTokensCommand extends ServiceLauncherBaseTest { + + public static YarnConfiguration config = createTestConfig() + + public static YarnConfiguration createTestConfig() { + def configuration = new YarnConfiguration() + configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032") + return configuration + } + + @Test + public void testBadSourceArgs() throws Throwable { + launchExpectingException(SliderClient, + config, + ActionTokensArgs.DUPLICATE_ARGS, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_SOURCE, "target/tokens.bin", + Arguments.ARG_OUTPUT, "target/tokens.bin", + ]) + } + + @Test + public void testKTNoPrincipal() throws Throwable { + launchExpectingException(SliderClient, + config, + ActionTokensArgs.MISSING_KT_PROVIDER, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_KEYTAB, "target/keytab", + ]) + } + + @Test + public void testPrincipalNoKT() throws Throwable { + launchExpectingException(SliderClient, + config, + ActionTokensArgs.MISSING_KT_PROVIDER, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_PRINCIPAL, "bob@REALM", + ]) + } + + /** + * A missing keytab is an error + * @throws Throwable + */ + @Test + public void testMissingKT() throws Throwable { + def ex = launchExpectingException(SliderClient, + config, + TokensOperation.E_NO_KEYTAB, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_PRINCIPAL, "bob@REALM", + Arguments.ARG_KEYTAB, "target/keytab", + ]) + if (!(ex instanceof NotFoundException)) { + throw ex + } + } + + @Test + public void testMissingSourceFile() throws Throwable { + def ex = launchExpectingException(SliderClient, + config, + TokensOperation.E_MISSING_SOURCE_FILE, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_SOURCE, "target/tokens.bin", + ]) + if (!(ex instanceof NotFoundException)) { + throw ex + } + } + + @Test + public void testListHarmlessWhenInsecure() throws Throwable { + execSliderCommand(0, config, [SliderActions.ACTION_TOKENS]) + } + + @Test + public void testCreateFailsWhenInsecure() throws Throwable { + def ex = launchExpectingException(SliderClient, + config, + TokensOperation.E_INSECURE, + [SliderActions.ACTION_TOKENS, + Arguments.ARG_OUTPUT, "target/tokens.bin", + ]) + if (!(ex instanceof BadClusterStateException)) { + throw ex + } + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/297e9319/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index cb6ce0e..0a3b040 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -990,7 +990,7 @@ class SliderTestUtils extends Assert { * of return code takes place * @param conf configuration * @param args arg list - * @return the return code + * @return the launcher */ protected static ServiceLauncher<SliderClient> execSliderCommand( Configuration conf, @@ -1005,6 +1005,24 @@ class SliderTestUtils extends Assert { return serviceLauncher } + /** + * Launch a slider command to a given exit code. + * Most failures will trigger exceptions; this is for the exit code of the runService() + * call. + * @param exitCode desired exit code + * @param conf configuration + * @param args arg list + * @return the launcher + */ + protected static ServiceLauncher<SliderClient> execSliderCommand( + int exitCode, + Configuration conf, + List args) { + ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf, args) + assert exitCode == serviceLauncher.serviceExitCode + serviceLauncher + } + public static ServiceLauncher launch(Class serviceClass, Configuration conf, List<Object> args) throws