Repository: incubator-slider Updated Branches: refs/heads/develop baa79a746 -> 97835815c
SLIDER-511 slider diagnostic command to list env variables and client configuration Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5456a58d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5456a58d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5456a58d Branch: refs/heads/develop Commit: 5456a58dff351141a96da633465361a0028f1395 Parents: 6c43743 Author: Steve Loughran <[email protected]> Authored: Mon Oct 13 19:05:03 2014 -0700 Committer: Steve Loughran <[email protected]> Committed: Mon Oct 13 19:05:03 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 47 +++++++++++---- .../common/params/ActionDiagnosticArgs.java | 7 ++- .../slider/common/params/ActionResolveArgs.java | 1 - .../apache/slider/common/params/Arguments.java | 22 +++---- .../slider/common/tools/ConfigHelper.java | 2 +- .../core/registry/SliderRegistryUtils.java | 62 ++++++++++++++++++++ .../TestStandaloneYarnRegistryAM.groovy | 9 +++ .../apache/slider/client/TestDiagnostics.groovy | 58 ++++++++++++++++++ .../slider/registry/TestRegistryPaths.groovy | 17 +++++- .../commands/DiagnosticsCommandIT.groovy | 36 ++++++++++++ 10 files changed, 235 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 750bfaa..e44db3b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -109,6 +109,7 @@ import org.apache.slider.core.launch.RunningApplication; import org.apache.slider.core.main.RunService; import org.apache.slider.core.persist.ConfPersister; import org.apache.slider.core.persist.LockAcquireFailedException; +import org.apache.slider.core.registry.SliderRegistryUtils; import org.apache.slider.core.registry.YarnAppListClient; import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; @@ -124,7 +125,6 @@ import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; -import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -439,6 +439,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * Create the zookeeper node associated with the calling user and the cluster */ @VisibleForTesting + @Deprecated public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException { String user = getUsername(); String zkPath = ZKIntegration.mkClusterPath(user, clusterName); @@ -511,8 +512,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.warn("Filesystem returned false from delete() operation"); } - if(!deleteZookeeperNode(clustername)) { - log.warn("Unable to perform node cleanup in Zookeeper."); + // rm the registry entry âdo not let this block the destroy operations + String registryPath = SliderRegistryUtils.registryPathForInstance( + clustername); + try { + getRegistryOperations().delete(registryPath, true); + } catch (IOException e) { + log.warn("Error deleting {}: {} ", registryPath, e, e); + } catch (SliderException e) { + log.warn("Error binding to registry {} ", e, e); } List<ApplicationReport> instances = findAllLiveInstances(clustername); @@ -531,9 +539,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionAmSuicide(String clustername, - ActionAMSuicideArgs args) throws - YarnException, - IOException { + ActionAMSuicideArgs args) throws YarnException, IOException { SliderClusterOperations clusterOperations = createClusterOperations(clustername); clusterOperations.amSuicide(args.message, args.exitcode, args.waittime); @@ -2210,7 +2216,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // the arguments args.validate(); RegistryOperations operations = getRegistryOperations(); - String path = args.path; + String path = SliderRegistryUtils.resolvePath(args.path); ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal(); try { if (args.list) { @@ -2351,7 +2357,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) { try { if (diagnosticArgs.client) { - actionDiagnosticClient(); + actionDiagnosticClient(diagnosticArgs); } else if (SliderUtils.isSet(diagnosticArgs.application)) { actionDiagnosticApplication(diagnosticArgs); } else if (SliderUtils.isSet(diagnosticArgs.slider)) { @@ -2442,7 +2448,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe //assign application name from param to each sub diagnostic function diagnosticArgs.application = diagnosticArgs.all; diagnosticArgs.slider = diagnosticArgs.all; - actionDiagnosticClient(); + actionDiagnosticClient(diagnosticArgs); actionDiagnosticApplication(diagnosticArgs); actionDiagnosticSlider(diagnosticArgs); actionDiagnosticYarn(diagnosticArgs); @@ -2579,7 +2585,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - private void actionDiagnosticClient() throws SliderException, IOException { + private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs) + throws SliderException, IOException { String currentCommandPath = SliderUtils.getCurrentCommandPath(); SliderVersionInfo.loadAndPrintVersionInfo(log); String clientConfigPath = SliderUtils.getClientConfigPath(); @@ -2589,6 +2596,25 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe + clientConfigPath); log.info(jdkInfo); + // verbose? + if (diagnosticArgs.verbose) { + // do the environment + Map<String, String> env = System.getenv(); + Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet()); + StringBuilder builder = new StringBuilder("Environment variables:\n"); + for (String key : envList) { + builder.append(key) + .append("=") + .append(env.get(key)) + .append("\n"); + } + log.info(builder.toString()); + + // then the config + log.info("Slider client configuration:\n" + + ConfigHelper.dumpConfigToString(getConfig())); + } + try { SliderUtils.validateSliderClientEnvironment(log); } catch (SliderException e) { @@ -2598,6 +2624,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.error(e.toString()); throw e; } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java index b2fde07..33abb30 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java @@ -9,11 +9,12 @@ public class ActionDiagnosticArgs extends AbstractActionArgs { public static final String USAGE = "Usage: " + SliderActions.ACTION_DIAGNOSTIC - + Arguments.ARG_CLIENT + "| " + + " " + + Arguments.ARG_CLIENT + " | " + Arguments.ARG_SLIDER + " <appname> " + "| " + Arguments.ARG_APPLICATION + " <appname> " + "| " - + Arguments.ARG_YARN + "| " - + Arguments.ARG_CREDENTIALS + "| " + + Arguments.ARG_YARN + " | " + + Arguments.ARG_CREDENTIALS + " | " + Arguments.ARG_ALL + " <appname> " + "| " + Arguments.ARG_LEVEL + " <appname> " + " [" + Arguments.ARG_VERBOSE + "] "; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java index 4759bcb..2ee075a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java @@ -26,7 +26,6 @@ import org.apache.slider.core.exceptions.UsageException; import java.io.File; -import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY; import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE; import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 4deebd7..59c7095 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -21,16 +21,22 @@ package org.apache.slider.common.params; /** * Here are all the arguments that may be parsed by the client or server * command lines. + * + * Important: Please keep the main list in alphabetical order + * so it is easier to see what arguments are there */ public interface Arguments { - + String ARG_ALL = "--all"; + String ARG_APPLICATION = "--application"; String ARG_APP_HOME = "--apphome"; + String ARG_CLIENT = "--client"; String ARG_CONFDIR = "--appconf"; String ARG_COMPONENT = "--component"; String ARG_COMPONENT_SHORT = "--comp"; String ARG_COMP_OPT= "--compopt"; String ARG_COMP_OPT_SHORT = "--co"; - + String ARG_CREDENTIALS = "--credentials"; + String ARG_DEBUG = "--debug"; String ARG_DEFINE = "-D"; String ARG_DEST = "--dest"; @@ -51,6 +57,7 @@ public interface Arguments { String ARG_ID = "--id"; String ARG_IMAGE = "--image"; String ARG_INTERNAL = "--internal"; + String ARG_LEVEL = "--level"; String ARG_LIST = "--list"; String ARG_LISTFILES = "--listfiles"; String ARG_LISTCONF = "--listconf"; @@ -58,14 +65,15 @@ public interface Arguments { String ARG_MANAGER = "--manager"; String ARG_MANAGER_SHORT = "--m"; String ARG_MESSAGE = "--message"; + String ARG_NAME = "--name"; String ARG_OPTION = "--option"; String ARG_OPTION_SHORT = "-O"; - String ARG_NAME = "--name"; String ARG_OUTPUT = "--out"; String ARG_OUTPUT_SHORT = "-o"; String ARG_PACKAGE = "--package"; String ARG_PATH = "--path"; String ARG_PROVIDER = "--provider"; + String ARG_QUEUE = "--queue"; String ARG_RESOURCES = "--resources"; String ARG_RES_COMP_OPT = "--rescompopt"; String ARG_RES_COMP_OPT_SHORT = "--rco"; @@ -73,6 +81,7 @@ public interface Arguments { String ARG_RESOURCE_OPT = "--resopt"; String ARG_RESOURCE_OPT_SHORT = "-ro"; String ARG_SERVICETYPE = "--servicetype"; + String ARG_SLIDER = "--slider"; String ARG_SYSPROP = "-S"; String ARG_TEMPLATE = "--template"; String ARG_VERBOSE = "--verbose"; @@ -81,14 +90,7 @@ public interface Arguments { String ARG_ZKPORT = "--zkport"; String ARG_ZKHOSTS = "--zkhosts"; String ARG_REPLACE_PKG = "--replacepkg"; - String ARG_CLIENT = "--client"; - String ARG_SLIDER = "--slider"; - String ARG_APPLICATION = "--application"; String ARG_YARN = "--yarn"; - String ARG_CREDENTIALS = "--credentials"; - String ARG_ALL = "--all"; - String ARG_LEVEL = "--level"; - String ARG_QUEUE = "--queue"; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index 31acb94..b710224 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -416,7 +416,7 @@ public class ConfigHelper { * @return listing in key=value style */ public static String dumpConfigToString(Configuration conf) { - TreeSet<String> sorted = sortedConfigKeys(conf); + Set<String> sorted = sortedConfigKeys(conf); StringBuilder builder = new StringBuilder(); for (String key : sorted) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java new file mode 100644 index 0000000..37b36ea --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.slider.common.SliderKeys; + +/** + * Miscellaneous methods to assist slider registry work + * + */ +public class SliderRegistryUtils { + + + /** + * Get the registry path for an instance under the user's home node + * @param instanceName application instance + * @return a path to the registry location for this application instance. + */ + public static String registryPathForInstance(String instanceName) { + return RegistryUtils.servicePath( + RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName + ); + } + + /** + * Process a path expanding it if needed. + * Validation is delegated to later as the core registry will need + * to do that anyway + * @param path path + * @return a path maybe with some expansion + */ + public static String resolvePath(String path) { + Preconditions.checkArgument(path!=null, "null path"); + Preconditions.checkArgument(!path.isEmpty(), "empty path"); + String newpath = path; + if (path.startsWith("~/")) { + // add user expansion + newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1); + } else if (path.equals("~")) { + newpath = RegistryUtils.homePathForCurrentUser(); + } + return newpath; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index e75cc57..a849be3 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -257,6 +257,15 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] == serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] + //resolve to the home directory + assert 0 == client.actionResolve(new ActionResolveArgs( + path: "~", list:true)) + + //resolve to the instance + assert 0 == client.actionResolve(new ActionResolveArgs( + path: "~/services/org-apache-slider/$clustername", + list: true)) + // hit the registry web page def registryEndpoint = serviceRecord.getExternalEndpoint( CustomRegistryConstants.REGISTRY_REST_API) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy new file mode 100644 index 0000000..57126f8 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.common.params.ActionDiagnosticArgs +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.test.SliderTestBase +import org.apache.slider.test.YarnMiniClusterTestBase +import org.apache.slider.test.YarnZKMiniClusterTestBase +import org.junit.Test + +@CompileStatic +@Slf4j +class TestDiagnostics extends YarnZKMiniClusterTestBase { + + @Test + public void testClientDiags() throws Throwable { + //launch fake master + String clustername = createMiniCluster("", configuration, 1, true) + ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [SliderActions.ACTION_DIAGNOSTIC, + Arguments.ARG_CLIENT] + ) + def client = launcher.service + def diagnostics = new ActionDiagnosticArgs() + diagnostics.client = true + diagnostics.verbose = true + describe("Verbose diagnostics") + + + client.actionDiagnostic(diagnostics) + + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy index be5ce78..5f95e0b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy @@ -19,6 +19,7 @@ package org.apache.slider.registry import org.apache.hadoop.registry.client.binding.RegistryUtils +import org.apache.slider.core.registry.SliderRegistryUtils import org.apache.slider.test.SliderTestUtils import org.junit.Assert import org.junit.Test @@ -44,6 +45,20 @@ class TestRegistryPaths { SliderTestUtils.skip("homedir filtering not yet in hadoop registry module") } } - + + @Test + public void testRegistryPathForInstance() throws Throwable { + def path = SliderRegistryUtils.registryPathForInstance("instance") + assert path.endsWith("/instance") + } + + @Test + public void testPathResolution() throws Throwable { + def home = RegistryUtils.homePathForCurrentUser() + assert home == SliderRegistryUtils.resolvePath("~") + assert (home +"/") == SliderRegistryUtils.resolvePath("~/") + assert (home +"/something") == SliderRegistryUtils.resolvePath("~/something") + assert ("~unresolved") == SliderRegistryUtils.resolvePath("~unresolved") + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy new file mode 100644 index 0000000..b210f4b --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.commands + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.funtest.framework.CommandTestBase +import org.junit.BeforeClass +import org.junit.Test + +@CompileStatic +@Slf4j +public class DiagnosticsCommandIT extends CommandTestBase { + + @Test + public void testListAll() throws Throwable { + assertSuccess(list(null)) + } + +}
