Repository: incubator-slider Updated Branches: refs/heads/develop 243c5b275 -> d9e9d5bcb
SLIDER-787 App Upgrade/Reconfig support in Slider (expose filters for list containers) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d9e9d5bc Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d9e9d5bc Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d9e9d5bc Branch: refs/heads/develop Commit: d9e9d5bcbfa652a2bdaf21bfb6041845cb3e60c2 Parents: 243c5b2 Author: Gour Saha <[email protected]> Authored: Thu Apr 23 17:35:49 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Thu Apr 23 17:35:49 2015 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 79 ++++++++++++-------- .../slider/common/params/ActionListArgs.java | 13 ++++ .../apache/slider/common/tools/SliderUtils.java | 47 +++++++++++- 3 files changed, 104 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d9e9d5bc/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index b7c4454..32a7120 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1701,11 +1701,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private void validateClientAndClusterResource(String clustername, ConfTreeOperations clientResources) throws BadClusterStateException, SliderException, IOException { - log.info("Validating client resource with cluster resource definition (components and instance count)"); - Map<String, Integer> clientComponentNameInstanceMap = new HashMap<>(); + log.info("Validating upgrade resource definition with current cluster " + + "state (components and instance count)"); + Map<String, Integer> clientComponentInstances = new HashMap<>(); for (String componentName : clientResources.getComponentNames()) { if (!SliderKeys.COMPONENT_AM.equals(componentName)) { - clientComponentNameInstanceMap.put(componentName, clientResources + clientComponentInstances.put(componentName, clientResources .getComponentOptInt(componentName, ResourceKeys.COMPONENT_INSTANCES, -1)); } @@ -1720,11 +1721,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe "Failed to validate client resource definition " + clustername + ": " + e); } - Map<String, Integer> clusterComponentNameInstanceMap = new HashMap<>(); + Map<String, Integer> clusterComponentInstances = new HashMap<>(); for (Map.Entry<String, Map<String, String>> component : clusterConf .getResources().components.entrySet()) { if (!SliderKeys.COMPONENT_AM.equals(component.getKey())) { - clusterComponentNameInstanceMap.put( + clusterComponentInstances.put( component.getKey(), Integer.decode(component.getValue().get( ResourceKeys.COMPONENT_INSTANCES))); @@ -1732,43 +1733,45 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } // client and cluster should be an exact match - Iterator<Map.Entry<String, Integer>> clientComponentNameInstanceIterator = clientComponentNameInstanceMap + Iterator<Map.Entry<String, Integer>> clientComponentInstanceIt = clientComponentInstances .entrySet().iterator(); - while (clientComponentNameInstanceIterator.hasNext()) { - Map.Entry<String, Integer> clientComponentNameInstanceEntry = clientComponentNameInstanceIterator + while (clientComponentInstanceIt.hasNext()) { + Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt .next(); - if (clusterComponentNameInstanceMap - .containsKey(clientComponentNameInstanceEntry.getKey())) { + if (clusterComponentInstances + .containsKey(clientComponentInstanceEntry.getKey())) { // compare instance count now and remove from both maps if they match - if (clusterComponentNameInstanceMap - .get(clientComponentNameInstanceEntry.getKey()) == clientComponentNameInstanceEntry + if (clusterComponentInstances + .get(clientComponentInstanceEntry.getKey()) == clientComponentInstanceEntry .getValue()) { - clusterComponentNameInstanceMap - .remove(clientComponentNameInstanceEntry.getKey()); - clientComponentNameInstanceIterator.remove(); + clusterComponentInstances + .remove(clientComponentInstanceEntry.getKey()); + clientComponentInstanceIt.remove(); } } } - if (!clientComponentNameInstanceMap.isEmpty() - || !clusterComponentNameInstanceMap.isEmpty()) { - log.error("Mismatch found in client and cluster resource specification"); - if (!clientComponentNameInstanceMap.isEmpty()) { - log.info(" Following are the client resources that do not match"); - for (Map.Entry<String, Integer> clientComponentNameInstanceEntry : clientComponentNameInstanceMap + if (!clientComponentInstances.isEmpty() + || !clusterComponentInstances.isEmpty()) { + log.error("Mismatch found in upgrade resource definition and cluster " + + "resource state"); + if (!clientComponentInstances.isEmpty()) { + log.info(" Following are the upgrade resource definitions that do " + + "not match"); + for (Map.Entry<String, Integer> clientComponentInstanceEntry : clientComponentInstances .entrySet()) { log.info(" Component Name: {}, Instance count: {}", - clientComponentNameInstanceEntry.getKey(), - clientComponentNameInstanceEntry.getValue()); + clientComponentInstanceEntry.getKey(), + clientComponentInstanceEntry.getValue()); } } - if (!clusterComponentNameInstanceMap.isEmpty()) { + if (!clusterComponentInstances.isEmpty()) { log.info(" Following are the cluster resources that do not match"); - for (Map.Entry<String, Integer> clusterComponentNameInstanceEntry : clusterComponentNameInstanceMap + for (Map.Entry<String, Integer> clusterComponentInstanceEntry : clusterComponentInstances .entrySet()) { log.info(" Component Name: {}, Instance count: {}", - clusterComponentNameInstanceEntry.getKey(), - clusterComponentNameInstanceEntry.getValue()); + clusterComponentInstanceEntry.getKey(), + clusterComponentInstanceEntry.getValue()); } } throw new BadConfigException("\n\nResource definition provided for " @@ -2611,6 +2614,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String state = args.state; boolean listContainers = args.containers; boolean verbose = args.verbose; + String version = args.version; + Set<String> components = args.components; if (live && !state.isEmpty()) { throw new BadCommandArgumentsException( @@ -2621,9 +2626,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe "Should specify an application instance with " + Arguments.ARG_CONTAINERS); } + // specifying both --version and --components with --containers is okay + if (StringUtils.isNotEmpty(version) && !listContainers) { + throw new BadCommandArgumentsException(Arguments.ARG_VERSION + + " can be specified only with " + Arguments.ARG_CONTAINERS); + } + if (!components.isEmpty() && !listContainers) { + throw new BadCommandArgumentsException(Arguments.ARG_COMPONENTS + + " can be specified only with " + Arguments.ARG_CONTAINERS); + } + // flag to indicate only services in a specific state are to be listed boolean listOnlyInState = live || !state.isEmpty(); - + YarnApplicationState min, max; if (live) { min = YarnApplicationState.NEW; @@ -2643,7 +2658,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.debug("No application instances found"); return EXIT_SUCCESS; } - + // and those the RM knows about List<ApplicationReport> instances = listSliderInstances(null); SliderUtils.sortApplicationsByMostRecent(instances); @@ -2682,10 +2697,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // a report listed++; // containers will be non-null when only one instance is requested - String details = SliderUtils.instanceDetailsToString(name, - report, - containers, - verbose); + String details = SliderUtils.instanceDetailsToString(name, report, + containers, version, components, verbose); print(details); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d9e9d5bc/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java index 00767e7..0bc5792 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java @@ -18,6 +18,9 @@ package org.apache.slider.common.params; +import java.util.HashSet; +import java.util.Set; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @@ -46,6 +49,16 @@ public class ActionListArgs extends AbstractActionArgs { description = "List containers of an application instance") public boolean containers; + @Parameter(names = {ARG_VERSION}, + description = "Filter containers by app version (used with " + + ARG_CONTAINERS + ")") + public String version; + + @Parameter(names = {ARG_COMPONENTS}, variableArity = true, + description = "Filter containers by component names (used with " + + ARG_CONTAINERS + ")") + public Set<String> components = new HashSet<>(0);; + /** * Get the min #of params expected * @return the min number of params in the {@link #parameters} field http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d9e9d5bc/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index f7c843c..dbac1e1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; @@ -668,6 +669,8 @@ public final class SliderUtils { public static String instanceDetailsToString(String name, ApplicationReport report, List<ContainerInformation> containers, + String version, + Set<String> components, boolean verbose) { // format strings String staticf = "%-30s"; @@ -693,7 +696,8 @@ public final class SliderUtils { } if (containers != null) { builder.append('\n'); - builder.append(SliderUtils.containersToString(containers)); + builder.append(SliderUtils.containersToString(containers, version, + components)); } } @@ -702,11 +706,16 @@ public final class SliderUtils { } public static String containersToString( - List<ContainerInformation> containers) { + List<ContainerInformation> containers, String version, + Set<String> components) { String containerf = " %-28s %30s %40s %s\n"; StringBuilder builder = new StringBuilder(512); builder.append("Containers:\n"); for (ContainerInformation container : containers) { + if (filter(container.appVersion, version) + || filter(container.component, components)) { + continue; + } builder.append(String.format(containerf, container.component, container.appVersion, container.containerId, container.host)); } @@ -714,6 +723,40 @@ public final class SliderUtils { } /** + * Filter a string value given a single filter + * + * @param value + * the string value to check + * @param filter + * a single string filter + * @return return true if value should be trapped, false if it should be let + * through + */ + public static boolean filter(String value, String filter) { + if (StringUtils.isEmpty(filter) || filter.equals(value)) { + return false; + } + return true; + } + + /** + * Filter a string value given a set of filters + * + * @param value + * the string value to check + * @param filters + * a set of string filters + * @return return true if value should be trapped, false if it should be let + * through + */ + public static boolean filter(String value, Set<String> filters) { + if (filters.isEmpty() || filters.contains(value)) { + return false; + } + return true; + } + + /** * Sorts the given list of application reports, most recently started * or finished instance first. *
