YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdea5e1d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdea5e1d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdea5e1d Branch: refs/heads/yarn-native-services Commit: fdea5e1d2b1d1967540570486fffb081ee032ea0 Parents: 038561c Author: Billie Rinaldi <bil...@apache.org> Authored: Tue Nov 29 13:14:45 2016 -0800 Committer: Jian He <jia...@apache.org> Committed: Tue Dec 13 14:46:44 2016 -0800 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 6 +- .../slider/client/SliderYarnClientImpl.java | 94 +++++++++++++------- .../apache/slider/common/tools/SliderUtils.java | 37 +++++++- .../slider/core/registry/YarnAppListClient.java | 21 ++++- .../providers/AbstractClientProvider.java | 28 +++++- .../providers/docker/DockerClientProvider.java | 8 ++ .../servicemonitor/YarnApplicationProbe.java | 12 +-- 7 files changed, 160 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ea10ed0..3f2df0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -198,6 +198,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -2083,7 +2084,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, - appOperations); + appOperations, clustername); Credentials credentials = null; if (clusterSecure) { @@ -3031,9 +3032,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe appstate.ordinal() < YarnApplicationState.FINISHED.ordinal(); } else { // scan for instance in single --state state - List<ApplicationReport> userInstances = yarnClient.listDeployedInstances(""); state = state.toUpperCase(Locale.ENGLISH); YarnApplicationState desiredState = extractYarnApplicationState(state); + List<ApplicationReport> userInstances = yarnClient + .listDeployedInstances("", EnumSet.of(desiredState), name); ApplicationReport foundInstance = yarnClient.findAppInInstanceList(userInstances, name, desiredState); if (foundInstance != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index d471cdb..258ef31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -53,6 +53,8 @@ import java.net.BindException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl { */ public List<ApplicationReport> listDeployedInstances(String user) throws YarnException, IOException { + return listDeployedInstances(user, null); + } + + /** + * List Slider <i>deployed</i>instances belonging to a specific user in a + * given set of states. + * <p> + * Deployed means: known about in the YARN cluster; it will include all apps + * in the specified set of states. + * + * @param user + * user: "" means all users + * @param appStates + * filter by a set of YarnApplicationState + * @return a possibly empty list of Slider AMs + * @throws YarnException + * @throws IOException + */ + public List<ApplicationReport> listDeployedInstances(String user, + EnumSet<YarnApplicationState> appStates) + throws YarnException, IOException { + return listDeployedInstances(user, appStates, null); + } + + /** + * List Slider <i>deployed</i>instances belonging to a specific user in a + * given set of states and filtered by an application name tag. + * <p> + * Deployed means: known about in the YARN cluster; it will include all apps + * in the specified set of states and tagged with the specified app name. + * + * @param user + * user: "" means all users + * @param appStates + * filter by a set of YarnApplicationState + * @param appname + * an application name tag in the format defined by + * {@link SliderUtils#createNameTag(String)} + * @return a possibly empty list of Slider AMs + * @throws YarnException + * @throws IOException + */ + public List<ApplicationReport> listDeployedInstances(String user, + EnumSet<YarnApplicationState> appStates, String appname) + throws YarnException, IOException { Preconditions.checkArgument(user != null, "Null User"); Set<String> types = new HashSet<>(1); types.add(SliderKeys.APP_TYPE); - List<ApplicationReport> allApps = getApplications(types); + Set<String> tags = null; + if (appname != null) { + tags = Collections.singleton(SliderUtils.createNameTag(appname)); + } + List<ApplicationReport> allApps = getApplications(types, appStates, tags); List<ApplicationReport> results = new ArrayList<>(); for (ApplicationReport report : allApps) { if (StringUtils.isEmpty(user) || user.equals(report.getUser())) { @@ -136,20 +187,11 @@ public class SliderYarnClientImpl extends YarnClientImpl { * @param appname application name * @return the list of all matching application instances */ - public List<ApplicationReport> findAllInstances(String user, - String appname) + public List<ApplicationReport> findAllInstances(String user, String appname) throws IOException, YarnException { Preconditions.checkArgument(appname != null, "Null application name"); - List<ApplicationReport> instances = listDeployedInstances(user); - List<ApplicationReport> results = - new ArrayList<>(instances.size()); - for (ApplicationReport report : instances) { - if (report.getName().equals(appname)) { - results.add(report); - } - } - return results; + return listDeployedInstances(user, null, appname); } /** @@ -204,14 +246,12 @@ public class SliderYarnClientImpl extends YarnClientImpl { // user wants all instances killed String user = getUsername(); log.info("Killing all applications belonging to {}", user); - Collection<ApplicationReport> instances = listDeployedInstances(user); + Collection<ApplicationReport> instances = listDeployedInstances(user, + SliderUtils.getAllLiveAppStates()); for (ApplicationReport instance : instances) { - if (isApplicationLive(instance)) { - ApplicationId appId = instance.getApplicationId(); - log.info("Killing Application {}", appId); - - killRunningApplication(appId, "forced kill"); - } + ApplicationId appId = instance.getApplicationId(); + log.info("Killing Application {}", appId); + killRunningApplication(appId, "forced kill"); } } else { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); @@ -290,21 +330,11 @@ public class SliderYarnClientImpl extends YarnClientImpl { * @return the list of all matching application instances */ public List<ApplicationReport> findAllLiveInstances(String user, - String appname) throws - YarnException, - IOException { + String appname) throws YarnException, IOException { Preconditions.checkArgument(StringUtils.isNotEmpty(appname), "Null/empty application name"); - List<ApplicationReport> instances = listDeployedInstances(user); - List<ApplicationReport> results = - new ArrayList<ApplicationReport>(instances.size()); - for (ApplicationReport app : instances) { - if (app.getName().equals(appname) - && isApplicationLive(app)) { - results.add(app); - } - } - return results; + return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(), + appname); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index b101d34..4457957 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -104,6 +104,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.EnumSet; import java.util.Enumeration; import java.util.HashMap; import java.util.List; @@ -2656,4 +2657,38 @@ public final class SliderUtils { } return prefix; } -} + + public static String createNameTag(String name) { + return "Name: " + name; + } + + public static String createVersionTag(String version) { + return "Version: " + version; + } + + public static String createDescriptionTag(String description) { + return "Description: " + description; + } + + /** + * Get all YarnApplicationState values which signify that an application is + * in RUNNING or pre-RUNNING state. + * + * @return all live app states + */ + public static EnumSet<YarnApplicationState> getAllLiveAppStates() { + return EnumSet.range(YarnApplicationState.NEW, + YarnApplicationState.RUNNING); + } + + /** + * Get all YarnApplicationState values which signify that an application is + * not live, which means it is in one of the post RUNNING states. + * + * @return all non live app states + */ + public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() { + return EnumSet.range(YarnApplicationState.FINISHED, + YarnApplicationState.KILLED); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java index 1bdfb9c..71cc193 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java @@ -86,7 +86,7 @@ public class YarnAppListClient { public ApplicationReport findInstance(String appname) throws YarnException, IOException { - List<ApplicationReport> instances = listInstances(null); + List<ApplicationReport> instances = listInstances(null, appname); return yarnClient.findClusterInInstanceList(instances, appname); } @@ -107,8 +107,25 @@ public class YarnAppListClient { */ public List<ApplicationReport> listInstances(String user) throws YarnException, IOException { + return listInstances(user, null); + } + + /** + * List all instances belonging to a specific user and a specific appname. + * + * @param user + * user if not the default. null means default, "" means all users, + * otherwise it is the name of a user + * @param appname + * application name set as an application tag + * @return a possibly empty list of AMs + * @throws YarnException + * @throws IOException + */ + public List<ApplicationReport> listInstances(String user, String appname) + throws YarnException, IOException { String listUser = user == null ? username : user; - return yarnClient.listDeployedInstances(listUser); + return yarnClient.listDeployedInstances(listUser, null, appname); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index f59c347..01444fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; @@ -37,6 +38,7 @@ import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.HashSet; import java.util.Set; import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; @@ -217,11 +219,35 @@ public abstract class AbstractClientProvider extends Configured { * @return the set of tags. */ public Set<String> getApplicationTags(SliderFileSystem fileSystem, - ConfTreeOperations appConf) throws SliderException { + ConfTreeOperations appConf, String appName) throws SliderException { return Collections.emptySet(); } /** + * Generates a fixed format of application tags given one or more of + * application name, version and description. This allows subsequent query for + * an application with a name only, version only or description only or any + * combination of those as filters. + * + * @param appName name of the application + * @param appVersion version of the application + * @param appDescription brief description of the application + * @return + */ + public final Set<String> createApplicationTags(String appName, + String appVersion, String appDescription) { + Set<String> tags = new HashSet<>(); + tags.add(SliderUtils.createNameTag(appName)); + if (appVersion != null) { + tags.add(SliderUtils.createVersionTag(appVersion)); + } + if (appDescription != null) { + tags.add(SliderUtils.createDescriptionTag(appDescription)); + } + return tags; + } + + /** * Process client operations for applications such as install, configure * @param fileSystem * @param registryOperations http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index d554427..86d87ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; @@ -93,4 +94,11 @@ public class DockerClientProvider extends AbstractClientProvider } } } + + @Override + public Set<String> getApplicationTags(SliderFileSystem fileSystem, + ConfTreeOperations appConf, String appName) throws SliderException { + return createApplicationTags(appName, null, null); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java index adf613c..92df048 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java @@ -68,23 +68,19 @@ public class YarnApplicationProbe extends Probe { */ @Override public ProbeStatus ping(boolean livePing) { - ProbeStatus status = new ProbeStatus(); try { - - List<ApplicationReport> instances = - yarnClient.listDeployedInstances(username); - ApplicationReport instance = - yarnClient.findClusterInInstanceList(instances, clustername); + List<ApplicationReport> instances = yarnClient + .listDeployedInstances(username, null, clustername); + ApplicationReport instance = yarnClient + .findClusterInInstanceList(instances, clustername); if (null == instance) { throw UnknownApplicationInstanceException.unknownInstance(clustername); } - status.succeed(this); } catch (Exception e) { status.fail(this, e); } return status; - } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org