YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40529b07 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40529b07 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40529b07 Branch: refs/heads/yarn-native-services Commit: 40529b0722de3c264d12b3a174f23d95f673f705 Parents: 0d69d71 Author: Gour Saha <gourks...@apache.org> Authored: Wed Nov 30 14:00:22 2016 -0800 Committer: Jian He <jia...@apache.org> Committed: Tue Dec 13 14:46:44 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/slider/api/OptionKeys.java | 14 ++++++++++++++ .../org/apache/slider/common/SliderKeys.java | 2 -- .../apache/slider/providers/ProviderUtils.java | 20 ++++++++++++++++---- 3 files changed, 30 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java index 434b1d9..988627d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java @@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys { * Prefix for export options: {@value} */ String EXPORT_PREFIX = "export."; + /** + * Type suffix for config file and package options: {@value} + */ String TYPE_SUFFIX = ".type"; + /** + * Name suffix for config file and package options: {@value} + */ String NAME_SUFFIX = ".name"; + /** + * Per component suffix for config file options: {@value} + */ + String PER_COMPONENT = ".per.component"; + /** + * Per group suffix for config file options: {@value} + */ + String PER_GROUP = ".per.group"; /** * Zookeeper quorum host list: {@value} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 3d25d33..adf40ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys { String APP_RESOURCES = "application.resources"; String APP_RESOURCES_DIR = "app/resources"; - String PER_COMPONENT = "per.component"; - String PER_GROUP = "per.group"; String APP_PACKAGES_DIR = "app/packages"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index 39986c1..bc237f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -614,8 +614,16 @@ public class ProviderUtils implements RoleKeys, SliderKeys { throw new BadConfigException("Config format " + configFileType + " doesn't exist"); } + boolean perComponent = appConf.getComponentOptBool(roleGroup, + OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys + .PER_COMPONENT, false); + boolean perGroup = appConf.getComponentOptBool(roleGroup, + OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys + .PER_GROUP, false); + localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(), - configFormat, configFileName, configs, env, fileSystem, clusterName); + configFormat, configFileName, configs, env, fileSystem, + clusterName, perComponent, perGroup); } } @@ -631,6 +639,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * @param env environment variables * @param fileSystem file system * @param clusterName app name + * @param perComponent true if file should be created per unique component + * @param perGroup true if file should be created per component group * @throws IOException file cannot be uploaded */ public void localizeConfigFile(ContainerLauncher launcher, @@ -639,7 +649,9 @@ public class ProviderUtils implements RoleKeys, SliderKeys { Map<String, Map<String, String>> configs, MapOperations env, SliderFileSystem fileSystem, - String clusterName) + String clusterName, + boolean perComponent, + boolean perGroup) throws IOException { if (launcher == null) { return; @@ -655,9 +667,9 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } String folder = null; - if ("true".equals(config.get(PER_COMPONENT))) { + if (perComponent) { folder = roleName; - } else if ("true".equals(config.get(PER_GROUP))) { + } else if (perGroup) { folder = roleGroup; } if (folder != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org