SLIDER-1107 fix AM-generated config publishing and race condition in config generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1699a714 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1699a714 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1699a714 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 1699a714dfcea6473d9256a7e0a4536976485acc Parents: c7a7c0d Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Thu May 12 18:22:08 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Thu May 12 18:51:38 2016 -0700 ---------------------------------------------------------------------- .../core/registry/docstore/ConfigUtils.java | 37 ++++++ .../docstore/PublishedConfiguration.java | 22 ---- .../PublishedConfigurationOutputter.java | 53 +-------- .../providers/agent/AgentProviderService.java | 113 ++++++++++++++----- 4 files changed, 120 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java index 7e6ee5a..2e1615b 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java @@ -17,6 +17,10 @@ */ package org.apache.slider.core.registry.docstore; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; + +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -24,6 +28,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; public class ConfigUtils { + public static final String TEMPLATE_FILE = "template.file"; + public static String replaceProps(Map<String, String> config, String content) { Map<String, String> tokens = new HashMap<>(); for (Entry<String, String> entry : config.entrySet()) { @@ -56,4 +62,35 @@ public class ConfigUtils { return newConfig; } + public static void prepConfigForTemplateOutputter(ConfigFormat configFormat, + Map<String, String> config, SliderFileSystem fileSystem, + String clusterName, String fileName) throws IOException { + if (!configFormat.equals(ConfigFormat.TEMPLATE)) { + return; + } + Path templateFile = null; + if (config.containsKey(TEMPLATE_FILE)) { + templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + config.get(TEMPLATE_FILE)); + } + if (!fileSystem.isFile(templateFile)) { + throw new IOException("config specified template file " + config + .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist"); + } + } + if (templateFile == null && fileName != null) { + templateFile = fileSystem.buildResourcePath(fileName); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + fileName); + } + } + if (fileSystem.isFile(templateFile)) { + config.put("content", fileSystem.cat(templateFile)); + } else { + config.put("content", ""); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java index cabdce6..477f7d3 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -20,7 +20,6 @@ package org.apache.slider.core.registry.docstore; import org.apache.hadoop.conf.Configuration; import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.exceptions.BadConfigException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.ObjectMapper; @@ -50,9 +49,6 @@ public class PublishedConfiguration { public Map<String, String> entries = new HashMap<>(); - public SliderFileSystem fileSystem; - public String clusterName; - public PublishedConfiguration() { } @@ -90,24 +86,6 @@ public class PublishedConfiguration { } /** - * Build a configuration from the entries - * @param description configuration description - * @param entries entries to put - * @param fileSystem Slider file system (source of configuration templates) - * @param clusterName cluster name - */ - public PublishedConfiguration(String description, - Iterable<Map.Entry<String, String>> entries, - SliderFileSystem fileSystem, - String clusterName) { - this.description = description; - putValues(entries); - this.fileSystem = fileSystem; - this.clusterName = clusterName; - } - - - /** * Is the configuration empty. This means either that it has not * been given any values, or it is stripped down copy set down over the * wire. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java index 23cfc8f..9bdcfcb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java @@ -23,9 +23,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.DumperOptions.FlowStyle; import org.yaml.snakeyaml.Yaml; @@ -34,7 +32,6 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; -import java.util.Map; import java.util.Properties; /** @@ -188,58 +185,10 @@ public abstract class PublishedConfigurationOutputter { } } - public static class TemplateOutputter extends PublishedConfigurationOutputter { - - public static final String TEMPLATE_FILE = "template.file"; - + public static class TemplateOutputter extends EnvOutputter { public TemplateOutputter(PublishedConfiguration owner) { super(owner); } - - @Override - public void save(File dest) throws IOException { - FileUtils.writeStringToFile(dest, asString(dest.getName()), - Charsets.UTF_8); - } - - public String asString(String fileName) throws IOException { - if (owner.fileSystem == null) { - throw new IOException("File system not specified for template " + - "configuration"); - } - Map<String,String> config = owner.entries; - SliderFileSystem fileSystem = owner.fileSystem; - Path templateFile = null; - if (config.containsKey(TEMPLATE_FILE)) { - templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); - if (!fileSystem.isFile(templateFile)) { - templateFile = fileSystem.buildResourcePath(owner.clusterName, - config.get(TEMPLATE_FILE)); - } - if (!fileSystem.isFile(templateFile)) { - throw new IOException("config specified template file " + config - .get(TEMPLATE_FILE) + " for config " + owner.description + - " but " + templateFile + " doesn't exist"); - } - } - if (templateFile == null && fileName != null) { - templateFile = fileSystem.buildResourcePath(fileName); - if (!fileSystem.isFile(templateFile)) { - templateFile = fileSystem.buildResourcePath(owner.clusterName, - fileName); - } - } - if (fileSystem.isFile(templateFile)) { - return ConfigUtils.replaceProps(config, fileSystem.cat(templateFile)); - } else { - return ""; - } - } - - @Override - public String asString() throws IOException { - return asString(null); - } } public static class YamlOutputter extends PublishedConfigurationOutputter { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index ab4da47..155ff09 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -170,6 +170,8 @@ public class AgentProviderService extends AbstractProviderService implements private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); private volatile Metainfo metaInfo = null; + private AggregateConf instanceDefinition = null; + private SliderFileSystem fileSystem = null; private Map<String, DefaultConfig> defaultConfigs = null; private ComponentCommandOrder commandOrder = null; private HeartbeatMonitor monitor; @@ -282,6 +284,8 @@ public class AgentProviderService extends AbstractProviderService implements if (metaInfo == null) { synchronized (syncLock) { if (metaInfo == null) { + this.instanceDefinition = instanceDefinition; + this.fileSystem = fileSystem; readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); @@ -679,21 +683,6 @@ public class AgentProviderService extends AbstractProviderService implements return uploadResource(resource, fileSystem, certsDir); } - private Path checkResourceExists(File resource, SliderFileSystem - fileSystem, String roleName) throws IOException { - Path dir; - if (roleName == null) { - dir = fileSystem.buildClusterResourcePath(getClusterName()); - } else { - dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName); - } - Path destPath = new Path(dir, resource.getName()); - if (fileSystem.getFileSystem().exists(destPath)) { - return destPath; - } - return null; - } - private Path uploadResource(File resource, SliderFileSystem fileSystem, String roleName) throws IOException { Path dir; @@ -705,8 +694,8 @@ public class AgentProviderService extends AbstractProviderService implements return uploadResource(resource, fileSystem, dir); } - private static Path uploadResource(File resource, SliderFileSystem fileSystem, - Path parentDir) throws IOException { + private static synchronized Path uploadResource(File resource, + SliderFileSystem fileSystem, Path parentDir) throws IOException { if (!fileSystem.getFileSystem().exists(parentDir)) { fileSystem.getFileSystem().mkdirs(parentDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); @@ -783,14 +772,19 @@ public class AgentProviderService extends AbstractProviderService implements } private void createConfigFile(SliderFileSystem fileSystem, File file, - ConfigFile configFile, Map<String, String> config) throws IOException { + ConfigFile configFile, Map<String, String> config) + throws IOException { ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); log.info("Writing {} file {}", configFormat, file); + ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, + fileSystem, getClusterName(), file.getName()); + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(configFile.getDictionaryName(), + config.entrySet()); PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(configFormat, - new PublishedConfiguration(configFile.getDictionaryName(), - config.entrySet(), fileSystem, getClusterName())); + publishedConfiguration); configurationOutputter.save(file); } @@ -820,16 +814,11 @@ public class AgentProviderService extends AbstractProviderService implements folder = roleGroup; } - Path destPath = checkResourceExists(localFile, fileSystem, folder); - if (destPath == null) { - log.info("Localizing {} configs to config file {} (destination {}) " + - "based on {} configs", - config.size(), localFile, fileName, configFile.getDictionaryName()); - createConfigFile(fileSystem, localFile, configFile, config); - destPath = uploadResource(localFile, fileSystem, folder); - } else { - log.info("Config already exists at {}, not recreating it", destPath); - } + log.info("Localizing {} configs to config file {} (destination {}) " + + "based on {} configs", config.size(), localFile, fileName, + configFile.getDictionaryName()); + createConfigFile(fileSystem, localFile, configFile, config); + Path destPath = uploadResource(localFile, fileSystem, folder); LocalResource configResource = fileSystem.createAmResource(destPath, LocalResourceType.FILE); @@ -1033,7 +1022,7 @@ public class AgentProviderService extends AbstractProviderService implements role = amState.getOwnedContainer(containerId); role.ip = status.getIp(); } - if(status.getHostname() != null & !status.getHostname().isEmpty()){ + if(status.getHostname() != null && !status.getHostname().isEmpty()){ role = amState.getOwnedContainer(containerId); role.hostname = status.getHostname(); } @@ -1369,6 +1358,68 @@ public class AgentProviderService extends AbstractProviderService implements } catch (URISyntaxException e) { throw new IOException(e); } + + // identify client component + Component client = null; + for (Component component : getMetaInfo().getApplication().getComponents()) { + if (component != null && component.getCategory().equals("CLIENT")) { + client = component; + break; + } + } + if (client == null) { + log.info("No client component specified, not publishing client configs"); + return; + } + + // register AM-generated client configs + ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); + MapOperations clientOperations = instanceDefinition.getAppConfOperations() + .getOrAddComponent(client.getName()); + if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, + false)) { + log.info("AM config generation is false, not publishing client configs"); + return; + } + + // build and localize configuration files + Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); + Map<String, String> tokens = null; + try { + tokens = getStandardTokenMap(appConf, client.getName(), client.getName()); + } catch (SliderException e) { + throw new IOException(e); + } + + for (ConfigFile configFile : getMetaInfo() + .getComponentConfigFiles(client.getName())) { + addNamedConfiguration(configFile.getDictionaryName(), + appConf.getGlobalOptions().options, configurations, tokens, null, + client.getName()); + if (appConf.getComponent(client.getName()) != null) { + addNamedConfiguration(configFile.getDictionaryName(), + appConf.getComponent(client.getName()).options, configurations, + tokens, null, client.getName()); + } + } + + //do a final replacement of re-used configs + dereferenceAllConfigs(configurations); + + for (ConfigFile configFile : getMetaInfo() + .getComponentConfigFiles(client.getName())) { + ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); + + Map<String, String> config = configurations.get(configFile.getDictionaryName()); + ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, + fileSystem, getClusterName(), + new File(configFile.getFileName()).getName()); + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(configFile.getDictionaryName(), + config.entrySet()); + getAmState().getPublishedSliderConfigurations().put( + configFile.getDictionaryName(), publishedConfiguration); + } } @Override
