SLIDER-481. Exports should allow a multiple line items per export and a more hierarchical structure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1ba58cd0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1ba58cd0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1ba58cd0 Branch: refs/heads/develop Commit: 1ba58cd041221c47aacca77a5ff06246fa3f2a84 Parents: 08d7aec Author: Sumit Mohanty <[email protected]> Authored: Wed Oct 15 15:34:39 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Oct 15 15:34:39 2014 -0700 ---------------------------------------------------------------------- app-packages/memcached/metainfo.xml | 18 +- .../org/apache/slider/client/SliderClient.java | 102 +++++++ .../common/params/ActionRegistryArgs.java | 19 +- .../apache/slider/common/params/Arguments.java | 2 + .../core/registry/docstore/ExportEntry.java | 120 +++++++++ .../docstore/PublishedConfiguration.java | 2 + .../registry/docstore/PublishedExports.java | 139 ++++++++++ .../docstore/PublishedExportsOutputter.java | 104 +++++++ .../registry/docstore/PublishedExportsSet.java | 100 +++++++ .../registry/info/CustomRegistryConstants.java | 3 + .../registry/retrieve/RegistryRetriever.java | 91 ++++++- .../providers/agent/AgentProviderService.java | 270 ++++++++++++++++--- .../agent/application/metadata/Component.java | 9 + .../application/metadata/MetainfoParser.java | 1 + .../slideram/SliderAMProviderService.java | 6 + .../appmaster/state/ProviderAppState.java | 7 + .../state/StateAccessForProviders.java | 7 + .../server/appmaster/web/rest/RestPaths.java | 1 + .../web/rest/publisher/PublisherResource.java | 31 ++- .../slider/client/TestClientBadArgs.groovy | 43 +++ .../agent/TestAgentProviderService.java | 186 ++++++++++++- .../framework/AgentCommandTestBase.groovy | 27 ++ .../funtest/lifecycle/AppsThroughAgentIT.groovy | 40 +++ 23 files changed, 1267 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/app-packages/memcached/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml index 5801ad2..0984dc9 100644 --- a/app-packages/memcached/metainfo.xml +++ b/app-packages/memcached/metainfo.xml @@ -23,17 +23,23 @@ <comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment> <version>1.0.0</version> <exportedConfigs>None</exportedConfigs> + <exportGroups> + <exportGroup> + <name>Servers</name> + <exports> + <export> + <name>host_port</name> + <value>${MEMCACHED_HOST}:${site.global.listen_port}</value> + </export> + </exports> + </exportGroup> + </exportGroups> <components> <component> <name>MEMCACHED</name> <category>MASTER</category> - <componentExports> - <componentExport> - <name>host_port</name> - <value>${THIS_HOST}:${site.global.listen_port}</value> - </componentExport> - </componentExports> + <compExports>Servers-host_port</compExports> <commandScript> <script>scripts/memcached.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 99fde78..7c73394 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -117,6 +117,9 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsOutputter; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; @@ -2331,11 +2334,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (registryArgs.listConf) { // list the configurations actionRegistryListConfigsYarn(registryArgs); + } else if (registryArgs.listExports) { + // list the exports + actionRegistryListExports(registryArgs); } else if (SliderUtils.isSet(registryArgs.getConf)) { // get a configuration PublishedConfiguration publishedConfiguration = actionRegistryGetConfig(registryArgs); outputConfig(publishedConfiguration, registryArgs); + } else if (SliderUtils.isSet(registryArgs.getExport)) { + // get a export group + PublishedExports publishedExports = + actionRegistryGetExport(registryArgs); + outputExport(publishedExports, registryArgs); } else { // it's an unknown command log.info(ActionRegistryArgs.USAGE); @@ -2744,6 +2755,34 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * list exports available for an instance + * + * @param registryArgs registry Arguments + * @throws YarnException YARN problems + * @throws IOException Network or other problems + */ + public void actionRegistryListExports(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + PublishedExportsSet exports = + retriever.getExports(!registryArgs.internal); + + for (String exportName : exports.keys()) { + if (!registryArgs.verbose) { + log.info("{}", exportName); + } else { + PublishedExports published = + exports.get(exportName); + log.info("{} : {}", + exportName, + published.description); + } + } + } + + /** * list configs available for an instance * * @param registryArgs registry Arguments @@ -2768,6 +2807,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * get a specific export group + * + * @param registryArgs registry Arguments + * + * @throws YarnException YARN problems + * @throws IOException Network or other problems + * @throws FileNotFoundException if the config is not found + */ + @VisibleForTesting + public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + boolean external = !registryArgs.internal; + PublishedExportsSet exports = + retriever.getExports(external); + + PublishedExports published = retriever.retrieveExports(exports, + registryArgs.getExport, + external); + return published; + } + + /** * write out the config. If a destination is provided and that dir is a * directory, the entry is written to it with the name provided + extension, * else it is printed to standard out. @@ -2807,6 +2871,44 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * write out the config + * @param published + * @param registryArgs + * @throws BadCommandArgumentsException + * @throws IOException + */ + private void outputExport(PublishedExports published, + ActionRegistryArgs registryArgs) throws + BadCommandArgumentsException, + IOException { + // decide whether or not to print + String entry = registryArgs.getExport; + String format = ConfigFormat.JSON.toString(); + ConfigFormat configFormat = ConfigFormat.resolve(format); + if (configFormat == null || configFormat != ConfigFormat.JSON) { + throw new BadCommandArgumentsException( + "Unknown/Unsupported format %s . Only JSON is supported.", format); + } + + PublishedExportsOutputter outputter = + PublishedExportsOutputter.createOutputter(configFormat, + published); + boolean print = registryArgs.out == null; + if (!print) { + File destFile; + destFile = registryArgs.out; + if (destFile.isDirectory()) { + // creating it under a directory + destFile = new File(destFile, entry + "." + format); + } + log.info("Destination path: {}", destFile); + outputter.save(destFile); + } else { + print(outputter.asString()); + } + } + + /** * Look up an instance * @return instance data * @throws SliderException other failures http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java index deae4eb..1e4aba5 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java @@ -47,8 +47,10 @@ public class ActionRegistryArgs extends AbstractActionArgs { + " (" + Arguments.ARG_LIST + "|" + Arguments.ARG_LISTCONF + "|" + + Arguments.ARG_LISTEXP + "|" + Arguments.ARG_LISTFILES + "|" - + Arguments.ARG_GETCONF + "> " + + Arguments.ARG_GETCONF + "|" + + Arguments.ARG_GETEXP + "> " + Arguments.ARG_NAME + " <name> " + " )" + "[" + Arguments.ARG_VERBOSE + "] " @@ -56,6 +58,8 @@ public class ActionRegistryArgs extends AbstractActionArgs { + "[" + Arguments.ARG_OUTPUT + " <filename> ] " + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] " + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] " + + System.getProperty("line.separator") + + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" ; public ActionRegistryArgs() { } @@ -90,7 +94,14 @@ public class ActionRegistryArgs extends AbstractActionArgs { description = "get configuration") public String getConf; - @Parameter(names = {ARG_LISTFILES}, + @Parameter(names = {ARG_LISTEXP}, + description = "list exports") + public boolean listExports; + @Parameter(names = {ARG_GETEXP}, + description = "get export") + public String getExport; + + @Parameter(names = {ARG_LISTFILES}, description = "list files") public String listFiles; @@ -135,8 +146,8 @@ public class ActionRegistryArgs extends AbstractActionArgs { super.validate(); //verify that at most one of the operations is set - int gets = s(getConf) + s(getFiles); - int lists = s(list) + s(listConf) + s(listFiles); + int gets = s(getConf) + s(getFiles) + s(getExport); + int lists = s(list) + s(listConf) + s(listFiles) + s(listExports); int set = lists + gets; if (set > 1) { throw new UsageException(USAGE); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 24ad442..2f7af70 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -52,6 +52,7 @@ public interface Arguments { String ARG_FORMAT = "--format"; String ARG_FORCE = "--force"; String ARG_GETCONF = "--getconf"; + String ARG_GETEXP = "--getexp"; String ARG_GETFILES = "--getfiles"; String ARG_HELP = "--help"; String ARG_ID = "--id"; @@ -60,6 +61,7 @@ public interface Arguments { String ARG_LEVEL = "--level"; String ARG_LIST = "--list"; String ARG_LISTCONF = "--listconf"; + String ARG_LISTEXP = "--listexp"; String ARG_LISTFILES = "--listfiles"; String ARG_LIVE = "--live"; String ARG_MANAGER = "--manager"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java new file mode 100644 index 0000000..4bcf6c1 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ExportEntry { + + /** + * The value of the export + */ + private String value; + /** + * The container id of the container that is responsible for the export + */ + private String containerId; + /** + * Tag associated with the container - its usually an identifier different than container id + * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ... + */ + private String tag; + /** + * An export can be at the level of a component or an application + */ + private String level; + /** + * The time when the export was updated + */ + private String updatedTime; + /** + * The time when the export expires + */ + private String validUntil; + + public ExportEntry() { + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + public String getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(String updatedTime) { + this.updatedTime = updatedTime; + } + + public String getValidUntil() { + return validUntil; + } + + public void setValidUntil(String validUntil) { + this.validUntil = validUntil; + } + + @Override + public String toString() { + return new StringBuilder("ExportEntry{"). + append("value='").append(value).append("',"). + append("containerId='").append(containerId).append("',"). + append("tag='").append(tag).append("',"). + append("level='").append(level).append("'"). + append("updatedTime='").append(updatedTime).append("'"). + append("validUntil='").append(validUntil).append("'"). + append(" }").toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java index cbc46f0..28f9d3d 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -23,6 +23,7 @@ import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.core.exceptions.BadConfigException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; import org.codehaus.jackson.map.annotate.JsonSerialize; import java.io.IOException; @@ -155,6 +156,7 @@ public class PublishedConfiguration { */ public String asJson() throws IOException { ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); String json = mapper.writeValueAsString(entries); return json; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java new file mode 100644 index 0000000..1919bfa --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExports { + + public String description; + public long updated; + public String updatedTime; + public Map<String, List<ExportEntry>> entries = new HashMap<String, List<ExportEntry>>(); + + public PublishedExports() { + } + + /** + * build an empty published configuration + * + * @param description configuration description + */ + public PublishedExports(String description) { + this.description = description; + } + + /** + * Build a configuration from the entries + * + * @param description configuration description + * @param entries entries to put + */ + public PublishedExports(String description, + Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.description = description; + putValues(entries); + } + + /** + * Is the configuration empty. This means either that it has not been given any values, or it is stripped down copy + * set down over the wire. + * + * @return + */ + public boolean isEmpty() { + return entries.isEmpty(); + } + + public long getUpdated() { + return updated; + } + + public void setUpdated(long updated) { + this.updated = updated; + this.updatedTime = new Date(updated).toString(); + } + + /** + * Set the values from an iterable (this includes a Hadoop Configuration and Java properties object). Any existing + * value set is discarded + * + * @param entries entries to put + */ + public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.entries = new HashMap<String, List<ExportEntry>>(); + for (Map.Entry<String, List<ExportEntry>> entry : entries) { + this.entries.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Return the values as json string + * + * @return + * + * @throws IOException + */ + public String asJson() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + String json = mapper.writeValueAsString(entries); + return json; + } + + /** + * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's + * values + * + * @return the copy + */ + public PublishedExports shallowCopy() { + PublishedExports that = new PublishedExports(); + that.description = this.description; + that.updated = this.updated; + that.updatedTime = this.updatedTime; + return that; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("PublishedConfiguration{"); + sb.append("description='").append(description).append('\''); + sb.append(" entries = ").append(entries.size()); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java new file mode 100644 index 0000000..b21e717 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** Output a published configuration */ +public abstract class PublishedExportsOutputter { + + protected final PublishedExports exports; + + protected PublishedExportsOutputter(PublishedExports exports) { + this.exports = exports; + } + + /** + * Create an outputter for the chosen format + * + * @param format format enumeration + * @param exports owning config + * @return the outputter + */ + + public static PublishedExportsOutputter createOutputter(ConfigFormat format, + PublishedExports exports) { + Preconditions.checkNotNull(exports); + switch (format) { + case JSON: + return new JsonOutputter(exports); + default: + throw new RuntimeException("Unsupported format :" + format); + } + } + + public void save(File dest) throws IOException { + FileOutputStream out = null; + try { + out = new FileOutputStream(dest); + save(out); + out.close(); + } finally { + org.apache.hadoop.io.IOUtils.closeStream(out); + } + } + + /** + * Save the content. The default saves the asString() value to the output stream + * + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + + /** + * Convert to a string + * + * @return + * @throws IOException + */ + public abstract String asString() throws IOException; + + public static class JsonOutputter extends PublishedExportsOutputter { + + public JsonOutputter(PublishedExports exports) { + super(exports); + } + + @Override + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + @Override + public String asString() throws IOException { + return exports.asJson(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java new file mode 100644 index 0000000..cdd35de --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.slider.server.appmaster.web.rest.RestPaths; +import org.apache.slider.server.services.utility.PatternValidator; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Represents a set of configurations for an application, component, etc. + * Json serialisable; accessors are synchronized + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExportsSet { + + private static final PatternValidator validator = new PatternValidator( + RestPaths.PUBLISHED_CONFIGURATION_REGEXP); + + public Map<String, PublishedExports> exports = + new HashMap<String, PublishedExports>(); + + public PublishedExportsSet() { + } + + /** + * Put a name -it will be converted to lower case before insertion. + * Any existing entry will be overwritten (that includes an entry + * with a different case in the original name) + * @param name name of entry + * @param export published export + * @throws IllegalArgumentException if not a valid name + */ + public void put(String name, PublishedExports export) { + String name1 = name.toLowerCase(Locale.ENGLISH); + validateName(name1); + exports.put(name1, export); + } + + /** + * Validate the name -restricting it to the set defined in + * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP} + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public static void validateName(String name) { + validator.validate(name); + + } + + public PublishedExports get(String name) { + return exports.get(name); + } + + public boolean contains(String name) { + return exports.containsKey(name); + } + + public int size() { + return exports.size(); + } + + public Set<String> keys() { + TreeSet<String> keys = new TreeSet<String>(); + keys.addAll(exports.keySet()); + return keys; + } + + public PublishedExportsSet shallowCopy() { + PublishedExportsSet that = new PublishedExportsSet(); + for (Map.Entry<String, PublishedExports> entry : + exports.entrySet()) { + that.put(entry.getKey(), entry.getValue().shallowCopy()); + } + return that; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java index 65c122f..67b9feb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java @@ -35,6 +35,9 @@ public class CustomRegistryConstants { public static final String PUBLISHER_CONFIGURATIONS_API = "org.apache.slider.publisher.configurations"; + public static final String PUBLISHER_EXPORTS_API = + "org.apache.slider.publisher.exports"; + public static final String PUBLISHER_DOCUMENTS_API = "org.apache.slider.publisher.documents"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java index 101efb2..a91f515 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java @@ -33,6 +33,8 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.ExceptionConverter; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +54,8 @@ public class RegistryRetriever { private final String externalConfigurationURL; private final String internalConfigurationURL; + private final String externalExportsURL; + private final String internalExportsURL; private static final Client jerseyClient; static { @@ -63,9 +67,12 @@ public class RegistryRetriever { jerseyClient.setFollowRedirects(true); } - public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL) { - this.externalConfigurationURL = externalConfigurationURL; - this.internalConfigurationURL = internalConfigurationURL; + public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL, + String externalExportsURL, String internalExportsURL) { + this.externalConfigurationURL = externalConfigurationURL; + this.internalConfigurationURL = internalConfigurationURL; + this.externalExportsURL = externalExportsURL; + this.internalExportsURL = internalExportsURL; } /** @@ -99,6 +106,29 @@ public class RegistryRetriever { } } externalConfigurationURL = url; + + internal = record.getInternalEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API); + url = null; + if (internal != null) { + List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType( + internal); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } + } + internalExportsURL = url; + external = record.getExternalEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API); + url = null; + if (external != null) { + List<String> addresses = + RegistryTypeUtils.retrieveAddressesUriType(external); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } + } + externalExportsURL = url; } /** @@ -138,6 +168,33 @@ public class RegistryRetriever { return confURL; } + protected String getExportURL(boolean external) throws FileNotFoundException { + String confURL = external ? externalExportsURL: internalExportsURL; + if (Strings.isStringEmpty(confURL)) { + throw new FileNotFoundException("No configuration URL"); + } + return confURL; + } + + /** + * Get the configurations of the registry + * @param external flag to indicate that it is the external entries to fetch + * @return the configuration sets + */ + public PublishedExportsSet getExports(boolean external) throws + FileNotFoundException, IOException { + + String exportsUrl = getExportURL(external); + try { + WebResource webResource = jsonResource(exportsUrl); + log.debug("GET {}", exportsUrl); + PublishedExportsSet exportSet = webResource.get(PublishedExportsSet.class); + return exportSet; + } catch (UniformInterfaceException e) { + throw ExceptionConverter.convertJerseyException(exportsUrl, e); + } + } + private WebResource resource(String url) { WebResource resource = jerseyClient.resource(url); return resource; @@ -174,7 +231,33 @@ public class RegistryRetriever { throw ExceptionConverter.convertJerseyException(confURL, e); } } - + + /** + * Get a complete export, with all values + * @param exportSet + * @param name name of the configuration + * @param external flag to indicate that it is an external configuration + * @return the retrieved config + * @throws IOException IO problems + */ + public PublishedExports retrieveExports(PublishedExportsSet exportSet, + String name, + boolean external) throws IOException { + if (!exportSet.contains(name)) { + throw new FileNotFoundException("Unknown export " + name); + } + String exportsURL = getExportURL(external); + exportsURL = SliderUtils.appendToURL(exportsURL, name); + try { + WebResource webResource = jsonResource(exportsURL); + PublishedExports publishedExports = + webResource.get(PublishedExports.class); + return publishedExports; + } catch (UniformInterfaceException e) { + throw ExceptionConverter.convertJerseyException(exportsURL, e); + } + } + @Override public String toString() { return super.toString() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 19b5ddd..fc7d935 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -52,7 +52,9 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.AbstractProviderService; import org.apache.slider.providers.ProviderCore; @@ -95,10 +97,13 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -106,6 +111,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -127,10 +133,15 @@ public class AgentProviderService extends AbstractProviderService implements private static final String CONTAINER_ID = "container_id"; private static final String GLOBAL_CONFIG_TAG = "global"; private static final String LOG_FOLDERS_TAG = "LogFolders"; + private static final String HOST_FOLDER_FORMAT = "%s:%s"; + private static final String CONTAINER_LOGS_TAG = "container_log_dirs"; + private static final String CONTAINER_PWDS_TAG = "container_work_dirs"; + private static final String COMPONENT_TAG = "component"; + private static final String APPLICATION_TAG = "application"; private static final String COMPONENT_DATA_TAG = "ComponentInstanceData"; private static final String SHARED_PORT_TAG = "SHARED"; private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}"; - private static final int MAX_LOG_ENTRIES = 20; + private static final int MAX_LOG_ENTRIES = 40; private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000; private final Object syncLock = new Object(); @@ -149,16 +160,25 @@ public class AgentProviderService extends AbstractProviderService implements new ConcurrentHashMap<String, ComponentInstanceState>(); private final Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap<String, Map<String, String>>(); - private final Map<String, Map<String, String>> exportGroups = - new ConcurrentHashMap<String, Map<String, String>>(); + private final Map<String, Map<String, List<ExportEntry>>> exportGroups = + new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>(); private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<String, Map<String, String>>(); - private final Map<String, String> workFolders = - Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) { + + private final Map<String, ExportEntry> logFolderExports = + Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) { + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_LOG_ENTRIES; + } + }); + private final Map<String, ExportEntry> workFolderExports = + Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) { protected boolean removeEldestEntry(Map.Entry eldest) { return size() > MAX_LOG_ENTRIES; } }); + private final Map<String, Set<String>> containerExportsMap = + new HashMap<String, Set<String>>(); /** * Create an instance of AgentProviderService @@ -491,7 +511,7 @@ public class AgentProviderService extends AbstractProviderService implements Map<String, String> folders = registration.getLogFolders(); if (folders != null && !folders.isEmpty()) { - publishLogFolderPaths(folders, containerId, roleName, hostFqdn); + publishFolderPaths(folders, containerId, roleName, hostFqdn); } } else { response.setResponseStatus(RegistrationStatus.FAILED); @@ -563,7 +583,7 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Component operation. Status: {}", result); if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) { - publishLogFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn()); + publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn()); } } @@ -639,6 +659,10 @@ public class AgentProviderService extends AbstractProviderService implements this.getAllocatedPorts(containerId).put(portname, portNo); if (instance != null) { try { + // if the returned value is not a single port number then there are no + // meaningful way for Slider to use it during export + // No need to error out as it may not be the responsibility of the component + // to allocate port or the component may need an array of ports instance.registerPortEndpoint(Integer.valueOf(portNo), portname); } catch (NumberFormatException e) { log.warn("Failed to parse {}: {}", portNo, e); @@ -648,6 +672,7 @@ public class AgentProviderService extends AbstractProviderService implements // component specific publishes processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName); + processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName); // and update registration entries if (instance != null) { @@ -697,7 +722,7 @@ public class AgentProviderService extends AbstractProviderService implements throw new IOException(e); } } - + @Override public void notifyContainerCompleted(ContainerId containerId) { if (containerId != null) { @@ -719,6 +744,25 @@ public class AgentProviderService extends AbstractProviderService implements } } } + + synchronized (this.containerExportsMap) { + Set<String> containerExportSets = containerExportsMap.get(containerIdStr); + if (containerExportSets != null) { + for (String containerExportStr : containerExportSets) { + String[] parts = containerExportStr.split(":"); + Map<String, List<ExportEntry>> exportGroup = getCurrentExports(parts[0]); + List<ExportEntry> exports = exportGroup.get(parts[1]); + List<ExportEntry> exportToRemove = new ArrayList<ExportEntry>(); + for (ExportEntry export : exports) { + if (containerIdStr.equals(export.getContainerId())) { + exportToRemove.add(export); + } + } + exports.removeAll(exportToRemove); + } + containerExportsMap.remove(containerIdStr); + } + } } } @@ -754,6 +798,16 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting + protected Map<String, ExportEntry> getLogFolderExports() { + return logFolderExports; + } + + @VisibleForTesting + protected Map<String, ExportEntry> getWorkFolderExports() { + return workFolderExports; + } + + @VisibleForTesting protected Metainfo getMetainfo() { return this.metainfo; } @@ -901,15 +955,59 @@ public class AgentProviderService extends AbstractProviderService implements * @param hostFqdn * @param roleName */ - protected void publishLogFolderPaths( + protected void publishFolderPaths( Map<String, String> folders, String containerId, String roleName, String hostFqdn) { - for (Map.Entry<String, String> entry: folders.entrySet()) { - workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, entry.getKey(), containerId), - entry.getValue()); + Date now = new Date(); + for (Map.Entry<String, String> entry : folders.entrySet()) { + ExportEntry exportEntry = new ExportEntry(); + exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, entry.getValue())); + exportEntry.setContainerId(containerId); + exportEntry.setLevel(COMPONENT_TAG); + exportEntry.setTag(roleName); + exportEntry.setUpdatedTime(now.toString()); + if (entry.getKey().equals("AGENT_LOG_ROOT")) { + synchronized (logFolderExports) { + getLogFolderExports().put(containerId, exportEntry); + } + } else { + synchronized (workFolderExports) { + getWorkFolderExports().put(containerId, exportEntry); + } + } + log.info("Updating log and pwd folders for container {}", containerId); } - publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, - (new HashMap<String, String>(this.workFolders)).entrySet()); + PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG); + exports.setUpdated(now.getTime()); + synchronized (logFolderExports) { + updateExportsFromList(exports, getLogFolderExports()); + } + getAmState().getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports); + + exports = new PublishedExports(CONTAINER_PWDS_TAG); + exports.setUpdated(now.getTime()); + synchronized (workFolderExports) { + updateExportsFromList(exports, getWorkFolderExports()); + } + getAmState().getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports); + } + + /** + * Update the export data from the map + * @param exports + * @param folderExports + */ + private void updateExportsFromList(PublishedExports exports, Map<String, ExportEntry> folderExports) { + Map<String, List<ExportEntry>> perComponentList = new HashMap<String, List<ExportEntry>>(); + for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) + { + String componentName = logEntry.getValue().getTag(); + if(!perComponentList.containsKey(componentName)) { + perComponentList.put(componentName, new ArrayList<ExportEntry>()); + } + perComponentList.get(componentName).add(logEntry.getValue()); + } + exports.putValues(perComponentList.entrySet()); } @@ -954,13 +1052,12 @@ public class AgentProviderService extends AbstractProviderService implements } } - List<ExportGroup> exportGroups = application.getExportGroups(); - boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty(); + List<ExportGroup> appExportGroups = application.getExportGroups(); + boolean hasExportGroups = appExportGroups != null && !appExportGroups.isEmpty(); Set<String> appExports = new HashSet(); String appExportsStr = getApplicationComponent(roleName).getAppExports(); - boolean hasNoAppExports = appExportsStr == null || appExportsStr.isEmpty(); - if (!hasNoAppExports) { + if (SliderUtils.isSet(appExportsStr)) { for (String appExport : appExportsStr.split(",")) { if (appExport.trim().length() > 0) { appExports.add(appExport.trim()); @@ -988,11 +1085,12 @@ public class AgentProviderService extends AbstractProviderService implements } Set<String> modifiedGroups = new HashSet<String>(); - for (ExportGroup exportGroup : exportGroups) { + for (ExportGroup exportGroup : appExportGroups) { List<Export> exports = exportGroup.getExports(); if (exports != null && !exports.isEmpty()) { String exportGroupName = exportGroup.getName(); - Map<String, String> map = getCurrentExports(exportGroupName); + ConcurrentHashMap<String, List<ExportEntry>> map = + (ConcurrentHashMap<String, List<ExportEntry>>)getCurrentExports(exportGroupName); for (Export export : exports) { if (canBeExported(exportGroupName, export.getName(), appExports)) { String value = export.getValue(); @@ -1002,7 +1100,12 @@ public class AgentProviderService extends AbstractProviderService implements value = value.replace(token, replaceTokens.get(token)); } } - map.put(export.getName(), value); + ExportEntry entry = new ExportEntry(); + entry.setLevel(APPLICATION_TAG); + entry.setValue(value); + entry.setUpdatedTime(new Date().toString()); + // over-write, app exports are singletons + map.put(export.getName(), new ArrayList(Arrays.asList(entry))); log.info("Preparing to publish. Key {} and Value {}", export.getName(), value); } } @@ -1024,11 +1127,11 @@ public class AgentProviderService extends AbstractProviderService implements return appExports.contains(String.format("%s-%s", exportGroupName, name)); } - protected Map<String, String> getCurrentExports(String groupName) { + protected Map<String, List<ExportEntry>> getCurrentExports(String groupName) { if (!this.exportGroups.containsKey(groupName)) { synchronized (this.exportGroups) { if (!this.exportGroups.containsKey(groupName)) { - this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>()); + this.exportGroups.put(groupName, new ConcurrentHashMap<String, List<ExportEntry>>()); } } } @@ -1037,10 +1140,24 @@ public class AgentProviderService extends AbstractProviderService implements } private void publishModifiedExportGroups(Set<String> modifiedGroups) { - synchronized (this.exportGroups) { - for (String groupName : modifiedGroups) { - publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet()); + for (String groupName : modifiedGroups) { + Map<String, List<ExportEntry>> entries = this.exportGroups.get(groupName); + + // Publish in old format for the time being + Map<String, String> simpleEntries = new HashMap<String, String>(); + for (Map.Entry<String, List<ExportEntry>> entry : entries.entrySet()) { + List<ExportEntry> exports = entry.getValue(); + if(exports != null && exports.size() > 0) { + // there is no support for multiple exports per name - so extract only the first one + simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); + } } + publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet()); + + PublishedExports exports = new PublishedExports(groupName); + exports.setUpdated(new Date().getTime()); + exports.putValues(entries.entrySet()); + getAmState().getPublishedExportsSet().put(groupName, exports); } } @@ -1095,14 +1212,102 @@ public class AgentProviderService extends AbstractProviderService implements } } + /** Publish component instance specific data if the component demands it */ + protected void processAndPublishComponentSpecificExports(Map<String, String> ports, + String containerId, + String hostFqdn, + String roleName) { + String portVarFormat = "${site.%s}"; + String hostNamePattern = "${" + roleName + "_HOST}"; + + List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups(); + Component component = getMetainfo().getApplicationComponent(roleName); + if (component != null && SliderUtils.isSet(component.getCompExports()) + && appExportGroups != null && appExportGroups.size() > 0) { + + Set<String> compExports = new HashSet(); + String compExportsStr = component.getCompExports(); + for (String appExport : compExportsStr.split(",")) { + if (appExport.trim().length() > 0) { + compExports.add(appExport.trim()); + } + } + + Date now = new Date(); + Set<String> modifiedGroups = new HashSet<String>(); + for (ExportGroup exportGroup : appExportGroups) { + List<Export> exports = exportGroup.getExports(); + if (exports != null && !exports.isEmpty()) { + String exportGroupName = exportGroup.getName(); + ConcurrentHashMap<String, List<ExportEntry>> map = + (ConcurrentHashMap<String, List<ExportEntry>>) getCurrentExports(exportGroupName); + for (Export export : exports) { + if (canBeExported(exportGroupName, export.getName(), compExports)) { + log.info("Attempting to publish {} of group {} for component type {}", + export.getName(), exportGroupName, roleName); + String templateToExport = export.getValue(); + for (String portName : ports.keySet()) { + boolean publishData = false; + String portValPattern = String.format(portVarFormat, portName); + if (templateToExport.contains(portValPattern)) { + templateToExport = templateToExport.replace(portValPattern, ports.get(portName)); + publishData = true; + } + if (templateToExport.contains(hostNamePattern)) { + templateToExport = templateToExport.replace(hostNamePattern, hostFqdn); + publishData = true; + } + if (publishData) { + ExportEntry entryToAdd = new ExportEntry(); + entryToAdd.setLevel(COMPONENT_TAG); + entryToAdd.setValue(templateToExport); + entryToAdd.setUpdatedTime(now.toString()); + entryToAdd.setContainerId(containerId); + + List<ExportEntry> existingList = + map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd))); + + // in-place edit, no lock needed + if (existingList != null) { + boolean updatedInPlace = false; + for (ExportEntry entry : existingList) { + if (containerId.equalsIgnoreCase(entry.getContainerId())) { + entryToAdd.setValue(templateToExport); + entryToAdd.setUpdatedTime(now.toString()); + updatedInPlace = true; + } + } + if (!updatedInPlace) { + existingList.add(entryToAdd); + } + } + + log.info("Publishing {} for name {} and container {}", + templateToExport, export.getName(), containerId); + modifiedGroups.add(exportGroupName); + synchronized (containerExportsMap) { + if (!containerExportsMap.containsKey(containerId)) { + containerExportsMap.put(containerId, new HashSet<String>()); + } + Set<String> containerExportMaps = containerExportsMap.get(containerId); + containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName())); + } + } + } + } + } + } + } + publishModifiedExportGroups(modifiedGroups); + } + } + private void publishComponentInstanceData() { Map<String, String> dataToPublish = new HashMap<String, String>(); - synchronized (this.componentInstanceData) { - for (String container : getComponentInstanceData().keySet()) { - for (String prop : getComponentInstanceData().get(container).keySet()) { - dataToPublish.put( - container + "." + prop, getComponentInstanceData().get(container).get(prop)); - } + for (String container : getComponentInstanceData().keySet()) { + for (String prop : getComponentInstanceData().get(container).keySet()) { + dataToPublish.put( + container + "." + prop, getComponentInstanceData().get(container).get(prop)); } } publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet()); @@ -1616,5 +1821,4 @@ public class AgentProviderService extends AbstractProviderService implements ""); } } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 9f3dd0f..418868c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -34,6 +34,7 @@ public class Component { String maxInstanceCount; String autoStartOnFailure; String appExports; + String compExports; CommandScript commandScript; List<ComponentExport> componentExports; @@ -82,6 +83,14 @@ public class Component { this.appExports = appExports; } + public String getCompExports() { + return compExports; + } + + public void setCompExports(String compExports) { + this.compExports = compExports; + } + public String getMinInstanceCount() { return minInstanceCount; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java index c92c265..1d8403f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -61,6 +61,7 @@ public class MetainfoParser { digester.addBeanPropertySetter("*/component/maxInstanceCount"); digester.addBeanPropertySetter("*/component/autoStartOnFailure"); digester.addBeanPropertySetter("*/component/appExports"); + digester.addBeanPropertySetter("*/component/compExports"); digester.addObjectCreate("*/componentExport", ComponentExport.class); digester.addBeanPropertySetter("*/componentExport/name"); digester.addBeanPropertySetter("*/componentExport/value"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java index 601c3f9..afe6428 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@ -146,6 +146,8 @@ public class SliderAMProviderService extends AbstractProviderService implements String configurationsURL = SliderUtils.appendToURL( publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET); + String exportsURL = SliderUtils.appendToURL( + publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS); serviceRecord.addExternalEndpoint( RegistryTypeUtils.webEndpoint( @@ -166,6 +168,10 @@ public class SliderAMProviderService extends AbstractProviderService implements RegistryTypeUtils.restEndpoint( CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, new URI(configurationsURL))); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API, + new URI(exportsURL))); } catch (URISyntaxException e) { throw new IOException(e); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index a0871ae..9c5da12 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.services.utility.PatternValidator; @@ -40,6 +41,7 @@ public class ProviderAppState implements StateAccessForProviders { private final Map<String, PublishedConfigSet> publishedConfigSets = new ConcurrentHashMap<String, PublishedConfigSet>(5); + private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet(); private static final PatternValidator validator = new PatternValidator( RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP); private String applicationName; @@ -66,6 +68,11 @@ public class ProviderAppState implements StateAccessForProviders { } @Override + public PublishedExportsSet getPublishedExportsSet() { + return publishedExportsSets; + } + + @Override public PublishedConfigSet getPublishedConfigSet(String name) { return publishedConfigSets.get(name); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 1714f75..b907b06 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import java.util.Collection; import java.util.List; @@ -51,6 +52,12 @@ public interface StateAccessForProviders { PublishedConfigSet getPublishedSliderConfigurations(); /** + * Get the published exports set + * @return + */ + PublishedExportsSet getPublishedExportsSet(); + + /** * Get a named published config set * @param name name to look up * @return the instance or null http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 93601ad..94f1e4c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -61,6 +61,7 @@ public class RestPaths { = "[a-z0-9][a-z0-9_.\\+-]*"; public static final String SLIDER_CONFIGSET = "slider"; + public static final String SLIDER_EXPORTS = "exports"; public static final String SLIDER_CLASSPATH = "classpath"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java index 5d8b657..e47bbb9 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java @@ -23,6 +23,8 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.docstore.UriMap; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -56,7 +58,10 @@ public class PublisherResource { protected static final Logger log = LoggerFactory.getLogger(PublisherResource.class); private final WebAppApi slider; - public static final String SET_NAME = + public static final String EXPORTS_NAME = "exports"; + public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME; + public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ; + public static final String SET_NAME = "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}"; private static final String CONFIG = SET_NAME + "/{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}"; @@ -101,7 +106,9 @@ public class PublisherResource { UriMap uriMap = new UriMap(); for (String name : appState.listConfigSets()) { uriMap.put(name, baseURL + name); + log.info("Tick tack {} and {}", name, baseURL); } + uriMap.put(EXPORTS_NAME, baseURL + EXPORTS_NAME); return uriMap; } @@ -114,6 +121,26 @@ public class PublisherResource { } @GET + @Path(EXPORTS_RESOURCES_PATH) + @Produces({MediaType.APPLICATION_JSON}) + public PublishedExportsSet gePublishedExports() { + + PublishedExportsSet set = appState.getPublishedExportsSet(); + return set.shallowCopy(); + } + + @GET + @Path(EXPORT_RESOURCE_PATH) + @Produces({MediaType.APPLICATION_JSON}) + public PublishedExports getAMExports2(@PathParam("exportname") String exportname, + @Context UriInfo uriInfo, + @Context HttpServletResponse res) { + init(res, uriInfo); + PublishedExportsSet set = appState.getPublishedExportsSet(); + return set.get(exportname); + } + + @GET @Path("/"+ SET_NAME) @Produces({MediaType.APPLICATION_JSON}) public PublishedConfigSet getPublishedConfiguration( @@ -129,7 +156,7 @@ public class PublisherResource { } private void logRequest(UriInfo uriInfo) { - log.debug(uriInfo.getRequestUri().toString()); + log.info(uriInfo.getRequestUri().toString()); } @GET http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy index d1f8a8f..7d596d6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy @@ -24,8 +24,10 @@ import org.apache.hadoop.conf.Configuration import org.apache.slider.common.params.ActionRegistryArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.ErrorStrings import org.apache.slider.core.exceptions.UsageException +import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.main.ServiceLauncherBaseTest import org.junit.Test @@ -88,4 +90,45 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { log.info(exception.toString()) } + @Test + public void testRegistryExportBadUsage1() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Expected a value after parameter --getexp", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_GETEXP]) + assert exception instanceof BadCommandArgumentsException + log.info(exception.toString()) + } + + @Test + public void testRegistryExportBadUsage2() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Expected a value after parameter --getexp", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP]) + assert exception instanceof BadCommandArgumentsException + log.info(exception.toString()) + } + + @Test + public void testRegistryExportBadUsage3() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Usage: registry", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP, + "export1"]) + assert exception instanceof UsageException + log.info(exception.toString()) + } }
