SLIDER-481. Exports should allow a multiple line items per export and a more hierarchical structure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/31e05853 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/31e05853 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/31e05853 Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports Commit: 31e058531a41d5169ee74b72a060af39a03a30a0 Parents: 34e4487 Author: Sumit Mohanty <[email protected]> Authored: Wed Oct 15 00:02:16 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Oct 15 00:02:16 2014 -0700 ---------------------------------------------------------------------- app-packages/memcached/metainfo.xml | 18 +- .../org/apache/slider/client/SliderClient.java | 102 + .../apache/slider/client/SliderClient.java.orig | 2913 ++++++++++++++++++ .../common/params/ActionRegistryArgs.java | 19 +- .../apache/slider/common/params/Arguments.java | 2 + .../core/registry/docstore/ExportEntry.java | 120 + .../registry/docstore/PublishedExports.java | 141 + .../docstore/PublishedExportsOutputter.java | 104 + .../registry/docstore/PublishedExportsSet.java | 100 + .../registry/info/CustomRegistryConstants.java | 3 + .../registry/retrieve/RegistryRetriever.java | 91 +- .../providers/agent/AgentProviderService.java | 266 +- .../agent/application/metadata/Component.java | 9 + .../application/metadata/MetainfoParser.java | 1 + .../slideram/SliderAMProviderService.java | 6 + .../appmaster/state/ProviderAppState.java | 7 + .../state/StateAccessForProviders.java | 7 + .../server/appmaster/web/rest/RestPaths.java | 1 + .../web/rest/publisher/PublisherResource.java | 31 +- .../slider/client/TestClientBadArgs.groovy | 43 + .../agent/TestAgentProviderService.java | 186 +- .../framework/AgentCommandTestBase.groovy | 27 + .../funtest/lifecycle/AppsThroughAgentIT.groovy | 40 + 23 files changed, 4176 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31e05853/app-packages/memcached/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml index 5801ad2..0984dc9 100644 --- a/app-packages/memcached/metainfo.xml +++ b/app-packages/memcached/metainfo.xml @@ -23,17 +23,23 @@ <comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment> <version>1.0.0</version> <exportedConfigs>None</exportedConfigs> + <exportGroups> + <exportGroup> + <name>Servers</name> + <exports> + <export> + <name>host_port</name> + <value>${MEMCACHED_HOST}:${site.global.listen_port}</value> + </export> + </exports> + </exportGroup> + </exportGroups> <components> <component> <name>MEMCACHED</name> <category>MASTER</category> - <componentExports> - <componentExport> - <name>host_port</name> - <value>${THIS_HOST}:${site.global.listen_port}</value> - </componentExport> - </componentExports> + <compExports>Servers-host_port</compExports> <commandScript> <script>scripts/memcached.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31e05853/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 50a7097..eeeee26 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -115,6 +115,9 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsOutputter; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; @@ -2285,11 +2288,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (registryArgs.listConf) { // list the configurations actionRegistryListConfigsYarn(registryArgs); + } else if (registryArgs.listExports) { + // list the exports + actionRegistryListExports(registryArgs); } else if (SliderUtils.isSet(registryArgs.getConf)) { // get a configuration PublishedConfiguration publishedConfiguration = actionRegistryGetConfig(registryArgs); outputConfig(publishedConfiguration, registryArgs); + } else if (SliderUtils.isSet(registryArgs.getExport)) { + // get a export group + PublishedExports publishedExports = + actionRegistryGetExport(registryArgs); + outputExport(publishedExports, registryArgs); } else { // it's an unknown command log.info(ActionRegistryArgs.USAGE); @@ -2698,6 +2709,34 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * list exports available for an instance + * + * @param registryArgs registry Arguments + * @throws YarnException YARN problems + * @throws IOException Network or other problems + */ + public void actionRegistryListExports(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + PublishedExportsSet exports = + retriever.getExports(!registryArgs.internal); + + for (String exportName : exports.keys()) { + if (!registryArgs.verbose) { + log.info("{}", exportName); + } else { + PublishedExports published = + exports.get(exportName); + log.info("{} : {}", + exportName, + published.description); + } + } + } + + /** * list configs available for an instance * * @param registryArgs registry Arguments @@ -2722,6 +2761,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * get a specific export group + * + * @param registryArgs registry Arguments + * + * @throws YarnException YARN problems + * @throws IOException Network or other problems + * @throws FileNotFoundException if the config is not found + */ + @VisibleForTesting + public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + boolean external = !registryArgs.internal; + PublishedExportsSet exports = + retriever.getExports(external); + + PublishedExports published = retriever.retrieveExports(exports, + registryArgs.getExport, + external); + return published; + } + + /** * write out the config. If a destination is provided and that dir is a * directory, the entry is written to it with the name provided + extension, * else it is printed to standard out. @@ -2761,6 +2825,44 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * write out the config + * @param published + * @param registryArgs + * @throws BadCommandArgumentsException + * @throws IOException + */ + private void outputExport(PublishedExports published, + ActionRegistryArgs registryArgs) throws + BadCommandArgumentsException, + IOException { + // decide whether or not to print + String entry = registryArgs.getExport; + String format = ConfigFormat.JSON.toString(); + ConfigFormat configFormat = ConfigFormat.resolve(format); + if (configFormat == null || configFormat != ConfigFormat.JSON) { + throw new BadCommandArgumentsException( + "Unknown/Unsupported format %s . Only JSON is supported.", format); + } + + PublishedExportsOutputter outputter = + PublishedExportsOutputter.createOutputter(configFormat, + published); + boolean print = registryArgs.out == null; + if (!print) { + File destFile; + destFile = registryArgs.out; + if (destFile.isDirectory()) { + // creating it under a directory + destFile = new File(destFile, entry + "." + format); + } + log.info("Destination path: {}", destFile); + outputter.save(destFile); + } else { + print(outputter.asString()); + } + } + + /** * Look up an instance * @return instance data * @throws SliderException other failures
