Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports eba79a4de -> 6f4c8b67e
SLIDER-481. Exports should allow a multiple line items per export and a more hierarchical structure Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f4c8b67 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f4c8b67 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f4c8b67 Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports Commit: 6f4c8b67e0d222581a7386bb4cdf8829f26877e7 Parents: eba79a4 Author: Sumit Mohanty <[email protected]> Authored: Sun Oct 12 00:19:00 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sun Oct 12 00:19:00 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 102 ++++++++++++++ .../common/params/ActionRegistryArgs.java | 19 ++- .../apache/slider/common/params/Arguments.java | 2 + .../core/registry/docstore/ExportEntry.java | 82 +++++++++++ .../registry/docstore/PublishedExports.java | 141 +++++++++++++++++++ .../docstore/PublishedExportsOutputter.java | 104 ++++++++++++++ .../registry/docstore/PublishedExportsSet.java | 100 +++++++++++++ .../registry/info/CustomRegistryConstants.java | 3 + .../registry/retrieve/RegistryRetriever.java | 91 +++++++++++- .../providers/agent/AgentProviderService.java | 87 ++++++++++-- .../slideram/SliderAMProviderService.java | 6 + .../appmaster/state/ProviderAppState.java | 7 + .../state/StateAccessForProviders.java | 7 + .../server/appmaster/web/rest/RestPaths.java | 1 + .../web/rest/publisher/PublisherResource.java | 31 +++- .../slider/client/TestClientBadArgs.groovy | 43 ++++++ .../framework/AgentCommandTestBase.groovy | 27 ++++ .../funtest/lifecycle/AppsThroughAgentIT.groovy | 40 ++++++ 18 files changed, 874 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 78f214e..9ca1079 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -114,6 +114,9 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsOutputter; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; @@ -2279,11 +2282,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (registryArgs.listConf) { // list the configurations actionRegistryListConfigsYarn(registryArgs); + } else if (registryArgs.listExports) { + // list the exports + actionRegistryListExports(registryArgs); } else if (SliderUtils.isSet(registryArgs.getConf)) { // get a configuration PublishedConfiguration publishedConfiguration = actionRegistryGetConfig(registryArgs); outputConfig(publishedConfiguration, registryArgs); + } else if (SliderUtils.isSet(registryArgs.getExport)) { + // get a export group + PublishedExports publishedExports = + actionRegistryGetExport(registryArgs); + outputExport(publishedExports, registryArgs); } else { // it's an unknown command log.info(ActionRegistryArgs.USAGE); @@ -2657,6 +2668,34 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * list exports available for an instance + * + * @param registryArgs registry Arguments + * @throws YarnException YARN problems + * @throws IOException Network or other problems + */ + public void actionRegistryListExports(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + PublishedExportsSet exports = + retriever.getExports(!registryArgs.internal); + + for (String exportName : exports.keys()) { + if (!registryArgs.verbose) { + log.info("{}", exportName); + } else { + PublishedExports published = + exports.get(exportName); + log.info("{} : {}", + exportName, + published.description); + } + } + } + + /** * list configs available for an instance * * @param registryArgs registry Arguments @@ -2681,6 +2720,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * get a specific export group + * + * @param registryArgs registry Arguments + * + * @throws YarnException YARN problems + * @throws IOException Network or other problems + * @throws FileNotFoundException if the config is not found + */ + @VisibleForTesting + public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs) + throws YarnException, IOException { + ServiceRecord instance = lookupServiceRecord(registryArgs); + + RegistryRetriever retriever = new RegistryRetriever(instance); + boolean external = !registryArgs.internal; + PublishedExportsSet exports = + retriever.getExports(external); + + PublishedExports published = retriever.retrieveExports(exports, + registryArgs.getExport, + external); + return published; + } + + /** * write out the config. If a destination is provided and that dir is a * directory, the entry is written to it with the name provided + extension, * else it is printed to standard out. @@ -2720,6 +2784,44 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * write out the config + * @param published + * @param registryArgs + * @throws BadCommandArgumentsException + * @throws IOException + */ + private void outputExport(PublishedExports published, + ActionRegistryArgs registryArgs) throws + BadCommandArgumentsException, + IOException { + // decide whether or not to print + String entry = registryArgs.getExport; + String format = ConfigFormat.JSON.toString(); + ConfigFormat configFormat = ConfigFormat.resolve(format); + if (configFormat == null || configFormat != ConfigFormat.JSON) { + throw new BadCommandArgumentsException( + "Unknown/Unsupported format %s . Only JSON is supported.", format); + } + + PublishedExportsOutputter outputter = + PublishedExportsOutputter.createOutputter(configFormat, + published); + boolean print = registryArgs.out == null; + if (!print) { + File destFile; + destFile = registryArgs.out; + if (destFile.isDirectory()) { + // creating it under a directory + destFile = new File(destFile, entry + "." + format); + } + log.info("Destination path: {}", destFile); + outputter.save(destFile); + } else { + print(outputter.asString()); + } + } + + /** * Look up an instance * @return instance data * @throws SliderException other failures http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java index df40ef1..133fde4 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java @@ -47,14 +47,18 @@ public class ActionRegistryArgs extends AbstractActionArgs { + " (" + Arguments.ARG_LIST + "|" + Arguments.ARG_LISTCONF + "|" + + Arguments.ARG_LISTEXP + "|" + Arguments.ARG_LISTFILES + "|" - + Arguments.ARG_GETCONF + "> " + + Arguments.ARG_GETCONF + "|" + + Arguments.ARG_GETEXP + "> " + Arguments.ARG_NAME + " <name> " + " )" + "[" + Arguments.ARG_VERBOSE + "] " + "[" + Arguments.ARG_OUTPUT + " <filename> ] " + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] " + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] " + + System.getProperty("line.separator") + + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" ; public ActionRegistryArgs() { } @@ -89,7 +93,14 @@ public class ActionRegistryArgs extends AbstractActionArgs { description = "get configuration") public String getConf; - @Parameter(names = {ARG_LISTFILES}, + @Parameter(names = {ARG_LISTEXP}, + description = "list exports") + public boolean listExports; + @Parameter(names = {ARG_GETEXP}, + description = "get export") + public String getExport; + + @Parameter(names = {ARG_LISTFILES}, description = "list files") public String listFiles; @@ -131,8 +142,8 @@ public class ActionRegistryArgs extends AbstractActionArgs { super.validate(); //verify that at most one of the operations is set - int gets = s(getConf) + s(getFiles); - int lists = s(list) + s(listConf) + s(listFiles); + int gets = s(getConf) + s(getFiles) + s(getExport); + int lists = s(list) + s(listConf) + s(listFiles) + s(listExports); int set = lists + gets; if (set > 1) { throw new UsageException(USAGE); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 4deebd7..a860579 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -46,6 +46,7 @@ public interface Arguments { String ARG_FORMAT = "--format"; String ARG_FORCE = "--force"; String ARG_GETCONF = "--getconf"; + String ARG_GETEXP = "--getexp"; String ARG_GETFILES = "--getfiles"; String ARG_HELP = "--help"; String ARG_ID = "--id"; @@ -54,6 +55,7 @@ public interface Arguments { String ARG_LIST = "--list"; String ARG_LISTFILES = "--listfiles"; String ARG_LISTCONF = "--listconf"; + String ARG_LISTEXP = "--listexp"; String ARG_LIVE = "--live"; String ARG_MANAGER = "--manager"; String ARG_MANAGER_SHORT = "--m"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java new file mode 100644 index 0000000..1db5093 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ExportEntry { + + private String value; + private String containerId; + private String tag; + private String level; + + public ExportEntry() { + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + @Override + public String toString() { + return new StringBuilder("ExportEntry{"). + append("value='").append(value).append("',"). + append("containerId='").append(containerId).append("',"). + append("tag='").append(tag).append("',"). + append("level='").append(level).append("'"). + append(" }").toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java new file mode 100644 index 0000000..d6d285c --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.commons.collections.IteratorUtils; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExports { + + public String description; + public long updated; + public String updatedTime; + public Map<String, List<ExportEntry>> entries = new HashMap<String, List<ExportEntry>>(); + + public PublishedExports() { + } + + /** + * build an empty published configuration + * + * @param description configuration description + */ + public PublishedExports(String description) { + this.description = description; + } + + /** + * Build a configuration from the entries + * + * @param description configuration description + * @param entries entries to put + */ + public PublishedExports(String description, + Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.description = description; + putValues(entries); + } + + /** + * Is the configuration empty. This means either that it has not been given any values, or it is stripped down copy + * set down over the wire. + * + * @return + */ + public boolean isEmpty() { + return entries.isEmpty(); + } + + public long getUpdated() { + return updated; + } + + public void setUpdated(long updated) { + this.updated = updated; + this.updatedTime = new Date(updated).toString(); + } + + /** + * Set the values from an iterable (this includes a Hadoop Configuration and Java properties object). Any existing + * value set is discarded + * + * @param entries entries to put + */ + public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.entries = new HashMap<String, List<ExportEntry>>(); + for (Map.Entry<String, List<ExportEntry>> entry : entries) { + this.entries.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Return the values as json string + * + * @return + * + * @throws IOException + */ + public String asJson() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + String json = mapper.writeValueAsString(entries); + return json; + } + + /** + * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's + * values + * + * @return the copy + */ + public PublishedExports shallowCopy() { + PublishedExports that = new PublishedExports(); + that.description = this.description; + that.updated = this.updated; + that.updatedTime = this.updatedTime; + return that; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("PublishedConfiguration{"); + sb.append("description='").append(description).append('\''); + sb.append(" entries = ").append(entries.size()); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java new file mode 100644 index 0000000..b21e717 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** Output a published configuration */ +public abstract class PublishedExportsOutputter { + + protected final PublishedExports exports; + + protected PublishedExportsOutputter(PublishedExports exports) { + this.exports = exports; + } + + /** + * Create an outputter for the chosen format + * + * @param format format enumeration + * @param exports owning config + * @return the outputter + */ + + public static PublishedExportsOutputter createOutputter(ConfigFormat format, + PublishedExports exports) { + Preconditions.checkNotNull(exports); + switch (format) { + case JSON: + return new JsonOutputter(exports); + default: + throw new RuntimeException("Unsupported format :" + format); + } + } + + public void save(File dest) throws IOException { + FileOutputStream out = null; + try { + out = new FileOutputStream(dest); + save(out); + out.close(); + } finally { + org.apache.hadoop.io.IOUtils.closeStream(out); + } + } + + /** + * Save the content. The default saves the asString() value to the output stream + * + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + + /** + * Convert to a string + * + * @return + * @throws IOException + */ + public abstract String asString() throws IOException; + + public static class JsonOutputter extends PublishedExportsOutputter { + + public JsonOutputter(PublishedExports exports) { + super(exports); + } + + @Override + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + @Override + public String asString() throws IOException { + return exports.asJson(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java new file mode 100644 index 0000000..cdd35de --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.slider.server.appmaster.web.rest.RestPaths; +import org.apache.slider.server.services.utility.PatternValidator; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Represents a set of configurations for an application, component, etc. + * Json serialisable; accessors are synchronized + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExportsSet { + + private static final PatternValidator validator = new PatternValidator( + RestPaths.PUBLISHED_CONFIGURATION_REGEXP); + + public Map<String, PublishedExports> exports = + new HashMap<String, PublishedExports>(); + + public PublishedExportsSet() { + } + + /** + * Put a name -it will be converted to lower case before insertion. + * Any existing entry will be overwritten (that includes an entry + * with a different case in the original name) + * @param name name of entry + * @param export published export + * @throws IllegalArgumentException if not a valid name + */ + public void put(String name, PublishedExports export) { + String name1 = name.toLowerCase(Locale.ENGLISH); + validateName(name1); + exports.put(name1, export); + } + + /** + * Validate the name -restricting it to the set defined in + * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP} + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public static void validateName(String name) { + validator.validate(name); + + } + + public PublishedExports get(String name) { + return exports.get(name); + } + + public boolean contains(String name) { + return exports.containsKey(name); + } + + public int size() { + return exports.size(); + } + + public Set<String> keys() { + TreeSet<String> keys = new TreeSet<String>(); + keys.addAll(exports.keySet()); + return keys; + } + + public PublishedExportsSet shallowCopy() { + PublishedExportsSet that = new PublishedExportsSet(); + for (Map.Entry<String, PublishedExports> entry : + exports.entrySet()) { + that.put(entry.getKey(), entry.getValue().shallowCopy()); + } + return that; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java index 65c122f..67b9feb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java @@ -35,6 +35,9 @@ public class CustomRegistryConstants { public static final String PUBLISHER_CONFIGURATIONS_API = "org.apache.slider.publisher.configurations"; + public static final String PUBLISHER_EXPORTS_API = + "org.apache.slider.publisher.exports"; + public static final String PUBLISHER_DOCUMENTS_API = "org.apache.slider.publisher.documents"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java index 101efb2..a91f515 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java @@ -33,6 +33,8 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.ExceptionConverter; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +54,8 @@ public class RegistryRetriever { private final String externalConfigurationURL; private final String internalConfigurationURL; + private final String externalExportsURL; + private final String internalExportsURL; private static final Client jerseyClient; static { @@ -63,9 +67,12 @@ public class RegistryRetriever { jerseyClient.setFollowRedirects(true); } - public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL) { - this.externalConfigurationURL = externalConfigurationURL; - this.internalConfigurationURL = internalConfigurationURL; + public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL, + String externalExportsURL, String internalExportsURL) { + this.externalConfigurationURL = externalConfigurationURL; + this.internalConfigurationURL = internalConfigurationURL; + this.externalExportsURL = externalExportsURL; + this.internalExportsURL = internalExportsURL; } /** @@ -99,6 +106,29 @@ public class RegistryRetriever { } } externalConfigurationURL = url; + + internal = record.getInternalEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API); + url = null; + if (internal != null) { + List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType( + internal); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } + } + internalExportsURL = url; + external = record.getExternalEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API); + url = null; + if (external != null) { + List<String> addresses = + RegistryTypeUtils.retrieveAddressesUriType(external); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } + } + externalExportsURL = url; } /** @@ -138,6 +168,33 @@ public class RegistryRetriever { return confURL; } + protected String getExportURL(boolean external) throws FileNotFoundException { + String confURL = external ? externalExportsURL: internalExportsURL; + if (Strings.isStringEmpty(confURL)) { + throw new FileNotFoundException("No configuration URL"); + } + return confURL; + } + + /** + * Get the configurations of the registry + * @param external flag to indicate that it is the external entries to fetch + * @return the configuration sets + */ + public PublishedExportsSet getExports(boolean external) throws + FileNotFoundException, IOException { + + String exportsUrl = getExportURL(external); + try { + WebResource webResource = jsonResource(exportsUrl); + log.debug("GET {}", exportsUrl); + PublishedExportsSet exportSet = webResource.get(PublishedExportsSet.class); + return exportSet; + } catch (UniformInterfaceException e) { + throw ExceptionConverter.convertJerseyException(exportsUrl, e); + } + } + private WebResource resource(String url) { WebResource resource = jerseyClient.resource(url); return resource; @@ -174,7 +231,33 @@ public class RegistryRetriever { throw ExceptionConverter.convertJerseyException(confURL, e); } } - + + /** + * Get a complete export, with all values + * @param exportSet + * @param name name of the configuration + * @param external flag to indicate that it is an external configuration + * @return the retrieved config + * @throws IOException IO problems + */ + public PublishedExports retrieveExports(PublishedExportsSet exportSet, + String name, + boolean external) throws IOException { + if (!exportSet.contains(name)) { + throw new FileNotFoundException("Unknown export " + name); + } + String exportsURL = getExportURL(external); + exportsURL = SliderUtils.appendToURL(exportsURL, name); + try { + WebResource webResource = jsonResource(exportsURL); + PublishedExports publishedExports = + webResource.get(PublishedExports.class); + return publishedExports; + } catch (UniformInterfaceException e) { + throw ExceptionConverter.convertJerseyException(exportsURL, e); + } + } + @Override public String toString() { return super.toString() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 44777c3..296537d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -52,7 +52,9 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.AbstractProviderService; import org.apache.slider.providers.ProviderCore; @@ -97,6 +99,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -127,10 +130,15 @@ public class AgentProviderService extends AbstractProviderService implements private static final String CONTAINER_ID = "container_id"; private static final String GLOBAL_CONFIG_TAG = "global"; private static final String LOG_FOLDERS_TAG = "LogFolders"; + private static final String HOST_FOLDER_FORMAT = "%s:%s"; + private static final String CONTAINER_LOGS_TAG = "container_log_dirs"; + private static final String CONTAINER_PWDS_TAG = "container_work_dirs"; + private static final String COMPONENT_TAG = "component"; + private static final String APPLICATION_TAG = "application"; private static final String COMPONENT_DATA_TAG = "ComponentInstanceData"; private static final String SHARED_PORT_TAG = "SHARED"; private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}"; - private static final int MAX_LOG_ENTRIES = 20; + private static final int MAX_LOG_ENTRIES = 40; private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000; private final Object syncLock = new Object(); @@ -153,13 +161,19 @@ public class AgentProviderService extends AbstractProviderService implements new ConcurrentHashMap<String, Map<String, String>>(); private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<String, Map<String, String>>(); - private final Map<String, String> workFolders = - Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) { + + private final Map<String, ExportEntry> logFolderExports = + Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) { + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_LOG_ENTRIES; + } + }); + private final Map<String, ExportEntry> workFolderExports = + Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) { protected boolean removeEldestEntry(Map.Entry eldest) { return size() > MAX_LOG_ENTRIES; } }); - /** * Create an instance of AgentProviderService */ @@ -749,6 +763,16 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting + protected Map<String, ExportEntry> getLogFolderExports() { + return logFolderExports; + } + + @VisibleForTesting + protected Map<String, ExportEntry> getWorkFolderExports() { + return workFolderExports; + } + + @VisibleForTesting protected Metainfo getMetainfo() { return this.metainfo; } @@ -898,13 +922,56 @@ public class AgentProviderService extends AbstractProviderService implements */ protected void publishLogFolderPaths( Map<String, String> folders, String containerId, String roleName, String hostFqdn) { - for (Map.Entry<String, String> entry: folders.entrySet()) { - workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, entry.getKey(), containerId), - entry.getValue()); + for (Map.Entry<String, String> entry : folders.entrySet()) { + ExportEntry exportEntry = new ExportEntry(); + exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, entry.getValue())); + exportEntry.setContainerId(containerId); + exportEntry.setLevel(COMPONENT_TAG); + exportEntry.setTag(roleName); + if (entry.getKey().equals("AGENT_LOG_ROOT")) { + synchronized (logFolderExports) { + getLogFolderExports().put(containerId, exportEntry); + } + } else { + synchronized (workFolderExports) { + getWorkFolderExports().put(containerId, exportEntry); + } + } + log.info("Updating log and pwd folders for container {}", containerId); } - publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, - (new HashMap<String, String>(this.workFolders)).entrySet()); + Date now = new Date(); + PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG); + exports.setUpdated(now.getTime()); + synchronized (logFolderExports) { + updateExportsFromList(exports, getLogFolderExports()); + } + getAmState().getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports); + + exports = new PublishedExports(CONTAINER_PWDS_TAG); + exports.setUpdated(now.getTime()); + synchronized (workFolderExports) { + updateExportsFromList(exports, getWorkFolderExports()); + } + getAmState().getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports); + } + + /** + * Update the export data from the map + * @param exports + * @param folderExports + */ + private void updateExportsFromList(PublishedExports exports, Map<String, ExportEntry> folderExports) { + Map<String, List<ExportEntry>> perComponentList = new HashMap<String, List<ExportEntry>>(); + for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) + { + String componentName = logEntry.getValue().getTag(); + if(!perComponentList.containsKey(componentName)) { + perComponentList.put(componentName, new ArrayList<ExportEntry>()); + } + perComponentList.get(componentName).add(logEntry.getValue()); + } + exports.putValues(perComponentList.entrySet()); } @@ -1101,6 +1168,8 @@ public class AgentProviderService extends AbstractProviderService implements } } publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet()); + + // add to the new exports } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java index 601c3f9..afe6428 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@ -146,6 +146,8 @@ public class SliderAMProviderService extends AbstractProviderService implements String configurationsURL = SliderUtils.appendToURL( publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET); + String exportsURL = SliderUtils.appendToURL( + publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS); serviceRecord.addExternalEndpoint( RegistryTypeUtils.webEndpoint( @@ -166,6 +168,10 @@ public class SliderAMProviderService extends AbstractProviderService implements RegistryTypeUtils.restEndpoint( CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, new URI(configurationsURL))); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_EXPORTS_API, + new URI(exportsURL))); } catch (URISyntaxException e) { throw new IOException(e); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index a0871ae..9c5da12 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.services.utility.PatternValidator; @@ -40,6 +41,7 @@ public class ProviderAppState implements StateAccessForProviders { private final Map<String, PublishedConfigSet> publishedConfigSets = new ConcurrentHashMap<String, PublishedConfigSet>(5); + private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet(); private static final PatternValidator validator = new PatternValidator( RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP); private String applicationName; @@ -66,6 +68,11 @@ public class ProviderAppState implements StateAccessForProviders { } @Override + public PublishedExportsSet getPublishedExportsSet() { + return publishedExportsSets; + } + + @Override public PublishedConfigSet getPublishedConfigSet(String name) { return publishedConfigSets.get(name); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index 1714f75..b907b06 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import java.util.Collection; import java.util.List; @@ -51,6 +52,12 @@ public interface StateAccessForProviders { PublishedConfigSet getPublishedSliderConfigurations(); /** + * Get the published exports set + * @return + */ + PublishedExportsSet getPublishedExportsSet(); + + /** * Get a named published config set * @param name name to look up * @return the instance or null http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 93601ad..94f1e4c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -61,6 +61,7 @@ public class RestPaths { = "[a-z0-9][a-z0-9_.\\+-]*"; public static final String SLIDER_CONFIGSET = "slider"; + public static final String SLIDER_EXPORTS = "exports"; public static final String SLIDER_CLASSPATH = "classpath"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java index 5d8b657..e47bbb9 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java @@ -23,6 +23,8 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.docstore.UriMap; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -56,7 +58,10 @@ public class PublisherResource { protected static final Logger log = LoggerFactory.getLogger(PublisherResource.class); private final WebAppApi slider; - public static final String SET_NAME = + public static final String EXPORTS_NAME = "exports"; + public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME; + public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ; + public static final String SET_NAME = "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}"; private static final String CONFIG = SET_NAME + "/{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}"; @@ -101,7 +106,9 @@ public class PublisherResource { UriMap uriMap = new UriMap(); for (String name : appState.listConfigSets()) { uriMap.put(name, baseURL + name); + log.info("Tick tack {} and {}", name, baseURL); } + uriMap.put(EXPORTS_NAME, baseURL + EXPORTS_NAME); return uriMap; } @@ -114,6 +121,26 @@ public class PublisherResource { } @GET + @Path(EXPORTS_RESOURCES_PATH) + @Produces({MediaType.APPLICATION_JSON}) + public PublishedExportsSet gePublishedExports() { + + PublishedExportsSet set = appState.getPublishedExportsSet(); + return set.shallowCopy(); + } + + @GET + @Path(EXPORT_RESOURCE_PATH) + @Produces({MediaType.APPLICATION_JSON}) + public PublishedExports getAMExports2(@PathParam("exportname") String exportname, + @Context UriInfo uriInfo, + @Context HttpServletResponse res) { + init(res, uriInfo); + PublishedExportsSet set = appState.getPublishedExportsSet(); + return set.get(exportname); + } + + @GET @Path("/"+ SET_NAME) @Produces({MediaType.APPLICATION_JSON}) public PublishedConfigSet getPublishedConfiguration( @@ -129,7 +156,7 @@ public class PublisherResource { } private void logRequest(UriInfo uriInfo) { - log.debug(uriInfo.getRequestUri().toString()); + log.info(uriInfo.getRequestUri().toString()); } @GET http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy index d1f8a8f..7d596d6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy @@ -24,8 +24,10 @@ import org.apache.hadoop.conf.Configuration import org.apache.slider.common.params.ActionRegistryArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.ErrorStrings import org.apache.slider.core.exceptions.UsageException +import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.main.ServiceLauncherBaseTest import org.junit.Test @@ -88,4 +90,45 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { log.info(exception.toString()) } + @Test + public void testRegistryExportBadUsage1() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Expected a value after parameter --getexp", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_GETEXP]) + assert exception instanceof BadCommandArgumentsException + log.info(exception.toString()) + } + + @Test + public void testRegistryExportBadUsage2() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Expected a value after parameter --getexp", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP]) + assert exception instanceof BadCommandArgumentsException + log.info(exception.toString()) + } + + @Test + public void testRegistryExportBadUsage3() throws Throwable { + def exception = launchExpectingException(SliderClient, + new Configuration(), + "Usage: registry", + [SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP, + "export1"]) + assert exception instanceof UsageException + log.info(exception.toString()) + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index ec7d3e3..04e4d0d 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -161,6 +161,33 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { return null; } + public static boolean containsString(SliderShell shell, String lookThisUp, int n = 1) { + int count = 0 + for (String str in shell.out) { + int subCount = countString(str, lookThisUp) + count = count + subCount + if (count == n) { + return true; + } + } + + return false; + } + + public static int countString(String str, String search) { + int count = 0 + if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) { + return count + } + + int index = str.indexOf(search, 0) + while (index > 0) { + index = str.indexOf(search, index + 1) + ++count + } + return count + } + public static String findLineEntryValue(SliderShell shell, String[] locaters) { String line = findLineEntry(shell, locaters); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f4c8b67/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy index 234275a..0796355 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy @@ -75,6 +75,46 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assertComponentCount(COMMAND_LOGGER, 2, shell) assertSuccess(shell) + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_LISTEXP]) + if(!containsString(shell, "container_log_dirs") || + !containsString(shell, "container_work_dirs")) { + logShell(shell) + assert fail("Should list default exports container_log_dirs or container_work_dirs") + } + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETEXP, + "container_log_dirs"]) + if(!containsString(shell, "\"tag\":\"COMMAND_LOGGER\",\"level\":\"component\"", 2)) { + logShell(shell) + assert fail("Should list 2 entries for log folders") + } + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETEXP, + "container_work_dirs"]) + if(!containsString(shell, "\"tag\":\"COMMAND_LOGGER\",\"level\":\"component\"", 2)) { + logShell(shell) + assert fail("Should list 2 entries for work folder") + } + assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.' } }
