YARN-5993. Allow native services quicklinks to be exported for each component. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4faaf9f3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4faaf9f3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4faaf9f3 Branch: refs/heads/yarn-native-services Commit: 4faaf9f326a4bd56db7f5a40f5b550180fe5f047 Parents: c64e27a Author: Gour Saha <[email protected]> Authored: Wed Dec 28 15:24:36 2016 -0800 Committer: Jian He <[email protected]> Committed: Tue Apr 18 11:24:18 2017 +0800 ---------------------------------------------------------------------- .../apache/slider/common/tools/SliderUtils.java | 8 +- .../core/registry/docstore/ExportEntry.java | 20 +++ .../registry/docstore/PublishedExports.java | 23 ++- .../apache/slider/providers/ProviderUtils.java | 133 ++++------------ .../providers/docker/DockerProviderService.java | 151 ++++++++++++------- .../server/appmaster/web/view/IndexBlock.java | 15 +- 6 files changed, 177 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index a107dfb..20c7831 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -183,17 +183,17 @@ public final class SliderUtils { return !isUnset(s); } - public static boolean isEmpty(List l) { + public static boolean isEmpty(Collection l) { return l == null || l.isEmpty(); } /** - * Probe for a list existing and not being empty - * @param l list + * Probe for a collection existing and not being empty + * @param l collection * @return true if the reference is valid and it contains entries */ - public static boolean isNotEmpty(List l) { + public static boolean isNotEmpty(Collection l) { return l != null && !l.isEmpty(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java index 4bcf6c1..dd6e034 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java @@ -107,6 +107,26 @@ public class ExportEntry { } @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ExportEntry that = (ExportEntry) o; + + if (value != null ? !value.equals(that.value) : that.value != null) + return false; + return containerId != null ? containerId.equals(that.containerId) : + that.containerId == null; + } + + @Override + public int hashCode() { + int result = value != null ? value.hashCode() : 0; + result = 31 * result + (containerId != null ? containerId.hashCode() : 0); + return result; + } + + @Override public String toString() { return new StringBuilder("ExportEntry{"). append("value='").append(value).append("',"). http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java index 0759b4e..58e67ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java @@ -26,8 +26,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; import java.io.IOException; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; /** * JSON-serializable description of a published key-val configuration. @@ -41,7 +43,7 @@ public class PublishedExports { public String description; public long updated; public String updatedTime; - public Map<String, List<ExportEntry>> entries = new HashMap<>(); + public Map<String, Set<ExportEntry>> entries = new HashMap<>(); public PublishedExports() { } @@ -62,7 +64,7 @@ public class PublishedExports { * @param entries entries to put */ public PublishedExports(String description, - Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + Iterable<Entry<String, Set<ExportEntry>>> entries) { this.description = description; putValues(entries); } @@ -87,15 +89,22 @@ public class PublishedExports { this.updatedTime = new Date(updated).toString(); } + + public Map<String, Set<ExportEntry>> sortedEntries() { + Map<String, Set<ExportEntry>> sortedEntries = new TreeMap<>(); + sortedEntries.putAll(entries); + return sortedEntries; + } + /** * Set the values from an iterable (this includes a Hadoop Configuration and Java properties * object). Any existing value set is discarded * - * @param entries entries to put + * @param values values to put */ - public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) { - this.entries = new HashMap<String, List<ExportEntry>>(); - for (Map.Entry<String, List<ExportEntry>> entry : entries) { + public void putValues(Iterable<Map.Entry<String, Set<ExportEntry>>> values) { + this.entries = new HashMap<>(); + for (Map.Entry<String, Set<ExportEntry>> entry : values) { this.entries.put(entry.getKey(), entry.getValue()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index 0d9ddec..f33db9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -75,6 +75,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.regex.Pattern; /** @@ -1054,32 +1055,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } /** - * Return a list of hostnames based on current ClusterNodes. - * @param values cluster nodes - * @return list of hosts - */ - public Iterable<String> getHostNamesList(Collection<ClusterNode> values) { - List<String> hosts = new ArrayList<>(); - for (ClusterNode cn : values) { - hosts.add(cn.hostname); - } - return hosts; - } - - /** - * Return a list of IPs based on current ClusterNodes. - * @param values cluster nodes - * @return list of hosts - */ - public Iterable<String> getIPsList(Collection<ClusterNode> values) { - List<String> hosts = new ArrayList<>(); - for (ClusterNode cn : values) { - hosts.add(cn.ip); - } - return hosts; - } - - /** * Update ServiceRecord in Registry with IP and hostname. * @param amState access to AM state * @param yarnRegistry acces to YARN registry @@ -1148,27 +1123,30 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * Publish an export group. * @param exportGroup export groups * @param amState access to AM state - * @param roleGroup component group + * @param groupName export group name */ - public void publishExportGroup(Map<String, List<ExportEntry>> exportGroup, - StateAccessForProviders amState, String roleGroup) { + public void publishExportGroup( + Map<String, Set<ExportEntry>> exportGroup, + StateAccessForProviders amState, String groupName) { // Publish in old format for the time being Map<String, String> simpleEntries = new HashMap<>(); - for (Entry<String, List<ExportEntry>> entry : exportGroup.entrySet()) { - List<ExportEntry> exports = entry.getValue(); + for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) { + Set<ExportEntry> exports = entry.getValue(); if (SliderUtils.isNotEmpty(exports)) { - // there is no support for multiple exports per name, so extract only - // the first one - simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); + Set<String> values = new TreeSet<>(); + for (ExportEntry export : exports) { + values.add(export.getValue()); + } + simpleEntries.put(entry.getKey(), StringUtils.join(",", values)); } } - publishApplicationInstanceData(roleGroup, roleGroup, + publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet(), amState); - PublishedExports exports = new PublishedExports(roleGroup); + PublishedExports exports = new PublishedExports(groupName); exports.setUpdated(new Date().getTime()); exports.putValues(exportGroup.entrySet()); - amState.getPublishedExportsSet().put(roleGroup, exports); + amState.getPublishedExportsSet().put(groupName, exports); } public Map<String, String> getExports(ConfTreeOperations appConf, @@ -1179,75 +1157,26 @@ public class ProviderUtils implements RoleKeys, SliderKeys { return exports; } - private static final String COMPONENT_TAG = "component"; - private static final String HOST_FOLDER_FORMAT = "%s:%s"; - private static final String CONTAINER_LOGS_TAG = "container_log_dirs"; - private static final String CONTAINER_PWDS_TAG = "container_work_dirs"; - - /** - * Format the folder locations and publish in the registry service. - * @param folders folder information - * @param containerId container ID - * @param hostFqdn host FQDN - * @param componentName component name - */ - public void publishFolderPaths(Map<String, String> folders, - String containerId, String componentName, String hostFqdn, - StateAccessForProviders amState, - Map<String, ExportEntry> logFolderExports, - Map<String, ExportEntry> workFolderExports) { - Date now = new Date(); - for (Map.Entry<String, String> entry : folders.entrySet()) { - ExportEntry exportEntry = new ExportEntry(); - exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, - entry.getValue())); - exportEntry.setContainerId(containerId); - exportEntry.setLevel(COMPONENT_TAG); - exportEntry.setTag(componentName); - exportEntry.setUpdatedTime(now.toString()); - if (entry.getKey().equals("AGENT_LOG_ROOT") || - entry.getKey().equals("LOG_DIR")) { - synchronized (logFolderExports) { - logFolderExports.put(containerId, exportEntry); - } - } else { - synchronized (workFolderExports) { - workFolderExports.put(containerId, exportEntry); - } - } - log.info("Updating log and pwd folders for container {}", containerId); - } - - PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG); - exports.setUpdated(now.getTime()); - synchronized (logFolderExports) { - updateExportsFromList(exports, logFolderExports); - } - amState.getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports); + public String getGroupKey(String roleGroup, ConfTreeOperations appConf) { + String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); + return getNameOrGroupKey(rolePrefix, roleGroup); + } - exports = new PublishedExports(CONTAINER_PWDS_TAG); - exports.setUpdated(now.getTime()); - synchronized (workFolderExports) { - updateExportsFromList(exports, workFolderExports); - } - amState.getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports); + public String getNameKey(String roleName, String roleGroup, + ConfTreeOperations appConf) { + String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); + return getNameOrGroupKey(rolePrefix, roleName); } - /** - * Update the export data from the map. - * @param exports published exports - * @param folderExports folder exports - */ - private void updateExportsFromList(PublishedExports exports, - Map<String, ExportEntry> folderExports) { - Map<String, List<ExportEntry>> perComponentList = new HashMap<>(); - for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) { - String componentName = logEntry.getValue().getTag(); - if (!perComponentList.containsKey(componentName)) { - perComponentList.put(componentName, new ArrayList<ExportEntry>()); + public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) { + if (!rolePrefix.isEmpty()) { + if (!roleNameOrGroup.startsWith(rolePrefix)) { + log.warn("Something went wrong, {} doesn't start with {}", + roleNameOrGroup, rolePrefix); + return null; } - perComponentList.get(componentName).add(logEntry.getValue()); + roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length()); } - exports.putValues(perComponentList.entrySet()); + return roleNameOrGroup.toUpperCase(Locale.ENGLISH); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 5edfa6a..63416cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -46,27 +46,24 @@ import org.apache.slider.providers.ProviderCore; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Scanner; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; -import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; - public class DockerProviderService extends AbstractProviderService implements ProviderCore, DockerKeys, @@ -77,10 +74,16 @@ public class DockerProviderService extends AbstractProviderService implements private static final ProviderUtils providerUtils = new ProviderUtils(log); private static final String EXPORT_GROUP = "quicklinks"; private static final String APPLICATION_TAG = "application"; + private static final String HOST_KEY_FORMAT = "${%s_HOST}"; + private static final String IP_KEY_FORMAT = "${%s_IP}"; + private static final String VARIABLE_INDICATOR = "${"; private String clusterName = null; private SliderFileSystem fileSystem = null; + private final Map<String, Set<ExportEntry>> exportMap = + new ConcurrentHashMap<>(); + protected DockerProviderService() { super("DockerProviderService"); } @@ -118,9 +121,6 @@ public class DockerProviderService extends AbstractProviderService implements String roleName = providerRole.name; String roleGroup = providerRole.group; - initializeApplicationConfiguration(instanceDefinition, fileSystem, - roleGroup); - log.info("Build launch context for Docker"); log.debug(instanceDefinition.toString()); @@ -280,6 +280,23 @@ public class DockerProviderService extends AbstractProviderService implements } @Override + public void notifyContainerCompleted(ContainerId containerId) { + if (containerId != null) { + String containerIdStr = containerId.toString(); + log.info("Removing container exports for {}", containerIdStr); + for (Set<ExportEntry> exportEntries : exportMap.values()) { + for (Iterator<ExportEntry> iter = exportEntries.iterator(); + iter.hasNext();) { + ExportEntry entry = iter.next(); + if (containerIdStr.equals(entry.getContainerId())) { + iter.remove(); + } + } + } + } + } + + @Override public boolean processContainerStatus(ContainerId containerId, ContainerStatus status) { log.debug("Handling container status: {}", status); @@ -301,7 +318,7 @@ public class DockerProviderService extends AbstractProviderService implements containerIdStr, roleName, status.getIPs(), status.getHost()); publishExportGroups(containerIdStr, roleName, roleGroup, - status.getHost()); + status.getHost(), status.getIPs()); return false; } @@ -312,17 +329,13 @@ public class DockerProviderService extends AbstractProviderService implements * are substituted with the actual hostnames of the containers. */ protected void publishExportGroups(String containerId, - String roleName, String roleGroup, String thisHost) { + String roleName, String roleGroup, String thisHost, List<String> ips) { ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); Map<String, String> exports = providerUtils.getExports( getAmState().getAppConfSnapshot(), roleGroup); - String hostKeyFormat = "${%s_HOST}"; - String hostNameKeyFormat = "${%s_HOSTNAME}"; - String ipKeyFormat = "${%s_IP}"; - // publish export groups if any Map<String, String> standardTokens = providerUtils.getStandardTokenMap( appConf, internalsConf, roleName, roleGroup, containerId, @@ -331,38 +344,33 @@ public class DockerProviderService extends AbstractProviderService implements appConf.getComponent(roleGroup).options, standardTokens); replaceTokens.putAll(standardTokens); - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - for (Map.Entry<String, Map<String, ClusterNode>> entry : - getAmState().getRoleClusterNodeMapping().entrySet()) { - String otherRolePrefix = appConf.getComponentOpt(entry.getKey(), - ROLE_PREFIX, ""); - if (!otherRolePrefix.equals(rolePrefix)) { - // hostname replacements are only made within role prefix groups - continue; - } - String key = entry.getKey(); - if (!rolePrefix.isEmpty()) { - if (!key.startsWith(rolePrefix)) { - log.warn("Something went wrong, {} doesn't start with {}", key, - rolePrefix); - continue; - } - key = key.substring(rolePrefix.length()); + String roleNameKey = providerUtils.getNameKey(roleName, roleGroup, + appConf); + String roleNameIPKey = null; + if (roleNameKey != null) { + replaceTokens.put(String.format(HOST_KEY_FORMAT, roleNameKey), thisHost); + roleNameIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, roleNameKey)); + } else { + // should not happen, but log if it does + log.info("Not replacing HOST or IP tokens because key was null for {}", + roleName); + } + String roleGroupKey = providerUtils.getGroupKey(roleGroup, appConf); + String roleGroupIPKey = null; + if (roleGroupKey != null) { + if (roleNameKey == null || !roleGroupKey.equals(roleNameKey)) { + replaceTokens.put(String.format(HOST_KEY_FORMAT, roleGroupKey), + thisHost); + roleGroupIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, + roleGroupKey)); } - key = key.toUpperCase(Locale.ENGLISH); - String host = providerUtils.getHostsList( - entry.getValue().values(), true).iterator().next(); - replaceTokens.put(String.format(hostKeyFormat, key), host); - String hostName = providerUtils.getHostNamesList( - entry.getValue().values()).iterator().next(); - replaceTokens.put(String.format(hostNameKeyFormat, key), hostName); - String ip = providerUtils.getIPsList( - entry.getValue().values()).iterator().next(); - replaceTokens.put(String.format(ipKeyFormat, key), ip); + } else { + // should not happen, but log if it does + log.info("Not replacing HOST or IP tokens because key was null for {}", + roleGroup); } replaceTokens.put("${THIS_HOST}", thisHost); - Map<String, List<ExportEntry>> entries = new HashMap<>(); for (Entry<String, String> export : exports.entrySet()) { String value = export.getValue(); // replace host names and site properties @@ -372,18 +380,53 @@ public class DockerProviderService extends AbstractProviderService implements value = value.replaceAll(Pattern.quote(token), entry.getValue()); } } - ExportEntry entry = new ExportEntry(); - entry.setLevel(APPLICATION_TAG); - entry.setValue(value); - entry.setUpdatedTime(new Date().toString()); - // over-write, app exports are singletons - entries.put(export.getKey(), new ArrayList(Arrays.asList(entry))); - log.info("Preparing to publish. Key {} and Value {}", - export.getKey(), value); + Set<String> values = new HashSet<>(); + for (String ip : ips) { + values.add(substituteIP(roleNameIPKey, roleGroupIPKey, ip, value)); + } + for (String exportValue : values) { + if (exportValue.contains(VARIABLE_INDICATOR)) { + // not all variables have been substituted, so do not export + continue; + } + ExportEntry entry = new ExportEntry(); + entry.setContainerId(containerId); + entry.setLevel(APPLICATION_TAG); + entry.setValue(exportValue); + entry.setUpdatedTime(new Date().toString()); + Set<ExportEntry> exportEntries = getExportEntries(export.getKey()); + exportEntries.add(entry); + log.info("Preparing to publish for {}. Key {} and Value {}", + roleName, export.getKey(), entry); + } + } + if (!exportMap.isEmpty()) { + providerUtils.publishExportGroup(exportMap, getAmState(), EXPORT_GROUP); } - if (!entries.isEmpty()) { - providerUtils.publishExportGroup(entries, getAmState(), EXPORT_GROUP); + } + + protected String substituteIP(String roleNameIPKey, String roleGroupIPKey, + String ip, String value) { + if (roleNameIPKey != null) { + value = value.replaceAll(roleNameIPKey, ip); } + if (roleGroupIPKey != null) { + value = value.replaceAll(roleGroupIPKey, ip); + } + return value; + } + + protected Set<ExportEntry> getExportEntries(String key) { + if (!this.exportMap.containsKey(key)) { + synchronized (this.exportMap) { + if (!this.exportMap.containsKey(key)) { + this.exportMap.put(key, Collections.newSetFromMap( + new ConcurrentHashMap<>())); + } + } + } + + return this.exportMap.get(key); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java index c3b9b6f..2f99b27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java @@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.StatusKeys; import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.api.types.RoleStatistics; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedExports; @@ -42,6 +41,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS; @@ -291,12 +291,15 @@ public class IndexBlock extends SliderHamletBlock { LI<UL<Hamlet>> item = ul.li(); item.span().$class("bold")._(export.description)._(); UL sublist = item.ul(); - for (Entry<String, List<ExportEntry>> entry : export.entries.entrySet()) { - LI sublistItem = sublist.li()._(entry.getKey()); - for (ExportEntry exportEntry : entry.getValue()) { - sublistItem._(exportEntry.getValue()); + for (Entry<String, Set<ExportEntry>> entry : export.sortedEntries() + .entrySet()) { + if (SliderUtils.isNotEmpty(entry.getValue())) { + LI sublistItem = sublist.li()._(entry.getKey()); + for (ExportEntry exportEntry : entry.getValue()) { + sublistItem._(exportEntry.getValue()); + } + sublistItem._(); } - sublistItem._(); } sublist._(); item._(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
