Repository: ambari Updated Branches: refs/heads/branch-1.6.0.slider 8df5a36ad -> 1d22431b6
AMBARI-5923. When slider app is frozen and thawed multiple times, multiple entries show up in apps table. - string comparison problems (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d22431b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d22431b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d22431b Branch: refs/heads/branch-1.6.0.slider Commit: 1d22431b6f8182cccd9cb318b85d4bf1c1e3948c Parents: 8df5a36 Author: Srimanth Gunturi <[email protected]> Authored: Thu May 29 17:11:25 2014 -0700 Committer: Srimanth Gunturi <[email protected]> Committed: Thu May 29 17:11:25 2014 -0700 ---------------------------------------------------------------------- .../slider/SliderAppsViewControllerImpl.java | 135 ++++++++++--------- 1 file changed, 73 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1d22431b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java index 5300008..3f3f24f 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java @@ -97,8 +97,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { private String getAppsFolderPath() { return viewContext - .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY) - + "/apps"; + .getAmbariProperty(org.apache.ambari.server.configuration.Configuration.RESOURCES_DIR_KEY) + + "/apps"; } private String getAppsCreateFolderPath() { @@ -183,7 +183,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (hdfsConfig != null && hdfsConfig.containsKey("dfs.namenode.http-address")) { viewUrls.put("dfs.namenode.http-address", - "http://"; + hdfsConfig.get("dfs.namenode.http-address")); + "http://"; + hdfsConfig.get("dfs.namenode.http-address")); } } // YARN URL @@ -195,7 +195,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (yarnConfig != null && yarnConfig.containsKey("yarn.resourcemanager.webapp.address")) { viewUrls.put("yarn.resourcemanager.webapp.address", "http://"; - + yarnConfig.get("yarn.resourcemanager.webapp.address")); + + yarnConfig.get("yarn.resourcemanager.webapp.address")); } } } else { @@ -218,7 +218,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { private String getApplicationIdString(ApplicationId appId) { return Long.toString(appId.getClusterTimestamp()) + "_" - + Integer.toString(appId.getId()); + + Integer.toString(appId.getId()); } private ApplicationId getApplicationId(String appIdString) { @@ -254,7 +254,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } private SliderApp createSliderAppObject(ApplicationReport yarnApp, - Set<String> properties, SliderClient sliderClient) { + Set<String> properties, SliderClient sliderClient) { if (yarnApp == null) { return null; } @@ -311,9 +311,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { List<SliderAppType> matchingAppTypes = getSliderAppTypes(null); if (matchingAppTypes != null && matchingAppTypes.size() > 0) { for (SliderAppType appType : matchingAppTypes) { - if ((appType.getTypeName() != null && appType.getTypeName().equalsIgnoreCase(app.getType())) && - (appType.getTypeVersion() != null - && appType.getTypeVersion().equalsIgnoreCase(app.getAppVersion()))) { + if ((appType.getTypeName() != null && appType.getTypeName() + .equalsIgnoreCase(app.getType())) + && (appType.getTypeVersion() != null && appType.getTypeVersion() + .equalsIgnoreCase(app.getAppVersion()))) { matchedAppType = appType; break; } @@ -349,7 +350,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (quickLinks != null && quickLinks.containsKey("JMX")) { String jmxUrl = quickLinks.get("JMX"); if (matchedAppType != null) { - app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, matchedAppType)); + app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, + matchedAppType)); } } Map<String, Map<String, String>> configs = sliderAppClient @@ -365,10 +367,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>(); for (Entry<String, Object> e : description.status.entrySet()) { @SuppressWarnings("unchecked") - Map<String, Map<String, Map<String, Object>>> - componentsObj = - (Map<String, Map<String, Map<String, Object>>>) e - .getValue(); + Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e + .getValue(); boolean isLive = "live".equals(e.getKey()); for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj .entrySet()) { @@ -382,7 +382,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appComponent .setCompletedContainers(new HashMap<String, Map<String, String>>()); componentTypeMap.put(componentEntry.getKey(), - appComponent); + appComponent); } for (Entry<String, Map<String, Object>> containerEntry : componentEntry .getValue().entrySet()) { @@ -395,19 +395,19 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Object containerPropertyValue = containerValues .get(containerProperty); containerDataMap.put(containerProperty, - containerPropertyValue.toString()); + containerPropertyValue.toString()); } if (isLive) { appComponent.getActiveContainers().put(containerId, - containerDataMap); + containerDataMap); } else { appComponent.getCompletedContainers().put( containerId, containerDataMap); } } appComponent.setInstanceCount(appComponent - .getActiveContainers().size() - + appComponent.getCompletedContainers().size()); + .getActiveContainers().size() + + appComponent.getCompletedContainers().size()); } } app.setComponents(componentTypeMap); @@ -415,16 +415,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } catch (UnknownApplicationInstanceException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); } catch (YarnException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { logger.warn( "Unable to determine app components for " - + yarnApp.getName(), e); + + yarnApp.getName(), e); throw new RuntimeException(e.getMessage(), e); } } else if (property.startsWith(METRICS_PREFIX)) { @@ -444,8 +444,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } if (quickLinks != null && quickLinks.containsKey("Metrics")) { String metricsUrl = quickLinks.get("Metrics"); - app.setMetrics( - sliderAppClient.getGangliaMetrics(metricsUrl, gangliaMetrics, null, viewContext, matchedAppType)); + app.setMetrics(sliderAppClient.getGangliaMetrics(metricsUrl, + gangliaMetrics, null, viewContext, matchedAppType)); } } } @@ -453,9 +453,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } /** - * Creates a new {@link SliderClient} initialized with appropriate configuration. If configuration was not determined, - * <code>null</code> is returned. - * + * Creates a new {@link SliderClient} initialized with appropriate + * configuration. If configuration was not determined, <code>null</code> is + * returned. + * * @return */ protected SliderClient getSliderClient() { @@ -472,7 +473,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { super.serviceInit(conf); // Override the default FS client to set the super user. FileSystem fs = FileSystem.get(FileSystem.getDefaultUri(getConfig()), - getConfig(), "yarn"); + getConfig(), "yarn"); SliderFileSystem fileSystem = new SliderFileSystem(fs, getConfig()); Field fsField = SliderClient.class .getDeclaredField("sliderFileSystem"); @@ -482,7 +483,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { }; try { sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, - new String[]{"usage"}); + new String[] { "usage" }); } catch (Exception e) { logger.warn("Unable to set SliderClient configs", e); throw new RuntimeException(e.getMessage(), e); @@ -495,15 +496,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } /** - * Dynamically determines Slider client configuration. If unable to determine, <code>null</code> is returned. - * + * Dynamically determines Slider client configuration. If unable to determine, + * <code>null</code> is returned. + * * @return */ private Configuration getSliderClientConfiguration() { AmbariCluster ambariCluster = getAmbariCluster(); if (ambariCluster != null) { AmbariService zkService = ambariClient.getService(ambariCluster, - "ZOOKEEPER"); + "ZOOKEEPER"); if (zkService != null && ambariCluster.getDesiredConfigs() != null && ambariCluster.getDesiredConfigs().containsKey("global") && ambariCluster.getDesiredConfigs().containsKey("yarn-site") @@ -538,7 +540,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { yarnConfig.set("yarn.log-aggregation-enable", "true"); yarnConfig.set("yarn.resourcemanager.address", rmAddress); yarnConfig.set("yarn.resourcemanager.scheduler.address", - rmSchedulerAddress); + rmSchedulerAddress); yarnConfig.set("fs.defaultFS", hdfsPath); yarnConfig.set("slider.zookeeper.quorum", zkQuorum.toString()); yarnConfig @@ -564,11 +566,17 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { List<ApplicationReport> yarnApps = sliderClient.listSliderInstances(null); for (ApplicationReport yarnApp : yarnApps) { SliderApp sliderAppObject = createSliderAppObject(yarnApp, properties, - sliderClient); + sliderClient); if (sliderAppObject != null) { if (sliderAppsMap.containsKey(sliderAppObject.getName())) { - if (sliderAppsMap.get(sliderAppObject.getName()).getId() - .compareTo(sliderAppObject.getId()) < 0) { + String mapId = sliderAppsMap.get(sliderAppObject.getName()).getId(); + String newId = sliderAppObject.getId(); + ApplicationId mapIdObj = getApplicationId(mapId); + ApplicationId newIdObj = getApplicationId(newId); + if (mapIdObj.getClusterTimestamp() < newIdObj.getClusterTimestamp() + || ((mapIdObj.getClusterTimestamp() == newIdObj + .getClusterTimestamp()) && mapIdObj.getId() < newIdObj + .getId())) { sliderAppsMap.put(sliderAppObject.getName(), sliderAppObject); } } else { @@ -641,7 +649,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { try { ZipFile zipFile = new ZipFile(appZip); Metainfo metainfo = new MetainfoParser().parse(zipFile - .getInputStream(zipFile.getEntry("metainfo.xml"))); + .getInputStream(zipFile.getEntry("metainfo.xml"))); // Create app type object if (metainfo.getServices() != null && metainfo.getServices().size() > 0) { @@ -685,11 +693,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { // appTypeComponent.setPriority(component.); if (component.getMinInstanceCount() != null) { appTypeComponent.setInstanceCount(Integer.parseInt(component - .getMinInstanceCount())); + .getMinInstanceCount())); } if (component.getMaxInstanceCount() != null) { appTypeComponent.setMaxInstanceCount(Integer - .parseInt(component.getMaxInstanceCount())); + .parseInt(component.getMaxInstanceCount())); } if (resourcesJson != null) { JsonElement componentJson = resourcesJson.getAsJsonObject() @@ -697,10 +705,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { .get(component.getName()); if (componentJson != null && componentJson.getAsJsonObject().has( - "yarn.role.priority")) { + "yarn.role.priority")) { appTypeComponent.setPriority(Integer.parseInt(componentJson - .getAsJsonObject().get("yarn.role.priority") - .getAsString())); + .getAsJsonObject().get("yarn.role.priority") + .getAsString())); } } appTypeComponent.setCategory(component.getCategory()); @@ -709,8 +717,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appType.setJmxMetrics(readMetrics(zipFile, "jmx_metrics.json")); appType.setGangliaMetrics(readMetrics(zipFile, - "ganglia_metrics.json")); - appType.setSupportedMetrics(getSupportedMetrics(appType.getGangliaMetrics())); + "ganglia_metrics.json")); + appType.setSupportedMetrics(getSupportedMetrics(appType + .getGangliaMetrics())); appType.setTypeComponents(appTypeComponentList); appTypes.add(appType); @@ -726,10 +735,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { return appTypes; } - private List<String> getSupportedMetrics(Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) { + private List<String> getSupportedMetrics( + Map<String, Map<String, Map<String, Metric>>> gangliaMetrics) { Set<String> supportedMetrics = new HashSet<String>(); if (gangliaMetrics != null && gangliaMetrics.size() > 0) { - for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics.values()) { + for (Map<String, Map<String, Metric>> compMetrics : gangliaMetrics + .values()) { for (Map<String, Metric> metrics : compMetrics.values()) { supportedMetrics.addAll(metrics.keySet()); } @@ -739,16 +750,16 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } Map<String, Map<String, Map<String, Metric>>> readMetrics(ZipFile zipFile, - String fileName) { + String fileName) { Map<String, Map<String, Map<String, Metric>>> metrics = null; try { InputStream inputStream = zipFile.getInputStream(zipFile - .getEntry(fileName)); + .getEntry(fileName)); ObjectMapper mapper = new ObjectMapper(); metrics = mapper.readValue(inputStream, - new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() { - }); + new TypeReference<Map<String, Map<String, Map<String, Metric>>>>() { + }); } catch (IOException e) { logger.info("Error reading metrics. " + e.getMessage()); } @@ -788,7 +799,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { appCount = ++createAppCounter; } File appCreateFolder = new File(appsCreateFolder, - Integer.toString(appCount)); + Integer.toString(appCount)); appCreateFolder.mkdirs(); File appConfigJsonFile = new File(appCreateFolder, "appConfig.json"); File resourcesJsonFile = new File(appCreateFolder, "resources.json"); @@ -799,20 +810,20 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { AmbariCluster cluster = ambariClient.getCluster(clusterInfo); Map<String, String> coreSiteConfigs = ambariClient.getConfiguration( clusterInfo, "core-site", cluster.getDesiredConfigs() - .get("core-site")); + .get("core-site")); String hdfsLocation = coreSiteConfigs.get("fs.defaultFS"); final ActionCreateArgs createArgs = new ActionCreateArgs(); createArgs.template = appConfigJsonFile; createArgs.resources = resourcesJsonFile; createArgs.image = new Path(hdfsLocation - + "/slider/agent/slider-agent.tar.gz"); + + "/slider/agent/slider-agent.tar.gz"); ClassLoader currentClassLoader = Thread.currentThread() .getContextClassLoader(); Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); try { ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); sliderClient.actionCreate(appName, createArgs); @@ -830,10 +841,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } private void saveAppResources(JsonArray componentsArray, - File resourcesJsonFile) throws IOException { + File resourcesJsonFile) throws IOException { JsonObject resourcesObj = new JsonObject(); resourcesObj.addProperty("schema", - "http://example.org/specification/v2.0.0";); + "http://example.org/specification/v2.0.0";); resourcesObj.add("metadata", new JsonObject()); resourcesObj.add("global", new JsonObject()); JsonObject componentsObj = new JsonObject(); @@ -843,11 +854,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { if (inputComponent.has("id")) { JsonObject componentValue = new JsonObject(); componentValue.addProperty("yarn.role.priority", - inputComponent.get("priority").getAsString()); + inputComponent.get("priority").getAsString()); componentValue.addProperty("yarn.component.instances", inputComponent .get("instanceCount").getAsString()); componentsObj.add(inputComponent.get("id").getAsString(), - componentValue); + componentValue); } } } @@ -865,7 +876,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } private void saveAppConfigs(JsonObject configs, JsonArray componentsArray, - File appConfigJsonFile) throws IOException { + File appConfigJsonFile) throws IOException { JsonObject appConfigs = new JsonObject(); appConfigs.addProperty("schema", "http://example.org/specification/v2.0.0";); appConfigs.add("metadata", new JsonObject()); @@ -876,7 +887,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { JsonObject inputComponent = componentsArray.get(i).getAsJsonObject(); if (inputComponent.has("id")) { componentsObj.add(inputComponent.get("id").getAsString(), - new JsonObject()); + new JsonObject()); } } } @@ -909,7 +920,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); ActionFreezeArgs freezeArgs = new ActionFreezeArgs(); @@ -938,7 +949,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { throw new ApplicationNotFoundException(appId); } ApplicationId applicationId = UserGroupInformation.getBestUGI(null, - "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { + "yarn").doAs(new PrivilegedExceptionAction<ApplicationId>() { public ApplicationId run() throws IOException, YarnException { SliderClient sliderClient = getSliderClient(); ActionThawArgs thawArgs = new ActionThawArgs();
