Repository: ambari Updated Branches: refs/heads/branch-2.5 6a88e44a7 -> b0d55af42
AMBARI-19755. Part2. Perf: start/stop all actions works much slower after few days of testing.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b0d55af4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b0d55af4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b0d55af4 Branch: refs/heads/branch-2.5 Commit: b0d55af42939195c3c7e842c9bf2d381500967c3 Parents: 6a88e44 Author: Vitaly Brodetskyi <[email protected]> Authored: Tue Jan 31 19:03:44 2017 +0200 Committer: Vitaly Brodetskyi <[email protected]> Committed: Tue Jan 31 19:03:44 2017 +0200 ---------------------------------------------------------------------- .../ambari/server/agent/HeartbeatMonitor.java | 2 +- .../AmbariCustomCommandExecutionHelper.java | 21 +++++++----- .../AmbariManagementControllerImpl.java | 36 +++++++++++++------- .../ambari/server/state/ConfigHelper.java | 16 +++++++-- .../org/apache/ambari/server/state/Host.java | 5 +++ .../server/state/cluster/ClusterImpl.java | 6 ++-- .../ambari/server/state/host/HostImpl.java | 15 ++++++-- .../svccomphost/ServiceComponentHostImpl.java | 25 ++++++++++---- 8 files changed, 90 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java index c53a73d..0042f53 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java @@ -258,7 +258,7 @@ public class HeartbeatMonitor implements Runnable { Collection<Config> clusterConfigs = cluster.getAllConfigs(); // creating list with desired config types to validate if cluster config actual - Set<String> desiredConfigTypes = cluster.getDesiredConfigs().keySet(); + Set<String> desiredConfigTypes = desiredConfigs.keySet(); // Apply global properties for this host from all config groups Map<String, Map<String, String>> allConfigTags = configHelper http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 93f4a8f..b601893 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -1148,28 +1148,33 @@ public class AmbariCustomCommandExecutionHelper { */ public String getRepoInfo(Cluster cluster, Host host) throws AmbariException { + return getRepoInfo(cluster, host.getOsType(), host.getOsFamily(), host.getHostName()); + } + + public String getRepoInfo(Cluster cluster, String hostOSType, String hostOSFamily, String hostName) throws AmbariException { + StackId stackId = cluster.getDesiredStackVersion(); Map<String, List<RepositoryInfo>> repos = ambariMetaInfo.getRepository( - stackId.getStackName(), stackId.getStackVersion()); + stackId.getStackName(), stackId.getStackVersion()); - String family = os_family.find(host.getOsType()); + String family = os_family.find(hostOSType); if (null == family) { - family = host.getOsFamily(); + family = hostOSFamily; } JsonElement gsonList = null; // !!! check for the most specific first - if (repos.containsKey(host.getOsType())) { - gsonList = gson.toJsonTree(repos.get(host.getOsType())); + if (repos.containsKey(hostOSType)) { + gsonList = gson.toJsonTree(repos.get(hostOSType)); } else if (null != family && repos.containsKey(family)) { gsonList = gson.toJsonTree(repos.get(family)); } else { LOG.warn("Could not retrieve repo information for host" - + ", hostname=" + host.getHostName() - + ", clusterName=" + cluster.getClusterName() - + ", stackInfo=" + stackId.getStackId()); + + ", hostname=" + hostName + + ", clusterName=" + cluster.getClusterName() + + ", stackInfo=" + stackId.getStackId()); } if (null != gsonList) { http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index e991e1d..c4df65f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -119,6 +119,7 @@ import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; import org.apache.ambari.server.orm.entities.ExtensionEntity; import org.apache.ambari.server.orm.entities.ExtensionLinkEntity; +import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.OperatingSystemEntity; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; @@ -228,6 +229,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle */ private static final String REQUEST_CONTEXT_PROPERTY = "context"; + private static final Type hostAttributesType = + new TypeToken<Map<String, String>>() {}.getType(); + private static final String CLUSTER_PHASE_PROPERTY = "phase"; private static final String CLUSTER_PHASE_INITIAL_INSTALL = "INITIAL_INSTALL"; private static final String CLUSTER_PHASE_INITIAL_START = "INITIAL_START"; @@ -2188,7 +2192,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle ServiceComponentHostEvent event, boolean skipFailure, ClusterVersionEntity effectiveClusterVersion, - boolean isUpgradeSuspended + boolean isUpgradeSuspended, + DatabaseType databaseType ) throws AmbariException { @@ -2199,7 +2204,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String componentName = scHost.getServiceComponentName(); String hostname = scHost.getHostName(); - String osFamily = clusters.getHost(hostname).getOsFamily(); + Host host = clusters.getHost(hostname); + HostEntity hostEntity = host.getHostEntity(); + Map<String, String> hostAttributes = gson.fromJson(hostEntity.getHostAttributes(), hostAttributesType); + String osFamily = host.getOSFamilyFromHostAttributes(hostAttributes); StackId stackId = cluster.getDesiredStackVersion(); ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); @@ -2208,12 +2216,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle serviceName, componentName); StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); + Map<String, ServiceInfo> servicesMap = ambariMetaInfo.getServices(stackInfo.getName(), stackInfo.getVersion()); ExecutionCommand execCmd = stage.getExecutionCommandWrapper(scHost.getHostName(), scHost.getServiceComponentName()).getExecutionCommand(); - Host host = clusters.getHost(scHost.getHostName()); - execCmd.setConfigurations(configurations); execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); @@ -2240,7 +2247,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle // Propagate HCFS service type info for (Service service : cluster.getServices().values()) { - ServiceInfo serviceInfoInstance = ambariMetaInfo.getService(stackId.getStackName(),stackId.getStackVersion(), service.getName()); + ServiceInfo serviceInfoInstance = servicesMap.get(service.getName()); LOG.debug("Iterating service type Instance in createHostAction: {}", serviceInfoInstance.getName()); String serviceType = serviceInfoInstance.getServiceType(); if (serviceType != null) { @@ -2344,7 +2351,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); } - String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, host); + String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, hostEntity.getOsType(), osFamily , hostname); if (LOG.isDebugEnabled()) { LOG.debug("Sending repo information to agent" + ", hostname=" + scHost.getHostName() @@ -2392,19 +2399,20 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); - Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); + Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stackInfo.getName(), stackInfo.getVersion()); + + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.USER, cluster, desiredConfigs, servicesMap, stackProperties); String userList = gson.toJson(userSet); hostParams.put(USER_LIST, userList); - Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs); + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.GROUP, cluster, desiredConfigs, servicesMap, stackProperties); String groupList = gson.toJson(groupSet); hostParams.put(GROUP_LIST, groupList); - Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs, servicesMap, stackProperties); String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); - DatabaseType databaseType = configs.getDatabaseType(); if (databaseType == DatabaseType.ORACLE) { hostParams.put(DB_DRIVER_FILENAME, configs.getOjdbcJarName()); } else if (databaseType == DatabaseType.MYSQL) { @@ -2567,6 +2575,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle // caching upgrade suspended boolean isUpgradeSuspended = cluster.isUpgradeSuspended(); + // caching database type + DatabaseType databaseType = configs.getDatabaseType(); + // smoke test any service that goes from installed to started Set<String> smokeTestServices = getServicesForSmokeTests(cluster, changedServices, changedScHosts, runSmokeTest); @@ -2899,7 +2910,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle scHost.setState(State.INSTALLED); } else { createHostAction(cluster, stage, scHost, configurations, configurationAttributes, configTags, - roleCommand, requestParameters, event, skipFailure, effectiveClusterVersion, isUpgradeSuspended); + roleCommand, requestParameters, event, skipFailure, effectiveClusterVersion, isUpgradeSuspended, databaseType); } } @@ -3037,8 +3048,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion(); boolean isUpgradeSuspended = cluster.isUpgradeSuspended(); + DatabaseType databaseType = configs.getDatabaseType(); createHostAction(cluster, stage, scHost, configurations, configurationAttributes, configTags, - roleCommand, null, null, false, effectiveClusterVersion, isUpgradeSuspended); + roleCommand, null, null, false, effectiveClusterVersion, isUpgradeSuspended, databaseType); ExecutionCommand ec = stage.getExecutionCommands().get(scHost.getHostName()).get(0).getExecutionCommand(); // createHostAction does not take a hostLevelParams but creates one http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 5ce2a90..95436c5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -537,8 +537,19 @@ public class ConfigHelper { } public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType, - Cluster cluster, Map<String, DesiredConfig> desiredConfigs) throws AmbariException { + Cluster cluster, Map<String, + DesiredConfig> desiredConfigs) throws AmbariException { StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); + Map<String, ServiceInfo> servicesMap = ambariMetaInfo.getServices(stack.getName(), stack.getVersion()); + Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); + + return getPropertyValuesWithPropertyType(propertyType, cluster, desiredConfigs, servicesMap, stackProperties); + } + + public Set<String> getPropertyValuesWithPropertyType(PropertyType propertyType, + Cluster cluster, Map<String, DesiredConfig> desiredConfigs, + Map<String, ServiceInfo> servicesMap, + Set<PropertyInfo> stackProperties) throws AmbariException { Map<String, Config> actualConfigs = new HashMap<>(); Set<String> result = new HashSet<String>(); @@ -549,7 +560,7 @@ public class ConfigHelper { } for (Service service : cluster.getServices().values()) { - Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), service.getName()); + Set<PropertyInfo> serviceProperties = new HashSet<PropertyInfo>(servicesMap.get(service.getName()).getProperties()); for (PropertyInfo serviceProperty : serviceProperties) { if (serviceProperty.getPropertyTypes().contains(propertyType)) { String stackPropertyConfigType = fileNameToConfigType(serviceProperty.getFilename()); @@ -561,7 +572,6 @@ public class ConfigHelper { } } - Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); for (PropertyInfo stackProperty : stackProperties) { if (stackProperty.getPropertyTypes().contains(propertyType)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java index 04b2104..241659a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java @@ -27,11 +27,14 @@ import org.apache.ambari.server.agent.DiskInfo; import org.apache.ambari.server.agent.HostInfo; import org.apache.ambari.server.agent.RecoveryReport; import org.apache.ambari.server.controller.HostResponse; +import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; public interface Host extends Comparable { + HostEntity getHostEntity(); + /** * @return the hostName */ @@ -172,6 +175,8 @@ public interface Host extends Comparable { */ String getOsFamily(); + String getOSFamilyFromHostAttributes(Map<String, String> hostAttributes); + /** * @param osType the osType to set */ http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 5dfa8ec..5da8012 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -2894,13 +2894,13 @@ public class ClusterImpl implements Cluster { Service service = clusterServices.get(serviceName); Map<String, ServiceComponent> components = service.getServiceComponents(); + Map<String, ServiceComponentHost> serviceComponentHosts = components.get(componentName).getServiceComponentHosts(); - if (!components.containsKey(componentName) || - components.get(componentName).getServiceComponentHosts().size() == 0) { + if (!components.containsKey(componentName) || serviceComponentHosts.size() == 0) { return Collections.emptySet(); } - return components.get(componentName).getServiceComponentHosts().keySet(); + return serviceComponentHosts.keySet(); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 3c7b040..328fe22 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@ -747,8 +747,18 @@ public class HostImpl implements Host { @Override public String getOsFamily() { Map<String, String> hostAttributes = getHostAttributes(); - String majorVersion = hostAttributes.get(OS_RELEASE_VERSION).split("\\.")[0]; - return hostAttributes.get(OSFAMILY) + majorVersion; + return getOSFamilyFromHostAttributes(hostAttributes); + } + + @Override + public String getOSFamilyFromHostAttributes(Map<String, String> hostAttributes) { + try { + String majorVersion = hostAttributes.get(OS_RELEASE_VERSION).split("\\.")[0]; + return hostAttributes.get(OSFAMILY) + majorVersion; + } catch(Exception e) { + LOG.error("Error while getting os family from host attributes:", e); + } + return null; } @Override @@ -1145,6 +1155,7 @@ public class HostImpl implements Host { } // Get the cached host entity or load it fresh through the DAO. + @Override public HostEntity getHostEntity() { return hostDAO.findById(hostId); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b0d55af4/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java index 1bcffe4..95de4e8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java @@ -1122,6 +1122,10 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { @Override public StackId getStackVersion() { HostComponentStateEntity schStateEntity = getStateEntity(); + return getStackVersionFromSCHStateEntity(schStateEntity); + } + + private StackId getStackVersionFromSCHStateEntity(HostComponentStateEntity schStateEntity) { if (schStateEntity == null) { return new StackId(); } @@ -1178,13 +1182,17 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { @Override public StackId getDesiredStackVersion() { HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity(); + return getDesiredStackVersionFromHostComponentDesiredStateEntity(desiredStateEntity); + } + + private StackId getDesiredStackVersionFromHostComponentDesiredStateEntity(HostComponentDesiredStateEntity desiredStateEntity) { if (desiredStateEntity != null) { StackEntity desiredStackEntity = desiredStateEntity.getDesiredStack(); return new StackId(desiredStackEntity.getStackName(), desiredStackEntity.getStackVersion()); } else { LOG.warn("Trying to fetch a member from an entity object that may " - + "have been previously deleted, serviceName = " + getServiceName() + ", " - + "componentName = " + getServiceComponentName() + ", " + "hostName = " + getHostName()); + + "have been previously deleted, serviceName = " + getServiceName() + ", " + + "componentName = " + getServiceComponentName() + ", " + "hostName = " + getHostName()); } return null; } @@ -1244,22 +1252,25 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { return null; } + StackId stackVersion = getStackVersionFromSCHStateEntity(hostComponentStateEntity); + HostComponentDesiredStateEntity hostComponentDesiredStateEntity = getDesiredStateEntity(); + String clusterName = serviceComponent.getClusterName(); String serviceName = serviceComponent.getServiceName(); String serviceComponentName = serviceComponent.getName(); String hostName = getHostName(); String publicHostName = getPublicHostName(); String state = getState().toString(); - String stackId = getStackVersion().getStackId(); - String desiredState = getDesiredState().toString(); - String desiredStackId = getDesiredStackVersion().getStackId(); + String stackId = stackVersion.getStackId(); + String desiredState = (hostComponentDesiredStateEntity == null) ? null : hostComponentDesiredStateEntity.getDesiredState().toString(); + String desiredStackId = getDesiredStackVersionFromHostComponentDesiredStateEntity(hostComponentDesiredStateEntity).getStackId(); HostComponentAdminState componentAdminState = getComponentAdminState(); UpgradeState upgradeState = hostComponentStateEntity.getUpgradeState(); String displayName = null; try { - ComponentInfo compInfo = ambariMetaInfo.getComponent(getStackVersion().getStackName(), - getStackVersion().getStackVersion(), serviceName, serviceComponentName); + ComponentInfo compInfo = ambariMetaInfo.getComponent(stackVersion.getStackName(), + stackVersion.getStackVersion(), serviceName, serviceComponentName); displayName = compInfo.getDisplayName(); } catch (AmbariException e) { displayName = serviceComponentName;
