Updated Branches: refs/heads/trunk 0b7be0d3a -> ae25391f8
AMBARI-2750. Perf: Start all services API call takes 18s. (Myroslav via mahadev) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/ae25391f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/ae25391f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/ae25391f Branch: refs/heads/trunk Commit: ae25391f89f1f95eaede4c3ae38d8e7723fe18ea Parents: 0b7be0d Author: Mahadev Konar <[email protected]> Authored: Sun Jul 28 19:07:37 2013 -0700 Committer: Mahadev Konar <[email protected]> Committed: Sun Jul 28 19:07:37 2013 -0700 ---------------------------------------------------------------------- .../AmbariManagementControllerImpl.java | 147 ++++++++++++++----- .../server/orm/dao/HostConfigMappingDAO.java | 48 +++++- .../server/orm/dao/HostRoleCommandDAO.java | 3 + .../orm/entities/HostConfigMappingEntity.java | 32 ++++ .../org/apache/ambari/server/state/Cluster.java | 13 ++ .../server/state/cluster/ClusterImpl.java | 77 +++++++--- .../server/state/cluster/ClustersImpl.java | 10 +- .../apache/ambari/server/utils/StageUtils.java | 3 +- .../src/main/resources/META-INF/persistence.xml | 1 + .../apache/ambari/server/orm/TestOrmImpl.java | 10 +- .../server/state/cluster/ClusterTest.java | 50 ++++--- 11 files changed, 313 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 1e2e950..495cb13 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -949,12 +949,13 @@ public class AmbariManagementControllerImpl implements } private void createHostAction(Cluster cluster, - Stage stage, ServiceComponentHost scHost, - Map<String, Map<String, String>> configurations, - Map<String, Map<String, String>> configTags, - RoleCommand command, - Map<String, String> commandParams, - ServiceComponentHostEvent event) throws AmbariException { + Stage stage, ServiceComponentHost scHost, + Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> configTags, + RoleCommand command, + Map<String, String> commandParams, + ServiceComponentHostEvent event, + Map<String, List<String>> clusterHostInfo) throws AmbariException { stage.addHostRoleExecutionCommand(scHost.getHostName(), Role.valueOf(scHost .getServiceComponentName()), command, @@ -964,16 +965,14 @@ public class AmbariManagementControllerImpl implements scHost.getServiceComponentName()).getExecutionCommand(); // Generate cluster host info - execCmd.setClusterHostInfo( - StageUtils.getClusterHostInfo( - clusters.getHostsForCluster(cluster.getClusterName()), cluster, hostsMap, injector)); + execCmd.setClusterHostInfo(clusterHostInfo); Host host = clusters.getHost(scHost.getHostName()); // Hack - Remove passwords from configs if (event.getServiceComponentName().equals(Role.HIVE_CLIENT.toString())) { Map<String, String> hiveConfigs = configurations.get(Configuration - .HIVE_CONFIG_TAG); + .HIVE_CONFIG_TAG); if (hiveConfigs != null) { hiveConfigs.remove(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY); } @@ -1014,18 +1013,32 @@ public class AmbariManagementControllerImpl implements params.put("mysql_jdbc_url" , this.mysqljdbcUrl); params.put("oracle_jdbc_url", this.ojdbcUrl); if (configs.getServerDBName().equalsIgnoreCase(Configuration - .ORACLE_DB_NAME)) { + .ORACLE_DB_NAME)) { params.put("db_driver_filename", configs.getOjdbcJarName()); } else if (configs.getServerDBName().equalsIgnoreCase(Configuration - .MYSQL_DB_NAME)) { + .MYSQL_DB_NAME)) { params.put("db_driver_filename", configs.getMySQLJarName()); } execCmd.setHostLevelParams(params); Map<String, String> roleParams = new TreeMap<String, String>(); execCmd.setRoleParams(roleParams); + } + + + private void createHostAction(Cluster cluster, + Stage stage, ServiceComponentHost scHost, + Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> configTags, + RoleCommand command, + Map<String, String> commandParams, + ServiceComponentHostEvent event) throws AmbariException { + + Map<String, List<String>> clusterHostInfo = StageUtils.getClusterHostInfo( + clusters.getHostsForCluster(cluster.getClusterName()), cluster, hostsMap, injector); - return; + createHostAction(cluster, stage, scHost, configurations, configTags, + command, commandParams, event, clusterHostInfo); } private synchronized Set<ClusterResponse> getClusters(ClusterRequest request) @@ -1972,15 +1985,18 @@ public class AmbariManagementControllerImpl implements } private void findConfigurationPropertiesWithOverrides( - Map<String, Map<String, String>> configurations, - Map<String, Map<String, String>> configTags, - Cluster cluster, String serviceName, String hostName) throws AmbariException { + Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> configTags, + Cluster cluster, String serviceName, String hostName, + Map<String, DesiredConfig> clusterDesiredConfigs, + Map<String, DesiredConfig> desiredConfigMap) throws AmbariException { + // Do not use host component config mappings. Instead, the rules are: // 1) Use the cluster desired config // 2) override (1) with service-specific overrides // 3) override (2) with host-specific overrides - for (Entry<String, DesiredConfig> entry : cluster.getDesiredConfigs().entrySet()) { + for (Entry<String, DesiredConfig> entry : clusterDesiredConfigs.entrySet()) { String type = entry.getKey(); String tag = entry.getValue().getVersion(); // 1) start with cluster config @@ -2001,8 +2017,8 @@ public class AmbariManagementControllerImpl implements } // 3) apply the host overrides, if any - Host host = clusters.getHost(hostName); - DesiredConfig dc = host.getDesiredConfigs(cluster.getClusterId()).get(type); + DesiredConfig dc = desiredConfigMap.get(type); + if (null != dc) { Config hostConfig = cluster.getConfig(type, dc.getVersion()); if (null != hostConfig) { @@ -2010,11 +2026,13 @@ public class AmbariManagementControllerImpl implements tags.put("host_override_tag", hostConfig.getVersionTag()); } } - + configurations.put(type, props); configTags.put(type, tags); } - + + + // HACK HACK HACK if the service has configs that are NOT included // in cluster-level, then use them anyway. THIS IS GENERALLY A BAD // IDEA, but is included for backward compatability. Do not check host @@ -2025,13 +2043,37 @@ public class AmbariManagementControllerImpl implements String type = c.getType(); if (!configurations.containsKey(type)) { configurations.put(type, new HashMap<String,String>(c.getProperties())); - + HashMap<String,String> tags = new HashMap<String,String>(); tags.put("tag", c.getVersionTag()); configTags.put(type, tags); } } - + } + + private void findConfigurationPropertiesWithOverrides( + Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> configTags, + Cluster cluster, String serviceName, String hostName, + Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException { + + Host host = clusters.getHost(hostName); + Map<String, DesiredConfig> desiredConfigMap = host.getDesiredConfigs(cluster.getClusterId()); + + findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, + serviceName, hostName, clusterDesiredConfigs, desiredConfigMap); + } + + private void findConfigurationPropertiesWithOverrides( + Map<String, Map<String, String>> configurations, + Map<String, Map<String, String>> configTags, + Cluster cluster, String serviceName, String hostName) throws AmbariException { + + Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs(); + + findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, + serviceName, hostName, clusterDesiredConfigs); + } private List<Stage> doStageCreation(Cluster cluster, @@ -2042,6 +2084,8 @@ public class AmbariManagementControllerImpl implements boolean runSmokeTest, boolean reconfigureClients) throws AmbariException { + Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs(); + // TODO handle different transitions? // Say HDFS to stopped and MR to started, what order should actions be done // in? @@ -2078,8 +2122,22 @@ public class AmbariManagementControllerImpl implements long stageId = 0; Stage stage = createNewStage(cluster, requestId.longValue(), requestContext); stage.setStageId(stageId); + + Set<String> hostnames = new HashSet<String>(); + for (Map<State, List<ServiceComponentHost>> stateListMap : changedScHosts.values()) { + for (List<ServiceComponentHost> serviceComponentHosts : stateListMap.values()) { + for (ServiceComponentHost serviceComponentHost : serviceComponentHosts) { + hostnames.add(serviceComponentHost.getHostName()); + } + } + } + + Map<String, Map<String, DesiredConfig>> configsByHosts = cluster.getHostsDesiredConfigs(hostnames); + //HACK String jobtrackerHost = this.getJobTrackerHost(cluster); + Map<String, List<String>> clusterHostInfo = StageUtils.getClusterHostInfo( + clusters.getHostsForCluster(cluster.getClusterName()), cluster, hostsMap, injector); for (String compName : changedScHosts.keySet()) { for (State newState : changedScHosts.get(compName).keySet()) { for (ServiceComponentHost scHost : @@ -2213,8 +2271,8 @@ public class AmbariManagementControllerImpl implements Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>(); Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>(); - findConfigurationPropertiesWithOverrides(configurations, configTags, - cluster, scHost.getServiceName(), scHost.getHostName()); + findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, scHost.getServiceName(), + scHost.getHostName(), clusterDesiredConfigs, configsByHosts.get(scHost.getHostName())); // HACK HACK HACK if ((!scHost.getHostName().equals(jobtrackerHost)) @@ -2227,7 +2285,7 @@ public class AmbariManagementControllerImpl implements } createHostAction(cluster, stage, scHost, configurations, configTags, - roleCommand, requestParameters, event); + roleCommand, requestParameters, event, clusterHostInfo); } } } @@ -2261,7 +2319,7 @@ public class AmbariManagementControllerImpl implements Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>(); findConfigurationPropertiesWithOverrides(configurations, configTags, - cluster, serviceName, clientHost); + cluster, serviceName, clientHost, clusterDesiredConfigs); stage.getExecutionCommandWrapper(clientHost, smokeTestRole).getExecutionCommand() @@ -2307,11 +2365,20 @@ public class AmbariManagementControllerImpl implements } } - @Transactional void updateServiceStates( Map<State, List<Service>> changedServices, Map<State, List<ServiceComponent>> changedComps, Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts) { + updateServiceStates(changedServices, changedComps, changedScHosts, null); + } + + @Transactional + void updateServiceStates( + Map<State, List<Service>> changedServices, + Map<State, List<ServiceComponent>> changedComps, + Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts, + Collection<ServiceComponentHost> ignoredScHosts + ) { if (changedServices != null) { for (Entry<State, List<Service>> entry : changedServices.entrySet()) { State newState = entry.getKey(); @@ -2345,6 +2412,12 @@ public class AmbariManagementControllerImpl implements } } } + + if (ignoredScHosts != null) { + for (ServiceComponentHost scHost : ignoredScHosts) { + scHost.setDesiredState(scHost.getState()); + } + } } private boolean isValidStateTransition(State oldState, @@ -2473,6 +2546,8 @@ public class AmbariManagementControllerImpl implements new HashMap<State, List<ServiceComponent>>(); Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts = new HashMap<String, Map<State, List<ServiceComponentHost>>>(); + Collection<ServiceComponentHost> ignoredScHosts = + new ArrayList<ServiceComponentHost>(); Set<String> clusterNames = new HashSet<String>(); Map<String, Set<String>> serviceNames = new HashMap<String, Set<String>>(); @@ -2628,7 +2703,7 @@ public class AmbariManagementControllerImpl implements continue; } if (newState == oldSchState) { - sch.setDesiredState(newState); + ignoredScHosts.add(sch); if (LOG.isDebugEnabled()) { LOG.debug("Ignoring ServiceComponentHost" + ", clusterName=" + request.getClusterName() @@ -2737,7 +2812,7 @@ public class AmbariManagementControllerImpl implements changedScHosts, null, requestProperties.get(REQUEST_CONTEXT_PROPERTY), runSmokeTest, reconfigureClients); persistStages(stages); - updateServiceStates(changedServices, changedComps, changedScHosts); + updateServiceStates(changedServices, changedComps, changedScHosts, ignoredScHosts); if (stages == null || stages.isEmpty()) { return null; } @@ -2759,6 +2834,8 @@ public class AmbariManagementControllerImpl implements new HashMap<State, List<ServiceComponent>>(); Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts = new HashMap<String, Map<State, List<ServiceComponentHost>>>(); + Collection<ServiceComponentHost> ignoredScHosts = + new ArrayList<ServiceComponentHost>(); Set<String> clusterNames = new HashSet<String>(); Map<String, Map<String, Set<String>>> componentNames = @@ -2935,7 +3012,7 @@ public class AmbariManagementControllerImpl implements } if (newState == oldSchState) { - sch.setDesiredState(newState); + ignoredScHosts.add(sch); if (LOG.isDebugEnabled()) { LOG.debug("Ignoring ServiceComponentHost" + ", clusterName=" + request.getClusterName() @@ -3024,7 +3101,7 @@ public class AmbariManagementControllerImpl implements changedComps, changedScHosts, null, requestProperties.get (REQUEST_CONTEXT_PROPERTY), runSmokeTest, false); persistStages(stages); - updateServiceStates(null, changedComps, changedScHosts); + updateServiceStates(null, changedComps, changedScHosts, ignoredScHosts); if (stages == null || stages.isEmpty()) { return null; } @@ -3126,6 +3203,8 @@ public class AmbariManagementControllerImpl implements Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts = new HashMap<String, Map<State, List<ServiceComponentHost>>>(); + Collection<ServiceComponentHost> ignoredScHosts = + new ArrayList<ServiceComponentHost>(); Set<String> clusterNames = new HashSet<String>(); Map<String, Map<String, Map<String, Set<String>>>> hostComponentNames = @@ -3267,7 +3346,7 @@ public class AmbariManagementControllerImpl implements State oldSchState = sch.getState(); // Client component reinstall allowed if (newState == oldSchState && !sc.isClientComponent()) { - sch.setDesiredState(newState); + ignoredScHosts.add(sch); if (LOG.isDebugEnabled()) { LOG.debug("Ignoring ServiceComponentHost" + ", clusterName=" + request.getClusterName() @@ -3410,7 +3489,7 @@ public class AmbariManagementControllerImpl implements List<Stage> stages = doStageCreation(cluster, null, null, changedScHosts, requestParameters, requestProperties.get(REQUEST_CONTEXT_PROPERTY), runSmokeTest, false); persistStages(stages); - updateServiceStates(null, null, changedScHosts); + updateServiceStates(null, null, changedScHosts, ignoredScHosts); if (stages == null || stages.isEmpty()) { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java index 3a59a5e..79fddb4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostConfigMappingDAO.java @@ -17,7 +17,7 @@ */ package org.apache.ambari.server.orm.dao; -import java.util.List; +import java.util.*; import javax.persistence.EntityManager; import javax.persistence.TypedQuery; @@ -47,6 +47,7 @@ public class HostConfigMappingDAO { return entityManagerProvider.get().merge(entity); } + @Transactional public List<HostConfigMappingEntity> findByType(long clusterId, String hostName, String type) { TypedQuery<HostConfigMappingEntity> query = entityManagerProvider.get().createQuery( "SELECT entity FROM HostConfigMappingEntity entity " + @@ -56,6 +57,7 @@ public class HostConfigMappingDAO { return daoUtils.selectList(query, Long.valueOf(clusterId), hostName, type); } + @Transactional public HostConfigMappingEntity findSelectedByType(long clusterId, String hostName, String type) { @@ -69,6 +71,7 @@ public class HostConfigMappingDAO { return daoUtils.selectSingle(query, Long.valueOf(clusterId), hostName, type); } + @Transactional public List<HostConfigMappingEntity> findSelected(long clusterId, String hostName) { TypedQuery<HostConfigMappingEntity> query = entityManagerProvider.get().createQuery( "SELECT entity FROM HostConfigMappingEntity entity " + @@ -79,6 +82,23 @@ public class HostConfigMappingDAO { return daoUtils.selectList(query, Long.valueOf(clusterId), hostName); } + @Transactional + public List<HostConfigMappingEntity> findSelectedByHosts(long clusterId, Collection<String> hostNames) { + + if (hostNames == null || hostNames.isEmpty()) { + return Collections.emptyList(); + } + + TypedQuery<HostConfigMappingEntity> query = entityManagerProvider.get().createQuery( + "SELECT entity FROM HostConfigMappingEntity entity " + + "WHERE entity.clusterId = ?1 AND entity.hostName IN ?2 " + + "AND entity.selected > 0", + HostConfigMappingEntity.class); + + return daoUtils.selectList(query, Long.valueOf(clusterId), hostNames); + } + + @Transactional public List<HostConfigMappingEntity> findSelectedHostsByType(long clusterId, String type) { TypedQuery<HostConfigMappingEntity> query = entityManagerProvider.get().createQuery( @@ -88,6 +108,32 @@ public class HostConfigMappingDAO { return daoUtils.selectList(query, Long.valueOf(clusterId), type); } + @Transactional + public Map<String, List<HostConfigMappingEntity>> findSelectedHostsByTypes(long clusterId, + Collection<String> types) { + TypedQuery<HostConfigMappingEntity> query = entityManagerProvider.get().createQuery( + "SELECT entity FROM HostConfigMappingEntity entity " + + "WHERE entity.clusterId = ?1 AND entity.type IN ?2 AND entity.selected > 0", + HostConfigMappingEntity.class); + + Map<String, List<HostConfigMappingEntity>> mappingsByType = new HashMap<String, List<HostConfigMappingEntity>>(); + + for (String type : types) { + if (!mappingsByType.containsKey(type)) { + mappingsByType.put(type, new ArrayList<HostConfigMappingEntity>()); + } + } + + if (!types.isEmpty()) { + List<HostConfigMappingEntity> mappings = daoUtils.selectList(query, clusterId, types); + for (HostConfigMappingEntity mapping : mappings) { + mappingsByType.get(mapping.getType()).add(mapping); + } + } + + return mappingsByType; + } + /** * @param clusterId * @param hostName http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index a1a4c2f..59a7a5f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@ -110,6 +110,9 @@ public class HostRoleCommandDAO { } @Transactional + /** + * NB: You cannot rely on return value if batch write is enabled + */ public int updateStatusByRequestId(long requestId, HostRoleStatus target, Collection<HostRoleStatus> sources) { Query query = entityManagerProvider.get().createQuery("UPDATE HostRoleCommandEntity command " + "SET command.status=?1 " + http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostConfigMappingEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostConfigMappingEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostConfigMappingEntity.java index 6c91eab..1411a67 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostConfigMappingEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostConfigMappingEntity.java @@ -129,4 +129,36 @@ public class HostConfigMappingEntity { user = userName; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HostConfigMappingEntity that = (HostConfigMappingEntity) o; + + if (selected != that.selected) return false; + if (clusterId != null ? !clusterId.equals(that.clusterId) : that.clusterId != null) return false; + if (createTimestamp != null ? !createTimestamp.equals(that.createTimestamp) : that.createTimestamp != null) + return false; + if (hostName != null ? !hostName.equals(that.hostName) : that.hostName != null) return false; + if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) return false; + if (type != null ? !type.equals(that.type) : that.type != null) return false; + if (user != null ? !user.equals(that.user) : that.user != null) return false; + if (versionTag != null ? !versionTag.equals(that.versionTag) : that.versionTag != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = clusterId != null ? clusterId.hashCode() : 0; + result = 31 * result + (hostName != null ? hostName.hashCode() : 0); + result = 31 * result + (type != null ? type.hashCode() : 0); + result = 31 * result + (createTimestamp != null ? createTimestamp.hashCode() : 0); + result = 31 * result + (versionTag != null ? versionTag.hashCode() : 0); + result = 31 * result + (serviceName != null ? serviceName.hashCode() : 0); + result = 31 * result + selected; + result = 31 * result + (user != null ? user.hashCode() : 0); + return result; + } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index 3321e48..fc97032 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -212,4 +212,17 @@ public interface Cluster { * @return cluster-global lock */ ReadWriteLock getClusterGlobalLock(); + + /** + * Fetch desired configs for list of hosts in cluster + * @param hostnames + * @return + */ + Map<String, Map<String, DesiredConfig>> getHostsDesiredConfigs(Collection<String> hostnames); + + /** + * Fetch desired configs for all hosts in cluster + * @return + */ + Map<String, Map<String, DesiredConfig>> getAllHostsDesiredConfigs(); } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 4e4e666..6ee0d9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -18,16 +18,8 @@ package org.apache.ambari.server.state.cluster; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -1027,6 +1019,7 @@ public class ClusterImpl implements Cluster { readWriteLock.readLock().lock(); try { Map<String, DesiredConfig> map = new HashMap<String, DesiredConfig>(); + Collection<String> types = new HashSet<String>(); for (ClusterConfigMappingEntity e : clusterEntity.getConfigMappingEntities()) { if (e.isSelected() > 0) { @@ -1035,20 +1028,22 @@ public class ClusterImpl implements Cluster { c.setVersion(e.getVersion()); c.setUser(e.getUser()); - List<HostConfigMappingEntity> hostMappings = - hostConfigMappingDAO.findSelectedHostsByType(clusterEntity.getClusterId(), - e.getType()); + map.put(e.getType(), c); + types.add(e.getType()); + } + } + + if (!map.isEmpty()) { + Map<String, List<HostConfigMappingEntity>> hostMappingsByType = + hostConfigMappingDAO.findSelectedHostsByTypes(clusterEntity.getClusterId(), types); - List<DesiredConfig.HostOverride> hosts = new ArrayList<DesiredConfig.HostOverride>(); - for (HostConfigMappingEntity mappingEntity : hostMappings) { - hosts.add(new DesiredConfig.HostOverride(mappingEntity.getHostName(), + for (Entry<String, DesiredConfig> entry : map.entrySet()) { + List<DesiredConfig.HostOverride> hostOverrides = new ArrayList<DesiredConfig.HostOverride>(); + for (HostConfigMappingEntity mappingEntity : hostMappingsByType.get(entry.getKey())) { + hostOverrides.add(new DesiredConfig.HostOverride(mappingEntity.getHostName(), mappingEntity.getVersion())); } - - c.setHostOverrides(hosts); - - map.put(e.getType(), c); - + entry.getValue().setHostOverrides(hostOverrides); } } @@ -1084,4 +1079,46 @@ public class ClusterImpl implements Cluster { } } + + @Override + public Map<String, Map<String, DesiredConfig>> getHostsDesiredConfigs(Collection<String> hostnames) { + + if (hostnames == null || hostnames.isEmpty()) { + return Collections.emptyMap(); + } + + List<HostConfigMappingEntity> mappingEntities = + hostConfigMappingDAO.findSelectedByHosts(clusterEntity.getClusterId(), hostnames); + + Map<String, Map<String, DesiredConfig>> desiredConfigsByHost = new HashMap<String, Map<String, DesiredConfig>>(); + + for (String hostname : hostnames) { + desiredConfigsByHost.put(hostname, new HashMap<String, DesiredConfig>()); + } + + for (HostConfigMappingEntity mappingEntity : mappingEntities) { + DesiredConfig desiredConfig = new DesiredConfig(); + desiredConfig.setVersion(mappingEntity.getVersion()); + desiredConfig.setServiceName(mappingEntity.getServiceName()); + desiredConfig.setUser(mappingEntity.getUser()); + + desiredConfigsByHost.get(mappingEntity.getHostName()).put(mappingEntity.getType(), desiredConfig); + } + + return desiredConfigsByHost; + } + + @Override + public Map<String, Map<String, DesiredConfig>> getAllHostsDesiredConfigs() { + + Collection<String> hostnames; + try { + hostnames = clusters.getHostsForCluster(clusterEntity.getClusterName()).keySet(); + } catch (AmbariException ignored) { + return Collections.emptyMap(); + } + + return getHostsDesiredConfigs(hostnames); + } + } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java index 444bf35..246d84d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java @@ -261,9 +261,10 @@ public class ClustersImpl implements Clusters { } @Override - @Transactional public Host getHost(String hostname) throws AmbariException { - loadClustersAndHosts(); + if (!clustersLoaded) { + loadClustersAndHosts(); + } r.lock(); try { if (!hosts.containsKey(hostname)) { @@ -541,10 +542,11 @@ public class ClustersImpl implements Clusters { } @Override - @Transactional public Map<String, Host> getHostsForCluster(String clusterName) throws AmbariException { - loadClustersAndHosts(); + if (!clustersLoaded) { + loadClustersAndHosts(); + } r.lock(); try { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 390529f..c4e9dcf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -193,6 +193,7 @@ public class StageUtils { Injector injector) throws AmbariException { Map<String, List<String>> info = new HashMap<String, List<String>>(); if (cluster.getServices() != null) { + String hostName = getHostName(); for (String serviceName : cluster.getServices().keySet()) { if (cluster.getServices().get(serviceName) != null) { for (String componentName : cluster.getServices().get(serviceName) @@ -218,7 +219,7 @@ public class StageUtils { Configuration configuration = injector.getInstance(Configuration.class); String url = configuration.getRcaDatabaseUrl(); if (url.contains(Configuration.HOSTNAME_MACRO)) { - url = url.replace(Configuration.HOSTNAME_MACRO, hostsMap.getHostMap(getHostName())); + url = url.replace(Configuration.HOSTNAME_MACRO, hostsMap.getHostMap(hostName)); } info.put("ambari_db_rca_url", Arrays.asList(url)); info.put("ambari_db_rca_driver", Arrays.asList(configuration.getRcaDatabaseDriver())); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/META-INF/persistence.xml b/ambari-server/src/main/resources/META-INF/persistence.xml index 13ac9c8..8e584c1 100644 --- a/ambari-server/src/main/resources/META-INF/persistence.xml +++ b/ambari-server/src/main/resources/META-INF/persistence.xml @@ -43,6 +43,7 @@ <!--<property name="javax.persistence.jdbc.url" value="jdbc:postgresql://localhost/ambari" />--> <!--<property name="javax.persistence.jdbc.driver" value="org.postgresql.Driver" />--> <property name="eclipselink.cache.size.default" value="3000" /> + <property name="eclipselink.jdbc.batch-writing" value="JDBC"/> </properties> </persistence-unit> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java index b7e2f19..586478d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/TestOrmImpl.java @@ -188,7 +188,15 @@ public class TestOrmImpl extends Assert { injector.getInstance(OrmTestHelper.class).createStageCommands(); HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class); int result = hostRoleCommandDAO.updateStatusByRequestId(0L, HostRoleStatus.ABORTED, Arrays.asList(HostRoleStatus.QUEUED, HostRoleStatus.IN_PROGRESS, HostRoleStatus.PENDING)); - assertEquals(2, result); + //result always 1 in batch mode + List<HostRoleCommandEntity> commandEntities = hostRoleCommandDAO.findByRequest(0L); + int count = 0; + for (HostRoleCommandEntity commandEntity : commandEntities) { + if (commandEntity.getStatus() == HostRoleStatus.ABORTED) { + count++; + } + } + assertEquals("Exactly two commands should be in aborted state", 2, count); } @Test http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/ae25391f/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index 2bdbb12..548ee45 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.state.cluster; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,27 +44,10 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.ClusterResponse; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.entities.ClusterEntity; -import org.apache.ambari.server.orm.entities.ClusterServiceEntity; -import org.apache.ambari.server.orm.entities.HostEntity; -import org.apache.ambari.server.orm.entities.HostStateEntity; -import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity; -import org.apache.ambari.server.state.AgentVersion; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigFactory; -import org.apache.ambari.server.state.DesiredConfig; +import org.apache.ambari.server.orm.dao.HostConfigMappingDAO; +import org.apache.ambari.server.orm.entities.*; +import org.apache.ambari.server.state.*; import org.apache.ambari.server.state.DesiredConfig.HostOverride; -import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.HostState; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentFactory; -import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceComponentHostFactory; -import org.apache.ambari.server.state.ServiceFactory; -import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.apache.ambari.server.state.host.HostHealthyHeartbeatEvent; import org.apache.ambari.server.state.host.HostRegistrationRequestEvent; @@ -475,4 +459,30 @@ public class ClusterTest { assertEquals(1, injector.getProvider(EntityManager.class).get(). createQuery("SELECT service FROM ClusterServiceEntity service").getResultList().size()); } + + @Test + public void testGetHostsDesiredConfigs() throws Exception { + Host host1 = clusters.getHost("h1"); + + Config config = configFactory.createNew(c1, "hdfs-site", new HashMap<String, String>(){{ + put("test", "test"); + }}); + config.setVersionTag("1"); + + host1.addDesiredConfig(c1.getClusterId(), true, "test", config); + + Map<String, Map<String, DesiredConfig>> configs = c1.getAllHostsDesiredConfigs(); + + assertTrue(configs.containsKey("h1")); + assertEquals(1, configs.get("h1").size()); + + List<String> hostnames = new ArrayList<String>(); + hostnames.add("h1"); + + configs = c1.getHostsDesiredConfigs(hostnames); + + assertTrue(configs.containsKey("h1")); + assertEquals(1, configs.get("h1").size()); + + } }
