http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ComponentResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ComponentResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ComponentResourceProvider.java index 1d5ee2b..99ecd76 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ComponentResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ComponentResourceProvider.java @@ -87,6 +87,8 @@ public class ComponentResourceProvider extends AbstractControllerResourceProvide protected static final String COMPONENT_TOTAL_COUNT_PROPERTY_ID = "ServiceComponentInfo/total_count"; protected static final String COMPONENT_STARTED_COUNT_PROPERTY_ID = "ServiceComponentInfo/started_count"; protected static final String COMPONENT_INSTALLED_COUNT_PROPERTY_ID = "ServiceComponentInfo/installed_count"; + protected static final String COMPONENT_INSTALLED_AND_MAINTENANCE_OFF_COUNT_PROPERTY_ID + = "ServiceComponentInfo/installed_and_maintenance_off_count"; protected static final String COMPONENT_INIT_COUNT_PROPERTY_ID = "ServiceComponentInfo/init_count"; protected static final String COMPONENT_UNKNOWN_COUNT_PROPERTY_ID = "ServiceComponentInfo/unknown_count"; protected static final String COMPONENT_INSTALL_FAILED_COUNT_PROPERTY_ID = "ServiceComponentInfo/install_failed_count"; @@ -126,6 +128,7 @@ public class ComponentResourceProvider extends AbstractControllerResourceProvide PROPERTY_IDS.add(COMPONENT_TOTAL_COUNT_PROPERTY_ID); PROPERTY_IDS.add(COMPONENT_STARTED_COUNT_PROPERTY_ID); PROPERTY_IDS.add(COMPONENT_INSTALLED_COUNT_PROPERTY_ID); + PROPERTY_IDS.add(COMPONENT_INSTALLED_AND_MAINTENANCE_OFF_COUNT_PROPERTY_ID); PROPERTY_IDS.add(COMPONENT_INIT_COUNT_PROPERTY_ID); PROPERTY_IDS.add(COMPONENT_UNKNOWN_COUNT_PROPERTY_ID); @@ -225,6 +228,7 @@ public class ComponentResourceProvider extends AbstractControllerResourceProvide setResourceProperty(resource, COMPONENT_TOTAL_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("totalCount"), requestedIds); setResourceProperty(resource, COMPONENT_STARTED_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("startedCount"), requestedIds); setResourceProperty(resource, COMPONENT_INSTALLED_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("installedCount"), requestedIds); + setResourceProperty(resource, COMPONENT_INSTALLED_AND_MAINTENANCE_OFF_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("installedAndMaintenanceOffCount"), requestedIds); setResourceProperty(resource, COMPONENT_INSTALL_FAILED_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("installFailedCount"), requestedIds); setResourceProperty(resource, COMPONENT_INIT_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("initCount"), requestedIds); setResourceProperty(resource, COMPONENT_UNKNOWN_COUNT_PROPERTY_ID, response.getServiceComponentStateCount().get("unknownCount"), requestedIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java index 2a45f02..737bfa4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigGroupResourceProvider.java @@ -62,6 +62,7 @@ import org.apache.ambari.server.state.ConfigFactory; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,6 +85,8 @@ public class ConfigGroupResourceProvider extends .getPropertyId("ConfigGroup", "group_name"); protected static final String CONFIGGROUP_TAG_PROPERTY_ID = PropertyHelper .getPropertyId("ConfigGroup", "tag"); + protected static final String CONFIGGROUP_SERVICENAME_PROPERTY_ID = PropertyHelper + .getPropertyId("ConfigGroup", "service_name"); protected static final String CONFIGGROUP_DESC_PROPERTY_ID = PropertyHelper .getPropertyId("ConfigGroup", "description"); protected static final String CONFIGGROUP_SCV_NOTE_ID = PropertyHelper @@ -562,8 +565,8 @@ public class ConfigGroupResourceProvider extends verifyHostList(cluster, hosts, request); - String serviceName = null; - if (request.getConfigs() != null && !request.getConfigs().isEmpty()) { + String serviceName = request.getServiceName(); + if (serviceName == null && !MapUtils.isEmpty(request.getConfigs())) { try { serviceName = cluster.getServiceForConfigTypes(request.getConfigs().keySet()); } catch (IllegalArgumentException e) { @@ -751,6 +754,7 @@ public class ConfigGroupResourceProvider extends (String) properties.get(CONFIGGROUP_CLUSTER_NAME_PROPERTY_ID), (String) properties.get(CONFIGGROUP_NAME_PROPERTY_ID), (String) properties.get(CONFIGGROUP_TAG_PROPERTY_ID), + (String) properties.get(CONFIGGROUP_SERVICENAME_PROPERTY_ID), (String) properties.get(CONFIGGROUP_DESC_PROPERTY_ID), null, null); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java index 074f8e1..5e5bff5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java @@ -106,8 +106,6 @@ public class DefaultProviderModule extends AbstractProviderModule { return new UpgradeItemResourceProvider(managementController); case UpgradeSummary: return new UpgradeSummaryResourceProvider(managementController); - case ClusterStackVersion: - return new ClusterStackVersionResourceProvider(managementController); case PreUpgradeCheck: return new PreUpgradeCheckResourceProvider(managementController); case HostStackVersion: http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java index 1d3e922..4814a33 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java @@ -52,6 +52,8 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid public static final String REPOSITORY_CLUSTER_STACK_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "cluster_version_id"); public static final String REPOSITORY_OS_TYPE_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "os_type"); public static final String REPOSITORY_BASE_URL_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "base_url"); + public static final String REPOSITORY_DISTRIBUTION_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "distribution"); + public static final String REPOSITORY_COMPONENTS_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "components"); public static final String REPOSITORY_REPO_ID_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "repo_id"); public static final String REPOSITORY_MIRRORS_LIST_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "mirrors_list"); public static final String REPOSITORY_DEFAULT_BASE_URL_PROPERTY_ID = PropertyHelper.getPropertyId("Repositories", "default_base_url"); @@ -74,6 +76,8 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid public static Set<String> propertyIds = new HashSet<String>() { { add(REPOSITORY_REPO_NAME_PROPERTY_ID); + add(REPOSITORY_DISTRIBUTION_PROPERTY_ID); + add(REPOSITORY_COMPONENTS_PROPERTY_ID); add(REPOSITORY_STACK_NAME_PROPERTY_ID); add(REPOSITORY_STACK_VERSION_PROPERTY_ID); add(REPOSITORY_OS_TYPE_PROPERTY_ID); @@ -154,6 +158,8 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid setResourceProperty(resource, REPOSITORY_STACK_NAME_PROPERTY_ID, response.getStackName(), requestedIds); setResourceProperty(resource, REPOSITORY_STACK_VERSION_PROPERTY_ID, response.getStackVersion(), requestedIds); setResourceProperty(resource, REPOSITORY_REPO_NAME_PROPERTY_ID, response.getRepoName(), requestedIds); + setResourceProperty(resource, REPOSITORY_DISTRIBUTION_PROPERTY_ID, response.getDistribution(), requestedIds); + setResourceProperty(resource, REPOSITORY_COMPONENTS_PROPERTY_ID, response.getComponents(), requestedIds); setResourceProperty(resource, REPOSITORY_BASE_URL_PROPERTY_ID, response.getBaseUrl(), requestedIds); setResourceProperty(resource, REPOSITORY_OS_TYPE_PROPERTY_ID, response.getOsType(), requestedIds); setResourceProperty(resource, REPOSITORY_REPO_ID_PROPERTY_ID, response.getRepoId(), requestedIds); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java index 44ef9f7..6455805 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java @@ -87,6 +87,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc public static final String REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "repository_version"); public static final String REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "display_name"); public static final String REPOSITORY_VERSION_HIDDEN_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "hidden"); + public static final String REPOSITORY_VERSION_RESOLVED_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "resolved"); public static final String SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID = new OperatingSystemResourceDefinition().getPluralName(); public static final String SUBRESOURCE_REPOSITORIES_PROPERTY_ID = new RepositoryResourceDefinition().getPluralName(); @@ -121,7 +122,8 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc REPOSITORY_VERSION_PARENT_ID, REPOSITORY_VERSION_HAS_CHILDREN, REPOSITORY_VERSION_AVAILABLE_SERVICES, - REPOSITORY_VERSION_STACK_SERVICES); + REPOSITORY_VERSION_STACK_SERVICES, + REPOSITORY_VERSION_RESOLVED_PROPERTY_ID); @SuppressWarnings("serial") public static Map<Type, String> keyPropertyIds = new ImmutableMap.Builder<Type, String>() @@ -257,7 +259,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc setResourceProperty(resource, REPOSITORY_VERSION_HIDDEN_PROPERTY_ID, entity.isHidden(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, entity.getVersion(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_TYPE_PROPERTY_ID, entity.getType(), requestedIds); - + setResourceProperty(resource, REPOSITORY_VERSION_RESOLVED_PROPERTY_ID, entity.isResolved(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_PARENT_ID, entity.getParentId(), requestedIds); List<RepositoryVersionEntity> children = entity.getChildren(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java index 3f8b6e9..526f173 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java @@ -427,6 +427,8 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider // do all validation checks validateCreateRequests(requests, clusters); + Set<Cluster> clustersSetFromRequests = new HashSet<>(); + for (ServiceRequest request : requests) { Cluster cluster = clusters.getCluster(request.getClusterName()); @@ -482,6 +484,12 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider // Initialize service widgets getManagementController().initializeWidgetsAndLayouts(cluster, s); + clustersSetFromRequests.add(cluster); + } + + // Create cluster widgets and layouts + for (Cluster cluster : clustersSetFromRequests) { + getManagementController().initializeWidgetsAndLayouts(cluster, null); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 0ff21a2..52f66bc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -731,11 +731,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider itemEntity.setText(wrapper.getText()); itemEntity.setTasks(wrapper.getTasksJson()); itemEntity.setHosts(wrapper.getHostsJson()); - itemEntities.add(itemEntity); injectVariables(configHelper, cluster, itemEntity); - makeServerSideStage(group, upgradeContext, effectiveRepositoryVersion, req, - itemEntity, (ServerSideActionTask) task, configUpgradePack); + if (makeServerSideStage(group, upgradeContext, effectiveRepositoryVersion, req, + itemEntity, (ServerSideActionTask) task, configUpgradePack)) { + itemEntities.add(itemEntity); + } } } } else { @@ -1184,7 +1185,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * upgrade * @throws AmbariException */ - private void makeServerSideStage(UpgradeGroupHolder group, UpgradeContext context, + private boolean makeServerSideStage(UpgradeGroupHolder group, UpgradeContext context, RepositoryVersionEntity effectiveRepositoryVersion, RequestStageContainer request, UpgradeItemEntity entity, ServerSideActionTask task, ConfigUpgradePack configUpgradePack) throws AmbariException { @@ -1201,6 +1202,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider String itemDetail = entity.getText(); String stageText = StringUtils.abbreviate(entity.getText(), 255); + boolean process = true; + switch (task.getType()) { case SERVER_ACTION: case MANUAL: { @@ -1236,6 +1239,13 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } case CONFIGURE: { ConfigureTask ct = (ConfigureTask) task; + + // !!! would prefer to do this in the sequence generator, but there's too many + // places to miss + if (context.getOrchestrationType().isRevertable() && !ct.supportsPatch) { + process = false; + } + Map<String, String> configurationChanges = ct.getConfigurationChanges(cluster, configUpgradePack); @@ -1266,6 +1276,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider break; } + if (!process) { + return false; + } + ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), Role.AMBARI_SERVER_ACTION.toString(), Collections.emptyList(), commandParams); @@ -1303,6 +1317,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider context.isComponentFailureAutoSkipped()); request.addStages(Collections.singletonList(stage)); + + return true; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java index 394a0bf..f94ac32 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -770,6 +770,10 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc repo.getRepositoryId()); repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID), repo.getName()); + repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_DISTRIBUTION_PROPERTY_ID), + repo.getDistribution()); + repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_COMPONENTS_PROPERTY_ID), + repo.getComponents()); repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_NAME_PROPERTY_ID), entity.getStackName()); repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID), http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java index e0c8be9e..f09e3ba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/AlertReceivedEvent.java @@ -27,6 +27,12 @@ import org.apache.ambari.server.state.Alert; */ public final class AlertReceivedEvent extends AlertEvent { + /** + * Constructor. + * + * @param clusterId + * @param alert + */ public AlertReceivedEvent(long clusterId, Alert alert) { super(clusterId, alert); } @@ -41,7 +47,7 @@ public final class AlertReceivedEvent extends AlertEvent { @Override public String toString() { StringBuilder buffer = new StringBuilder("AlertReceivedEvent{"); - buffer.append("clusterId=").append(m_clusterId); + buffer.append("cluserId=").append(m_clusterId); buffer.append(", alerts=").append(getAlerts()); buffer.append("}"); return buffer.toString(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java index cdb86ac..f0574d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterConfigFinishedEvent.java @@ -23,17 +23,27 @@ package org.apache.ambari.server.events; * cluster configuration is successfully updated. */ public class ClusterConfigFinishedEvent extends AmbariEvent { + + private final long clusterId; private final String clusterName; - public ClusterConfigFinishedEvent(String clusterName) { + public ClusterConfigFinishedEvent(long clusterId, String clusterName) { super(AmbariEventType.CLUSTER_CONFIG_FINISHED); + this.clusterId = clusterId; this.clusterName = clusterName; } /** + * Get the cluster id + * @return + */ + public long getClusterId() { + return clusterId; + } + + /** * Get the cluster name - * * @return */ public String getClusterName() { @@ -46,6 +56,7 @@ public class ClusterConfigFinishedEvent extends AmbariEvent { @Override public String toString() { StringBuilder buffer = new StringBuilder("ClusterConfigChangedEvent{"); + buffer.append("clusterId=").append(getClusterId()); buffer.append("clusterName=").append(getClusterName()); buffer.append("}"); return buffer.toString(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java index aa29894..1944761 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java @@ -124,8 +124,16 @@ public class DistributeRepositoriesActionListener { if (null != repoVersion && StringUtils.isNotBlank(actualVersion)) { if (!StringUtils.equals(repoVersion.getVersion(), actualVersion)) { repoVersion.setVersion(actualVersion); + repoVersion.setResolved(true); repoVersionDAO.merge(repoVersion); repositoryVersion = actualVersion; + } else { + // the reported versions are the same - we should ensure that the + // repo is resolved + if (!repoVersion.isResolved()) { + repoVersion.setResolved(true); + repoVersionDAO.merge(repoVersion); + } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index 394e9f9..4329cdb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@ -98,8 +98,16 @@ public class StackVersionListener { if (null != rve) { String currentRepoVersion = rve.getVersion(); if (!StringUtils.equals(currentRepoVersion, newVersion)) { - rve.setVersion(newVersion); + rve.setVersion(newVersion); + rve.setResolved(true); + repositoryVersionDAO.merge(rve); + } else { + // the reported versions are the same - we should ensure that the repo + // is resolved + if (!rve.isResolved()) { + rve.setResolved(true); repositoryVersionDAO.merge(rve); + } } } } @@ -236,4 +244,4 @@ public class StackVersionListener { sch.setUpgradeState(upgradeState); sch.recalculateHostVersionState(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java index 1ef415a..096a395 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java @@ -31,11 +31,13 @@ import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.google.inject.Injector; +import com.google.inject.Singleton; /** * RoleCommandOrderProvider which caches RoleCommandOrder objects for a cluster to avoid the cost of construction of * RoleCommandOrder objects each time. */ +@Singleton public class CachedRoleCommandOrderProvider implements RoleCommandOrderProvider { private static final Logger LOG = LoggerFactory.getLogger(CachedRoleCommandOrderProvider.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java index b137705..26670fc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java @@ -653,12 +653,10 @@ public class DBAccessorImpl implements DBAccessor { } @Override - public void alterColumn(String tableName, DBColumnInfo columnInfo) - throws SQLException { + public void alterColumn(String tableName, DBColumnInfo columnInfo) throws SQLException { //varchar extension only (derby limitation, but not too much for others), if (dbmsHelper.supportsColumnTypeChange()) { - String statement = dbmsHelper.getAlterColumnStatement(tableName, - columnInfo); + String statement = dbmsHelper.getAlterColumnStatement(tableName, columnInfo); executeQuery(statement); } else { //use addColumn: add_tmp-update-drop-rename for Derby http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java index 3ca9d3a..227a935 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertDefinitionDAO.java @@ -37,6 +37,7 @@ import org.apache.ambari.server.orm.entities.AlertGroupEntity; import org.apache.ambari.server.state.alert.AlertDefinition; import org.apache.ambari.server.state.alert.AlertDefinitionFactory; import org.apache.ambari.server.state.alert.Scope; +import org.apache.ambari.server.state.alert.SourceType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -308,6 +309,18 @@ public class AlertDefinitionDAO { } /** + * @return all definitions with the given sourceType + */ + @RequiresSession + public List<AlertDefinitionEntity> findBySourceType(Long clusterId, SourceType sourceType) { + return daoUtils.selectList( + entityManagerProvider.get() + .createNamedQuery("AlertDefinitionEntity.findBySourceType", AlertDefinitionEntity.class) + .setParameter("clusterId", clusterId) + .setParameter("sourceType", sourceType)); + } + + /** * Persists a new alert definition, also creating the associated * {@link AlertGroupEntity} relationship for the definition's service default * group. Fires an {@link AlertDefinitionRegistrationEvent}. http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java index d0f8d0b..a1b6fbe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java @@ -356,9 +356,29 @@ public class ClusterDAO { * the entity to merge (not {@code null}). * @return the managed entity which was merged (never {@code null}). */ + @Transactional public ClusterConfigEntity merge(ClusterConfigEntity clusterConfigEntity) { + return merge(clusterConfigEntity, false); + } + + /** + * Merge the specified entity into the current persistence context. + * + * @param clusterConfigEntity + * the entity to merge (not {@code null}). + * @param flush + * if {@code true} then {@link EntityManager#flush()} will be invoked + * immediately after the merge. + * @return the managed entity which was merged (never {@code null}). + */ + @Transactional + public ClusterConfigEntity merge(ClusterConfigEntity clusterConfigEntity, boolean flush) { EntityManager entityManager = entityManagerProvider.get(); - return entityManager.merge(clusterConfigEntity); + ClusterConfigEntity clusterConfigEntityRes = entityManager.merge(clusterConfigEntity); + if(flush) { + entityManager.flush(); + } + return clusterConfigEntityRes; } @Transactional http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index f634d89..010ccec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@ -39,6 +39,7 @@ import javax.persistence.metamodel.SingularAttribute; import org.apache.ambari.annotations.TransactionalLock; import org.apache.ambari.annotations.TransactionalLock.LockArea; import org.apache.ambari.annotations.TransactionalLock.LockType; +import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; @@ -964,6 +965,24 @@ public class HostRoleCommandDAO { } /** + * Gets the most recently run service check grouped by the command's role + * (which is the only way to identify the service it was for!?) + * + * @param clusterId + * the ID of the cluster to get the service checks for. + */ + @RequiresSession + public List<LastServiceCheckDTO> getLatestServiceChecksByRole(long clusterId) { + TypedQuery<LastServiceCheckDTO> query = entityManagerProvider.get().createNamedQuery( + "HostRoleCommandEntity.findLatestServiceChecksByRole", LastServiceCheckDTO.class); + + query.setParameter("clusterId", clusterId); + query.setParameter("roleCommand", RoleCommand.SERVICE_CHECK); + + return daoUtils.selectList(query); + } + + /** * The {@link HostRoleCommandPredicateVisitor} is used to convert an Ambari * {@link Predicate} into a JPA {@link javax.persistence.criteria.Predicate}. */ @@ -1010,4 +1029,32 @@ public class HostRoleCommandDAO { return Sets.newHashSet(taskIds); } + + /** + * A simple DTO for storing the most recent service check time for a given + * {@link Role}. + */ + public static class LastServiceCheckDTO { + + /** + * The role. + */ + public final String role; + + /** + * The time that the service check ended. + */ + public final long endTime; + + /** + * Constructor. + * + * @param role + * @param endTime + */ + public LastServiceCheckDTO(String role, long endTime) { + this.role = role; + this.endTime = endTime; + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java index 94f5fa5..978540a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceConfigDAO.java @@ -197,7 +197,7 @@ public class ServiceConfigDAO { stackId.getStackVersion()); TypedQuery<ServiceConfigEntity> query = entityManagerProvider.get().createNamedQuery( - "ServiceConfigEntity.findServiceConfigsByStack", + "ServiceConfigEntity.findAllServiceConfigsByStack", ServiceConfigEntity.class); query.setParameter("clusterId", clusterId); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java index de203be..2f4941b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/AlertDefinitionEntity.java @@ -72,8 +72,8 @@ import org.apache.ambari.server.state.alert.SourceType; @NamedQuery(name = "AlertDefinitionEntity.findByService", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.serviceName = :serviceName AND ad.clusterId = :clusterId"), @NamedQuery(name = "AlertDefinitionEntity.findByServiceAndComponent", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.serviceName = :serviceName AND ad.componentName = :componentName AND ad.clusterId = :clusterId"), @NamedQuery(name = "AlertDefinitionEntity.findByServiceMaster", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.serviceName IN :services AND ad.scope = :scope AND ad.clusterId = :clusterId AND ad.componentName IS NULL"), - @NamedQuery(name = "AlertDefinitionEntity.findByIds", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.definitionId IN :definitionIds")}) - + @NamedQuery(name = "AlertDefinitionEntity.findByIds", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.definitionId IN :definitionIds"), + @NamedQuery(name = "AlertDefinitionEntity.findBySourceType", query = "SELECT ad FROM AlertDefinitionEntity ad WHERE ad.clusterId = :clusterId AND ad.sourceType = :sourceType")}) public class AlertDefinitionEntity { @Id http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java index bfc83ca..0eea7e6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java @@ -60,22 +60,54 @@ import org.apache.commons.lang.ArrayUtils; , initialValue = 1 ) @NamedQueries({ - @NamedQuery(name = "HostRoleCommandEntity.findTaskIdsByRequestStageIds", query = "SELECT command.taskId FROM HostRoleCommandEntity command WHERE command.stageId = :stageId AND command.requestId = :requestId"), - @NamedQuery(name = "HostRoleCommandEntity.findCountByCommandStatuses", query = "SELECT COUNT(command.taskId) FROM HostRoleCommandEntity command WHERE command.status IN :statuses"), - @NamedQuery(name = "HostRoleCommandEntity.findByRequestIdAndStatuses", query="SELECT task FROM HostRoleCommandEntity task WHERE task.requestId=:requestId AND task.status IN :statuses ORDER BY task.taskId ASC"), - @NamedQuery(name = "HostRoleCommandEntity.findTasksByStatusesOrderByIdDesc", query = "SELECT task FROM HostRoleCommandEntity task WHERE task.requestId = :requestId AND task.status IN :statuses ORDER BY task.taskId DESC"), - @NamedQuery(name = "HostRoleCommandEntity.findNumTasksAlreadyRanInStage", query = "SELECT COUNT(task.taskId) FROM HostRoleCommandEntity task WHERE task.requestId = :requestId AND task.taskId > :taskId AND task.stageId > :stageId AND task.status NOT IN :statuses"), - @NamedQuery(name = "HostRoleCommandEntity.findByCommandStatuses", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.status IN :statuses ORDER BY command.requestId, command.stageId"), - @NamedQuery(name = "HostRoleCommandEntity.findByHostId", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostId=:hostId"), - @NamedQuery(name = "HostRoleCommandEntity.findByHostRole", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity.hostName=:hostName AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role ORDER BY command.taskId"), - @NamedQuery(name = "HostRoleCommandEntity.findByHostRoleNullHost", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity IS NULL AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role"), - @NamedQuery(name = "HostRoleCommandEntity.findByStatusBetweenStages", query = "SELECT command FROM HostRoleCommandEntity command WHERE command.requestId = :requestId AND command.stageId >= :minStageId AND command.stageId <= :maxStageId AND command.status = :status"), - @NamedQuery(name = "HostRoleCommandEntity.updateAutoSkipExcludeRoleCommand", query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand <> :roleCommand"), - @NamedQuery(name = "HostRoleCommandEntity.updateAutoSkipForRoleCommand", query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand = :roleCommand"), - @NamedQuery(name = "HostRoleCommandEntity.removeByTaskIds", query = "DELETE FROM HostRoleCommandEntity command WHERE command.taskId IN :taskIds"), - @NamedQuery(name = "HostRoleCommandEntity.findHostsByCommandStatus", query = "SELECT DISTINCT(host.hostName) FROM HostRoleCommandEntity command, HostEntity host WHERE (command.requestId >= :iLowestRequestIdInProgress AND command.requestId <= :iHighestRequestIdInProgress) AND command.status IN :statuses AND command.hostId = host.hostId AND host.hostName IS NOT NULL"), - @NamedQuery(name = "HostRoleCommandEntity.getBlockingHostsForRequest", query = "SELECT DISTINCT(host.hostName) FROM HostRoleCommandEntity command, HostEntity host WHERE command.requestId >= :lowerRequestIdInclusive AND command.requestId < :upperRequestIdExclusive AND command.status IN :statuses AND command.isBackgroundCommand=0 AND command.hostId = host.hostId AND host.hostName IS NOT NULL") - + @NamedQuery( + name = "HostRoleCommandEntity.findTaskIdsByRequestStageIds", + query = "SELECT command.taskId FROM HostRoleCommandEntity command WHERE command.stageId = :stageId AND command.requestId = :requestId"), + @NamedQuery( + name = "HostRoleCommandEntity.findCountByCommandStatuses", + query = "SELECT COUNT(command.taskId) FROM HostRoleCommandEntity command WHERE command.status IN :statuses"), + @NamedQuery( + name = "HostRoleCommandEntity.findByRequestIdAndStatuses", + query = "SELECT task FROM HostRoleCommandEntity task WHERE task.requestId=:requestId AND task.status IN :statuses ORDER BY task.taskId ASC"), + @NamedQuery( + name = "HostRoleCommandEntity.findTasksByStatusesOrderByIdDesc", + query = "SELECT task FROM HostRoleCommandEntity task WHERE task.requestId = :requestId AND task.status IN :statuses ORDER BY task.taskId DESC"), + @NamedQuery( + name = "HostRoleCommandEntity.findNumTasksAlreadyRanInStage", + query = "SELECT COUNT(task.taskId) FROM HostRoleCommandEntity task WHERE task.requestId = :requestId AND task.taskId > :taskId AND task.stageId > :stageId AND task.status NOT IN :statuses"), + @NamedQuery( + name = "HostRoleCommandEntity.findByCommandStatuses", + query = "SELECT command FROM HostRoleCommandEntity command WHERE command.status IN :statuses ORDER BY command.requestId, command.stageId"), + @NamedQuery( + name = "HostRoleCommandEntity.findByHostId", + query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostId=:hostId"), + @NamedQuery( + name = "HostRoleCommandEntity.findByHostRole", + query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity.hostName=:hostName AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role ORDER BY command.taskId"), + @NamedQuery( + name = "HostRoleCommandEntity.findByHostRoleNullHost", + query = "SELECT command FROM HostRoleCommandEntity command WHERE command.hostEntity IS NULL AND command.requestId=:requestId AND command.stageId=:stageId AND command.role=:role"), + @NamedQuery( + name = "HostRoleCommandEntity.findByStatusBetweenStages", + query = "SELECT command FROM HostRoleCommandEntity command WHERE command.requestId = :requestId AND command.stageId >= :minStageId AND command.stageId <= :maxStageId AND command.status = :status"), + @NamedQuery( + name = "HostRoleCommandEntity.updateAutoSkipExcludeRoleCommand", + query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand <> :roleCommand"), + @NamedQuery( + name = "HostRoleCommandEntity.updateAutoSkipForRoleCommand", + query = "UPDATE HostRoleCommandEntity command SET command.autoSkipOnFailure = :autoSkipOnFailure WHERE command.requestId = :requestId AND command.roleCommand = :roleCommand"), + @NamedQuery( + name = "HostRoleCommandEntity.removeByTaskIds", + query = "DELETE FROM HostRoleCommandEntity command WHERE command.taskId IN :taskIds"), + @NamedQuery( + name = "HostRoleCommandEntity.findHostsByCommandStatus", + query = "SELECT DISTINCT(host.hostName) FROM HostRoleCommandEntity command, HostEntity host WHERE (command.requestId >= :iLowestRequestIdInProgress AND command.requestId <= :iHighestRequestIdInProgress) AND command.status IN :statuses AND command.hostId = host.hostId AND host.hostName IS NOT NULL"), + @NamedQuery( + name = "HostRoleCommandEntity.getBlockingHostsForRequest", + query = "SELECT DISTINCT(host.hostName) FROM HostRoleCommandEntity command, HostEntity host WHERE command.requestId >= :lowerRequestIdInclusive AND command.requestId < :upperRequestIdExclusive AND command.status IN :statuses AND command.isBackgroundCommand=0 AND command.hostId = host.hostId AND host.hostName IS NOT NULL"), + @NamedQuery( + name = "HostRoleCommandEntity.findLatestServiceChecksByRole", + query = "SELECT NEW org.apache.ambari.server.orm.dao.HostRoleCommandDAO.LastServiceCheckDTO(command.role, MAX(command.endTime)) FROM HostRoleCommandEntity command WHERE command.roleCommand = :roleCommand AND command.endTime > 0 AND command.stage.clusterId = :clusterId GROUP BY command.role ORDER BY command.role ASC") }) public class HostRoleCommandEntity { http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java index bad8195..6d7498b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryEntity.java @@ -23,6 +23,8 @@ package org.apache.ambari.server.orm.entities; public class RepositoryEntity { private String name; + private String distribution; + private String components; private String baseUrl; private String repositoryId; private String mirrorsList; @@ -36,6 +38,22 @@ public class RepositoryEntity { this.name = name; } + public String getDistribution() { + return distribution; + } + + public void setDistribution(String distribution) { + this.distribution = distribution; + } + + public String getComponents() { + return components; + } + + public void setComponents(String components) { + this.components = components; + } + public String getBaseUrl() { return baseUrl; } @@ -60,6 +78,8 @@ public class RepositoryEntity { RepositoryEntity that = (RepositoryEntity) o; if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (distribution != null ? !distribution.equals(that.distribution) : that.distribution != null) return false; + if (components != null ? !components.equals(that.components) : that.components != null) return false; if (baseUrl != null ? !baseUrl.equals(that.baseUrl) : that.baseUrl != null) return false; if (repositoryId != null ? !repositoryId.equals(that.repositoryId) : that.repositoryId != null) return false; @@ -69,6 +89,8 @@ public class RepositoryEntity { @Override public int hashCode() { int result = name != null ? name.hashCode() : 0; + result = 31 * result + (distribution != null ? distribution.hashCode() : 0); + result = 31 * result + (components != null ? components.hashCode() : 0); result = 31 * result + (baseUrl != null ? baseUrl.hashCode() : 0); result = 31 * result + (repositoryId != null ? repositoryId.hashCode() : 0); return result; http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index bf9e678..2b56b11 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@ -147,6 +147,16 @@ public class RepositoryVersionEntity { @Column(name = "hidden", nullable = false, insertable = true, updatable = true) private short isHidden = 0; + /** + * Repositories can't be trusted until they have been deployed and we've + * detected their actual version. Most of the time, things match up, but + * editing a VDF could causes the version to be misrepresented. Once we have + * received the correct version of the repository (normally after it's been + * installed), then we can set this flag to {@code true}. + */ + @Column(name = "resolved", nullable = false) + private short resolved = 0; + @ManyToOne @JoinColumn(name = "parent_id") private RepositoryVersionEntity parent; @@ -217,6 +227,13 @@ public class RepositoryVersionEntity { return version; } + /** + * Sets the version on this repository version entity. If the version is + * confirmed as correct, then the called should also set + * {@link #setResolved(boolean)}. + * + * @param version + */ public void setVersion(String version) { this.version = version; } @@ -459,4 +476,24 @@ public class RepositoryVersionEntity { this.isHidden = (short) (isHidden ? 1 : 0); } + /** + * Gets whether this repository has been installed and has reported back its + * actual version. + * + * @return {@code true} if the version for this repository can be trusted, + * {@code false} otherwise. + */ + public boolean isResolved() { + return resolved == 1; + } + + /** + * Sets whether this repository has been installed and has reported back its + * actual version. + * + * @param resolved + */ + public void setResolved(boolean resolved) { + this.resolved = resolved ? (short) 1 : (short) 0; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java index 7f0408f..0c254ef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java @@ -76,7 +76,7 @@ public class UpgradeItemEntity { private String tasks = null; @Basic - @Column(name = "item_text", length = 1024) + @Column(name = "item_text") private String itemText = null; @Basic http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java b/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java index 95fc973..e804961 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/scheduler/ExecutionScheduleManager.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.regex.Pattern; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; @@ -116,6 +117,8 @@ public class ExecutionScheduleManager { protected static final String REQUESTS_TIMEDOUT_TASKS_KEY = "timed_out_task_count"; protected static final String REQUESTS_TOTAL_TASKS_KEY = "task_count"; + protected static final Pattern CONTAINS_API_VERSION_PATTERN = Pattern.compile("^/?" + DEFAULT_API_PATH+ ".*"); + @Inject public ExecutionScheduleManager(Configuration configuration, ExecutionScheduler executionScheduler, @@ -663,7 +666,7 @@ public class ExecutionScheduleManager { } protected BatchRequestResponse performApiGetRequest(String relativeUri, boolean queryAllFields) { - WebResource webResource = ambariWebResource.path(completeRelativeUri(relativeUri)); + WebResource webResource = extendApiResource(ambariWebResource, relativeUri); if (queryAllFields) { webResource = webResource.queryParam("fields", "*"); } @@ -679,7 +682,8 @@ public class ExecutionScheduleManager { protected BatchRequestResponse performApiRequest(String relativeUri, String body, String method, Integer userId) { ClientResponse response; try { - response = ambariWebResource.path(completeRelativeUri(relativeUri)).header(USER_ID_HEADER, userId).method(method, ClientResponse.class, body); + response = extendApiResource(ambariWebResource, relativeUri) + .header(USER_ID_HEADER, userId).method(method, ClientResponse.class, body); } catch (UniformInterfaceException e) { response = e.getResponse(); } @@ -796,15 +800,18 @@ public class ExecutionScheduleManager { } } - private String completeRelativeUri(String relativeUri){ - if (StringUtils.isNotEmpty(relativeUri) - && !(relativeUri.startsWith("api/v1") || relativeUri.startsWith("/api/v1"))){ - if (relativeUri.charAt(0) != '/') { - relativeUri = '/' + relativeUri; - } - return "api/v1" + relativeUri; + /** + * Returns the absolute web resource with {@link #DEFAULT_API_PATH} + * @param webResource Ambari WebResource as provided by the client {@link #ambariWebResource} + * @param relativeUri relative request URI + * @return Extended WebResource + */ + protected WebResource extendApiResource(WebResource webResource, String relativeUri) { + WebResource result = webResource; + if (StringUtils.isNotEmpty(relativeUri) && !CONTAINS_API_VERSION_PATTERN.matcher(relativeUri).matches()) { + result = webResource.path(DEFAULT_API_PATH); } - return relativeUri; + return result.path(relativeUri); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java index d6b8ffc..3db844a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java @@ -76,12 +76,13 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer Map<String, Map<String, String>> currentConfigurations, Map<String, Map<String, String>> kerberosConfigurations, boolean includeAmbariIdentity, - Map<String, Set<String>> propertiesToBeIgnored) throws AmbariException { + Map<String, Set<String>> propertiesToBeIgnored, + boolean excludeHeadless) throws AmbariException { List<Component> components = new ArrayList<>(); for (ServiceComponentHost each : schToProcess) { components.add(Component.fromServiceComponentHost(each)); } - processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored); + processServiceComponents(cluster, kerberosDescriptor, components, identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, includeAmbariIdentity, propertiesToBeIgnored, excludeHeadless); } protected void processServiceComponents(Cluster cluster, KerberosDescriptor kerberosDescriptor, @@ -90,7 +91,8 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer Map<String, Map<String, String>> currentConfigurations, Map<String, Map<String, String>> kerberosConfigurations, boolean includeAmbariIdentity, - Map<String, Set<String>> propertiesToBeIgnored) throws AmbariException { + Map<String, Set<String>> propertiesToBeIgnored, + boolean excludeHeadless) throws AmbariException { actionLog.writeStdOut("Processing Kerberos identities and configurations"); @@ -141,7 +143,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer // Add service-level principals (and keytabs) kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, serviceIdentities, - identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore); KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName); @@ -156,7 +158,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer // Add component-level principals (and keytabs) kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities, - identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + identityFilter, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } @@ -177,7 +179,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity); kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities, - identityFilter, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations); + identityFilter, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, excludeHeadless); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java index a23ab5d..4396a2b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java @@ -217,9 +217,14 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { return commandReport; } + boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL)); + boolean onlyKeytabWrite = "true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE)); + boolean grabKeytabFromCache = regenerateKeytabs && onlyKeytabWrite; + // if grabKeytabFromCache=true we will try to get keytab from cache and send to agent, it will be true for + // headless cached keytabs if (password == null) { - if (hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || kerberosPrincipalHostDAO - .exists(evaluatedPrincipal, hostEntity.getHostId())) { + if (!grabKeytabFromCache && (hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || kerberosPrincipalHostDAO + .exists(evaluatedPrincipal, hostEntity.getHostId()))) { // There is nothing to do for this since it must already exist and we don't want to // regenerate the keytab message = String.format("Skipping keytab file for %s, missing password indicates nothing to do", evaluatedPrincipal); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java index 81e345a..ddf3d1b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java @@ -36,6 +36,6 @@ public interface KerberosIdentityDataFile extends KerberosDataFile { String KEYTAB_FILE_GROUP_NAME = "keytab_file_group_name"; String KEYTAB_FILE_GROUP_ACCESS = "keytab_file_group_access"; String KEYTAB_FILE_IS_CACHABLE = "keytab_file_is_cachable"; - + String ONLY_KEYTAB_WRITE = "only_keytab_write"; } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java index f55c6f4..ea742bd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java @@ -68,7 +68,8 @@ public class KerberosIdentityDataFileWriter extends AbstractKerberosDataFileWrit String principal, String principalType, String keytabFilePath, String keytabFileOwnerName, String keytabFileOwnerAccess, String keytabFileGroupName, - String keytabFileGroupAccess, String keytabFileCanCache) + String keytabFileGroupAccess, String keytabFileCanCache, + String onlyKeytabWrite) throws IOException { super.appendRecord(hostName, serviceName, @@ -80,7 +81,8 @@ public class KerberosIdentityDataFileWriter extends AbstractKerberosDataFileWrit keytabFileOwnerAccess, keytabFileGroupName, keytabFileGroupAccess, - keytabFileCanCache); + keytabFileCanCache, + onlyKeytabWrite); } @Override @@ -95,6 +97,7 @@ public class KerberosIdentityDataFileWriter extends AbstractKerberosDataFileWrit KEYTAB_FILE_OWNER_ACCESS, KEYTAB_FILE_GROUP_NAME, KEYTAB_FILE_GROUP_ACCESS, - KEYTAB_FILE_IS_CACHABLE); + KEYTAB_FILE_IS_CACHABLE, + ONLY_KEYTAB_WRITE); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java index 4e63f4a..f56e946 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java @@ -108,7 +108,7 @@ public class PrepareDisableKerberosServerAction extends AbstractPrepareKerberosS Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore); + configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, false); // Add auth-to-local configurations to the set of changes Map<String, Set<String>> authToLocalProperties = kerberosHelper.translateConfigurationSpecifications(kerberosDescriptor.getAllAuthToLocalProperties()); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java index e13f033..3ec84fa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java @@ -117,7 +117,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, true, propertiesToIgnore); + configurations, kerberosConfigurations, true, propertiesToIgnore, false); // Calculate the set of configurations to update and replace any variables // using the previously calculated Map of configurations for the host. http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java index 00c82a5..49828cb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java @@ -33,6 +33,7 @@ import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,7 +95,7 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore); + configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, !CollectionUtils.isEmpty(getHostFilter())); kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, propertiesToRemove, true); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java index 5ec0692..451f802 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java @@ -202,6 +202,11 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { // longer used finalizeHostRepositoryVersions(cluster); + if (upgradeContext.getOrchestrationType() == RepositoryType.STANDARD) { + outSB.append(String.format("Finalizing the version for cluster %s.\n", cluster.getClusterName())); + cluster.setCurrentStackVersion(cluster.getDesiredStackVersion()); + } + // mark revertable if (repositoryType.isRevertable() && direction == Direction.UPGRADE) { UpgradeEntity upgrade = cluster.getUpgradeInProgress(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java index 697f1d1..30bc47f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java @@ -310,7 +310,7 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction { // Add service-level principals (and keytabs) kerberosHelper.addIdentities(null, serviceIdentities, - null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false); propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore); KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName); @@ -325,7 +325,7 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction { // Add component-level principals (and keytabs) kerberosHelper.addIdentities(null, componentIdentities, - null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations); + null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } @@ -346,7 +346,7 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction { List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity); kerberosHelper.addIdentities(null, componentIdentities, - null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations); + null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, false); propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerUsersyncConfigCalculation.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerUsersyncConfigCalculation.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerUsersyncConfigCalculation.java new file mode 100644 index 0000000..67e1dee --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerUsersyncConfigCalculation.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.upgrades; + +import java.text.MessageFormat; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; + +import com.google.inject.Inject; + +/** + * Computes Ranger Usersync ldap grouphierarchylevels property. This class is only used when upgrading from + * HDP-2.6.x to HDP-2.6.y. + */ + +public class RangerUsersyncConfigCalculation extends AbstractServerAction { + private static final String RANGER_USERSYNC_CONFIG_TYPE = "ranger-ugsync-site"; + private static final String RANGER_ENV_CONFIG_TYPE = "ranger-env"; + + @Inject + private Clusters m_clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException { + + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = m_clusters.getCluster(clusterName); + String outputMsg = ""; + + Config rangerUsersyncConfig = cluster.getDesiredConfigByType(RANGER_USERSYNC_CONFIG_TYPE); + + if (null == rangerUsersyncConfig) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Config type {0} not found, skipping updating property in same.", RANGER_USERSYNC_CONFIG_TYPE), ""); + } + + String ldapGroupHierarchy = "0"; + + if (rangerUsersyncConfig.getProperties().containsKey("ranger.usersync.ldap.grouphierarchylevels")) { + ldapGroupHierarchy = rangerUsersyncConfig.getProperties().get("ranger.usersync.ldap.grouphierarchylevels"); + } else { + Map<String, String> targetRangerUsersyncConfig = rangerUsersyncConfig.getProperties(); + targetRangerUsersyncConfig.put("ranger.usersync.ldap.grouphierarchylevels", ldapGroupHierarchy); + rangerUsersyncConfig.setProperties(targetRangerUsersyncConfig); + rangerUsersyncConfig.save(); + + outputMsg = outputMsg + MessageFormat.format("Successfully updated {0} config type.\n", RANGER_USERSYNC_CONFIG_TYPE); + } + + Config rangerEnvConfig = cluster.getDesiredConfigByType(RANGER_ENV_CONFIG_TYPE); + + if (null == rangerEnvConfig) { + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", + MessageFormat.format("Config type {0} not found, skipping updating property in same.", RANGER_ENV_CONFIG_TYPE), ""); + } + + String enableSyncNestedGroup = "false"; + + if (!ldapGroupHierarchy.equals("0") ) { + enableSyncNestedGroup = "true"; + } + + Map<String, String> targetRangerEnvConfig = rangerEnvConfig.getProperties(); + targetRangerEnvConfig.put("is_nested_groupsync_enabled", enableSyncNestedGroup); + rangerEnvConfig.setProperties(targetRangerEnvConfig); + rangerEnvConfig.save(); + + outputMsg = outputMsg + MessageFormat.format("Successfully updated {0} config type.\n", RANGER_ENV_CONFIG_TYPE); + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outputMsg, ""); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java index d43bdfa..073fd82 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java @@ -189,6 +189,8 @@ public class RepoUtil { re.setBaseUrl(repoInfo.getBaseUrl()); re.setName(repoInfo.getRepoName()); re.setRepositoryId(repoInfo.getRepoId()); + re.setDistribution(repoInfo.getDistribution()); + re.setComponents(repoInfo.getComponents()); return re; } @@ -209,4 +211,4 @@ class RepositoryFolderAndXml { this.repoDir = repoDir; this.repoXml = repoXml; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java index db9d178..2992027 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java @@ -18,18 +18,28 @@ package org.apache.ambari.server.stack; -import java.io.File; +import java.net.URI; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.MetainfoDAO; -import org.apache.ambari.server.state.stack.LatestRepoCallable; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.RepoUrlInfoCallable; +import org.apache.ambari.server.state.stack.RepoUrlInfoCallable.RepoUrlInfoResult; +import org.apache.ambari.server.state.stack.RepoVdfCallable; +import org.apache.commons.collections.MapUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Provides external functionality to the Stack framework. @@ -46,15 +56,12 @@ public class StackContext { private ActionMetadata actionMetaData; /** - * Operating System families - */ - private OsFamily osFamily; - - /** * Executor used to get latest repo url's */ - private LatestRepoQueryExecutor repoUpdateExecutor = new LatestRepoQueryExecutor(); + private LatestRepoQueryExecutor repoUpdateExecutor; + private final static Logger LOG = LoggerFactory.getLogger(StackContext.class); + private static final int THREAD_COUNT = 10; /** @@ -67,7 +74,7 @@ public class StackContext { public StackContext(MetainfoDAO metaInfoDAO, ActionMetadata actionMetaData, OsFamily osFamily) { this.metaInfoDAO = metaInfoDAO; this.actionMetaData = actionMetaData; - this.osFamily = osFamily; + repoUpdateExecutor = new LatestRepoQueryExecutor(osFamily); } /** @@ -85,9 +92,8 @@ public class StackContext { * @param url external repo information URL * @param stack stack module */ - public void registerRepoUpdateTask(String url, StackModule stack) { - repoUpdateExecutor.addTask(new LatestRepoCallable(url, - new File(stack.getStackDirectory().getRepoDir()), stack.getModuleInfo(), osFamily)); + public void registerRepoUpdateTask(URI uri, StackModule stack) { + repoUpdateExecutor.addTask(uri, stack); } /** @@ -115,16 +121,16 @@ public class StackContext { /** * Registered tasks */ - private Collection<LatestRepoCallable> tasks = new ArrayList<>(); + private Map<URI, RepoUrlInfoCallable> tasks = new HashMap<>(); /** * Task futures */ - Collection<Future<Void>> futures = new ArrayList<>(); + Collection<Future<?>> futures = new ArrayList<>(); /** * Underlying executor */ - private ExecutorService executor = Executors.newSingleThreadExecutor(new ThreadFactory() { + private ExecutorService executor = Executors.newFixedThreadPool(THREAD_COUNT, new ThreadFactory() { @Override public Thread newThread(Runnable r) { return new Thread(r, "Stack Version Loading Thread"); @@ -132,23 +138,89 @@ public class StackContext { }); + private OsFamily m_family; + + private LatestRepoQueryExecutor(OsFamily family) { + m_family = family; + } + /** - * Add a task. - * - * @param task task to be added + * @param uri + * uri to load + * @param stackModule + * the stack module */ - public void addTask(LatestRepoCallable task) { - tasks.add(task); + public void addTask(URI uri, StackModule stackModule) { + RepoUrlInfoCallable callable = null; + if (tasks.containsKey(uri)) { + callable = tasks.get(uri); + } else { + callable = new RepoUrlInfoCallable(uri); + tasks.put(uri, callable); + } + + callable.addStack(stackModule); } /** * Execute all tasks. */ public void execute() { - for (LatestRepoCallable task : tasks) { - futures.add(executor.submit(task)); + + long currentTime = System.nanoTime(); + List<Future<Map<StackModule, RepoUrlInfoResult>>> results = new ArrayList<>(); + + // !!! first, load the *_urlinfo.json files and block for completion + try { + results = executor.invokeAll(tasks.values(), 2, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.warn("Could not load urlinfo as the executor was interrupted", e); + return; + } finally { + LOG.info("Loaded urlinfo in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - currentTime) + "ms"); } + + List<Map<StackModule, RepoUrlInfoResult>> urlInfoResults = new ArrayList<>(); + // !!! now load all the VDF _by version_ in a new thread. + for (Future<Map<StackModule, RepoUrlInfoResult>> future : results) { + try { + urlInfoResults.add(future.get()); + } catch (Exception e) { + LOG.error("Could not load repo results", e.getCause()); + } + } + + currentTime = System.nanoTime(); + for (Map<StackModule, RepoUrlInfoResult> urlInfoResult : urlInfoResults) { + for (Entry<StackModule, RepoUrlInfoResult> entry : urlInfoResult.entrySet()) { + StackModule stackModule = entry.getKey(); + RepoUrlInfoResult result = entry.getValue(); + + if (null != result) { + if (MapUtils.isNotEmpty(result.getManifest())) { + for (Entry<String, Map<String, URI>> manifestEntry : result.getManifest().entrySet()) { + futures.add(executor.submit(new RepoVdfCallable(stackModule, manifestEntry.getKey(), + manifestEntry.getValue(), m_family))); + } + } + + if (MapUtils.isNotEmpty(result.getLatestVdf())) { + futures.add(executor.submit( + new RepoVdfCallable(stackModule, result.getLatestVdf(), m_family))); + } + } + } + } + executor.shutdown(); + + try { + executor.awaitTermination(2, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.warn("Loading all VDF was interrupted", e.getCause()); + } finally { + LOG.info("Loaded all VDF in " + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - currentTime) + "ms"); + } } /** @@ -157,7 +229,7 @@ public class StackContext { * @return true if all tasks have completed; false otherwise */ public boolean hasCompleted() { - for (Future<Void> f : futures) { + for (Future<?> f : futures) { if (! f.isDone()) { return false; }
