Repository: ambari Updated Branches: refs/heads/branch-2.5 e93c13d0d -> acb6e246f refs/heads/trunk 70d8da7cd -> a5b8230ae
AMBARI-18651. HDP-2.5 installation allows ZKFC to advertise version (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/acb6e246 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/acb6e246 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/acb6e246 Branch: refs/heads/branch-2.5 Commit: acb6e246ffcf53c4583363bc1249a0b43b8ccaa6 Parents: e93c13d Author: Lisnichenko Dmitro <[email protected]> Authored: Wed Oct 26 18:53:16 2016 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Wed Oct 26 18:53:16 2016 +0300 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../ambari/server/events/AmbariEvent.java | 5 + .../server/events/StackUpgradeFinishEvent.java | 41 +++++++ .../upgrade/StackUpgradeFinishListener.java | 83 ++++++++++++++ .../listeners/upgrade/StackVersionListener.java | 53 ++++++--- .../publishers/VersionEventPublisher.java | 9 +- .../upgrades/FinalizeUpgradeAction.java | 8 ++ .../ambari/server/state/ServiceComponent.java | 6 ++ .../server/state/ServiceComponentImpl.java | 41 +++---- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 20 ++++ .../HDP/2.0.6/properties/stack_features.json | 5 + .../stacks/HDP/2.5/services/HDFS/metainfo.xml | 6 ++ .../upgrade/StackUpgradeFinishListenerTest.java | 108 +++++++++++++++++++ .../upgrade/StackVersionListenerTest.java | 47 ++++++-- 14 files changed, 384 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 1396bd8..d1428d4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -101,3 +101,4 @@ class StackFeature: SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support" ATLAS_HBASE_SETUP = "atlas_hbase_setup" RANGER_HIVE_PLUGIN_JDBC_URL = "ranger_hive_plugin_jdbc_url" + ZKFC_VERSION_ADVERTISED = "zkfc_version_advertised" http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java index 912c441..1d3ec39 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java @@ -118,6 +118,11 @@ public abstract class AmbariEvent { SERVICE_COMPONENT_RECOVERY_CHANGED, /** + * Stack upgrade or downgrade finishes + */ + FINALIZE_UPGRADE_FINISH, + + /** * Cluster configuration changed. */ CLUSTER_CONFIG_CHANGED; http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java new file mode 100644 index 0000000..d5745dc --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.events; + +import org.apache.ambari.server.state.Cluster; + +public class StackUpgradeFinishEvent extends ClusterEvent { + + + public Cluster getCluster() { + return cluster; + } + + protected final Cluster cluster; + + /** + * Constructor. + * + * @param cluster + */ + public StackUpgradeFinishEvent(Cluster cluster) { + super(AmbariEventType.FINALIZE_UPGRADE_FINISH, cluster.getClusterId()); + this.cluster = cluster; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java new file mode 100644 index 0000000..b1bffef --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.events.listeners.upgrade; + + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.EagerSingleton; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.events.StackUpgradeFinishEvent; +import org.apache.ambari.server.events.publishers.VersionEventPublisher; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.eventbus.AllowConcurrentEvents; +import com.google.common.eventbus.Subscribe; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; + +/** + * The {@link StackUpgradeFinishListener} class handles updating component info + * after stack upgrade or downgrade finish + */ +@Singleton +@EagerSingleton +public class StackUpgradeFinishListener { + /** + * Logger. + */ + private final static Logger LOG = LoggerFactory.getLogger(StackUpgradeFinishListener.class); + @Inject + Provider<AmbariMetaInfo> ambariMetaInfo; + + /** + * Constructor. + * + * @param eventPublisher the publisher + */ + @Inject + public StackUpgradeFinishListener(VersionEventPublisher eventPublisher) { + eventPublisher.register(this); + } + + @Subscribe + @AllowConcurrentEvents + public void onAmbariEvent(StackUpgradeFinishEvent event) { + LOG.debug("Received event {}", event); + + Cluster cluster = event.getCluster(); + + //update component info due to new stack + for (Service service : cluster.getServices().values()) { + for (ServiceComponent sc : service.getServiceComponents().values()) { + try { + sc.updateComponentInfo(); + } catch (AmbariException e) { + if (LOG.isErrorEnabled()) { + LOG.error("Caught AmbariException when update component info", e); + } + } + } + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index 87247eb..f5a5b0c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@ -22,11 +22,13 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.EagerSingleton; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; import org.apache.ambari.server.events.publishers.VersionEventPublisher; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.State; @@ -38,6 +40,7 @@ import org.slf4j.LoggerFactory; import com.google.common.eventbus.AllowConcurrentEvents; import com.google.common.eventbus.Subscribe; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; /** @@ -66,6 +69,9 @@ public class StackVersionListener { @Inject private RepositoryVersionDAO repositoryVersionDAO; + @Inject + Provider<AmbariMetaInfo> ambariMetaInfo; + /** * Constructor. * @@ -110,8 +116,14 @@ public class StackVersionListener { // Update host component version value if needed try { + AmbariMetaInfo metaInfo = ambariMetaInfo.get(); + ComponentInfo componentInfo = metaInfo.getComponent(cluster.getDesiredStackVersion().getStackName(), + cluster.getDesiredStackVersion().getStackVersion(), sch.getServiceName(), sch.getServiceComponentName()); ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(sch.getServiceComponentName()); - if(!sc.isVersionAdvertised() && StringUtils.isNotBlank(newVersion) + if (componentInfo.isVersionAdvertised() && StringUtils.isNotBlank(newVersion) + && !UNKNOWN_VERSION.equalsIgnoreCase(newVersion)) { + processComponentAdvertisedVersion(cluster, sch, newVersion, sc); + } else if(!sc.isVersionAdvertised() && StringUtils.isNotBlank(newVersion) && !UNKNOWN_VERSION.equalsIgnoreCase(newVersion)) { LOG.error("ServiceComponent {0} doesn't advertise version, " + "however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update", @@ -119,17 +131,8 @@ public class StackVersionListener { } else { if (UNKNOWN_VERSION.equals(sc.getDesiredVersion())) { processUnknownDesiredVersion(cluster, sc, sch, newVersion); - } else if (StringUtils.isNotBlank(newVersion)) { - String previousVersion = sch.getVersion(); - if (previousVersion == null || UNKNOWN_VERSION.equalsIgnoreCase(previousVersion)) { - // value may be "UNKNOWN" when upgrading from older Ambari versions - // or if host component reports it's version for the first time - sch.setUpgradeState(UpgradeState.NONE); - sch.setVersion(newVersion); - bootstrapVersion(cluster, sch); - } else if (!StringUtils.equals(previousVersion, newVersion)) { // - processComponentVersionChange(cluster, sc, sch, newVersion); - } + } else { + processComponentAdvertisedVersion(cluster, sch, newVersion, sc); } } } catch (Exception e) { @@ -142,6 +145,32 @@ public class StackVersionListener { } /** + * Update host component version + * or + * Bootstrap cluster/repo version when version is reported for the first time + * @param cluster target cluster + * @param sch target host component + * @param newVersion advertised version + * @param sc target service component + * @throws AmbariException + */ + private void processComponentAdvertisedVersion(Cluster cluster, ServiceComponentHost sch, String newVersion, ServiceComponent sc) throws AmbariException { + if (StringUtils.isBlank(newVersion)) { + return; + } + String previousVersion = sch.getVersion(); + if (previousVersion == null || UNKNOWN_VERSION.equalsIgnoreCase(previousVersion)) { + // value may be "UNKNOWN" when upgrading from older Ambari versions + // or if host component reports it's version for the first time + sch.setUpgradeState(UpgradeState.NONE); + sch.setVersion(newVersion); + bootstrapVersion(cluster, sch); + } else if (!StringUtils.equals(previousVersion, newVersion)) { + processComponentVersionChange(cluster, sc, sch, newVersion); + } + } + + /** * Bootstrap cluster/repo version when version is reported for the first time * @param cluster target cluster * @param sch target host component http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java index 5b32c4e..710707e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java @@ -20,11 +20,12 @@ package org.apache.ambari.server.events.publishers; import com.google.common.eventbus.EventBus; import com.google.inject.Singleton; -import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; + +import org.apache.ambari.server.events.ClusterEvent; /** * The {@link VersionEventPublisher} is used to publish instances of - * {@link HostComponentVersionAdvertisedEvent} to any {@link com.google.common.eventbus.Subscribe} interested. + * {@link ClusterEvent} to any {@link com.google.common.eventbus.Subscribe} interested. * It uses a single-threaded, serial {@link EventBus}. */ @Singleton @@ -44,11 +45,11 @@ public class VersionEventPublisher { /** * Publishes the specified event to all registered listeners that * {@link com.google.common.eventbus.Subscribe} to any of the - * {@link HostComponentVersionAdvertisedEvent} instances. + * {@link ClusterEvent} instances. * * @param event the event */ - public void publish(HostComponentVersionAdvertisedEvent event) { + public void publish(ClusterEvent event) { m_eventBus.post(event); } http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java index a07d0e6..e73651e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java @@ -32,6 +32,9 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; +import org.apache.ambari.server.events.StackUpgradeFinishEvent; +import org.apache.ambari.server.events.publishers.VersionEventPublisher; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostComponentStateDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; @@ -126,6 +129,9 @@ public class FinalizeUpgradeAction extends AbstractServerAction { @Inject private AmbariMetaInfo ambariMetaInfo; + @Inject + VersionEventPublisher versionEventPublisher; + @Override public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException { @@ -300,6 +306,7 @@ public class FinalizeUpgradeAction extends AbstractServerAction { String.format("Finalizing the version for %d host(s).\n", hostVersionsAllowed.size())); cluster.mapHostVersions(hostsToUpdate, upgradingClusterVersion, RepositoryVersionState.CURRENT); + versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster)); // Reset upgrade state cluster.setUpgradeEntity(null); @@ -447,6 +454,7 @@ public class FinalizeUpgradeAction extends AbstractServerAction { // ensure that when downgrading, we set the desired back to the // original value cluster.setDesiredStackVersion(currentClusterStackId); + versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster)); // Reset upgrade state cluster.setUpgradeEntity(null); http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java index f91a958..e93ab9a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java @@ -59,6 +59,12 @@ public interface ServiceComponent { void setDesiredVersion(String version); + /** + * Refresh Component info due to current stack + * @throws AmbariException + */ + void updateComponentInfo() throws AmbariException; + Map<String, ServiceComponentHost> getServiceComponentHosts(); ServiceComponentHost getServiceComponentHost(String hostname) http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java index f383e80..f9c0eb7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java @@ -60,10 +60,10 @@ public class ServiceComponentImpl implements ServiceComponent { private final Service service; private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); private final String componentName; - private final String displayName; - private final boolean isClientComponent; - private final boolean isMasterComponent; - private final boolean isVersionAdvertised; + private String displayName; + private boolean isClientComponent; + private boolean isMasterComponent; + private boolean isVersionAdvertised; private final ServiceComponentDesiredStateDAO serviceComponentDesiredStateDAO; @@ -73,6 +73,8 @@ public class ServiceComponentImpl implements ServiceComponent { private final AmbariEventPublisher eventPublisher; + private AmbariMetaInfo ambariMetaInfo; + private final ConcurrentMap<String, ServiceComponentHost> hostComponents = new ConcurrentHashMap<String, ServiceComponentHost>(); /** @@ -93,6 +95,7 @@ public class ServiceComponentImpl implements ServiceComponent { StackDAO stackDAO, AmbariEventPublisher eventPublisher) throws AmbariException { + this.ambariMetaInfo = ambariMetaInfo; this.service = service; this.componentName = componentName; this.serviceComponentDesiredStateDAO = serviceComponentDesiredStateDAO; @@ -113,6 +116,14 @@ public class ServiceComponentImpl implements ServiceComponent { desiredStateEntity.setRecoveryEnabled(false); desiredStateEntity.setDesiredStack(stackEntity); + updateComponentInfo(); + + persistEntities(desiredStateEntity); + desiredStateEntityId = desiredStateEntity.getId(); + } + + public void updateComponentInfo() throws AmbariException { + StackId stackId = service.getDesiredStackVersion(); try { ComponentInfo compInfo = ambariMetaInfo.getComponent(stackId.getStackName(), stackId.getStackVersion(), service.getName(), componentName); @@ -128,9 +139,6 @@ public class ServiceComponentImpl implements ServiceComponent { + ", componentName=" + componentName + ", stackInfo=" + stackId.getStackId()); } - - persistEntities(desiredStateEntity); - desiredStateEntityId = desiredStateEntity.getId(); } @AssistedInject @@ -149,27 +157,12 @@ public class ServiceComponentImpl implements ServiceComponent { this.serviceComponentHostFactory = serviceComponentHostFactory; this.stackDAO = stackDAO; this.eventPublisher = eventPublisher; + this.ambariMetaInfo = ambariMetaInfo; desiredStateEntityId = serviceComponentDesiredStateEntity.getId(); componentName = serviceComponentDesiredStateEntity.getComponentName(); - StackId stackId = service.getDesiredStackVersion(); - try { - ComponentInfo compInfo = ambariMetaInfo.getComponent( - stackId.getStackName(), stackId.getStackVersion(), service.getName(), - componentName); - isClientComponent = compInfo.isClient(); - isMasterComponent = compInfo.isMaster(); - isVersionAdvertised = compInfo.isVersionAdvertised(); - displayName = compInfo.getDisplayName(); - } catch (ObjectNotFoundException e) { - throw new AmbariException("Trying to create a ServiceComponent" - + " not recognized in stack info" - + ", clusterName=" + service.getCluster().getClusterName() - + ", serviceName=" + service.getName() - + ", componentName=" + componentName - + ", stackInfo=" + stackId.getStackId()); - } + updateComponentInfo(); for (HostComponentStateEntity hostComponentStateEntity : serviceComponentDesiredStateEntity.getHostComponentStateEntities()) { HostComponentDesiredStateEntityPK pk = new HostComponentDesiredStateEntityPK(); http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index aa0ab0f..9dfe666 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -27,15 +27,26 @@ from resource_management.core.exceptions import Fail from resource_management.core.resources.system import Directory from resource_management.core.resources.service import Service from resource_management.core import shell +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions.security_commons import build_expectations from resource_management.libraries.functions.security_commons import cached_kinit_executor from resource_management.libraries.functions.security_commons import get_params_from_filesystem from resource_management.libraries.functions.security_commons import validate_security_config_properties from resource_management.libraries.functions.security_commons import FILE_TYPE_XML +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.script import Script +from resource_management.libraries.functions.version_select_util import get_component_version class ZkfcSlave(Script): + def get_component_name(self): + import params + if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version): + return "hadoop-hdfs-zkfc" + pass + def install(self, env): import params env.set_params(params) @@ -178,6 +189,15 @@ def initialize_ha_zookeeper(params): Logger.error('HA state initialization in ZooKeeper threw an exception. Reason %s' %(str(ex))) return False + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Stack Upgrade pre-restart") + import params + env.set_params(params) + if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \ + and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-hdfs-zkfc", params.version) + @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class ZkfcSlaveWindows(ZkfcSlave): def start(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index dbde58b..93e7bdf 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -308,6 +308,11 @@ "name": "ranger_hive_plugin_jdbc_url", "description": "Handle Ranger hive repo config jdbc url change for stack 2.5 (AMBARI-18386)", "min_version": "2.5.0.0" + }, + { + "name": "zkfc_version_advertised", + "description": "ZKFC advertise version", + "min_version": "2.5.0.0" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml index a3e4a64..89c128c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml @@ -21,6 +21,12 @@ <service> <name>HDFS</name> <version>2.7.1.2.5</version> + <components> + <component> + <name>ZKFC</name> + <versionAdvertised>true</versionAdvertised> + </component> + </components> </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java new file mode 100644 index 0000000..2bfd176 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.events.listeners.upgrade; + +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.events.StackUpgradeFinishEvent; +import org.apache.ambari.server.events.publishers.VersionEventPublisher; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.UpgradeEntity; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.UpgradeState; +import org.easymock.EasyMockRunner; +import org.easymock.EasyMockSupport; +import org.easymock.TestSubject; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + + + +/** + * StackVersionListener tests. + */ +@RunWith(EasyMockRunner.class) +public class StackUpgradeFinishListenerTest extends EasyMockSupport { + + private static final String INVALID_NEW_VERSION = "1.2.3.4-5678"; + private static final String VALID_NEW_VERSION = "2.4.0.0-1000"; + private static final String SERVICE_COMPONENT_NAME = "Some component name"; + private static final String SERVICE_NAME = "Service name"; + private static final Long CLUSTER_ID = 1L; + private static final String UNKNOWN_VERSION = "UNKNOWN"; + private static final String VALID_PREVIOUS_VERSION = "2.2.0.0"; + private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity(); + private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity(); + public static final String STACK_NAME = "HDP-2.4.0.0"; + public static final String STACK_VERSION = "2.4.0.0"; + + private Cluster cluster; + private ServiceComponentHost sch; + private Service service; + private ServiceComponent serviceComponent; + private VersionEventPublisher publisher = new VersionEventPublisher(); + + @TestSubject + private StackUpgradeFinishListener listener = new StackUpgradeFinishListener(publisher); + + + @Before + public void setup() throws Exception { + cluster = createNiceMock(Cluster.class); + serviceComponent = createNiceMock(ServiceComponent.class); + service = createNiceMock(Service.class); + Map<String, Service> services = new HashMap<>(); + services.put("mock_service",service); + Map<String, ServiceComponent> components = new HashMap<>(); + components.put("mock_component", serviceComponent); + + expect(cluster.getServices()).andReturn(services); + expect(service.getServiceComponents()).andReturn(components); + serviceComponent.updateComponentInfo(); + } + + @Test + public void testupdateComponentInfo() throws AmbariException { + replayAll(); + + sendEventAndVerify(); + } + + + private void sendEventAndVerify() { + StackUpgradeFinishEvent event = new StackUpgradeFinishEvent(cluster); + listener.onAmbariEvent(event); + + verifyAll(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/acb6e246/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java index d22622e..bd9a340 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java @@ -17,31 +17,42 @@ */ package org.apache.ambari.server.events.listeners.upgrade; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import java.lang.reflect.Field; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; import org.apache.ambari.server.events.publishers.VersionEventPublisher; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.UpgradeState; +import org.easymock.EasyMockRunner; import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.easymock.TestSubject; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.inject.Provider; import junit.framework.Assert; /** * StackVersionListener tests. */ +@RunWith(EasyMockRunner.class) public class StackVersionListenerTest extends EasyMockSupport { private static final String INVALID_NEW_VERSION = "1.2.3.4-5678"; @@ -53,12 +64,23 @@ public class StackVersionListenerTest extends EasyMockSupport { private static final String VALID_PREVIOUS_VERSION = "2.2.0.0"; private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity(); private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity(); + public static final String STACK_NAME = "HDP-2.4.0.0"; + public static final String STACK_VERSION = "2.4.0.0"; private Cluster cluster; private ServiceComponentHost sch; private Service service; private ServiceComponent serviceComponent; - private VersionEventPublisher publisher; + private VersionEventPublisher publisher = new VersionEventPublisher(); + private AmbariMetaInfo ambariMetaInfo; + private ComponentInfo componentInfo; + private StackId stackId; + + @TestSubject + private StackVersionListener listener = new StackVersionListener(publisher); + + @Mock + private Provider<AmbariMetaInfo> ambariMetaInfoProvider; @Before public void setup() throws Exception { @@ -66,13 +88,23 @@ public class StackVersionListenerTest extends EasyMockSupport { sch = createNiceMock(ServiceComponentHost.class); service = createNiceMock(Service.class); serviceComponent = createNiceMock(ServiceComponent.class); - publisher = createNiceMock(VersionEventPublisher.class); + componentInfo = createNiceMock(ComponentInfo.class); + stackId = createNiceMock(StackId.class); + ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); + + expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo); + expect(ambariMetaInfo.getComponent(anyString(),anyString(),anyString(),anyString())).andReturn(componentInfo); + + expect(cluster.getDesiredStackVersion()).andReturn(stackId).atLeastOnce(); + expect(stackId.getStackName()).andReturn(STACK_NAME); + expect(stackId.getStackVersion()).andReturn(STACK_VERSION); expect(cluster.getClusterId()).andReturn(CLUSTER_ID); - expect(cluster.getService(SERVICE_NAME)).andReturn(service); - expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent); - expect(sch.getServiceName()).andReturn(SERVICE_NAME); - expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME); + + expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce(); + expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce(); + expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce(); + expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce(); } @Test @@ -269,7 +301,6 @@ public class StackVersionListenerTest extends EasyMockSupport { String newVersion = VALID_NEW_VERSION; HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, sch, newVersion, 1L); - StackVersionListener listener = new StackVersionListener(publisher); // !!! avoid injector for test class Field field = StackVersionListener.class.getDeclaredField("repositoryVersionDAO"); field.setAccessible(true); @@ -298,7 +329,6 @@ public class StackVersionListenerTest extends EasyMockSupport { replayAll(); // !!! avoid injector for test class - StackVersionListener listener = new StackVersionListener(publisher); Field field = StackVersionListener.class.getDeclaredField("repositoryVersionDAO"); field.setAccessible(true); @@ -317,7 +347,6 @@ public class StackVersionListenerTest extends EasyMockSupport { private void sendEventAndVerify(String newVersion) { HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, sch, newVersion); - StackVersionListener listener = new StackVersionListener(publisher); listener.onAmbariEvent(event); verifyAll();
