AMBARI-21526 - ZKFC Doesn't Update its Version On Some Upgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/963f2b4c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/963f2b4c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/963f2b4c Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 963f2b4cf2793abf40625de5da0ab489c28ea72e Parents: fb2c202 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed Jul 19 16:49:01 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Wed Jul 19 20:05:01 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/stack_features.py | 2 +- .../listeners/upgrade/StackVersionListener.java | 31 ++++++++++-- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 8 +--- .../upgrade/StackVersionListenerTest.java | 50 +++++++++++++------- 4 files changed, 62 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 2c66728..3fcce82 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -122,7 +122,7 @@ def get_stack_feature_version(config): # if this is not an upgrade, then we take the simple path if upgrade_direction is None: Logger.info( - "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format( + "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3} -> {4}".format( stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks)) return version_for_stack_feature_checks http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java index 3af5520..4183b51 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java @@ -22,6 +22,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.EagerSingleton; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; import org.apache.ambari.server.events.publishers.VersionEventPublisher; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; @@ -29,8 +30,10 @@ import org.apache.ambari.server.orm.entities.ClusterVersionEntity; import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.UpgradeState; import org.apache.commons.lang.StringUtils; @@ -39,6 +42,7 @@ import org.slf4j.LoggerFactory; import com.google.common.eventbus.Subscribe; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; /** @@ -66,6 +70,13 @@ public class StackVersionListener { private RepositoryVersionDAO repositoryVersionDAO; /** + * Used for looking up a component's advertising version status given a stack + * and name. + */ + @Inject + private Provider<AmbariMetaInfo> ambariMetaInfoProvider; + + /** * Constructor. * * @param eventPublisher the publisher @@ -108,20 +119,30 @@ public class StackVersionListener { // Update host component version value if needed try { - ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent( - sch.getServiceComponentName()); + // get the component information for the desired stack; if a component + // moves from UNKNOWN to providing a version, we must do the version + // advertised check against the target stack + StackId desiredStackId = sch.getDesiredStackVersion(); + + AmbariMetaInfo ambariMetaInfo = ambariMetaInfoProvider.get(); + ComponentInfo componentInfo = ambariMetaInfo.getComponent(desiredStackId.getStackName(), + desiredStackId.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName()); // not advertising a version, do nothing - if (!sc.isVersionAdvertised()) { + if (!componentInfo.isVersionAdvertised()) { // that's odd; a version came back - log it and still do nothing if (!StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) { - LOG.debug( + LOG.warn( "ServiceComponent {} doesn't advertise version, however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update", - sc.getName(), sch.getServiceComponentName(), sch.getHostName(), newVersion); + sch.getServiceComponentName(), sch.getServiceComponentName(), sch.getHostName(), + newVersion); } return; } + ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent( + sch.getServiceComponentName()); + boolean desiredVersionIsCurrentlyUnknown = StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, sc.getDesiredVersion()); http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index dff1548..db68544 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -30,11 +30,6 @@ from resource_management.core import shell from resource_management.libraries.functions import conf_select, stack_select from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.security_commons import build_expectations -from resource_management.libraries.functions.security_commons import cached_kinit_executor -from resource_management.libraries.functions.security_commons import get_params_from_filesystem -from resource_management.libraries.functions.security_commons import validate_security_config_properties -from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.script import Script from resource_management.core.resources.zkmigrator import ZkMigrator @@ -140,8 +135,7 @@ class ZkfcSlaveDefault(ZkfcSlave): Logger.info("Executing Stack Upgrade pre-restart") import params env.set_params(params) - if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \ - and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + if check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version_for_stack_feature_checks): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-zkfc", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java index f20c46c..3a4f5b9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java @@ -32,6 +32,7 @@ import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; @@ -81,6 +82,12 @@ public class StackVersionListenerTest extends EasyMockSupport { @Mock private Provider<AmbariMetaInfo> ambariMetaInfoProvider; + @Mock + private ComponentInfo componentInfo; + + @Mock + private AmbariMetaInfo ambariMetaInfo; + @Before public void setup() throws Exception { cluster = createNiceMock(Cluster.class); @@ -89,22 +96,30 @@ public class StackVersionListenerTest extends EasyMockSupport { serviceComponent = createNiceMock(ServiceComponent.class); stackId = createNiceMock(StackId.class); + expect(cluster.getService(SERVICE_NAME)).andReturn(service).anyTimes(); expect(cluster.getDesiredStackVersion()).andReturn(stackId).anyTimes(); expect(stackId.getStackName()).andReturn(STACK_NAME).anyTimes(); expect(stackId.getStackVersion()).andReturn(STACK_VERSION).anyTimes(); expect(cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce(); - expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce(); - expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce(); + expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).anyTimes(); + expect(serviceComponent.getDesiredStackVersion()).andReturn(stackId).anyTimes(); + expect(sch.getDesiredStackVersion()).andReturn(stackId).atLeastOnce(); expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce(); expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce(); + + expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo).atLeastOnce(); + expect(ambariMetaInfo.getComponent(STACK_NAME, STACK_VERSION, SERVICE_NAME, + SERVICE_COMPONENT_NAME)).andReturn(componentInfo).atLeastOnce(); + + injectMocks(listener); } @Test public void testRecalculateHostVersionStateWhenVersionIsNullAndNewVersionIsNotBlank() throws AmbariException { expect(sch.getVersion()).andReturn(null).atLeastOnce(); expect(serviceComponent.getDesiredVersion()).andReturn(INVALID_NEW_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setVersion(INVALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(null).once(); @@ -117,7 +132,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateHostVersionStateWhenVersionIsUnknownAndNewVersionIsNotBlank() throws AmbariException { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setVersion(INVALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(null).once(); @@ -130,7 +145,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionIsNullAndNewVersionIsValid() throws AmbariException { expect(sch.getVersion()).andReturn(null).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setVersion(VALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once(); @@ -145,7 +160,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionIsUnknownAndNewVersionIsValid() throws AmbariException { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setVersion(VALID_NEW_VERSION); expectLastCall().once(); expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once(); @@ -160,7 +175,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateHostVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsNotValid() throws AmbariException { expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); serviceComponent.setDesiredVersion(INVALID_NEW_VERSION); expectLastCall().once(); sch.setUpgradeState(UpgradeState.NONE); @@ -177,7 +192,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsValid() throws AmbariException { expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); serviceComponent.setDesiredVersion(VALID_NEW_VERSION); expectLastCall().once(); sch.setUpgradeState(UpgradeState.NONE); @@ -195,7 +210,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testRecalculateClusterVersionStateWhenVersionNotAdvertised() throws AmbariException { - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.FALSE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(false).atLeastOnce(); replayAll(); sendEventAndVerify(VALID_NEW_VERSION); } @@ -203,7 +218,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testNoActionTakenOnNullVersion() { - expect(serviceComponent.isVersionAdvertised()).andReturn(true); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); resetAll(); replayAll(); @@ -215,7 +230,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(cluster.getUpgradeInProgress()).andReturn(EasyMock.niceMock(UpgradeEntity.class)).atLeastOnce(); expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce(); expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setUpgradeState(UpgradeState.COMPLETE); expectLastCall().once(); @@ -230,7 +245,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class)).atLeastOnce(); expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce(); expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setUpgradeState(UpgradeState.VERSION_MISMATCH); expectLastCall().once(); @@ -246,7 +261,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce(); expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY).atLeastOnce(); expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setUpgradeState(UpgradeState.COMPLETE); expectLastCall().once(); @@ -260,7 +275,7 @@ public class StackVersionListenerTest extends EasyMockSupport { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce(); expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce(); expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); sch.setUpgradeState(UpgradeState.NONE); expectLastCall().once(); @@ -276,7 +291,7 @@ public class StackVersionListenerTest extends EasyMockSupport { public void testSetUpgradeStateToVersionMismatchByDefaultWhenHostAndNewVersionsAreValid() { expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce(); expect(serviceComponent.getDesiredVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); // !!! VERSION_MISMATCH might need to be allowed to happen when not in an // upgrade @@ -295,7 +310,7 @@ public class StackVersionListenerTest extends EasyMockSupport { @Test public void testSetRepositoryVersion() throws Exception { expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); - expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); RepositoryVersionDAO dao = createNiceMock(RepositoryVersionDAO.class); RepositoryVersionEntity entity = createNiceMock(RepositoryVersionEntity.class); @@ -327,6 +342,9 @@ public class StackVersionListenerTest extends EasyMockSupport { */ @Test public void testRepositoryVersionNotSetDuringUpgrade() throws Exception { + expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce(); + expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce(); + // this call will make it seem like there is an upgrade in progress expect(cluster.getUpgradeInProgress()).andReturn( createNiceMock(UpgradeEntity.class)).atLeastOnce();
