This is an automated email from the ASF dual-hosted git repository. mpapirkovskyy pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push: new 65ee542 AMBARI-24476. Disable autostart during blueprint deploy. (#2073) 65ee542 is described below commit 65ee5420056d63a6e2fcbbeacca4f92fd1ca09b2 Author: Myroslav Papirkovskyi <mpapirkovs...@apache.org> AuthorDate: Wed Aug 15 12:40:41 2018 +0300 AMBARI-24476. Disable autostart during blueprint deploy. (#2073) * AMBARI-24476. Disable autostart during blueprint deploy. (mpapirkovskyy) * AMBARI-24476. Disable autostart during blueprint deploy. (mpapirkovskyy) * AMBARI-24476. Disable autostart during blueprint deploy. (mpapirkovskyy) * AMBARI-24476. Disable autostart during blueprint deploy. Agent UT fixes. (mpapirkovskyy) --- .../main/python/ambari_agent/RecoveryManager.py | 14 +++++++++ .../test/python/ambari_agent/TestActionQueue.py | 4 +-- .../src/test/python/ambari_agent/TestAlerts.py | 4 +-- .../python/ambari_agent/TestRecoveryManager.py | 16 +++++----- .../ambari/server/agent/ExecutionCommand.java | 1 + .../controller/AmbariManagementControllerImpl.java | 2 ++ .../apache/ambari/server/events/AmbariEvent.java | 5 +++ .../events/ClusterProvisionStartedEvent.java | 35 +++++++++++++++++++++ .../ambari/server/orm/entities/ClusterEntity.java | 14 +++++++++ .../server/state/BlueprintProvisioningState.java | 34 ++++++++++++++++++++ .../org/apache/ambari/server/state/Cluster.java | 4 +++ .../ambari/server/state/cluster/ClusterImpl.java | 36 ++++++++++++++++++++++ .../ambari/server/topology/TopologyManager.java | 2 ++ .../ambari/server/upgrade/UpgradeCatalog271.java | 10 ++++++ .../src/main/resources/Ambari-DDL-Derby-CREATE.sql | 1 + .../src/main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../main/resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../topology/ClusterDeployWithStartOnlyTest.java | 3 ++ ...terInstallWithoutStartOnComponentLevelTest.java | 3 ++ .../topology/ClusterInstallWithoutStartTest.java | 3 ++ .../server/topology/TopologyManagerTest.java | 20 ++++++++++-- .../server/upgrade/UpgradeCatalog271Test.java | 33 ++++++++++++++++++++ 25 files changed, 234 insertions(+), 15 deletions(-) diff --git a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py index 0ce65c9..64ae873 100644 --- a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py +++ b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py @@ -33,6 +33,7 @@ class RecoveryManager: * Generate INSTALL command * Generate START command """ + BLUEPRINT_STATE_IN_PROGRESS = 'IN_PROGRESS' COMMAND_TYPE = "commandType" PAYLOAD_LEVEL = "payloadLevel" SERVICE_NAME = "serviceName" @@ -97,6 +98,7 @@ class RecoveryManager: self.active_command_count = 0 self.cluster_id = None self.initializer_module = initializer_module + self.metadata_cache = initializer_module.metadata_cache self.actions = {} self.update_config(6, 60, 5, 12, recovery_enabled, auto_start_only, auto_install_start) @@ -109,6 +111,14 @@ class RecoveryManager: with self.__active_command_lock: self.active_command_count -= 1 + def is_blueprint_provisioning(self): + try: + blueprint_state = self.metadata_cache[self.cluster_id]['clusterLevelParams']['blueprint_provisioning_state'] + except KeyError: + blueprint_state = 'NONE' + + return blueprint_state == RecoveryManager.BLUEPRINT_STATE_IN_PROGRESS + def has_active_command(self): return self.active_command_count > 0 @@ -640,6 +650,10 @@ class RecoveryManager: logger.info("Recovery is paused, tasks waiting in pipeline for this host.") return None + if self.is_blueprint_provisioning(): + logger.info("Recovery is paused, blueprint is being provisioned.") + return None + if self.enabled(): command_id = self.get_unique_task_id() command = { diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py index 57740b4..ac14da4 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py +++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py @@ -508,7 +508,7 @@ class TestActionQueue(TestCase): initializer_module = InitializerModule() initializer_module.init() initializer_module.config = config - initializer_module.recovery_manager = RecoveryManager(tempfile.mktemp()) + initializer_module.recovery_manager = RecoveryManager(MagicMock()) initializer_module.recovery_manager.update_config(5, 5, 1, 11, True, False, False) with patch("__builtin__.open") as open_mock: @@ -952,7 +952,7 @@ class TestActionQueue(TestCase): get_mock, process_command_mock, gpeo_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() - dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp()) + dummy_controller.recovery_manager = RecoveryManager(MagicMock()) config = MagicMock() gpeo_mock.return_value = 0 config.get_parallel_exec_option = gpeo_mock diff --git a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py index 6fde9b0..47e38f0 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py +++ b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py @@ -113,7 +113,7 @@ class TestAlerts(TestCase): cluster_configuration = self.__get_cluster_configuration() self.__update_cluster_configuration(cluster_configuration, {}) - rm = RecoveryManager(tempfile.mktemp(), True) + rm = RecoveryManager(MagicMock(), True) alert = RecoveryAlert(definition_json, definition_json['source'], self.config, rm) alert.set_helpers(collector, cluster_configuration, MagicMock()) alert.set_cluster("c1", "0", "c6401.ambari.apache.org") @@ -871,7 +871,7 @@ class TestAlerts(TestCase): self.assertEquals(alert._get_reporting_text(alert.RESULT_WARNING), '{0}') self.assertEquals(alert._get_reporting_text(alert.RESULT_CRITICAL), '{0}') - rm = RecoveryManager(tempfile.mktemp()) + rm = RecoveryManager(MagicMock()) definition_json['source']['type'] = 'RECOVERY' alert = RecoveryAlert(definition_json, definition_json['source'], self.config, rm) self.assertEquals(alert._get_reporting_text(alert.RESULT_OK), 'No recovery operations executed for {2}{0}.') diff --git a/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py b/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py index 8195315..e5947cf 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py +++ b/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py @@ -201,7 +201,7 @@ class _TestRecoveryManager(TestCase): self.assertFalse(rm.execute("NODEMANAGER2")) def test_recovery_required(self): - rm = RecoveryManager(True, False) + rm = RecoveryManager(MagicMock(), False) rm.update_config(12, 5, 1, 15, True, False, False, ) rm.update_recovery_config({'recoveryConfig':{'components':[ {'component_name': 'NODEMANAGER', 'service_name': 'YARN', 'desired_state': 'INSTALLED'} @@ -235,7 +235,7 @@ class _TestRecoveryManager(TestCase): rm.update_desired_status("NODEMANAGER", "STARTED") self.assertTrue(rm.requires_recovery("NODEMANAGER")) - rm = RecoveryManager(True, True) + rm = RecoveryManager(MagicMock(), True) rm.update_current_status("NODEMANAGER", "INIT") rm.update_desired_status("NODEMANAGER", "INSTALLED") @@ -251,7 +251,7 @@ class _TestRecoveryManager(TestCase): def test_recovery_required2(self): - rm = RecoveryManager(True, True) + rm = RecoveryManager(MagicMock(), True) rm.update_config(15, 5, 1, 16, True, False, False) rm.update_recovery_config({'recoveryConfig':{'components':[ {'component_name': 'NODEMANAGER', 'service_name': 'YARN', 'desired_state': 'INSTALLED'} @@ -260,7 +260,7 @@ class _TestRecoveryManager(TestCase): rm.update_desired_status("NODEMANAGER", "STARTED") self.assertTrue(rm.requires_recovery("NODEMANAGER")) - rm = RecoveryManager( True, True) + rm = RecoveryManager( MagicMock(), True) rm.update_config(15, 5, 1, 16, True, False, False) rm.update_recovery_config({'recoveryConfig':{'components':[ {'component_name': 'NODEMANAGER', 'service_name': 'YARN', 'desired_state': 'INSTALLED'} @@ -273,7 +273,7 @@ class _TestRecoveryManager(TestCase): rm.update_desired_status("DATANODE", "STARTED") self.assertFalse(rm.requires_recovery("DATANODE")) - rm = RecoveryManager(True, True) + rm = RecoveryManager(MagicMock(), True) rm.update_config(15, 5, 1, 16, True, False, False) rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") @@ -346,7 +346,7 @@ class _TestRecoveryManager(TestCase): time_mock.side_effect = \ [1000, 1001, 1104, 1105, 1106, 1807, 1808, 1809, 1810, 1811, 1812] - rm = RecoveryManager(True) + rm = RecoveryManager(MagicMock()) rm.update_config(5, 5, 0, 11, True, False, False) command1 = copy.deepcopy(self.command) @@ -428,7 +428,7 @@ class _TestRecoveryManager(TestCase): def test_reset_if_window_passed_since_last_attempt(self, time_mock): time_mock.side_effect = \ [1000, 1071, 1372] - rm = RecoveryManager(True) + rm = RecoveryManager(MagicMock()) rm.update_config(2, 5, 1, 4, True, True, False) @@ -447,7 +447,7 @@ class _TestRecoveryManager(TestCase): @patch.object(RecoveryManager, "_now_") def test_is_action_info_stale(self, time_mock): - rm = RecoveryManager(True) + rm = RecoveryManager(MagicMock()) rm.update_config(5, 60, 5, 16, True, False, False) time_mock.return_value = 0 diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 7d9964c..ea994ff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -550,6 +550,7 @@ public class ExecutionCommand extends AgentCommand { String USER_LIST = "user_list"; String GROUP_LIST = "group_list"; String USER_GROUPS = "user_groups"; + String BLUEPRINT_PROVISIONING_STATE = "blueprint_provisioning_state"; String NOT_MANAGED_HDFS_PATH_LIST = "not_managed_hdfs_path_list"; String REFRESH_TOPOLOGY = "refresh_topology"; String HOST_SYS_PREPPED = "host_sys_prepped"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 8e7196f..a5cee17 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -24,6 +24,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.BLUEPRINT_PROVISIONING_STATE; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLUSTER_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; @@ -5743,6 +5744,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle clusterLevelParams.putAll(getMetadataClusterLevelConfigsParams(cluster, stackId)); clusterLevelParams.put(CLUSTER_NAME, cluster.getClusterName()); clusterLevelParams.put(HOOKS_FOLDER, configs.getProperty(Configuration.HOOKS_FOLDER)); + clusterLevelParams.put(BLUEPRINT_PROVISIONING_STATE, cluster.getBlueprintProvisioningState().toString()); return clusterLevelParams; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java index 1d12377..610f50d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java @@ -118,6 +118,11 @@ public abstract class AmbariEvent { CLUSTER_PROVISIONED, /** + * The cluster provision was started. + */ + CLUSTER_PROVISION_STARTED, + + /** * The service component recovery enabled field changed. */ SERVICE_COMPONENT_RECOVERY_CHANGED, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterProvisionStartedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterProvisionStartedEvent.java new file mode 100644 index 0000000..e5fff17 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/ClusterProvisionStartedEvent.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.events; + +/** + * Fired when cluster provisioning was started (only for blueprints currently). + */ +public class ClusterProvisionStartedEvent extends AmbariEvent { + + private final long clusterId; + + public ClusterProvisionStartedEvent(long clusterId) { + super(AmbariEventType.CLUSTER_PROVISION_STARTED); + this.clusterId = clusterId; + } + + public long getClusterId() { + return clusterId; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java index c22449c..fa94d7a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterEntity.java @@ -42,6 +42,7 @@ import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.TableGenerator; +import org.apache.ambari.server.state.BlueprintProvisioningState; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.State; @@ -95,6 +96,11 @@ public class ClusterEntity { @Column(name = "cluster_info", insertable = true, updatable = true) private String clusterInfo = ""; + @Basic + @Enumerated(value = EnumType.STRING) + @Column(name = "blueprint_provisioning_state", insertable = true, updatable = true) + private BlueprintProvisioningState blueprintProvisioningState = BlueprintProvisioningState.NONE; + /** * Unidirectional one-to-one association to {@link StackEntity} */ @@ -347,4 +353,12 @@ public class ClusterEntity { public void setUpgradeEntity(UpgradeEntity upgradeEntity) { this.upgradeEntity = upgradeEntity; } + + public BlueprintProvisioningState getBlueprintProvisioningState() { + return blueprintProvisioningState; + } + + public void setBlueprintProvisioningState(BlueprintProvisioningState blueprintProvisioningState) { + this.blueprintProvisioningState = blueprintProvisioningState; + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/BlueprintProvisioningState.java b/ambari-server/src/main/java/org/apache/ambari/server/state/BlueprintProvisioningState.java new file mode 100644 index 0000000..bbba05a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/BlueprintProvisioningState.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.state; + +public enum BlueprintProvisioningState { + /** + * Initial state. + */ + NONE, + /** + * Blueprint provisioning in progress. + */ + IN_PROGRESS, + /** + * Blueprint provisioning was completed. + */ + FINISHED +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index c201310..1b0e4b4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -245,6 +245,10 @@ public interface Cluster { */ void setProvisioningState(State provisioningState); + BlueprintProvisioningState getBlueprintProvisioningState(); + + void setBlueprintProvisioningState(BlueprintProvisioningState blueprintProvisioningState); + /** * Gets the cluster's security type. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 821bdbc..eafb863 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -56,6 +56,7 @@ import org.apache.ambari.server.ServiceComponentHostNotFoundException; import org.apache.ambari.server.ServiceComponentNotFoundException; import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; +import org.apache.ambari.server.agent.stomp.MetadataHolder; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariSessionManager; @@ -69,6 +70,7 @@ import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMet import org.apache.ambari.server.events.AmbariEvent.AmbariEventType; import org.apache.ambari.server.events.ClusterConfigChangedEvent; import org.apache.ambari.server.events.ClusterEvent; +import org.apache.ambari.server.events.ClusterProvisionStartedEvent; import org.apache.ambari.server.events.ClusterProvisionedEvent; import org.apache.ambari.server.events.ConfigsUpdateEvent; import org.apache.ambari.server.events.jpa.EntityManagerCacheInvalidationEvent; @@ -111,6 +113,7 @@ import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.orm.entities.TopologyRequestEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; import org.apache.ambari.server.security.authorization.AuthorizationException; +import org.apache.ambari.server.state.BlueprintProvisioningState; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ClusterHealthReport; import org.apache.ambari.server.state.Clusters; @@ -280,6 +283,9 @@ public class ClusterImpl implements Cluster { @Inject private STOMPComponentsDeleteHandler STOMPComponentsDeleteHandler; + @Inject + private MetadataHolder metadataHolder; + /** * Data access object used for looking up stacks from the database. */ @@ -967,6 +973,19 @@ public class ClusterImpl implements Cluster { } @Override + public BlueprintProvisioningState getBlueprintProvisioningState() { + ClusterEntity clusterEntity = getClusterEntity(); + return clusterEntity.getBlueprintProvisioningState(); + } + + @Override + public void setBlueprintProvisioningState(BlueprintProvisioningState blueprintProvisioningState) { + ClusterEntity clusterEntity = getClusterEntity(); + clusterEntity.setBlueprintProvisioningState(blueprintProvisioningState); + clusterEntity = clusterDAO.merge(clusterEntity); + } + + @Override public SecurityType getSecurityType() { SecurityType securityType = null; ClusterEntity clusterEntity = getClusterEntity(); @@ -2791,6 +2810,23 @@ public class ClusterImpl implements Cluster { LOG.warn("Failed to remove temporary configurations: {} / {}", e.getKey(), e.getValue(), ex); } } + changeBlueprintProvisioningState(BlueprintProvisioningState.FINISHED); + } + } + + @Subscribe + public void onClusterProvisionStarted(ClusterProvisionStartedEvent event) { + if (event.getClusterId() == getClusterId()) { + changeBlueprintProvisioningState(BlueprintProvisioningState.IN_PROGRESS); + } + } + + private void changeBlueprintProvisioningState(BlueprintProvisioningState newState) { + setBlueprintProvisioningState(newState); + try { + metadataHolder.updateData(controller.getClusterMetadataOnConfigsUpdate(this)); + } catch (AmbariException e) { + LOG.error("Metadata update failed after setting blueprint provision state to {}", newState, e); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 36f1ad0..7ca1c1d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -61,6 +61,7 @@ import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.events.AmbariEvent; import org.apache.ambari.server.events.ClusterConfigFinishedEvent; +import org.apache.ambari.server.events.ClusterProvisionStartedEvent; import org.apache.ambari.server.events.ClusterProvisionedEvent; import org.apache.ambari.server.events.HostsRemovedEvent; import org.apache.ambari.server.events.RequestFinishedEvent; @@ -351,6 +352,7 @@ public class TopologyManager { ambariContext.persistInstallStateForUI(clusterName, stack.getName(), stack.getVersion()); clusterProvisionWithBlueprintCreateRequests.put(clusterId, logicalRequest); + ambariEventPublisher.publish(new ClusterProvisionStartedEvent(clusterId)); return getRequestStatus(logicalRequest.getRequestId()); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java index b8abb3c..854b358 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java @@ -35,6 +35,7 @@ import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.dao.DaoUtils; import org.apache.ambari.server.orm.entities.ServiceConfigEntity; +import org.apache.ambari.server.state.BlueprintProvisioningState; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -55,6 +56,8 @@ public class UpgradeCatalog271 extends AbstractUpgradeCatalog { private static final String SERVICE_CONFIG_MAPPING_TABLE = "serviceconfigmapping"; private static final String CLUSTER_CONFIG_TABLE = "clusterconfig"; + protected static final String CLUSTERS_TABLE = "clusters"; + protected static final String CLUSTERS_BLUEPRINT_PROVISIONING_STATE_COLUMN = "blueprint_provisioning_state"; @Inject DaoUtils daoUtils; @@ -91,6 +94,7 @@ public class UpgradeCatalog271 extends AbstractUpgradeCatalog { */ @Override protected void executeDDLUpdates() throws AmbariException, SQLException { + addBlueprintProvisioningState(); } /** @@ -260,4 +264,10 @@ public class UpgradeCatalog271 extends AbstractUpgradeCatalog { dba.executeQuery(serviceConfigMappingRemoveSQL); dba.executeQuery(clusterConfigRemoveSQL); } + + protected void addBlueprintProvisioningState() throws SQLException { + dbAccessor.addColumn(CLUSTERS_TABLE, + new DBAccessor.DBColumnInfo(CLUSTERS_BLUEPRINT_PROVISIONING_STATE_COLUMN, String.class, 255, + BlueprintProvisioningState.NONE, true)); + } } \ No newline at end of file diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 66f4701..680c8fa 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -58,6 +58,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR(255) NOT NULL, cluster_name VARCHAR(100) NOT NULL UNIQUE, provisioning_state VARCHAR(255) NOT NULL DEFAULT 'INIT', + blueprint_provisioning_state VARCHAR(255) DEFAULT 'NONE', security_type VARCHAR(32) NOT NULL DEFAULT 'NONE', desired_cluster_state VARCHAR(255) NOT NULL, desired_stack_id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index e028957..143669e 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -78,6 +78,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR(255) NOT NULL, cluster_name VARCHAR(100) NOT NULL UNIQUE, provisioning_state VARCHAR(255) NOT NULL DEFAULT 'INIT', + blueprint_provisioning_state VARCHAR(255) DEFAULT 'NONE', security_type VARCHAR(32) NOT NULL DEFAULT 'NONE', desired_cluster_state VARCHAR(255) NOT NULL, desired_stack_id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 3a54f33..90d6f9b 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -58,6 +58,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR2(255) NULL, cluster_name VARCHAR2(100) NOT NULL UNIQUE, provisioning_state VARCHAR2(255) DEFAULT 'INIT' NOT NULL, + blueprint_provisioning_state VARCHAR2(255) DEFAULT 'NONE', security_type VARCHAR2(32) DEFAULT 'NONE' NOT NULL, desired_cluster_state VARCHAR2(255) NULL, desired_stack_id NUMBER(19) NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 1611130..702e9d3 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -58,6 +58,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR(255) NOT NULL, cluster_name VARCHAR(100) NOT NULL UNIQUE, provisioning_state VARCHAR(255) NOT NULL DEFAULT 'INIT', + blueprint_provisioning_state VARCHAR(255) DEFAULT 'NONE', security_type VARCHAR(32) NOT NULL DEFAULT 'NONE', desired_cluster_state VARCHAR(255) NOT NULL, desired_stack_id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index bf6f63d..dae27b0 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -57,6 +57,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR(255) NOT NULL, cluster_name VARCHAR(100) NOT NULL UNIQUE, provisioning_state VARCHAR(255) NOT NULL DEFAULT 'INIT', + blueprint_provisioning_state VARCHAR(255) DEFAULT 'NONE', security_type VARCHAR(32) NOT NULL DEFAULT 'NONE', desired_cluster_state VARCHAR(255) NOT NULL, desired_stack_id NUMERIC(19) NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 03fe222..0a6fc30 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -71,6 +71,7 @@ CREATE TABLE clusters ( cluster_info VARCHAR(255) NOT NULL, cluster_name VARCHAR(100) NOT NULL UNIQUE, provisioning_state VARCHAR(255) NOT NULL DEFAULT 'INIT', + blueprint_provisioning_state VARCHAR(255) DEFAULT 'NONE', security_type VARCHAR(32) NOT NULL DEFAULT 'NONE', desired_cluster_state VARCHAR(255) NOT NULL, desired_stack_id BIGINT NOT NULL, diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index aecc6cb..3a98ab0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -55,6 +55,7 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.entities.TopologyLogicalRequestEntity; import org.apache.ambari.server.security.encryption.CredentialStoreService; import org.apache.ambari.server.state.Cluster; @@ -161,6 +162,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock(type = MockType.NICE) + private AmbariEventPublisher eventPublisher; private final Configuration stackConfig = new Configuration(new HashMap<>(), diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index a4b2160..46e3ff9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -56,6 +56,7 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.entities.TopologyLogicalRequestEntity; import org.apache.ambari.server.security.encryption.CredentialStoreService; import org.apache.ambari.server.state.Cluster; @@ -158,6 +159,8 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock(type = MockType.NICE) + private AmbariEventPublisher eventPublisher; private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index d89c8ca..61face9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -56,6 +56,7 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.entities.TopologyLogicalRequestEntity; import org.apache.ambari.server.security.encryption.CredentialStoreService; import org.apache.ambari.server.state.Cluster; @@ -160,6 +161,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock(type = MockType.NICE) + private AmbariEventPublisher eventPublisher; private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 34b7828..cc9e7e9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -58,6 +58,8 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.events.ClusterProvisionStartedEvent; +import org.apache.ambari.server.events.ClusterProvisionedEvent; import org.apache.ambari.server.events.RequestFinishedEvent; import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.dao.SettingDAO; @@ -378,12 +380,12 @@ public class TopologyManagerTest { PowerMock.verify(System.class); verify(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, - requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); + requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO, eventPublisher); PowerMock.reset(System.class); reset(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, - requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); + requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO, eventPublisher); } @Test @@ -396,6 +398,16 @@ public class TopologyManagerTest { } @Test + public void testBlueprintProvisioningStateEvent() throws Exception { + expect(persistedState.getAllRequests()).andReturn(Collections.emptyMap()).anyTimes(); + eventPublisher.publish(anyObject(ClusterProvisionStartedEvent.class)); + expectLastCall().once(); + replayAll(); + + topologyManager.provisionCluster(request); + } + + @Test public void testAddKerberosClientAtTopologyInit() throws Exception { Map<ClusterTopology, List<LogicalRequest>> allRequests = new HashMap<>(); List<LogicalRequest> requestList = new ArrayList<>(); @@ -437,6 +449,8 @@ public class TopologyManagerTest { expect(persistedState.getProvisionRequest(CLUSTER_ID)).andReturn(logicalRequest).anyTimes(); expect(logicalRequest.isFinished()).andReturn(true).anyTimes(); expect(logicalRequest.isSuccessful()).andReturn(true).anyTimes(); + eventPublisher.publish(anyObject(ClusterProvisionedEvent.class)); + expectLastCall().once(); replayAll(); topologyManager.provisionCluster(request); requestFinished(); @@ -541,7 +555,7 @@ public class TopologyManagerTest { configurationRequest, configurationRequest2, configurationRequest3, executor, persistedState, clusterTopologyMock, securityConfigurationFactory, credentialStoreService, clusterController, resourceProvider, mockFuture, requestStatusResponse, logicalRequest, settingDAO, - configureClusterTaskFactory, configureClusterTask); + configureClusterTaskFactory, configureClusterTask, eventPublisher); } @Test(expected = InvalidTopologyException.class) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java index a1780d7..63247da 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java @@ -18,13 +18,17 @@ package org.apache.ambari.server.upgrade; +import static org.apache.ambari.server.upgrade.UpgradeCatalog271.CLUSTERS_BLUEPRINT_PROVISIONING_STATE_COLUMN; +import static org.apache.ambari.server.upgrade.UpgradeCatalog271.CLUSTERS_TABLE; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.createMockBuilder; import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.newCapture; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.startsWith; import static org.easymock.EasyMock.verify; @@ -37,12 +41,14 @@ import java.util.Map; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariManagementControllerImpl; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.state.BlueprintProvisioningState; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.StackId; import org.easymock.Capture; +import org.easymock.CaptureType; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.Assert; @@ -53,6 +59,33 @@ import com.google.inject.Injector; public class UpgradeCatalog271Test { @Test + public void testExecuteDDLUpdates() throws Exception { + EasyMockSupport easyMockSupport = new EasyMockSupport(); + Injector injector = easyMockSupport.createNiceMock(Injector.class); + DBAccessor dbAccessor = easyMockSupport.createNiceMock(DBAccessor.class); + + Capture<DBAccessor.DBColumnInfo> blueprintProvisioningStateColumnCapture = newCapture(CaptureType.ALL); + dbAccessor.addColumn(eq(CLUSTERS_TABLE), capture(blueprintProvisioningStateColumnCapture)); + expectLastCall().once(); + + replay(dbAccessor, injector); + + UpgradeCatalog271 upgradeCatalog271 = new UpgradeCatalog271(injector); + upgradeCatalog271.dbAccessor = dbAccessor; + upgradeCatalog271.executeDDLUpdates(); + + DBAccessor.DBColumnInfo capturedBlueprintProvisioningStateColumn = + blueprintProvisioningStateColumnCapture.getValue(); + Assert.assertEquals(CLUSTERS_BLUEPRINT_PROVISIONING_STATE_COLUMN, + capturedBlueprintProvisioningStateColumn.getName()); + Assert.assertEquals(BlueprintProvisioningState.NONE, capturedBlueprintProvisioningStateColumn.getDefaultValue()); + Assert.assertEquals(String.class, capturedBlueprintProvisioningStateColumn.getType()); + + easyMockSupport.verifyAll(); + + } + + @Test public void testExecuteDMLUpdates() throws Exception { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); Method updateRangerLogDirConfigs = UpgradeCatalog271.class.getDeclaredMethod("updateRangerLogDirConfigs");