Repository: ambari Updated Branches: refs/heads/trunk 060ae2cdb -> 08dc84acb
AMBARI-4894. Allow upgrade stack to take URL as a parameter (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/08dc84ac Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/08dc84ac Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/08dc84ac Branch: refs/heads/trunk Commit: 08dc84acb4e29fdd657a9f0d6937005ef8e5ece7 Parents: 060ae2c Author: Nate Cole <[email protected]> Authored: Fri Feb 28 15:26:11 2014 -0500 Committer: Nate Cole <[email protected]> Committed: Fri Feb 28 16:13:55 2014 -0500 ---------------------------------------------------------------------- .../server/upgrade/StackUpgradeHelper.java | 49 +++++--- .../ambari/server/upgrade/StackUpgradeUtil.java | 59 ++++++++- ambari-server/src/main/python/ambari-server.py | 20 ++- .../server/upgrade/StackUpgradeUtilTest.java | 123 +++++++++++++++++++ .../src/test/python/TestAmbariServer.py | 48 +++++++- 5 files changed, 269 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java index b9fec20..77ec069 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java @@ -17,25 +17,28 @@ */ package org.apache.ambari.server.upgrade; -import com.google.gson.Gson; -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.persist.PersistService; -import com.google.inject.persist.Transactional; -import org.apache.ambari.server.controller.ControllerModule; -import org.apache.ambari.server.orm.DBAccessor; -import org.apache.ambari.server.orm.dao.MetainfoDAO; -import org.apache.ambari.server.orm.entities.MetainfoEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.ArrayList; import java.util.InputMismatchException; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; + +import javax.ws.rs.core.UriInfo; + +import org.apache.ambari.server.controller.ControllerModule; +import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.dao.MetainfoDAO; +import org.apache.ambari.server.orm.entities.MetainfoEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; +import com.google.inject.persist.Transactional; public class StackUpgradeHelper { private static final Logger LOG = LoggerFactory.getLogger @@ -88,14 +91,18 @@ public class StackUpgradeHelper { /** * Change the stack id in the Ambari DB. - * @param stackId + * @param stackInfo * @throws SQLException */ - private void updateStackVersion(Map<String, String> stackId) throws SQLException { - if (stackId == null || stackId.isEmpty()) { - throw new IllegalArgumentException("Empty stack id. " + stackId); + public void updateStackVersion(Map<String, String> stackInfo) throws Exception { + if (stackInfo == null || stackInfo.isEmpty()) { + throw new IllegalArgumentException("Empty stack id. " + stackInfo); } - Iterator<Map.Entry<String, String>> stackIdEntry = stackId.entrySet().iterator(); + + String repoUrl = stackInfo.remove("repo_url"); + String repoUrlOs = stackInfo.remove("repo_url_os"); + + Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator(); Map.Entry<String, String> stackEntry = stackIdEntry.next(); String stackName = stackEntry.getKey(); @@ -105,6 +112,10 @@ public class StackUpgradeHelper { "stackVersion = "+ stackVersion); stackUpgradeUtil.updateStackDetails(stackName, stackVersion); + + if (null != repoUrl) { + stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs); + } dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'"); } @@ -139,11 +150,13 @@ public class StackUpgradeHelper { "actions: " + stackUpgradeHelper.getValidActions()); } + stackUpgradeHelper.startPersistenceService(); Map values = gson.fromJson(valueMap, Map.class); if (action.equals(STACK_ID_UPDATE_ACTION)) { stackUpgradeHelper.updateStackVersion(values); + } else if (action.equals(METAINFO_UPDATE_ACTION)) { stackUpgradeHelper.updateMetaInfo(values); http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java index 75189cc..55697dc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java @@ -17,10 +17,14 @@ */ package org.apache.ambari.server.upgrade; -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.persist.Transactional; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Scanner; +import java.util.Set; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.ClusterStateDAO; import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO; @@ -33,9 +37,13 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity; +import org.apache.ambari.server.state.OperatingSystemInfo; import org.apache.ambari.server.state.StackId; -import java.util.ArrayList; -import java.util.List; + +import com.google.gson.Gson; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.persist.Transactional; public class StackUpgradeUtil { @Inject @@ -139,4 +147,43 @@ public class StackUpgradeUtil { } + + /** + * @param stackName + * @param stackVersion + * @param localRepo + */ + public void updateLocalRepo(String stackName, String stackVersion, + String repoUrl, String repoUrlOs) throws Exception { + + if (null == repoUrl || + repoUrl.isEmpty() || + !repoUrl.startsWith("http")) + return; + + String server = repoUrl; + + String[] oses = new String[0]; + + if (null != repoUrlOs) { + oses = repoUrlOs.split(","); + } + + AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class); + + if (0 == oses.length) { + // do them all + for (OperatingSystemInfo osi : ami.getOperatingSystems(stackName, stackVersion)) { + ami.updateRepoBaseURL(stackName, stackVersion, osi.getOsType(), + stackName + "-" + stackVersion, server); + } + + } else { + for (String os : oses) { + ami.updateRepoBaseURL(stackName, stackVersion, os, + stackName + "-" + stackVersion, server); + } + } + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 30b778c..c4fa246 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -2487,7 +2487,7 @@ def stop(args): ### Stack upgrade ### -def upgrade_stack(args, stack_id): +def upgrade_stack(args, stack_id, repo_url = None, repo_url_os = None): if not is_root(): err = 'Ambari-server upgradestack should be run with ' \ 'root-level privileges' @@ -2495,7 +2495,7 @@ def upgrade_stack(args, stack_id): check_database_name_property() stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP) - retcode = run_stack_upgrade(stack_name, stack_version) + retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os) if not retcode == 0: raise FatalException(retcode, 'Stack upgrade failed.') @@ -2606,7 +2606,7 @@ def run_schema_upgrade(version): print_error_msg("Error executing schema upgrade, please check the server logs.") return retcode -def run_stack_upgrade(stackName, stackVersion): +def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os): jdk_path = find_jdk() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " @@ -2615,6 +2615,10 @@ def run_stack_upgrade(stackName, stackVersion): return 1 stackId = {} stackId[stackName] = stackVersion + if repo_url is not None: + stackId['repo_url'] = repo_url + if repo_url_os is not None: + stackId['repo_url_os'] = repo_url_os command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(), "updateStackId", @@ -4070,7 +4074,15 @@ def main(): upgrade(options) elif action == UPGRADE_STACK_ACTION: stack_id = args[1] - upgrade_stack(options, stack_id) + repo_url = None + repo_url_os = None + + if len(args) > 2: + repo_url = args[2] + if len(args) > 3: + repo_url_os = args[3] + + upgrade_stack(options, stack_id, repo_url, repo_url_os) elif action == LDAP_SETUP_ACTION: setup_ldap() elif action == SETUP_SECURITY_ACTION: http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java new file mode 100644 index 0000000..5cda29c --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.upgrade; + +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.orm.GuiceJpaInitializer; +import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.orm.dao.MetainfoDAO; +import org.apache.ambari.server.orm.entities.MetainfoEntity; +import org.apache.ambari.server.state.RepositoryInfo; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; + +/** + * Tests the StackUpgradeHelper + */ +public class StackUpgradeUtilTest { + + private Injector injector; + + @Before + public void setup() throws Exception { + injector = Guice.createInjector(new InMemoryDefaultTestModule()); + injector.getInstance(GuiceJpaInitializer.class); + } + + @After + public void teardown() throws Exception { + injector.getInstance(PersistService.class).stop(); + } + + private void reset(String stackName, String stackVersion) throws Exception { + AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class); + + for (Entry<String, List<RepositoryInfo>> entry : ami.getRepository(stackName, stackVersion).entrySet()) { + for (RepositoryInfo ri : entry.getValue()) { + if (-1 == ri.getRepoId().indexOf("epel")) { + ami.updateRepoBaseURL(stackName, stackVersion, + ri.getOsType(), ri.getRepoId(), ri.getDefaultBaseUrl()); + } + } + } + + } + + @Test + public void testUpgradeStack() throws Exception { + StackUpgradeUtil stackUpgradeUtil = injector.getInstance(StackUpgradeUtil.class); + + String stackName = "HDP"; + String stackVersion = "1.3.0"; + String localRepoUrl = "http://foo.bar";; + + // check updating all + stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, null); + + MetainfoDAO dao = injector.getInstance(MetainfoDAO.class); + + Collection<MetainfoEntity> entities = dao.findAll(); + Assert.assertTrue(entities.size() > 0); + + for (MetainfoEntity entity : entities) { + Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/")); + Assert.assertEquals(localRepoUrl, entity.getMetainfoValue()); + } + + reset (stackName, stackVersion); + entities = dao.findAll(); + Assert.assertTrue(0 == entities.size()); + + // check updating only centos6 + stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6"); + + entities = dao.findAll(); + for (MetainfoEntity entity : entities) { + Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6")); + Assert.assertEquals(localRepoUrl, entity.getMetainfoValue()); + } + + reset (stackName, stackVersion); + entities = dao.findAll(); + Assert.assertTrue(0 == entities.size()); + + // check updating only centos6 and centos5 + stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6,centos5"); + + entities = dao.findAll(); + for (MetainfoEntity entity : entities) { + Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6") || + entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos5")); + Assert.assertEquals(localRepoUrl, entity.getMetainfoValue()); + } + + + } + + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index d1053aa..ca85b77 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -2740,7 +2740,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV ambari_server.upgrade_stack(args, 'HDP-2.0') self.assertTrue(run_stack_upgrade_mock.called) - run_stack_upgrade_mock.assert_called_with("HDP", "2.0") + run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None) @patch.object(ambari_server, 'get_conf_dir') @patch.object(ambari_server, 'get_ambari_classpath') @@ -2754,7 +2754,51 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV get_conf_dir_mock.return_value = '/etc/conf' stackIdMap = {'HDP' : '2.0'} - ambari_server.run_stack_upgrade('HDP', '2.0') + ambari_server.run_stack_upgrade('HDP', '2.0', None, None) + + self.assertTrue(jdk_path_mock.called) + self.assertTrue(get_ambari_classpath_mock.called) + self.assertTrue(get_conf_dir_mock.called) + self.assertTrue(run_os_command_mock.called) + run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 ' + 'org.apache.ambari.server.upgrade.StackUpgradeHelper ' + 'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1') + + @patch.object(ambari_server, 'get_conf_dir') + @patch.object(ambari_server, 'get_ambari_classpath') + @patch.object(ambari_server, 'run_os_command') + @patch.object(ambari_server, 'find_jdk') + def test_run_stack_upgrade_with_url(self, jdk_path_mock, run_os_command_mock, + get_ambari_classpath_mock, get_conf_dir_mock): + jdk_path_mock.return_value = "/usr/lib/java" + run_os_command_mock.return_value = (0, None, None) + get_ambari_classpath_mock.return_value = 'test:path12' + get_conf_dir_mock.return_value = '/etc/conf' + stackIdMap = {'HDP' : '2.0', 'repo_url' : 'http://test.com'} + + ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', None) + + self.assertTrue(jdk_path_mock.called) + self.assertTrue(get_ambari_classpath_mock.called) + self.assertTrue(get_conf_dir_mock.called) + self.assertTrue(run_os_command_mock.called) + run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 ' + 'org.apache.ambari.server.upgrade.StackUpgradeHelper ' + 'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1') + + @patch.object(ambari_server, 'get_conf_dir') + @patch.object(ambari_server, 'get_ambari_classpath') + @patch.object(ambari_server, 'run_os_command') + @patch.object(ambari_server, 'find_jdk') + def test_run_stack_upgrade_with_url_os(self, jdk_path_mock, run_os_command_mock, + get_ambari_classpath_mock, get_conf_dir_mock): + jdk_path_mock.return_value = "/usr/lib/java" + run_os_command_mock.return_value = (0, None, None) + get_ambari_classpath_mock.return_value = 'test:path12' + get_conf_dir_mock.return_value = '/etc/conf' + stackIdMap = {'HDP' : '2.0', 'repo_url': 'http://test.com', 'repo_url_os': 'centos5,centos6'} + + ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', 'centos5,centos6') self.assertTrue(jdk_path_mock.called) self.assertTrue(get_ambari_classpath_mock.called)
