AMBARI-22454. ambari-server upgrade to 2.6.1 should surface the GPL agreement (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3df5ae74 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3df5ae74 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3df5ae74 Branch: refs/heads/branch-2.6 Commit: 3df5ae74a5917923d254a47ba116816e899dfbce Parents: 4b01b65 Author: Andrew Onishuk <[email protected]> Authored: Fri Nov 17 13:44:58 2017 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Fri Nov 17 13:44:58 2017 +0200 ---------------------------------------------------------------------- .../server/upgrade/AbstractUpgradeCatalog.java | 8 ++ .../server/upgrade/SchemaUpgradeHelper.java | 27 +++++ .../ambari/server/upgrade/UpgradeCatalog.java | 7 ++ .../server/upgrade/UpgradeCatalog261.java | 118 +++++++++++++++++++ .../python/ambari_server/serverConfiguration.py | 22 +++- .../main/python/ambari_server/serverSetup.py | 20 +--- .../main/python/ambari_server/serverUpgrade.py | 18 ++- .../src/test/python/TestAmbariServer.py | 2 +- 8 files changed, 200 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java index dac871e..5ecb638 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java @@ -140,6 +140,7 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { new HashMap<>(); protected String ambariUpgradeConfigUpdatesFileName; + private Map<String,String> upgradeJsonOutput = new HashMap<>(); @Inject public AbstractUpgradeCatalog(Injector injector) { @@ -258,6 +259,13 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { } /** + * {@inheritDoc} + */ + public Map<String,String> getUpgradeJsonOutput() { + return upgradeJsonOutput; + } + + /** * Update metainfo to new version. */ @Transactional http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index a2dea40..963fd26 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -29,7 +29,11 @@ import java.util.Date; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.audit.AuditLoggerModule; import org.apache.ambari.server.configuration.Configuration; @@ -56,6 +60,7 @@ public class SchemaUpgradeHelper { private DBAccessor dbAccessor; private Configuration configuration; private static final String[] rcaTableNames = {"workflow", "job", "task", "taskAttempt", "hdfsEvent", "mapreduceEvent", "clusterEvent"}; + static final Gson gson = new GsonBuilder().create(); @Inject public SchemaUpgradeHelper(Set<UpgradeCatalog> allUpgradeCatalogs, @@ -184,6 +189,7 @@ public class SchemaUpgradeHelper { catalogBinder.addBinding().to(UpgradeCatalog251.class); catalogBinder.addBinding().to(UpgradeCatalog252.class); catalogBinder.addBinding().to(UpgradeCatalog260.class); + catalogBinder.addBinding().to(UpgradeCatalog261.class); catalogBinder.addBinding().to(UpdateAlertScriptPaths.class); catalogBinder.addBinding().to(FinalUpgradeCatalog.class); @@ -254,6 +260,26 @@ public class SchemaUpgradeHelper { } } + public void outputUpgradeJsonOutput(List<UpgradeCatalog> upgradeCatalogs) + throws AmbariException { + LOG.info("Combining upgrade json output."); + Map<String,String> combinedUpgradeJsonOutput = new HashMap<>(); + + if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) { + for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) { + try { + combinedUpgradeJsonOutput.putAll(upgradeCatalog.getUpgradeJsonOutput()); + + } catch (Exception e) { + LOG.error("Upgrade failed. ", e); + throw new AmbariException(e.getMessage(), e); + } + } + } + String content = gson.toJson(combinedUpgradeJsonOutput); + System.out.println(content); + } + public void resetUIState() throws AmbariException { LOG.info("Resetting UI state."); try { @@ -424,6 +450,7 @@ public class SchemaUpgradeHelper { schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName); schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs); + schemaUpgradeHelper.outputUpgradeJsonOutput(upgradeCatalogs); schemaUpgradeHelper.resetUIState(); http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java index 21273fd..a005951 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.upgrade; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import org.apache.ambari.server.AmbariException; @@ -88,4 +90,9 @@ public interface UpgradeCatalog { * Update schema version in the database to the Target one */ void updateDatabaseSchemaVersion(); + + /* + Get upgrade json output, which is sent to python executing process. + */ + Map<String,String> getUpgradeJsonOutput(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java new file mode 100644 index 0000000..7f463aa --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog261.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.upgrade; + +import com.google.inject.Inject; +import com.google.inject.Injector; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.Map; + +/** + * The {@link UpgradeCatalog261} upgrades Ambari from 2.6.0 to 2.6.1. + */ +public class UpgradeCatalog261 extends AbstractUpgradeCatalog { + private static final String CORE_SITE = "core-site"; + private static final String COMPRESSION_CODECS_PROPERTY = "io.compression.codecs"; + private static final String COMPRESSION_CODECS_LZO = "com.hadoop.compression.lzo"; + private static final String LZO_ENABLED_JSON_KEY = "lzo_enabled"; + + private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog261.class); + + /** + * Constructor. + * + * @param injector + */ + @Inject + public UpgradeCatalog261(Injector injector) { + super(injector); + } + + /** + * {@inheritDoc} + */ + @Override + public String getSourceVersion() { + return "2.6.0"; + } + + /** + * {@inheritDoc} + */ + @Override + public String getTargetVersion() { + return "2.6.1"; + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDDLUpdates() throws AmbariException, SQLException { + } + + /** + * {@inheritDoc} + */ + @Override + protected void executePreDMLUpdates() throws AmbariException, SQLException { + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDMLUpdates() throws AmbariException, SQLException { + // TODO: make this map with clusterids as keys + this.getUpgradeJsonOutput().put(LZO_ENABLED_JSON_KEY, isLzoEnabled().toString()); + } + + protected Boolean isLzoEnabled() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + + Config coreSite = cluster.getDesiredConfigByType(CORE_SITE); + if (coreSite != null) { + Map<String, String> coreSiteProperties = coreSite.getProperties(); + + if (coreSiteProperties.containsKey(COMPRESSION_CODECS_PROPERTY)) { + String compressionCodecs = coreSiteProperties.get(COMPRESSION_CODECS_PROPERTY); + + if(compressionCodecs.contains(COMPRESSION_CODECS_LZO)) { + return true; + } + } + } + } + } + } + return false; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 262e944..6e19b48 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -1149,9 +1149,25 @@ def update_ambari_env(): return 0 -def prompt_gpl_agreement(): - result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True) - return str(result).lower() +def write_gpl_license_accepted(is_silent=False): + properties = get_ambari_properties() + if properties == -1: + err = "Error getting ambari properties" + raise FatalException(-1, err) + + + if GPL_LICENSE_ACCEPTED_PROPERTY in properties.keys() and properties.get_property(GPL_LICENSE_ACCEPTED_PROPERTY).lower() == "true": + return True + + if is_silent: + result = options.accept_gpl + else: + result = get_YN_input(GPL_LICENSE_PROMPT_TEXT, True) + + properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, str(result).lower()) + update_properties(properties) + + return result def update_ambari_properties(): prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir()) http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverSetup.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverSetup.py b/ambari-server/src/main/python/ambari_server/serverSetup.py index 8a14066..4ffe0bc 100644 --- a/ambari-server/src/main/python/ambari_server/serverSetup.py +++ b/ambari-server/src/main/python/ambari_server/serverSetup.py @@ -38,7 +38,7 @@ from ambari_commons.str_utils import compress_backslashes from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map, check_jdbc_drivers from ambari_server.serverConfiguration import configDefaults, JDKRelease, \ get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME, get_missing_properties, \ - get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, prompt_gpl_agreement, write_property, \ + get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_gpl_license_accepted, write_property, \ JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \ JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \ VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES, GPL_LICENSE_ACCEPTED_PROPERTY @@ -1081,20 +1081,6 @@ def check_setup_already_done(): return not bool(get_missing_properties(properties, property_set=SETUP_DONE_PROPERTIES)) -def write_gpl_license_accepted(options): - properties = get_ambari_properties() - if properties == -1: - err = "Error getting ambari properties" - raise FatalException(-1, err) - - if get_silent(): - result = str(options.accept_gpl).lower() - else: - result = prompt_gpl_agreement() - - properties.process_pair(GPL_LICENSE_ACCEPTED_PROPERTY, result) - update_properties(properties) - # # Setup the Ambari Server. # @@ -1142,8 +1128,8 @@ def setup(options): err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e) raise FatalException(e.code, err) - print 'Prompting GPL software agreement...' - write_gpl_license_accepted(options) + print 'Checking GPL software agreement...' + write_gpl_license_accepted(is_silent=get_silent()) print 'Completing setup...' retcode = configure_os_settings() http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/main/python/ambari_server/serverUpgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index 421adfc..a864609 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -28,6 +28,7 @@ import re import glob import optparse import logging +import ambari_simplejson as json from ambari_commons.exceptions import FatalException from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, get_verbose @@ -41,7 +42,7 @@ from ambari_server.serverConfiguration import configDefaults, get_resources_loca update_database_name_property, get_admin_views_dir, get_views_dir, get_views_jars, \ AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY, \ SETUP_OR_UPGRADE_MSG, update_krb_jaas_login_properties, AMBARI_KRB_JAAS_LOGIN_FILE, get_db_type, update_ambari_env, \ - AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir + AMBARI_ENV_FILE, JDBC_DATABASE_PROPERTY, get_default_views_dir, write_gpl_license_accepted from ambari_server.setupSecurity import adjust_directory_permissions, \ generate_env, ensure_can_start_under_current_user from ambari_server.utils import compare_versions @@ -73,6 +74,11 @@ SCHEMA_UPGRADE_DEBUG = False SUSPEND_START_MODE = False +INSALLED_LZO_WITHOUT_GPL_TEXT = "By saying no, Ambari will not automatically install LZO on any new host in the cluster." + \ +"It is up to you to ensure LZO is installed and configured appropriately." + \ +"Without LZO being installed and configured data compressed with LZO will not be readable. " + \ +"Are you sure you want to proceed? [y/n] (n)?" + def load_stack_values(version, filename): import xml.etree.ElementTree as ET values = {} @@ -146,6 +152,10 @@ def run_schema_upgrade(args): environ = generate_env(args, ambari_user, current_user) (retcode, stdout, stderr) = run_os_command(command, env=environ) + upgrade_response = json.loads(stdout) + + check_gpl_license_approved(upgrade_response) + print_info_msg("Return code from schema upgrade command, retcode = {0}".format(str(retcode)), True) if stdout: print_info_msg("Console output from schema upgrade command:", True) @@ -161,6 +171,12 @@ def run_schema_upgrade(args): print_info_msg('Schema upgrade completed', True) return retcode +def check_gpl_license_approved(upgrade_response): + if 'lzo_enabled' not in upgrade_response or upgrade_response['lzo_enabled'].lower() != "true": + return + + while not write_gpl_license_accepted() and not get_YN_input(INSALLED_LZO_WITHOUT_GPL_TEXT, False): + pass # # Upgrades the Ambari Server. http://git-wip-us.apache.org/repos/asf/ambari/blob/3df5ae74/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 840b9a6..a53d274 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -4938,7 +4938,7 @@ class TestAmbariServer(TestCase): ensure_can_start_under_current_user_mock, get_jdbc_mock, ensure_jdbc_driver_is_installed_mock): java_exe_path_mock.return_value = "/usr/lib/java/bin/java" - run_os_command_mock.return_value = (0, None, None) + run_os_command_mock.return_value = (0, '{"lzo_enabled":"false"}', None) get_conf_dir_mock.return_value = '/etc/conf' command = '/usr/lib/java/bin/java -cp /etc/conf' + os.pathsep + 'test' + os.pathsep + 'path12' + \ os.pathsep +'/path/to/jdbc.jar ' \
