AMBARI-14385. Robust Handling of Ambari properites and DB state. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/65752c9b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/65752c9b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/65752c9b Branch: refs/heads/branch-dev-patch-upgrade Commit: 65752c9bd95af7f64002e1b1da6b734b3bd99d8f Parents: f697f4a Author: Andrew Onishuk <[email protected]> Authored: Tue Dec 29 13:33:42 2015 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Tue Dec 29 13:33:42 2015 +0200 ---------------------------------------------------------------------- .../ambari/server/controller/AmbariServer.java | 7 +- .../controller/utilities/DatabaseChecker.java | 115 ++++++++++++++++++- ambari-server/src/main/python/ambari-server.py | 4 + .../python/ambari_server/serverConfiguration.py | 14 +++ .../src/main/python/ambari_server_main.py | 13 ++- 5 files changed, 148 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index c6f90c0..cc69134 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -260,8 +260,11 @@ public class AmbariServer { server.setSessionIdManager(sessionIdManager); Server serverForAgent = new Server(); - DatabaseChecker.checkDBVersion(); - DatabaseChecker.checkDBConsistency(); + if (System.getProperty("skipDatabaseConsistencyValidation") == null) { + DatabaseChecker.checkDBVersion(); + DatabaseChecker.checkDBConsistency(); + DatabaseChecker.checkDBConfigsConsistency(); + } try { ClassPathXmlApplicationContext parentSpringAppContext = http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java index c4a4e4c..4839d21 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java @@ -19,22 +19,31 @@ package org.apache.ambari.server.controller.utilities; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.MetainfoDAO; +import org.apache.ambari.server.orm.entities.ClusterConfigEntity; +import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity; import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ClusterServiceEntity; +import org.apache.ambari.server.orm.entities.ClusterStateEntity; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.MetainfoEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity; +import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.State; import org.apache.ambari.server.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -142,7 +151,108 @@ public class DatabaseChecker { if (checkPassed) { LOG.info("DB consistency check passed."); } else { - LOG.error("DB consistency check failed."); + String errorMessage = "DB consistency check failed. Run \"ambari-server start --skip-database-validation\" to skip validation."; + LOG.error(errorMessage); + throw new AmbariException(errorMessage); + } + } + + private static boolean clusterConfigsContainTypeAndTag(Collection<ClusterConfigEntity> clusterConfigEntities, + String typeName, String tag) { + for (ClusterConfigEntity clusterConfigEntity : clusterConfigEntities) { + if (typeName.equals(clusterConfigEntity.getType()) && tag.equals(clusterConfigEntity.getTag())) { + return true; + } + } + return false; + } + + /** + * Validates configuration consistency. Checks that every config type from stack is presented in + * ClusterConfigMapping. Checks that for each config type exactly one is selected. Checks that ClusterConfig + * contains type_names and tags from ClusterConfigMapping. + * + * @throws AmbariException if check failed + */ + public static void checkDBConfigsConsistency() throws AmbariException { + LOG.info("Checking DB configs consistency"); + + boolean checkPassed = true; + + if (ambariMetaInfo == null) { + ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); + } + + ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class); + List<ClusterEntity> clusters = clusterDAO.findAll(); + if (clusters != null) { + for (ClusterEntity clusterEntity : clusters) { + Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities(); + Collection<ClusterConfigEntity> clusterConfigEntities = clusterEntity.getClusterConfigEntities(); + + if (configMappingEntities != null) { + Map<String, Integer> selectedCountForType = new HashMap<>(); + for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities) { + String typeName = clusterConfigMappingEntity.getType(); + if (clusterConfigMappingEntity.isSelected() > 0) { + int selectedCount = selectedCountForType.get(typeName) != null ? selectedCountForType.get(typeName) : 0; + selectedCountForType.put(typeName, selectedCount + 1); + + // Check that ClusterConfig contains type_name and tag from ClusterConfigMapping + if (!clusterConfigsContainTypeAndTag(clusterConfigEntities, typeName, clusterConfigMappingEntity.getTag())) { + checkPassed = false; + LOG.error("ClusterConfig does not contain mapping for type_name=" + typeName + " tag=" + + clusterConfigMappingEntity.getTag()); + } + } else { + if (!selectedCountForType.containsKey(typeName)) { + selectedCountForType.put(typeName, 0); + } + } + } + + // Check that every config type from stack is presented in ClusterConfigMapping + Collection<ClusterServiceEntity> clusterServiceEntities = clusterEntity.getClusterServiceEntities(); + ClusterStateEntity clusterStateEntity = clusterEntity.getClusterStateEntity(); + if (clusterStateEntity != null) { + StackEntity currentStack = clusterStateEntity.getCurrentStack(); + StackInfo stack = ambariMetaInfo.getStack(currentStack.getStackName(), currentStack.getStackVersion()); + + for (ClusterServiceEntity clusterServiceEntity : clusterServiceEntities) { + if (!State.INIT.equals(clusterServiceEntity.getServiceDesiredStateEntity().getDesiredState())) { + String serviceName = clusterServiceEntity.getServiceName(); + ServiceInfo serviceInfo = ambariMetaInfo.getService(stack.getName(), stack.getVersion(), serviceName); + for (String configTypeName : serviceInfo.getConfigTypeAttributes().keySet()) { + if (selectedCountForType.get(configTypeName) == null) { + checkPassed = false; + LOG.error("ClusterConfigMapping does not contain mapping for service=" + serviceName + " type_name=" + + configTypeName); + } else { + // Check that for each config type exactly one is selected + if (selectedCountForType.get(configTypeName) == 0) { + checkPassed = false; + LOG.error("ClusterConfigMapping selected count is 0 for service=" + serviceName + " type_name=" + + configTypeName); + } else if (selectedCountForType.get(configTypeName) > 1) { + checkPassed = false; + LOG.error("ClusterConfigMapping selected count is more than 1 for service=" + serviceName + + " type_name=" + configTypeName); + } + } + } + } + } + } + } + } + } + + if (checkPassed) { + LOG.info("DB configs consistency check passed."); + } else { + String errorMessage = "DB configs consistency check failed. Run \"ambari-server start --skip-database-validation\" to skip validation."; + LOG.error(errorMessage); + throw new AmbariException(errorMessage); } } @@ -169,7 +279,8 @@ public class DatabaseChecker { String error = "Current database store version is not compatible with " + "current server version" + ", serverVersion=" + serverVersion - + ", schemaVersion=" + schemaVersion; + + ", schemaVersion=" + schemaVersion + + " Run \"ambari-server start --skip-database-validation\" to skip validation"; LOG.warn(error); throw new AmbariException(error); } http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 41fdfb6..9f13ac5 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -316,6 +316,8 @@ def init_parser_options(parser): help="Database user password") parser.add_option('--jdbc-driver', default=None, dest="jdbc_driver", help="Specifies the path to the JDBC driver JAR file") + parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation") + parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip database consistency validation", dest="skip_database_validation") # -b and -i the remaining available short options # -h reserved for help @@ -380,6 +382,8 @@ def init_parser_options(parser): dest="jdbc_db") parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name") parser.add_option('--version-display-name', default=None, help="Display name of desired repo version", dest="desired_repo_version") + parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation") + parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip database consistency validation", dest="skip_database_validation") parser.add_option('--force-version', action="store_true", default=False, help="Force version to current", dest="force_repo_version") parser.add_option('--version', dest="stack_versions", default=None, action="append", type="string", help="Specify stack version that needs to be enabled. All other stacks versions will be disabled") http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 223c608..aeb4d5b 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -177,6 +177,11 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s DEFAULT_DB_NAME = "ambari" +COMMON_SERVICES_PATH_PROPERTY = 'common.services.path' +WEBAPP_DIR_PROPERTY = 'webapp.dir' +REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH, \ + WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY] + class ServerDatabaseType(object): internal = 0 remote = 1 @@ -1303,3 +1308,12 @@ def get_stack_location(properties): if stack_location is None: stack_location = configDefaults.STACK_LOCATION_DEFAULT return stack_location + +def get_missing_properties(properties): + missing_propertiers = [] + for property in REQUIRED_PROPERTIES: + value = properties[property] + if not value: + missing_propertiers.append(property) + + return missing_propertiers \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server_main.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py index cf8e8c0..5c98bf4 100644 --- a/ambari-server/src/main/python/ambari_server_main.py +++ b/ambari-server/src/main/python/ambari_server_main.py @@ -32,7 +32,7 @@ from ambari_server.serverConfiguration import configDefaults, find_jdk, get_amba get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key, read_ambari_user, \ get_is_active_instance, \ PID_NAME, SECURITY_KEY_ENV_VAR_NAME, SECURITY_MASTER_KEY_LOCATION, \ - SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file + SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file, get_missing_properties from ambari_server.serverUtils import refresh_stack_hash from ambari_server.setupHttps import get_fqdn from ambari_server.setupSecurity import generate_env, \ @@ -237,6 +237,13 @@ def server_process_main(options, scmStatus=None): properties = get_ambari_properties() + if not options.skip_properties_validation: + missing_properties = get_missing_properties(properties) + if missing_properties: + err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \ + "use \"--skip-properties-validation\"" + raise FatalException(1, err) + # Preparations if is_root(): print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT @@ -267,6 +274,10 @@ def server_process_main(options, scmStatus=None): suspend_start = (debug_mode & 2) or SUSPEND_START_MODE suspend_mode = 'y' if suspend_start else 'n' + if options.skip_database_validation: + global jvm_args + jvm_args += " -DskipDatabaseConsistencyValidation" + param_list = generate_child_process_param_list(ambari_user, java_exe, serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start, suspend_mode)
