Repository: ambari Updated Branches: refs/heads/branch-2.5 a16cb5ad9 -> c1bf05941
AMBARI-18048. ambari-server upgrade should report all the config properties it added/modified.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c1bf0594 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c1bf0594 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c1bf0594 Branch: refs/heads/branch-2.5 Commit: c1bf05941544c945f0ab5412b7dadcb4708697ca Parents: a16cb5a Author: Vitaly Brodetskyi <[email protected]> Authored: Thu Aug 25 15:38:23 2016 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Thu Aug 25 15:38:23 2016 +0300 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 143 +++++++++++++++++++ .../server/upgrade/AbstractUpgradeCatalog.java | 66 +++++++-- .../server/upgrade/SchemaUpgradeHelper.java | 41 +++--- .../ambari/server/upgrade/UpgradeCatalog.java | 7 + .../ambari/server/upgrade/UpgradeTest.java | 6 +- 5 files changed, 233 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c1bf0594/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index fae9378..3539025 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -20,8 +20,10 @@ package org.apache.ambari.server.configuration; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.io.Writer; import java.security.cert.CertificateException; import java.security.interfaces.RSAPublicKey; import java.util.ArrayList; @@ -52,6 +54,7 @@ import org.apache.ambari.server.security.encryption.CertificateUtils; import org.apache.ambari.server.security.encryption.CredentialProvider; import org.apache.ambari.server.state.services.MetricsRetrievalService; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.upgrade.AbstractUpgradeCatalog; import org.apache.ambari.server.utils.AmbariPath; import org.apache.ambari.server.utils.Parallel; import org.apache.ambari.server.utils.ShellCommandUtil; @@ -62,9 +65,13 @@ import org.apache.commons.lang.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Multimap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -96,6 +103,7 @@ public class Configuration { public static final String VIEWS_REMOVE_UNDEPLOYED = "views.remove.undeployed"; public static final String VIEWS_REMOVE_UNDEPLOYED_DEFAULT = "false"; public static final String WEBAPP_DIR = "webapp.dir"; + public static final String AMBARI_LOG_FILE = "log4j.properties"; public static final String BOOTSTRAP_SCRIPT = "bootstrap.script"; public static final String BOOTSTRAP_SCRIPT_DEFAULT = AmbariPath.getPath("/usr/bin/ambari_bootstrap"); public static final String BOOTSTRAP_SETUP_AGENT_SCRIPT = "bootstrap.setup_agent.script"; @@ -803,6 +811,8 @@ public class Configuration { Configuration.class); private Properties properties; + private Properties log4jProperties = new Properties(); + private String ambariUpgradeConfigUpdatesFilePath; private JsonObject hostChangesJson; private Map<String, String> configsMap; private Map<String, String> agentConfigsMap; @@ -1158,6 +1168,139 @@ public class Configuration { return properties; } + /** + * Find, read, and parse the log4j.properties file. + * @return the properties that were found or empty if no file was found + */ + public Properties getLog4jProperties() { + if (!log4jProperties.isEmpty()) { + return log4jProperties; + } + + //Get log4j.properties file stream from classpath + InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(AMBARI_LOG_FILE); + + if (inputStream == null) { + throw new RuntimeException(AMBARI_LOG_FILE + " not found in classpath"); + } + + // load the properties + try { + log4jProperties.load(inputStream); + inputStream.close(); + } catch (FileNotFoundException fnf) { + LOG.info("No configuration file " + AMBARI_LOG_FILE + " found in classpath.", fnf); + } catch (IOException ie) { + throw new IllegalArgumentException("Can't read configuration file " + + AMBARI_LOG_FILE, ie); + } + + return log4jProperties; + } + + + public void wrtiteToAmbariUpgradeConfigUpdatesFile(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, + String configType, String serviceName, String wrtiteToAmbariUpgradeConfigUpdatesFile) { + try { + if (ambariUpgradeConfigUpdatesFilePath == null) { + Properties log4jProperties = getLog4jProperties(); + if (log4jProperties != null) { + String logPath = log4jProperties.getProperty("ambari.log.dir"); + String rootPath = log4jProperties.getProperty("ambari.root.dir"); + logPath = StringUtils.replace(logPath, "${ambari.root.dir}", rootPath); + logPath = StringUtils.replace(logPath, "//", "/"); + if (StringUtils.isNotEmpty(logPath)) { + ambariUpgradeConfigUpdatesFilePath = logPath + File.separator + wrtiteToAmbariUpgradeConfigUpdatesFile; + } + } else { + LOG.warn("Log4j properties are not available"); + } + } + } catch(Exception e) { + LOG.warn("Failed to create log file name or get path for it:", e); + } + + if (StringUtils.isNotEmpty(ambariUpgradeConfigUpdatesFilePath)) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + Writer fileWriter = null; + try { + JsonObject rootJson = readFileToJSON(ambariUpgradeConfigUpdatesFilePath); + buildServiceJson(propertiesToLog, configType, serviceName, rootJson); + + fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath); + gson.toJson(rootJson, fileWriter); + } catch (IllegalArgumentException e) { + JsonObject rootJson = new JsonObject(); + buildServiceJson(propertiesToLog, configType, serviceName, rootJson); + + try { + fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath); + gson.toJson(rootJson, fileWriter); + } catch (IOException e1) { + LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e); + } + } catch (IOException e) { + LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e); + } finally { + try { + fileWriter.close(); + } catch (IOException e) { + LOG.error("Unable to close file " + ambariUpgradeConfigUpdatesFilePath, e); + } + } + } + } + + private void buildServiceJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, + String configType, String serviceName, JsonObject rootJson) { + JsonElement serviceJson = null; + serviceJson = rootJson.get(serviceName); + JsonObject serviceJsonObject = null; + if (serviceJson != null) { + serviceJsonObject = serviceJson.getAsJsonObject(); + } else { + serviceJsonObject = new JsonObject(); + } + buildConfigJson(propertiesToLog, serviceJsonObject, configType); + if (serviceName == null) { + serviceName = "General"; + } + + rootJson.add(serviceName, serviceJsonObject); + } + + private void buildConfigJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, + JsonObject serviceJson, String configType) { + JsonElement configJson = null; + configJson = serviceJson.get(configType); + JsonObject configJsonObject = null; + if (configJson != null) { + configJsonObject = configJson.getAsJsonObject(); + } else { + configJsonObject = new JsonObject(); + } + buildConfigUpdateTypes(propertiesToLog, configJsonObject); + serviceJson.add(configType, configJsonObject); + } + + private void buildConfigUpdateTypes(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, + JsonObject configJson) { + for (AbstractUpgradeCatalog.ConfigUpdateType configUpdateType : propertiesToLog.keySet()) { + JsonElement currentConfigUpdateType = configJson.get(configUpdateType.getDescription()); + JsonObject currentConfigUpdateTypeJsonObject = null; + if (currentConfigUpdateType != null) { + currentConfigUpdateTypeJsonObject = currentConfigUpdateType.getAsJsonObject(); + } else { + currentConfigUpdateTypeJsonObject = new JsonObject(); + } + for (Entry<String, String> property : propertiesToLog.get(configUpdateType)) { + currentConfigUpdateTypeJsonObject.add(property.getKey(), new JsonPrimitive(property.getValue())); + } + configJson.add(configUpdateType.getDescription(), currentConfigUpdateTypeJsonObject); + } + } + + public Map<String, String> getDatabaseConnectorNames() { File file = new File(Configuration.class.getClassLoader().getResource(CONFIG_FILE).getPath()); Long currentConfigLastModifiedDate = file.lastModified(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1bf0594/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java index 57cce3f..517d665 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java @@ -17,6 +17,11 @@ */ package org.apache.ambari.server.upgrade; +import javax.persistence.EntityManager; +import javax.xml.bind.JAXBException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -24,6 +29,7 @@ import java.io.StringReader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.AbstractMap; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -39,11 +45,6 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.persistence.EntityManager; -import javax.xml.bind.JAXBException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -84,7 +85,9 @@ import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.xml.sax.InputSource; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; @@ -140,6 +143,8 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { private static final Map<String, UpgradeCatalog> upgradeCatalogMap = new HashMap<String, UpgradeCatalog>(); + protected String ambariUpgradeConfigUpdatesFileName; + @Inject public AbstractUpgradeCatalog(Injector injector) { this.injector = injector; @@ -553,11 +558,22 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { oldConfigProperties = oldConfig.getProperties(); } + Multimap<ConfigUpdateType, Entry<String, String>> propertiesToLog = ArrayListMultimap.create(); + String serviceName = cluster.getServiceByConfigType(configType); + Map<String, String> mergedProperties = - mergeProperties(oldConfigProperties, properties, updateIfExists); + mergeProperties(oldConfigProperties, properties, updateIfExists, propertiesToLog); if (removePropertiesList != null) { - mergedProperties = removeProperties(mergedProperties, removePropertiesList); + mergedProperties = removeProperties(mergedProperties, removePropertiesList, propertiesToLog); + } + + if (propertiesToLog.size() > 0) { + try { + configuration.wrtiteToAmbariUpgradeConfigUpdatesFile(propertiesToLog, configType, serviceName, ambariUpgradeConfigUpdatesFileName); + } catch(Exception e) { + LOG.error("Write to config updates file failed:", e); + } } if (!Maps.difference(oldConfigProperties, mergedProperties).areEqual()) { @@ -642,28 +658,53 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { private Map<String, String> mergeProperties(Map<String, String> originalProperties, Map<String, String> newProperties, - boolean updateIfExists) { + boolean updateIfExists, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog) { Map<String, String> properties = new HashMap<String, String>(originalProperties); for (Map.Entry<String, String> entry : newProperties.entrySet()) { - if (!properties.containsKey(entry.getKey()) || updateIfExists) { + if (!properties.containsKey(entry.getKey())) { properties.put(entry.getKey(), entry.getValue()); + propertiesToLog.put(ConfigUpdateType.ADDED, entry); + } + if (updateIfExists) { + properties.put(entry.getKey(), entry.getValue()); + propertiesToLog.put(ConfigUpdateType.UPDATED, entry); } } return properties; } - private Map<String, String> removeProperties(Map<String, String> originalProperties, Set<String> removeList){ + private Map<String, String> removeProperties(Map<String, String> originalProperties, + Set<String> removeList, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog){ Map<String, String> properties = new HashMap<String, String>(); properties.putAll(originalProperties); for (String removeProperty: removeList){ if (originalProperties.containsKey(removeProperty)){ properties.remove(removeProperty); + propertiesToLog.put(ConfigUpdateType.REMOVED, new AbstractMap.SimpleEntry<String, String>(removeProperty, "")); } } return properties; } + public enum ConfigUpdateType { + ADDED("Added"), + UPDATED("Updated"), + REMOVED("Removed"); + + + private final String description; + + + private ConfigUpdateType(String description) { + this.description = description; + } + + public String getDescription() { + return description; + } + } + /** * Iterates through a collection of AbstractKerberosDescriptorContainers to find and update * identity descriptor references. @@ -895,6 +936,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { } @Override + public void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName) { + this.ambariUpgradeConfigUpdatesFileName = ambariUpgradeConfigUpdatesFileName; + } + + @Override public void upgradeData() throws AmbariException, SQLException { executeDMLUpdates(); updateTezHistoryUrlBase(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1bf0594/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index ebfb267..3f7821e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -17,32 +17,35 @@ */ package org.apache.ambari.server.upgrade; -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.multibindings.Multibinder; -import com.google.inject.persist.PersistService; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.audit.AuditLoggerModule; -import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.ControllerModule; -import org.apache.ambari.server.orm.DBAccessor; -import org.apache.ambari.server.utils.EventBusSynchronizer; -import org.apache.ambari.server.utils.VersionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Properties; import java.util.Set; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.audit.AuditLoggerModule; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.ControllerModule; +import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.utils.EventBusSynchronizer; +import org.apache.ambari.server.utils.VersionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.multibindings.Multibinder; +import com.google.inject.persist.PersistService; + public class SchemaUpgradeHelper { private static final Logger LOG = LoggerFactory.getLogger (SchemaUpgradeHelper.class); @@ -225,12 +228,13 @@ public class SchemaUpgradeHelper { } } - public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs) throws AmbariException { + public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs, String ambariUpgradeConfigUpdatesFileName) throws AmbariException { LOG.info("Executing DML changes."); if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) { for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) { try { + upgradeCatalog.setConfigUpdatesFileName(ambariUpgradeConfigUpdatesFileName); upgradeCatalog.upgradeData(); } catch (Exception e) { LOG.error("Upgrade failed. ", e); @@ -344,13 +348,16 @@ public class SchemaUpgradeHelper { List<UpgradeCatalog> upgradeCatalogs = schemaUpgradeHelper.getUpgradePath(sourceVersion, targetVersion); + String date = new SimpleDateFormat("MM-dd-yyyy_HH:mm:ss").format(new Date()); + String ambariUpgradeConfigUpdatesFileName = "ambari_upgrade_config_changes_" + date + ".json"; + schemaUpgradeHelper.executeUpgrade(upgradeCatalogs); schemaUpgradeHelper.startPersistenceService(); schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs); - schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs); + schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName); schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs); http://git-wip-us.apache.org/repos/asf/ambari/blob/c1bf0594/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java index 41c2215..21273fd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java @@ -47,6 +47,13 @@ public interface UpgradeCatalog { void upgradeData() throws AmbariException, SQLException; /** + * Set the file name, to store all config changes during upgrade + * @param ambariUpgradeConfigUpdatesFileName + */ + void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName); + + + /** * Defines if Upgrade Catalog should be executed last * @return */ http://git-wip-us.apache.org/repos/asf/ambari/blob/c1bf0594/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java index a1fd32d..dd745e6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.upgrade; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; @@ -74,8 +76,6 @@ import com.google.inject.Key; import com.google.inject.TypeLiteral; import com.google.inject.persist.PersistService; -import static org.junit.Assert.assertTrue; - @RunWith(Parameterized.class) public class UpgradeTest { private static final Logger LOG = LoggerFactory.getLogger(UpgradeTest.class); @@ -221,7 +221,7 @@ public class UpgradeTest { schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs); - schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs); + schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, "test"); schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);
