Repository: ambari Updated Branches: refs/heads/branch-2.5 c1bf05941 -> 567e94be9
Revert "AMBARI-18048. ambari-server upgrade should report all the config properties it added/modified.(vbrodetskyi)" This reverts commit c1bf05941544c945f0ab5412b7dadcb4708697ca. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/567e94be Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/567e94be Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/567e94be Branch: refs/heads/branch-2.5 Commit: 567e94be9829d0b486df118256848deeb11fa00a Parents: c1bf059 Author: Vitaly Brodetskyi <[email protected]> Authored: Thu Aug 25 18:39:32 2016 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Thu Aug 25 18:39:32 2016 +0300 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 143 ------------------- .../server/upgrade/AbstractUpgradeCatalog.java | 66 ++------- .../server/upgrade/SchemaUpgradeHelper.java | 41 +++--- .../ambari/server/upgrade/UpgradeCatalog.java | 7 - .../ambari/server/upgrade/UpgradeTest.java | 6 +- 5 files changed, 30 insertions(+), 233 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/567e94be/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 3539025..fae9378 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -20,10 +20,8 @@ package org.apache.ambari.server.configuration; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; -import java.io.Writer; import java.security.cert.CertificateException; import java.security.interfaces.RSAPublicKey; import java.util.ArrayList; @@ -54,7 +52,6 @@ import org.apache.ambari.server.security.encryption.CertificateUtils; import org.apache.ambari.server.security.encryption.CredentialProvider; import org.apache.ambari.server.state.services.MetricsRetrievalService; import org.apache.ambari.server.state.stack.OsFamily; -import org.apache.ambari.server.upgrade.AbstractUpgradeCatalog; import org.apache.ambari.server.utils.AmbariPath; import org.apache.ambari.server.utils.Parallel; import org.apache.ambari.server.utils.ShellCommandUtil; @@ -65,13 +62,9 @@ import org.apache.commons.lang.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Multimap; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.google.gson.JsonPrimitive; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -103,7 +96,6 @@ public class Configuration { public static final String VIEWS_REMOVE_UNDEPLOYED = "views.remove.undeployed"; public static final String VIEWS_REMOVE_UNDEPLOYED_DEFAULT = "false"; public static final String WEBAPP_DIR = "webapp.dir"; - public static final String AMBARI_LOG_FILE = "log4j.properties"; public static final String BOOTSTRAP_SCRIPT = "bootstrap.script"; public static final String BOOTSTRAP_SCRIPT_DEFAULT = AmbariPath.getPath("/usr/bin/ambari_bootstrap"); public static final String BOOTSTRAP_SETUP_AGENT_SCRIPT = "bootstrap.setup_agent.script"; @@ -811,8 +803,6 @@ public class Configuration { Configuration.class); private Properties properties; - private Properties log4jProperties = new Properties(); - private String ambariUpgradeConfigUpdatesFilePath; private JsonObject hostChangesJson; private Map<String, String> configsMap; private Map<String, String> agentConfigsMap; @@ -1168,139 +1158,6 @@ public class Configuration { return properties; } - /** - * Find, read, and parse the log4j.properties file. - * @return the properties that were found or empty if no file was found - */ - public Properties getLog4jProperties() { - if (!log4jProperties.isEmpty()) { - return log4jProperties; - } - - //Get log4j.properties file stream from classpath - InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(AMBARI_LOG_FILE); - - if (inputStream == null) { - throw new RuntimeException(AMBARI_LOG_FILE + " not found in classpath"); - } - - // load the properties - try { - log4jProperties.load(inputStream); - inputStream.close(); - } catch (FileNotFoundException fnf) { - LOG.info("No configuration file " + AMBARI_LOG_FILE + " found in classpath.", fnf); - } catch (IOException ie) { - throw new IllegalArgumentException("Can't read configuration file " + - AMBARI_LOG_FILE, ie); - } - - return log4jProperties; - } - - - public void wrtiteToAmbariUpgradeConfigUpdatesFile(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, - String configType, String serviceName, String wrtiteToAmbariUpgradeConfigUpdatesFile) { - try { - if (ambariUpgradeConfigUpdatesFilePath == null) { - Properties log4jProperties = getLog4jProperties(); - if (log4jProperties != null) { - String logPath = log4jProperties.getProperty("ambari.log.dir"); - String rootPath = log4jProperties.getProperty("ambari.root.dir"); - logPath = StringUtils.replace(logPath, "${ambari.root.dir}", rootPath); - logPath = StringUtils.replace(logPath, "//", "/"); - if (StringUtils.isNotEmpty(logPath)) { - ambariUpgradeConfigUpdatesFilePath = logPath + File.separator + wrtiteToAmbariUpgradeConfigUpdatesFile; - } - } else { - LOG.warn("Log4j properties are not available"); - } - } - } catch(Exception e) { - LOG.warn("Failed to create log file name or get path for it:", e); - } - - if (StringUtils.isNotEmpty(ambariUpgradeConfigUpdatesFilePath)) { - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - Writer fileWriter = null; - try { - JsonObject rootJson = readFileToJSON(ambariUpgradeConfigUpdatesFilePath); - buildServiceJson(propertiesToLog, configType, serviceName, rootJson); - - fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath); - gson.toJson(rootJson, fileWriter); - } catch (IllegalArgumentException e) { - JsonObject rootJson = new JsonObject(); - buildServiceJson(propertiesToLog, configType, serviceName, rootJson); - - try { - fileWriter = new FileWriter(ambariUpgradeConfigUpdatesFilePath); - gson.toJson(rootJson, fileWriter); - } catch (IOException e1) { - LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e); - } - } catch (IOException e) { - LOG.error("Unable to write data into " + ambariUpgradeConfigUpdatesFilePath, e); - } finally { - try { - fileWriter.close(); - } catch (IOException e) { - LOG.error("Unable to close file " + ambariUpgradeConfigUpdatesFilePath, e); - } - } - } - } - - private void buildServiceJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, - String configType, String serviceName, JsonObject rootJson) { - JsonElement serviceJson = null; - serviceJson = rootJson.get(serviceName); - JsonObject serviceJsonObject = null; - if (serviceJson != null) { - serviceJsonObject = serviceJson.getAsJsonObject(); - } else { - serviceJsonObject = new JsonObject(); - } - buildConfigJson(propertiesToLog, serviceJsonObject, configType); - if (serviceName == null) { - serviceName = "General"; - } - - rootJson.add(serviceName, serviceJsonObject); - } - - private void buildConfigJson(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, - JsonObject serviceJson, String configType) { - JsonElement configJson = null; - configJson = serviceJson.get(configType); - JsonObject configJsonObject = null; - if (configJson != null) { - configJsonObject = configJson.getAsJsonObject(); - } else { - configJsonObject = new JsonObject(); - } - buildConfigUpdateTypes(propertiesToLog, configJsonObject); - serviceJson.add(configType, configJsonObject); - } - - private void buildConfigUpdateTypes(Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog, - JsonObject configJson) { - for (AbstractUpgradeCatalog.ConfigUpdateType configUpdateType : propertiesToLog.keySet()) { - JsonElement currentConfigUpdateType = configJson.get(configUpdateType.getDescription()); - JsonObject currentConfigUpdateTypeJsonObject = null; - if (currentConfigUpdateType != null) { - currentConfigUpdateTypeJsonObject = currentConfigUpdateType.getAsJsonObject(); - } else { - currentConfigUpdateTypeJsonObject = new JsonObject(); - } - for (Entry<String, String> property : propertiesToLog.get(configUpdateType)) { - currentConfigUpdateTypeJsonObject.add(property.getKey(), new JsonPrimitive(property.getValue())); - } - configJson.add(configUpdateType.getDescription(), currentConfigUpdateTypeJsonObject); - } - } - - public Map<String, String> getDatabaseConnectorNames() { File file = new File(Configuration.class.getClassLoader().getResource(CONFIG_FILE).getPath()); Long currentConfigLastModifiedDate = file.lastModified(); http://git-wip-us.apache.org/repos/asf/ambari/blob/567e94be/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java index 517d665..57cce3f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java @@ -17,11 +17,6 @@ */ package org.apache.ambari.server.upgrade; -import javax.persistence.EntityManager; -import javax.xml.bind.JAXBException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -29,7 +24,6 @@ import java.io.StringReader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.AbstractMap; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -45,6 +39,11 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.persistence.EntityManager; +import javax.xml.bind.JAXBException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -85,9 +84,7 @@ import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.xml.sax.InputSource; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; @@ -143,8 +140,6 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { private static final Map<String, UpgradeCatalog> upgradeCatalogMap = new HashMap<String, UpgradeCatalog>(); - protected String ambariUpgradeConfigUpdatesFileName; - @Inject public AbstractUpgradeCatalog(Injector injector) { this.injector = injector; @@ -558,22 +553,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { oldConfigProperties = oldConfig.getProperties(); } - Multimap<ConfigUpdateType, Entry<String, String>> propertiesToLog = ArrayListMultimap.create(); - String serviceName = cluster.getServiceByConfigType(configType); - Map<String, String> mergedProperties = - mergeProperties(oldConfigProperties, properties, updateIfExists, propertiesToLog); + mergeProperties(oldConfigProperties, properties, updateIfExists); if (removePropertiesList != null) { - mergedProperties = removeProperties(mergedProperties, removePropertiesList, propertiesToLog); - } - - if (propertiesToLog.size() > 0) { - try { - configuration.wrtiteToAmbariUpgradeConfigUpdatesFile(propertiesToLog, configType, serviceName, ambariUpgradeConfigUpdatesFileName); - } catch(Exception e) { - LOG.error("Write to config updates file failed:", e); - } + mergedProperties = removeProperties(mergedProperties, removePropertiesList); } if (!Maps.difference(oldConfigProperties, mergedProperties).areEqual()) { @@ -658,53 +642,28 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { private Map<String, String> mergeProperties(Map<String, String> originalProperties, Map<String, String> newProperties, - boolean updateIfExists, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog) { + boolean updateIfExists) { Map<String, String> properties = new HashMap<String, String>(originalProperties); for (Map.Entry<String, String> entry : newProperties.entrySet()) { - if (!properties.containsKey(entry.getKey())) { + if (!properties.containsKey(entry.getKey()) || updateIfExists) { properties.put(entry.getKey(), entry.getValue()); - propertiesToLog.put(ConfigUpdateType.ADDED, entry); - } - if (updateIfExists) { - properties.put(entry.getKey(), entry.getValue()); - propertiesToLog.put(ConfigUpdateType.UPDATED, entry); } } return properties; } - private Map<String, String> removeProperties(Map<String, String> originalProperties, - Set<String> removeList, Multimap<AbstractUpgradeCatalog.ConfigUpdateType, Entry<String, String>> propertiesToLog){ + private Map<String, String> removeProperties(Map<String, String> originalProperties, Set<String> removeList){ Map<String, String> properties = new HashMap<String, String>(); properties.putAll(originalProperties); for (String removeProperty: removeList){ if (originalProperties.containsKey(removeProperty)){ properties.remove(removeProperty); - propertiesToLog.put(ConfigUpdateType.REMOVED, new AbstractMap.SimpleEntry<String, String>(removeProperty, "")); } } return properties; } - public enum ConfigUpdateType { - ADDED("Added"), - UPDATED("Updated"), - REMOVED("Removed"); - - - private final String description; - - - private ConfigUpdateType(String description) { - this.description = description; - } - - public String getDescription() { - return description; - } - } - /** * Iterates through a collection of AbstractKerberosDescriptorContainers to find and update * identity descriptor references. @@ -936,11 +895,6 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog { } @Override - public void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName) { - this.ambariUpgradeConfigUpdatesFileName = ambariUpgradeConfigUpdatesFileName; - } - - @Override public void upgradeData() throws AmbariException, SQLException { executeDMLUpdates(); updateTezHistoryUrlBase(); http://git-wip-us.apache.org/repos/asf/ambari/blob/567e94be/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index 3f7821e..ebfb267 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -17,19 +17,11 @@ */ package org.apache.ambari.server.upgrade; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Properties; -import java.util.Set; - +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.multibindings.Multibinder; +import com.google.inject.persist.PersistService; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.audit.AuditLoggerModule; import org.apache.ambari.server.configuration.Configuration; @@ -40,11 +32,16 @@ import org.apache.ambari.server.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.multibindings.Multibinder; -import com.google.inject.persist.PersistService; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.Set; public class SchemaUpgradeHelper { private static final Logger LOG = LoggerFactory.getLogger @@ -228,13 +225,12 @@ public class SchemaUpgradeHelper { } } - public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs, String ambariUpgradeConfigUpdatesFileName) throws AmbariException { + public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs) throws AmbariException { LOG.info("Executing DML changes."); if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) { for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) { try { - upgradeCatalog.setConfigUpdatesFileName(ambariUpgradeConfigUpdatesFileName); upgradeCatalog.upgradeData(); } catch (Exception e) { LOG.error("Upgrade failed. ", e); @@ -348,16 +344,13 @@ public class SchemaUpgradeHelper { List<UpgradeCatalog> upgradeCatalogs = schemaUpgradeHelper.getUpgradePath(sourceVersion, targetVersion); - String date = new SimpleDateFormat("MM-dd-yyyy_HH:mm:ss").format(new Date()); - String ambariUpgradeConfigUpdatesFileName = "ambari_upgrade_config_changes_" + date + ".json"; - schemaUpgradeHelper.executeUpgrade(upgradeCatalogs); schemaUpgradeHelper.startPersistenceService(); schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs); - schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, ambariUpgradeConfigUpdatesFileName); + schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs); schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs); http://git-wip-us.apache.org/repos/asf/ambari/blob/567e94be/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java index 21273fd..41c2215 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java @@ -47,13 +47,6 @@ public interface UpgradeCatalog { void upgradeData() throws AmbariException, SQLException; /** - * Set the file name, to store all config changes during upgrade - * @param ambariUpgradeConfigUpdatesFileName - */ - void setConfigUpdatesFileName(String ambariUpgradeConfigUpdatesFileName); - - - /** * Defines if Upgrade Catalog should be executed last * @return */ http://git-wip-us.apache.org/repos/asf/ambari/blob/567e94be/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java index dd745e6..a1fd32d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java @@ -18,8 +18,6 @@ package org.apache.ambari.server.upgrade; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; @@ -76,6 +74,8 @@ import com.google.inject.Key; import com.google.inject.TypeLiteral; import com.google.inject.persist.PersistService; +import static org.junit.Assert.assertTrue; + @RunWith(Parameterized.class) public class UpgradeTest { private static final Logger LOG = LoggerFactory.getLogger(UpgradeTest.class); @@ -221,7 +221,7 @@ public class UpgradeTest { schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs); - schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs, "test"); + schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs); schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);
