Repository: ambari Updated Branches: refs/heads/trunk 7826a3ff4 -> 1aa5bcdd8
AMBARI-6746. Ambari Server upgrade to take care of service config versions. (mpapirkovskyy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1aa5bcdd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1aa5bcdd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1aa5bcdd Branch: refs/heads/trunk Commit: 1aa5bcdd8426bad4c175a34fd9bbc62ead757c4c Parents: 7826a3f Author: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Authored: Tue Aug 5 19:30:50 2014 +0300 Committer: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Committed: Mon Aug 11 13:05:05 2014 +0300 ---------------------------------------------------------------------- .../server/api/util/StackExtensionHelper.java | 7 ++ .../ServiceConfigApplicationEntity.java | 2 +- .../apache/ambari/server/state/ServiceInfo.java | 16 ++++ .../server/state/cluster/ClusterImpl.java | 9 +- .../server/upgrade/UpgradeCatalog150.java | 11 ++- .../server/upgrade/UpgradeCatalog170.java | 89 ++++++++++++++++++++ .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +- .../stacks/HDP/1.3.2/services/HIVE/metainfo.xml | 7 ++ .../stacks/HDP/2.0.6/services/HIVE/metainfo.xml | 6 ++ .../stacks/HDP/2.1/services/FALCON/metainfo.xml | 4 + .../ambari/server/upgrade/UpgradeTest.java | 41 ++++----- 11 files changed, 167 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java index 65efa77..0670d9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java @@ -149,6 +149,13 @@ public class StackExtensionHelper { parentService.getConfigTypes() != null ? parentService.getConfigTypes() : Collections.<String, Map<String, Map<String, String>>>emptyMap()); + mergedServiceInfo.setExcludedConfigTypes( + childService.getExcludedConfigTypes() != null ? + childService.getExcludedConfigTypes() : + parentService.getExcludedConfigTypes() != null ? + parentService.getExcludedConfigTypes() : + Collections.<String>emptySet() + ); mergedServiceInfo.setRestartRequiredAfterChange( (childService.isRestartRequiredAfterChange() != null) http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java index cae46e3..ba5adb5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java @@ -53,7 +53,7 @@ public class ServiceConfigApplicationEntity { @Basic @Column(name = "user_name") - private String user; + private String user = "_db"; @ManyToOne @JoinColumn(name = "service_config_id", referencedColumnName = "service_config_id") http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java index ecc5c11..ac1c9b5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java @@ -21,6 +21,7 @@ package org.apache.ambari.server.state; import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -70,6 +71,10 @@ public class ServiceInfo { @XmlElement(name="config-type") private List<String> configDependencies; + @XmlElementWrapper(name="excluded-config-types") + @XmlElement(name="config-type") + private Set<String> excludedConfigTypes; + @XmlTransient private Map<String, Map<String, Map<String, String>>> configTypes; @@ -452,4 +457,15 @@ public class ServiceInfo { public File getAlertsFile() { return alertsFile; } + + /** + * @return config types this service contains configuration for, but which are primarily related to another service + */ + public Set<String> getExcludedConfigTypes() { + return excludedConfigTypes; + } + + public void setExcludedConfigTypes(Set<String> excludedConfigTypes) { + this.excludedConfigTypes = excludedConfigTypes; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 472c7a6..3daf32a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -235,13 +235,18 @@ public class ClusterImpl implements Cluster { LOG.error("Service config versioning disabled due to exception: ", e); return serviceConfigTypes; } - for (String serviceName : serviceInfoMap.keySet()) { + for (Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) { + String serviceName = entry.getKey(); + ServiceInfo serviceInfo = entry.getValue(); //collect config types for service Set<PropertyInfo> properties = ambariMetaInfo.getProperties(desiredStackVersion.getStackName(), desiredStackVersion.getStackVersion(), serviceName); for (PropertyInfo property : properties) { int extIndex = property.getFilename().indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX); String configType = property.getFilename().substring(0, extIndex); - serviceConfigTypes.put(serviceName, configType); + if (serviceInfo.getExcludedConfigTypes() == null || + !serviceInfo.getExcludedConfigTypes().contains(configType)) { + serviceConfigTypes.put(serviceName, configType); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java index 29b0418..620c076 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java @@ -312,16 +312,18 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog { LOG.error(msg); throw new AmbariException(msg); } else if (!dbAccessor.tableHasData(tableName)) { - String query; + String query = null; if (dbType.equals(Configuration.POSTGRES_DB_NAME)) { query = getPostgresRequestUpgradeQuery(); } else if (dbType.equals(Configuration.ORACLE_DB_NAME)) { query = getOracleRequestUpgradeQuery(); - } else { + } else if (Configuration.MYSQL_DB_NAME.equals(dbType)) { query = getMysqlRequestUpgradeQuery(); } - dbAccessor.executeQuery(query); + if (query != null) { + dbAccessor.executeQuery(query); + } } else { LOG.info("Table {} already filled", tableName); } @@ -329,7 +331,8 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog { // Drop old constraints // ======================================================================== if (Configuration.POSTGRES_DB_NAME.equals(dbType) - || Configuration.MYSQL_DB_NAME.equals(dbType)) { + || Configuration.MYSQL_DB_NAME.equals(dbType) + || Configuration.DERBY_DB_NAME.equals(dbType)) { //recreate old constraints to sync with oracle dbAccessor.dropConstraint("clusterconfigmapping", "FK_clusterconfigmapping_cluster_id"); http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java index 9fbed00..31123ce 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java @@ -206,6 +206,95 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { } addAlertingFrameworkDDL(); + + //service config versions changes + + //remove old artifacts (for versions <=1.4.1) which depend on tables changed + //actually these had to be dropped in UC150, but some of tables were never used, and others were just cleared + if (dbAccessor.tableExists("componentconfigmapping")) { + dbAccessor.dropTable("componentconfigmapping"); + } + if (dbAccessor.tableExists("hostcomponentconfigmapping")) { + dbAccessor.dropTable("hostcomponentconfigmapping"); + } + if (dbAccessor.tableExists("hcdesiredconfigmapping")) { + dbAccessor.dropTable("hcdesiredconfigmapping"); + } + if (dbAccessor.tableExists("serviceconfigmapping")) { + dbAccessor.dropTable("serviceconfigmapping"); + } + + dbAccessor.dropConstraint("confgroupclusterconfigmapping", "FK_confg"); + + if (Configuration.ORACLE_DB_NAME.equals(dbType) + || Configuration.MYSQL_DB_NAME.equals(dbType) + || Configuration.DERBY_DB_NAME.equals(dbType)) { + dbAccessor.executeQuery("ALTER TABLE clusterconfig DROP PRIMARY KEY", true); + } else if (Configuration.POSTGRES_DB_NAME.equals(dbType)) { + dbAccessor.executeQuery("ALTER TABLE clusterconfig DROP CONSTRAINT clusterconfig_pkey CASCADE", true); + } + + dbAccessor.addColumn("clusterconfig", new DBColumnInfo("config_id", Long.class, null, null, true)); + + if (Configuration.ORACLE_DB_NAME.equals(dbType)) { + //sequence looks to be simpler than rownum + if (dbAccessor.tableHasData("clusterconfig")) { + dbAccessor.executeQuery("CREATE SEQUENCE TEMP_SEQ " + + " START WITH 1 " + + " MAXVALUE 999999999999999999999999999 " + + " MINVALUE 1 " + + " NOCYCLE " + + " NOCACHE " + + " NOORDER"); + dbAccessor.executeQuery("UPDATE clusterconfig SET config_id = TEMP_SEQ.NEXTVAL"); + dbAccessor.dropSequence("TEMP_SEQ"); + } + } else if (Configuration.MYSQL_DB_NAME.equals(dbType)) { + if (dbAccessor.tableHasData("clusterconfig")) { + dbAccessor.executeQuery("UPDATE viewinstance " + + "SET config_id = (SELECT @a := @a + 1 FROM (SELECT @a := 1) s)"); + } + } else if (Configuration.POSTGRES_DB_NAME.equals(dbType)) { + if (dbAccessor.tableHasData("clusterconfig")) { + //window functions like row_number were added in 8.4, workaround for earlier versions (redhat/centos 5) + dbAccessor.executeQuery("CREATE SEQUENCE temp_seq START WITH 1"); + dbAccessor.executeQuery("UPDATE clusterconfig SET config_id = nextval('temp_seq')"); + dbAccessor.dropSequence("temp_seq"); + } + } + + //upgrade unit test workaround + if (Configuration.DERBY_DB_NAME.equals(dbType)) { + dbAccessor.executeQuery("ALTER TABLE clusterconfig ALTER COLUMN config_id DEFAULT 0"); + dbAccessor.executeQuery("ALTER TABLE clusterconfig ALTER COLUMN config_id NOT NULL"); + } + + dbAccessor.executeQuery("ALTER TABLE clusterconfig ADD PRIMARY KEY (config_id)"); + + columns.clear(); + columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false)); + columns.add(new DBColumnInfo("cluster_id", Long.class, null, null, false)); + columns.add(new DBColumnInfo("service_name", String.class, null, null, false)); + columns.add(new DBColumnInfo("version", Long.class, null, null, false)); + columns.add(new DBColumnInfo("create_timestamp", Long.class, null, null, false)); + dbAccessor.createTable("serviceconfig", columns, "service_config_id"); + + columns.clear(); + columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false)); + columns.add(new DBColumnInfo("config_id", Long.class, null, null, false)); + dbAccessor.createTable("serviceconfigmapping", columns, "service_config_id", "config_id"); + + columns.clear(); + columns.add(new DBColumnInfo("apply_id", Long.class, null, null, false)); + columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false)); + columns.add(new DBColumnInfo("apply_timestamp", Long.class, null, null, false)); + columns.add(new DBColumnInfo("user_name", String.class, null, "_db", false)); + dbAccessor.createTable("serviceconfigapplication", columns, "apply_id"); + + dbAccessor.addFKConstraint("confgroupclusterconfigmapping", "FK_confg", + new String[]{"cluster_id", "config_type", "version_tag"}, "clusterconfig", + new String[]{"cluster_id", "type_name", "version_tag"}, true); + } http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index b1d2eba..abaf35c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -116,7 +116,7 @@ ALTER TABLE serviceconfigmapping ADD CONSTRAINT FK_scvm_scv FOREIGN KEY (service ALTER TABLE serviceconfigmapping ADD CONSTRAINT FK_scvm_config FOREIGN KEY (config_id) REFERENCES clusterconfig(config_id); ALTER TABLE serviceconfigapplication ADD CONSTRAINT FK_scva_scv FOREIGN KEY (service_config_id) REFERENCES serviceconfig(service_config_id); ALTER TABLE configgroup ADD CONSTRAINT FK_configgroup_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id); -ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_confg FOREIGN KEY (version_tag, config_type, cluster_id) REFERENCES clusterconfig (version_tag, type_name, cluster_id); +ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_confg FOREIGN KEY (cluster_id, config_type, version_tag) REFERENCES clusterconfig (cluster_id, type_name, version_tag); ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_cgccm_gid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id); ALTER TABLE configgrouphostmapping ADD CONSTRAINT FK_cghm_cgid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id); ALTER TABLE configgrouphostmapping ADD CONSTRAINT FK_cghm_hname FOREIGN KEY (host_name) REFERENCES hosts (host_name); http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml index 2157fb3..afd2f74 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml @@ -181,6 +181,13 @@ <config-type>hive-env</config-type> </configuration-dependencies> + <excluded-config-types> + <config-type>hive-env</config-type> + <config-type>hive-site</config-type> + <config-type>hive-exec-log4j</config-type> + <config-type>hive-log4j</config-type> + </excluded-config-types> + </service> </services> http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml index a3dc4fd..5433c8a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml @@ -193,6 +193,12 @@ <config-type>hive-site</config-type> <config-type>hive-env</config-type> </configuration-dependencies> + <excluded-config-types> + <config-type>hive-env</config-type> + <config-type>hive-site</config-type> + <config-type>hive-exec-log4j</config-type> + <config-type>hive-log4j</config-type> + </excluded-config-types> </service> </services> http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml index e24b13c..bc66161 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml @@ -92,6 +92,10 @@ <config-type>falcon-runtime.properties</config-type> </configuration-dependencies> + <excluded-config-types> + <config-type>oozie-site</config-type> + </excluded-config-types> + </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/1aa5bcdd/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java index 60b0eb1..f651d10 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java @@ -34,6 +34,8 @@ import org.apache.ambari.server.security.CertificateManager; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.utils.VersionUtils; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +47,7 @@ import java.util.*; import static org.junit.Assert.assertTrue; +@RunWith(Parameterized.class) public class UpgradeTest { private static final Logger LOG = LoggerFactory.getLogger(UpgradeTest.class); @@ -53,11 +56,14 @@ public class UpgradeTest { "1.4.3", "1.4.2", "1.4.1", "1.4.0", "1.2.5", "1.2.4", "1.2.3"); //TODO add all private static String DROP_DERBY_URL = "jdbc:derby:memory:myDB/ambari;drop=true"; + + private final String sourceVersion; private Properties properties = new Properties(); private Injector injector; - public UpgradeTest() { + public UpgradeTest(String sourceVersion) { + this.sourceVersion = sourceVersion; properties.setProperty(Configuration.SERVER_PERSISTENCE_TYPE_KEY, "remote"); properties.setProperty(Configuration.SERVER_JDBC_URL_KEY, Configuration.JDBC_IN_MEMORY_URL); properties.setProperty(Configuration.SERVER_JDBC_DRIVER_KEY, Configuration.JDBC_IN_MEMROY_DRIVER); @@ -79,27 +85,15 @@ public class UpgradeTest { } String targetVersion = getLastVersion(); - List<String> failedVersions = new ArrayList<String>(); - - for (String version : VERSIONS) { - injector = Guice.createInjector(new ControllerModule(properties)); - - try { - createSourceDatabase(version); - - performUpgrade(targetVersion); - - testUpgradedSchema(); - } catch (Exception e) { - failedVersions.add(version); - e.printStackTrace(); - } - dropDatabase(); + injector = Guice.createInjector(new ControllerModule(properties)); + LOG.info("Testing upgrade from version {} to {}", sourceVersion, targetVersion); - } + createSourceDatabase(sourceVersion); + performUpgrade(targetVersion); + testUpgradedSchema(); - assertTrue("Upgrade test failed for version: " + failedVersions, failedVersions.isEmpty()); + dropDatabase(); } @@ -215,5 +209,14 @@ public class UpgradeTest { } + @Parameterized.Parameters + public static Collection<Object[]> data() { + Collection<Object[]> data = new ArrayList<Object[]>(); + for (String s : VERSIONS) { + data.add(new Object[]{s}); + } + return data; + } + }
