http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog211.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog211.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog211.java deleted file mode 100644 index eb835ef..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog211.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.upgrade; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.MessageFormat; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.configuration.Configuration.DatabaseType; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; -import org.apache.ambari.server.orm.dao.DaoUtils; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.jdbc.support.JdbcUtils; - -import com.google.inject.Inject; -import com.google.inject.Injector; - - -/** - * Upgrade catalog for version 2.1.1. - */ -public class UpgradeCatalog211 extends AbstractUpgradeCatalog { - private static final String HOST_COMPONENT_STATE_TABLE = "hostcomponentstate"; - private static final String HOST_COMPONENT_STATE_ID_COLUMN = "id"; - private static final String HOST_COMPONENT_STATE_INDEX = "idx_host_component_state"; - - /** - * Logger. - */ - private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog211.class); - - // this "id holder" is a field only for a test that verifies "big" 4 digit+ - // numbers are formatted correctly - private AtomicLong m_hcsId = new AtomicLong(1); - - - @Inject - DaoUtils daoUtils; - - // ----- Constructors ------------------------------------------------------ - - /** - * Don't forget to register new UpgradeCatalogs in {@link SchemaUpgradeHelper.UpgradeHelperModule#configure()} - * - * @param injector Guice injector to track dependencies and uses bindings to inject them. - */ - @Inject - public UpgradeCatalog211(Injector injector) { - super(injector); - - daoUtils = injector.getInstance(DaoUtils.class); - } - - // ----- UpgradeCatalog ---------------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getTargetVersion() { - return "2.1.1"; - } - - // ----- AbstractUpgradeCatalog -------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getSourceVersion() { - return "2.1.0"; - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDDLUpdates() throws AmbariException, SQLException { - // change out the PK on hostcomponentstate - executeHostComponentStateDDLUpdates(); - - // make viewinstanceproperty.value & viewinstancedata.value nullable - dbAccessor.setColumnNullable("viewinstanceproperty", "value", true); - dbAccessor.setColumnNullable("viewinstancedata", "value", true); - - } - - /** - * {@inheritDoc} - */ - @Override - protected void executePreDMLUpdates() throws AmbariException, SQLException { - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDMLUpdates() throws AmbariException, SQLException { - addNewConfigurationsFromXml(); - updateExistingConfigurations(); - } - - // ----- UpgradeCatalog211 -------------------------------------------- - - /** - * Iterates over the set of clusters to call service-specific configuration - * update routines. - * - * @throws AmbariException - * if an error occurs while updating the configurations - */ - protected void updateExistingConfigurations() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if ((clusterMap != null) && !clusterMap.isEmpty()) { - // Iterate through the clusters and perform any configuration updates - for (final Cluster cluster : clusterMap.values()) { - updateKerberosConfigurations(cluster); - - /* ********************************************************* - * Add additional configuration update methods here - * ********************************************************* */ - } - } - } - } - - /** - * Updates the Kerberos configurations for the given cluster - * <p/> - * Performs the following updates: - * <ul> - * <li>Rename <code>create_attributes_template</code> to - * <code>ad_create_attributes_template</code></li> - * </ul> - * - * @param cluster - * the cluster - * @throws AmbariException - * if an error occurs while updating the configurations - */ - protected void updateKerberosConfigurations(Cluster cluster) throws AmbariException { - Config config = cluster.getDesiredConfigByType("kerberos-env"); - - if (config != null) { - // Rename create_attributes_template to ad_create_attributes_template - String value = config.getProperties().get("create_attributes_template"); - Map<String, String> updates = Collections.singletonMap("ad_create_attributes_template", value); - Set<String> removes = Collections.singleton("create_attributes_template"); - - updateConfigurationPropertiesForCluster(cluster, "kerberos-env", updates, removes, true, false); - } - } - - /** - * Perform the DDL updates required to add a new Primary Key ID column to the - * {@code hostcomponentstate} table. This will perform the following actions: - * <ul> - * <li>Add a new column to hostcomponentstate named id</li> - * <li>Populated id with an incrementing long, then make it non-NULL</li> - * <li>Drop the existing PK on hostcomponentstate</li> - * <li>Add a new surrogate PK on hostcomponentstate on the id column</li> - * <li>Add an index on hostcomponentstate for host_id, component_name, - * service_name, cluster_id</li> - * </ul> - * - * @throws AmbariException - * @throws SQLException - */ - private void executeHostComponentStateDDLUpdates() throws AmbariException, SQLException { - if (!dbAccessor.tableHasPrimaryKey(HOST_COMPONENT_STATE_TABLE, HOST_COMPONENT_STATE_ID_COLUMN)) { - // add the new column, nullable for now until we insert unique IDs - dbAccessor.addColumn(HOST_COMPONENT_STATE_TABLE, - new DBColumnInfo(HOST_COMPONENT_STATE_ID_COLUMN, Long.class, null, null, true)); - - Statement statement = null; - ResultSet resultSet = null; - try { - statement = dbAccessor.getConnection().createStatement(); - if (statement != null) { - String selectSQL = MessageFormat.format( - "SELECT id, cluster_id, service_name, component_name, host_id FROM {0} ORDER BY {1} {2}", - HOST_COMPONENT_STATE_TABLE, "id", "DESC"); - - resultSet = statement.executeQuery(selectSQL); - while (resultSet.next()) { - final Long clusterId = resultSet.getLong("cluster_id"); - final String serviceName = resultSet.getString("service_name"); - final String componentName = resultSet.getString("component_name"); - final Long hostId = resultSet.getLong("host_id"); - final Long idKey = resultSet.getLong("id"); - - if (idKey != 0 && m_hcsId.get() == 1) { - m_hcsId.set(idKey); - m_hcsId.getAndIncrement(); - } else if(idKey == 0) { - String updateSQL = MessageFormat.format( - "UPDATE {0} SET {1} = {2,number,#} WHERE cluster_id = {3} AND service_name = ''{4}'' AND component_name = ''{5}'' and host_id = {6,number,#}", - HOST_COMPONENT_STATE_TABLE, HOST_COMPONENT_STATE_ID_COLUMN, m_hcsId.getAndIncrement(), - clusterId, serviceName, componentName, hostId); - - dbAccessor.executeQuery(updateSQL); - } - } - } - } finally { - JdbcUtils.closeResultSet(resultSet); - JdbcUtils.closeStatement(statement); - } - - // make the column NON NULL now - dbAccessor.alterColumn(HOST_COMPONENT_STATE_TABLE, - new DBColumnInfo(HOST_COMPONENT_STATE_ID_COLUMN, Long.class, null, null, false)); - - // Add sequence for hostcomponentstate id - addSequence("hostcomponentstate_id_seq", m_hcsId.get(), false); - - // drop the current PK - String primaryKeyConstraintName = null; - Configuration.DatabaseType databaseType = configuration.getDatabaseType(); - switch (databaseType) { - case POSTGRES: { - primaryKeyConstraintName = "hostcomponentstate_pkey"; - break; - } - case ORACLE: - case SQL_SERVER: { - // Oracle and SQL Server require us to lookup the PK name - primaryKeyConstraintName = dbAccessor.getPrimaryKeyConstraintName( - HOST_COMPONENT_STATE_TABLE); - - break; - } - default: - break; - } - - if (databaseType == DatabaseType.MYSQL) { - String mysqlDropQuery = MessageFormat.format("ALTER TABLE {0} DROP PRIMARY KEY", - HOST_COMPONENT_STATE_TABLE); - - dbAccessor.executeQuery(mysqlDropQuery, true); - } else { - // warn if we can't find it - if (null == primaryKeyConstraintName) { - LOG.warn("Unable to determine the primary key constraint name for {}", - HOST_COMPONENT_STATE_TABLE); - } else { - dbAccessor.dropPKConstraint(HOST_COMPONENT_STATE_TABLE, primaryKeyConstraintName, true); - } - } - - // create a new PK, matching the name of the constraint found in the SQL - // files - dbAccessor.addPKConstraint(HOST_COMPONENT_STATE_TABLE, "pk_hostcomponentstate", "id"); - - // create index, ensuring column order matches that of the SQL files - dbAccessor.createIndex(HOST_COMPONENT_STATE_INDEX, HOST_COMPONENT_STATE_TABLE, "host_id", - "component_name", "service_name", "cluster_id"); - } - } -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java deleted file mode 100644 index 8eb2654..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog212.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.upgrade; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; -import org.apache.ambari.server.orm.dao.DaoUtils; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.utils.VersionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.inject.Inject; -import com.google.inject.Injector; - - -/** - * Upgrade catalog for version 2.1.2. - */ -public class UpgradeCatalog212 extends AbstractUpgradeCatalog { - private static final String HIVE_SITE = "hive-site"; - private static final String HIVE_ENV = "hive-env"; - private static final String HBASE_ENV = "hbase-env"; - private static final String HBASE_SITE = "hbase-site"; - private static final String CLUSTER_ENV = "cluster-env"; - private static final String OOZIE_ENV = "oozie-env"; - - private static final String TOPOLOGY_REQUEST_TABLE = "topology_request"; - private static final String CLUSTERS_TABLE = "clusters"; - private static final String CLUSTERS_TABLE_CLUSTER_ID_COLUMN = "cluster_id"; - private static final String TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN = "cluster_name"; - private static final String TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN = "cluster_id"; - private static final String TOPOLOGY_REQUEST_CLUSTER_ID_FK_CONSTRAINT_NAME = "FK_topology_request_cluster_id"; - - private static final String HOST_ROLE_COMMAND_TABLE = "host_role_command"; - private static final String HOST_ROLE_COMMAND_SKIP_COLUMN = "auto_skip_on_failure"; - - /** - * Logger. - */ - private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog212.class); - - @Inject - DaoUtils daoUtils; - - // ----- Constructors ------------------------------------------------------ - - /** - * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()} - * - * @param injector Guice injector to track dependencies and uses bindings to inject them. - */ - @Inject - public UpgradeCatalog212(Injector injector) { - super(injector); - - daoUtils = injector.getInstance(DaoUtils.class); - } - - // ----- UpgradeCatalog ---------------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getTargetVersion() { - return "2.1.2"; - } - - // ----- AbstractUpgradeCatalog -------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getSourceVersion() { - return "2.1.1"; - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDDLUpdates() throws AmbariException, SQLException { - executeTopologyDDLUpdates(); - executeHostRoleCommandDDLUpdates(); - } - - private void executeTopologyDDLUpdates() throws AmbariException, SQLException { - dbAccessor.addColumn(TOPOLOGY_REQUEST_TABLE, new DBColumnInfo(TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN, - Long.class, null, null, true)); - // TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN will be deleted in PreDML. We need a cluster name to set cluster id. - // dbAccessor.dropColumn(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN); - // dbAccessor.setColumnNullable(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN, false); - // dbAccessor.addFKConstraint(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_ID_FK_CONSTRAINT_NAME, - // TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN, CLUSTERS_TABLE, CLUSTERS_TABLE_CLUSTER_ID_COLUMN, false); - } - - /** - * {@inheritDoc} - */ - @Override - protected void executePreDMLUpdates() throws AmbariException, SQLException { - if (dbAccessor.tableHasColumn(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN)) { - addClusterIdToTopology(); - finilizeTopologyDDL(); - } else { - LOG.debug("The column: [ {} ] has already been dropped from table: [ {} ]. Skipping preDMLUpdate logic.", - TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN, TOPOLOGY_REQUEST_TABLE); - } - } - - protected void finilizeTopologyDDL() throws AmbariException, SQLException { - dbAccessor.dropColumn(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_NAME_COLUMN); - dbAccessor.setColumnNullable(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN, false); - dbAccessor.addFKConstraint(TOPOLOGY_REQUEST_TABLE, TOPOLOGY_REQUEST_CLUSTER_ID_FK_CONSTRAINT_NAME, - TOPOLOGY_REQUEST_CLUSTER_ID_COLUMN, CLUSTERS_TABLE, CLUSTERS_TABLE_CLUSTER_ID_COLUMN, false); - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDMLUpdates() throws AmbariException, SQLException { - addNewConfigurationsFromXml(); - addMissingConfigs(); - } - - protected void addClusterIdToTopology() throws AmbariException, SQLException { - Map<String, Long> clusterNameIdMap = new HashMap<>(); - try (Statement statement = dbAccessor.getConnection().createStatement(); - ResultSet rs = statement.executeQuery("SELECT DISTINCT cluster_name, cluster_id FROM clusters"); - ) { - while (rs.next()) { - long clusterId = rs.getLong("cluster_id"); - String clusterName = rs.getString("cluster_name"); - clusterNameIdMap.put(clusterName, clusterId); - } - } - - for (String clusterName : clusterNameIdMap.keySet()) { - try (PreparedStatement preparedStatement = dbAccessor.getConnection().prepareStatement("UPDATE topology_request " + - "SET cluster_id=? WHERE cluster_name=?"); - ) { - preparedStatement.setLong(1, clusterNameIdMap.get(clusterName)); - preparedStatement.setString(2, clusterName); - preparedStatement.executeUpdate(); - } - } - - // Set cluster id for all null values. - // Useful if cluster was renamed and cluster name does not match. - if (clusterNameIdMap.entrySet().size() >= 1) { - try (PreparedStatement preparedStatement = dbAccessor.getConnection().prepareStatement("UPDATE topology_request " + - "SET cluster_id=? WHERE cluster_id IS NULL"); - ) { - preparedStatement.setLong(1, clusterNameIdMap.entrySet().iterator().next().getValue()); - preparedStatement.executeUpdate(); - } - } - if (clusterNameIdMap.entrySet().size() == 0) { - LOG.warn("Cluster not found. topology_request.cluster_id is not set"); - } - if (clusterNameIdMap.entrySet().size() > 1) { - LOG.warn("Found more than one cluster. topology_request.cluster_id can be incorrect if you have renamed the cluster."); - } - } - - protected void addMissingConfigs() throws AmbariException { - updateHiveConfigs(); - updateOozieConfigs(); - updateHbaseAndClusterConfigurations(); - updateKafkaConfigurations(); - updateStormConfigs(); - removeDataDirMountConfig(); - } - - protected void updateStormConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if ((clusterMap != null) && !clusterMap.isEmpty()) { - // Iterate through the clusters and perform any configuration updates - for (final Cluster cluster : clusterMap.values()) { - Set<String> removes = new HashSet<>(); - removes.add("topology.metrics.consumer.register"); - updateConfigurationPropertiesForCluster(cluster, "storm-site", - new HashMap<String, String>(), removes, false, false); - } - } - } - } - - protected void updateKafkaConfigurations() throws AmbariException { - Map<String, String> properties = new HashMap<>(); - properties.put("external.kafka.metrics.exclude.prefix", - "kafka.network.RequestMetrics,kafka.server.DelayedOperationPurgatory," + - "kafka.server.BrokerTopicMetrics.BytesRejectedPerSec"); - properties.put("external.kafka.metrics.include.prefix", - "kafka.network.RequestMetrics.ResponseQueueTimeMs.request.OffsetCommit.98percentile," + - "kafka.network.RequestMetrics.ResponseQueueTimeMs.request.Offsets.95percentile," + - "kafka.network.RequestMetrics.ResponseSendTimeMs.request.Fetch.95percentile," + - "kafka.network.RequestMetrics.RequestsPerSec.request"); - - updateConfigurationProperties("kafka-broker", properties, false, false); - } - - protected void updateHbaseAndClusterConfigurations() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if ((clusterMap != null) && !clusterMap.isEmpty()) { - // Iterate through the clusters and perform any configuration updates - for (final Cluster cluster : clusterMap.values()) { - Config hbaseEnvProps = cluster.getDesiredConfigByType(HBASE_ENV); - Config hbaseSiteProps = cluster.getDesiredConfigByType(HBASE_SITE); - - if (hbaseEnvProps != null) { - // Remove override_hbase_uid from hbase-env and add override_uid to cluster-env - String value = hbaseEnvProps.getProperties().get("override_hbase_uid"); - if (value != null) { - Map<String, String> updates = new HashMap<>(); - Set<String> removes = new HashSet<>(); - updates.put("override_uid", value); - removes.add("override_hbase_uid"); - updateConfigurationPropertiesForCluster(cluster, HBASE_ENV, new HashMap<String, String>(), removes, false, true); - updateConfigurationPropertiesForCluster(cluster, CLUSTER_ENV, updates, true, false); - } else { - updateOverrideUIDClusterConfig("false", cluster); - } - } else { - updateOverrideUIDClusterConfig("false", cluster); - } - - if (hbaseSiteProps != null) { - String value = hbaseSiteProps.getProperties().get("hbase.bucketcache.size"); - if (value != null) { - if (value.endsWith("m")) { - value = value.substring(0, value.length() - 1); - Map<String, String> updates = new HashMap<>(); - updates.put("hbase.bucketcache.size", value); - updateConfigurationPropertiesForCluster(cluster, HBASE_SITE, updates, true, false); - } - } - } - } - } - } - } - - /** - * Set override_uid to false during the upgrade to retain UIDs already set on the cluster - * This is necessary for upgrading a third party Ambari/stack distribution from - * Ambari version 2.1.0 where HBase does not have override_hbase_uid. - * */ - private void updateOverrideUIDClusterConfig(String toOverride, Cluster cluster) throws AmbariException{ - Map<String, String> updates = new HashMap<>(); - updates.put("override_uid", toOverride); - updateConfigurationPropertiesForCluster(cluster, CLUSTER_ENV, updates, true, false); - } - - protected void updateHiveConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance( - AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - Service service = cluster.getServices().get("HIVE"); - - if (null == service) { - continue; - } - - StackId stackId = service.getDesiredStackId(); - - String content = null; - Boolean isHiveSitePresent = cluster.getDesiredConfigByType(HIVE_SITE) != null; - Boolean isStackNotLess22 = (stackId != null && stackId.getStackName().equals("HDP") && - VersionUtils.compareVersions(stackId.getStackVersion(), "2.2") >= 0); - - if (cluster.getDesiredConfigByType(HIVE_ENV) != null && isStackNotLess22) { - Map<String, String> hiveEnvProps = new HashMap<>(); - content = cluster.getDesiredConfigByType(HIVE_ENV).getProperties().get("content"); - if(content != null) { - content = updateHiveEnvContent(content); - hiveEnvProps.put("content", content); - } - updateConfigurationPropertiesForCluster(cluster, HIVE_ENV, hiveEnvProps, true, true); - } - - if (isHiveSitePresent && isStackNotLess22) { - Set<String> hiveSiteRemoveProps = new HashSet<>(); - hiveSiteRemoveProps.add("hive.heapsize"); - hiveSiteRemoveProps.add("hive.optimize.mapjoin.mapreduce"); - hiveSiteRemoveProps.add("hive.server2.enable.impersonation"); - hiveSiteRemoveProps.add("hive.auto.convert.sortmerge.join.noconditionaltask"); - - updateConfigurationPropertiesForCluster(cluster, HIVE_SITE, new HashMap<String, String>(), hiveSiteRemoveProps, false, true); - } - } - } - } - } - - protected void updateOozieConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - Config oozieEnv = cluster.getDesiredConfigByType(OOZIE_ENV); - if (oozieEnv != null) { - Map<String, String> oozieEnvProperties = oozieEnv.getProperties(); - - String hostname = oozieEnvProperties.get("oozie_hostname"); - String db_type = oozieEnvProperties.get("oozie_database"); - String final_db_host = null; - // fix for empty hostname after 1.7 -> 2.1.x+ upgrade - if (hostname != null && db_type != null && hostname.equals("")) { - switch (db_type.toUpperCase()) { - case "EXISTING MYSQL DATABASE": - final_db_host = oozieEnvProperties.get("oozie_existing_mysql_host"); - break; - case "EXISTING POSTGRESQL DATABASE": - final_db_host = oozieEnvProperties.get("oozie_existing_postgresql_host"); - break; - case "EXISTING ORACLE DATABASE": - final_db_host = oozieEnvProperties.get("oozie_existing_oracle_host"); - break; - default: - final_db_host = null; - break; - } - if (final_db_host != null) { - Map<String, String> newProperties = new HashMap<>(); - newProperties.put("oozie_hostname", final_db_host); - updateConfigurationPropertiesForCluster(cluster, OOZIE_ENV, newProperties, true, true); - } - } - } - } - } - } - } - - protected String updateHiveEnvContent(String hiveEnvContent) { - if(hiveEnvContent == null) { - return null; - } - String oldHeapSizeRegex = "export HADOOP_HEAPSIZE=\"\\{\\{hive_heapsize\\}\\}\"\\s*\\n" + - "export HADOOP_CLIENT_OPTS=\"-Xmx\\$\\{HADOOP_HEAPSIZE\\}m \\$HADOOP_CLIENT_OPTS\""; - String newAuxJarPath = ""; - return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newAuxJarPath)); - } - - /** - * DDL changes for {@link #HOST_ROLE_COMMAND_TABLE}. - * - * @throws AmbariException - * @throws SQLException - */ - private void executeHostRoleCommandDDLUpdates() throws AmbariException, SQLException { - dbAccessor.addColumn(HOST_ROLE_COMMAND_TABLE, - new DBColumnInfo(HOST_ROLE_COMMAND_SKIP_COLUMN, Integer.class, 1, 0, false)); - } - - protected void removeDataDirMountConfig() throws AmbariException { - Set<String> properties = new HashSet<>(); - properties.add("dfs.datanode.data.dir.mount.file"); - - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - if (clusterMap != null && !clusterMap.isEmpty()) { - for (final Cluster cluster : clusterMap.values()) { - removeConfigurationPropertiesFromCluster(cluster, "hadoop-env", properties); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2121.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2121.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2121.java deleted file mode 100644 index ab41b99..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2121.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.upgrade; - -import java.sql.SQLException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.orm.dao.DaoUtils; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.DesiredConfig; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.StackId; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.inject.Inject; -import com.google.inject.Injector; - -/** - * Upgrade catalog for version 2.1.2.1 - */ -public class UpgradeCatalog2121 extends AbstractUpgradeCatalog { - - /** - * Logger. - */ - private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog2121.class); - - @Inject - DaoUtils daoUtils; - - private static final String OOZIE_SITE_CONFIG = "oozie-site"; - private static final String OOZIE_AUTHENTICATION_KERBEROS_NAME_RULES = "oozie.authentication.kerberos.name.rules"; - - // ----- Constructors ------------------------------------------------------ - - /** - * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()} - * - * @param injector Guice injector to track dependencies and uses bindings to inject them. - */ - @Inject - public UpgradeCatalog2121(Injector injector) { - super(injector); - - daoUtils = injector.getInstance(DaoUtils.class); - } - - // ----- UpgradeCatalog ---------------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getTargetVersion() { - return "2.1.2.1"; - } - - // ----- AbstractUpgradeCatalog -------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getSourceVersion() { - return "2.1.2"; - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDDLUpdates() throws AmbariException, SQLException { - - } - - /** - * {@inheritDoc} - */ - @Override - protected void executePreDMLUpdates() throws AmbariException, SQLException { - - } - - /** - * {@inheritDoc} - */ - @Override - protected void executeDMLUpdates() throws AmbariException, SQLException { - updatePHDConfigs(); - updateOozieConfigs(); - } - - /** - * Update PHD stack configs - * @throws AmbariException - */ - protected void updatePHDConfigs() throws AmbariException { - - Map<String, String> replacements = new LinkedHashMap<>(); - replacements.put("-Dstack.name=\\{\\{\\s*stack_name\\s*\\}\\}\\s*", ""); - replacements.put("-Dstack.name=\\$\\{stack.name\\}\\s*", ""); - replacements.put("-Dstack.version=\\{\\{\\s*stack_version_buildnum\\s*\\}\\}", "-Dhdp.version=\\$HDP_VERSION"); - replacements.put("-Dstack.version=\\$\\{stack.version\\}", "-Dhdp.version=\\$\\{hdp.version\\}"); - replacements.put("\\{\\{\\s*stack_name\\s*\\}\\}", "phd"); - replacements.put("\\$\\{stack.name\\}", "phd"); - replacements.put("\\$\\{stack.version\\}", "\\$\\{hdp.version\\}"); - - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - if (clusters != null) { - Map<String, Cluster> clusterMap = clusters.getClusters(); - if ((clusterMap != null) && !clusterMap.isEmpty()) { - // Iterate through the clusters and perform any configuration updates - Set<StackId> stackIds = new HashSet<>(); - - for (final Cluster cluster : clusterMap.values()) { - for (Service service : cluster.getServices().values()) { - StackId currentStackVersion = service.getDesiredStackId(); - - if (stackIds.contains(currentStackVersion)) { - continue; - } else { - stackIds.add(currentStackVersion); - } - - String currentStackName = currentStackVersion != null? currentStackVersion.getStackName() : null; - if (currentStackName != null && currentStackName.equalsIgnoreCase("PHD")) { - // Update configs only if PHD stack is deployed - Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); - if(desiredConfigs != null && !desiredConfigs.isEmpty()) { - for (Map.Entry<String, DesiredConfig> dc : desiredConfigs.entrySet()) { - String configType = dc.getKey(); - DesiredConfig desiredConfig = dc.getValue(); - String configTag = desiredConfig.getTag(); - Config config = cluster.getConfig(configType, configTag); - - Map<String, String> properties = config.getProperties(); - if(properties != null && !properties.isEmpty()) { - Map<String, String> updates = new HashMap<>(); - for (Map.Entry<String, String> property : properties.entrySet()) { - String propertyKey = property.getKey(); - String propertyValue = property.getValue(); - String modifiedPropertyValue = propertyValue; - for (String regex : replacements.keySet()) { - modifiedPropertyValue = modifiedPropertyValue.replaceAll(regex, replacements.get(regex)); - } - if (!modifiedPropertyValue.equals(propertyValue)) { - updates.put(propertyKey, modifiedPropertyValue); - } - } - if (!updates.isEmpty()) { - updateConfigurationPropertiesForCluster(cluster, configType, updates, true, false); - } - } - } - } - } - } - } - } - } - } - - protected void updateOozieConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { - Config oozieSiteProps = cluster.getDesiredConfigByType(OOZIE_SITE_CONFIG); - if (oozieSiteProps != null) { - // Remove oozie.authentication.kerberos.name.rules if empty - String oozieAuthKerbRules = oozieSiteProps.getProperties().get(OOZIE_AUTHENTICATION_KERBEROS_NAME_RULES); - if (StringUtils.isBlank(oozieAuthKerbRules)) { - Set<String> removeProperties = new HashSet<>(); - removeProperties.add(OOZIE_AUTHENTICATION_KERBEROS_NAME_RULES); - updateConfigurationPropertiesForCluster(cluster, OOZIE_SITE_CONFIG, new HashMap<String, String>(), removeProperties, true, false); - } - } - } - - } -} -