AMBARI-21490 Ambari Schema Upgrade fails : Unknown column 'from_version' in 'field list' (dgrinenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/651fe3d7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/651fe3d7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/651fe3d7 Branch: refs/heads/branch-feature-AMBARI-21348 Commit: 651fe3d706185992e107486ee69059ec8eccf909 Parents: 606c5ca Author: Dmytro Grinenko <[email protected]> Authored: Mon Jul 17 18:51:26 2017 +0300 Committer: Dmytro Grinenko <[email protected]> Committed: Mon Jul 17 18:51:26 2017 +0300 ---------------------------------------------------------------------- .../apache/ambari/server/orm/DBAccessor.java | 14 ++ .../ambari/server/orm/DBAccessorImpl.java | 24 +++ .../server/upgrade/UpgradeCatalog220.java | 197 +------------------ .../server/upgrade/UpgradeCatalog252.java | 11 +- 4 files changed, 48 insertions(+), 198 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/651fe3d7/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java index 899426e..0313698 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java @@ -680,6 +680,20 @@ public interface DBAccessor { void moveColumnToAnotherTable(String sourceTableName, DBColumnInfo sourceColumn, String sourceIDFieldName, String targetTableName, DBColumnInfo targetColumn, String targetIDFieldName, Object initialValue) throws SQLException; + /** + * Remove all rows from the table + * @param tableName name of the table + */ + void clearTable(String tableName) throws SQLException; + + /** + * Reset all rows with {@code value} for {@code columnName} column + * @param tableName name of the table + * @param columnName name of the column name to be update + * @param value data to use for update + */ + void clearTableColumn(String tableName, String columnName, Object value) throws SQLException; + enum DbType { ORACLE, MYSQL, http://git-wip-us.apache.org/repos/asf/ambari/blob/651fe3d7/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java index 2256d44..38d60e9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java @@ -1442,4 +1442,28 @@ public class DBAccessorImpl implements DBAccessor { dropColumn(sourceTableName, sourceColumn.getName()); } } + + /** + * Remove all rows from the table + * + * @param tableName name of the table + */ + @Override + public void clearTable(String tableName) throws SQLException{ + String sqlQuery = "DELETE FROM " + convertObjectName(tableName); + executeQuery(sqlQuery); + } + + /** + * Reset all rows with {@code value} for {@code columnName} column + * + * @param tableName name of the table + * @param columnName + * @param value data to use for update + */ + @Override + public void clearTableColumn(String tableName, String columnName, Object value) throws SQLException { + String sqlQuery = String.format("UPDATE %s SET %s = ?", convertObjectName(tableName), convertObjectName(columnName)); + executePreparedUpdate(sqlQuery, value); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/651fe3d7/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java index 8ee7943..0bf60af 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java @@ -455,22 +455,10 @@ public class UpgradeCatalog220 extends AbstractUpgradeCatalog { protected void executeStackUpgradeDDLUpdates() throws SQLException, AmbariException { final Configuration.DatabaseType databaseType = configuration.getDatabaseType(); - // Add columns - if (!dbAccessor.tableHasColumn(UPGRADE_TABLE, UPGRADE_PACKAGE_COL)) { - LOG.info("Adding upgrade_package column to upgrade table."); - dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_PACKAGE_COL, String.class, 255, null, true)); - } - if (!dbAccessor.tableHasColumn(UPGRADE_TABLE, UPGRADE_TYPE_COL)) { - LOG.info("Adding upgrade_type column to upgrade table."); - dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_TYPE_COL, String.class, 32, null, true)); - } - - // Populate values in upgrade table. - boolean success = populateUpgradeTable(); + dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_PACKAGE_COL, String.class, 255, null, true)); + dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_TYPE_COL, String.class, 32, null, true)); - if (!success) { - throw new AmbariException("Errors found while populating the upgrade table with values for columns upgrade_type and upgrade_package."); - } + // Tip: skipping data update as we clearing this table at UpgradeCatalog252 DDL if (dbAccessor.tableHasColumn(REPO_VERSION_TABLE, UPGRADE_PACKAGE_COL)) { LOG.info("Dropping upgrade_package column from repo_version table."); @@ -500,184 +488,7 @@ public class UpgradeCatalog220 extends AbstractUpgradeCatalog { } } - /** - * Populate the upgrade table with values for the columns upgrade_type and upgrade_package. - * The upgrade_type will default to {@code org.apache.ambari.server.state.stack.upgrade.UpgradeType.ROLLING} - * whereas the upgrade_package will be calculated. - * @return {@code} true on success, and {@code} false otherwise. - */ - private boolean populateUpgradeTable() { - boolean success = true; - Statement statement = null; - ResultSet rs = null; - try { - statement = dbAccessor.getConnection().createStatement(); - if (statement != null) { - // Need to use SQL since the schema is changing and some of the columns have not yet been added.. - rs = statement.executeQuery("SELECT upgrade_id, cluster_id, from_version, to_version, direction, upgrade_package, upgrade_type FROM upgrade"); - if (rs != null) { - try { - while (rs.next()) { - final long upgradeId = rs.getLong("upgrade_id"); - final long clusterId = rs.getLong("cluster_id"); - final String fromVersion = rs.getString("from_version"); - final String toVersion = rs.getString("to_version"); - final Direction direction = Direction.valueOf(rs.getString("direction")); - // These two values are likely null. - String upgradePackage = rs.getString("upgrade_package"); - String upgradeType = rs.getString("upgrade_type"); - - LOG.info(MessageFormat.format("Populating rows for the upgrade table record with " + - "upgrade_id: {0,number,#}, cluster_id: {1,number,#}, from_version: {2}, to_version: {3}, direction: {4}", - upgradeId, clusterId, fromVersion, toVersion, direction)); - - // Set all upgrades that have been done so far to type "rolling" - if (StringUtils.isEmpty(upgradeType)) { - LOG.info("Updating the record's upgrade_type to " + UpgradeType.ROLLING); - dbAccessor.executeQuery("UPDATE upgrade SET upgrade_type = '" + UpgradeType.ROLLING + "' WHERE upgrade_id = " + upgradeId); - } - - if (StringUtils.isEmpty(upgradePackage)) { - String version = null; - StackEntity stack = null; - - if (direction == Direction.UPGRADE) { - version = toVersion; - } else if (direction == Direction.DOWNGRADE) { - // TODO AMBARI-12698, this is going to be a problem. - // During a downgrade, the "to_version" is overwritten to the source version, but the "from_version" - // doesn't swap. E.g., - // upgrade_id | from_version | to_version | direction - // ------------+--------------+--------------+---------- - // 1 | 2.2.6.0-2800 | 2.3.0.0-2557 | UPGRADE - // 2 | 2.2.6.0-2800 | 2.2.6.0-2800 | DOWNGRADE - version = fromVersion; - } - - ClusterEntity cluster = clusterDAO.findById(clusterId); - - if (null != cluster) { - stack = cluster.getDesiredStack(); - upgradePackage = calculateUpgradePackage(stack, version); - } else { - LOG.error("Could not find a cluster with cluster_id " + clusterId); - } - - if (!StringUtils.isEmpty(upgradePackage)) { - LOG.info("Updating the record's upgrade_package to " + upgradePackage); - dbAccessor.executeQuery("UPDATE upgrade SET upgrade_package = '" + upgradePackage + "' WHERE upgrade_id = " + upgradeId); - } else { - success = false; - LOG.error("Unable to populate column upgrade_package for record in table upgrade with id " + upgradeId); - } - } - } - } catch (Exception e) { - success = false; - e.printStackTrace(); - LOG.error("Unable to populate the upgrade_type and upgrade_package columns of the upgrade table. " + e); - } - } - } - } catch (Exception e) { - success = false; - e.printStackTrace(); - LOG.error("Failed to retrieve records from the upgrade table to populate the upgrade_type and upgrade_package columns. Exception: " + e); - } finally { - try { - if (rs != null) { - rs.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - ; - } - } - return success; - } - - /** - * Find the single Repo Version for the given stack and version, and return - * its upgrade_package column. Because the upgrade_package column is going to - * be removed from this entity, must use raw SQL instead of the entity class. - * <p/> - * It's possible that there is an invalid version listed in the upgrade table. - * For example: - * - * <pre> - * upgrade - * 1 2 1295 2.2.0.0-2041 2.2.4.2-2 UPGRADE - * 2 2 1296 2.2.0.0-2041 2.2.0.0-2041 DOWNGRADE - * 3 2 1299 2.2.0.0-2041 2.2.4.2 UPGRADE - * - * repo_version - * 1 2.2.0.0-2041 HDP-2.2.0.0-2041 upgrade-2.2 - * 2 2.2.4.2-2 HDP-2.2.4.2-2 upgrade-2.2 - * </pre> - * - * Notice that it's possible for the {@code upgrade} table to include entries - * for a repo version which does not exist; {@code 2.2.4.2}. In these cases, - * this method will attempt a "best match". - * - * @param stack - * Stack - * @param version - * Stack version - * @return The value of the upgrade_package column, or null if not found. - */ - - private String calculateUpgradePackage(StackEntity stack, String version) { - String upgradePackage = null; - // Find the corresponding repo_version, and extract its upgrade_package - if (null != version && null != stack) { - RepositoryVersionEntity repoVersion = repositoryVersionDAO.findByStackNameAndVersion(stack.getStackName(), version); - - // a null repoVersion means there's mismatch between the upgrade and repo_version table; - // use a best-guess approach based on the Stack - if( null == repoVersion ){ - List<RepositoryVersionEntity> bestMatches = repositoryVersionDAO.findByStack(new StackId(stack)); - if (!bestMatches.isEmpty()) { - repoVersion = bestMatches.get(0); - } - } - - // our efforts have failed; we have no idea what to use; return null as per the contract of the method - if( null == repoVersion ) { - return null; - } - - Statement statement = null; - ResultSet rs = null; - try { - statement = dbAccessor.getConnection().createStatement(); - if (statement != null) { - // Need to use SQL since the schema is changing and the entity will no longer have the upgrade_package column. - rs = statement.executeQuery("SELECT upgrade_package FROM repo_version WHERE repo_version_id = " + repoVersion.getId()); - if (rs != null && rs.next()) { - upgradePackage = rs.getString("upgrade_package"); - } - } - } catch (Exception e) { - LOG.error("Failed to retrieve upgrade_package for repo_version record with id " + repoVersion.getId() + ". Exception: " + e.getMessage()); - } finally { - try { - if (rs != null) { - rs.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - ; - } - } - } - return upgradePackage; - } - - /** + /** * {@inheritDoc} */ @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/651fe3d7/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java index 6d2ab84..3c8686c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java @@ -127,11 +127,12 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog { * @throws SQLException */ private void addRepositoryColumnsToUpgradeTable() throws SQLException { - dbAccessor.executeQuery(String.format("UPDATE %s SET %s = NULL", CLUSTERS_TABLE, UPGRADE_ID_COLUMN)); - dbAccessor.executeQuery(String.format("DELETE FROM %s", SERVICE_COMPONENT_HISTORY_TABLE)); - dbAccessor.executeQuery(String.format("DELETE FROM %s", UPGRADE_ITEM_TABLE)); - dbAccessor.executeQuery(String.format("DELETE FROM %s", UPGRADE_GROUP_TABLE)); - dbAccessor.executeQuery(String.format("DELETE FROM %s", UPGRADE_TABLE)); + dbAccessor.clearTableColumn(CLUSTERS_TABLE, UPGRADE_ID_COLUMN, null); + dbAccessor.clearTable(SERVICE_COMPONENT_HISTORY_TABLE); + dbAccessor.clearTable(SERVICE_COMPONENT_HISTORY_TABLE); + dbAccessor.clearTable(UPGRADE_ITEM_TABLE); + dbAccessor.clearTable(UPGRADE_GROUP_TABLE); + dbAccessor.clearTable(UPGRADE_TABLE); dbAccessor.dropColumn(UPGRADE_TABLE, "to_version"); dbAccessor.dropColumn(UPGRADE_TABLE, "from_version");
