HIVE-11911 : The stats table limits are too large for innodb (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f73157fe Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f73157fe Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f73157fe Branch: refs/heads/beeline-cli Commit: f73157fe45a0c9ea7efeef11ca1c02e47136a63c Parents: cdc65dc Author: Sergey Shelukhin <[email protected]> Authored: Wed Sep 23 14:39:23 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Wed Sep 23 14:39:23 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java | 13 +++++++++++-- .../hive/ql/stats/jdbc/JDBCStatsSetupConstants.java | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f73157fe/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java index 4228957..aeb3d27 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java @@ -289,7 +289,16 @@ public class JDBCStatsPublisher implements StatsPublisher { boolean tblExists = rs.next(); if (!tblExists) { // Table does not exist, create it String createTable = JDBCStatsUtils.getCreate(""); - stmt.executeUpdate(createTable); + try { + stmt.executeUpdate(createTable); + } catch (SQLException ex) { + String msg = ex.getMessage(); + if (msg != null && msg.contains("Specified key was too long")) { + throw new RuntimeException(msg + "; try using innodb with " + + "Barracuda file format and innodb_large_prefix", ex); + } + throw ex; + } } else { // Upgrade column name to allow for longer paths. String idColName = JDBCStatsUtils.getIdColumnName(); @@ -301,7 +310,7 @@ public class JDBCStatsPublisher implements StatsPublisher { colSize = rs.getInt("COLUMN_SIZE"); if (colSize < JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE) { String alterTable = JDBCStatsUtils.getAlterIdColumn(); - stmt.executeUpdate(alterTable); + stmt.executeUpdate(alterTable); } } else { LOG.warn("Failed to update " + idColName + " - column not found"); http://git-wip-us.apache.org/repos/asf/hive/blob/f73157fe/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java index 17e109a..e39fc5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java @@ -34,6 +34,6 @@ public final class JDBCStatsSetupConstants { public static final String PART_STAT_RAW_DATA_SIZE_COLUMN_NAME = "RAW_DATA_SIZE"; - // MySQL - 65535, SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large. - public static final int ID_COLUMN_VARCHAR_SIZE = 4000; + // MySQL - 3072/3 (innodb+utf8), SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large. + public static final int ID_COLUMN_VARCHAR_SIZE = 1000; }
