HIVE-13354 : Add ability to specify Compaction options per table and per request (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e0504d9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e0504d9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e0504d9 Branch: refs/heads/branch-1 Commit: 6e0504d9a8552471f7137a513abe2a0a15c124de Parents: abaf882 Author: Wei Zheng <w...@apache.org> Authored: Fri May 27 15:13:53 2016 -0700 Committer: Wei Zheng <w...@apache.org> Committed: Fri May 27 15:13:53 2016 -0700 ---------------------------------------------------------------------- .../hive/ql/txn/compactor/TestCompactor.java | 164 ++ metastore/if/hive_metastore.thrift | 1 + .../upgrade/derby/036-HIVE-13354.derby.sql | 2 + .../derby/hive-txn-schema-1.3.0.derby.sql | 2 + .../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 1 + .../upgrade/mssql/021-HIVE-13354.mssql.sql | 2 + .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 2 + .../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 1 + .../upgrade/mysql/036-HIVE-13354.mysql.sql | 2 + .../mysql/hive-txn-schema-1.3.0.mysql.sql | 2 + .../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 1 + .../upgrade/oracle/036-HIVE-13354.oracle.sql | 2 + .../oracle/hive-txn-schema-1.3.0.oracle.sql | 2 + .../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 1 + .../postgres/035-HIVE-13354.postgres.sql | 2 + .../postgres/hive-txn-schema-1.3.0.postgres.sql | 2 + .../upgrade-1.2.0-to-1.3.0.postgres.sql | 1 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 1814 ++++++++-------- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 640 +++--- .../gen/thrift/gen-cpp/hive_metastore_types.h | 10 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../hive/metastore/api/CompactionRequest.java | 166 +- .../hive/metastore/api/FireEventRequest.java | 32 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../metastore/api/InsertEventRequestData.java | 32 +- .../api/NotificationEventResponse.java | 36 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ThriftHiveMetastore.java | 1948 +++++++++--------- .../gen-php/metastore/ThriftHiveMetastore.php | 1098 +++++----- .../src/gen/thrift/gen-php/metastore/Types.php | 171 +- .../hive_metastore/ThriftHiveMetastore.py | 738 +++---- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 103 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 4 +- .../hive/metastore/HiveMetaStoreClient.java | 14 + .../hadoop/hive/metastore/IMetaStoreClient.java | 18 + .../hive/metastore/txn/CompactionInfo.java | 30 +- .../metastore/txn/CompactionTxnHandler.java | 11 +- .../hadoop/hive/metastore/txn/TxnDbUtil.java | 2 + .../hadoop/hive/metastore/txn/TxnHandler.java | 8 + .../hadoop/hive/metastore/txn/TxnUtils.java | 54 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 5 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 4 +- .../hive/ql/plan/AlterTableSimpleDesc.java | 8 + .../hive/ql/txn/compactor/CompactorMR.java | 121 +- .../hadoop/hive/ql/txn/compactor/Initiator.java | 22 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 9 + .../hive/ql/txn/compactor/TestWorker.java | 9 +- 49 files changed, 4016 insertions(+), 3393 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 37bbab8..9c8bcc1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -14,6 +14,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CompactionRequest; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; @@ -34,6 +35,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcStruct; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.mapred.JobConf; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.streaming.DelimitedInputWriter; import org.apache.hive.hcatalog.streaming.HiveEndPoint; @@ -61,6 +63,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; /** */ @@ -834,6 +837,137 @@ public class TestCompactor { connection.close(); } } + + /** + * Users have the choice of specifying compaction related tblproperties either in CREATE TABLE + * statement or in ALTER TABLE .. COMPACT statement. This tests both cases. + * @throws Exception + */ + @Test + public void testTableProperties() throws Exception { + String tblName1 = "ttp1"; // plain acid table + String tblName2 = "ttp2"; // acid table with customized tblproperties + executeStatementOnDriver("drop table if exists " + tblName1, driver); + executeStatementOnDriver("drop table if exists " + tblName2, driver); + executeStatementOnDriver("CREATE TABLE " + tblName1 + "(a INT, b STRING) " + + " CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver); + executeStatementOnDriver("CREATE TABLE " + tblName2 + "(a INT, b STRING) " + + " CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES (" + + "'transactional'='true'," + + "'compactor.mapreduce.map.memory.mb'='2048'," + // 2048 MB memory for compaction map job + "'compactorthreshold.hive.compactor.delta.num.threshold'='4'," + // minor compaction if more than 4 delta dirs + "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.5'" + // major compaction if more than 50% + ")", driver); + + // Insert 5 rows to both tables + executeStatementOnDriver("insert into " + tblName1 + " values (1, 'a')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (2, 'b')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (3, 'c')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (4, 'd')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (5, 'e')", driver); + + executeStatementOnDriver("insert into " + tblName2 + " values (1, 'a')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (2, 'b')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (3, 'c')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (4, 'd')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (5, 'e')", driver); + + runInitiator(conf); + + // Compactor should only schedule compaction for ttp2 (delta.num.threshold=4), not ttp1 + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(1, rsp.getCompacts().size()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(CompactionType.MAJOR, rsp.getCompacts().get(0).getType()); // type is MAJOR since there's no base yet + + // Finish the scheduled compaction for ttp2, and manually compact ttp1, to make them comparable again + executeStatementOnDriver("alter table " + tblName1 + " compact 'major'", driver); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(2, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp1", rsp.getCompacts().get(1).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(1).getState()); + // compact ttp2, by running the Worker explicitly, in order to get the reference to the compactor MR job + AtomicBoolean stop = new AtomicBoolean(true); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + JobConf job = t.getMrJob(); + Assert.assertEquals("2048", job.get("mapreduce.map.memory.mb")); // 2048 comes from tblproperties + // Compact ttp1 + stop = new AtomicBoolean(true); + t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + job = t.getMrJob(); + Assert.assertEquals("1024", job.get("mapreduce.map.memory.mb")); // 1024 is the default value + // Clean up + runCleaner(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(2, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp1", rsp.getCompacts().get(1).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(1).getState()); + + // Insert one more row - this should trigger hive.compactor.delta.pct.threshold to be reached for ttp2 + executeStatementOnDriver("insert into " + tblName1 + " values (6, 'f')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (6, 'f')", driver); + + // Intentionally set this high so that it will not trigger major compaction for ttp1. + // Only trigger major compaction for ttp2 (delta.pct.threshold=0.5) because of the newly inserted row (actual pct: 0.66) + conf.setFloatVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD, 0.8f); + runInitiator(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(4, rsp.getCompacts().size()); + Assert.assertEquals("ttp1", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(1).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(1).getState()); + + // Finish the scheduled compaction for ttp2 + runWorker(conf); + runWorker(conf); + runCleaner(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(4, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(0).getState()); + + // Now test tblproperties specified on ALTER TABLE .. COMPACT .. statement + executeStatementOnDriver("insert into " + tblName2 + " values (7, 'g')", driver); + executeStatementOnDriver("alter table " + tblName2 + " compact 'major'" + + " with overwrite tblproperties (" + + "'compactor.mapreduce.map.memory.mb'='3072'," + + "'tblprops.orc.compress.size'='8192')", driver); + + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(5, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + + // Run the Worker explicitly, in order to get the reference to the compactor MR job + stop = new AtomicBoolean(true); + t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + job = t.getMrJob(); + Assert.assertEquals("3072", job.get("mapreduce.map.memory.mb")); + Assert.assertTrue(job.get("hive.compactor.table.props").contains("orc.compress.size4:8192")); + } + private void writeBatch(StreamingConnection connection, DelimitedInputWriter writer, boolean closeEarly) throws InterruptedException, StreamingException { @@ -957,4 +1091,34 @@ public class TestCompactor { } } + + static void runInitiator(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Initiator t = new Initiator(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } + + static void runWorker(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } + + static void runCleaner(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Cleaner t = new Cleaner(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index f3d1d42..c7b6fd8 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -674,6 +674,7 @@ struct CompactionRequest { 3: optional string partitionname, 4: required CompactionType type, 5: optional string runas, + 6: optional map<string, string> properties } struct ShowCompactRequest { http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql b/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql index 480c19e..634dd73 100644 --- a/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql @@ -82,6 +82,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -102,6 +103,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql index 1b9e171..d4484bd 100644 --- a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql @@ -11,5 +11,6 @@ RUN '030-HIVE-12823.derby.sql'; RUN '031-HIVE-12831.derby.sql'; RUN '032-HIVE-12832.derby.sql'; RUN '035-HIVE-13395.derby.sql'; +RUN '036-HIVE-13354.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql b/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql new file mode 100644 index 0000000..518b142 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES nvarchar(2048) NULL; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES nvarchar(2048) NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql index 7e0e24f..931edfc 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql @@ -863,6 +863,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_PARTITION nvarchar(767) NULL, CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES nvarchar(2048) NULL, CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, @@ -882,6 +883,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION nvarchar(767) NULL, CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES nvarchar(2048) NULL, CC_WORKER_ID nvarchar(128) NULL, CC_START bigint NULL, CC_END bigint NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql index 18da152..32782eb 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql @@ -11,6 +11,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; :r 016-HIVE-12831.mssql.sql; :r 017-HIVE-12832.mssql.sql; :r 020-HIVE-13395.mssql.sql; +:r 021-HIVE-13354.mssql.sql UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql b/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql index e852fc9..21b6c9b 100644 --- a/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql @@ -85,6 +85,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -100,6 +101,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql index 021b802..284f366 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql @@ -12,6 +12,7 @@ SOURCE 030-HIVE-12823.mysql.sql; SOURCE 031-HIVE-12831.mysql.sql; SOURCE 032-HIVE-12832.mysql.sql; SOURCE 035-HIVE-13395.mysql.sql; +SOURCE 036-HIVE-13354.mysql.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql b/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql index 199ff4c..89f6374 100644 --- a/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START NUMBER(19), CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START NUMBER(19), CC_END NUMBER(19), http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql index ce86e67..7673b6e 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql @@ -11,6 +11,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; @031-HIVE-12381.oracle.sql; @032-HIVE-12832.oracle.sql; @035-HIVE-13395.oracle.sql; +@036-HIVE-13354.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql b/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql index b606f81..f998d18 100644 --- a/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/6e0504d9/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql index 624dde6..260f5c2 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql @@ -11,6 +11,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0'; \i 030-HIVE-12831.postgres.sql; \i 031-HIVE-12832.postgres.sql; \i 034-HIVE-13395.postgres.sql; +\i 035-HIVE-13354.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='1.3.0', "VERSION_COMMENT"='Hive release version 1.3.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0';
