This is an automated email from the ASF dual-hosted git repository. klcopp pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 37e61a72990 HIVE-26280: Copy more data into COMPLETED_COMPACTIONS for better supportability (Karen Coppage, reviewed by Denys Kuzmenko) 37e61a72990 is described below commit 37e61a7299018027ac7d811a183365a8b509f242 Author: Karen Coppage <klc...@apache.org> AuthorDate: Tue Jun 14 09:17:53 2022 +0200 HIVE-26280: Copy more data into COMPLETED_COMPACTIONS for better supportability (Karen Coppage, reviewed by Denys Kuzmenko) Closes #3340. --- .../hive/hive-schema-4.0.0-alpha-2.hive.sql | 33 +++- ...upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql | 194 +++++++++++++++++++++ .../test/results/clientpositive/llap/sysdb.q.out | 15 +- .../hadoop/hive/metastore/txn/CompactionInfo.java | 9 + .../hive/metastore/txn/CompactionTxnHandler.java | 17 +- .../sql/derby/hive-schema-4.0.0-alpha-2.derby.sql | 3 + ...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql | 5 + .../sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql | 3 + ...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql | 5 + .../sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql | 3 + ...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql | 5 + .../oracle/hive-schema-4.0.0-alpha-2.oracle.sql | 3 + ...grade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql | 5 + .../hive-schema-4.0.0-alpha-2.postgres.sql | 3 + ...ade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql | 4 + .../upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql | 5 + 16 files changed, 301 insertions(+), 11 deletions(-) diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql index 61cafd8ca9e..f6dd0af61a0 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-2.hive.sql @@ -1090,6 +1090,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `COMPACTION_QUEUE` ( `CQ_HIGHEST_WRITE_ID` bigint, `CQ_HADOOP_JOB_ID` string, `CQ_ERROR_MESSAGE` string, + `CQ_NEXT_TXN_ID` bigint, + `CQ_TXN_ID` bigint, + `CQ_COMMIT_TIME` bigint, `CQ_INITIATOR_ID` string, `CQ_INITIATOR_VERSION` string, `CQ_WORKER_VERSION` string, @@ -1114,6 +1117,9 @@ TBLPROPERTIES ( \"COMPACTION_QUEUE\".\"CQ_HIGHEST_WRITE_ID\", \"COMPACTION_QUEUE\".\"CQ_HADOOP_JOB_ID\", \"COMPACTION_QUEUE\".\"CQ_ERROR_MESSAGE\", + \"COMPACTION_QUEUE\".\"CQ_NEXT_TXN_ID\", + \"COMPACTION_QUEUE\".\"CQ_TXN_ID\", + \"COMPACTION_QUEUE\".\"CQ_COMMIT_TIME\", \"COMPACTION_QUEUE\".\"CQ_INITIATOR_ID\", \"COMPACTION_QUEUE\".\"CQ_INITIATOR_VERSION\", \"COMPACTION_QUEUE\".\"CQ_WORKER_VERSION\", @@ -1138,6 +1144,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `COMPLETED_COMPACTIONS` ( `CC_HIGHEST_WRITE_ID` bigint, `CC_HADOOP_JOB_ID` string, `CC_ERROR_MESSAGE` string, + `CC_NEXT_TXN_ID` bigint, + `CC_TXN_ID` bigint, + `CC_COMMIT_TIME` bigint, `CC_INITIATOR_ID` string, `CC_INITIATOR_VERSION` string, `CC_WORKER_VERSION` string @@ -1162,6 +1171,9 @@ TBLPROPERTIES ( \"COMPLETED_COMPACTIONS\".\"CC_HIGHEST_WRITE_ID\", \"COMPLETED_COMPACTIONS\".\"CC_HADOOP_JOB_ID\", \"COMPLETED_COMPACTIONS\".\"CC_ERROR_MESSAGE\", + \"COMPLETED_COMPACTIONS\".\"CC_NEXT_TXN_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_TXN_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_COMMIT_TIME\", \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_ID\", \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_VERSION\", \"COMPLETED_COMPACTIONS\".\"CC_WORKER_VERSION\" @@ -1186,11 +1198,16 @@ CREATE OR REPLACE VIEW `COMPACTIONS` `C_DURATION`, `C_HADOOP_JOB_ID`, `C_RUN_AS`, + `C_ERROR_MESSAGE`, + `C_NEXT_TXN_ID`, + `C_TXN_ID`, + `C_COMMIT_TIME`, `C_HIGHEST_WRITE_ID`, `C_INITIATOR_HOST`, `C_INITIATOR_ID`, `C_INITIATOR_VERSION`, - `C_CLEANER_START` + `C_CLEANER_START`, + `C_TBLPROPERTIES` ) AS SELECT CC_ID, @@ -1209,11 +1226,16 @@ SELECT CASE WHEN CC_END IS NULL THEN cast (null as string) ELSE CC_END-CC_START END, CC_HADOOP_JOB_ID, CC_RUN_AS, + CC_ERROR_MESSAGE, + CC_NEXT_TXN_ID, + CC_TXN_ID, + CC_COMMIT_TIME, CC_HIGHEST_WRITE_ID, CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[0] END, CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[size(split(CC_INITIATOR_ID,"-"))-1] END, CC_INITIATOR_VERSION, - NULL + NULL, + CC_TBLPROPERTIES FROM COMPLETED_COMPACTIONS UNION ALL SELECT @@ -1232,11 +1254,16 @@ SELECT cast (null as string), CQ_HADOOP_JOB_ID, CQ_RUN_AS, + CQ_ERROR_MESSAGE, + CQ_NEXT_TXN_ID, + CQ_TXN_ID, + CQ_COMMIT_TIME, CQ_HIGHEST_WRITE_ID, CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[0] END, CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[size(split(CQ_INITIATOR_ID,"-"))-1] END, CQ_INITIATOR_VERSION, - CQ_CLEANER_START + CQ_CLEANER_START, + CQ_TBLPROPERTIES FROM COMPACTION_QUEUE; CREATE EXTERNAL TABLE IF NOT EXISTS `SCHEDULED_QUERIES` ( diff --git a/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql b/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql index a81d6881edc..73d9a5234d7 100644 --- a/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql +++ b/metastore/scripts/upgrade/hive/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.hive.sql @@ -16,4 +16,198 @@ TBLPROPERTIES ( FROM \"MIN_HISTORY_LEVEL\"" ); +-- HIVE-26280 +DROP TABLE IF EXISTS `COMPACTION_QUEUE`; +CREATE EXTERNAL TABLE IF NOT EXISTS `COMPACTION_QUEUE` ( + `CQ_ID` bigint, + `CQ_DATABASE` string, + `CQ_TABLE` string, + `CQ_PARTITION` string, + `CQ_STATE` string, + `CQ_TYPE` string, + `CQ_TBLPROPERTIES` string, + `CQ_WORKER_ID` string, + `CQ_ENQUEUE_TIME` bigint, + `CQ_START` bigint, + `CQ_RUN_AS` string, + `CQ_HIGHEST_WRITE_ID` bigint, + `CQ_HADOOP_JOB_ID` string, + `CQ_ERROR_MESSAGE` string, + `CQ_NEXT_TXN_ID` bigint, + `CQ_TXN_ID` bigint, + `CQ_COMMIT_TIME` bigint, + `CQ_INITIATOR_ID` string, + `CQ_INITIATOR_VERSION` string, + `CQ_WORKER_VERSION` string, + `CQ_CLEANER_START` bigint +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"COMPACTION_QUEUE\".\"CQ_ID\", + \"COMPACTION_QUEUE\".\"CQ_DATABASE\", + \"COMPACTION_QUEUE\".\"CQ_TABLE\", + \"COMPACTION_QUEUE\".\"CQ_PARTITION\", + \"COMPACTION_QUEUE\".\"CQ_STATE\", + \"COMPACTION_QUEUE\".\"CQ_TYPE\", + \"COMPACTION_QUEUE\".\"CQ_TBLPROPERTIES\", + \"COMPACTION_QUEUE\".\"CQ_WORKER_ID\", + \"COMPACTION_QUEUE\".\"CQ_ENQUEUE_TIME\", + \"COMPACTION_QUEUE\".\"CQ_START\", + \"COMPACTION_QUEUE\".\"CQ_RUN_AS\", + \"COMPACTION_QUEUE\".\"CQ_HIGHEST_WRITE_ID\", + \"COMPACTION_QUEUE\".\"CQ_HADOOP_JOB_ID\", + \"COMPACTION_QUEUE\".\"CQ_ERROR_MESSAGE\", + \"COMPACTION_QUEUE\".\"CQ_NEXT_TXN_ID\", + \"COMPACTION_QUEUE\".\"CQ_TXN_ID\", + \"COMPACTION_QUEUE\".\"CQ_COMMIT_TIME\", + \"COMPACTION_QUEUE\".\"CQ_INITIATOR_ID\", + \"COMPACTION_QUEUE\".\"CQ_INITIATOR_VERSION\", + \"COMPACTION_QUEUE\".\"CQ_WORKER_VERSION\", + \"COMPACTION_QUEUE\".\"CQ_CLEANER_START\" +FROM \"COMPACTION_QUEUE\" +" +); + +DROP TABLE IF EXISTS `COMPLETED_COMPACTIONS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `COMPLETED_COMPACTIONS` ( + `CC_ID` bigint, + `CC_DATABASE` string, + `CC_TABLE` string, + `CC_PARTITION` string, + `CC_STATE` string, + `CC_TYPE` string, + `CC_TBLPROPERTIES` string, + `CC_WORKER_ID` string, + `CC_ENQUEUE_TIME` bigint, + `CC_START` bigint, + `CC_END` bigint, + `CC_RUN_AS` string, + `CC_HIGHEST_WRITE_ID` bigint, + `CC_HADOOP_JOB_ID` string, + `CC_ERROR_MESSAGE` string, + `CC_NEXT_TXN_ID` bigint, + `CC_TXN_ID` bigint, + `CC_COMMIT_TIME` bigint, + `CC_INITIATOR_ID` string, + `CC_INITIATOR_VERSION` string, + `CC_WORKER_VERSION` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"COMPLETED_COMPACTIONS\".\"CC_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_DATABASE\", + \"COMPLETED_COMPACTIONS\".\"CC_TABLE\", + \"COMPLETED_COMPACTIONS\".\"CC_PARTITION\", + \"COMPLETED_COMPACTIONS\".\"CC_STATE\", + \"COMPLETED_COMPACTIONS\".\"CC_TYPE\", + \"COMPLETED_COMPACTIONS\".\"CC_TBLPROPERTIES\", + \"COMPLETED_COMPACTIONS\".\"CC_WORKER_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_ENQUEUE_TIME\", + \"COMPLETED_COMPACTIONS\".\"CC_START\", + \"COMPLETED_COMPACTIONS\".\"CC_END\", + \"COMPLETED_COMPACTIONS\".\"CC_RUN_AS\", + \"COMPLETED_COMPACTIONS\".\"CC_HIGHEST_WRITE_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_HADOOP_JOB_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_ERROR_MESSAGE\", + \"COMPLETED_COMPACTIONS\".\"CC_NEXT_TXN_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_TXN_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_COMMIT_TIME\", + \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_ID\", + \"COMPLETED_COMPACTIONS\".\"CC_INITIATOR_VERSION\", + \"COMPLETED_COMPACTIONS\".\"CC_WORKER_VERSION\" +FROM \"COMPLETED_COMPACTIONS\" +" +); + +CREATE OR REPLACE VIEW `COMPACTIONS` +( + `C_ID`, + `C_CATALOG`, + `C_DATABASE`, + `C_TABLE`, + `C_PARTITION`, + `C_TYPE`, + `C_STATE`, + `C_WORKER_HOST`, + `C_WORKER_ID`, + `C_WORKER_VERSION`, + `C_ENQUEUE_TIME`, + `C_START`, + `C_DURATION`, + `C_HADOOP_JOB_ID`, + `C_RUN_AS`, + `C_ERROR_MESSAGE`, + `C_NEXT_TXN_ID`, + `C_TXN_ID`, + `C_COMMIT_TIME`, + `C_HIGHEST_WRITE_ID`, + `C_INITIATOR_HOST`, + `C_INITIATOR_ID`, + `C_INITIATOR_VERSION`, + `C_CLEANER_START`, + `C_TBLPROPERTIES` +) AS +SELECT + CC_ID, + 'default', + CC_DATABASE, + CC_TABLE, + CC_PARTITION, + CASE WHEN CC_TYPE = 'i' THEN 'minor' WHEN CC_TYPE = 'a' THEN 'major' ELSE 'UNKNOWN' END, + CASE WHEN CC_STATE = 'f' THEN 'failed' WHEN CC_STATE = 's' THEN 'succeeded' + WHEN CC_STATE = 'a' THEN 'did not initiate' WHEN CC_STATE = 'c' THEN 'refused' ELSE 'UNKNOWN' END, + CASE WHEN CC_WORKER_ID IS NULL THEN cast (null as string) ELSE split(CC_WORKER_ID,"-")[0] END, + CASE WHEN CC_WORKER_ID IS NULL THEN cast (null as string) ELSE split(CC_WORKER_ID,"-")[size(split(CC_WORKER_ID,"-"))-1] END, + CC_WORKER_VERSION, + CC_ENQUEUE_TIME, + CC_START, + CASE WHEN CC_END IS NULL THEN cast (null as string) ELSE CC_END-CC_START END, + CC_HADOOP_JOB_ID, + CC_RUN_AS, + CC_ERROR_MESSAGE, + CC_NEXT_TXN_ID, + CC_TXN_ID, + CC_COMMIT_TIME, + CC_HIGHEST_WRITE_ID, + CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[0] END, + CASE WHEN CC_INITIATOR_ID IS NULL THEN cast (null as string) ELSE split(CC_INITIATOR_ID,"-")[size(split(CC_INITIATOR_ID,"-"))-1] END, + CC_INITIATOR_VERSION, + NULL, + CC_TBLPROPERTIES +FROM COMPLETED_COMPACTIONS +UNION ALL +SELECT + CQ_ID, + 'default', + CQ_DATABASE, + CQ_TABLE, + CQ_PARTITION, + CASE WHEN CQ_TYPE = 'i' THEN 'minor' WHEN CQ_TYPE = 'a' THEN 'major' ELSE 'UNKNOWN' END, + CASE WHEN CQ_STATE = 'i' THEN 'initiated' WHEN CQ_STATE = 'w' THEN 'working' WHEN CQ_STATE = 'r' THEN 'ready for cleaning' ELSE 'UNKNOWN' END, + CASE WHEN CQ_WORKER_ID IS NULL THEN NULL ELSE split(CQ_WORKER_ID,"-")[0] END, + CASE WHEN CQ_WORKER_ID IS NULL THEN NULL ELSE split(CQ_WORKER_ID,"-")[size(split(CQ_WORKER_ID,"-"))-1] END, + CQ_WORKER_VERSION, + CQ_ENQUEUE_TIME, + CQ_START, + cast (null as string), + CQ_HADOOP_JOB_ID, + CQ_RUN_AS, + CQ_ERROR_MESSAGE, + CQ_NEXT_TXN_ID, + CQ_TXN_ID, + CQ_COMMIT_TIME, + CQ_HIGHEST_WRITE_ID, + CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[0] END, + CASE WHEN CQ_INITIATOR_ID IS NULL THEN NULL ELSE split(CQ_INITIATOR_ID,"-")[size(split(CQ_INITIATOR_ID,"-"))-1] END, + CQ_INITIATOR_VERSION, + CQ_CLEANER_START, + CQ_TBLPROPERTIES +FROM COMPACTION_QUEUE; + SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2'; diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index b5c0869bba6..d2bf198a4ad 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -470,6 +470,7 @@ columns_v2 comment columns_v2 integer_idx columns_v2 type_name compaction_queue cq_cleaner_start +compaction_queue cq_commit_time compaction_queue cq_database compaction_queue cq_enqueue_time compaction_queue cq_error_message @@ -478,12 +479,14 @@ compaction_queue cq_highest_write_id compaction_queue cq_id compaction_queue cq_initiator_id compaction_queue cq_initiator_version +compaction_queue cq_next_txn_id compaction_queue cq_partition compaction_queue cq_run_as compaction_queue cq_start compaction_queue cq_state compaction_queue cq_table compaction_queue cq_tblproperties +compaction_queue cq_txn_id compaction_queue cq_type compaction_queue cq_worker_id compaction_queue cq_worker_version @@ -491,12 +494,14 @@ compactions c_catalog compactions c_catalog compactions c_cleaner_start compactions c_cleaner_start +compactions c_commit_time compactions c_database compactions c_database compactions c_duration compactions c_duration compactions c_enqueue_time compactions c_enqueue_time +compactions c_error_message #### A masked pattern was here #### compactions c_highest_write_id compactions c_highest_write_id @@ -508,6 +513,7 @@ compactions c_initiator_id compactions c_initiator_id compactions c_initiator_version compactions c_initiator_version +compactions c_next_txn_id compactions c_partition compactions c_partition compactions c_run_as @@ -518,6 +524,8 @@ compactions c_state compactions c_state compactions c_table compactions c_table +compactions c_tblproperties +compactions c_txn_id compactions c_type compactions c_type compactions c_worker_host @@ -526,6 +534,7 @@ compactions c_worker_id compactions c_worker_id compactions c_worker_version compactions c_worker_version +completed_compactions cc_commit_time completed_compactions cc_database completed_compactions cc_end completed_compactions cc_enqueue_time @@ -535,12 +544,14 @@ completed_compactions cc_highest_write_id completed_compactions cc_id completed_compactions cc_initiator_id completed_compactions cc_initiator_version +completed_compactions cc_next_txn_id completed_compactions cc_partition completed_compactions cc_run_as completed_compactions cc_start completed_compactions cc_state completed_compactions cc_table completed_compactions cc_tblproperties +completed_compactions cc_txn_id completed_compactions cc_type completed_compactions cc_worker_id completed_compactions cc_worker_version @@ -1561,8 +1572,8 @@ POSTHOOK: Input: sys@compaction_queue POSTHOOK: Input: sys@compactions POSTHOOK: Input: sys@completed_compactions #### A masked pattern was here #### -1 default default scr_txn NULL major initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL -2 default default scr_txn_2 NULL minor initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL +1 default default scr_txn NULL major initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL NULL +2 default default scr_txn_2 NULL minor initiated NULL NULL NULL #Masked# NULL NULL NULL NULL NULL NULL NULL NULL NULL #Masked# manual 4.0.0-alpha-2-SNAPSHOT NULL NULL PREHOOK: query: use INFORMATION_SCHEMA PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:information_schema diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java index 7ac42380a50..e6fe08b9357 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java @@ -59,6 +59,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> { public boolean tooManyAborts = false; public boolean hasOldAbort = false; public long retryRetention = 0; + public long nextTxnId = 0; + public long txnId = 0; + public long commitTime = 0; /** * The highest write id that the compaction job will pay attention to. @@ -202,6 +205,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> { fullCi.initiatorId = rs.getString(17); fullCi.initiatorVersion = rs.getString(18); fullCi.retryRetention = rs.getLong(19); + fullCi.nextTxnId = rs.getLong(20); + fullCi.txnId = rs.getLong(21); + fullCi.commitTime = rs.getLong(22); return fullCi; } static void insertIntoCompletedCompactions(PreparedStatement pStmt, CompactionInfo ci, long endTime) throws SQLException, MetaException { @@ -224,6 +230,9 @@ public class CompactionInfo implements Comparable<CompactionInfo> { pStmt.setString(17, ci.workerVersion); pStmt.setString(18, ci.initiatorId); pStmt.setString(19, ci.initiatorVersion); + pStmt.setLong(20, ci.nextTxnId); + pStmt.setLong(21, ci.txnId); + pStmt.setLong(22, ci.commitTime); } public static CompactionInfo compactionStructToInfo(CompactionInfoStruct cr) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java index 5753c7863e9..ac2243530f0 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java @@ -64,7 +64,8 @@ class CompactionTxnHandler extends TxnHandler { + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", " + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", " + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", " - + "\"CQ_RETRY_RETENTION\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?"; + + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\"" + + " FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?"; private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY = "SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " + "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?"; @@ -535,12 +536,14 @@ class CompactionTxnHandler extends TxnHandler { + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", " + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", " + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", " - + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\") " + + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", " + + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\") " + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", " + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", " + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", " - + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\"" + + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", " + + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\" " + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?"; pStmt = dbConn.prepareStatement(s); pStmt.setLong(1, info.id); @@ -1363,7 +1366,8 @@ class CompactionTxnHandler extends TxnHandler { + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", " + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", " + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", " - + "\"CQ_RETRY_RETENTION\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?"); + + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\" " + + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?"); pStmt.setLong(1, ci.id); rs = pStmt.executeQuery(); if (rs.next()) { @@ -1405,8 +1409,9 @@ class CompactionTxnHandler extends TxnHandler { + "(\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " + "\"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " + "\"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " - + "\"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\") " - + "VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?,?,?,?,?)"); + + "\"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," + + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\") " + + "VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?)"); CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn)); int updCount = pStmt.executeUpdate(); LOG.debug("Inserted " + updCount + " entries into COMPLETED_COMPACTIONS"); diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql index 172382a7da7..949c144c86b 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0-alpha-2.derby.sql @@ -655,6 +655,9 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_META_INFO varchar(2048) for bit data, CC_HADOOP_JOB_ID varchar(32), CC_ERROR_MESSAGE clob, + CC_NEXT_TXN_ID bigint, + CC_TXN_ID bigint, + CC_COMMIT_TIME bigint, CC_INITIATOR_ID varchar(128), CC_INITIATOR_VERSION varchar(128), CC_WORKER_VERSION varchar(128) diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql index 22c56389d85..bf17bd33380 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql @@ -1,4 +1,9 @@ -- Upgrade MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2 +-- HIVE-26280 +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID bigint; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID bigint; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME bigint; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1; diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql index 0fd9303b279..b12ab46fc4c 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0-alpha-2.mssql.sql @@ -1073,6 +1073,9 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_META_INFO varbinary(2048) NULL, CC_HADOOP_JOB_ID nvarchar(128) NULL, CC_ERROR_MESSAGE varchar(max) NULL, + CC_NEXT_TXN_ID bigint NULL, + CC_TXN_ID bigint NULL, + CC_COMMIT_TIME bigint NULL, CC_INITIATOR_ID nvarchar(128) NULL, CC_INITIATOR_VERSION nvarchar(128) NULL, CC_WORKER_VERSION nvarchar(128) NULL, diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql index 1b1ff800a86..7dd16b77dfe 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql @@ -1,5 +1,10 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE; +-- HIVE-26280 +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID bigint NULL; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID bigint NULL; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME bigint NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql index c98cd53ee6a..2bbe7b1445b 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0-alpha-2.mysql.sql @@ -1111,6 +1111,9 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_META_INFO varbinary(2048), CC_HADOOP_JOB_ID varchar(32), CC_ERROR_MESSAGE mediumtext, + CC_NEXT_TXN_ID bigint, + CC_TXN_ID bigint, + CC_COMMIT_TIME bigint, CC_INITIATOR_ID varchar(128), CC_INITIATOR_VERSION varchar(128), CC_WORKER_VERSION varchar(128) diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql index a773532bb7d..09eace89b49 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql @@ -1,5 +1,10 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE; +-- HIVE-26280 +ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_NEXT_TXN_ID` bigint; +ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_TXN_ID` bigint; +ALTER TABLE `COMPLETED_COMPACTIONS` ADD COLUMN `CC_COMMIT_TIME` bigint; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql index 0d2b8ad5f8e..29d6f0ffddf 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0-alpha-2.oracle.sql @@ -1118,6 +1118,9 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_META_INFO BLOB, CC_HADOOP_JOB_ID varchar2(32), CC_ERROR_MESSAGE CLOB, + CC_NEXT_TXN_ID NUMBER(19), + CC_TXN_ID NUMBER(19), + CC_COMMIT_TIME NUMBER(19), CC_INITIATOR_ID varchar(128), CC_INITIATOR_VERSION varchar(128), CC_WORKER_VERSION varchar(128) diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql index 79eadf14809..c0fd8f231d3 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.oracle.sql @@ -1,5 +1,10 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS Status from dual; +-- HIVE-26280 +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_NEXT_TXN_ID NUMBER(19); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TXN_ID NUMBER(19); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_COMMIT_TIME NUMBER(19); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0-alpha-2', VERSION_COMMENT='Hive release version 4.0.0-alpha-2' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2' AS Status from dual; diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql index 4be408996ab..b72b8d5ca0f 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0-alpha-2.postgres.sql @@ -1827,6 +1827,9 @@ CREATE TABLE "COMPLETED_COMPACTIONS" ( "CC_META_INFO" bytea, "CC_HADOOP_JOB_ID" varchar(32), "CC_ERROR_MESSAGE" text, + "CC_NEXT_TXN_ID" bigint, + "CC_TXN_ID" bigint, + "CC_COMMIT_TIME" bigint, "CC_INITIATOR_ID" varchar(128), "CC_INITIATOR_VERSION" varchar(128), "CC_WORKER_VERSION" varchar(128) diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql index fa03cd9f295..2dad36782f3 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.postgres.sql @@ -1,5 +1,9 @@ SELECT 'Upgrading MetaStore schema from 4.0.0-alpha-1 to 4.0.0-alpha-2'; +-- HIVE-26280 +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_NEXT_TXN_ID" bigint; +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_TXN_ID" bigint; +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_COMMIT_TIME" bigint; -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0-alpha-2', "VERSION_COMMENT"='Hive release version 4.0.0-alpha-2' where "VER_ID"=1; diff --git a/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql b/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql index be224ab4619..ae712a92f20 100644 --- a/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql +++ b/standalone-metastore/metastore-server/src/test/resources/sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-2.postgres.sql @@ -131,6 +131,11 @@ CREATE TABLE "COMPACTION_METRICS_CACHE" ( -- HIVE-25993 ALTER TABLE "COMPACTION_QUEUE" ADD "CQ_RETRY_RETENTION" integer NOT NULL DEFAULT 0; +-- HIVE-26280 +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_NEXT_TXN_ID" bigint; +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_TXN_ID" bigint; +ALTER TABLE "COMPLETED_COMPACTIONS" ADD "CC_COMMIT_TIME" bigint; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0-alpha-2', "VERSION_COMMENT"='Hive release version 4.0.0-alpha-2' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.1.3000 to 4.0.0-alpha-2';