Repository: hive Updated Branches: refs/heads/master 6d713b656 -> 840dd431f
HIVE-20862: QueryId no longer shows up in the logs (Eugene Koifman reviewed by Vaibhav Gumashta) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/840dd431 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/840dd431 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/840dd431 Branch: refs/heads/master Commit: 840dd431f3772772fc57060e27e3f2bee72a8936 Parents: 6d713b6 Author: Eugene Koifman <ekoif...@apache.org> Authored: Wed Nov 7 17:11:14 2018 -0800 Committer: Eugene Koifman <ekoif...@apache.org> Committed: Wed Nov 7 17:11:14 2018 -0800 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 13 ++++++++----- .../apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java | 2 +- .../apache/hadoop/hive/ql/session/SessionState.java | 1 + .../hadoop/hive/metastore/txn/TxnCommonUtils.java | 1 + 4 files changed, 11 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index ff5086a..57f71a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -543,7 +543,8 @@ public class Driver implements IDriver { LockedDriverState.setLockedDriverState(lDrvState); - String queryId = queryState.getQueryId(); + final String queryId = Strings.isNullOrEmpty(queryState.getQueryId()) ? + QueryPlan.makeQueryId() : queryState.getQueryId(); SparkSession ss = SessionState.get().getSparkSession(); if (ss != null) { @@ -1484,7 +1485,7 @@ public class Driver implements IDriver { throw new LockException("Unexpected tables in compaction: " + txnTables); } String fullTableName = txnTables.get(0); - txnWriteIds = new ValidTxnWriteIdList(0L); // No transaction for the compaction for now. + txnWriteIds = new ValidTxnWriteIdList(0L); // No transaction for the compaction for now. todo: Since MM compaction is a query, a txn has been opened at this point txnWriteIds.addTableValidWriteIdList(compactionWriteIds); } else { txnWriteIds = txnMgr.getValidWriteIds(txnTables, txnString); @@ -1672,8 +1673,10 @@ public class Driver implements IDriver { queryTxnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState); // This check is for controlling the correctness of the current state if (queryTxnMgr.recordSnapshot(plan) && !validTxnListsGenerated) { - throw new IllegalStateException("calling recordValidTxn() more than once in the same " + - JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId())); + throw new IllegalStateException( + "Need to record valid WriteID list but there is no valid TxnID list (" + + JavaUtils.txnIdToString(queryTxnMgr.getCurrentTxnId()) + + ", queryId:" + plan.getQueryId() + ")"); } if (plan.hasAcidResourcesInQuery() || hasAcidDdl) { @@ -2230,7 +2233,7 @@ public class Driver implements IDriver { } Metrics metrics = MetricsFactory.getInstance(); - String queryId = queryState.getQueryId(); + String queryId = plan.getQueryId(); // Get the query string from the conf file as the compileInternal() method might // hide sensitive information during query redaction. String queryStr = conf.getQueryString(); http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java index ba1f1ff..12c1027 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java @@ -262,7 +262,7 @@ public interface HiveTxnManager { /** * For resources that support MVCC, the state of the DB must be recorded for the duration of the - * operation/transaction. Returns {@code true} if current statment needs to do this. + * operation/transaction. Returns {@code true} if current statement needs to do this. */ boolean recordSnapshot(QueryPlan queryPlan); http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index ee7c940..455ffc3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -501,6 +501,7 @@ public class SessionState { * it's not coupled to the executing thread. Since tests run against Derby which often wedges * under concurrent access, tests must use a single thead and simulate concurrent access. * For example, {@code TestDbTxnManager2} + * @return previous {@link HiveTxnManager} or null */ @VisibleForTesting public HiveTxnManager setTxnMgr(HiveTxnManager mgr) { http://git-wip-us.apache.org/repos/asf/hive/blob/840dd431/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java index 94cb18d..c61a997 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java @@ -41,6 +41,7 @@ public class TxnCommonUtils { * @return a valid txn list. */ public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) { + assert currentTxn <= txns.getTxn_high_water_mark(); /* * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0 * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
