HIVE-13126: Clean up MapJoinOperator properly to avoid object cache reuse with unintentional states (Wei Zheng via Vikram Dixit K)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2bf0d92d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2bf0d92d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2bf0d92d Branch: refs/heads/llap Commit: 2bf0d92d10b183017b5cacc2e308a7aedcd30a40 Parents: a154a09 Author: vikram <[email protected]> Authored: Wed Feb 24 10:01:31 2016 -0800 Committer: vikram <[email protected]> Committed: Wed Feb 24 10:01:31 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java | 7 +++++++ 1 file changed, 7 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2bf0d92d/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 91b5ca7..4d2447b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -152,6 +152,11 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen]; hashTblInitedOnce = false; + // Reset grace hashjoin context so that there is no state maintained when operator/work is + // retrieved from object cache + hybridMapJoinLeftover = false; + firstSmallTable = null; + generateMapMetaData(); final ExecMapperContext mapContext = getExecContext(); @@ -662,6 +667,8 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem // firstSmallTable has reference to the spilled big table rows. HashPartition partition = firstSmallTable.getHashPartitions()[partitionId]; ObjectContainer bigTable = partition.getMatchfileObjContainer(); + LOG.info("Hybrid Grace Hash Join: Going to process spilled big table rows in partition " + + partitionId + ". Number of rows: " + bigTable.size()); while (bigTable.hasNext()) { Object row = bigTable.next(); process(row, conf.getPosBigTable());
