Repository: hive Updated Branches: refs/heads/branch-3 3f904ff79 -> 86361df3a
HIVE-19946: VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different JVMs (Teddy Choi via Sergey Shelukhin, Matt McCline) Signed-off-by: Zoltan Haindrich <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86361df3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86361df3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86361df3 Branch: refs/heads/branch-3 Commit: 86361df3aeb556ee6811e9f0409f8efd6a1a5d48 Parents: 3f904ff Author: Teddy Choi <[email protected]> Authored: Tue Jun 26 09:26:28 2018 +0200 Committer: Teddy Choi <[email protected]> Committed: Wed Jun 27 15:43:12 2018 +0900 ---------------------------------------------------------------------- .../hive/ql/exec/vector/VectorMapOperator.java | 5 ++--- .../hive/ql/exec/vector/VectorizedRowBatchCtx.java | 15 --------------- .../ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 4 ++-- 3 files changed, 4 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/86361df3/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 2542e03..bd70991 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -803,10 +803,9 @@ public class VectorMapOperator extends AbstractMapOperator { VectorizedRowBatch batch = (VectorizedRowBatch) value; numRows += batch.size; if (hasRowIdentifier) { - if (batchContext.getRecordIdColumnVector() == null) { + final int idx = batchContext.findVirtualColumnNum(VirtualColumn.ROWID); + if (idx < 0) { setRowIdentiferToNull(batch); - } else { - batch.cols[rowIdentifierColumnNum] = batchContext.getRecordIdColumnVector(); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/86361df3/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java index ca17d4b..fa056e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java @@ -77,13 +77,6 @@ public class VectorizedRowBatchCtx { private int partitionColumnCount; private int virtualColumnCount; private VirtualColumn[] neededVirtualColumns; - /** - * A record ID column is a virtual column, so it should be separated from normal data column - * processes. A recordIdColumnVector contains RecordIdentifier information in a - * StructColumnVector. It has three LongColumnVectors as its fields; original write IDs, - * bucket IDs, and row IDs. - */ - private StructColumnVector recordIdColumnVector; private String[] scratchColumnTypeNames; private DataTypePhysicalVariation[] scratchDataTypePhysicalVariations; @@ -210,14 +203,6 @@ public class VectorizedRowBatchCtx { return scratchDataTypePhysicalVariations; } - public StructColumnVector getRecordIdColumnVector() { - return this.recordIdColumnVector; - } - - public void setRecordIdColumnVector(StructColumnVector recordIdColumnVector) { - this.recordIdColumnVector = recordIdColumnVector; - } - /** * Initializes the VectorizedRowBatch context based on an scratch column type names and * object inspector. http://git-wip-us.apache.org/repos/asf/hive/blob/86361df3/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index e181d8c..1841cfa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -483,8 +483,8 @@ public class VectorizedOrcAcidRowBatchReader } } if (rowIdProjected) { - // TODO: could we just do: int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID); value.cols[ix] = recordIdColumnVector; - rbCtx.setRecordIdColumnVector(recordIdColumnVector); + int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID); + value.cols[ix] = recordIdColumnVector; } progress = baseReader.getProgress(); return true;
