spark git commit: [SPARK-20474] Fixing OnHeapColumnVector reallocation
Repository: spark Updated Branches: refs/heads/branch-2.2 6709bcf6e -> e278876ba [SPARK-20474] Fixing OnHeapColumnVector reallocation ## What changes were proposed in this pull request? OnHeapColumnVector reallocation copies to the new storage data up to 'elementsAppended'. This variable is only updated when using the ColumnVector.appendX API, while ColumnVector.putX is more commonly used. ## How was this patch tested? Tested using existing unit tests. Author: Michal Szafranski Closes #17773 from michal-databricks/spark-20474. (cherry picked from commit a277ae80a2836e6533b338d2b9c4e59ed8a1daae) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e278876b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e278876b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e278876b Branch: refs/heads/branch-2.2 Commit: e278876ba3d66d3fb249df59c3de8d78ca25c5f0 Parents: 6709bcf Author: Michal Szafranski Authored: Wed Apr 26 12:47:37 2017 -0700 Committer: Reynold Xin Committed: Wed Apr 26 12:47:50 2017 -0700 -- .../vectorized/OnHeapColumnVector.java | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e278876b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java -- diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java index 9b410ba..94ed322 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java @@ -410,53 +410,53 @@ public final class OnHeapColumnVector extends ColumnVector { int[] newLengths = new int[newCapacity]; int[] newOffsets = new int[newCapacity]; if (this.arrayLengths != null) { -System.arraycopy(this.arrayLengths, 0, newLengths, 0, elementsAppended); -System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, elementsAppended); +System.arraycopy(this.arrayLengths, 0, newLengths, 0, capacity); +System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, capacity); } arrayLengths = newLengths; arrayOffsets = newOffsets; } else if (type instanceof BooleanType) { if (byteData == null || byteData.length < newCapacity) { byte[] newData = new byte[newCapacity]; -if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended); +if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity); byteData = newData; } } else if (type instanceof ByteType) { if (byteData == null || byteData.length < newCapacity) { byte[] newData = new byte[newCapacity]; -if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended); +if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity); byteData = newData; } } else if (type instanceof ShortType) { if (shortData == null || shortData.length < newCapacity) { short[] newData = new short[newCapacity]; -if (shortData != null) System.arraycopy(shortData, 0, newData, 0, elementsAppended); +if (shortData != null) System.arraycopy(shortData, 0, newData, 0, capacity); shortData = newData; } } else if (type instanceof IntegerType || type instanceof DateType || DecimalType.is32BitDecimalType(type)) { if (intData == null || intData.length < newCapacity) { int[] newData = new int[newCapacity]; -if (intData != null) System.arraycopy(intData, 0, newData, 0, elementsAppended); +if (intData != null) System.arraycopy(intData, 0, newData, 0, capacity); intData = newData; } } else if (type instanceof LongType || type instanceof TimestampType || DecimalType.is64BitDecimalType(type)) { if (longData == null || longData.length < newCapacity) { long[] newData = new long[newCapacity]; -if (longData != null) System.arraycopy(longData, 0, newData, 0, elementsAppended); +if (longData != null) System.arraycopy(longData, 0, newData, 0, capacity); longData = newData; } } else if (type instanceof FloatType) { if (floatData == null || floatData.length < newCapacity) { float[] newData = new float[newCapacity]; -if (floatData != null) System.arraycopy(floatData, 0, newData, 0, elementsAppended); +if (floatData != null) System.arraycopy(floatData, 0
spark git commit: [SPARK-20474] Fixing OnHeapColumnVector reallocation
Repository: spark Updated Branches: refs/heads/master 99c6cf9ef -> a277ae80a [SPARK-20474] Fixing OnHeapColumnVector reallocation ## What changes were proposed in this pull request? OnHeapColumnVector reallocation copies to the new storage data up to 'elementsAppended'. This variable is only updated when using the ColumnVector.appendX API, while ColumnVector.putX is more commonly used. ## How was this patch tested? Tested using existing unit tests. Author: Michal Szafranski Closes #17773 from michal-databricks/spark-20474. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a277ae80 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a277ae80 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a277ae80 Branch: refs/heads/master Commit: a277ae80a2836e6533b338d2b9c4e59ed8a1daae Parents: 99c6cf9 Author: Michal Szafranski Authored: Wed Apr 26 12:47:37 2017 -0700 Committer: Reynold Xin Committed: Wed Apr 26 12:47:37 2017 -0700 -- .../vectorized/OnHeapColumnVector.java | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a277ae80/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java -- diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java index 9b410ba..94ed322 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java @@ -410,53 +410,53 @@ public final class OnHeapColumnVector extends ColumnVector { int[] newLengths = new int[newCapacity]; int[] newOffsets = new int[newCapacity]; if (this.arrayLengths != null) { -System.arraycopy(this.arrayLengths, 0, newLengths, 0, elementsAppended); -System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, elementsAppended); +System.arraycopy(this.arrayLengths, 0, newLengths, 0, capacity); +System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, capacity); } arrayLengths = newLengths; arrayOffsets = newOffsets; } else if (type instanceof BooleanType) { if (byteData == null || byteData.length < newCapacity) { byte[] newData = new byte[newCapacity]; -if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended); +if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity); byteData = newData; } } else if (type instanceof ByteType) { if (byteData == null || byteData.length < newCapacity) { byte[] newData = new byte[newCapacity]; -if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended); +if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity); byteData = newData; } } else if (type instanceof ShortType) { if (shortData == null || shortData.length < newCapacity) { short[] newData = new short[newCapacity]; -if (shortData != null) System.arraycopy(shortData, 0, newData, 0, elementsAppended); +if (shortData != null) System.arraycopy(shortData, 0, newData, 0, capacity); shortData = newData; } } else if (type instanceof IntegerType || type instanceof DateType || DecimalType.is32BitDecimalType(type)) { if (intData == null || intData.length < newCapacity) { int[] newData = new int[newCapacity]; -if (intData != null) System.arraycopy(intData, 0, newData, 0, elementsAppended); +if (intData != null) System.arraycopy(intData, 0, newData, 0, capacity); intData = newData; } } else if (type instanceof LongType || type instanceof TimestampType || DecimalType.is64BitDecimalType(type)) { if (longData == null || longData.length < newCapacity) { long[] newData = new long[newCapacity]; -if (longData != null) System.arraycopy(longData, 0, newData, 0, elementsAppended); +if (longData != null) System.arraycopy(longData, 0, newData, 0, capacity); longData = newData; } } else if (type instanceof FloatType) { if (floatData == null || floatData.length < newCapacity) { float[] newData = new float[newCapacity]; -if (floatData != null) System.arraycopy(floatData, 0, newData, 0, elementsAppended); +if (floatData != null) System.arraycopy(floatData, 0, newData, 0, capacity); floatData = newData; } } else if (type instanceof DoubleTyp