Repository: parquet-mr Updated Branches: refs/heads/master 6dad1e3bd -> 044de16c1
PARQUET-623: Fix DeltaByteArrayReader#skip. Previously, this passed the skip to the underlying readers, but would not update previous and would corrupt values or cause exceptions. Author: Ryan Blue <[email protected]> Closes #366 from rdblue/PARQUET-623-fix-delta-byte-array-skip and squashes the following commits: f85800c [Ryan Blue] PARQUET-623: Fix DeltaByteArrayReader#skip. Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/044de16c Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/044de16c Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/044de16c Branch: refs/heads/master Commit: 044de16c14076019f87763b7b58c45664ee57c11 Parents: 6dad1e3 Author: Ryan Blue <[email protected]> Authored: Thu Sep 8 14:22:30 2016 -0700 Committer: Julien Le Dem <[email protected]> Committed: Thu Sep 8 14:22:30 2016 -0700 ---------------------------------------------------------------------- .../values/deltastrings/DeltaByteArrayReader.java | 4 ++-- .../values/deltastrings/TestDeltaByteArray.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java index c2cfd6d..742b515 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java @@ -55,8 +55,8 @@ public class DeltaByteArrayReader extends ValuesReader implements RequiresPrevio @Override public void skip() { - prefixLengthReader.skip(); - suffixReader.skip(); + // read the next value to skip so that previous is correct. + readBytes(); } @Override http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java ---------------------------------------------------------------------- diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java index 770f4dc..4f8f40c 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java @@ -19,6 +19,7 @@ package org.apache.parquet.column.values.deltastrings; import java.io.IOException; +import java.nio.ByteBuffer; import org.junit.Test; import org.junit.Assert; @@ -50,6 +51,13 @@ public class TestDeltaByteArray { } @Test + public void testRandomStringsWithSkip() throws Exception { + DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator()); + DeltaByteArrayReader reader = new DeltaByteArrayReader(); + assertReadWriteWithSkip(writer, reader, randvalues); + } + + @Test public void testLengths() throws IOException { DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator()); ValuesReader reader = new DeltaBinaryPackingValuesReader(); @@ -81,6 +89,16 @@ public class TestDeltaByteArray { } } + private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArrayReader reader, String[] vals) throws Exception { + Utils.writeData(writer, vals); + + reader.initFromPage(vals.length, writer.getBytes().toByteBuffer(), 0); + for (int i = 0; i < vals.length; i += 2) { + Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes()); + reader.skip(); + } + } + @Test public void testWriterReset() throws Exception { DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
