orc git commit: ORC-367: Fix incorrect reads of boolean columns after a seek.

2018-05-22 Thread omalley 
Repository: orc
Updated Branches:
  refs/heads/branch-1.5 90250bb02 -> be1036df6


ORC-367: Fix incorrect reads of boolean columns after a seek.

Fix #272

Signed-off-by: Owen O'Malley 


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/be1036df
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/be1036df
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/be1036df

Branch: refs/heads/branch-1.5
Commit: be1036df64829ba13cacc2c413c9dfd8c080099d
Parents: 90250bb
Author: Owen O'Malley 
Authored: Tue May 22 14:43:06 2018 -0700
Committer: Owen O'Malley 
Committed: Tue May 22 15:06:44 2018 -0700

--
 .../org/apache/orc/impl/BitFieldReader.java |  8 ++---
 .../org/apache/orc/impl/TestBitFieldReader.java | 35 
 2 files changed, 38 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/orc/blob/be1036df/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
--
diff --git a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java 
b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
index 9102555..5daa204 100644
--- a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
+++ b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
@@ -24,9 +24,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 
 public final class BitFieldReader {
   private final RunLengthByteReader input;
-  /** The number of bits in one item. Non-test code always uses 1. */
   private int current;
-  private byte currentIdx;
+  private byte currentIdx = 8;
 
   public BitFieldReader(InStream input) {
 this.input = new RunLengthByteReader(input);
@@ -42,8 +41,7 @@ public final class BitFieldReader {
   }
 
   public int next() throws IOException {
-// mod 8
-if ((currentIdx & 7) == 0) {
+if (currentIdx > 7) {
   readByte();
 }
 
@@ -86,7 +84,7 @@ public final class BitFieldReader {
   readByte();
   currentIdx = (byte) consumed;
 } else {
-  currentIdx = 0;
+  currentIdx = 8;
 }
   }
 

http://git-wip-us.apache.org/repos/asf/orc/blob/be1036df/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
--
diff --git a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java 
b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
index 156f10a..f7a2a5c 100644
--- a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
+++ b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
@@ -111,4 +111,39 @@ public class TestBitFieldReader {
   in.skip(0);
 }
   }
+
+  @Test
+  public void testSeekSkip() throws Exception {
+TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+BitFieldWriter out = new BitFieldWriter(
+new OutStream("test", 100, null, collect), 1);
+final int COUNT = 256;
+TestInStream.PositionCollector posn = new TestInStream.PositionCollector();
+for(int i=0; i < COUNT; ++i) {
+  if (i == 200) {
+out.getPosition(posn);
+  }
+  if (i < COUNT/2) {
+out.write(i & 1);
+  } else {
+out.write((i/3) & 1);
+  }
+}
+out.flush();
+ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+inBuf.flip();
+BitFieldReader in = new BitFieldReader(InStream.create("test", new 
ByteBuffer[]{inBuf},
+new long[]{0}, inBuf.remaining(), null, 100));
+in.seek(posn);
+in.skip(10);
+for(int r = 210; r < COUNT; ++r) {
+  int x = (int) in.next();
+  if (r < COUNT/2) {
+assertEquals(r & 1, x);
+  } else {
+assertEquals((r/3) & 1, x);
+  }
+}
+  }
 }

orc git commit: ORC-367: Fix incorrect reads of boolean columns after a seek.

2018-05-22 Thread omalley 
Repository: orc
Updated Branches:
  refs/heads/master 5120baaef -> b0504f55e


ORC-367: Fix incorrect reads of boolean columns after a seek.

Fix #272

Signed-off-by: Owen O'Malley 


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/b0504f55
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/b0504f55
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/b0504f55

Branch: refs/heads/master
Commit: b0504f55e4a68de0c73b89ab5bccb2a673efe9b4
Parents: 5120baa
Author: Owen O'Malley 
Authored: Tue May 22 14:43:06 2018 -0700
Committer: Owen O'Malley 
Committed: Tue May 22 14:45:30 2018 -0700

--
 .../org/apache/orc/impl/BitFieldReader.java |  8 ++---
 .../org/apache/orc/impl/TestBitFieldReader.java | 35 
 2 files changed, 38 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
--
diff --git a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java 
b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
index 9102555..5daa204 100644
--- a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
+++ b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
@@ -24,9 +24,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 
 public final class BitFieldReader {
   private final RunLengthByteReader input;
-  /** The number of bits in one item. Non-test code always uses 1. */
   private int current;
-  private byte currentIdx;
+  private byte currentIdx = 8;
 
   public BitFieldReader(InStream input) {
 this.input = new RunLengthByteReader(input);
@@ -42,8 +41,7 @@ public final class BitFieldReader {
   }
 
   public int next() throws IOException {
-// mod 8
-if ((currentIdx & 7) == 0) {
+if (currentIdx > 7) {
   readByte();
 }
 
@@ -86,7 +84,7 @@ public final class BitFieldReader {
   readByte();
   currentIdx = (byte) consumed;
 } else {
-  currentIdx = 0;
+  currentIdx = 8;
 }
   }
 

http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
--
diff --git a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java 
b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
index 156f10a..f7a2a5c 100644
--- a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
+++ b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
@@ -111,4 +111,39 @@ public class TestBitFieldReader {
   in.skip(0);
 }
   }
+
+  @Test
+  public void testSeekSkip() throws Exception {
+TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+BitFieldWriter out = new BitFieldWriter(
+new OutStream("test", 100, null, collect), 1);
+final int COUNT = 256;
+TestInStream.PositionCollector posn = new TestInStream.PositionCollector();
+for(int i=0; i < COUNT; ++i) {
+  if (i == 200) {
+out.getPosition(posn);
+  }
+  if (i < COUNT/2) {
+out.write(i & 1);
+  } else {
+out.write((i/3) & 1);
+  }
+}
+out.flush();
+ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+inBuf.flip();
+BitFieldReader in = new BitFieldReader(InStream.create("test", new 
ByteBuffer[]{inBuf},
+new long[]{0}, inBuf.remaining(), null, 100));
+in.seek(posn);
+in.skip(10);
+for(int r = 210; r < COUNT; ++r) {
+  int x = (int) in.next();
+  if (r < COUNT/2) {
+assertEquals(r & 1, x);
+  } else {
+assertEquals((r/3) & 1, x);
+  }
+}
+  }
 }