Updated Branches: refs/heads/master 39bb519c8 -> 1db6ca01d
TAJO-47: RowFile has the duplicated initialization problem and unflipped ByteBuffer problem. (jihoon) Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/1db6ca01 Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/1db6ca01 Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/1db6ca01 Branch: refs/heads/master Commit: 1db6ca01d42288387725c2cb3ca0fafed450f2f9 Parents: 39bb519 Author: Jihoon Son <[email protected]> Authored: Fri May 3 10:07:57 2013 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri May 3 10:07:57 2013 +0900 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../src/test/java/tajo/storage/TestRowFile.java | 2 + .../src/main/java/tajo/storage/RowFile.java | 24 ++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e2cae30..db43b6d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -32,6 +32,9 @@ Release 0.2.0 - unreleased BUG FIXES + TAJO-47: RowFile has the duplicated initialization problem and unflipped + ByteBuffer problem. (jihoon) + TAJO-38: Update class comment in TaskAttemptContext from Korean to English (hsaputra) http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java b/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java index 4e98d8b..cd3058f 100644 --- a/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java +++ b/tajo-core/tajo-core-backend/src/test/java/tajo/storage/TestRowFile.java @@ -118,6 +118,7 @@ public class TestRowFile { int tupleCnt = 0; start = System.currentTimeMillis(); Scanner scanner = new RowFile.RowFileScanner(conf, meta, fragment); + scanner.init(); while ((tuple=scanner.next()) != null) { tupleCnt++; // System.out.println(tuple.toString()); @@ -137,6 +138,7 @@ public class TestRowFile { System.out.println("range: " + fileStart + ", " + fileLen); fragment = new Fragment("test.tbl", dataPath, meta, fileStart, fileLen, null); scanner = new RowFile.RowFileScanner(conf, meta, fragment); + scanner.init(); while ((tuple=scanner.next()) != null) { if (!idSet.remove(tuple.get(0).asInt())) { System.out.println("duplicated! " + tuple.get(0).asInt()); http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/1db6ca01/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java index 4ee8cec..119fcf4 100644 --- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java +++ b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/RowFile.java @@ -79,8 +79,6 @@ public class RowFile { tupleHeaderSize = nullFlags.size() + (2 * Short.SIZE/8); this.start = fragment.getStartOffset(); this.end = this.start + fragment.getLength(); - - init(); } public void init() throws IOException { @@ -99,7 +97,6 @@ public class RowFile { } bufferStartPos = in.getPos(); fillBuffer(); - fillBuffer(); // due to the bug of FSDataInputStream.read(ByteBuffer) if (start != 0) { // TODO: improve @@ -137,22 +134,29 @@ public class RowFile { return Arrays.equals(checkSync, sync); } - private boolean fillBuffer() throws IOException { + private int fillBuffer() throws IOException { bufferStartPos += buffer.position(); buffer.compact(); + int remain = buffer.remaining(); int read = in.read(buffer); - if (read < 0) { - return false; + if (read == -1) { + buffer.flip(); + return read; } else { + int totalRead = read; + if (remain > totalRead) { + read = in.read(buffer); + totalRead += read > 0 ? read : 0; + } buffer.flip(); - return true; + return totalRead; } } @Override public Tuple next() throws IOException { while (buffer.remaining() < SYNC_SIZE) { - if (!fillBuffer()) { + if (fillBuffer() < 0) { return null; } } @@ -167,7 +171,7 @@ public class RowFile { } while (buffer.remaining() < tupleHeaderSize) { - if (!fillBuffer()) { + if (fillBuffer() < 0) { return null; } } @@ -182,7 +186,7 @@ public class RowFile { int tupleSize = buffer.getShort(); while (buffer.remaining() < (tupleSize)) { - if (!fillBuffer()) { + if (fillBuffer() < 0) { return null; } }
