DRILL-498: cast to varchar only returns one row
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e2bb5418 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e2bb5418 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e2bb5418 Branch: refs/heads/master Commit: e2bb54188e060b79bc8bc478390af01a7ab91f10 Parents: 7ec8669 Author: Steven Phillips <[email protected]> Authored: Wed Apr 9 20:03:37 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sat Apr 19 21:07:28 2014 -0700 ---------------------------------------------------------------------- .../exec/store/parquet/VarLenBinaryReader.java | 51 ++++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e2bb5418/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java index d9e498e..09d19a8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java @@ -87,20 +87,16 @@ public class VarLenBinaryReader { NullableVarBinaryVector currNullVec; // write the first 0 offset for (ColumnReader columnReader : columns) { - currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector(); - currVec.getAccessor().getOffsetVector().getData().writeInt(0); columnReader.bytesReadInCurrentPass = 0; columnReader.valuesReadInCurrentPass = 0; } // same for the nullable columns for (NullableVarLengthColumn columnReader : nullableColumns) { - currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector(); - currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData().writeInt(0); columnReader.bytesReadInCurrentPass = 0; columnReader.valuesReadInCurrentPass = 0; columnReader.nullsRead = 0; } - do { + outer: do { lengthVarFieldsInCurrentRecord = 0; for (ColumnReader columnReader : columns) { if (recordsReadInCurrentPass == columnReader.valueVecHolder.getValueVector().getValueCapacity()){ @@ -121,6 +117,11 @@ public class VarLenBinaryReader { columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes, (int) columnReader.pageReadStatus.readPosInBytes); lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits; + + if (columnReader.bytesReadInCurrentPass + columnReader.dataTypeLengthInBits > ((VarBinaryVector) columnReader.valueVecHolder.getValueVector()).getData().capacity()) { + break outer; + } + } for (NullableVarLengthColumn columnReader : nullableColumns) { // check to make sure there is capacity for the next value (for nullables this is a check to see if there is @@ -150,57 +151,55 @@ public class VarLenBinaryReader { (int) columnReader.pageReadStatus.readPosInBytes); lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits; + if (columnReader.bytesReadInCurrentPass + columnReader.dataTypeLengthInBits > ((NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector()).getData().capacity()) { + break outer; + } } // check that the next record will fit in the batch if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord > parentReader.getBatchSize()){ - break; - } - else{ - recordsReadInCurrentPass++; + break outer; } for (ColumnReader columnReader : columns) { bytes = columnReader.pageReadStatus.pageDataByteArray; currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector(); // again, I am re-purposing the unused field here, it is a length n BYTES, not bits - currVec.getAccessor().getOffsetVector().getData().writeInt((int) columnReader.bytesReadInCurrentPass + - columnReader.dataTypeLengthInBits - 4 * (int) columnReader.valuesReadInCurrentPass); - currVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4, - columnReader.dataTypeLengthInBits); + boolean success = currVec.getMutator().setSafe(columnReader.valuesReadInCurrentPass, bytes, + (int) columnReader.pageReadStatus.readPosInBytes + 4, columnReader.dataTypeLengthInBits); + assert success; columnReader.pageReadStatus.readPosInBytes += columnReader.dataTypeLengthInBits + 4; columnReader.bytesReadInCurrentPass += columnReader.dataTypeLengthInBits + 4; columnReader.pageReadStatus.valuesRead++; columnReader.valuesReadInCurrentPass++; - currVec.getMutator().setValueCount((int)recordsReadInCurrentPass); } for (NullableVarLengthColumn columnReader : nullableColumns) { bytes = columnReader.pageReadStatus.pageDataByteArray; currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector(); // again, I am re-purposing the unused field here, it is a length n BYTES, not bits - currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData() - .writeInt( - (int) columnReader.bytesReadInCurrentPass + - columnReader.dataTypeLengthInBits - 4 * (columnReader.valuesReadInCurrentPass - - (columnReader.currentValNull ? Math.max (0, columnReader.nullsRead - 1) : columnReader.nullsRead))); - columnReader.currentValNull = false; - if (columnReader.dataTypeLengthInBits > 0){ - currNullVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4, - columnReader.dataTypeLengthInBits); - ((NullableVarBinaryVector)columnReader.valueVecHolder.getValueVector()).getMutator().setIndexDefined(columnReader.valuesReadInCurrentPass); + if (!columnReader.currentValNull && columnReader.dataTypeLengthInBits > 0){ + boolean success = currNullVec.getMutator().setSafe(columnReader.valuesReadInCurrentPass, bytes, + (int) columnReader.pageReadStatus.readPosInBytes + 4, columnReader.dataTypeLengthInBits); + assert success; } + columnReader.currentValNull = false; if (columnReader.dataTypeLengthInBits > 0){ columnReader.pageReadStatus.readPosInBytes += columnReader.dataTypeLengthInBits + 4; columnReader.bytesReadInCurrentPass += columnReader.dataTypeLengthInBits + 4; } columnReader.pageReadStatus.valuesRead++; columnReader.valuesReadInCurrentPass++; - currNullVec.getMutator().setValueCount((int)recordsReadInCurrentPass); - // reached the end of a page if ( columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) { columnReader.pageReadStatus.next(); } } + recordsReadInCurrentPass++; } while (recordsReadInCurrentPass < recordsToReadInThisPass); + for (VarLengthColumn columnReader : columns) { + columnReader.valueVecHolder.getValueVector().getMutator().setValueCount((int) recordsReadInCurrentPass); + } + for (NullableVarLengthColumn columnReader : nullableColumns) { + columnReader.valueVecHolder.getValueVector().getMutator().setValueCount((int) recordsReadInCurrentPass); + } return recordsReadInCurrentPass; } } \ No newline at end of file
