shangxinli commented on code in PR #1139:
URL: https://github.com/apache/parquet-mr/pull/1139#discussion_r1438296891
##########
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -1063,6 +1068,69 @@ public ColumnChunkPageReadStore readFilteredRowGroup(int
blockIndex, RowRanges r
return internalReadFilteredRowGroup(block, rowRanges,
getColumnIndexStore(blockIndex));
}
+ /**
+ * Read data in all parts via either vectored IO or serial IO.
+ * @param allParts all parts to be read.
+ * @param builder used to build chunk list to read the pages for the
different columns.
+ * @throws IOException any IOE.
+ */
+ private void readAllPartsVectoredOrNormal(List<ConsecutivePartList>
allParts, ChunkListBuilder builder)
+ throws IOException {
+ boolean isVectoredIO = options.useHadoopVectoredIO()
+ && f.readVectoredAvailable()
+ && partsLengthValidForVectoredIO(allParts);
+ if (isVectoredIO) {
+ readVectored(allParts, builder);
+ } else {
+ for (ConsecutivePartList consecutiveChunks : allParts) {
+ consecutiveChunks.readAll(f, builder);
+ }
+ }
+ }
+
+ /**
+ * Vectored IO doesn't support reading ranges of size greater than
+ * Integer.MAX_VALUE.
+ * @param allParts all parts to read.
+ * @return true or false.
+ */
+ private boolean partsLengthValidForVectoredIO(List<ConsecutivePartList>
allParts) {
+ for (ConsecutivePartList consecutivePart : allParts) {
+ if (consecutivePart.length >= Integer.MAX_VALUE) {
+ LOG.debug("Part length {} greater than Integer.MAX_VALUE thus
disabling vectored IO", consecutivePart.length);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Read all parts through vectored IO.
+ * @param allParts all parts to be read.
+ * @param builder used to build chunk list to read the pages for the
different columns.
+ * @throws IOException any IOE.
+ */
+ private void readVectored(List<ConsecutivePartList> allParts,
+ ChunkListBuilder builder) throws IOException {
+
+ List<ParquetFileRange> ranges = new ArrayList<>(allParts.size());
+ for (ConsecutivePartList consecutiveChunks : allParts) {
+ Preconditions.checkArgument(consecutiveChunks.length < Integer.MAX_VALUE,
+ "Invalid length %s for vectored read operation. It must be less than
max integer value.",
+ consecutiveChunks.length);
+ ranges.add(new ParquetFileRange(consecutiveChunks.offset, (int)
consecutiveChunks.length));
+ }
+ LOG.debug("Doing vectored IO for ranges {}", ranges);
+ ByteBufferAllocator allocator = options.getAllocator();
+ //blocking or asynchronous vectored read.
Review Comment:
The comment is not very clear. Can you move it to be above the definition
of the readVectorized(...) where you can make more comments to make it more
clear
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
