(orc) branch main updated: ORC-1570: Add `supportVectoredIO` API to `HadoopShimsCurrent` and use it

[dongjoon]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/main by this push:
     new 68f4a8bb9 ORC-1570: Add `supportVectoredIO` API to 
`HadoopShimsCurrent` and use it
68f4a8bb9 is described below

commit 68f4a8bb970cb370faba6b907ae93164e3243042
Author: Dongjoon Hyun <dongj...@apache.org>
AuthorDate: Thu Jan 4 00:23:10 2024 -0800

    ORC-1570: Add `supportVectoredIO` API to `HadoopShimsCurrent` and use it
    
    ### What changes were proposed in this pull request?
    
    This PR aims to improve `HadoopShimsCurrent` by adding `supportVectoredIO`.
    
    ### Why are the changes needed?
    
    Hadoop Vectored IO exists at Apache Hadoop 3.3.5+ via HADOOP-18103.
    
    ### How was this patch tested?
    
    Pass the CIs.
    
    Closes #1725 from dongjoon-hyun/ORC-1570.
    
    Authored-by: Dongjoon Hyun <dongj...@apache.org>
    Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
 java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java |  3 ++-
 java/shims/src/java/org/apache/orc/impl/HadoopShims.java      | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java 
b/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
index 717e49737..0eabb421e 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
@@ -48,6 +48,7 @@ import java.util.function.Supplier;
  */
 public class RecordReaderUtils {
   private static final HadoopShims SHIMS = HadoopShimsFactory.get();
+  private static final boolean supportVectoredIO = SHIMS.supportVectoredIO();
   private static final Logger LOG = 
LoggerFactory.getLogger(RecordReaderUtils.class);
 
   private static class DefaultDataReader implements DataReader {
@@ -107,7 +108,7 @@ public class RecordReaderUtils {
     public BufferChunkList readFileData(BufferChunkList range,
                                         boolean doForceDirect
                                         ) throws IOException {
-      if (zcr == null) {
+      if (supportVectoredIO && zcr == null) {
         RecordReaderUtils.readDiskRangesVectored(file, range, doForceDirect);
       } else {
         RecordReaderUtils.readDiskRanges(file, zcr, range, doForceDirect,
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShims.java 
b/java/shims/src/java/org/apache/orc/impl/HadoopShims.java
index d15768de8..2ae0364f2 100644
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShims.java
+++ b/java/shims/src/java/org/apache/orc/impl/HadoopShims.java
@@ -20,6 +20,7 @@ package org.apache.orc.impl;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.util.VersionInfo;
 import org.apache.orc.EncryptionAlgorithm;
 
 import java.io.Closeable;
@@ -131,6 +132,15 @@ public interface HadoopShims {
    */
   boolean endVariableLengthBlock(OutputStream output) throws IOException;
 
+  default boolean supportVectoredIO() {
+    // HADOOP-18103 is available since Apache Hadoop 3.3.5+
+    String[] versionParts = VersionInfo.getVersion().split("[.]");
+    int major = Integer.parseInt(versionParts[0]);
+    int minor = Integer.parseInt(versionParts[1]);
+    int patch = Integer.parseInt(versionParts[2]);
+    return major == 3 && (minor > 3 || (minor == 3 && patch > 4));
+  }
+
   /**
    * The known KeyProviders for column encryption.
    * These are identical to OrcProto.KeyProviderKind.