This is an automated email from the ASF dual-hosted git repository.
william pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new e8374e3b8 ORC-1569: Remove HadoopShimsPre2_3, HadoopShimsPre2_6,
HadoopShimsPre2_7 classes
e8374e3b8 is described below
commit e8374e3b8170b02d5e870617dd73537e2f835f59
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jan 3 22:50:33 2024 -0800
ORC-1569: Remove HadoopShimsPre2_3, HadoopShimsPre2_6, HadoopShimsPre2_7
classes
### What changes were proposed in this pull request?
This PR aims to remove `HadoopShimsPre2_3`, `HadoopShimsPre2_6`,
`HadoopShimsPre2_7` classes and use `HadoopShimsCurrent` always.
### Why are the changes needed?
1. `HadoopShimsCurrent` supports not only Apache Hadoop 3+ but also Apache
Hadoop 2.7+.
2. Apache ORC 2.0 uses Hadoop 3.x shaded client doesn't need old shims for
Hadoop 2.6 and olders.
- #1509
In addition, Apache Spark community also has been using the shaded Hadoop
client since Spark 3.2 (SPARK-33212) and dropped `Hadoop 2` profile via
[SPARK-42452](https://issues.apache.org/jira/browse/SPARK-42452) completely at
Spark 3.5.0.
### How was this patch tested?
Pass the CIs.
Closes #1724 from dongjoon-hyun/ORC-1569.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: William Hyun <[email protected]>
---
.../org/apache/orc/impl/HadoopShimsFactory.java | 18 +-----
.../org/apache/orc/impl/HadoopShimsPre2_3.java | 60 --------------------
.../org/apache/orc/impl/HadoopShimsPre2_6.java | 60 --------------------
.../org/apache/orc/impl/HadoopShimsPre2_7.java | 66 ----------------------
4 files changed, 2 insertions(+), 202 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java
b/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java
index 1cd98e067..1366d3d4f 100644
--- a/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/HadoopShimsFactory.java
@@ -32,11 +32,6 @@ public class HadoopShimsFactory {
private static final String CURRENT_SHIM_NAME =
"org.apache.orc.impl.HadoopShimsCurrent";
- private static final String PRE_2_6_SHIM_NAME =
- "org.apache.orc.impl.HadoopShimsPre2_6";
- private static final String PRE_2_7_SHIM_NAME =
- "org.apache.orc.impl.HadoopShimsPre2_7";
-
private static HadoopShims SHIMS = null;
private static HadoopShims createShimByName(String name) {
@@ -57,18 +52,9 @@ public class HadoopShimsFactory {
int major = Integer.parseInt(versionParts[0]);
int minor = Integer.parseInt(versionParts[1]);
if (major < 2 || (major == 2 && minor < 7)) {
- LOG.warn("Hadoop " + VersionInfo.getVersion() + " support is
deprecated. " +
- "Please upgrade to Hadoop 2.7.3 or above.");
- }
- if (major < 2 || (major == 2 && minor < 3)) {
- SHIMS = new HadoopShimsPre2_3();
- } else if (major == 2 && minor < 6) {
- SHIMS = createShimByName(PRE_2_6_SHIM_NAME);
- } else if (major == 2 && minor < 7) {
- SHIMS = createShimByName(PRE_2_7_SHIM_NAME);
- } else {
- SHIMS = createShimByName(CURRENT_SHIM_NAME);
+ LOG.warn("Hadoop " + VersionInfo.getVersion() + " support is
dropped.");
}
+ SHIMS = createShimByName(CURRENT_SHIM_NAME);
}
return SHIMS;
}
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java
b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java
deleted file mode 100644
index d53a377b2..000000000
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_3.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Random;
-
-/**
- * Shims for versions of Hadoop up to and including 2.2.x
- */
-public class HadoopShimsPre2_3 implements HadoopShims {
-
- HadoopShimsPre2_3() {
- }
-
- @Override
- public DirectDecompressor getDirectDecompressor(
- DirectCompressionType codec) {
- return null;
- }
-
- @Override
- public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
- ByteBufferPoolShim pool
- ) throws IOException {
- /* not supported */
- return null;
- }
-
- @Override
- public boolean endVariableLengthBlock(OutputStream output) {
- return false;
- }
-
- @Override
- public KeyProvider getHadoopKeyProvider(Configuration conf, Random random) {
- return new NullKeyProvider();
- }
-
-}
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java
b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java
deleted file mode 100644
index b59171d50..000000000
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_6.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Random;
-
-/**
- * Shims for versions of Hadoop less than 2.6
- * <p>
- * Adds support for:
- * <ul>
- * <li>Direct buffer decompression</li>
- * <li>Zero copy</li>
- * </ul>
- */
-public class HadoopShimsPre2_6 implements HadoopShims {
-
- @Override
- public DirectDecompressor getDirectDecompressor(DirectCompressionType codec)
{
- return HadoopShimsCurrent.getDecompressor(codec);
- }
-
- @Override
- public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
- ByteBufferPoolShim pool
- ) throws IOException {
- return ZeroCopyShims.getZeroCopyReader(in, pool);
- }
-
- @Override
- public boolean endVariableLengthBlock(OutputStream output) {
- return false;
- }
-
- @Override
- public KeyProvider getHadoopKeyProvider(Configuration conf, Random random) {
- return new NullKeyProvider();
- }
-}
diff --git a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
b/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
deleted file mode 100644
index f8ad0ee06..000000000
--- a/java/shims/src/java/org/apache/orc/impl/HadoopShimsPre2_7.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Random;
-
-/**
- * Shims for versions of Hadoop less than 2.7.
- * <p>
- * Adds support for:
- * <ul>
- * <li>Crypto</li>
- * </ul>
- */
-public class HadoopShimsPre2_7 implements HadoopShims {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(HadoopShimsPre2_7.class);
-
-
- @Override
- public DirectDecompressor getDirectDecompressor(DirectCompressionType codec)
{
- return HadoopShimsCurrent.getDecompressor(codec);
- }
-
- @Override
- public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
- ByteBufferPoolShim pool
- ) throws IOException {
- return ZeroCopyShims.getZeroCopyReader(in, pool);
- }
-
- @Override
- public boolean endVariableLengthBlock(OutputStream output) {
- return false;
- }
-
- @Override
- public KeyProvider getHadoopKeyProvider(Configuration conf,
- Random random) throws IOException {
- return HadoopShimsCurrent.createKeyProvider(conf, random);
- }
-}
