This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 56c8aa5f1c4 HADOOP-19204. VectorIO regression: empty ranges are now rejected (#6887) 56c8aa5f1c4 is described below commit 56c8aa5f1c4a0336f69083c742e2504ccc828d7d Author: Steve Loughran <ste...@cloudera.com> AuthorDate: Wed Jun 19 12:05:24 2024 +0100 HADOOP-19204. VectorIO regression: empty ranges are now rejected (#6887) - restore old outcome: no-op - test this - update spec This is a critical fix for vector IO and MUST be cherrypicked to all branches with that feature Contributed by Steve Loughran --- .../src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java | 9 ++++++++- .../src/site/markdown/filesystem/fsdatainputstream.md | 1 - .../hadoop/fs/contract/AbstractContractVectoredReadTest.java | 11 +++++++++++ .../java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java | 7 +++---- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java index 493b8c3a33d..fa0440620a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java @@ -294,7 +294,14 @@ public final class VectoredReadUtils { final Optional<Long> fileLength) throws EOFException { requireNonNull(input, "Null input list"); - checkArgument(!input.isEmpty(), "Empty input list"); + + if (input.isEmpty()) { + // this may seem a pathological case, but it was valid + // before and somehow Spark can call it through parquet. + LOG.debug("Empty input list"); + return input; + } + final List<? extends FileRange> sortedRanges; if (input.size() == 1) { diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md index 6cbb54ea701..db844a94e39 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md @@ -474,7 +474,6 @@ No empty lists. ```python if ranges = null raise NullPointerException -if ranges.len() = 0 raise IllegalArgumentException if allocate = null raise NullPointerException ``` diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java index d6a1fb1f0b7..aa478f3af63 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java @@ -340,6 +340,17 @@ public abstract class AbstractContractVectoredReadTest extends AbstractFSContrac } } + @Test + public void testEmptyRanges() throws Exception { + List<FileRange> fileRanges = new ArrayList<>(); + try (FSDataInputStream in = openVectorFile()) { + in.readVectored(fileRanges, allocate); + Assertions.assertThat(fileRanges) + .describedAs("Empty ranges must stay empty") + .isEmpty(); + } + } + /** * Test to validate EOF ranges. * <p> diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java index 2a290058cae..3fd3fe4d1f4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestVectoredReadUtils.java @@ -702,12 +702,11 @@ public class TestVectoredReadUtils extends HadoopTestBase { } /** - * Empty ranges cannot be sorted. + * Empty ranges are allowed. */ @Test - public void testEmptyRangesRaisesIllegalArgument() throws Throwable { - intercept(IllegalArgumentException.class, - () -> validateAndSortRanges(Collections.emptyList(), Optional.empty())); + public void testEmptyRangesAllowed() throws Throwable { + validateAndSortRanges(Collections.emptyList(), Optional.empty()); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org