hhhizzz commented on code in PR #11791:
URL: https://github.com/apache/arrow-rs/pull/11791#discussion_r4386471620
##########
parquet/src/arrow/array_reader/byte_array.rs:
##########
@@ -767,4 +779,82 @@ mod tests {
assert_eq!(output.offsets.as_slice(), &[0, 3]);
assert_eq!(decoder.max_remaining_values, 1);
}
+
+ #[test]
+ fn test_delta_length_checked_offset() {
+ let max = i32::MAX as usize;
+ assert_eq!(
+ ByteArrayDecoderDeltaLength::checked_offset::<i32>(max - 1,
1).unwrap(),
+ i32::MAX
+ );
+ assert_eq!(
+ ByteArrayDecoderDeltaLength::checked_offset::<i32>(max,
0).unwrap(),
+ i32::MAX
+ );
+ for (offset, additional) in [(max, 1), (0, max + 1), (usize::MAX, 1)] {
+ let error =
+ ByteArrayDecoderDeltaLength::checked_offset::<i32>(offset,
additional).unwrap_err();
+ assert_eq!(
+ error.to_string(),
+ "Parquet error: index overflow decoding byte array"
+ );
+ }
+ assert_eq!(
+ ByteArrayDecoderDeltaLength::checked_offset::<i64>(max,
1).unwrap(),
+ i32::MAX as i64 + 1
+ );
+ assert!(ByteArrayDecoderDeltaLength::checked_offset::<i64>(usize::MAX,
1).is_err());
+ }
+
+ #[test]
+ fn test_delta_length_read_append_and_skip() {
+ let (pages, _) = byte_array_all_encodings(vec!["ab", "", "cde", "f"]);
+ let (_, page) = pages
+ .into_iter()
+ .find(|(encoding, _)| *encoding ==
Encoding::DELTA_LENGTH_BYTE_ARRAY)
+ .unwrap();
+ let mut decoder = ByteArrayDecoderDeltaLength::new(page,
true).unwrap();
+ let mut output = OffsetBuffer::<i32>::with_capacity(0);
+ output.try_push(b"prefix", true).unwrap();
+
+ assert_eq!(decoder.read(&mut output, 2).unwrap(), 2);
+ assert_eq!(output.values.as_slice(), b"prefixab");
+ assert_eq!(output.offsets.as_slice(), &[0, 6, 8, 8]);
+ assert_eq!(decoder.skip(1).unwrap(), 1);
+ assert_eq!(decoder.read(&mut output, 1).unwrap(), 1);
+ assert_eq!(output.values.as_slice(), b"prefixabf");
+ assert_eq!(output.offsets.as_slice(), &[0, 6, 8, 8, 9]);
+ assert_eq!(decoder.read(&mut output, 1).unwrap(), 0);
+ }
+
+ #[test]
+ #[ignore = "allocates about 2 GiB; run --ignored --test-threads=1"]
Review Comment:
My intended use case is to run this myself while developing and validating
the follow-up fixes in this series, rather than expect maintainers to remember
to run it. Once capacity-safe decoding and batch splitting are in place, I plan
to replace this interim panic-to-error check with end-to-end tests that verify
successful reads of more than 2 GiB of logical payload across valid output
batches. Ideally, that coverage would run automatically in a suitable CI job.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
