jaideeppyne commented on code in PR #11136:
URL: https://github.com/apache/arrow-rs/pull/11136#discussion_r4575072636
##########
parquet/src/encodings/decoding.rs:
##########
@@ -2175,6 +2175,70 @@ mod tests {
test_byte_stream_split_decode::<Int64Type>(data, -1);
}
+ #[test]
+ fn test_byte_stream_split_uses_encoded_value_count() {
+ let mut decoder = ByteStreamSplitDecoder::<DoubleType>::new();
+
+ for count in [8, 63, usize::MAX] {
+ decoder.set_data(Bytes::from(vec![0; 64]), count).unwrap();
+ assert_eq!(decoder.values_left(), 8);
+ let mut values = vec![0.; 63];
+ assert_eq!(decoder.get(&mut values[..3]).unwrap(), 3);
+ assert_eq!(decoder.skip(2).unwrap(), 2);
+ assert_eq!(decoder.get(&mut values).unwrap(), 3);
+ assert_eq!(decoder.values_left(), 0);
+ assert_eq!(decoder.get(&mut values).unwrap(), 0);
+ }
+ }
+
+ #[test]
+ fn test_byte_stream_split_rejects_invalid_data_length() {
+ let mut decoder = ByteStreamSplitDecoder::<DoubleType>::new();
+
+ let err = decoder.set_data(Bytes::from(vec![0; 63]), 8).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: Invalid BYTE_STREAM_SPLIT data length 63 for
values of size 8"
+ );
+ }
+
+ #[test]
+ fn test_byte_stream_split_rejects_more_values_than_expected() {
+ let mut decoder = ByteStreamSplitDecoder::<DoubleType>::new();
+ let err = decoder.set_data(Bytes::from(vec![0; 64]), 7).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: Invalid BYTE_STREAM_SPLIT data: 8 encoded values
exceeds the 7 values expected"
+ );
+
+ let mut decoder =
VariableWidthByteStreamSplitDecoder::<FixedLenByteArrayType>::new(5);
+ let err = decoder.set_data(Bytes::from(vec![0; 40]), 7).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Parquet error: Invalid BYTE_STREAM_SPLIT data: 8 encoded values
exceeds the 7 values expected"
+ );
+ }
+
+ #[test]
+ fn test_byte_stream_split_flba_uses_encoded_value_count() {
Review Comment:
The end-to-end test performs double reads. A distinct decoder handles
fixed-length byte arrays, so the new test verifies that decoder directly. It
writes eight values, then reads three, skips two, and requests additional
values. It should return the three remaining values, followed by zero, even
when the requested count is larger. The test employs 5-byte and 16-byte values
to exercise both the generic and optimized decoding paths.
##########
parquet/tests/arrow_reader/bad_data.rs:
##########
@@ -136,6 +136,19 @@ fn test_arrow_gh_47662() {
);
}
+#[test]
+fn test_arrow_rs_gh_11136_byte_stream_split_truncated_page() {
+ // Checked-in malformed page: declared value count exceeds
BYTE_STREAM_SPLIT payload.
+ let file =
Bytes::from_static(include_bytes!("ARROW-RS-GH-11136-bss-truncated.parquet"));
+ let reader = ArrowReaderBuilder::try_new(file).unwrap().build().unwrap();
+ let err = reader.collect::<Result<Vec<_>, _>>().unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("insufficient values read from column"),
Review Comment:
The new decoder error detects a payload containing more values than the
provided count permits. A payload with fewer values may still be valid because,
on a V1 page, that count can include nulls that occupy no space in the value
stream. Therefore, the decoder returns only the values it can actually read. In
this test, the column reader still requires additional values to complete, so
it reports the shortfall.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]