sdf-jkl commented on code in PR #11539:
URL: https://github.com/apache/arrow-rs/pull/11539#discussion_r4648758961
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -733,11 +746,38 @@ where
#[cfg(test)]
mod tests {
use super::*;
+ use crate::encodings::alp::ALP_DEFAULT_LOG_VECTOR_SIZE;
+
+ /// The spec-recommended size the encoder defaults to; most tests pin to
it so
+ /// existing coverage keeps measuring the default path.
+ const DEFAULT_VECTOR_SIZE: usize = 1 << ALP_DEFAULT_LOG_VECTOR_SIZE;
use crate::data_type::{DoubleType, FloatType};
use crate::encodings::decoding::Decoder;
use crate::encodings::decoding::alp_decoder::AlpDecoder;
Review Comment:
Nit: `DEFAULT_VECTOR_SIZE` ended up between the `use` lines.
```suggestion
use crate::data_type::{DoubleType, FloatType};
use crate::encodings::alp::ALP_DEFAULT_LOG_VECTOR_SIZE;
use crate::encodings::decoding::Decoder;
use crate::encodings::decoding::alp_decoder::AlpDecoder;
/// The spec-recommended size the encoder defaults to; most tests pin to
it so
/// existing coverage keeps measuring the default path.
const DEFAULT_VECTOR_SIZE: usize = 1 << ALP_DEFAULT_LOG_VECTOR_SIZE;
```
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -733,11 +746,38 @@ where
#[cfg(test)]
mod tests {
use super::*;
+ use crate::encodings::alp::ALP_DEFAULT_LOG_VECTOR_SIZE;
+
+ /// The spec-recommended size the encoder defaults to; most tests pin to
it so
+ /// existing coverage keeps measuring the default path.
+ const DEFAULT_VECTOR_SIZE: usize = 1 << ALP_DEFAULT_LOG_VECTOR_SIZE;
use crate::data_type::{DoubleType, FloatType};
use crate::encodings::decoding::Decoder;
use crate::encodings::decoding::alp_decoder::AlpDecoder;
/// Encode `values` and read them back through the decoder.
+ /// Encode at `vector_size` and read back. The decoder takes the size from
the
+ /// page header, so it needs no configuration of its own.
+ fn roundtrip_with_log_vector_size<T: DataType>(
+ values: &[T::T],
+ log_vector_size: u8,
+ ) -> Vec<T::T>
+ where
+ T::T: AlpFloat,
+ <T::T as AlpFloat>::Exact: Send,
+ {
+ let options = AlpEncoderOptions::try_new(log_vector_size).unwrap();
+ let mut encoder = AlpEncoder::<T>::new_with_options(options);
+ encoder.put(values).unwrap();
+ let page = encoder.flush_buffer().unwrap();
+
+ let mut decoder = AlpDecoder::<T>::new();
+ decoder.set_data(page, values.len()).unwrap();
+ let mut out = vec![T::T::default(); values.len()];
+ assert_eq!(decoder.get(&mut out).unwrap(), values.len());
+ out
+ }
+
Review Comment:
With the suggestion on `test_roundtrip_all_vector_sizes` below, this helper
has no other caller. Removing it also gives `roundtrip` its doc line back.
```suggestion
/// Encode `values` and read them back through the decoder.
```
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -616,6 +618,8 @@ where
scratch: Scratch<T::T>,
/// The page built incrementally, used for every page after the first.
streaming: StreamingPage<T::T>,
+ /// Values per vector, from the column's `log_vector_size`.
+ vector_size: usize,
Review Comment:
This duplicates `StreamingPage::vector_size`. Could `AlpEncoder` read
`self.streaming.vector_size` instead? That also drops the `vector_size: _` in
`put`.
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -1125,4 +1167,55 @@ mod tests {
"a leading empty page must not affect encoding of the first
nonempty page"
);
}
+ /// Every `log_vector_size` the format permits must round-trip, including
the
+ /// 3 and 15 bounds, and across both the buffered first page and the
streaming path.
+ #[test]
+ fn test_roundtrip_all_vector_sizes() {
+ let values: Vec<f64> = (0..5000).map(|i| (i as f64) * 0.01 -
7.5).collect();
+ for log_vector_size in 3u8..=15 {
+ let out = roundtrip_with_log_vector_size::<DoubleType>(&values,
log_vector_size);
+ assert_bits_eq(&out, &values);
+ }
+ }
+
+ /// A non-default vector size must survive the streaming path, where the
size
+ /// governs the carry across `put` boundaries.
+ #[test]
+ fn test_streaming_with_non_default_vector_size() {
+ let page1: Vec<f64> = (0..3000).map(|i| (i as f64) * 0.01).collect();
+ let page2: Vec<f64> = (0..3000).map(|i| (i as f64) * 0.03 -
7.0).collect();
+ let options = AlpEncoderOptions::try_new(13).unwrap();
+ let vector_size = options.vector_size();
+
+ let mut encoder = AlpEncoder::<DoubleType>::new_with_options(options);
+ encoder.put(&page1).unwrap();
+ let _ = encoder.flush_buffer().unwrap();
+
+ // Feed page 2 in chunks that straddle vector boundaries.
+ for chunk in page2.chunks(997) {
+ encoder.put(chunk).unwrap();
+ }
+ let streamed = encoder.flush_buffer().unwrap();
+
+ // Same values, same preset, encoded in one pass.
+ let preset = build_preset(&page1, vector_size);
+ let mut scratch = Scratch::<f64>::new();
+ let reference = encode_page(&page2, &preset, vector_size, &mut
scratch).unwrap();
+ assert_eq!(streamed.as_ref(), reference.as_slice());
+ }
+
+ /// The configured size must reach the page header, which is what lets the
+ /// decoder read the page back without being told.
+ #[test]
+ fn test_vector_size_recorded_in_page_header() {
+ let values: Vec<f64> = (0..2000).map(|i| (i as f64) * 0.01).collect();
+ for log_vector_size in [3u8, 10, 13, 15] {
+ let options = AlpEncoderOptions::try_new(log_vector_size).unwrap();
+ let mut encoder =
AlpEncoder::<DoubleType>::new_with_options(options);
+ encoder.put(&values).unwrap();
+ let page = encoder.flush_buffer().unwrap();
+ // log_vector_size is byte 2 of the ALP header.
+ assert_eq!(page[2], log_vector_size);
+ }
+ }
Review Comment:
`test_alp_encoder_options_reach_the_encoder` in `mod.rs` checks the same
`page[2]` byte through the full `WriterProperties` path, so I think this one
can go.
```suggestion
```
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -1125,4 +1167,55 @@ mod tests {
"a leading empty page must not affect encoding of the first
nonempty page"
);
}
+ /// Every `log_vector_size` the format permits must round-trip, including
the
+ /// 3 and 15 bounds, and across both the buffered first page and the
streaming path.
+ #[test]
+ fn test_roundtrip_all_vector_sizes() {
+ let values: Vec<f64> = (0..5000).map(|i| (i as f64) * 0.01 -
7.5).collect();
+ for log_vector_size in 3u8..=15 {
+ let out = roundtrip_with_log_vector_size::<DoubleType>(&values,
log_vector_size);
+ assert_bits_eq(&out, &values);
+ }
+ }
Review Comment:
This only exercises the buffered first page, since there is one `put` and
one `flush`. Feeding two pages through the same encoder in 997-value `put`s
covers the streaming path and the carry across `put`s for all 13 sizes, which
is what the doc comment promises. With this change, the test also fails if the
carry split keeps a hardcoded 1024, which nothing else in the suite catches.
```suggestion
}
/// Every `log_vector_size` the format permits must round-trip,
including the
/// 3 and 15 bounds, on both the buffered first page and a streamed
second page.
#[test]
fn test_roundtrip_all_vector_sizes() {
let values: Vec<f64> = (0..5000).map(|i| (i as f64) * 0.01 -
7.5).collect();
for log_vector_size in 3u8..=15 {
let options =
AlpEncoderOptions::try_new(log_vector_size).unwrap();
let mut encoder =
AlpEncoder::<DoubleType>::new_with_options(options);
for _ in 0..2 {
for chunk in values.chunks(997) {
encoder.put(chunk).unwrap();
}
let page = encoder.flush_buffer().unwrap();
let mut decoder = AlpDecoder::<DoubleType>::new();
decoder.set_data(page, values.len()).unwrap();
let mut out = vec![0.0; values.len()];
assert_eq!(decoder.get(&mut out).unwrap(), values.len());
assert_bits_eq(&out, &values);
}
}
}
```
##########
parquet/src/encodings/encoding/alp_encoder.rs:
##########
@@ -1125,4 +1167,55 @@ mod tests {
"a leading empty page must not affect encoding of the first
nonempty page"
);
}
+ /// Every `log_vector_size` the format permits must round-trip, including
the
+ /// 3 and 15 bounds, and across both the buffered first page and the
streaming path.
+ #[test]
+ fn test_roundtrip_all_vector_sizes() {
+ let values: Vec<f64> = (0..5000).map(|i| (i as f64) * 0.01 -
7.5).collect();
+ for log_vector_size in 3u8..=15 {
+ let out = roundtrip_with_log_vector_size::<DoubleType>(&values,
log_vector_size);
+ assert_bits_eq(&out, &values);
+ }
+ }
+
+ /// A non-default vector size must survive the streaming path, where the
size
+ /// governs the carry across `put` boundaries.
+ #[test]
+ fn test_streaming_with_non_default_vector_size() {
+ let page1: Vec<f64> = (0..3000).map(|i| (i as f64) * 0.01).collect();
+ let page2: Vec<f64> = (0..3000).map(|i| (i as f64) * 0.03 -
7.0).collect();
+ let options = AlpEncoderOptions::try_new(13).unwrap();
+ let vector_size = options.vector_size();
+
+ let mut encoder = AlpEncoder::<DoubleType>::new_with_options(options);
+ encoder.put(&page1).unwrap();
+ let _ = encoder.flush_buffer().unwrap();
+
+ // Feed page 2 in chunks that straddle vector boundaries.
+ for chunk in page2.chunks(997) {
+ encoder.put(chunk).unwrap();
+ }
+ let streamed = encoder.flush_buffer().unwrap();
+
+ // Same values, same preset, encoded in one pass.
+ let preset = build_preset(&page1, vector_size);
+ let mut scratch = Scratch::<f64>::new();
+ let reference = encode_page(&page2, &preset, vector_size, &mut
scratch).unwrap();
+ assert_eq!(streamed.as_ref(), reference.as_slice());
+ }
Review Comment:
With the 997-value `put`s in `test_roundtrip_all_vector_sizes` above, I
think this test can go. It is `test_streaming_matches_buffered` with a
different size, and at 8192 values per vector, 3000 values never fill a vector:
every `put` extends `carry`, and `finish` encodes one partial vector. So the
`chunks_exact` loop and the carry split never run at a non-default size here.
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]