AdamGS commented on code in PR #4389:
URL: https://github.com/apache/arrow-rs/pull/4389#discussion_r1225212489
##########
parquet/src/column/writer/mod.rs:
##########
@@ -2220,12 +2332,201 @@ mod tests {
);
}
+ /// Verify min/max value truncation in the column index works as expected
+ #[test]
+ fn test_column_offset_index_metadata_truncating() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+ let props = Default::default();
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 3];
+ // This is the expected min value - "aaa..."
+ data[0].set_data(ByteBufferPtr::new(vec![97_u8; 200]));
+ // This is the expected max value - "ZZZ..."
+ data[1].set_data(ByteBufferPtr::new(vec![112_u8; 200]));
+ data[2].set_data(ByteBufferPtr::new(vec![98_u8; 200]));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(3, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_ne!(stats.min_bytes(),
column_index_min_value.as_slice());
+ assert_ne!(stats.max_bytes(),
column_index_max_value.as_slice());
+
+ assert_eq!(
+ column_index_min_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+ assert_eq!(column_index_min_value.as_slice(), &[97_u8; 64]);
+ assert_eq!(
+ column_index_max_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+
+ // We expect the last byte to be incremented
+ assert_eq!(
+ *column_index_max_value.last().unwrap(),
+ *column_index_max_value.first().unwrap() + 1
+ );
+ } else {
+ panic!("expecting Statistics::FixedLenByteArray");
+ }
+ } else {
+ panic!("metadata missing statistics");
+ }
+ }
+
+ #[test]
+ fn test_column_offset_index_truncating_spec_example() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+
+ // Truncate values at 1 byte
+ let builder =
+
WriterProperties::builder().set_column_index_truncate_length(Some(1));
+ let props = Arc::new(builder.build());
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 1];
+ // This is the expected min value
+ data[0].set_data(ByteBufferPtr::new(
+ String::from("Blart Versenwald III").into_bytes(),
+ ));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(1, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(_stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ assert_ne!(column_index_min_value, column_index_max_value);
+
+ assert_eq!(column_index_min_value.len(), 1);
+ assert_eq!(column_index_max_value.len(), 1);
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_eq!("B".as_bytes(), column_index_min_value.as_slice());
+ // In this case, they are equal because the max value is
shorter than the default threshold
Review Comment:
Yeah sorry I didn't mean to push these changes, still in the middle of
working stuff out.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
