AdamGS commented on code in PR #4389:
URL: https://github.com/apache/arrow-rs/pull/4389#discussion_r1225328334
##########
parquet/src/column/writer/mod.rs:
##########
@@ -2220,12 +2332,201 @@ mod tests {
);
}
+ /// Verify min/max value truncation in the column index works as expected
+ #[test]
+ fn test_column_offset_index_metadata_truncating() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+ let props = Default::default();
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 3];
+ // This is the expected min value - "aaa..."
+ data[0].set_data(ByteBufferPtr::new(vec![97_u8; 200]));
+ // This is the expected max value - "ZZZ..."
+ data[1].set_data(ByteBufferPtr::new(vec![112_u8; 200]));
+ data[2].set_data(ByteBufferPtr::new(vec![98_u8; 200]));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(3, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_ne!(stats.min_bytes(),
column_index_min_value.as_slice());
+ assert_ne!(stats.max_bytes(),
column_index_max_value.as_slice());
+
+ assert_eq!(
+ column_index_min_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+ assert_eq!(column_index_min_value.as_slice(), &[97_u8; 64]);
+ assert_eq!(
+ column_index_max_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+
+ // We expect the last byte to be incremented
+ assert_eq!(
+ *column_index_max_value.last().unwrap(),
+ *column_index_max_value.first().unwrap() + 1
+ );
+ } else {
+ panic!("expecting Statistics::FixedLenByteArray");
+ }
+ } else {
+ panic!("metadata missing statistics");
+ }
+ }
+
+ #[test]
+ fn test_column_offset_index_truncating_spec_example() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+
+ // Truncate values at 1 byte
+ let builder =
+
WriterProperties::builder().set_column_index_truncate_length(Some(1));
+ let props = Arc::new(builder.build());
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 1];
+ // This is the expected min value
+ data[0].set_data(ByteBufferPtr::new(
+ String::from("Blart Versenwald III").into_bytes(),
+ ));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(1, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(_stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ assert_ne!(column_index_min_value, column_index_max_value);
+
+ assert_eq!(column_index_min_value.len(), 1);
+ assert_eq!(column_index_max_value.len(), 1);
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_eq!("B".as_bytes(), column_index_min_value.as_slice());
+ // In this case, they are equal because the max value is
shorter than the default threshold
+ assert_eq!("C".as_bytes(), column_index_max_value.as_slice());
+ } else {
+ panic!("expecting Statistics::FixedLenByteArray");
+ }
+ } else {
+ panic!("metadata missing statistics");
+ }
+ }
+
#[test]
fn test_send() {
fn test<T: Send>() {}
test::<ColumnWriterImpl<Int32Type>>();
}
+ #[test]
+ fn test_increment() {
+ let mut v = vec![0, 0, 0];
+ increment(&mut v);
+ assert_eq!(&v, &[0, 0, 1]);
+
+ // Handle overflow
+ let mut v = vec![0, 255, 255];
+ increment(&mut v);
+ assert_eq!(&v, &[1, 255, 255]);
+
+ // No-op if all bytes are u8::MAX
+ let mut v = vec![255, 255, 255];
+ increment(&mut v);
+ assert_eq!(&v, &[255, 255, 255]);
Review Comment:
Sorry, not sure I understand the question here.
Seems like parquet-mr doesn't truncate slices like `&[255, 255, 255]`, will
push a version that behaves the same soon.
##########
parquet/src/column/writer/mod.rs:
##########
@@ -2220,12 +2332,201 @@ mod tests {
);
}
+ /// Verify min/max value truncation in the column index works as expected
+ #[test]
+ fn test_column_offset_index_metadata_truncating() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+ let props = Default::default();
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 3];
+ // This is the expected min value - "aaa..."
+ data[0].set_data(ByteBufferPtr::new(vec![97_u8; 200]));
+ // This is the expected max value - "ZZZ..."
+ data[1].set_data(ByteBufferPtr::new(vec![112_u8; 200]));
+ data[2].set_data(ByteBufferPtr::new(vec![98_u8; 200]));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(3, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_ne!(stats.min_bytes(),
column_index_min_value.as_slice());
+ assert_ne!(stats.max_bytes(),
column_index_max_value.as_slice());
+
+ assert_eq!(
+ column_index_min_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+ assert_eq!(column_index_min_value.as_slice(), &[97_u8; 64]);
+ assert_eq!(
+ column_index_max_value.len(),
+ DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
+ );
+
+ // We expect the last byte to be incremented
+ assert_eq!(
+ *column_index_max_value.last().unwrap(),
+ *column_index_max_value.first().unwrap() + 1
+ );
+ } else {
+ panic!("expecting Statistics::FixedLenByteArray");
+ }
+ } else {
+ panic!("metadata missing statistics");
+ }
+ }
+
+ #[test]
+ fn test_column_offset_index_truncating_spec_example() {
+ // write data
+ // and check the offset index and column index
+ let page_writer = get_test_page_writer();
+
+ // Truncate values at 1 byte
+ let builder =
+
WriterProperties::builder().set_column_index_truncate_length(Some(1));
+ let props = Arc::new(builder.build());
+ let mut writer =
+ get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0,
props);
+
+ let mut data = vec![FixedLenByteArray::default(); 1];
+ // This is the expected min value
+ data[0].set_data(ByteBufferPtr::new(
+ String::from("Blart Versenwald III").into_bytes(),
+ ));
+
+ writer.write_batch(&data, None, None).unwrap();
+
+ writer.flush_data_pages().unwrap();
+
+ let r = writer.close().unwrap();
+ let column_index = r.column_index.unwrap();
+ let offset_index = r.offset_index.unwrap();
+
+ assert_eq!(1, r.rows_written);
+
+ // column index
+ assert_eq!(1, column_index.null_pages.len());
+ assert_eq!(1, offset_index.page_locations.len());
+ assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
+ assert!(!column_index.null_pages[0]);
+ assert_eq!(0, column_index.null_counts.as_ref().unwrap()[0]);
+
+ if let Some(stats) = r.metadata.statistics() {
+ assert!(stats.has_min_max_set());
+ assert_eq!(stats.null_count(), 0);
+ assert_eq!(stats.distinct_count(), None);
+ if let Statistics::FixedLenByteArray(_stats) = stats {
+ let column_index_min_value =
column_index.min_values.get(0).unwrap();
+ let column_index_max_value =
column_index.max_values.get(0).unwrap();
+
+ assert_ne!(column_index_min_value, column_index_max_value);
+
+ assert_eq!(column_index_min_value.len(), 1);
+ assert_eq!(column_index_max_value.len(), 1);
+
+ // Column index stats are truncated, while the column chunk's
aren't.
+ assert_eq!("B".as_bytes(), column_index_min_value.as_slice());
+ // In this case, they are equal because the max value is
shorter than the default threshold
+ assert_eq!("C".as_bytes(), column_index_max_value.as_slice());
+ } else {
+ panic!("expecting Statistics::FixedLenByteArray");
+ }
+ } else {
+ panic!("metadata missing statistics");
+ }
+ }
+
#[test]
fn test_send() {
fn test<T: Send>() {}
test::<ColumnWriterImpl<Int32Type>>();
}
+ #[test]
+ fn test_increment() {
+ let mut v = vec![0, 0, 0];
+ increment(&mut v);
+ assert_eq!(&v, &[0, 0, 1]);
+
+ // Handle overflow
+ let mut v = vec![0, 255, 255];
+ increment(&mut v);
+ assert_eq!(&v, &[1, 255, 255]);
+
+ // No-op if all bytes are u8::MAX
+ let mut v = vec![255, 255, 255];
+ increment(&mut v);
+ assert_eq!(&v, &[255, 255, 255]);
Review Comment:
Sorry, not sure I understand the question here, but it seems like parquet-mr
doesn't truncate slices like `&[255, 255, 255]`, will push a version that
behaves the same soon.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
