Rich-T-kid commented on code in PR #10654:
URL: https://github.com/apache/arrow-rs/pull/10654#discussion_r3784564569
##########
parquet/src/arrow/arrow_writer/mod.rs:
##########
@@ -1112,6 +1117,32 @@ impl ArrowColumnWriter {
}
fn write_internal(&mut self, levels: &ArrayLevels) -> Result<()> {
+ if let Some(seen) = &mut self.distinct_values_seen {
+ let array = levels.array();
+ let non_null = levels.non_null_indices();
+ match array.as_any_dictionary_opt() {
+ Some(dict) => {
+ // For dictionary arrays, hash the integer keys rather
than the actual values.
+ // Key cardinality equals value cardinality, so
distinct-value counting stays
+ // correct while avoiding the cost of hashing
arbitrary-length values.
Review Comment:
a similar approach may or may not be worth it for REE's, worth benchmarking
first
##########
parquet/src/arrow/arrow_writer/mod.rs:
##########
@@ -1894,6 +1939,109 @@ fn chunk_contiguous_vec(arena: Vec<u8>, chunk_size:
usize) -> Vec<FixedLenByteAr
values
}
+/// Hash a byte slice to a u64 for NDV tracking.
+#[inline]
+fn hash_bytes(bytes: &[u8]) -> u64 {
+ twox_hash::XxHash64::oneshot(0, bytes)
+}
+
+/// Returns the byte width of an Arrow dictionary key type, or 0 if
unsupported.
+fn arrow_key_byte_width(dt: &ArrowDataType) -> usize {
+ match dt {
+ ArrowDataType::Int8 | ArrowDataType::UInt8 => 1,
+ ArrowDataType::Int16 | ArrowDataType::UInt16 => 2,
+ ArrowDataType::Int32 | ArrowDataType::UInt32 => 4,
+ ArrowDataType::Int64 | ArrowDataType::UInt64 => 8,
+ _ => 0,
+ }
+}
+
+/// Returns the fixed byte width for primitive Arrow types, or `None` for
variable-length types.
+fn fixed_byte_width(dt: &ArrowDataType) -> Option<usize> {
+ use ArrowDataType::*;
+ match dt {
+ Int8 | UInt8 => Some(1),
+ Int16 | UInt16 | Float16 => Some(2),
+ Int32 | UInt32 | Float32 | Date32 | Time32(_) | Decimal32(_, _) =>
Some(4),
+ Int64
+ | UInt64
+ | Float64
+ | Date64
+ | Time64(_)
+ | Timestamp(_, _)
+ | Duration(_)
+ | Decimal64(_, _) => Some(8),
+ Interval(IntervalUnit::YearMonth) => Some(4),
+ Interval(IntervalUnit::DayTime) => Some(8),
+ Interval(IntervalUnit::MonthDayNano) => Some(16),
+ Decimal128(_, _) => Some(16),
+ Decimal256(_, _) => Some(32),
+ _ => None,
+ }
+}
+
+/// Hash the non-null values in `array` (at `non_null_indices`) into `seen`.
+///
+/// Handles primitive, boolean, fixed-size-binary, and variable-length
(Utf8/Binary)
+/// arrays. Unsupported types are silently skipped, leaving `seen` unchanged
for
+/// those values (NDV is best-effort).
+fn update_distinct_values_seen(
+ array: &dyn arrow_array::Array,
+ non_null_indices: &[usize],
+ seen: &mut DistinctValuesSet,
+) {
+ let data = array.to_data();
+ let offset = data.offset();
+
+ match array.data_type() {
+ ArrowDataType::Boolean => {
+ let arr = array
+ .as_any()
+ .downcast_ref::<arrow_array::BooleanArray>()
+ .unwrap();
+ for &row in non_null_indices {
+ seen.insert(arr.value(row) as u64);
+ }
+ }
+ ArrowDataType::Utf8 | ArrowDataType::Binary => {
+ let offsets = data.buffers()[0].typed_data::<i32>();
+ let values = data.buffers()[1].as_slice();
+ for &row in non_null_indices {
+ let start = offsets[offset + row] as usize;
+ let end = offsets[offset + row + 1] as usize;
+ seen.insert(hash_bytes(&values[start..end]));
+ }
+ }
+ ArrowDataType::LargeUtf8 | ArrowDataType::LargeBinary => {
+ let offsets = data.buffers()[0].typed_data::<i64>();
+ let values = data.buffers()[1].as_slice();
+ for &row in non_null_indices {
+ let start = offsets[offset + row] as usize;
+ let end = offsets[offset + row + 1] as usize;
+ seen.insert(hash_bytes(&values[start..end]));
+ }
+ }
+ ArrowDataType::FixedSizeBinary(byte_width) => {
+ let byte_width = *byte_width as usize;
+ let buffer = data.buffers()[0].as_slice();
+ for &row in non_null_indices {
+ let start = (offset + row) * byte_width;
+ seen.insert(hash_bytes(&buffer[start..start + byte_width]));
+ }
+ }
+ data_type => {
+ if let Some(width) = fixed_byte_width(data_type) {
+ let buffer = data.buffers()[0].as_slice();
+ for &row in non_null_indices {
+ let pos = (offset + row) * width;
+ seen.insert(hash_bytes(&buffer[pos..pos + width]));
+ }
+ }
+ // Utf8View, BinaryView, nested types: skip
+ }
+ }
+}
Review Comment:
In a follow up PR it'd be nice to add some benchmarks for this
##########
parquet/src/arrow/arrow_writer/mod.rs:
##########
@@ -1894,6 +1939,109 @@ fn chunk_contiguous_vec(arena: Vec<u8>, chunk_size:
usize) -> Vec<FixedLenByteAr
values
}
+/// Hash a byte slice to a u64 for NDV tracking.
+#[inline]
+fn hash_bytes(bytes: &[u8]) -> u64 {
+ twox_hash::XxHash64::oneshot(0, bytes)
+}
+
+/// Returns the byte width of an Arrow dictionary key type, or 0 if
unsupported.
+fn arrow_key_byte_width(dt: &ArrowDataType) -> usize {
+ match dt {
+ ArrowDataType::Int8 | ArrowDataType::UInt8 => 1,
+ ArrowDataType::Int16 | ArrowDataType::UInt16 => 2,
+ ArrowDataType::Int32 | ArrowDataType::UInt32 => 4,
+ ArrowDataType::Int64 | ArrowDataType::UInt64 => 8,
+ _ => 0,
+ }
+}
+
+/// Returns the fixed byte width for primitive Arrow types, or `None` for
variable-length types.
+fn fixed_byte_width(dt: &ArrowDataType) -> Option<usize> {
+ use ArrowDataType::*;
+ match dt {
+ Int8 | UInt8 => Some(1),
+ Int16 | UInt16 | Float16 => Some(2),
+ Int32 | UInt32 | Float32 | Date32 | Time32(_) | Decimal32(_, _) =>
Some(4),
+ Int64
+ | UInt64
+ | Float64
+ | Date64
+ | Time64(_)
+ | Timestamp(_, _)
+ | Duration(_)
+ | Decimal64(_, _) => Some(8),
+ Interval(IntervalUnit::YearMonth) => Some(4),
+ Interval(IntervalUnit::DayTime) => Some(8),
+ Interval(IntervalUnit::MonthDayNano) => Some(16),
+ Decimal128(_, _) => Some(16),
+ Decimal256(_, _) => Some(32),
+ _ => None,
+ }
+}
+
+/// Hash the non-null values in `array` (at `non_null_indices`) into `seen`.
+///
+/// Handles primitive, boolean, fixed-size-binary, and variable-length
(Utf8/Binary)
+/// arrays. Unsupported types are silently skipped, leaving `seen` unchanged
for
+/// those values (NDV is best-effort).
+fn update_distinct_values_seen(
+ array: &dyn arrow_array::Array,
+ non_null_indices: &[usize],
+ seen: &mut DistinctValuesSet,
+) {
+ let data = array.to_data();
+ let offset = data.offset();
+
+ match array.data_type() {
+ ArrowDataType::Boolean => {
+ let arr = array
+ .as_any()
+ .downcast_ref::<arrow_array::BooleanArray>()
+ .unwrap();
+ for &row in non_null_indices {
+ seen.insert(arr.value(row) as u64);
+ }
+ }
+ ArrowDataType::Utf8 | ArrowDataType::Binary => {
+ let offsets = data.buffers()[0].typed_data::<i32>();
+ let values = data.buffers()[1].as_slice();
+ for &row in non_null_indices {
+ let start = offsets[offset + row] as usize;
+ let end = offsets[offset + row + 1] as usize;
+ seen.insert(hash_bytes(&values[start..end]));
+ }
+ }
+ ArrowDataType::LargeUtf8 | ArrowDataType::LargeBinary => {
+ let offsets = data.buffers()[0].typed_data::<i64>();
+ let values = data.buffers()[1].as_slice();
+ for &row in non_null_indices {
+ let start = offsets[offset + row] as usize;
+ let end = offsets[offset + row + 1] as usize;
+ seen.insert(hash_bytes(&values[start..end]));
+ }
+ }
+ ArrowDataType::FixedSizeBinary(byte_width) => {
+ let byte_width = *byte_width as usize;
+ let buffer = data.buffers()[0].as_slice();
+ for &row in non_null_indices {
+ let start = (offset + row) * byte_width;
+ seen.insert(hash_bytes(&buffer[start..start + byte_width]));
+ }
+ }
+ data_type => {
+ if let Some(width) = fixed_byte_width(data_type) {
+ let buffer = data.buffers()[0].as_slice();
+ for &row in non_null_indices {
+ let pos = (offset + row) * width;
+ seen.insert(hash_bytes(&buffer[pos..pos + width]));
+ }
+ }
+ // Utf8View, BinaryView, nested types: skip
Review Comment:
Can also make a follow up issue to support the rest of the types
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
