Jefffrey commented on code in PR #10812:
URL: https://github.com/apache/arrow-rs/pull/10812#discussion_r3871471009
##########
arrow-select/src/take.rs:
##########
@@ -652,76 +651,142 @@ where
OffsetType::Native: OffsetSizeTrait,
PrimitiveArray<OffsetType>: From<Vec<OffsetType::Native>>,
{
- let list_offsets = values.value_offsets();
+ let src_offsets = values.value_offsets();
let child_data = values.values().to_data();
let nulls = take_nulls(values.nulls(), indices);
- let mut new_offsets = Vec::with_capacity(indices.len() + 1);
- new_offsets.push(OffsetType::Native::zero());
+ let mut dst_offsets = Vec::with_capacity(indices.len() + 1);
+ dst_offsets.push(OffsetType::Native::zero());
- let use_nulls = child_data.null_count() > 0;
+ let field = values.value_field().clone();
+
+ let is_primitive_child = child_data.null_count() == 0 &&
child_data.data_type().is_primitive();
+
+ if is_primitive_child {
+ let values_buf = &child_data.buffers()[0];
+ let bytes_per_value = if !child_data.is_empty() {
Review Comment:
is this the same as
[`DataType::primitive_width()`](https://docs.rs/arrow/latest/arrow/datatypes/enum.DataType.html#method.primitive_width)
##########
arrow-select/src/take.rs:
##########
@@ -652,76 +651,142 @@ where
OffsetType::Native: OffsetSizeTrait,
PrimitiveArray<OffsetType>: From<Vec<OffsetType::Native>>,
{
- let list_offsets = values.value_offsets();
+ let src_offsets = values.value_offsets();
let child_data = values.values().to_data();
let nulls = take_nulls(values.nulls(), indices);
- let mut new_offsets = Vec::with_capacity(indices.len() + 1);
- new_offsets.push(OffsetType::Native::zero());
+ let mut dst_offsets = Vec::with_capacity(indices.len() + 1);
+ dst_offsets.push(OffsetType::Native::zero());
- let use_nulls = child_data.null_count() > 0;
+ let field = values.value_field().clone();
+
+ let is_primitive_child = child_data.null_count() == 0 &&
child_data.data_type().is_primitive();
+
+ if is_primitive_child {
+ let values_buf = &child_data.buffers()[0];
+ let bytes_per_value = if !child_data.is_empty() {
+ values_buf.len() / child_data.len()
+ } else {
+ 0
+ };
+ let child_buf_offset = child_data.offset() * bytes_per_value;
+
+ let avg_row_len = child_data
+ .len()
+ .checked_div(values.len().max(1))
+ .unwrap_or(0);
+ let mut dst_buf = MutableBuffer::new(
+ avg_row_len
+ .saturating_mul(indices.len())
+ .saturating_mul(bytes_per_value),
+ );
+
+ let mut child_len = OffsetType::Native::zero();
+
+ match nulls.as_ref().filter(|n| n.null_count() > 0) {
+ None => {
+ for &idx in indices.values() {
+ let row = idx.as_usize();
+ let start = child_buf_offset + src_offsets[row].as_usize()
* bytes_per_value;
+ let end = child_buf_offset + src_offsets[row +
1].as_usize() * bytes_per_value;
+ dst_buf.extend_from_slice(&values_buf[start..end]);
+ child_len += src_offsets[row + 1] - src_offsets[row];
+ dst_offsets.push(child_len);
+ }
+ }
+ Some(valid) => {
+ let mut prev = 0;
+ for vidx in valid.valid_indices() {
+ // Fill offsets for null values between the two valid
indices.
+ if prev < vidx {
+ dst_offsets.extend(std::iter::repeat_n(child_len, vidx
- prev));
+ }
+ let row = indices.value(vidx).as_usize();
+ let start = child_buf_offset + src_offsets[row].as_usize()
* bytes_per_value;
+ let end = child_buf_offset + src_offsets[row +
1].as_usize() * bytes_per_value;
+ dst_buf.extend_from_slice(&values_buf[start..end]);
+ child_len += src_offsets[row + 1] - src_offsets[row];
+ dst_offsets.push(child_len);
+ prev = vidx + 1;
+ }
+ dst_offsets.extend(std::iter::repeat_n(child_len,
indices.len() - prev));
+ }
+ }
+
+ assert_eq!(
Review Comment:
this should probably be a debug assert or an error, to avoid adding a new
panic point
##########
arrow-select/src/take.rs:
##########
@@ -2088,6 +2153,26 @@ mod tests {
assert_eq!(&output, &expected)
}
+ #[test]
+ // Fast path (primitive child, no child nulls) with null indices —
verifies offset backfill for null slots.
+ fn test_take_list_primitive_child_null_indices() {
+ let list = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3), Some(4)]),
+ Some(vec![Some(5), Some(6)]),
+ ]);
Review Comment:
maybe vary the list sizes instead of having them be uniform
##########
arrow-select/src/take.rs:
##########
@@ -652,76 +651,142 @@ where
OffsetType::Native: OffsetSizeTrait,
PrimitiveArray<OffsetType>: From<Vec<OffsetType::Native>>,
{
- let list_offsets = values.value_offsets();
+ let src_offsets = values.value_offsets();
let child_data = values.values().to_data();
let nulls = take_nulls(values.nulls(), indices);
- let mut new_offsets = Vec::with_capacity(indices.len() + 1);
- new_offsets.push(OffsetType::Native::zero());
+ let mut dst_offsets = Vec::with_capacity(indices.len() + 1);
+ dst_offsets.push(OffsetType::Native::zero());
- let use_nulls = child_data.null_count() > 0;
+ let field = values.value_field().clone();
+
+ let is_primitive_child = child_data.null_count() == 0 &&
child_data.data_type().is_primitive();
+
+ if is_primitive_child {
+ let values_buf = &child_data.buffers()[0];
+ let bytes_per_value = if !child_data.is_empty() {
+ values_buf.len() / child_data.len()
+ } else {
+ 0
+ };
+ let child_buf_offset = child_data.offset() * bytes_per_value;
+
+ let avg_row_len = child_data
+ .len()
+ .checked_div(values.len().max(1))
+ .unwrap_or(0);
+ let mut dst_buf = MutableBuffer::new(
+ avg_row_len
+ .saturating_mul(indices.len())
+ .saturating_mul(bytes_per_value),
+ );
+
+ let mut child_len = OffsetType::Native::zero();
+
+ match nulls.as_ref().filter(|n| n.null_count() > 0) {
+ None => {
+ for &idx in indices.values() {
+ let row = idx.as_usize();
+ let start = child_buf_offset + src_offsets[row].as_usize()
* bytes_per_value;
+ let end = child_buf_offset + src_offsets[row +
1].as_usize() * bytes_per_value;
+ dst_buf.extend_from_slice(&values_buf[start..end]);
+ child_len += src_offsets[row + 1] - src_offsets[row];
Review Comment:
codex pointed out `child_len` here could overflow
##########
arrow-select/src/take.rs:
##########
@@ -652,76 +651,142 @@ where
OffsetType::Native: OffsetSizeTrait,
PrimitiveArray<OffsetType>: From<Vec<OffsetType::Native>>,
{
- let list_offsets = values.value_offsets();
+ let src_offsets = values.value_offsets();
let child_data = values.values().to_data();
let nulls = take_nulls(values.nulls(), indices);
- let mut new_offsets = Vec::with_capacity(indices.len() + 1);
- new_offsets.push(OffsetType::Native::zero());
+ let mut dst_offsets = Vec::with_capacity(indices.len() + 1);
+ dst_offsets.push(OffsetType::Native::zero());
- let use_nulls = child_data.null_count() > 0;
+ let field = values.value_field().clone();
+
+ let is_primitive_child = child_data.null_count() == 0 &&
child_data.data_type().is_primitive();
+
+ if is_primitive_child {
+ let values_buf = &child_data.buffers()[0];
+ let bytes_per_value = if !child_data.is_empty() {
+ values_buf.len() / child_data.len()
+ } else {
+ 0
+ };
+ let child_buf_offset = child_data.offset() * bytes_per_value;
+
+ let avg_row_len = child_data
+ .len()
+ .checked_div(values.len().max(1))
+ .unwrap_or(0);
+ let mut dst_buf = MutableBuffer::new(
+ avg_row_len
+ .saturating_mul(indices.len())
+ .saturating_mul(bytes_per_value),
+ );
+
+ let mut child_len = OffsetType::Native::zero();
+
+ match nulls.as_ref().filter(|n| n.null_count() > 0) {
+ None => {
+ for &idx in indices.values() {
+ let row = idx.as_usize();
+ let start = child_buf_offset + src_offsets[row].as_usize()
* bytes_per_value;
+ let end = child_buf_offset + src_offsets[row +
1].as_usize() * bytes_per_value;
+ dst_buf.extend_from_slice(&values_buf[start..end]);
+ child_len += src_offsets[row + 1] - src_offsets[row];
+ dst_offsets.push(child_len);
+ }
+ }
+ Some(valid) => {
+ let mut prev = 0;
+ for vidx in valid.valid_indices() {
+ // Fill offsets for null values between the two valid
indices.
+ if prev < vidx {
+ dst_offsets.extend(std::iter::repeat_n(child_len, vidx
- prev));
+ }
+ let row = indices.value(vidx).as_usize();
+ let start = child_buf_offset + src_offsets[row].as_usize()
* bytes_per_value;
+ let end = child_buf_offset + src_offsets[row +
1].as_usize() * bytes_per_value;
+ dst_buf.extend_from_slice(&values_buf[start..end]);
+ child_len += src_offsets[row + 1] - src_offsets[row];
Review Comment:
same here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
