Dandandan commented on code in PR #7614:
URL: https://github.com/apache/arrow-rs/pull/7614#discussion_r2132974750
##########
arrow-array/src/builder/generic_bytes_view_builder.rs:
##########
@@ -201,10 +201,40 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
let b = b.get_unchecked(start..end);
let view = make_view(b, block, offset);
- self.views_builder.append(view);
+ self.views_buffer.push(view);
self.null_buffer_builder.append_non_null();
}
+ /// Appends an array to the builder.
+ /// This will flush any in-progress block and append the data buffers
+ /// and add the (adapted) views.
+ pub fn append_array(&mut self, array: &GenericByteViewArray<T>) {
+ self.flush_in_progress();
+ self.completed.extend(array.data_buffers().iter().cloned());
+
+ if self.completed.is_empty() {
Review Comment:
For me this is a all-positive change 🎉 I really like the 2x improvement on
inlined views.
```
concat utf8_view all_inline max_str_len=12 null_density=0
time: [25.775 µs 27.046 µs 28.546 µs]
change: [−55.148% −53.600% −51.725%] (p = 0.00 <
0.05)
Performance has improved.
Found 23 outliers among 100 measurements (23.00%)
7 (7.00%) low mild
5 (5.00%) high mild
11 (11.00%) high severe
concat utf8_view max_str_len=20 null_density=0
time: [53.475 µs 53.934 µs 54.689 µs]
change: [−7.1979% −6.4937% −5.5943%] (p = 0.00 <
0.05)
Performance has improved.
Found 7 outliers among 100 measurements (7.00%)
1 (1.00%) low mild
3 (3.00%) high mild
3 (3.00%) high severe
concat utf8_view max_str_len=128 null_density=0
time: [52.999 µs 53.106 µs 53.245 µs]
change: [−7.0948% −6.6147% −6.0319%] (p = 0.00 <
0.05)
Performance has improved.
Found 15 outliers among 100 measurements (15.00%)
7 (7.00%) high mild
8 (8.00%) high severe
concat utf8_view all_inline max_str_len=12 null_density=0.2
time: [25.532 µs 25.741 µs 25.943 µs]
change: [−56.519% −56.108% −55.710%] (p = 0.00 <
0.05)
Performance has improved.
Found 11 outliers among 100 measurements (11.00%)
8 (8.00%) low mild
3 (3.00%) high mild
concat utf8_view max_str_len=20 null_density=0.2
time: [56.062 µs 56.342 µs 56.688 µs]
change: [−6.1050% −5.3513% −4.6591%] (p = 0.00 <
0.05)
Performance has improved.
Found 5 outliers among 100 measurements (5.00%)
1 (1.00%) low mild
4 (4.00%) high severe
concat utf8_view max_str_len=128 null_density=0.2
time: [55.406 µs 55.623 µs 55.891 µs]
change: [−6.0536% −5.4045% −4.8059%] (p = 0.00 <
0.05)
Performance has improved.
Found 14 outliers among 100 measurements (14.00%)
1 (1.00%) low mild
1 (1.00%) high mild
12 (12.00%) high severe
```
##########
arrow-array/src/builder/generic_bytes_view_builder.rs:
##########
@@ -201,10 +201,40 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
let b = b.get_unchecked(start..end);
let view = make_view(b, block, offset);
- self.views_builder.append(view);
+ self.views_buffer.push(view);
self.null_buffer_builder.append_non_null();
}
+ /// Appends an array to the builder.
+ /// This will flush any in-progress block and append the data buffers
+ /// and add the (adapted) views.
+ pub fn append_array(&mut self, array: &GenericByteViewArray<T>) {
+ self.flush_in_progress();
+ self.completed.extend(array.data_buffers().iter().cloned());
+
+ if self.completed.is_empty() {
Review Comment:
For me this is a all-positive change 🎉 I really like the >2x improvement
on inlined views.
```
concat utf8_view all_inline max_str_len=12 null_density=0
time: [25.775 µs 27.046 µs 28.546 µs]
change: [−55.148% −53.600% −51.725%] (p = 0.00 <
0.05)
Performance has improved.
Found 23 outliers among 100 measurements (23.00%)
7 (7.00%) low mild
5 (5.00%) high mild
11 (11.00%) high severe
concat utf8_view max_str_len=20 null_density=0
time: [53.475 µs 53.934 µs 54.689 µs]
change: [−7.1979% −6.4937% −5.5943%] (p = 0.00 <
0.05)
Performance has improved.
Found 7 outliers among 100 measurements (7.00%)
1 (1.00%) low mild
3 (3.00%) high mild
3 (3.00%) high severe
concat utf8_view max_str_len=128 null_density=0
time: [52.999 µs 53.106 µs 53.245 µs]
change: [−7.0948% −6.6147% −6.0319%] (p = 0.00 <
0.05)
Performance has improved.
Found 15 outliers among 100 measurements (15.00%)
7 (7.00%) high mild
8 (8.00%) high severe
concat utf8_view all_inline max_str_len=12 null_density=0.2
time: [25.532 µs 25.741 µs 25.943 µs]
change: [−56.519% −56.108% −55.710%] (p = 0.00 <
0.05)
Performance has improved.
Found 11 outliers among 100 measurements (11.00%)
8 (8.00%) low mild
3 (3.00%) high mild
concat utf8_view max_str_len=20 null_density=0.2
time: [56.062 µs 56.342 µs 56.688 µs]
change: [−6.1050% −5.3513% −4.6591%] (p = 0.00 <
0.05)
Performance has improved.
Found 5 outliers among 100 measurements (5.00%)
1 (1.00%) low mild
4 (4.00%) high severe
concat utf8_view max_str_len=128 null_density=0.2
time: [55.406 µs 55.623 µs 55.891 µs]
change: [−6.0536% −5.4045% −4.8059%] (p = 0.00 <
0.05)
Performance has improved.
Found 14 outliers among 100 measurements (14.00%)
1 (1.00%) low mild
1 (1.00%) high mild
12 (12.00%) high severe
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
