HippoBaro commented on code in PR #9831:
URL: https://github.com/apache/arrow-rs/pull/9831#discussion_r3198174667
##########
parquet/src/arrow/arrow_writer/levels.rs:
##########
@@ -754,17 +734,129 @@ impl LevelInfoBuilder {
/// The data necessary to write a primitive Arrow array to parquet, taking
into account
/// any non-primitive parents it may have in the arrow representation
+#[derive(Debug, Clone)]
+pub(crate) enum LevelData {
+ Absent,
+ Materialized(Vec<i16>),
+ Uniform { value: i16, count: usize },
+}
+
+impl PartialEq for LevelData {
+ fn eq(&self, other: &Self) -> bool {
+ match (self, other) {
+ (Self::Absent, Self::Absent) => true,
+ (Self::Materialized(a), Self::Materialized(b)) => a == b,
+ (Self::Uniform { value: v, count: n }, Self::Materialized(b))
+ | (Self::Materialized(b), Self::Uniform { value: v, count: n }) =>
{
+ b.len() == *n && b.iter().all(|x| x == v)
+ }
+ (
+ Self::Uniform {
+ value: v1,
+ count: n1,
+ },
+ Self::Uniform {
+ value: v2,
+ count: n2,
+ },
+ ) => v1 == v2 && n1 == n2,
+ _ => false,
+ }
+ }
+}
+
+impl Eq for LevelData {}
+
+impl LevelData {
+ fn new(present: bool) -> Self {
+ match present {
+ true => Self::Materialized(Vec::new()),
+ false => Self::Absent,
+ }
+ }
+
+ pub(crate) fn as_ref(&self) -> LevelDataRef<'_> {
+ match self {
+ Self::Absent => LevelDataRef::Absent,
+ Self::Materialized(values) => LevelDataRef::Materialized(values),
+ Self::Uniform { value, count } => LevelDataRef::Uniform {
+ value: *value,
+ count: *count,
+ },
+ }
+ }
+
+ pub(crate) fn slice(&self, offset: usize, len: usize) -> Self {
+ match self {
+ Self::Absent => Self::Absent,
+ Self::Materialized(values) =>
Self::Materialized(values[offset..offset + len].to_vec()),
Review Comment:
Yeah, I think it is possible if we switch CDC chunking to borrow chunk views
using `LevelDataRef::slice`. That would avoid copying materialized level
slices, but it is a larger change because `slice_for_chunk` also owns and
rewrites `non_null_indices`.
I am not a user of CDC right now, so I wouldn't have anything real to
benchmark this against. I might come back to it in the future.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
