yordan-pavlov commented on a change in pull request #1054:
URL: https://github.com/apache/arrow-rs/pull/1054#discussion_r785354757
##########
File path: parquet/src/arrow/record_reader/definition_levels.rs
##########
@@ -91,10 +131,335 @@ impl DefinitionLevelBuffer {
&self,
range: Range<usize>,
) -> impl Iterator<Item = usize> + '_ {
- let max_def_level = self.max_level;
- let slice = self.buffer.as_slice();
- range.rev().filter(move |x| slice[*x] == max_def_level)
+ assert_eq!(range.start, self.len);
+ iter_set_bits_rev(self.nulls().as_slice())
+ }
+
+ fn nulls(&self) -> &BooleanBufferBuilder {
+ match &self.inner {
+ BufferInner::Full { nulls, .. } => nulls,
+ BufferInner::Mask { nulls } => nulls,
+ }
+ }
+}
+
+impl LevelsBufferSlice for DefinitionLevelBuffer {
+ fn capacity(&self) -> usize {
+ usize::MAX
+ }
+
+ fn count_nulls(&self, range: Range<usize>, _max_level: i16) -> usize {
+ let total_count = range.end - range.start;
+ let range = range.start + self.len..range.end + self.len;
+ total_count - count_set_bits(self.nulls().as_slice(), range)
+ }
+}
+
+pub struct DefinitionLevelDecoder {
+ max_level: i16,
+ encoding: Encoding,
+ data: Option<ByteBufferPtr>,
+ column_decoder: Option<ColumnLevelDecoderImpl>,
+ packed_decoder: Option<PackedDecoder>,
+}
+
+impl ColumnLevelDecoder for DefinitionLevelDecoder {
+ type Slice = DefinitionLevelBuffer;
+
+ fn new(max_level: i16, encoding: Encoding, data: ByteBufferPtr) -> Self {
+ Self {
+ max_level,
+ encoding,
+ data: Some(data),
+ column_decoder: None,
+ packed_decoder: None,
+ }
+ }
+
+ fn read(
+ &mut self,
+ writer: &mut Self::Slice,
+ range: Range<usize>,
+ ) -> crate::errors::Result<usize> {
+ match &mut writer.inner {
+ BufferInner::Full {
+ levels,
+ nulls,
+ max_level,
+ } => {
+ assert_eq!(self.max_level, *max_level);
+ assert_eq!(range.start + writer.len, nulls.len());
+
+ let decoder = match self.data.take() {
+ Some(data) => self.column_decoder.insert(
Review comment:
it looks like the intention is that self.data will only be used once (to
create a ColumnLevelDecoderImpl) and if that's the case, why not move the
entire match statement in the constructor?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
