zeroshade commented on code in PR #38367:
URL: https://github.com/apache/arrow/pull/38367#discussion_r1369238377
##########
go/parquet/internal/encoding/boolean_decoder.go:
##########
@@ -109,3 +114,76 @@ func (dec *PlainBooleanDecoder) DecodeSpaced(out []bool,
nullCount int, validBit
}
return dec.Decode(out)
}
+
+type RleBooleanDecoder struct {
+ decoder
+
+ rleDec *utils.RleDecoder
+}
+
+func (RleBooleanDecoder) Type() parquet.Type {
+ return parquet.Types.Boolean
+}
+
+func (dec *RleBooleanDecoder) SetData(nvals int, data []byte) error {
+ dec.nvals = nvals
+
+ if len(data) < 4 {
+ return fmt.Errorf("invalid length - %d (corrupt data page?)",
len(data))
+ }
+
+ // load the first 4 bytes in little-endian which indicates the length
+ nbytes := binary.LittleEndian.Uint32(data[:4])
+ if nbytes > uint32(len(data)-4) {
+ return fmt.Errorf("received invalid number of bytes - %d
(corrupt data page?)", nbytes)
+ }
+
+ dec.data = data[4:]
+ if dec.rleDec == nil {
+ dec.rleDec = utils.NewRleDecoder(bytes.NewReader(dec.data), 1)
+ } else {
+ dec.rleDec.Reset(bytes.NewReader(dec.data), 1)
+ }
+ return nil
+}
+
+func (dec *RleBooleanDecoder) Decode(out []bool) (int, error) {
+ max := shared_utils.MinInt(len(out), dec.nvals)
+
+ var (
+ buf [1024]uint64
+ n = max
+ )
+
+ for n > 0 {
+ batch := shared_utils.MinInt(len(buf), n)
+ decoded := dec.rleDec.GetBatch(buf[:batch])
+ if decoded != batch {
+ return max - n, io.ErrUnexpectedEOF
+ }
+
+ for i := 0; i < batch; i++ {
+ out[i] = buf[i] != 0
+ }
+ n -= batch
+ out = out[batch:]
+ }
+
+ dec.nvals -= max
+ return max, nil
+}
+
+func (dec *RleBooleanDecoder) DecodeSpaced(out []bool, nullCount int,
validBits []byte, validBitsOffset int64) (int, error) {
+ if nullCount > 0 {
+ toRead := len(out) - nullCount
+ valuesRead, err := dec.Decode(out[:toRead])
+ if err != nil {
+ return 0, err
+ }
+ if valuesRead != toRead {
+ return valuesRead, xerrors.New("parquet: rle boolean
decoder: number of values / definition levels read did not match")
Review Comment:
at the time I originally wrote the parquet code, `xerrors` was used for
particular benefits of wrapping and otherwise that weren't available in the
stdlib. Since then, all the features of `xerrors` have been folded into the go
stdlib and there really isn't a reason to use it anymore, I intend to phase it
out as I make changes to the code. So I'm going to fix this to just use
`errors`, it was my mistake to propagate `xerrors` here so thanks for catching
it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
