mapleFU commented on code in PR #34526:
URL: https://github.com/apache/arrow/pull/34526#discussion_r1138944982
##########
cpp/src/parquet/encoding.cc:
##########
@@ -2838,6 +2839,109 @@ class DeltaLengthByteArrayDecoder : public DecoderImpl,
std::shared_ptr<ResizableBuffer> buffered_data_;
};
+// ----------------------------------------------------------------------
+// RLE_BOOLEAN_ENCODER
+
+class RleBooleanEncoder final : public EncoderImpl, virtual public
BooleanEncoder {
+ public:
+ explicit RleBooleanEncoder(const ColumnDescriptor* descr,
::arrow::MemoryPool* pool)
+ : EncoderImpl(descr, Encoding::RLE, pool) {}
+
+ int64_t EstimatedDataEncodedSize() override {
+ return kRleLengthInBytes + MaxRleBufferSize();
+ }
+
+ std::shared_ptr<Buffer> FlushValues() override;
+
+ void Put(const T* buffer, int num_values) override;
+ void Put(const ::arrow::Array& values) override {
+ if (values.type_id() != ::arrow::Type::BOOL) {
+ throw ParquetException("RleBooleanEncoder expects BooleanArray, got ",
+ values.type()->ToString());
+ }
+ const auto& boolean_array = checked_cast<const
::arrow::BooleanArray&>(values);
+ if (values.null_count() == 0) {
+ for (int i = 0; i < boolean_array.length(); ++i) {
+ // null_count == 0, so just call Value directly is ok.
+ buffered_append_values_.push_back(boolean_array.Value(i));
+ }
+ } else {
+ PARQUET_THROW_NOT_OK(::arrow::VisitArraySpanInline<::arrow::BooleanType>(
+ *boolean_array.data(),
+ [&](bool value) {
+ buffered_append_values_.push_back(value);
+ return Status::OK();
+ },
+ []() { return Status::OK(); }));
+ }
+ }
+
+ void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits,
+ int64_t valid_bits_offset) override {
+ if (valid_bits != NULLPTR) {
+ PARQUET_ASSIGN_OR_THROW(auto buffer, ::arrow::AllocateBuffer(num_values
* sizeof(T),
+
this->memory_pool()));
+ T* data = reinterpret_cast<T*>(buffer->mutable_data());
+ int num_valid_values = ::arrow::util::internal::SpacedCompress<T>(
+ src, num_values, valid_bits, valid_bits_offset, data);
+ Put(data, num_valid_values);
+ } else {
+ Put(src, num_values);
+ }
+ }
+
+ void Put(const std::vector<bool>& src, int num_values) override;
+
+ protected:
+ template <typename SequenceType>
+ void PutImpl(const SequenceType& src, int num_values);
+
+ int MaxRleBufferSize() const noexcept {
+ return
RlePreserveBufferSize(static_cast<int>(buffered_append_values_.size()),
+ kBitWidth);
+ }
+
+ constexpr static int32_t kBitWidth = 1;
+ /// 4 bytes in little-endian, which indicates the length.
+ constexpr static int32_t kRleLengthInBytes = 4;
+
+ // std::vector<bool> in C++ is tricky, because it's a bitmap.
+ // Here RleBooleanEncoder will only append values into it, and
+ // dump values into Buffer, so using it here is ok.
+ std::vector<bool> buffered_append_values_;
Review Comment:
Nice catch, I think it won't be too large, because it's just a bitmap. But
`ArrowPoolVector<bool>` seems much better?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
