pitrou commented on code in PR #45622:
URL: https://github.com/apache/arrow/pull/45622#discussion_r1983452076
##########
cpp/src/parquet/decoder.cc:
##########
@@ -1675,13 +1675,90 @@ class DeltaLengthByteArrayDecoder : public DecoderImpl,
num_valid_values_ = num_length;
}
+ Status DecodeArrowDenseFastPath(
+ int num_values, int null_count, const uint8_t* valid_bits,
+ int64_t valid_bits_offset, typename
EncodingTraits<ByteArrayType>::Accumulator* out,
+ int* out_num_values) {
+ int max_values = num_values - null_count;
+ if (num_values - null_count > num_valid_values_) {
+ throw ParquetException("Expected to decode ", num_values - null_count,
+ " values, but can decode decoded ",
num_valid_values_,
+ " values.");
+ }
+ const int32_t* length_ptr = buffered_length_->data_as<int32_t>() +
length_idx_;
+ int bytes_offset = len_ - decoder_->bytes_left();
+ const uint8_t* data_ptr = data_ + bytes_offset;
+ const int64_t initial_offset = out->builder->value_data_length();
+ auto* offsets_builder = out->builder->offsets_builder();
+ auto* value_data_builder = out->builder->value_data_builder();
+ // Phase1: get total length of binary data and append to value_data_builder
+ int64_t accum_length = 0;
+ for (int i = 0; i < max_values; ++i) {
+ if (ARROW_PREDICT_FALSE(length_ptr[i] < 0)) {
+ return Status::Invalid("negative string delta length");
+ }
+ accum_length += length_ptr[i];
+ }
+ if (ARROW_PREDICT_FALSE(accum_length >
std::numeric_limits<int32_t>::max())) {
+ return Status::Invalid("excess expansion in DELTA_BYTE_ARRAY");
+ }
+ if (ARROW_PREDICT_FALSE(decoder_->bytes_left() < accum_length)) {
+ return Status::Invalid("Binary data is too short");
+ }
+ RETURN_NOT_OK(out->builder->ValidateOverflow(accum_length));
+ // Append the binary data
+ ARROW_RETURN_NOT_OK(value_data_builder->Append(data_ptr, accum_length));
+ // Phase2: append offsets
+ RETURN_NOT_OK(offsets_builder->Reserve(num_values));
+ accum_length = 0;
+ if (valid_bits == nullptr) {
+ for (int i = 0; i < max_values; ++i) {
Review Comment:
IIUC, `VisitNullBitmapInline` can also handle the case where `valid_bits ==
nullptr` since it uses `OptionalBitBlockCounter` under the hood.
##########
cpp/src/parquet/decoder.cc:
##########
@@ -1675,13 +1675,90 @@ class DeltaLengthByteArrayDecoder : public DecoderImpl,
num_valid_values_ = num_length;
}
+ Status DecodeArrowDenseFastPath(
+ int num_values, int null_count, const uint8_t* valid_bits,
+ int64_t valid_bits_offset, typename
EncodingTraits<ByteArrayType>::Accumulator* out,
+ int* out_num_values) {
+ int max_values = num_values - null_count;
Review Comment:
Can you call this `num_non_null_values`?
##########
cpp/src/parquet/decoder.cc:
##########
@@ -1675,13 +1675,90 @@ class DeltaLengthByteArrayDecoder : public DecoderImpl,
num_valid_values_ = num_length;
}
+ Status DecodeArrowDenseFastPath(
+ int num_values, int null_count, const uint8_t* valid_bits,
+ int64_t valid_bits_offset, typename
EncodingTraits<ByteArrayType>::Accumulator* out,
+ int* out_num_values) {
+ int max_values = num_values - null_count;
+ if (num_values - null_count > num_valid_values_) {
+ throw ParquetException("Expected to decode ", num_values - null_count,
+ " values, but can decode decoded ",
num_valid_values_,
+ " values.");
+ }
+ const int32_t* length_ptr = buffered_length_->data_as<int32_t>() +
length_idx_;
+ int bytes_offset = len_ - decoder_->bytes_left();
+ const uint8_t* data_ptr = data_ + bytes_offset;
+ const int64_t initial_offset = out->builder->value_data_length();
+ auto* offsets_builder = out->builder->offsets_builder();
+ auto* value_data_builder = out->builder->value_data_builder();
+ // Phase1: get total length of binary data and append to value_data_builder
+ int64_t accum_length = 0;
+ for (int i = 0; i < max_values; ++i) {
+ if (ARROW_PREDICT_FALSE(length_ptr[i] < 0)) {
+ return Status::Invalid("negative string delta length");
+ }
+ accum_length += length_ptr[i];
+ }
+ if (ARROW_PREDICT_FALSE(accum_length >
std::numeric_limits<int32_t>::max())) {
Review Comment:
Since `CanFit` returned true, can this actually happen?
##########
cpp/src/parquet/encoding_benchmark.cc:
##########
@@ -1284,6 +1305,11 @@ BENCHMARK_DEFINE_F(BM_ArrowBinaryPlain,
DecodeArrowNonNull_Dense)
BENCHMARK_REGISTER_F(BM_ArrowBinaryPlain, DecodeArrowNonNull_Dense)
->Range(MIN_RANGE, MAX_RANGE);
+BENCHMARK_DEFINE_F(BM_ArrowBinaryDeltaLength, DL_DecodeArrow_Dense)
Review Comment:
Why "DL"?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
