pitrou commented on code in PR #37641:
URL: https://github.com/apache/arrow/pull/37641#discussion_r1324752764
##########
cpp/src/parquet/encoding_benchmark.cc:
##########
@@ -737,6 +737,90 @@ static void
BM_DeltaLengthDecodingSpacedByteArray(benchmark::State& state) {
BENCHMARK(BM_PlainDecodingSpacedByteArray)->Apply(ByteArrayCustomArguments);
BENCHMARK(BM_DeltaLengthDecodingSpacedByteArray)->Apply(ByteArrayCustomArguments);
+void prefixed_random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray*
out,
+ int min_size, int max_size, double
prefixed_probability) {
+ std::default_random_engine gen(seed);
+ std::uniform_int_distribution<int> dist_size(min_size, max_size);
+ std::uniform_int_distribution<int> dist_byte(0, 255);
+ std::bernoulli_distribution dist_has_prefix(prefixed_probability);
+ std::uniform_real_distribution<double> dist_prefix_length(0, 1);
+
+ for (int i = 0; i < n; ++i) {
+ int len = dist_size(gen);
+ out[i].len = len;
+ out[i].ptr = buf;
+
+ bool do_prefix = dist_has_prefix(gen) && i > 0;
+ int prefix_len = 0;
+ if (do_prefix) {
+ int max_prefix_len = std::min(len, static_cast<int>(out[i - 1].len));
+ prefix_len = static_cast<int>(std::ceil(max_prefix_len *
dist_prefix_length(gen)));
+ }
+ for (int j = 0; j < prefix_len; ++j) {
+ buf[j] = out[i - 1].ptr[j];
+ }
+ for (int j = prefix_len; j < len; ++j) {
+ buf[j] = static_cast<uint8_t>(dist_byte(gen));
+ }
+ buf += len;
+ }
+}
+
+static void BM_DeltaEncodingByteArray(benchmark::State& state) {
+ // Using arrow generator to generate random data.
+ int32_t max_length = static_cast<int32_t>(state.range(0));
+ int32_t array_size = static_cast<int32_t>(state.range(1));
+ auto encoder = MakeTypedEncoder<ByteArrayType>(Encoding::DELTA_BYTE_ARRAY);
+ std::vector<ByteArray> values;
+ std::vector<uint8_t> buf(max_length * array_size);
+ values.resize(array_size);
+ prefixed_random_byte_array(array_size, /*seed=*/0, buf.data(), values.data(),
Review Comment:
Ideally, though, we should try to generate data for which the encoding is
very space-efficient. Is it the case here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
