zanmato1984 commented on code in PR #45822:
URL: https://github.com/apache/arrow/pull/45822#discussion_r2009872726
##########
cpp/src/arrow/compute/row/grouper_test.cc:
##########
@@ -780,21 +897,23 @@ TEST(Grouper, FloatingPointKey) {
TEST(Grouper, StringKey) {
for (auto ty : {utf8(), large_utf8(), fixed_size_binary(2)}) {
- SCOPED_TRACE("key type: " + ty->ToString());
-
- TestGrouper g({ty});
-
- g.ExpectConsume(R"([["eh"], ["eh"]])", "[0, 0]");
-
- g.ExpectConsume(R"([["eh"], ["eh"]])", "[0, 0]");
-
- g.ExpectConsume(R"([["be"], [null]])", "[1, 2]");
+ ARROW_SCOPED_TRACE("key type = ", *ty);
Review Comment:
Why is this not testing `Populate`?
##########
cpp/src/arrow/compute/row/grouper.cc:
##########
@@ -649,11 +726,29 @@ struct GrouperFastImpl : public Grouper {
cols_[icol] = col_base.Slice(offset, num_rows);
}
+ std::shared_ptr<arrow::Buffer> group_ids, null_bitmap;
+ // If we need to return the group ids, then allocate a buffer of group ids
+ // for all rows, otherwise each minibatch will reuse the same buffer.
+ const int64_t groups_ids_size =
+ (mode == GrouperMode::kPopulate) ? minibatch_size_max_ : num_rows;
+ ARROW_ASSIGN_OR_RAISE(group_ids, AllocateBuffer(sizeof(uint32_t) *
groups_ids_size,
+ ctx_->memory_pool()));
+ if (mode == GrouperMode::kLookup) {
+ ARROW_ASSIGN_OR_RAISE(null_bitmap,
+ AllocateBitmap(groups_ids_size,
ctx_->memory_pool()));
+ }
+
// Split into smaller mini-batches
//
for (uint32_t start_row = 0; start_row < num_rows;) {
uint32_t batch_size_next =
std::min(static_cast<uint32_t>(minibatch_size_),
static_cast<uint32_t>(num_rows) -
start_row);
+ uint32_t* batch_group_ids = group_ids->mutable_data_as<uint32_t>() +
+ ((mode == GrouperMode::kPopulate) ? 0 :
start_row);
+ if (mode == GrouperMode::kLookup) {
+ // Zero-initialize
+ memset(batch_group_ids, 0, batch_size_next * sizeof(uint32_t));
+ }
Review Comment:
Is it possible to zero the full length of `group_ids` outside the for loop?
Maybe in the `if (mode == GrouperMode::kLookup)` statement right above?
##########
cpp/src/arrow/compute/row/grouper_test.cc:
##########
@@ -898,42 +1064,88 @@ TEST(Grouper, DoubleStringInt64Key) {
g.ExpectConsume(R"([[-0.0, "be", 7], [0.0, "be", 7]])", "[3, 4]");
}
-TEST(Grouper, RandomInt64Keys) {
- TestGrouper g({int64()});
+FieldVector AnnotateForRandomGeneration(FieldVector fields) {
+ for (auto& field : fields) {
+ // For each field, constrain random generation to ensure that group ids
+ // can appear more than once.
+ if (is_integer(*field->type())) {
+ field =
+ field->WithMergedMetadata(key_value_metadata({"min", "max"}, {"100",
"10000"}));
+ } else if (is_binary_like(*field->type())) {
+ // (note this is unsupported for large binary types)
+ field = field->WithMergedMetadata(key_value_metadata({"unique"},
{"100"}));
+ }
+ field = field->WithMergedMetadata(key_value_metadata({"null_probability"},
{"0.1"}));
+ }
+ return fields;
+}
+
+void TestRandomConsume(TestGrouper g) {
Review Comment:
Ditto, why no cases for `Populate`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
