rok commented on code in PR #44470:
URL: https://github.com/apache/arrow/pull/44470#discussion_r2079461120
##########
cpp/src/arrow/dataset/file_test.cc:
##########
@@ -353,6 +361,153 @@ TEST_F(TestFileSystemDataset, WriteProjected) {
}
}
+// This kernel delays execution for some specific scalar values,
+// which guarantees the writing phase sees out-of-order exec batches
+Status delay(compute::KernelContext* ctx, const compute::ExecSpan& batch,
+ compute::ExecResult* out) {
+ const ArraySpan& input = batch[0].array;
+ const auto* input_values = input.GetValues<uint32_t>(1);
+ uint8_t* output_values = out->array_span()->buffers[1].data;
+
+ // Boolean data is stored in 1 bit per value
+ for (int64_t i = 0; i < input.length; ++i) {
+ if (input_values[i] % 16 == 0) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+ bit_util::SetBitTo(output_values, i, true);
+ }
+
+ return Status::OK();
+}
+
+// A fragment with start=0 will defer ScanBatchesAsync returning a batch
generator
+// This guarantees a dataset of multiple fragments produces out-of-order
batches
+class MockFragment : public Fragment {
+ public:
+ explicit MockFragment(uint32_t start, int64_t rows_per_batch, int
num_batches,
+ const std::shared_ptr<Schema>& schema)
+ : Fragment(compute::literal(true), schema),
+ start_(start),
+ rows_per_batch_(rows_per_batch),
+ num_batches_(num_batches) {}
+
+ Result<RecordBatchGenerator> ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) override {
+ // One fragment requires some more time than others
Review Comment:
Agreed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
