lidavidm commented on a change in pull request #9589:
URL: https://github.com/apache/arrow/pull/9589#discussion_r609824953
##########
File path: r/src/dataset.cpp
##########
@@ -463,4 +488,69 @@ void dataset___Dataset__Write(
StopIfNotOk(ds::FileSystemDataset::Write(opts, scanner));
}
+namespace arrow {
+
+Result<std::shared_ptr<Table>> TakeRows(std::shared_ptr<Array> indices,
+ ds::Scanner* scanner) {
+ if (indices->null_count() != 0) {
+ return Status::NotImplemented("null take indices");
+ }
+
+ if (indices->type_id() != Type::INT64) {
+ ARROW_ASSIGN_OR_RAISE(indices, compute::Cast(*indices, int64()));
+ }
+
+ std::shared_ptr<Array> unsort_indices;
+ {
+ ARROW_ASSIGN_OR_RAISE(auto sort_indices, compute::SortIndices(*indices));
+ ARROW_ASSIGN_OR_RAISE(indices, compute::Take(*indices, *sort_indices));
+ ARROW_ASSIGN_OR_RAISE(unsort_indices, compute::SortIndices(*sort_indices));
+ }
+
+ RecordBatchVector out_batches;
+
+ auto raw_indices = static_cast<const Int64Array&>(*indices).raw_values();
+ int64_t offset = 0, row_begin = 0;
+
+ ARROW_ASSIGN_OR_RAISE(auto batch_it, scanner->ScanBatches());
+ for (auto maybe_batch : batch_it) {
+ ARROW_ASSIGN_OR_RAISE(auto batch, maybe_batch);
+
+ if (offset == indices->length()) break;
+ // DCHECK_LT(offset, indices->length());
+
+ int64_t length = 0;
+ while (offset + length < indices->length()) {
+ auto rel_index = raw_indices[offset + length] - row_begin;
+ if (rel_index >= batch.batch->num_rows()) break;
+ ++length;
+ }
+ // DCHECK_LE(offset + length, indices->length());
+
+ Datum rel_indices = indices->Slice(offset, length);
+ ARROW_ASSIGN_OR_RAISE(rel_indices, compute::Subtract(rel_indices,
Datum(row_begin)));
+
+ ARROW_ASSIGN_OR_RAISE(Datum out_batch, compute::Take(batch.batch,
rel_indices));
+ out_batches.push_back(out_batch.record_batch());
+
+ offset += length;
+ row_begin += batch.batch->num_rows();
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ Datum out, Table::FromRecordBatches(scanner->schema(),
std::move(out_batches)));
+
+ ARROW_ASSIGN_OR_RAISE(out, compute::Take(out, unsort_indices));
+ return out.table();
+}
+
+} // namespace arrow
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> dataset___Scanner__TakeRows(
Review comment:
Hmm, Ben split it into two functions (a pure-Arrow implementation, and
the R binding) presumably for convenience. But maybe we can just port the
implementation into the C++ library and expose it to Python as well?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
