nealrichardson commented on a change in pull request #9589:
URL: https://github.com/apache/arrow/pull/9589#discussion_r609822376
##########
File path: r/src/dataset.cpp
##########
@@ -463,4 +488,69 @@ void dataset___Dataset__Write(
StopIfNotOk(ds::FileSystemDataset::Write(opts, scanner));
}
+namespace arrow {
+
+Result<std::shared_ptr<Table>> TakeRows(std::shared_ptr<Array> indices,
+ ds::Scanner* scanner) {
+ if (indices->null_count() != 0) {
+ return Status::NotImplemented("null take indices");
+ }
+
+ if (indices->type_id() != Type::INT64) {
+ ARROW_ASSIGN_OR_RAISE(indices, compute::Cast(*indices, int64()));
+ }
+
+ std::shared_ptr<Array> unsort_indices;
+ {
+ ARROW_ASSIGN_OR_RAISE(auto sort_indices, compute::SortIndices(*indices));
+ ARROW_ASSIGN_OR_RAISE(indices, compute::Take(*indices, *sort_indices));
+ ARROW_ASSIGN_OR_RAISE(unsort_indices, compute::SortIndices(*sort_indices));
+ }
+
+ RecordBatchVector out_batches;
+
+ auto raw_indices = static_cast<const Int64Array&>(*indices).raw_values();
+ int64_t offset = 0, row_begin = 0;
+
+ ARROW_ASSIGN_OR_RAISE(auto batch_it, scanner->ScanBatches());
+ for (auto maybe_batch : batch_it) {
+ ARROW_ASSIGN_OR_RAISE(auto batch, maybe_batch);
+
+ if (offset == indices->length()) break;
+ // DCHECK_LT(offset, indices->length());
+
+ int64_t length = 0;
+ while (offset + length < indices->length()) {
+ auto rel_index = raw_indices[offset + length] - row_begin;
+ if (rel_index >= batch.batch->num_rows()) break;
+ ++length;
+ }
+ // DCHECK_LE(offset + length, indices->length());
+
+ Datum rel_indices = indices->Slice(offset, length);
+ ARROW_ASSIGN_OR_RAISE(rel_indices, compute::Subtract(rel_indices,
Datum(row_begin)));
+
+ ARROW_ASSIGN_OR_RAISE(Datum out_batch, compute::Take(batch.batch,
rel_indices));
+ out_batches.push_back(out_batch.record_batch());
+
+ offset += length;
+ row_begin += batch.batch->num_rows();
+ }
+
+ ARROW_ASSIGN_OR_RAISE(
+ Datum out, Table::FromRecordBatches(scanner->schema(),
std::move(out_batches)));
+
+ ARROW_ASSIGN_OR_RAISE(out, compute::Take(out, unsort_indices));
+ return out.table();
+}
+
+} // namespace arrow
+
+// [[arrow::export]]
+std::shared_ptr<arrow::Table> dataset___Scanner__TakeRows(
Review comment:
Maybe this is a naive question but why are there two versions of
TakeRows?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
