otegami commented on code in PR #37137:
URL: https://github.com/apache/arrow/pull/37137#discussion_r1300043297
##########
ruby/red-arrow/ext/arrow/raw-records.cpp:
##########
@@ -144,6 +144,118 @@ namespace red_arrow {
// The number of columns.
const int n_columns_;
};
+
+ class RawRecordsProducer : private Converter, public arrow::ArrayVisitor {
+ public:
+ explicit RawRecordsProducer()
+ : Converter(),
+ record_(Qnil) {
+ }
+
+ void produce(const arrow::RecordBatch& record_batch) {
+ rb::protect([&] {
+ auto n_columns = record_batch.num_columns();
+ const auto n_rows = record_batch.num_rows();
+ for (int64_t i = 0; i < n_rows; ++i) {
+ record_ = rb_ary_new_capa(n_columns);
+ row_offset_ = i;
+ for (int i = 0; i < n_columns; ++i) {
+ const auto array = record_batch.column(i).get();
+ column_index_ = i;
+ check_status(array->Accept(this),
+ "[record-batch][each-raw-record]");
+ }
+ rb_yield(record_);
+ }
+ return Qnil;
+ });
+ }
+
+ void produce(const arrow::Table& table) {
+ rb::protect([&] {
+ int n_columns = table.num_columns();
+ const auto& base_column = table.column(0);
+ for (int i = 0; i < base_column->num_chunks(); ++i) {
+ int chunk_length = base_column->chunk(i)->length();
+ for (int j = 0; j < chunk_length; ++j) {
+ row_offset_ = j;
+ record_ = rb_ary_new_capa(n_columns);
+ for (int k = 0; k < n_columns; ++k) {
+ column_index_ = k;
+ const auto& array = table.column(k)->chunk(i);
+ check_status(array->Accept(this),
+ "[table][each-raw-record]");
+ }
+ rb_yield(record_);
+ }
+ }
Review Comment:
> We can't assume that all columns have the same chunk layout.
Thank you so much. I misunderstood this point🙏
##########
ruby/red-arrow/ext/arrow/raw-records.cpp:
##########
@@ -144,6 +144,118 @@ namespace red_arrow {
// The number of columns.
const int n_columns_;
};
+
+ class RawRecordsProducer : private Converter, public arrow::ArrayVisitor {
+ public:
+ explicit RawRecordsProducer()
+ : Converter(),
+ record_(Qnil) {
+ }
+
+ void produce(const arrow::RecordBatch& record_batch) {
+ rb::protect([&] {
+ auto n_columns = record_batch.num_columns();
+ const auto n_rows = record_batch.num_rows();
+ for (int64_t i = 0; i < n_rows; ++i) {
+ record_ = rb_ary_new_capa(n_columns);
+ row_offset_ = i;
+ for (int i = 0; i < n_columns; ++i) {
+ const auto array = record_batch.column(i).get();
+ column_index_ = i;
+ check_status(array->Accept(this),
+ "[record-batch][each-raw-record]");
+ }
+ rb_yield(record_);
+ }
+ return Qnil;
+ });
+ }
+
+ void produce(const arrow::Table& table) {
+ rb::protect([&] {
+ int n_columns = table.num_columns();
+ const auto& base_column = table.column(0);
+ for (int i = 0; i < base_column->num_chunks(); ++i) {
+ int chunk_length = base_column->chunk(i)->length();
+ for (int j = 0; j < chunk_length; ++j) {
+ row_offset_ = j;
+ record_ = rb_ary_new_capa(n_columns);
+ for (int k = 0; k < n_columns; ++k) {
+ column_index_ = k;
+ const auto& array = table.column(k)->chunk(i);
+ check_status(array->Accept(this),
+ "[table][each-raw-record]");
+ }
+ rb_yield(record_);
+ }
+ }
Review Comment:
> We can't assume that all columns have the same chunk layout.
Thank you so much. I misunderstood this point🙏
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
