lidavidm commented on code in PR #49698:
URL: https://github.com/apache/arrow/pull/49698#discussion_r3073353971
##########
cpp/src/arrow/ipc/reader.cc:
##########
@@ -613,16 +614,90 @@ Status DecompressBuffers(Compression::type compression,
const IpcReadOptions& op
});
}
+// Helper class to run post-ArrayLoader steps:
+// buffer decompression, dictionary resolution, buffer re-alignment.
+struct RecordBatchLoader {
+ Result<std::shared_ptr<RecordBatch>> CreateRecordBatch(ArrayDataVector
columns) {
+ ARROW_ASSIGN_OR_RAISE(auto filtered_columns,
CreateColumns(std::move(columns)));
+
+ std::shared_ptr<Schema> filtered_schema;
+ if (!inclusion_mask_.empty()) {
+ FieldVector filtered_fields;
+ for (int i = 0; i < schema_->num_fields(); ++i) {
+ if (inclusion_mask_[i]) {
+ filtered_fields.push_back(schema_->field(i));
+ }
+ }
+ filtered_schema = schema(std::move(filtered_fields),
schema_->metadata());
+ } else {
+ filtered_schema = schema_;
+ }
+
+ return RecordBatch::Make(std::move(filtered_schema), batch_length_,
+ std::move(filtered_columns));
+ }
+
+ Result<ArrayDataVector> CreateColumns(ArrayDataVector columns,
+ bool resolve_dictionaries = true) {
+ if (resolve_dictionaries) {
+ // Dictionary resolution needs to happen on the unfiltered columns,
+ // because fields are mapped structurally (by path in the original
schema).
+ RETURN_NOT_OK(ResolveDictionaries(columns, *context_.dictionary_memo,
+ context_.options.memory_pool));
+ }
+
+ ArrayDataVector filtered_columns;
+ if (!inclusion_mask_.empty()) {
+ FieldVector filtered_fields;
+ for (int i = 0; i < schema_->num_fields(); ++i) {
+ if (inclusion_mask_[i]) {
+ filtered_columns.push_back(std::move(columns[i]));
Review Comment:
nit: given the code is more encapsulated from its users now, is it worth
DCHECKing that the column is not-nullptr and actually got loaded here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
