sahil1105 commented on code in PR #43661:
URL: https://github.com/apache/arrow/pull/43661#discussion_r1736860944
##########
cpp/src/arrow/dataset/file_parquet.cc:
##########
@@ -555,6 +562,66 @@ Future<std::shared_ptr<parquet::arrow::FileReader>>
ParquetFileFormat::GetReader
});
}
+struct CastingGenerator {
+ CastingGenerator(RecordBatchGenerator source, std::shared_ptr<Schema>
final_schema,
+ const std::unordered_set<std::string>& cols_to_skip,
+ MemoryPool* pool = default_memory_pool())
+ : source_(source),
+ final_schema_(final_schema),
+ cols_to_skip_(cols_to_skip),
+ exec_ctx(std::make_shared<compute::ExecContext>(pool)) {}
+
+ Future<std::shared_ptr<RecordBatch>> operator()() {
+ return this->source_().Then([this](const std::shared_ptr<RecordBatch>&
next)
+ -> Result<std::shared_ptr<RecordBatch>> {
+ if (IsIterationEnd(next) || this->final_schema_ == nullptr) {
+ return next;
+ }
+ std::vector<std::shared_ptr<Array>> out_cols;
+ std::vector<std::shared_ptr<Field>> out_schema_fields;
+
+ bool changed = false;
+ for (const auto& field : this->final_schema_->fields()) {
+ FieldRef field_ref = FieldRef(field->name());
+ ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Array> column,
+ field_ref.GetOneOrNone(*next));
+ if (column) {
+ if (this->cols_to_skip_.count(field->name())) {
+ // Maintain the original input type.
+ out_schema_fields.emplace_back(field->WithType(column->type()));
+ out_cols.emplace_back(std::move(column));
Review Comment:
Yes, to skip the cast and leave them as they are.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
