adriangb commented on code in PR #3052:
URL: https://github.com/apache/datafusion-comet/pull/3052#discussion_r2688986063
##########
native/core/src/execution/planner.rs:
##########
@@ -4004,18 +3995,15 @@ mod tests {
}
}
- let source =
ParquetSource::default().with_schema_adapter_factory(Arc::new(
- SparkSchemaAdapterFactory::new(
Review Comment:
If you're removing a `SparkSchemaAdapterFactory` you need to replace it with
a `SparkPhysicalExprAdapterFactory`.
##########
native/core/src/parquet/parquet_exec.rs:
##########
@@ -78,7 +78,26 @@ pub(crate) fn init_datasource_exec(
encryption_enabled,
);
- let mut parquet_source = ParquetSource::new(table_parquet_options);
+ // Determine the schema to use for ParquetSource
+ let table_schema = if let Some(ref data_schema) = data_schema {
Review Comment:
There is also `TableSchema::with_table_partition_cols` which might make this
easier
##########
native/core/src/execution/planner.rs:
##########
@@ -4004,18 +3995,15 @@ mod tests {
}
}
- let source =
ParquetSource::default().with_schema_adapter_factory(Arc::new(
- SparkSchemaAdapterFactory::new(
- SparkParquetOptions::new(EvalMode::Ansi, "", false),
- None,
- ),
- ))?;
+ let source = Arc::new(
+ ParquetSource::new(Arc::new(read_schema.clone()))
+ .with_table_parquet_options(TableParquetOptions::new()),
+ ) as Arc<dyn FileSource>;
let object_store_url = ObjectStoreUrl::local_filesystem();
- let file_scan_config =
- FileScanConfigBuilder::new(object_store_url, read_schema.into(),
source)
- .with_file_groups(file_groups)
- .build();
+ let file_scan_config = FileScanConfigBuilder::new(object_store_url,
source)
+ .with_file_groups(file_groups)
Review Comment:
```suggestion
.with_file_groups(file_groups)
.with_expr_adapter(Some(Arc::new(
SparkPhysicalExprAdapterFactory::new(SparkParquetOptions::new(EvalMode::Ansi,
"", false), None),
))
```
or something like that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
