tustvold commented on code in PR #5056:
URL: https://github.com/apache/arrow-datafusion/pull/5056#discussion_r1086598305
##########
datafusion/core/src/physical_plan/file_format/json.rs:
##########
@@ -162,40 +154,61 @@ impl ExecutionPlan for NdJsonExec {
}
struct JsonOpener {
- options: DecoderOptions,
- file_schema: SchemaRef,
+ batch_size: usize,
+ projected_schema: SchemaRef,
file_compression_type: FileCompressionType,
object_store: Arc<dyn ObjectStore>,
}
impl FileOpener for JsonOpener {
fn open(&self, file_meta: FileMeta) -> Result<FileOpenFuture> {
- let options = self.options.clone();
- let schema = self.file_schema.clone();
let store = self.object_store.clone();
+ let schema = self.projected_schema.clone();
+ let batch_size = self.batch_size;
+
let file_compression_type = self.file_compression_type.to_owned();
Ok(Box::pin(async move {
match store.get(file_meta.location()).await? {
GetResult::File(file, _) => {
- let decoder = file_compression_type.convert_read(file)?;
- let reader = json::Reader::new(decoder, schema.clone(),
options);
+ let bytes = file_compression_type.convert_read(file)?;
+ let reader = RawReaderBuilder::new(schema)
+ .with_batch_size(batch_size)
+ .build(BufReader::new(bytes))?;
Ok(futures::stream::iter(reader).boxed())
}
GetResult::Stream(s) => {
+ let mut decoder = RawDecoder::try_new(schema, batch_size)?;
Review Comment:
I think this interface is pretty cool, it avoids needing to scan the byte
stream looking for newlines, and consequently should add some additional
performance on top of the faster performance of `RawDecoder` in general
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
