BubbaJoe commented on issue #5635:
URL:
https://github.com/apache/arrow-datafusion/issues/5635#issuecomment-1480427571
`main.rs`
```
use datafusion::error::Result;
use datafusion::{
arrow::{
array::{
ArrayRef,
StringArray,
Date32Builder,
Array,
},
datatypes::DataType,
},
physical_plan::functions::make_scalar_function,
logical_expr::Volatility,
prelude::{
create_udf,
SessionContext,
NdJsonReadOptions,
},
error::DataFusionError,
};
use std::sync::Arc;
use chrono::DateTime;
#[tokio::main]
async fn main() -> Result<()> {
let ctx = SessionContext::new();
// let testdata = datafusion::test_util::parquet_test_data();
// register parquet file with the execution context
ctx.register_json(
"users",
&format!("./data.jsonl"),
NdJsonReadOptions::default().
file_extension(".jsonl"),
).await?;
let to_date = make_scalar_function(to_date);
let to_date = create_udf(
"to_date",
vec![DataType::Utf8],
Arc::new(DataType::Date32),
Volatility::Immutable,
to_date,
);
ctx.register_udf(to_date.clone());
let batches = {
let df = ctx.sql(
"SELECT to_date(\"date\") as a from \
'users' group by a;").await?;
df.collect().await?
};
let list: Vec<_> = datafusion::arrow::json::writer::
record_batches_to_json_rows(&batches[..])?;
println!("{:?}", list);
Ok(())
}
fn to_date(args: &[ArrayRef]) -> Result<ArrayRef> {
if args.is_empty() || args.len() > 1 {
return Err(DataFusionError::Internal(format!(
"to_date was called with {} arguments. It requires only 1.",
args.len()
)));
}
let arg = &args[0].as_any().downcast_ref::<StringArray>().unwrap();
let mut builder = Date32Builder::new();
for date_string in arg.iter() {
let date_time = match DateTime::parse_from_rfc3339(
date_string.expect("date_string is null"),
) {
Ok(dt) => dt,
Err(e) => {
// builder.append_value((date_time.timestamp() / 86400) as
i32);
return Result::Err(DataFusionError::Internal(e.to_string()));
}
};
builder.append_value((date_time.timestamp() / 86400) as i32);
}
Ok(Arc::new(builder.finish()))
}
```
`Cargo.toml`
```
[package]
name = "df-test"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at
https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { version = "1.0", features = ["full"] }
datafusion = { version = "20.0"}
chrono = "0.4.24"
```
`data.jsonl`
```
{"date":"2012-04-23T18:25:43.511Z","id":0}
{"date":"2012-04-23T18:25:43.511Z","id":1}
{"date":"2012-04-23T18:25:43.511Z","id":2}
{"date":"2012-04-23T18:25:43.511Z","id":3}
{"date":"2012-04-23T18:25:43.511Z","id":4}
{"date":"2012-04-23T18:25:43.511Z","id":5}
{"date":"2012-04-23T18:25:43.511Z","id":6}
{"date":"2012-04-23T18:25:43.511Z","id":7}
{"date":"2012-04-23T18:25:43.511Z","id":8}
{"date":"2012-04-23T18:25:43.511Z","id":9}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
