matthewgapp commented on code in PR #5197:
URL: https://github.com/apache/arrow-rs/pull/5197#discussion_r1458351360
##########
arrow-json/src/writer.rs:
##########
@@ -469,11 +469,67 @@ fn set_column_for_json_rows(
row.insert(col_name.to_string(),
serde_json::Value::Object(obj));
}
}
+ DataType::Decimal128(_precision, _scale) |
DataType::Decimal256(_precision, _scale) => {
+ to_json_float(rows, array, col_name, explicit_nulls)?;
+ }
_ => {
return Err(ArrowError::JsonError(format!(
- "data type {:?} not supported in nested map for json writer",
+ "data type {:?} not supported for json writer",
array.data_type()
- )))
+ )));
+ }
+ }
+ Ok(())
+}
+
+fn to_json_float(
+ rows: &mut [Option<JsonMap<String, Value>>],
+ array: &ArrayRef,
+ col_name: &str,
+ explicit_nulls: bool,
+) -> Result<(), ArrowError> {
+ let options = FormatOptions::default();
+ let formatter = ArrayFormatter::try_new(array.as_ref(), &options)?;
Review Comment:
that makes sense. I'm thinking we could take advantage of serde_json's
ability to parse from string, with the optional compile time feature flag to
allow arbitrary precision numbers which are stored as a string.
See here
https://github.com/serde-rs/json/blob/6d44b9fac9269b4decf76acac5d68e8ec9d10c58/src/de.rs#L107
See here
https://github.com/serde-rs/json/blob/6d44b9fac9269b4decf76acac5d68e8ec9d10c58/src/de.rs#L932
We could expose the arbitrary precision feature flag on arrow_json so the
user can decide or we could just turn it on by default.
Below is what I'd propose. I'm working on tests to prove this out. Let me
know your thoughts @tustvold
```rust
fn to_json_float(
rows: &mut [Option<JsonMap<String, Value>>],
array: &ArrayRef,
col_name: &str,
explicit_nulls: bool,
) -> Result<(), ArrowError> {
let options = FormatOptions::default();
let formatter = ArrayFormatter::try_new(array.as_ref(), &options)?;
let nulls = array.nulls();
let rows = rows
.iter_mut()
.enumerate()
.filter_map(|(idx, maybe_row)| maybe_row.as_mut().map(|row| (idx,
row)));
for (idx, row) in rows {
let maybe_value = nulls
.map(|x| x.is_valid(idx))
.unwrap_or(true)
.then(|| {
let num_string = formatter.value(idx).to_string();
serde_json::Number::from_str(&num_string)
.map_err(|e| {
ArrowError::CastError(format!(
"Cannot convert {} to json number: {}",
formatter.value(idx),
e
))
})
.map(Value::Number)
})
.map_or_else(|| Ok(None), |result| result.map(Some));
if let Some(j) = maybe_value? {
row.insert(col_name.to_string(), j);
} else if explicit_nulls {
row.insert(col_name.to_string(), Value::Null);
}
}
Ok(())
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
