Dandandan commented on a change in pull request #8938:
URL: https://github.com/apache/arrow/pull/8938#discussion_r544583608
##########
File path: rust/arrow/src/json/reader.rs
##########
@@ -1175,8 +1239,103 @@ impl Decoder {
}
Ok(Arc::new(builder.finish()) as ArrayRef)
}
+
+ /// Read the primitive list's values into ArrayData
+ fn read_primitive_list_values<T>(&self, rows: &[Value]) -> ArrayDataRef
+ where
+ T: ArrowPrimitiveType + ArrowNumericType,
+ T::Native: num::NumCast,
+ {
+ let values = rows
+ .iter()
+ .filter_map(|row| {
+ // read values from list
+ if let Value::Array(values) = row {
+ Some(
+ values
+ .iter()
+ .map(|value| {
+ let v: Option<T::Native> =
+ value.as_f64().and_then(num::cast::cast);
+ v
+ })
+ .collect::<Vec<Option<T::Native>>>(),
+ )
+ } else if let Value::Number(value) = row {
+ // handle the scalar number case
+ let v: Option<T::Native> =
value.as_f64().and_then(num::cast::cast);
+ v.map(|v| vec![Some(v)])
+ } else {
+ None
+ }
+ })
+ .flatten()
+ .collect::<Vec<Option<T::Native>>>();
+ let array = PrimitiveArray::<T>::from_iter(values.iter());
+ array.data()
+ }
+}
+
+/// Reads a JSON value as a string, regardless of its type.
+/// This is useful if the expected datatype is a string, in which case we
preserve
+/// all the values regardless of they type.
+///
+/// Applying `value.to_string()` unfortunately results in an escaped string,
which
+/// is not what we want.
+#[inline(always)]
+fn json_value_as_string(value: &Value) -> Option<String> {
+ match value {
+ Value::Null => None,
+ Value::String(string) => Some(string.clone()),
+ _ => Some(value.to_string()),
+ }
}
+/// Flattens a list of JSON values, by flattening lists, and treating all
other values as
+/// single-value lists.
+/// This is used to read into nested lists (list of list, list of struct) and
non-dictionary lists.
+#[inline]
+fn flatten_json_values(values: &[Value]) -> Vec<Value> {
+ values
+ .iter()
+ .map(|row| {
+ if let Value::Array(values) = row {
+ values.clone()
Review comment:
Could return iterators here instead of cloning/ generating new vecs?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
