This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11402-03d40357b73c02e6454c7d352d6552f01bf500a8 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit deeb7256110c6c2325db66ad741c5f56749d3b93 Author: Rishabh Bhardwaj <[email protected]> AuthorDate: Mon Oct 26 04:15:37 2026 +0000 perf(parquet-variant): Avoid intermediate string allocations in JSON serialization (#11402) # Which issue does this PR close? - Closes #11396 # Rationale for this change Avoid unnecessary temporary `String` allocations when serializing JSON values by writing directly to the output buffer. # What changes are included in this PR? - Use `serde_json::to_writer` for direct JSON serialization. - Avoid unnecessary intermediate `String` allocations and copies. - Preserve the existing JSON escaping and serialization behavior. --------- Co-authored-by: rizzabh <[email protected]> Co-authored-by: Jeffrey Vo <[email protected]> Co-authored-by: Kosta Tarasov <[email protected]> --- parquet-variant-json/src/to_json.rs | 43 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/parquet-variant-json/src/to_json.rs b/parquet-variant-json/src/to_json.rs index 930ba935dc..4a0dd8f0ef 100644 --- a/parquet-variant-json/src/to_json.rs +++ b/parquet-variant-json/src/to_json.rs @@ -192,26 +192,20 @@ impl VariantToJson for Variant<'_, '_> { } Variant::Time(time) => write!(buffer, "\"{}\"", format_time_ntz_str(time))?, Variant::Binary(bytes) => { - // Encode binary as base64 string + // Encode binary as base64 string and serialize it directly into the writer let base64_str = format_binary_base64(bytes); - let json_str = serde_json::to_string(&base64_str).map_err(|e| { - ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}")) - })?; - write!(buffer, "{json_str}")? + serde_json::to_writer(&mut *buffer, &base64_str) + .map_err(|e| json_encoding_error("JSON encoding error", e))? } Variant::String(s) => { - // Use serde_json to properly escape the string - let json_str = serde_json::to_string(s).map_err(|e| { - ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}")) - })?; - write!(buffer, "{json_str}")? + // Serialize directly into the writer, escaping as needed + serde_json::to_writer(&mut *buffer, s) + .map_err(|e| json_encoding_error("JSON encoding error", e))? } Variant::ShortString(s) => { - // Use serde_json to properly escape the string - let json_str = serde_json::to_string(s.as_str()).map_err(|e| { - ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}")) - })?; - write!(buffer, "{json_str}")? + // Serialize directly into the writer, escaping as needed + serde_json::to_writer(&mut *buffer, s.as_str()) + .map_err(|e| json_encoding_error("JSON encoding error", e))? } Variant::Uuid(uuid) => { write!(buffer, "\"{uuid}\"")?; @@ -365,6 +359,16 @@ fn format_time_ntz_str(time: &chrono::NaiveTime) -> String { } } +/// Maps a [`serde_json`] serialization failure, keeping writer failures as +/// [`ArrowError::IoError`] just like [`write!`] does +fn json_encoding_error(context: &str, error: serde_json::Error) -> ArrowError { + if error.is_io() { + ArrowError::IoError(error.to_string(), error.into()) + } else { + ArrowError::InvalidArgumentError(format!("{context}: {error}")) + } +} + /// Convert object fields to JSON fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) -> Result<(), ArrowError> { write!(buffer, "{{")?; @@ -378,11 +382,10 @@ fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) -> Resul } first = false; - // Write the key (properly escaped) - let json_key = serde_json::to_string(key).map_err(|e| { - ArrowError::InvalidArgumentError(format!("JSON key encoding error: {e}")) - })?; - write!(buffer, "{json_key}:")?; + // Write the key (properly escaped), directly into the writer + serde_json::to_writer(&mut *buffer, key) + .map_err(|e| json_encoding_error("JSON key encoding error", e))?; + write!(buffer, ":")?; // Recursively convert the value value.to_json(buffer)?;
