(arrow-rs) branch main updated: RecordBatchDecoder: skip RecordBatch validation when `skip_validation` property is enabled (#7509)

[alamb]

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new 5a278c6d63 RecordBatchDecoder: skip RecordBatch validation when 
`skip_validation` property is enabled (#7509)
5a278c6d63 is described below

commit 5a278c6d63d5faeaaf00be30ef8cc57ed4c859aa
Author: Nils Koch <m...@nilskch.dev>
AuthorDate: Mon May 19 21:33:18 2025 +0100

    RecordBatchDecoder: skip RecordBatch validation when `skip_validation` 
property is enabled (#7509)
    
    * RecordBatchDecoder: respect skip_validation property
    
    * Update arrow-ipc/src/reader.rs
    
    Co-authored-by: Andrew Lamb <and...@nerdnetworks.org>
    
    * Update arrow-ipc/src/reader.rs
    
    Co-authored-by: Andrew Lamb <and...@nerdnetworks.org>
    
    ---------
    
    Co-authored-by: Andrew Lamb <and...@nerdnetworks.org>
---
 arrow-ipc/src/reader.rs | 40 ++++++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/arrow-ipc/src/reader.rs b/arrow-ipc/src/reader.rs
index 83dc5702dc..7f9b4b2937 100644
--- a/arrow-ipc/src/reader.rs
+++ b/arrow-ipc/src/reader.rs
@@ -490,13 +490,25 @@ impl<'a> RecordBatchDecoder<'a> {
                     self.skip_field(field, &mut variadic_counts)?;
                 }
             }
-            assert!(variadic_counts.is_empty());
+
             arrays.sort_by_key(|t| t.0);
-            RecordBatch::try_new_with_options(
-                Arc::new(schema.project(projection)?),
-                arrays.into_iter().map(|t| t.1).collect(),
-                &options,
-            )
+
+            let schema = Arc::new(schema.project(projection)?);
+            let columns = arrays.into_iter().map(|t| t.1).collect::<Vec<_>>();
+
+            if self.skip_validation.get() {
+                // Safety: setting `skip_validation` requires `unsafe`, user 
assures data is valid
+                unsafe {
+                    Ok(RecordBatch::new_unchecked(
+                        schema,
+                        columns,
+                        self.batch.length() as usize,
+                    ))
+                }
+            } else {
+                assert!(variadic_counts.is_empty());
+                RecordBatch::try_new_with_options(schema, columns, &options)
+            }
         } else {
             let mut children = vec![];
             // keep track of index as lists require more than one node
@@ -504,8 +516,20 @@ impl<'a> RecordBatchDecoder<'a> {
                 let child = self.create_array(field, &mut variadic_counts)?;
                 children.push(child);
             }
-            assert!(variadic_counts.is_empty());
-            RecordBatch::try_new_with_options(schema, children, &options)
+
+            if self.skip_validation.get() {
+                // Safety: setting `skip_validation` requires `unsafe`, user 
assures data is valid
+                unsafe {
+                    Ok(RecordBatch::new_unchecked(
+                        schema,
+                        children,
+                        self.batch.length() as usize,
+                    ))
+                }
+            } else {
+                assert!(variadic_counts.is_empty());
+                RecordBatch::try_new_with_options(schema, children, &options)
+            }
         }
     }