tustvold commented on code in PR #6840:
URL: https://github.com/apache/arrow-rs/pull/6840#discussion_r1883013129
##########
parquet/src/arrow/schema/mod.rs:
##########
@@ -225,27 +219,134 @@ pub(crate) fn
add_encoded_arrow_schema_to_metadata(schema: &Schema, props: &mut
}
}
-/// Convert arrow schema to parquet schema
+/// Converter for arrow schema to parquet schema
///
-/// The name of the root schema element defaults to `"arrow_schema"`, this can
be
-/// overridden with [`arrow_to_parquet_schema_with_root`]
-pub fn arrow_to_parquet_schema(schema: &Schema, coerce_types: bool) ->
Result<SchemaDescriptor> {
- arrow_to_parquet_schema_with_root(schema, "arrow_schema", coerce_types)
+/// Example:
+/// ```
+/// # use std::sync::Arc;
+/// # use arrow_schema::{Field, Schema, DataType};
+/// # use parquet::arrow::ArrowToParquetSchemaConverter;
+/// use parquet::schema::types::{SchemaDescriptor, Type};
+/// use parquet::basic; // note there are two `Type`s in the following example
+/// let arrow_schema = Schema::new(vec![
+/// Field::new("a", DataType::Int64, true),
+/// Field::new("b", DataType::Date32, true),
+/// ]);
+///
+/// let parquet_schema = ArrowToParquetSchemaConverter::new(&arrow_schema)
+/// .build()
+/// .unwrap();
+///
+/// let expected_parquet_schema = SchemaDescriptor::new(
+/// Arc::new(
+/// Type::group_type_builder("arrow_schema")
+/// .with_fields(vec![
+/// Arc::new(
+/// Type::primitive_type_builder("a", basic::Type::INT64)
+/// .build().unwrap()
+/// ),
+/// Arc::new(
+/// Type::primitive_type_builder("b", basic::Type::INT32)
+/// .with_converted_type(basic::ConvertedType::DATE)
+/// .with_logical_type(Some(basic::LogicalType::Date))
+/// .build().unwrap()
+/// ),
+/// ])
+/// .build().unwrap()
+/// )
+/// );
+///
+/// assert_eq!(parquet_schema, expected_parquet_schema);
+/// ```
+#[derive(Debug)]
+pub struct ArrowToParquetSchemaConverter<'a> {
Review Comment:
```suggestion
pub struct ArrowSchemaConverter<'a> {
```
Perhaps, I don't feel strongly but might reduce the word salad
##########
parquet/src/arrow/schema/mod.rs:
##########
@@ -225,27 +219,134 @@ pub(crate) fn
add_encoded_arrow_schema_to_metadata(schema: &Schema, props: &mut
}
}
-/// Convert arrow schema to parquet schema
+/// Converter for arrow schema to parquet schema
///
-/// The name of the root schema element defaults to `"arrow_schema"`, this can
be
-/// overridden with [`arrow_to_parquet_schema_with_root`]
-pub fn arrow_to_parquet_schema(schema: &Schema, coerce_types: bool) ->
Result<SchemaDescriptor> {
- arrow_to_parquet_schema_with_root(schema, "arrow_schema", coerce_types)
+/// Example:
+/// ```
+/// # use std::sync::Arc;
+/// # use arrow_schema::{Field, Schema, DataType};
+/// # use parquet::arrow::ArrowToParquetSchemaConverter;
+/// use parquet::schema::types::{SchemaDescriptor, Type};
+/// use parquet::basic; // note there are two `Type`s in the following example
+/// let arrow_schema = Schema::new(vec![
+/// Field::new("a", DataType::Int64, true),
+/// Field::new("b", DataType::Date32, true),
+/// ]);
+///
+/// let parquet_schema = ArrowToParquetSchemaConverter::new(&arrow_schema)
+/// .build()
+/// .unwrap();
+///
+/// let expected_parquet_schema = SchemaDescriptor::new(
+/// Arc::new(
+/// Type::group_type_builder("arrow_schema")
+/// .with_fields(vec![
+/// Arc::new(
+/// Type::primitive_type_builder("a", basic::Type::INT64)
+/// .build().unwrap()
+/// ),
+/// Arc::new(
+/// Type::primitive_type_builder("b", basic::Type::INT32)
+/// .with_converted_type(basic::ConvertedType::DATE)
+/// .with_logical_type(Some(basic::LogicalType::Date))
+/// .build().unwrap()
+/// ),
+/// ])
+/// .build().unwrap()
+/// )
+/// );
+///
+/// assert_eq!(parquet_schema, expected_parquet_schema);
+/// ```
+#[derive(Debug)]
+pub struct ArrowToParquetSchemaConverter<'a> {
+ /// The schema to convert
+ schema: &'a Schema,
+ /// Name of the root schema in Parquet
+ schema_root: &'a str,
+ /// Should we coerce Arrow types to compatible Parquet types?
+ ///
+ /// See docs on [Self::with_coerce_types]`
+ coerce_types: bool,
}
-/// Convert arrow schema to parquet schema specifying the name of the root
schema element
-pub fn arrow_to_parquet_schema_with_root(
- schema: &Schema,
- root: &str,
- coerce_types: bool,
-) -> Result<SchemaDescriptor> {
- let fields = schema
- .fields()
- .iter()
- .map(|field| arrow_to_parquet_type(field, coerce_types).map(Arc::new))
- .collect::<Result<_>>()?;
- let group = Type::group_type_builder(root).with_fields(fields).build()?;
- Ok(SchemaDescriptor::new(Arc::new(group)))
+impl<'a> ArrowToParquetSchemaConverter<'a> {
+ /// Create a new converter
+ pub fn new(schema: &'a Schema) -> Self {
+ Self {
+ schema,
+ schema_root: "arrow_schema",
+ coerce_types: false,
+ }
+ }
+
+ /// Should Arrow types be coerced into Parquet native types (default
`false`).
+ ///
+ /// Setting this option to `true` will result in Parquet files that can be
+ /// read by more readers, but may lose precision for Arrow types such as
+ /// [`DataType::Date64`] which have no direct [corresponding Parquet type].
+ ///
+ /// By default, this converter does not coerce to native Parquet types.
Enabling type
+ /// coercion allows for meaningful representations that do not require
+ /// downstream readers to consider the embedded Arrow schema, and can allow
+ /// for greater compatibility with other Parquet implementations. However,
+ /// type coercion also prevents data from being losslessly round-tripped.
+ ///
+ /// # Discussion
+ ///
+ /// Some Arrow types such as `Date64`, `Timestamp` and `Interval` have no
+ /// corresponding Parquet logical type. Thus, they can not be losslessly
+ /// round-tripped when stored using the appropriate Parquet logical type.
+ /// For example, some Date64 values may be truncated when stored with
+ /// parquet's native 32 bit date type.
+ ///
+ /// For [`List`] and [`Map`] types, some Parquet readers expect certain
+ /// schema elements to have specific names (earlier versions of the spec
+ /// were somewhat ambiguous on this point). Type coercion will use the
names
+ /// prescribed by the Parquet specification, potentially losing naming
+ /// metadata from the Arrow schema.
+ ///
+ /// [`List`]:
https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
+ /// [`Map`]:
https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps
+ /// [corresponding Parquet type]:
https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date
+ ///
+ pub fn with_coerce_types(mut self, coerce_types: bool) -> Self {
+ self.coerce_types = coerce_types;
+ self
+ }
+
+ /// Set the root schema element name (defaults to `"arrow_schema"`).
+ pub fn schema_root(mut self, schema_root: &'a str) -> Self {
Review Comment:
```suggestion
pub fn with_root(mut self, schema_root: &'a str) -> Self {
```
Perhaps? For consistency and because the schema is perhaps redundant?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
