This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 44b644d47c Cleanup parquet type builders (#4706)
44b644d47c is described below
commit 44b644d47ccb2172de54f4dc729caae487f7851d
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Thu Aug 17 10:22:23 2023 +0100
Cleanup parquet type builders (#4706)
* Cleanup parquet type builders
* Update parquet-derive
---
parquet/src/arrow/arrow_reader/mod.rs | 8 +--
parquet/src/arrow/schema/mod.rs | 24 ++++----
parquet/src/file/footer.rs | 4 +-
parquet/src/file/metadata.rs | 2 +-
parquet/src/file/writer.rs | 16 +++---
parquet/src/record/reader.rs | 2 +-
parquet/src/schema/mod.rs | 2 +-
parquet/src/schema/parser.rs | 40 ++++++-------
parquet/src/schema/printer.rs | 36 ++++++------
parquet/src/schema/types.rs | 105 ++++++++++++++++------------------
parquet_derive/src/lib.rs | 2 +-
11 files changed, 114 insertions(+), 127 deletions(-)
diff --git a/parquet/src/arrow/arrow_reader/mod.rs
b/parquet/src/arrow/arrow_reader/mod.rs
index 7e4423b864..f7cecabb01 100644
--- a/parquet/src/arrow/arrow_reader/mod.rs
+++ b/parquet/src/arrow/arrow_reader/mod.rs
@@ -1669,7 +1669,7 @@ mod tests {
_ => -1,
};
- let mut fields = vec![Arc::new(
+ let fields = vec![Arc::new(
Type::primitive_type_builder("leaf", T::get_physical_type())
.with_repetition(repetition)
.with_converted_type(converted_type)
@@ -1680,7 +1680,7 @@ mod tests {
let schema = Arc::new(
Type::group_type_builder("test_schema")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()
.unwrap(),
);
@@ -2026,7 +2026,7 @@ mod tests {
#[test]
fn test_dictionary_preservation() {
- let mut fields = vec![Arc::new(
+ let fields = vec![Arc::new(
Type::primitive_type_builder("leaf", PhysicalType::BYTE_ARRAY)
.with_repetition(Repetition::OPTIONAL)
.with_converted_type(ConvertedType::UTF8)
@@ -2036,7 +2036,7 @@ mod tests {
let schema = Arc::new(
Type::group_type_builder("test_schema")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()
.unwrap(),
);
diff --git a/parquet/src/arrow/schema/mod.rs b/parquet/src/arrow/schema/mod.rs
index cd6e8046cc..bcfc2f884c 100644
--- a/parquet/src/arrow/schema/mod.rs
+++ b/parquet/src/arrow/schema/mod.rs
@@ -37,7 +37,7 @@ use crate::basic::{
};
use crate::errors::{ParquetError, Result};
use crate::file::{metadata::KeyValue, properties::WriterProperties};
-use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type, TypePtr};
+use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type};
mod complex;
mod primitive;
@@ -230,13 +230,13 @@ pub(crate) fn add_encoded_arrow_schema_to_metadata(
/// Convert arrow schema to parquet schema
pub fn arrow_to_parquet_schema(schema: &Schema) -> Result<SchemaDescriptor> {
- let fields: Result<Vec<TypePtr>> = schema
+ let fields = schema
.fields()
.iter()
.map(|field| arrow_to_parquet_type(field).map(Arc::new))
- .collect();
+ .collect::<Result<_>>()?;
let group = Type::group_type_builder("arrow_schema")
- .with_fields(&mut fields?)
+ .with_fields(fields)
.build()?;
Ok(SchemaDescriptor::new(Arc::new(group)))
}
@@ -476,9 +476,9 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
}
DataType::List(f) | DataType::FixedSizeList(f, _) |
DataType::LargeList(f) => {
Type::group_type_builder(name)
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
Type::group_type_builder("list")
- .with_fields(&mut
vec![Arc::new(arrow_to_parquet_type(f)?)])
+ .with_fields(vec![Arc::new(arrow_to_parquet_type(f)?)])
.with_repetition(Repetition::REPEATED)
.build()?,
)])
@@ -493,21 +493,21 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
);
}
// recursively convert children to types/nodes
- let fields: Result<Vec<TypePtr>> = fields
+ let fields = fields
.iter()
.map(|f| arrow_to_parquet_type(f).map(Arc::new))
- .collect();
+ .collect::<Result<_>>()?;
Type::group_type_builder(name)
- .with_fields(&mut fields?)
+ .with_fields(fields)
.with_repetition(repetition)
.build()
}
DataType::Map(field, _) => {
if let DataType::Struct(struct_fields) = field.data_type() {
Type::group_type_builder(name)
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
Type::group_type_builder(field.name())
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(arrow_to_parquet_type(&Field::new(
struct_fields[0].name(),
struct_fields[0].data_type().clone(),
@@ -534,7 +534,7 @@ fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
DataType::Union(_, _) => unimplemented!("See ARROW-8817."),
DataType::Dictionary(_, ref value) => {
// Dictionary encoding not handled at the schema level
- let dict_field = Field::new(name, *value.clone(),
field.is_nullable());
+ let dict_field =
field.clone().with_data_type(value.as_ref().clone());
arrow_to_parquet_type(&dict_field)
}
DataType::RunEndEncoded(_, _) => Err(arrow_err!(
diff --git a/parquet/src/file/footer.rs b/parquet/src/file/footer.rs
index f4fb2534c2..21de63e0c2 100644
--- a/parquet/src/file/footer.rs
+++ b/parquet/src/file/footer.rs
@@ -184,7 +184,7 @@ mod tests {
#[test]
fn test_metadata_column_orders_parse() {
// Define simple schema, we do not need to provide logical types.
- let mut fields = vec![
+ let fields = vec![
Arc::new(
SchemaType::primitive_type_builder("col1", Type::INT32)
.build()
@@ -197,7 +197,7 @@ mod tests {
),
];
let schema = SchemaType::group_type_builder("schema")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()
.unwrap();
let schema_descr = SchemaDescriptor::new(Arc::new(schema));
diff --git a/parquet/src/file/metadata.rs b/parquet/src/file/metadata.rs
index a5e2de6b06..aaa3d28e20 100644
--- a/parquet/src/file/metadata.rs
+++ b/parquet/src/file/metadata.rs
@@ -1112,7 +1112,7 @@ mod tests {
/// Returns sample schema descriptor so we can create column metadata.
fn get_test_schema_descr() -> SchemaDescPtr {
let schema = SchemaType::group_type_builder("schema")
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
SchemaType::primitive_type_builder("a", Type::INT32)
.build()
diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs
index c31b9dc474..cafb176135 100644
--- a/parquet/src/file/writer.rs
+++ b/parquet/src/file/writer.rs
@@ -775,7 +775,7 @@ mod tests {
let file = tempfile::tempfile().unwrap();
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
types::Type::primitive_type_builder("col1", Type::INT32)
.build()
.unwrap(),
@@ -801,7 +801,7 @@ mod tests {
let file = tempfile::tempfile().unwrap();
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
types::Type::primitive_type_builder("col1",
Type::INT32)
.with_repetition(Repetition::REQUIRED)
@@ -848,7 +848,7 @@ mod tests {
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
types::Type::primitive_type_builder("col1", Type::INT32)
.build()
.unwrap(),
@@ -871,7 +871,7 @@ mod tests {
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
types::Type::primitive_type_builder("col1", Type::INT32)
.build()
.unwrap(),
@@ -920,7 +920,7 @@ mod tests {
);
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![field.clone()])
+ .with_fields(vec![field.clone()])
.build()
.unwrap(),
);
@@ -963,7 +963,7 @@ mod tests {
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
types::Type::primitive_type_builder("col1",
Type::INT32)
.build()
@@ -1310,7 +1310,7 @@ mod tests {
{
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
types::Type::primitive_type_builder("col1",
D::get_physical_type())
.with_repetition(Repetition::REQUIRED)
.build()
@@ -1468,7 +1468,7 @@ mod tests {
) {
let schema = Arc::new(
types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
types::Type::primitive_type_builder("col1", Type::INT32)
.with_repetition(Repetition::REQUIRED)
.build()
diff --git a/parquet/src/record/reader.rs b/parquet/src/record/reader.rs
index 780e982248..3416386c97 100644
--- a/parquet/src/record/reader.rs
+++ b/parquet/src/record/reader.rs
@@ -274,7 +274,7 @@ impl TreeBuilder {
let required_field = Type::group_type_builder(field.name())
.with_repetition(Repetition::REQUIRED)
.with_converted_type(field.get_basic_info().converted_type())
- .with_fields(&mut Vec::from(field.get_fields()))
+ .with_fields(field.get_fields().to_vec())
.build()?;
path.pop();
diff --git a/parquet/src/schema/mod.rs b/parquet/src/schema/mod.rs
index 1ebee2e06e..ead7f1d2c0 100644
--- a/parquet/src/schema/mod.rs
+++ b/parquet/src/schema/mod.rs
@@ -45,7 +45,7 @@
//! .unwrap();
//!
//! let schema = Type::group_type_builder("schema")
-//! .with_fields(&mut vec![Arc::new(field_a), Arc::new(field_b)])
+//! .with_fields(vec![Arc::new(field_a), Arc::new(field_b)])
//! .build()
//! .unwrap();
//!
diff --git a/parquet/src/schema/parser.rs b/parquet/src/schema/parser.rs
index c09f13603d..9af0f328a7 100644
--- a/parquet/src/schema/parser.rs
+++ b/parquet/src/schema/parser.rs
@@ -205,9 +205,8 @@ impl<'a> Parser<'a> {
.tokenizer
.next()
.ok_or_else(|| general_err!("Expected name, found None"))?;
- let mut fields = self.parse_child_types()?;
Type::group_type_builder(name)
- .with_fields(&mut fields)
+ .with_fields(self.parse_child_types()?)
.build()
}
_ => Err(general_err!("Message type does not start with
'message'")),
@@ -290,17 +289,14 @@ impl<'a> Parser<'a> {
None
};
- let mut fields = self.parse_child_types()?;
let mut builder = Type::group_type_builder(name)
.with_logical_type(logical_type)
.with_converted_type(converted_type)
- .with_fields(&mut fields);
+ .with_fields(self.parse_child_types()?)
+ .with_id(id);
if let Some(rep) = repetition {
builder = builder.with_repetition(rep);
}
- if let Some(id) = id {
- builder = builder.with_id(id);
- }
builder.build()
}
@@ -516,17 +512,15 @@ impl<'a> Parser<'a> {
};
assert_token(self.tokenizer.next(), ";")?;
- let mut builder = Type::primitive_type_builder(name, physical_type)
+ Type::primitive_type_builder(name, physical_type)
.with_repetition(repetition)
.with_logical_type(logical_type)
.with_converted_type(converted_type)
.with_length(length)
.with_precision(precision)
- .with_scale(scale);
- if let Some(id) = id {
- builder = builder.with_id(id);
- }
- builder.build()
+ .with_scale(scale)
+ .with_id(id)
+ .build()
}
}
@@ -845,7 +839,7 @@ mod tests {
let message = parse(schema).unwrap();
let expected = Type::group_type_builder("root")
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
Type::primitive_type_builder(
"f1",
@@ -906,16 +900,16 @@ mod tests {
let message = parse(schema).unwrap();
let expected = Type::group_type_builder("root")
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
Type::group_type_builder("a0")
.with_repetition(Repetition::REQUIRED)
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
Type::group_type_builder("a1")
.with_repetition(Repetition::OPTIONAL)
.with_logical_type(Some(LogicalType::List))
.with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
Type::primitive_type_builder(
"a2",
PhysicalType::BYTE_ARRAY,
@@ -933,10 +927,10 @@ mod tests {
.with_repetition(Repetition::OPTIONAL)
.with_logical_type(Some(LogicalType::List))
.with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(
+ .with_fields(vec![Arc::new(
Type::group_type_builder("b2")
.with_repetition(Repetition::REPEATED)
- .with_fields(&mut vec![
+ .with_fields(vec![
Arc::new(
Type::primitive_type_builder(
"b3",
@@ -984,7 +978,7 @@ mod tests {
";
let message = parse(schema).unwrap();
- let mut fields = vec![
+ let fields = vec![
Arc::new(
Type::primitive_type_builder("_1", PhysicalType::INT32)
.with_repetition(Repetition::REQUIRED)
@@ -1027,7 +1021,7 @@ mod tests {
];
let expected = Type::group_type_builder("root")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()
.unwrap();
assert_eq!(message, expected);
@@ -1051,7 +1045,7 @@ mod tests {
";
let message = parse(schema).unwrap();
- let mut fields = vec![
+ let fields = vec![
Arc::new(
Type::primitive_type_builder("_1", PhysicalType::INT32)
.with_repetition(Repetition::REQUIRED)
@@ -1135,7 +1129,7 @@ mod tests {
];
let expected = Type::group_type_builder("root")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()
.unwrap();
assert_eq!(message, expected);
diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs
index ad4acb0cb8..12624513ac 100644
--- a/parquet/src/schema/printer.rs
+++ b/parquet/src/schema/printer.rs
@@ -695,40 +695,40 @@ mod tests {
let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
.with_repetition(Repetition::REQUIRED)
.with_converted_type(ConvertedType::INT_32)
- .with_id(0)
+ .with_id(Some(0))
.build();
let f2 = Type::primitive_type_builder("f2",
PhysicalType::BYTE_ARRAY)
.with_converted_type(ConvertedType::UTF8)
- .with_id(1)
+ .with_id(Some(1))
.build();
let f3 = Type::primitive_type_builder("f3",
PhysicalType::BYTE_ARRAY)
.with_logical_type(Some(LogicalType::String))
- .with_id(1)
+ .with_id(Some(1))
.build();
let f4 =
Type::primitive_type_builder("f4",
PhysicalType::FIXED_LEN_BYTE_ARRAY)
.with_repetition(Repetition::REPEATED)
.with_converted_type(ConvertedType::INTERVAL)
.with_length(12)
- .with_id(2)
+ .with_id(Some(2))
.build();
- let mut struct_fields = vec![
+ let struct_fields = vec![
Arc::new(f1.unwrap()),
Arc::new(f2.unwrap()),
Arc::new(f3.unwrap()),
];
let field = Type::group_type_builder("field")
.with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut struct_fields)
- .with_id(1)
+ .with_fields(struct_fields)
+ .with_id(Some(1))
.build()
.unwrap();
- let mut fields = vec![Arc::new(field), Arc::new(f4.unwrap())];
+ let fields = vec![Arc::new(field), Arc::new(f4.unwrap())];
let message = Type::group_type_builder("schema")
- .with_fields(&mut fields)
- .with_id(2)
+ .with_fields(fields)
+ .with_id(Some(2))
.build()
.unwrap();
p.print(&message);
@@ -756,7 +756,7 @@ mod tests {
.with_repetition(Repetition::OPTIONAL)
.with_logical_type(Some(LogicalType::List))
.with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(a2)])
+ .with_fields(vec![Arc::new(a2)])
.build()
.unwrap();
@@ -773,7 +773,7 @@ mod tests {
let b2 = Type::group_type_builder("b2")
.with_repetition(Repetition::REPEATED)
.with_converted_type(ConvertedType::NONE)
- .with_fields(&mut vec![Arc::new(b3), Arc::new(b4)])
+ .with_fields(vec![Arc::new(b3), Arc::new(b4)])
.build()
.unwrap();
@@ -781,18 +781,18 @@ mod tests {
.with_repetition(Repetition::OPTIONAL)
.with_logical_type(Some(LogicalType::List))
.with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(b2)])
+ .with_fields(vec![Arc::new(b2)])
.build()
.unwrap();
let a0 = Type::group_type_builder("a0")
.with_repetition(Repetition::REQUIRED)
- .with_fields(&mut vec![Arc::new(a1), Arc::new(b1)])
+ .with_fields(vec![Arc::new(a1), Arc::new(b1)])
.build()
.unwrap();
let message = Type::group_type_builder("root")
- .with_fields(&mut vec![Arc::new(a0)])
+ .with_fields(vec![Arc::new(a0)])
.build()
.unwrap();
@@ -815,7 +815,7 @@ mod tests {
let field = Type::group_type_builder("field")
.with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut vec![Arc::new(f1), Arc::new(f2)])
+ .with_fields(vec![Arc::new(f1), Arc::new(f2)])
.build()
.unwrap();
@@ -827,7 +827,7 @@ mod tests {
.unwrap();
let message = Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(field), Arc::new(f3)])
+ .with_fields(vec![Arc::new(field), Arc::new(f3)])
.build()
.unwrap();
@@ -861,7 +861,7 @@ mod tests {
.unwrap();
let message = Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(f1), Arc::new(f2)])
+ .with_fields(vec![Arc::new(f1), Arc::new(f2)])
.build()
.unwrap();
diff --git a/parquet/src/schema/types.rs b/parquet/src/schema/types.rs
index fd22cedeac..bed85268ff 100644
--- a/parquet/src/schema/types.rs
+++ b/parquet/src/schema/types.rs
@@ -220,52 +220,51 @@ impl<'a> PrimitiveTypeBuilder<'a> {
}
/// Sets [`Repetition`](crate::basic::Repetition) for this field and
returns itself.
- pub fn with_repetition(mut self, repetition: Repetition) -> Self {
- self.repetition = repetition;
- self
+ pub fn with_repetition(self, repetition: Repetition) -> Self {
+ Self { repetition, ..self }
}
/// Sets [`ConvertedType`](crate::basic::ConvertedType) for this field and
returns itself.
- pub fn with_converted_type(mut self, converted_type: ConvertedType) ->
Self {
- self.converted_type = converted_type;
- self
+ pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
+ Self {
+ converted_type,
+ ..self
+ }
}
/// Sets [`LogicalType`](crate::basic::LogicalType) for this field and
returns itself.
/// If only the logical type is populated for a primitive type, the
converted type
/// will be automatically populated, and can thus be omitted.
- pub fn with_logical_type(mut self, logical_type: Option<LogicalType>) ->
Self {
- self.logical_type = logical_type;
- self
+ pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
+ Self {
+ logical_type,
+ ..self
+ }
}
/// Sets type length and returns itself.
/// This is only applied to FIXED_LEN_BYTE_ARRAY and INT96 (INTERVAL)
types, because
/// they maintain fixed size underlying byte array.
/// By default, value is `0`.
- pub fn with_length(mut self, length: i32) -> Self {
- self.length = length;
- self
+ pub fn with_length(self, length: i32) -> Self {
+ Self { length, ..self }
}
/// Sets precision for Parquet DECIMAL physical type and returns itself.
/// By default, it equals to `0` and used only for decimal context.
- pub fn with_precision(mut self, precision: i32) -> Self {
- self.precision = precision;
- self
+ pub fn with_precision(self, precision: i32) -> Self {
+ Self { precision, ..self }
}
/// Sets scale for Parquet DECIMAL physical type and returns itself.
/// By default, it equals to `0` and used only for decimal context.
- pub fn with_scale(mut self, scale: i32) -> Self {
- self.scale = scale;
- self
+ pub fn with_scale(self, scale: i32) -> Self {
+ Self { scale, ..self }
}
/// Sets optional field id and returns itself.
- pub fn with_id(mut self, id: i32) -> Self {
- self.id = Some(id);
- self
+ pub fn with_id(self, id: Option<i32>) -> Self {
+ Self { id, ..self }
}
/// Creates a new `PrimitiveType` instance from the collected attributes.
@@ -560,28 +559,30 @@ impl<'a> GroupTypeBuilder<'a> {
}
/// Sets [`ConvertedType`](crate::basic::ConvertedType) for this field and
returns itself.
- pub fn with_converted_type(mut self, converted_type: ConvertedType) ->
Self {
- self.converted_type = converted_type;
- self
+ pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
+ Self {
+ converted_type,
+ ..self
+ }
}
/// Sets [`LogicalType`](crate::basic::LogicalType) for this field and
returns itself.
- pub fn with_logical_type(mut self, logical_type: Option<LogicalType>) ->
Self {
- self.logical_type = logical_type;
- self
+ pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
+ Self {
+ logical_type,
+ ..self
+ }
}
/// Sets a list of fields that should be child nodes of this field.
/// Returns updated self.
- pub fn with_fields(mut self, fields: &mut Vec<TypePtr>) -> Self {
- self.fields.append(fields);
- self
+ pub fn with_fields(self, fields: Vec<TypePtr>) -> Self {
+ Self { fields, ..self }
}
/// Sets optional field id and returns itself.
- pub fn with_id(mut self, id: i32) -> Self {
- self.id = Some(id);
- self
+ pub fn with_id(self, id: Option<i32>) -> Self {
+ Self { id, ..self }
}
/// Creates a new `GroupType` instance from the gathered attributes.
@@ -1093,16 +1094,14 @@ fn from_thrift_helper(
let scale = elements[index].scale.unwrap_or(-1);
let precision = elements[index].precision.unwrap_or(-1);
let name = &elements[index].name;
- let mut builder = Type::primitive_type_builder(name, physical_type)
+ let builder = Type::primitive_type_builder(name, physical_type)
.with_repetition(repetition)
.with_converted_type(converted_type)
.with_logical_type(logical_type)
.with_length(length)
.with_precision(precision)
- .with_scale(scale);
- if let Some(id) = field_id {
- builder = builder.with_id(id);
- }
+ .with_scale(scale)
+ .with_id(field_id);
Ok((index + 1, Arc::new(builder.build()?)))
}
Some(n) => {
@@ -1122,7 +1121,8 @@ fn from_thrift_helper(
let mut builder = Type::group_type_builder(&elements[index].name)
.with_converted_type(converted_type)
.with_logical_type(logical_type)
- .with_fields(&mut fields);
+ .with_fields(fields)
+ .with_id(field_id);
if let Some(rep) = repetition {
// Sometimes parquet-cpp and parquet-mr set repetition level
REQUIRED or
// REPEATED for root node.
@@ -1135,9 +1135,6 @@ fn from_thrift_helper(
builder = builder.with_repetition(rep);
}
}
- if let Some(id) = field_id {
- builder = builder.with_id(id);
- }
Ok((next_index, Arc::new(builder.build().unwrap())))
}
}
@@ -1243,7 +1240,7 @@ mod tests {
bit_width: 32,
is_signed: true,
}))
- .with_id(0)
+ .with_id(Some(0))
.build();
assert!(result.is_ok());
@@ -1525,22 +1522,22 @@ mod tests {
fn test_group_type() {
let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
.with_converted_type(ConvertedType::INT_32)
- .with_id(0)
+ .with_id(Some(0))
.build();
assert!(f1.is_ok());
let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
.with_converted_type(ConvertedType::UTF8)
- .with_id(1)
+ .with_id(Some(1))
.build();
assert!(f2.is_ok());
- let mut fields = vec![Arc::new(f1.unwrap()), Arc::new(f2.unwrap())];
+ let fields = vec![Arc::new(f1.unwrap()), Arc::new(f2.unwrap())];
let result = Type::group_type_builder("foo")
.with_repetition(Repetition::REPEATED)
.with_logical_type(Some(LogicalType::List))
- .with_fields(&mut fields)
- .with_id(1)
+ .with_fields(fields)
+ .with_id(Some(1))
.build();
assert!(result.is_ok());
@@ -1630,17 +1627,17 @@ mod tests {
let list = Type::group_type_builder("records")
.with_repetition(Repetition::REPEATED)
.with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(item1), Arc::new(item2),
Arc::new(item3)])
+ .with_fields(vec![Arc::new(item1), Arc::new(item2),
Arc::new(item3)])
.build()?;
let bag = Type::group_type_builder("bag")
.with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut vec![Arc::new(list)])
+ .with_fields(vec![Arc::new(list)])
.build()?;
fields.push(Arc::new(bag));
let schema = Type::group_type_builder("schema")
.with_repetition(Repetition::REPEATED)
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()?;
let descr = SchemaDescriptor::new(Arc::new(schema));
@@ -1789,13 +1786,9 @@ mod tests {
// function to create a new group type for testing
fn test_new_group_type(name: &str, repetition: Repetition, types:
Vec<Type>) -> Type {
- let mut fields = Vec::new();
- for tpe in types {
- fields.push(Arc::new(tpe))
- }
Type::group_type_builder(name)
.with_repetition(repetition)
- .with_fields(&mut fields)
+ .with_fields(types.into_iter().map(Arc::new).collect())
.build()
.unwrap()
}
diff --git a/parquet_derive/src/lib.rs b/parquet_derive/src/lib.rs
index 0f875401f0..c6641cd809 100644
--- a/parquet_derive/src/lib.rs
+++ b/parquet_derive/src/lib.rs
@@ -130,7 +130,7 @@ pub fn parquet_record_writer(input:
proc_macro::TokenStream) -> proc_macro::Toke
#field_types
);*;
let group = ParquetType::group_type_builder("rust_schema")
- .with_fields(&mut fields)
+ .with_fields(fields)
.build()?;
Ok(group.into())
}
