sdf-jkl commented on code in PR #11333:
URL: https://github.com/apache/arrow-rs/pull/11333#discussion_r4648667303
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -153,6 +165,148 @@ impl From<EncodingArgs> for Encoding {
}
}
+/// Parses a `--column-encoding` value of the form `<COLUMN>=<ENCODING>`.
+///
+/// The value is split at the last `=`, as encoding names never contain one.
+fn parse_column_encoding(value: &str) -> Result<(String, EncodingArgs),
String> {
+ let (column, encoding) = value
+ .rsplit_once('=')
+ .ok_or_else(|| format!("expected <COLUMN>=<ENCODING>, got
'{value}'"))?;
+ if column.is_empty() {
+ return Err(format!("missing column in '{value}'"));
+ }
+ let encoding = EncodingArgs::from_str(encoding, true).map_err(|_| {
+ let possible_values: Vec<_> = EncodingArgs::value_variants()
+ .iter()
+ .filter_map(|v| v.to_possible_value())
+ .map(|v| v.get_name().to_string())
+ .collect();
+ format!(
+ "invalid encoding '{encoding}' in '{value}', possible values: {}",
+ possible_values.join(", ")
+ )
+ })?;
+ Ok((column.to_string(), encoding))
+}
+
+/// Checks that `encoding` can be set as the encoding of a column at all.
+fn check_encoding_settable(encoding: EncodingArgs) -> Result<(), String> {
+ match encoding {
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary =>
Err(format!(
+ "{} cannot be set as a column encoding, use --dictionary-enabled
instead",
+ Encoding::from(encoding)
+ )),
+ EncodingArgs::BitPacked => Err(format!(
+ "{} is deprecated and not supported for writing",
+ Encoding::from(encoding)
+ )),
+ _ => Ok(()),
+ }
+}
+
+/// Checks that the writer supports `encoding` for columns of `physical_type`.
+///
+/// This mirrors the encoders that the writer can create, so that an
+/// unsupported combination is reported before the output file is created
+/// instead of failing part way through the rewrite.
+fn check_encoding(encoding: EncodingArgs, physical_type: PhysicalType) ->
Result<(), String> {
+ use PhysicalType::*;
+ let supported: &[PhysicalType] = match encoding {
+ EncodingArgs::Plain => return Ok(()),
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary |
EncodingArgs::BitPacked => {
+ return check_encoding_settable(encoding);
+ }
+ EncodingArgs::Rle => &[BOOLEAN],
+ EncodingArgs::DeltaBinaryPacked => &[INT32, INT64],
+ EncodingArgs::DeltaLengthByteArray => &[BYTE_ARRAY],
+ EncodingArgs::DeltaByteArray => &[BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY],
+ EncodingArgs::ByteStreamSplit => &[INT32, INT64, FLOAT, DOUBLE,
FIXED_LEN_BYTE_ARRAY],
+ EncodingArgs::Alp => &[FLOAT, DOUBLE],
+ };
+ if supported.contains(&physical_type) {
+ return Ok(());
+ }
+ let mut types: Vec<_> = supported.iter().map(|t| t.to_string()).collect();
+ let last = types.pop().unwrap();
+ let types = match types.is_empty() {
+ true => last,
+ false => format!("{} and {last}", types.join(", ")),
+ };
+ let encoding = Encoding::from(encoding);
+ Err(format!("{encoding} supports only {types}"))
+}
Review Comment:
`check_encoding` can take the library `Encoding` (what
`WriterProperties::encoding` returns, see the comment on the encoding setup)
and the column, and build the whole message. `{supported:?}` replaces the "X, Y
and Z" formatting. `check_encoding_settable` stays as it is, it is called at
parse time and once for `--encoding` (see the other comments).
```suggestion
/// Checks that `encoding` can be set as the encoding of a column at all.
fn check_encoding_settable(encoding: EncodingArgs) -> Result<(), String> {
match encoding {
EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary =>
Err(format!(
"{} cannot be set as a column encoding, use --dictionary-enabled
instead",
Encoding::from(encoding)
)),
EncodingArgs::BitPacked => Err(format!(
"{} is deprecated and not supported for writing",
Encoding::from(encoding)
)),
_ => Ok(()),
}
}
/// Rejects encodings the writer cannot write for `column`. The writer
panics on
/// them instead of returning an error (#10964).
fn check_encoding(column: &ColumnDescriptor, encoding: Encoding) ->
Result<(), String> {
use PhysicalType::*;
let supported: &[PhysicalType] = match encoding {
Encoding::RLE => &[BOOLEAN],
Encoding::DELTA_BINARY_PACKED => &[INT32, INT64],
Encoding::DELTA_LENGTH_BYTE_ARRAY => &[BYTE_ARRAY],
Encoding::DELTA_BYTE_ARRAY => &[BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY],
Encoding::BYTE_STREAM_SPLIT => &[INT32, INT64, FLOAT, DOUBLE,
FIXED_LEN_BYTE_ARRAY],
Encoding::ALP => &[FLOAT, DOUBLE],
_ => return Ok(()),
};
if supported.contains(&column.physical_type()) {
return Ok(());
}
Err(format!(
"column '{}' ({}): {encoding} supports only {supported:?}",
column.path().string(),
column.physical_type()
))
}
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -153,6 +165,148 @@ impl From<EncodingArgs> for Encoding {
}
}
+/// Parses a `--column-encoding` value of the form `<COLUMN>=<ENCODING>`.
+///
+/// The value is split at the last `=`, as encoding names never contain one.
+fn parse_column_encoding(value: &str) -> Result<(String, EncodingArgs),
String> {
+ let (column, encoding) = value
+ .rsplit_once('=')
+ .ok_or_else(|| format!("expected <COLUMN>=<ENCODING>, got
'{value}'"))?;
+ if column.is_empty() {
+ return Err(format!("missing column in '{value}'"));
+ }
+ let encoding = EncodingArgs::from_str(encoding, true).map_err(|_| {
+ let possible_values: Vec<_> = EncodingArgs::value_variants()
+ .iter()
+ .filter_map(|v| v.to_possible_value())
+ .map(|v| v.get_name().to_string())
+ .collect();
+ format!(
+ "invalid encoding '{encoding}' in '{value}', possible values: {}",
+ possible_values.join(", ")
+ )
+ })?;
+ Ok((column.to_string(), encoding))
+}
Review Comment:
Two small things here: `false` makes this case sensitive like `--encoding`
(today `--encoding PLAIN` is rejected but `--column-encoding id=PLAIN` is
accepted), and the empty-column check can go because `=alp` is already reported
as an unknown column `''`. clap also prints the full value in its error, so the
`got '{value}'` part repeats it. Calling `check_encoding_settable` here makes
`x=rle-dictionary` a normal clap error, instead of a panic in
`set_column_encoding`.
```suggestion
/// Parses a `--column-encoding` value of the form `<COLUMN>=<ENCODING>`.
fn parse_column_encoding(value: &str) -> Result<(String, EncodingArgs),
String> {
let (column, encoding) = value
.rsplit_once('=')
.ok_or("expected <COLUMN>=<ENCODING>")?;
let encoding = EncodingArgs::from_str(encoding, false).map_err(|_| {
let possible_values: Vec<_> = EncodingArgs::value_variants()
.iter()
.filter_map(|v| v.to_possible_value())
.map(|v| v.get_name().to_string())
.collect();
format!(
"invalid encoding '{encoding}' in '{value}', possible values:
{}",
possible_values.join(", ")
)
})?;
check_encoding_settable(encoding)?;
Ok((column.to_string(), encoding))
}
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -348,10 +522,19 @@ fn main() {
writer_properties_builder =
writer_properties_builder.set_compression(compression);
}
- // setup encoding
+ // setup encoding, checking it against the schema that the writer will use
+ let output_schema = ArrowSchemaConverter::new()
+ .with_coerce_types(args.coerce_types.unwrap_or(DEFAULT_COERCE_TYPES))
+ .convert(&parquet_reader.schema())
+ .expect("convert schema");
+ let column_encodings =
+ resolve_column_encodings(&output_schema, args.encoding,
&args.column_encoding)?;
if let Some(value) = args.encoding {
writer_properties_builder =
writer_properties_builder.set_encoding(value.into());
}
+ for (column, encoding) in column_encodings {
+ writer_properties_builder =
writer_properties_builder.set_column_encoding(column, encoding);
+ }
Review Comment:
Here only the column names need the schema. The check itself can run on the
built properties, so the override precedence comes from `WriterProperties`
instead of being computed here. That part is also outside the diff, right after
`let writer_properties = writer_properties_builder.build();`:
```rust
let errors: Vec<_> = output_schema
.columns()
.iter()
.filter_map(|c| check_encoding(c,
writer_properties.encoding(c.path())?).err())
.collect();
if !errors.is_empty() {
return Err(errors.join("\n"));
}
```
It is still before `File::create`, so a bad combination still leaves no
output file. `check_encoding_settable` has to run before `set_encoding`,
because the builder panics on the dictionary encodings. I dropped the list of
leaf columns from the unknown-column error because on a wide file it gets very
long.
```suggestion
// setup encoding, resolving columns against the schema that the writer
will use
let output_schema = ArrowSchemaConverter::new()
.with_coerce_types(args.coerce_types.unwrap_or(DEFAULT_COERCE_TYPES))
.convert(&parquet_reader.schema())
.expect("convert schema");
if let Some(value) = args.encoding {
check_encoding_settable(value).map_err(|e| format!("--encoding:
{e}"))?;
writer_properties_builder =
writer_properties_builder.set_encoding(value.into());
}
for (name, encoding) in &args.column_encoding {
let column = output_schema
.columns()
.iter()
.find(|c| c.path().string() == *name)
.ok_or_else(|| format!("unknown column '{name}' in
--column-encoding"))?;
writer_properties_builder = writer_properties_builder
.set_column_encoding(column.path().clone(), (*encoding).into());
}
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -435,4 +618,389 @@ fn main() {
}
parquet_writer.close().expect("finalizing file");
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::path::Path;
+ use std::sync::Arc;
+
+ use arrow_array::{
+ ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array,
Float64Array, Int32Array,
+ Int64Array, RecordBatch, StringArray,
+ };
+ use parquet::schema::parser::parse_message_type;
+
+ const ALL_ENCODINGS: [EncodingArgs; 10] = [
+ EncodingArgs::Plain,
+ EncodingArgs::PlainDictionary,
+ EncodingArgs::Rle,
+ EncodingArgs::BitPacked,
+ EncodingArgs::DeltaBinaryPacked,
+ EncodingArgs::DeltaLengthByteArray,
+ EncodingArgs::DeltaByteArray,
+ EncodingArgs::RleDictionary,
+ EncodingArgs::ByteStreamSplit,
+ EncodingArgs::Alp,
+ ];
+
+ fn schema() -> SchemaDescriptor {
+ let message = "
+ message schema {
+ REQUIRED BOOLEAN b;
+ REQUIRED INT32 i;
+ REQUIRED FLOAT f;
+ REQUIRED BYTE_ARRAY s (UTF8);
+ OPTIONAL group g {
+ REQUIRED DOUBLE d;
+ }
+ }";
+ SchemaDescriptor::new(Arc::new(parse_message_type(message).unwrap()))
+ }
+
+ fn column_encodings(values: &[&str]) -> Vec<(String, EncodingArgs)> {
+ values
+ .iter()
+ .map(|v| parse_column_encoding(v).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn test_parse_column_encoding() {
+ let parse = |v| parse_column_encoding(v).unwrap();
+ assert_eq!(parse("price=alp"), ("price".to_string(),
EncodingArgs::Alp));
+ assert_eq!(parse("a.b=PLAIN"), ("a.b".to_string(),
EncodingArgs::Plain));
+ assert_eq!(
+ parse("x=delta-binary-packed"),
+ ("x".to_string(), EncodingArgs::DeltaBinaryPacked)
+ );
+ // Only the last `=` separates the column from the encoding
+ assert_eq!(parse("a=b=plain"), ("a=b".to_string(),
EncodingArgs::Plain));
+
+ let err = |v| parse_column_encoding(v).unwrap_err();
+ assert_eq!(err("price"), "expected <COLUMN>=<ENCODING>, got 'price'");
+ assert_eq!(err("=alp"), "missing column in '=alp'");
+ assert!(
+ err("price=snappy").starts_with(
+ "invalid encoding 'snappy' in 'price=snappy', possible values:
plain, "
+ ),
+ "{}",
+ err("price=snappy")
+ );
+ assert!(err("price=").starts_with("invalid encoding '' in 'price='"));
+ }
+
+ #[test]
+ fn test_parse_args_column_encoding() {
+ let args = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--encoding",
+ "plain",
+ "--column-encoding",
+ "f=alp",
+ "--column-encoding",
+ "g.d=byte-stream-split",
+ ])
+ .unwrap();
+ assert_eq!(args.encoding, Some(EncodingArgs::Plain));
+ assert_eq!(
+ args.column_encoding,
+ column_encodings(&["f=alp", "g.d=byte-stream-split"])
+ );
+
+ let err = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--column-encoding",
+ "f",
+ ])
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ValueValidation);
+ }
+
+ #[test]
+ fn test_check_encoding_messages() {
+ assert_eq!(
+ check_encoding(EncodingArgs::Alp,
PhysicalType::INT32).unwrap_err(),
+ "ALP supports only FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::Rle,
PhysicalType::INT32).unwrap_err(),
+ "RLE supports only BOOLEAN"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::ByteStreamSplit,
PhysicalType::BYTE_ARRAY).unwrap_err(),
+ "BYTE_STREAM_SPLIT supports only INT32, INT64, FLOAT, DOUBLE and
FIXED_LEN_BYTE_ARRAY"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::RleDictionary,
PhysicalType::INT32).unwrap_err(),
+ "RLE_DICTIONARY cannot be set as a column encoding, use
--dictionary-enabled instead"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::BitPacked,
PhysicalType::INT32).unwrap_err(),
+ "BIT_PACKED is deprecated and not supported for writing"
+ );
+ for physical_type in [PhysicalType::INT96, PhysicalType::BYTE_ARRAY] {
+ assert!(check_encoding(EncodingArgs::Plain,
physical_type).is_ok());
+ }
+ }
+
+ /// Returns whether the writer can write `array` with `encoding`.
+ ///
+ /// Some unsupported encodings make the column writer panic rather than
+ /// return an error, so panics count as unsupported too.
+ fn writer_supports(array: ArrayRef, encoding: Encoding) -> bool {
+ let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .set_encoding(encoding)
+ .build();
+ let write = || {
+ let mut writer = ArrowWriter::try_new(vec![], batch.schema(),
Some(props))?;
+ writer.write(&batch)?;
+ writer.close()
+ };
+ matches!(
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(write)),
+ Ok(Ok(_))
+ )
+ }
+
+ /// `check_encoding` must agree with what the writer actually supports
+ #[test]
+ fn test_check_encoding_matches_writer() {
+ let arrays: [(PhysicalType, ArrayRef); 7] = [
+ (
+ PhysicalType::BOOLEAN,
+ Arc::new(BooleanArray::from(vec![true, false])),
+ ),
+ (PhysicalType::INT32, Arc::new(Int32Array::from(vec![1, 2]))),
+ (PhysicalType::INT64, Arc::new(Int64Array::from(vec![1, 2]))),
+ (
+ PhysicalType::FLOAT,
+ Arc::new(Float32Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::DOUBLE,
+ Arc::new(Float64Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::BYTE_ARRAY,
+ Arc::new(StringArray::from(vec!["a", "b"])),
+ ),
+ (
+ PhysicalType::FIXED_LEN_BYTE_ARRAY,
+ Arc::new(FixedSizeBinaryArray::try_from_iter([b"ab",
b"cd"].into_iter()).unwrap()),
+ ),
+ ];
+ for (physical_type, array) in arrays {
+ for encoding in ALL_ENCODINGS {
+ let checked = check_encoding(encoding, physical_type);
+ if matches!(
+ encoding,
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary
+ ) {
+ // The writer properties builder panics on these
+ assert!(checked.is_err());
+ continue;
+ }
+ assert_eq!(
+ checked.is_ok(),
+ writer_supports(array.clone(), encoding.into()),
+ "{encoding:?} for {physical_type}: {checked:?}"
+ );
+ }
+ }
+ }
Review Comment:
This test is the important one, it keeps the table honest. It can get the
physical type from `ArrowSchemaConverter` instead of pairing each array with a
type by hand, and loop over `EncodingArgs::value_variants()`. Encodings that
`check_encoding_settable` rejects never reach the writer, so the test can skip
them. (I checked that it fails when an entry in the table is wrong, e.g. ALP
with only `[DOUBLE]`.)
```suggestion
#[test]
fn test_check_encoding_matches_writer() {
let arrays: [ArrayRef; 7] = [
Arc::new(BooleanArray::from(vec![true, false])),
Arc::new(Int32Array::from(vec![1, 2])),
Arc::new(Int64Array::from(vec![1, 2])),
Arc::new(Float32Array::from(vec![1.5, 2.5])),
Arc::new(Float64Array::from(vec![1.5, 2.5])),
Arc::new(StringArray::from(vec!["a", "b"])),
Arc::new(FixedSizeBinaryArray::try_from_iter([b"ab",
b"cd"].into_iter()).unwrap()),
];
for array in arrays {
let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
let schema = ArrowSchemaConverter::new()
.convert(&batch.schema())
.unwrap();
for encoding in EncodingArgs::value_variants() {
if check_encoding_settable(*encoding).is_err() {
continue;
}
let props = WriterProperties::builder()
.set_dictionary_enabled(false)
.set_encoding((*encoding).into())
.build();
let write = || {
let mut writer = ArrowWriter::try_new(vec![],
batch.schema(), Some(props))?;
writer.write(&batch)?;
writer.close()
};
let written = matches!(
std::panic::catch_unwind(std::panic::AssertUnwindSafe(write)),
Ok(Ok(_))
);
let checked = check_encoding(schema.column(0).as_ref(),
(*encoding).into());
assert_eq!(checked.is_ok(), written, "{encoding:?}:
{checked:?}");
}
}
}
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -435,4 +618,389 @@ fn main() {
}
parquet_writer.close().expect("finalizing file");
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::path::Path;
+ use std::sync::Arc;
+
+ use arrow_array::{
+ ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array,
Float64Array, Int32Array,
+ Int64Array, RecordBatch, StringArray,
+ };
+ use parquet::schema::parser::parse_message_type;
+
+ const ALL_ENCODINGS: [EncodingArgs; 10] = [
+ EncodingArgs::Plain,
+ EncodingArgs::PlainDictionary,
+ EncodingArgs::Rle,
+ EncodingArgs::BitPacked,
+ EncodingArgs::DeltaBinaryPacked,
+ EncodingArgs::DeltaLengthByteArray,
+ EncodingArgs::DeltaByteArray,
+ EncodingArgs::RleDictionary,
+ EncodingArgs::ByteStreamSplit,
+ EncodingArgs::Alp,
+ ];
+
+ fn schema() -> SchemaDescriptor {
+ let message = "
+ message schema {
+ REQUIRED BOOLEAN b;
+ REQUIRED INT32 i;
+ REQUIRED FLOAT f;
+ REQUIRED BYTE_ARRAY s (UTF8);
+ OPTIONAL group g {
+ REQUIRED DOUBLE d;
+ }
+ }";
+ SchemaDescriptor::new(Arc::new(parse_message_type(message).unwrap()))
+ }
+
+ fn column_encodings(values: &[&str]) -> Vec<(String, EncodingArgs)> {
+ values
+ .iter()
+ .map(|v| parse_column_encoding(v).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn test_parse_column_encoding() {
+ let parse = |v| parse_column_encoding(v).unwrap();
+ assert_eq!(parse("price=alp"), ("price".to_string(),
EncodingArgs::Alp));
+ assert_eq!(parse("a.b=PLAIN"), ("a.b".to_string(),
EncodingArgs::Plain));
+ assert_eq!(
+ parse("x=delta-binary-packed"),
+ ("x".to_string(), EncodingArgs::DeltaBinaryPacked)
+ );
+ // Only the last `=` separates the column from the encoding
+ assert_eq!(parse("a=b=plain"), ("a=b".to_string(),
EncodingArgs::Plain));
+
+ let err = |v| parse_column_encoding(v).unwrap_err();
+ assert_eq!(err("price"), "expected <COLUMN>=<ENCODING>, got 'price'");
+ assert_eq!(err("=alp"), "missing column in '=alp'");
+ assert!(
+ err("price=snappy").starts_with(
+ "invalid encoding 'snappy' in 'price=snappy', possible values:
plain, "
+ ),
+ "{}",
+ err("price=snappy")
+ );
+ assert!(err("price=").starts_with("invalid encoding '' in 'price='"));
+ }
+
+ #[test]
+ fn test_parse_args_column_encoding() {
+ let args = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--encoding",
+ "plain",
+ "--column-encoding",
+ "f=alp",
+ "--column-encoding",
+ "g.d=byte-stream-split",
+ ])
+ .unwrap();
+ assert_eq!(args.encoding, Some(EncodingArgs::Plain));
+ assert_eq!(
+ args.column_encoding,
+ column_encodings(&["f=alp", "g.d=byte-stream-split"])
+ );
+
+ let err = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--column-encoding",
+ "f",
+ ])
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ValueValidation);
+ }
+
+ #[test]
+ fn test_check_encoding_messages() {
+ assert_eq!(
+ check_encoding(EncodingArgs::Alp,
PhysicalType::INT32).unwrap_err(),
+ "ALP supports only FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::Rle,
PhysicalType::INT32).unwrap_err(),
+ "RLE supports only BOOLEAN"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::ByteStreamSplit,
PhysicalType::BYTE_ARRAY).unwrap_err(),
+ "BYTE_STREAM_SPLIT supports only INT32, INT64, FLOAT, DOUBLE and
FIXED_LEN_BYTE_ARRAY"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::RleDictionary,
PhysicalType::INT32).unwrap_err(),
+ "RLE_DICTIONARY cannot be set as a column encoding, use
--dictionary-enabled instead"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::BitPacked,
PhysicalType::INT32).unwrap_err(),
+ "BIT_PACKED is deprecated and not supported for writing"
+ );
+ for physical_type in [PhysicalType::INT96, PhysicalType::BYTE_ARRAY] {
+ assert!(check_encoding(EncodingArgs::Plain,
physical_type).is_ok());
+ }
+ }
+
+ /// Returns whether the writer can write `array` with `encoding`.
+ ///
+ /// Some unsupported encodings make the column writer panic rather than
+ /// return an error, so panics count as unsupported too.
+ fn writer_supports(array: ArrayRef, encoding: Encoding) -> bool {
+ let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .set_encoding(encoding)
+ .build();
+ let write = || {
+ let mut writer = ArrowWriter::try_new(vec![], batch.schema(),
Some(props))?;
+ writer.write(&batch)?;
+ writer.close()
+ };
+ matches!(
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(write)),
+ Ok(Ok(_))
+ )
+ }
+
+ /// `check_encoding` must agree with what the writer actually supports
+ #[test]
+ fn test_check_encoding_matches_writer() {
+ let arrays: [(PhysicalType, ArrayRef); 7] = [
+ (
+ PhysicalType::BOOLEAN,
+ Arc::new(BooleanArray::from(vec![true, false])),
+ ),
+ (PhysicalType::INT32, Arc::new(Int32Array::from(vec![1, 2]))),
+ (PhysicalType::INT64, Arc::new(Int64Array::from(vec![1, 2]))),
+ (
+ PhysicalType::FLOAT,
+ Arc::new(Float32Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::DOUBLE,
+ Arc::new(Float64Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::BYTE_ARRAY,
+ Arc::new(StringArray::from(vec!["a", "b"])),
+ ),
+ (
+ PhysicalType::FIXED_LEN_BYTE_ARRAY,
+ Arc::new(FixedSizeBinaryArray::try_from_iter([b"ab",
b"cd"].into_iter()).unwrap()),
+ ),
+ ];
+ for (physical_type, array) in arrays {
+ for encoding in ALL_ENCODINGS {
+ let checked = check_encoding(encoding, physical_type);
+ if matches!(
+ encoding,
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary
+ ) {
+ // The writer properties builder panics on these
+ assert!(checked.is_err());
+ continue;
+ }
+ assert_eq!(
+ checked.is_ok(),
+ writer_supports(array.clone(), encoding.into()),
+ "{encoding:?} for {physical_type}: {checked:?}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn test_resolve_column_encodings() {
+ let schema = schema();
+ let resolve = |encoding, values: &[&str]| {
+ resolve_column_encodings(&schema, encoding,
&column_encodings(values))
+ };
+
+ assert_eq!(resolve(None, &[]).unwrap(), vec![]);
+ assert_eq!(resolve(Some(EncodingArgs::Plain), &[]).unwrap(), vec![]);
+ assert_eq!(
+ resolve(
+ Some(EncodingArgs::Plain),
+ &["f=alp", "g.d=byte-stream-split", "i=delta-binary-packed"]
+ )
+ .unwrap(),
+ vec![
+ (ColumnPath::from("f"), Encoding::ALP),
+ (
+ ColumnPath::new(vec!["g".to_string(), "d".to_string()]),
+ Encoding::BYTE_STREAM_SPLIT
+ ),
+ (ColumnPath::from("i"), Encoding::DELTA_BINARY_PACKED),
+ ]
+ );
+
+ // The global encoding is checked for every column without its own
encoding
+ assert_eq!(
+ resolve(Some(EncodingArgs::Alp), &["b=rle",
"i=plain"]).unwrap_err(),
+ "--encoding for column 's' (BYTE_ARRAY): ALP supports only FLOAT
and DOUBLE \
+ (set another encoding for this column with --column-encoding
's=<ENCODING>')"
+ );
+ assert_eq!(
+ resolve(
+ Some(EncodingArgs::Alp),
+ &["b=rle", "i=plain", "s=delta-byte-array"]
+ )
+ .unwrap(),
+ vec![
+ (ColumnPath::from("b"), Encoding::RLE),
+ (ColumnPath::from("i"), Encoding::PLAIN),
+ (ColumnPath::from("s"), Encoding::DELTA_BYTE_ARRAY),
+ ]
+ );
+
+ assert_eq!(
+ resolve(None, &["s=alp"]).unwrap_err(),
+ "--column-encoding for column 's' (BYTE_ARRAY): ALP supports only
FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ resolve(None, &["d=alp"]).unwrap_err(),
+ "--column-encoding refers to unknown column 'd', leaf columns are:
b, i, f, s, g.d"
+ );
+ assert_eq!(
+ resolve(None, &["f=alp", "f=plain"]).unwrap_err(),
+ "--column-encoding is given more than once for column 'f'"
+ );
+ // Encodings that cannot be set at all are reported once
+ assert_eq!(
+ resolve(Some(EncodingArgs::RleDictionary), &[]).unwrap_err(),
+ "--encoding: RLE_DICTIONARY cannot be set as a column encoding, \
+ use --dictionary-enabled instead"
+ );
+ // Every problem is reported
+ let err = resolve(Some(EncodingArgs::Rle), &["f=alp",
"x=plain"]).unwrap_err();
+ let lines: Vec<_> = err.lines().collect();
+ assert_eq!(lines.len(), 4, "{err}");
+ assert!(lines[0].starts_with("--column-encoding refers to unknown
column 'x'"));
+ assert!(lines[1].starts_with("--encoding for column 'i' (INT32): RLE
supports only"));
+ assert!(lines[2].starts_with("--encoding for column 's'
(BYTE_ARRAY)"));
+ assert!(lines[3].starts_with("--encoding for column 'g.d' (DOUBLE)"));
+ }
+
Review Comment:
With the resolution done by `WriterProperties` there is nothing left here
for this test to check that the library does not already test.
```suggestion
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -32,19 +32,31 @@
//! ```
//! cargo run --features=cli --bin parquet-rewrite -- -i XYZ.parquet -o
XYZ2.parquet
//! ```
+//!
+//! Encodings can be set for all columns with `--encoding`, and for individual
+//! leaf columns, by their dot-separated path, with `--column-encoding`:
+//! ```
+//! parquet-rewrite -i XYZ.parquet -o XYZ2.parquet --dictionary-enabled false \
+//! --encoding plain --column-encoding price=alp --column-encoding
a.b=delta-binary-packed
+//! ```
+use std::collections::HashSet;
use std::fs::File;
use arrow_array::RecordBatchReader;
-use clap::{Parser, ValueEnum, builder::PossibleValue};
+use clap::{CommandFactory, Parser, ValueEnum, builder::PossibleValue,
error::ErrorKind};
use parquet::{
- arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder},
- basic::{BrotliLevel, Compression, Encoding, GzipLevel, ZstdLevel},
+ arrow::{ArrowSchemaConverter, ArrowWriter,
arrow_reader::ParquetRecordBatchReaderBuilder},
+ basic::{BrotliLevel, Compression, Encoding, GzipLevel, Type as
PhysicalType, ZstdLevel},
file::{
- properties::{BloomFilterPosition, EnabledStatistics, WriterProperties,
WriterVersion},
+ properties::{
+ BloomFilterPosition, DEFAULT_COERCE_TYPES, EnabledStatistics,
WriterProperties,
+ WriterVersion,
+ },
reader::FileReader,
serialized_reader::SerializedFileReader,
},
+ schema::types::{ColumnDescriptor, ColumnPath, SchemaDescriptor},
Review Comment:
Same here, only `ColumnDescriptor` is still used.
```suggestion
schema::types::ColumnDescriptor,
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -32,19 +32,31 @@
//! ```
//! cargo run --features=cli --bin parquet-rewrite -- -i XYZ.parquet -o
XYZ2.parquet
//! ```
+//!
+//! Encodings can be set for all columns with `--encoding`, and for individual
+//! leaf columns, by their dot-separated path, with `--column-encoding`:
+//! ```
+//! parquet-rewrite -i XYZ.parquet -o XYZ2.parquet --dictionary-enabled false \
+//! --encoding plain --column-encoding price=alp --column-encoding
a.b=delta-binary-packed
+//! ```
+use std::collections::HashSet;
Review Comment:
Not needed once `resolve_column_encodings` goes (see below).
```suggestion
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -435,4 +618,389 @@ fn main() {
}
parquet_writer.close().expect("finalizing file");
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::path::Path;
+ use std::sync::Arc;
+
+ use arrow_array::{
+ ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array,
Float64Array, Int32Array,
+ Int64Array, RecordBatch, StringArray,
+ };
+ use parquet::schema::parser::parse_message_type;
+
+ const ALL_ENCODINGS: [EncodingArgs; 10] = [
+ EncodingArgs::Plain,
+ EncodingArgs::PlainDictionary,
+ EncodingArgs::Rle,
+ EncodingArgs::BitPacked,
+ EncodingArgs::DeltaBinaryPacked,
+ EncodingArgs::DeltaLengthByteArray,
+ EncodingArgs::DeltaByteArray,
+ EncodingArgs::RleDictionary,
+ EncodingArgs::ByteStreamSplit,
+ EncodingArgs::Alp,
+ ];
+
+ fn schema() -> SchemaDescriptor {
+ let message = "
+ message schema {
+ REQUIRED BOOLEAN b;
+ REQUIRED INT32 i;
+ REQUIRED FLOAT f;
+ REQUIRED BYTE_ARRAY s (UTF8);
+ OPTIONAL group g {
+ REQUIRED DOUBLE d;
+ }
+ }";
+ SchemaDescriptor::new(Arc::new(parse_message_type(message).unwrap()))
+ }
+
+ fn column_encodings(values: &[&str]) -> Vec<(String, EncodingArgs)> {
+ values
+ .iter()
+ .map(|v| parse_column_encoding(v).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn test_parse_column_encoding() {
+ let parse = |v| parse_column_encoding(v).unwrap();
+ assert_eq!(parse("price=alp"), ("price".to_string(),
EncodingArgs::Alp));
+ assert_eq!(parse("a.b=PLAIN"), ("a.b".to_string(),
EncodingArgs::Plain));
+ assert_eq!(
+ parse("x=delta-binary-packed"),
+ ("x".to_string(), EncodingArgs::DeltaBinaryPacked)
+ );
+ // Only the last `=` separates the column from the encoding
+ assert_eq!(parse("a=b=plain"), ("a=b".to_string(),
EncodingArgs::Plain));
+
+ let err = |v| parse_column_encoding(v).unwrap_err();
+ assert_eq!(err("price"), "expected <COLUMN>=<ENCODING>, got 'price'");
+ assert_eq!(err("=alp"), "missing column in '=alp'");
+ assert!(
+ err("price=snappy").starts_with(
+ "invalid encoding 'snappy' in 'price=snappy', possible values:
plain, "
+ ),
+ "{}",
+ err("price=snappy")
+ );
+ assert!(err("price=").starts_with("invalid encoding '' in 'price='"));
+ }
+
+ #[test]
+ fn test_parse_args_column_encoding() {
+ let args = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--encoding",
+ "plain",
+ "--column-encoding",
+ "f=alp",
+ "--column-encoding",
+ "g.d=byte-stream-split",
+ ])
+ .unwrap();
+ assert_eq!(args.encoding, Some(EncodingArgs::Plain));
+ assert_eq!(
+ args.column_encoding,
+ column_encodings(&["f=alp", "g.d=byte-stream-split"])
+ );
+
+ let err = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--column-encoding",
+ "f",
+ ])
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ValueValidation);
+ }
+
+ #[test]
+ fn test_check_encoding_messages() {
+ assert_eq!(
+ check_encoding(EncodingArgs::Alp,
PhysicalType::INT32).unwrap_err(),
+ "ALP supports only FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::Rle,
PhysicalType::INT32).unwrap_err(),
+ "RLE supports only BOOLEAN"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::ByteStreamSplit,
PhysicalType::BYTE_ARRAY).unwrap_err(),
+ "BYTE_STREAM_SPLIT supports only INT32, INT64, FLOAT, DOUBLE and
FIXED_LEN_BYTE_ARRAY"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::RleDictionary,
PhysicalType::INT32).unwrap_err(),
+ "RLE_DICTIONARY cannot be set as a column encoding, use
--dictionary-enabled instead"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::BitPacked,
PhysicalType::INT32).unwrap_err(),
+ "BIT_PACKED is deprecated and not supported for writing"
+ );
+ for physical_type in [PhysicalType::INT96, PhysicalType::BYTE_ARRAY] {
+ assert!(check_encoding(EncodingArgs::Plain,
physical_type).is_ok());
+ }
+ }
+
Review Comment:
I would drop these. `test_parse_args_column_encoding` tests clap itself,
`test_check_encoding_messages` only checks message wording, and
`test_parse_column_encoding` covers a 5-line split. `ALL_ENCODINGS` can be
`EncodingArgs::value_variants()`, and `schema()` / `column_encodings()` only
serve `test_resolve_column_encodings`. The `std::path::Path` and
`parse_message_type` imports above are then unused too.
```suggestion
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -435,4 +618,389 @@ fn main() {
}
parquet_writer.close().expect("finalizing file");
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::path::Path;
+ use std::sync::Arc;
+
+ use arrow_array::{
+ ArrayRef, BooleanArray, FixedSizeBinaryArray, Float32Array,
Float64Array, Int32Array,
+ Int64Array, RecordBatch, StringArray,
+ };
+ use parquet::schema::parser::parse_message_type;
+
+ const ALL_ENCODINGS: [EncodingArgs; 10] = [
+ EncodingArgs::Plain,
+ EncodingArgs::PlainDictionary,
+ EncodingArgs::Rle,
+ EncodingArgs::BitPacked,
+ EncodingArgs::DeltaBinaryPacked,
+ EncodingArgs::DeltaLengthByteArray,
+ EncodingArgs::DeltaByteArray,
+ EncodingArgs::RleDictionary,
+ EncodingArgs::ByteStreamSplit,
+ EncodingArgs::Alp,
+ ];
+
+ fn schema() -> SchemaDescriptor {
+ let message = "
+ message schema {
+ REQUIRED BOOLEAN b;
+ REQUIRED INT32 i;
+ REQUIRED FLOAT f;
+ REQUIRED BYTE_ARRAY s (UTF8);
+ OPTIONAL group g {
+ REQUIRED DOUBLE d;
+ }
+ }";
+ SchemaDescriptor::new(Arc::new(parse_message_type(message).unwrap()))
+ }
+
+ fn column_encodings(values: &[&str]) -> Vec<(String, EncodingArgs)> {
+ values
+ .iter()
+ .map(|v| parse_column_encoding(v).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn test_parse_column_encoding() {
+ let parse = |v| parse_column_encoding(v).unwrap();
+ assert_eq!(parse("price=alp"), ("price".to_string(),
EncodingArgs::Alp));
+ assert_eq!(parse("a.b=PLAIN"), ("a.b".to_string(),
EncodingArgs::Plain));
+ assert_eq!(
+ parse("x=delta-binary-packed"),
+ ("x".to_string(), EncodingArgs::DeltaBinaryPacked)
+ );
+ // Only the last `=` separates the column from the encoding
+ assert_eq!(parse("a=b=plain"), ("a=b".to_string(),
EncodingArgs::Plain));
+
+ let err = |v| parse_column_encoding(v).unwrap_err();
+ assert_eq!(err("price"), "expected <COLUMN>=<ENCODING>, got 'price'");
+ assert_eq!(err("=alp"), "missing column in '=alp'");
+ assert!(
+ err("price=snappy").starts_with(
+ "invalid encoding 'snappy' in 'price=snappy', possible values:
plain, "
+ ),
+ "{}",
+ err("price=snappy")
+ );
+ assert!(err("price=").starts_with("invalid encoding '' in 'price='"));
+ }
+
+ #[test]
+ fn test_parse_args_column_encoding() {
+ let args = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--encoding",
+ "plain",
+ "--column-encoding",
+ "f=alp",
+ "--column-encoding",
+ "g.d=byte-stream-split",
+ ])
+ .unwrap();
+ assert_eq!(args.encoding, Some(EncodingArgs::Plain));
+ assert_eq!(
+ args.column_encoding,
+ column_encodings(&["f=alp", "g.d=byte-stream-split"])
+ );
+
+ let err = Args::try_parse_from([
+ "parquet-rewrite",
+ "-i",
+ "in.parquet",
+ "-o",
+ "out.parquet",
+ "--column-encoding",
+ "f",
+ ])
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ValueValidation);
+ }
+
+ #[test]
+ fn test_check_encoding_messages() {
+ assert_eq!(
+ check_encoding(EncodingArgs::Alp,
PhysicalType::INT32).unwrap_err(),
+ "ALP supports only FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::Rle,
PhysicalType::INT32).unwrap_err(),
+ "RLE supports only BOOLEAN"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::ByteStreamSplit,
PhysicalType::BYTE_ARRAY).unwrap_err(),
+ "BYTE_STREAM_SPLIT supports only INT32, INT64, FLOAT, DOUBLE and
FIXED_LEN_BYTE_ARRAY"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::RleDictionary,
PhysicalType::INT32).unwrap_err(),
+ "RLE_DICTIONARY cannot be set as a column encoding, use
--dictionary-enabled instead"
+ );
+ assert_eq!(
+ check_encoding(EncodingArgs::BitPacked,
PhysicalType::INT32).unwrap_err(),
+ "BIT_PACKED is deprecated and not supported for writing"
+ );
+ for physical_type in [PhysicalType::INT96, PhysicalType::BYTE_ARRAY] {
+ assert!(check_encoding(EncodingArgs::Plain,
physical_type).is_ok());
+ }
+ }
+
+ /// Returns whether the writer can write `array` with `encoding`.
+ ///
+ /// Some unsupported encodings make the column writer panic rather than
+ /// return an error, so panics count as unsupported too.
+ fn writer_supports(array: ArrayRef, encoding: Encoding) -> bool {
+ let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .set_encoding(encoding)
+ .build();
+ let write = || {
+ let mut writer = ArrowWriter::try_new(vec![], batch.schema(),
Some(props))?;
+ writer.write(&batch)?;
+ writer.close()
+ };
+ matches!(
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(write)),
+ Ok(Ok(_))
+ )
+ }
+
+ /// `check_encoding` must agree with what the writer actually supports
+ #[test]
+ fn test_check_encoding_matches_writer() {
+ let arrays: [(PhysicalType, ArrayRef); 7] = [
+ (
+ PhysicalType::BOOLEAN,
+ Arc::new(BooleanArray::from(vec![true, false])),
+ ),
+ (PhysicalType::INT32, Arc::new(Int32Array::from(vec![1, 2]))),
+ (PhysicalType::INT64, Arc::new(Int64Array::from(vec![1, 2]))),
+ (
+ PhysicalType::FLOAT,
+ Arc::new(Float32Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::DOUBLE,
+ Arc::new(Float64Array::from(vec![1.5, 2.5])),
+ ),
+ (
+ PhysicalType::BYTE_ARRAY,
+ Arc::new(StringArray::from(vec!["a", "b"])),
+ ),
+ (
+ PhysicalType::FIXED_LEN_BYTE_ARRAY,
+ Arc::new(FixedSizeBinaryArray::try_from_iter([b"ab",
b"cd"].into_iter()).unwrap()),
+ ),
+ ];
+ for (physical_type, array) in arrays {
+ for encoding in ALL_ENCODINGS {
+ let checked = check_encoding(encoding, physical_type);
+ if matches!(
+ encoding,
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary
+ ) {
+ // The writer properties builder panics on these
+ assert!(checked.is_err());
+ continue;
+ }
+ assert_eq!(
+ checked.is_ok(),
+ writer_supports(array.clone(), encoding.into()),
+ "{encoding:?} for {physical_type}: {checked:?}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn test_resolve_column_encodings() {
+ let schema = schema();
+ let resolve = |encoding, values: &[&str]| {
+ resolve_column_encodings(&schema, encoding,
&column_encodings(values))
+ };
+
+ assert_eq!(resolve(None, &[]).unwrap(), vec![]);
+ assert_eq!(resolve(Some(EncodingArgs::Plain), &[]).unwrap(), vec![]);
+ assert_eq!(
+ resolve(
+ Some(EncodingArgs::Plain),
+ &["f=alp", "g.d=byte-stream-split", "i=delta-binary-packed"]
+ )
+ .unwrap(),
+ vec![
+ (ColumnPath::from("f"), Encoding::ALP),
+ (
+ ColumnPath::new(vec!["g".to_string(), "d".to_string()]),
+ Encoding::BYTE_STREAM_SPLIT
+ ),
+ (ColumnPath::from("i"), Encoding::DELTA_BINARY_PACKED),
+ ]
+ );
+
+ // The global encoding is checked for every column without its own
encoding
+ assert_eq!(
+ resolve(Some(EncodingArgs::Alp), &["b=rle",
"i=plain"]).unwrap_err(),
+ "--encoding for column 's' (BYTE_ARRAY): ALP supports only FLOAT
and DOUBLE \
+ (set another encoding for this column with --column-encoding
's=<ENCODING>')"
+ );
+ assert_eq!(
+ resolve(
+ Some(EncodingArgs::Alp),
+ &["b=rle", "i=plain", "s=delta-byte-array"]
+ )
+ .unwrap(),
+ vec![
+ (ColumnPath::from("b"), Encoding::RLE),
+ (ColumnPath::from("i"), Encoding::PLAIN),
+ (ColumnPath::from("s"), Encoding::DELTA_BYTE_ARRAY),
+ ]
+ );
+
+ assert_eq!(
+ resolve(None, &["s=alp"]).unwrap_err(),
+ "--column-encoding for column 's' (BYTE_ARRAY): ALP supports only
FLOAT and DOUBLE"
+ );
+ assert_eq!(
+ resolve(None, &["d=alp"]).unwrap_err(),
+ "--column-encoding refers to unknown column 'd', leaf columns are:
b, i, f, s, g.d"
+ );
+ assert_eq!(
+ resolve(None, &["f=alp", "f=plain"]).unwrap_err(),
+ "--column-encoding is given more than once for column 'f'"
+ );
+ // Encodings that cannot be set at all are reported once
+ assert_eq!(
+ resolve(Some(EncodingArgs::RleDictionary), &[]).unwrap_err(),
+ "--encoding: RLE_DICTIONARY cannot be set as a column encoding, \
+ use --dictionary-enabled instead"
+ );
+ // Every problem is reported
+ let err = resolve(Some(EncodingArgs::Rle), &["f=alp",
"x=plain"]).unwrap_err();
+ let lines: Vec<_> = err.lines().collect();
+ assert_eq!(lines.len(), 4, "{err}");
+ assert!(lines[0].starts_with("--column-encoding refers to unknown
column 'x'"));
+ assert!(lines[1].starts_with("--encoding for column 'i' (INT32): RLE
supports only"));
+ assert!(lines[2].starts_with("--encoding for column 's'
(BYTE_ARRAY)"));
+ assert!(lines[3].starts_with("--encoding for column 'g.d' (DOUBLE)"));
+ }
+
+ fn write_input(path: &Path) -> RecordBatch {
+ let batch = RecordBatch::try_from_iter([
+ (
+ "price",
+ Arc::new(Float64Array::from(vec![1.25, 2.5, 3.75])) as
ArrayRef,
+ ),
+ ("id", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef),
+ (
+ "name",
+ Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef,
+ ),
+ ])
+ .unwrap();
+ let mut writer =
+ ArrowWriter::try_new(File::create(path).unwrap(), batch.schema(),
None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ batch
+ }
+
+ fn rewrite_args(input: &Path, output: &Path, extra: &[&str]) -> Args {
+ let mut args = vec![
+ "parquet-rewrite",
+ "-i",
+ input.to_str().unwrap(),
+ "-o",
+ output.to_str().unwrap(),
+ ];
+ args.extend_from_slice(extra);
+ Args::try_parse_from(args).unwrap()
+ }
+
+ #[test]
+ fn test_rewrite_with_column_encodings() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.parquet");
+ let output = dir.path().join("output.parquet");
+ let batch = write_input(&input);
+
+ let args = rewrite_args(
+ &input,
+ &output,
+ &[
+ "--dictionary-enabled",
+ "false",
+ "--encoding",
+ "plain",
+ "--column-encoding",
+ "price=alp",
+ "--column-encoding",
+ "id=delta-binary-packed",
+ ],
+ );
+ rewrite(args).unwrap();
+
+ let reader =
SerializedFileReader::new(File::open(&output).unwrap()).unwrap();
+ let encodings: Vec<Vec<Encoding>> = reader
+ .metadata()
+ .row_group(0)
+ .columns()
+ .iter()
+ .map(|c| c.encodings().collect())
+ .collect();
+ assert!(encodings[0].contains(&Encoding::ALP), "{encodings:?}");
+ assert!(
+ encodings[1].contains(&Encoding::DELTA_BINARY_PACKED),
+ "{encodings:?}"
+ );
+ assert!(encodings[2].contains(&Encoding::PLAIN), "{encodings:?}");
+ assert!(!encodings[2].contains(&Encoding::ALP), "{encodings:?}");
+
+ let rewritten: Vec<_> =
+
ParquetRecordBatchReaderBuilder::try_new(File::open(&output).unwrap())
+ .unwrap()
+ .build()
+ .unwrap()
+ .collect::<Result<_, _>>()
+ .unwrap();
+ assert_eq!(rewritten, vec![batch]);
+ }
+
+ #[test]
+ fn test_rewrite_rejects_unsupported_encoding_before_creating_output() {
+ let dir = tempfile::tempdir().unwrap();
+ let input = dir.path().join("input.parquet");
+ let output = dir.path().join("output.parquet");
+ write_input(&input);
+
+ let args = rewrite_args(&input, &output, &["--encoding", "alp"]);
+ let err = rewrite(args).unwrap_err();
+ assert!(
+ err.contains("column 'id' (INT32): ALP supports only"),
+ "{err}"
+ );
+ assert!(
+ err.contains("column 'name' (BYTE_ARRAY): ALP supports only"),
+ "{err}"
+ );
+ assert!(!err.contains("price"), "{err}");
+ assert!(!output.exists());
+
+ let args = rewrite_args(&input, &output, &["--column-encoding",
"name=alp"]);
+ let err = rewrite(args).unwrap_err();
+ assert!(
+ err.contains("column 'name' (BYTE_ARRAY): ALP supports only"),
+ "{err}"
+ );
+ assert!(!output.exists());
+ }
Review Comment:
These two can be one test: the error and no output file, the unknown column,
and an override that makes the same rewrite pass and lands in the column chunk
metadata. Reading the data back tests the writer, not this tool.
```suggestion
#[test]
fn test_rewrite_checks_encodings_before_creating_output() {
let dir = tempfile::tempdir().unwrap();
let input = dir.path().join("input.parquet");
let output = dir.path().join("output.parquet");
let batch = RecordBatch::try_from_iter([
(
"price",
Arc::new(Float64Array::from(vec![1.5, 2.5])) as ArrayRef,
),
("id", Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef),
])
.unwrap();
let mut writer =
ArrowWriter::try_new(File::create(&input).unwrap(),
batch.schema(), None).unwrap();
writer.write(&batch).unwrap();
writer.close().unwrap();
let rewrite_with = |extra: &[&str]| {
let mut argv = vec!["parquet-rewrite", "-i",
input.to_str().unwrap()];
argv.extend([
"-o",
output.to_str().unwrap(),
"--dictionary-enabled",
"false",
]);
argv.extend(["--encoding", "alp"]);
argv.extend_from_slice(extra);
rewrite(Args::try_parse_from(argv).unwrap())
};
assert_eq!(
rewrite_with(&[]).unwrap_err(),
"column 'id' (INT32): ALP supports only [FLOAT, DOUBLE]"
);
assert_eq!(
rewrite_with(&["--column-encoding", "x=plain"]).unwrap_err(),
"unknown column 'x' in --column-encoding"
);
assert!(!output.exists());
rewrite_with(&["--column-encoding",
"id=delta-binary-packed"]).unwrap();
let reader =
SerializedFileReader::new(File::open(&output).unwrap()).unwrap();
let columns = reader.metadata().row_group(0).columns();
assert!(columns[0].encodings().any(|e| e == Encoding::ALP));
assert!(
columns[1]
.encodings()
.any(|e| e == Encoding::DELTA_BINARY_PACKED)
);
}
```
##########
parquet/src/bin/parquet-rewrite.rs:
##########
@@ -153,6 +165,148 @@ impl From<EncodingArgs> for Encoding {
}
}
+/// Parses a `--column-encoding` value of the form `<COLUMN>=<ENCODING>`.
+///
+/// The value is split at the last `=`, as encoding names never contain one.
+fn parse_column_encoding(value: &str) -> Result<(String, EncodingArgs),
String> {
+ let (column, encoding) = value
+ .rsplit_once('=')
+ .ok_or_else(|| format!("expected <COLUMN>=<ENCODING>, got
'{value}'"))?;
+ if column.is_empty() {
+ return Err(format!("missing column in '{value}'"));
+ }
+ let encoding = EncodingArgs::from_str(encoding, true).map_err(|_| {
+ let possible_values: Vec<_> = EncodingArgs::value_variants()
+ .iter()
+ .filter_map(|v| v.to_possible_value())
+ .map(|v| v.get_name().to_string())
+ .collect();
+ format!(
+ "invalid encoding '{encoding}' in '{value}', possible values: {}",
+ possible_values.join(", ")
+ )
+ })?;
+ Ok((column.to_string(), encoding))
+}
+
+/// Checks that `encoding` can be set as the encoding of a column at all.
+fn check_encoding_settable(encoding: EncodingArgs) -> Result<(), String> {
+ match encoding {
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary =>
Err(format!(
+ "{} cannot be set as a column encoding, use --dictionary-enabled
instead",
+ Encoding::from(encoding)
+ )),
+ EncodingArgs::BitPacked => Err(format!(
+ "{} is deprecated and not supported for writing",
+ Encoding::from(encoding)
+ )),
+ _ => Ok(()),
+ }
+}
+
+/// Checks that the writer supports `encoding` for columns of `physical_type`.
+///
+/// This mirrors the encoders that the writer can create, so that an
+/// unsupported combination is reported before the output file is created
+/// instead of failing part way through the rewrite.
+fn check_encoding(encoding: EncodingArgs, physical_type: PhysicalType) ->
Result<(), String> {
+ use PhysicalType::*;
+ let supported: &[PhysicalType] = match encoding {
+ EncodingArgs::Plain => return Ok(()),
+ EncodingArgs::PlainDictionary | EncodingArgs::RleDictionary |
EncodingArgs::BitPacked => {
+ return check_encoding_settable(encoding);
+ }
+ EncodingArgs::Rle => &[BOOLEAN],
+ EncodingArgs::DeltaBinaryPacked => &[INT32, INT64],
+ EncodingArgs::DeltaLengthByteArray => &[BYTE_ARRAY],
+ EncodingArgs::DeltaByteArray => &[BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY],
+ EncodingArgs::ByteStreamSplit => &[INT32, INT64, FLOAT, DOUBLE,
FIXED_LEN_BYTE_ARRAY],
+ EncodingArgs::Alp => &[FLOAT, DOUBLE],
+ };
+ if supported.contains(&physical_type) {
+ return Ok(());
+ }
+ let mut types: Vec<_> = supported.iter().map(|t| t.to_string()).collect();
+ let last = types.pop().unwrap();
+ let types = match types.is_empty() {
+ true => last,
+ false => format!("{} and {last}", types.join(", ")),
+ };
+ let encoding = Encoding::from(encoding);
+ Err(format!("{encoding} supports only {types}"))
+}
+
+/// Resolves `--encoding` and `--column-encoding` against the leaf columns of
+/// the output `schema`, returning the encoding to set for each column given
+/// by `--column-encoding`.
+///
+/// Returns an error describing every problem found if a column does not
+/// exist, is given more than once, or does not support its encoding (from
+/// `--column-encoding`, or else from `--encoding`).
+fn resolve_column_encodings(
+ schema: &SchemaDescriptor,
+ encoding: Option<EncodingArgs>,
+ column_encodings: &[(String, EncodingArgs)],
+) -> Result<Vec<(ColumnPath, Encoding)>, String> {
+ let describe = |column: &ColumnDescriptor| {
+ format!(
+ "column '{}' ({})",
+ column.path().string(),
+ column.physical_type()
+ )
+ };
+
+ let mut errors = vec![];
+ let mut resolved = Vec::with_capacity(column_encodings.len());
+ let mut seen = HashSet::new();
+ for (name, column_encoding) in column_encodings {
+ if !seen.insert(name.as_str()) {
+ errors.push(format!(
+ "--column-encoding is given more than once for column '{name}'"
+ ));
+ continue;
+ }
+ let Some(column) = schema.columns().iter().find(|c| c.path().string()
== *name) else {
+ let columns: Vec<_> = schema.columns().iter().map(|c|
c.path().string()).collect();
+ errors.push(format!(
+ "--column-encoding refers to unknown column '{name}', leaf
columns are: {}",
+ columns.join(", ")
+ ));
+ continue;
+ };
+ match check_encoding(*column_encoding, column.physical_type()) {
+ Ok(()) => resolved.push((column.path().clone(),
(*column_encoding).into())),
+ Err(e) => errors.push(format!("--column-encoding for {}: {e}",
describe(column))),
+ }
+ }
+
+ if let Some(encoding) = encoding {
+ if let Err(e) = check_encoding_settable(encoding) {
+ // Reported once rather than for every column
+ errors.push(format!("--encoding: {e}"));
+ } else {
+ for column in schema.columns() {
+ if seen.contains(column.path().string().as_str()) {
+ continue;
+ }
+ if let Err(e) = check_encoding(encoding,
column.physical_type()) {
+ errors.push(format!(
+ "--encoding for {}: {e} (set another encoding for this
column \
+ with --column-encoding '{}=<ENCODING>')",
+ describe(column),
+ column.path().string()
+ ));
+ }
+ }
+ }
+ }
+
+ match errors.is_empty() {
+ true => Ok(resolved),
+ false => Err(errors.join("\n")),
+ }
+}
+
Review Comment:
`WriterProperties::encoding(&ColumnPath)` already resolves "the column's
encoding if set, else the default one", so this function rebuilds what the
properties do. With the properties doing it, the two loops, the `seen` set and
the duplicate error all go away. Giving a column twice then just means the last
one wins, like any repeated `set_column_encoding`.
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]