This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 20786809e3 Prettify RunEndEncoded datatype display (#10840)
20786809e3 is described below
commit 20786809e3edc0fdc0532642d89edbf5b9c202a3
Author: RIchard Baah <[email protected]>
AuthorDate: Wed Sep 9 11:58:08 2026 -0400
Prettify RunEndEncoded datatype display (#10840)
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes #8351.
# Rationale for this change
the current display method for `DataType:REE` is a big verbose & not
pretty.
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
# What changes are included in this PR?
Before: `RunEndEncoded("run_ends": non-null Int16, "values": Utf8)`
After: `RunEndEncoded("run_ends": Int16, "values": Utf8)`
note - the values field can still display null if the field is nullable
thanks to `format_field`, the non-null was only removed from the run_end
field because it cannot be null as noted by the arrow spec
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
# Are these changes tested?
yes
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
# Are there any user-facing changes?
yes, users printing REE can expect a different output format now.
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
If there are any breaking changes to public APIs, please call them out.
-->
---
arrow-schema/src/datatype_display.rs | 127 +++++++++++++++++++++++++++++------
arrow-schema/src/datatype_parse.rs | 70 +++++++++++++++++--
arrow-schema/src/field.rs | 8 +++
3 files changed, 178 insertions(+), 27 deletions(-)
diff --git a/arrow-schema/src/datatype_display.rs
b/arrow-schema/src/datatype_display.rs
index 354c3a2b35..9bb40852b3 100644
--- a/arrow-schema/src/datatype_display.rs
+++ b/arrow-schema/src/datatype_display.rs
@@ -16,6 +16,7 @@
// under the License.
use crate::DataType;
+use crate::Field;
use crate::Metadata;
use std::fmt;
use std::fmt::Display;
@@ -173,11 +174,25 @@ impl Display for DataType {
Ok(())
}
Self::RunEndEncoded(run_ends_field, values_field) => {
+ let default_names = run_ends_field.name() ==
Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME
+ && values_field.name() ==
Field::REE_VALUES_FIELD_DEFAULT_NAME;
+ let no_metadata =
+ run_ends_field.metadata().is_empty() &&
values_field.metadata().is_empty();
write!(f, "RunEndEncoded(")?;
- let run_ends_str = format_field(run_ends_field);
- let values_str = format_field(values_field);
-
- write!(f, "{run_ends_str}, {values_str})")?;
+ if default_names && no_metadata {
+ let re_null = format_nullability(run_ends_field);
+ let v_null = format_nullability(values_field);
+ write!(
+ f,
+ "{re_null}{}, {v_null}{})",
+ run_ends_field.data_type(),
+ values_field.data_type(),
+ )?;
+ } else {
+ let run_ends_str = format_field(run_ends_field);
+ let values_str = format_field(values_field);
+ write!(f, "{run_ends_str}, {values_str})")?;
+ }
Ok(())
}
}
@@ -474,24 +489,96 @@ mod tests {
#[test]
fn test_display_run_end_encoded() {
- let run_ends_field = Arc::new(Field::new("run_ends", DataType::UInt32,
false));
- let values_field = Arc::new(Field::new("values", DataType::Int32,
true));
- let ree_data_type = DataType::RunEndEncoded(run_ends_field.clone(),
values_field.clone());
- let ree_data_type_string = ree_data_type.to_string();
- let expected_string = "RunEndEncoded(\"run_ends\": non-null UInt32,
\"values\": Int32)";
- assert_eq!(ree_data_type_string, expected_string);
+ // Compact form: default field names
+ let run_ends_field = Arc::new(Field::new(
+ Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
+ DataType::UInt32,
+ false,
+ ));
+ let values_field = Arc::new(Field::new(
+ Field::REE_VALUES_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ true,
+ ));
+ let ree = DataType::RunEndEncoded(run_ends_field.clone(),
values_field.clone());
+ assert_eq!(ree.to_string(), "RunEndEncoded(non-null UInt32, Int32)");
+
+ // Compact form: non-null values
+ let run_ends_field = Arc::new(Field::new(
+ Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ false,
+ ));
+ let values_field_str = Arc::new(Field::new(
+ Field::REE_VALUES_FIELD_DEFAULT_NAME,
+ DataType::Utf8,
+ false,
+ ));
+ let ree2 = DataType::RunEndEncoded(run_ends_field, values_field_str);
+ assert_eq!(
+ ree2.to_string(),
+ "RunEndEncoded(non-null Int32, non-null Utf8)"
+ );
- // Test with metadata
- let mut run_ends_field_with_metadata = Field::new("run_ends",
DataType::UInt32, false);
- let metadata = HashMap::from([("key".to_string(),
"value".to_string())]);
- run_ends_field_with_metadata.set_metadata(metadata);
- let ree_data_type_with_metadata =
- DataType::RunEndEncoded(Arc::new(run_ends_field_with_metadata),
values_field.clone());
- let ree_data_type_with_metadata_string =
ree_data_type_with_metadata.to_string();
- let expected_string_with_metadata = "RunEndEncoded(\"run_ends\":
non-null UInt32, metadata: {\"key\": \"value\"}, \"values\": Int32)";
+ // Verbose form: metadata on values field triggers verbose form
+ let run_ends_field = Arc::new(Field::new(
+ Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ false,
+ ));
+ let mut values_with_meta =
+ Field::new(Field::REE_VALUES_FIELD_DEFAULT_NAME, DataType::Utf8,
true);
+ values_with_meta.set_metadata(HashMap::from([("k".to_string(),
"v".to_string())]));
+ let ree_meta = DataType::RunEndEncoded(run_ends_field,
Arc::new(values_with_meta));
assert_eq!(
- ree_data_type_with_metadata_string,
- expected_string_with_metadata
+ ree_meta.to_string(),
+ "RunEndEncoded(\"run_ends\": non-null Int32, \"values\": Utf8,
metadata: {\"k\": \"v\"})"
+ );
+
+ // Verbose form: non-default field name on values
+ let run_ends_field = Arc::new(Field::new(
+ Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ false,
+ ));
+ let named_values = Arc::new(Field::new("named_values", DataType::Utf8,
false));
+ let ree3 = DataType::RunEndEncoded(run_ends_field, named_values);
+ assert_eq!(
+ ree3.to_string(),
+ "RunEndEncoded(\"run_ends\": non-null Int32, \"named_values\":
non-null Utf8)"
+ );
+
+ // Verbose form: non-default field name on run_ends
+ let custom_re = Arc::new(Field::new("re", DataType::Int32, false));
+ let values_field = Arc::new(Field::new(
+ Field::REE_VALUES_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ true,
+ ));
+ let ree4 = DataType::RunEndEncoded(custom_re, values_field);
+ assert_eq!(
+ ree4.to_string(),
+ "RunEndEncoded(\"re\": non-null Int32, \"values\": Int32)"
+ );
+
+ // Verbose form: metadata on both fields
+ let mut run_ends_with_meta = Field::new(
+ Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
+ DataType::Int32,
+ false,
+ );
+ run_ends_with_meta.set_metadata(HashMap::from([(
+ "source".to_string(),
+ "encoder".to_string(),
+ )]));
+ let mut values_with_meta2 =
+ Field::new(Field::REE_VALUES_FIELD_DEFAULT_NAME, DataType::Utf8,
true);
+ values_with_meta2.set_metadata(HashMap::from([("locale".to_string(),
"en".to_string())]));
+ let ree5 =
+ DataType::RunEndEncoded(Arc::new(run_ends_with_meta),
Arc::new(values_with_meta2));
+ assert_eq!(
+ ree5.to_string(),
+ "RunEndEncoded(\"run_ends\": non-null Int32, metadata:
{\"source\": \"encoder\"}, \"values\": Utf8, metadata: {\"locale\": \"en\"})"
);
}
diff --git a/arrow-schema/src/datatype_parse.rs
b/arrow-schema/src/datatype_parse.rs
index f20a11cc8b..84f24e7bac 100644
--- a/arrow-schema/src/datatype_parse.rs
+++ b/arrow-schema/src/datatype_parse.rs
@@ -597,13 +597,36 @@ impl<'a> Parser<'a> {
}
}
- /// Parses the next RunEndEncoded (called after `RunEndEncoded` has been
consumed)
- /// E.g: RunEndEncoded("run_ends": UInt32, "values": nonnull Int32)
+ /// Parses the next RunEndEncoded (called after `RunEndEncoded` has been
consumed).
+ ///
+ /// Compact form (default field names): `RunEndEncoded(non-null Int32,
non-null Utf8)`
+ /// Verbose form (custom field names): `RunEndEncoded("re": Int32, "v":
non-null Utf8)`
fn parse_run_end_encoded(&mut self) -> ArrowResult<DataType> {
self.expect_token(Token::LParen)?;
- let run_ends = self.parse_field()?;
- self.expect_token(Token::Comma)?;
- let values = self.parse_field()?;
+
+ // Distinguish compact from verbose by peeking: verbose starts with a
double-quoted name.
+ let verbose = matches!(
+ self.tokenizer.peek(),
+ Some(Ok(Token::DoubleQuotedString(_)))
+ );
+
+ let (run_ends, values) = if verbose {
+ let run_ends = self.parse_ree_verbose_field()?;
+ self.expect_token(Token::Comma)?;
+ let values = self.parse_ree_verbose_field()?;
+ (run_ends.with_nullable(false), values)
+ } else {
+ self.parse_opt_nullable(); // run_ends is always non-null; consume
the token if present
+ let re_type = self.parse_next_type()?;
+ self.expect_token(Token::Comma)?;
+ let v_nullable = self.parse_opt_nullable();
+ let v_type = self.parse_next_type()?;
+ (
+ Field::new(Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME, re_type,
false),
+ Field::new(Field::REE_VALUES_FIELD_DEFAULT_NAME, v_type,
v_nullable),
+ )
+ };
+
self.expect_token(Token::RParen)?;
Ok(DataType::RunEndEncoded(
Arc::new(run_ends),
@@ -611,6 +634,15 @@ impl<'a> Parser<'a> {
))
}
+ /// Parses `"name": [non-null] Type` used in the verbose REE form.
+ fn parse_ree_verbose_field(&mut self) -> ArrowResult<Field> {
+ let name = self.parse_double_quoted_string("RunEndEncoded field")?;
+ self.expect_token(Token::Colon)?;
+ let nullable = self.parse_opt_nullable();
+ let data_type = self.parse_next_type()?;
+ Ok(Field::new(name, data_type, nullable))
+ }
+
/// consume the next token and return `false` if the field is `nonnull`.
fn parse_opt_nullable(&mut self) -> bool {
let tok = self
@@ -1222,15 +1254,39 @@ mod test {
),
DataType::RunEndEncoded(
Arc::new(Field::new(
- "nested_run_end_encoded",
+ "run_ends",
DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::UInt32,
false)),
Arc::new(Field::new("values", DataType::Int32, true)),
),
- true,
+ false,
)),
Arc::new(Field::new("values", DataType::Int32, true)),
),
+ // non-default field names trigger verbose display form
+ DataType::RunEndEncoded(
+ Arc::new(Field::new(
+ "run_ends",
+ DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::UInt32,
false)),
+ Arc::new(Field::new("values", DataType::Int32, true)),
+ ),
+ false,
+ )),
+ Arc::new(Field::new("named_values", DataType::Int32, false)),
+ ),
+ // verbose form with non-null inner values
+ DataType::RunEndEncoded(
+ Arc::new(Field::new(
+ "run_ends",
+ DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::UInt32,
false)),
+ Arc::new(Field::new("values", DataType::Int32, false)),
+ ),
+ false,
+ )),
+ Arc::new(Field::new("named_values", DataType::Int32, false)),
+ ),
]
}
diff --git a/arrow-schema/src/field.rs b/arrow-schema/src/field.rs
index adccb9c7f5..feebdc29cf 100644
--- a/arrow-schema/src/field.rs
+++ b/arrow-schema/src/field.rs
@@ -162,6 +162,14 @@ impl Field {
///
/// See [Arrow
Spec](https://github.com/apache/arrow/blob/b19c4761b558ade94ae05743062d92aacedef10e/format/Schema.fbs#L127-L138))
pub const MAP_VALUE_FIELD_DEFAULT_NAME: &'static str = "value";
+ /// Default field name for the run-ends field for RunEndEncoded
+ ///
+ /// See [Arrow
Spec](https://arrow.apache.org/docs/format/Columnar.html#run-end-encoded-layout)
+ pub const REE_RUN_ENDS_FIELD_DEFAULT_NAME: &'static str = "run_ends";
+ /// Default field name for the values field for RunEndEncoded
+ ///
+ /// See [Arrow
Spec](https://arrow.apache.org/docs/format/Columnar.html#run-end-encoded-layout)
+ pub const REE_VALUES_FIELD_DEFAULT_NAME: &'static str = "values";
/// Creates a new field with the given name, data type, and nullability
///
