This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new bc4e672607 Remove the unmaintained paste dependency from arrow (#10303)
bc4e672607 is described below
commit bc4e672607f00587349b1308f6cf717fc6518848
Author: Phoenix <[email protected]>
AuthorDate: Wed Jul 15 13:31:41 2026 +0800
Remove the unmaintained paste dependency from arrow (#10303)
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes #7598 .
# Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
# What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
This PR removes the workspace's remaining dependency on the unmaintained
`paste` crate.
# Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
Yes.
# Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
If there are any breaking changes to public APIs, please call them out.
-->
No
@dentiny
CC @alamb
---------
Signed-off-by: Jiawei Zhao <[email protected]>
---
Cargo.lock | 9 -
arrow-flight/Cargo.toml | 5 +-
arrow-flight/src/sql/mod.rs | 163 ++++++++---------
parquet-variant/Cargo.toml | 1 -
parquet-variant/src/decoder.rs | 11 +-
parquet/Cargo.toml | 1 -
parquet/src/arrow/arrow_reader/statistics.rs | 259 +++++++++++++++++----------
parquet/tests/variant_integration.rs | 8 +-
8 files changed, 250 insertions(+), 207 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index d08bb0c1b5..d781954e85 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -355,7 +355,6 @@ dependencies = [
"http-body",
"hyper-util",
"once_cell",
- "paste",
"pin-project-lite",
"prost",
"prost-types",
@@ -2359,7 +2358,6 @@ dependencies = [
"parquet-variant",
"parquet-variant-compute",
"parquet-variant-json",
- "paste",
"rand 0.9.4",
"ring",
"seq-macro",
@@ -2397,7 +2395,6 @@ dependencies = [
"half",
"indexmap",
"num-traits",
- "paste",
"rand 0.9.4",
"simdutf8",
"uuid",
@@ -2452,12 +2449,6 @@ dependencies = [
"uuid",
]
-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
[[package]]
name = "percent-encoding"
version = "2.3.2"
diff --git a/arrow-flight/Cargo.toml b/arrow-flight/Cargo.toml
index c4541fdb60..e9fc019ed1 100644
--- a/arrow-flight/Cargo.toml
+++ b/arrow-flight/Cargo.toml
@@ -43,7 +43,6 @@ base64 = { version = "0.22", default-features = false,
features = ["std"] }
bytes = { version = "1", default-features = false }
futures = { version = "0.3", default-features = false, features = ["alloc"] }
once_cell = { version = "1", optional = true }
-paste = { version = "1.0" , optional = true }
prost = { version = "0.14.1", default-features = false, features = ["derive"] }
# For Timestamp type
prost-types = { version = "0.14.1", default-features = false }
@@ -62,7 +61,7 @@ all-features = true
[features]
default = []
-flight-sql = ["dep:arrow-arith", "dep:arrow-ord", "dep:arrow-row",
"dep:arrow-select", "dep:arrow-string", "dep:once_cell", "dep:paste"]
+flight-sql = ["dep:arrow-arith", "dep:arrow-ord", "dep:arrow-row",
"dep:arrow-select", "dep:arrow-string", "dep:once_cell"]
# TODO: Remove in the next release
flight-sql-experimental = ["flight-sql"]
tls-aws-lc= ["tonic/tls-aws-lc"]
@@ -112,4 +111,4 @@ required-features = ["cli", "flight-sql", "tls-ring"]
[[bench]]
name = "flight"
path = "benches/flight.rs"
-harness = false
\ No newline at end of file
+harness = false
diff --git a/arrow-flight/src/sql/mod.rs b/arrow-flight/src/sql/mod.rs
index e076f7aa07..1bf7b1a1b3 100644
--- a/arrow-flight/src/sql/mod.rs
+++ b/arrow-flight/src/sql/mod.rs
@@ -40,7 +40,6 @@
//! [`metadata`]: crate::sql::metadata
use arrow_schema::ArrowError;
use bytes::Bytes;
-use paste::paste;
use prost::Message;
#[allow(clippy::all)]
@@ -124,113 +123,95 @@ pub trait ProstMessageExt: prost::Message + Default {
fn as_any(&self) -> Any;
}
-/// Macro to coerce a token to an item, specifically
-/// to build the `Commands` enum.
-///
-/// See: <https://danielkeep.github.io/tlborm/book/blk-ast-coercion.html>
-macro_rules! as_item {
- ($i:item) => {
- $i
- };
-}
-
macro_rules! prost_message_ext {
($($name:tt,)*) => {
- paste! {
+ /// Helper to convert to/from protobuf [`Any`] message
+ /// to a specific FlightSQL command message.
+ ///
+ /// # Example
+ /// ```rust
+ /// # use arrow_flight::sql::{Any, CommandStatementQuery, Command};
+ /// let flightsql_message = CommandStatementQuery {
+ /// query: "SELECT * FROM foo".to_string(),
+ /// transaction_id: None,
+ /// };
+ ///
+ /// // Given a packed FlightSQL Any message
+ /// let any_message = Any::pack(&flightsql_message).unwrap();
+ ///
+ /// // decode it to Command:
+ /// match Command::try_from(any_message).unwrap() {
+ /// Command::CommandStatementQuery(decoded) => {
+ /// assert_eq!(flightsql_message, decoded);
+ /// }
+ /// _ => panic!("Unexpected decoded message"),
+ /// }
+ /// ```
+ #[derive(Clone, Debug, PartialEq)]
+ pub enum Command {
$(
- const [<$name:snake:upper _TYPE_URL>]: &'static str =
concat!("type.googleapis.com/arrow.flight.protocol.sql.", stringify!($name));
- )*
+ #[doc = concat!(stringify!($name), "variant")]
+ $name($name),)*
- as_item! {
- /// Helper to convert to/from protobuf [`Any`] message
- /// to a specific FlightSQL command message.
- ///
- /// # Example
- /// ```rust
- /// # use arrow_flight::sql::{Any, CommandStatementQuery,
Command};
- /// let flightsql_message = CommandStatementQuery {
- /// query: "SELECT * FROM foo".to_string(),
- /// transaction_id: None,
- /// };
- ///
- /// // Given a packed FlightSQL Any message
- /// let any_message = Any::pack(&flightsql_message).unwrap();
- ///
- /// // decode it to Command:
- /// match Command::try_from(any_message).unwrap() {
- /// Command::CommandStatementQuery(decoded) => {
- /// assert_eq!(flightsql_message, decoded);
- /// }
- /// _ => panic!("Unexpected decoded message"),
- /// }
- /// ```
- #[derive(Clone, Debug, PartialEq)]
- pub enum Command {
- $(
- #[doc = concat!(stringify!($name), "variant")]
- $name($name),)*
+ /// Any message that is not any FlightSQL command.
+ Unknown(Any),
+ }
- /// Any message that is not any FlightSQL command.
- Unknown(Any),
+ impl Command {
+ /// Convert the command to [`Any`].
+ pub fn into_any(self) -> Any {
+ match self {
+ $(
+ Self::$name(cmd) => cmd.as_any(),
+ )*
+ Self::Unknown(any) => any,
}
}
- impl Command {
- /// Convert the command to [`Any`].
- pub fn into_any(self) -> Any {
- match self {
- $(
- Self::$name(cmd) => cmd.as_any(),
- )*
- Self::Unknown(any) => any,
- }
- }
-
- /// Get the URL for the command.
- pub fn type_url(&self) -> &str {
- match self {
- $(
- Self::$name(_) => [<$name:snake:upper _TYPE_URL>],
- )*
- Self::Unknown(any) => any.type_url.as_str(),
- }
+ /// Get the URL for the command.
+ pub fn type_url(&self) -> &str {
+ match self {
+ $(
+ Self::$name(_) => <$name as ProstMessageExt>::type_url(),
+ )*
+ Self::Unknown(any) => any.type_url.as_str(),
}
}
+ }
- impl TryFrom<Any> for Command {
- type Error = ArrowError;
+ impl TryFrom<Any> for Command {
+ type Error = ArrowError;
- fn try_from(any: Any) -> Result<Self, Self::Error> {
- match any.type_url.as_str() {
- $(
- [<$name:snake:upper _TYPE_URL>]
- => {
- let m: $name =
Message::decode(&*any.value).map_err(|err| {
- ArrowError::ParseError(format!("Unable to
decode Any value: {err}"))
- })?;
- Ok(Self::$name(m))
- }
- )*
- _ => Ok(Self::Unknown(any)),
- }
+ fn try_from(any: Any) -> Result<Self, Self::Error> {
+ match any.type_url.as_str() {
+ $(
+ concat!("type.googleapis.com/arrow.flight.protocol.sql.",
stringify!($name))
+ => {
+ let m: $name =
Message::decode(&*any.value).map_err(|err| {
+ ArrowError::ParseError(format!("Unable to
decode Any value: {err}"))
+ })?;
+ Ok(Self::$name(m))
+ }
+ )*
+ _ => Ok(Self::Unknown(any)),
}
}
+ }
- $(
- impl ProstMessageExt for $name {
- fn type_url() -> &'static str {
- [<$name:snake:upper _TYPE_URL>]
- }
+ $(
+ impl ProstMessageExt for $name {
+ fn type_url() -> &'static str {
+ concat!("type.googleapis.com/arrow.flight.protocol.sql.",
stringify!($name))
+ }
- fn as_any(&self) -> Any {
- Any {
- type_url: <$name>::type_url().to_string(),
- value: self.encode_to_vec().into(),
- }
+ fn as_any(&self) -> Any {
+ Any {
+ type_url: <$name>::type_url().to_string(),
+ value: self.encode_to_vec().into(),
}
}
- )*
- }
+ }
+ )*
};
}
@@ -361,7 +342,7 @@ mod tests {
let cmd: Command = any.try_into().unwrap();
assert!(matches!(cmd, Command::CommandStatementQuery(_)));
- assert_eq!(cmd.type_url(), COMMAND_STATEMENT_QUERY_TYPE_URL);
+ assert_eq!(cmd.type_url(), CommandStatementQuery::type_url());
// Unknown variant
diff --git a/parquet-variant/Cargo.toml b/parquet-variant/Cargo.toml
index ba186432d2..c42a060794 100644
--- a/parquet-variant/Cargo.toml
+++ b/parquet-variant/Cargo.toml
@@ -47,7 +47,6 @@ name = "parquet_variant"
bench = false
[dev-dependencies]
-paste = { version = "1.0" }
criterion = { workspace = true, default-features = false }
rand = { version = "0.9", default-features = false, features = [
"std",
diff --git a/parquet-variant/src/decoder.rs b/parquet-variant/src/decoder.rs
index 9f15c0facd..d75f51e7cb 100644
--- a/parquet-variant/src/decoder.rs
+++ b/parquet-variant/src/decoder.rs
@@ -367,21 +367,22 @@ pub(crate) fn decode_short_string(
#[cfg(test)]
mod tests {
use super::*;
- use paste::paste;
macro_rules! test_decoder_bounds {
($test_name:ident, $data:expr, $decode_fn:ident, $expected:expr) => {
- paste! {
+ mod $test_name {
+ use super::*;
+
#[test]
- fn [<$test_name _exact_length>]() {
+ fn exact_length() {
let result = $decode_fn(&$data).unwrap();
assert_eq!(result, $expected);
}
#[test]
- fn [<$test_name _truncated_length>]() {
+ fn truncated_length() {
// Remove the last byte of data so that there is not
enough to decode
- let truncated_data = &$data[.. $data.len() - 1];
+ let truncated_data = &$data[..$data.len() - 1];
let result = $decode_fn(truncated_data);
assert!(matches!(result,
Err(ArrowError::InvalidArgumentError(_))));
}
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
index da449b3e57..f3342f03fc 100644
--- a/parquet/Cargo.toml
+++ b/parquet/Cargo.toml
@@ -71,7 +71,6 @@ futures = { version = "0.3", default-features = false,
features = ["std"], optio
tokio = { version = "1.0", optional = true, default-features = false, features
= ["macros", "rt", "io-util"] }
hashbrown = { version = "0.17", default-features = false }
twox-hash = { version = "2.0", default-features = false, features =
["xxhash64"] }
-paste = { version = "1.0" }
half = { version = "2.1", default-features = false, features = ["num-traits"] }
crc32fast = { version = "1.4.2", optional = true, default-features = false }
simdutf8 = { workspace = true , optional = true }
diff --git a/parquet/src/arrow/arrow_reader/statistics.rs
b/parquet/src/arrow/arrow_reader/statistics.rs
index 99bf28c8c7..c096448b6b 100644
--- a/parquet/src/arrow/arrow_reader/statistics.rs
+++ b/parquet/src/arrow/arrow_reader/statistics.rs
@@ -47,7 +47,6 @@ use arrow_array::{
use arrow_buffer::{NullBufferBuilder, i256};
use arrow_schema::{DataType, Field, Schema, TimeUnit};
use half::f16;
-use paste::paste;
use std::sync::Arc;
// Convert the bytes array to i32.
@@ -356,73 +355,125 @@ make_decimal_stats_iterator!(
from_bytes_to_i256
);
-/// Special macro to combine the statistics iterators for min and max using
the [`mod@paste`] macro.
+/// Special macro to combine the statistics iterators for min and max.
/// This is used to avoid repeating the same code for min and max statistics
extractions
///
/// Parameters:
-/// stat_type_prefix: The prefix of the statistics iterator type (e.g. `Min`
or `Max`)
/// data_type: The data type of the statistics (e.g. `DataType::Int32`)
/// iterator: The iterator of [`ParquetStatistics`] to extract the statistics
from.
macro_rules! get_statistics {
- ($stat_type_prefix: ident, $data_type: ident, $iterator: ident,
$physical_type: ident) => {
- paste! {
+ (Min, $data_type: ident, $iterator: ident, $physical_type: ident) => {
+ get_statistics!(
+ $data_type,
+ $iterator,
+ $physical_type,
+ MinBooleanStatsIterator,
+ MinInt32StatsIterator,
+ MinInt64StatsIterator,
+ MinFloatStatsIterator,
+ MinDoubleStatsIterator,
+ MinByteArrayStatsIterator,
+ MinFixedLenByteArrayStatsIterator,
+ MinDecimal32StatsIterator,
+ MinDecimal64StatsIterator,
+ MinDecimal128StatsIterator,
+ MinDecimal256StatsIterator,
+ min_statistics
+ )
+ };
+ (Max, $data_type: ident, $iterator: ident, $physical_type: ident) => {
+ get_statistics!(
+ $data_type,
+ $iterator,
+ $physical_type,
+ MaxBooleanStatsIterator,
+ MaxInt32StatsIterator,
+ MaxInt64StatsIterator,
+ MaxFloatStatsIterator,
+ MaxDoubleStatsIterator,
+ MaxByteArrayStatsIterator,
+ MaxFixedLenByteArrayStatsIterator,
+ MaxDecimal32StatsIterator,
+ MaxDecimal64StatsIterator,
+ MaxDecimal128StatsIterator,
+ MaxDecimal256StatsIterator,
+ max_statistics
+ )
+ };
+ (
+ $data_type: ident,
+ $iterator: ident,
+ $physical_type: ident,
+ $boolean_iter: ident,
+ $int32_iter: ident,
+ $int64_iter: ident,
+ $float_iter: ident,
+ $double_iter: ident,
+ $byte_array_iter: ident,
+ $fixed_len_byte_array_iter: ident,
+ $decimal32_iter: ident,
+ $decimal64_iter: ident,
+ $decimal128_iter: ident,
+ $decimal256_iter: ident,
+ $dictionary_statistics: ident
+ ) => {
match $data_type {
DataType::Boolean => Ok(Arc::new(BooleanArray::from_iter(
- [<$stat_type_prefix
BooleanStatsIterator>]::new($iterator).map(|x| x.copied()),
+ $boolean_iter::new($iterator).map(|x| x.copied()),
))),
DataType::Int8 => Ok(Arc::new(Int8Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
+ $int32_iter::new($iterator).map(|x| {
x.and_then(|x| i8::try_from(*x).ok())
}),
))),
DataType::Int16 => Ok(Arc::new(Int16Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
+ $int32_iter::new($iterator).map(|x| {
x.and_then(|x| i16::try_from(*x).ok())
}),
))),
DataType::Int32 => Ok(Arc::new(Int32Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int32_iter::new($iterator).map(|x| x.copied()),
))),
DataType::Int64 => Ok(Arc::new(Int64Array::from_iter(
- [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int64_iter::new($iterator).map(|x| x.copied()),
))),
DataType::UInt8 => Ok(Arc::new(UInt8Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
+ $int32_iter::new($iterator).map(|x| {
x.and_then(|x| u8::try_from(*x).ok())
}),
))),
DataType::UInt16 => Ok(Arc::new(UInt16Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
+ $int32_iter::new($iterator).map(|x| {
x.and_then(|x| u16::try_from(*x).ok())
}),
))),
DataType::UInt32 => Ok(Arc::new(UInt32Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| x.map(|x| *x as u32)),
+ $int32_iter::new($iterator).map(|x| x.map(|x| *x as u32)),
))),
DataType::UInt64 => Ok(Arc::new(UInt64Array::from_iter(
- [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.map(|x| *x as u64)),
+ $int64_iter::new($iterator).map(|x| x.map(|x| *x as u64)),
))),
DataType::Float16 => Ok(Arc::new(Float16Array::from_iter(
- [<$stat_type_prefix
FixedLenByteArrayStatsIterator>]::new($iterator).map(|x| x.and_then(|x| {
+ $fixed_len_byte_array_iter::new($iterator).map(|x|
x.and_then(|x| {
from_bytes_to_f16(x)
})),
))),
DataType::Float32 => Ok(Arc::new(Float32Array::from_iter(
- [<$stat_type_prefix
FloatStatsIterator>]::new($iterator).map(|x| x.copied()),
+ $float_iter::new($iterator).map(|x| x.copied()),
))),
DataType::Float64 => Ok(Arc::new(Float64Array::from_iter(
- [<$stat_type_prefix
DoubleStatsIterator>]::new($iterator).map(|x| x.copied()),
+ $double_iter::new($iterator).map(|x| x.copied()),
))),
DataType::Date32 => Ok(Arc::new(Date32Array::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int32_iter::new($iterator).map(|x| x.copied()),
))),
DataType::Date64 if $physical_type == Some(PhysicalType::INT32) =>
Ok(Arc::new(Date64Array::from_iter(
- [<$stat_type_prefix Int32StatsIterator>]::new($iterator)
+ $int32_iter::new($iterator)
.map(|x| x.map(|x| i64::from(*x) * 24 * 60 * 60 *
1000))))),
DataType::Date64 if $physical_type == Some(PhysicalType::INT64) =>
Ok(Arc::new(Date64Array::from_iter(
- [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.copied()),))),
+ $int64_iter::new($iterator).map(|x| x.copied()),))),
DataType::Timestamp(unit, timezone) =>{
- let iter = [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.copied());
+ let iter = $int64_iter::new($iterator).map(|x| x.copied());
Ok(match unit {
TimeUnit::Second =>
Arc::new(TimestampSecondArray::from_iter(iter).with_timezone_opt(timezone.clone())),
TimeUnit::Millisecond =>
Arc::new(TimestampMillisecondArray::from_iter(iter).with_timezone_opt(timezone.clone())),
@@ -433,10 +484,10 @@ macro_rules! get_statistics {
DataType::Time32(unit) => {
Ok(match unit {
TimeUnit::Second => Arc::new(Time32SecondArray::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int32_iter::new($iterator).map(|x| x.copied()),
)),
TimeUnit::Millisecond =>
Arc::new(Time32MillisecondArray::from_iter(
- [<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int32_iter::new($iterator).map(|x| x.copied()),
)),
_ => {
let len = $iterator.count();
@@ -448,10 +499,10 @@ macro_rules! get_statistics {
DataType::Time64(unit) => {
Ok(match unit {
TimeUnit::Microsecond =>
Arc::new(Time64MicrosecondArray::from_iter(
- [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int64_iter::new($iterator).map(|x| x.copied()),
)),
TimeUnit::Nanosecond =>
Arc::new(Time64NanosecondArray::from_iter(
- [<$stat_type_prefix
Int64StatsIterator>]::new($iterator).map(|x| x.copied()),
+ $int64_iter::new($iterator).map(|x| x.copied()),
)),
_ => {
let len = $iterator.count();
@@ -461,13 +512,13 @@ macro_rules! get_statistics {
})
},
DataType::Binary => Ok(Arc::new(BinaryArray::from_iter(
- [<$stat_type_prefix ByteArrayStatsIterator>]::new($iterator)
+ $byte_array_iter::new($iterator)
))),
DataType::LargeBinary => Ok(Arc::new(LargeBinaryArray::from_iter(
- [<$stat_type_prefix ByteArrayStatsIterator>]::new($iterator)
+ $byte_array_iter::new($iterator)
))),
DataType::Utf8 => {
- let iterator = [<$stat_type_prefix
ByteArrayStatsIterator>]::new($iterator);
+ let iterator = $byte_array_iter::new($iterator);
let mut builder = StringBuilder::new();
for x in iterator {
let Some(x) = x else {
@@ -485,7 +536,7 @@ macro_rules! get_statistics {
Ok(Arc::new(builder.finish()))
},
DataType::LargeUtf8 => {
- let iterator = [<$stat_type_prefix
ByteArrayStatsIterator>]::new($iterator);
+ let iterator = $byte_array_iter::new($iterator);
let mut builder = LargeStringBuilder::new();
for x in iterator {
let Some(x) = x else {
@@ -503,7 +554,7 @@ macro_rules! get_statistics {
Ok(Arc::new(builder.finish()))
},
DataType::FixedSizeBinary(size) => {
- let iterator = [<$stat_type_prefix
FixedLenByteArrayStatsIterator>]::new($iterator);
+ let iterator = $fixed_len_byte_array_iter::new($iterator);
let mut builder = FixedSizeBinaryBuilder::new(*size);
for x in iterator {
let Some(x) = x else {
@@ -523,33 +574,33 @@ macro_rules! get_statistics {
},
DataType::Decimal32(precision, scale) => {
let arr = Decimal32Array::from_iter(
- [<$stat_type_prefix
Decimal32StatsIterator>]::new($iterator)
+ $decimal32_iter::new($iterator)
).with_precision_and_scale(*precision, *scale)?;
Ok(Arc::new(arr))
},
DataType::Decimal64(precision, scale) => {
let arr = Decimal64Array::from_iter(
- [<$stat_type_prefix
Decimal64StatsIterator>]::new($iterator)
+ $decimal64_iter::new($iterator)
).with_precision_and_scale(*precision, *scale)?;
Ok(Arc::new(arr))
},
DataType::Decimal128(precision, scale) => {
let arr = Decimal128Array::from_iter(
- [<$stat_type_prefix
Decimal128StatsIterator>]::new($iterator)
+ $decimal128_iter::new($iterator)
).with_precision_and_scale(*precision, *scale)?;
Ok(Arc::new(arr))
},
DataType::Decimal256(precision, scale) => {
let arr = Decimal256Array::from_iter(
- [<$stat_type_prefix
Decimal256StatsIterator>]::new($iterator)
+ $decimal256_iter::new($iterator)
).with_precision_and_scale(*precision, *scale)?;
Ok(Arc::new(arr))
},
DataType::Dictionary(_, value_type) => {
- [<$stat_type_prefix:lower _ statistics>](value_type,
$iterator, $physical_type)
+ $dictionary_statistics(value_type, $iterator, $physical_type)
},
DataType::Utf8View => {
- let iterator = [<$stat_type_prefix
ByteArrayStatsIterator>]::new($iterator);
+ let iterator = $byte_array_iter::new($iterator);
let mut builder = StringViewBuilder::new();
for x in iterator {
let Some(x) = x else {
@@ -567,7 +618,7 @@ macro_rules! get_statistics {
Ok(Arc::new(builder.finish()))
},
DataType::BinaryView => {
- let iterator = [<$stat_type_prefix
ByteArrayStatsIterator>]::new($iterator);
+ let iterator = $byte_array_iter::new($iterator);
let mut builder = BinaryViewBuilder::new();
for x in iterator {
let Some(x) = x else {
@@ -597,22 +648,45 @@ macro_rules! get_statistics {
// don't know how to extract statistics, so return a null array
Ok(new_null_array($data_type, len))
}
- }}}
+ }
+ };
}
macro_rules! get_data_page_statistics {
- ($stat_type_prefix: ident, $data_type: ident, $iterator: ident,
$physical_type: ident) => {
- {
- let chunks: Vec<(usize, &ColumnIndexMetaData)> =
$iterator.collect();
- let capacity: usize = chunks.iter().map(|c| c.0).sum();
- paste! {
- match $data_type {
+ (Min, $data_type: ident, $iterator: ident, $physical_type: ident) => {
+ get_data_page_statistics!(
+ $data_type,
+ $iterator,
+ $physical_type,
+ min_values_iter,
+ min_page_statistics
+ )
+ };
+ (Max, $data_type: ident, $iterator: ident, $physical_type: ident) => {
+ get_data_page_statistics!(
+ $data_type,
+ $iterator,
+ $physical_type,
+ max_values_iter,
+ max_page_statistics
+ )
+ };
+ (
+ $data_type: ident,
+ $iterator: ident,
+ $physical_type: ident,
+ $values_iter: ident,
+ $page_statistics: ident
+ ) => {{
+ let chunks: Vec<(usize, &ColumnIndexMetaData)> = $iterator.collect();
+ let capacity: usize = chunks.iter().map(|c| c.0).sum();
+ match $data_type {
DataType::Boolean => {
let mut b = BooleanBuilder::with_capacity(capacity);
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BOOLEAN(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
b.append_option(val.copied());
}
}
@@ -627,7 +701,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|&x|
u8::try_from(x).ok())),
);
}
@@ -642,7 +716,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|&x|
u16::try_from(x).ok())),
);
}
@@ -657,7 +731,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|&x| x as u32)),
);
}
@@ -672,7 +746,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|&x| x as u64)),
);
}
@@ -687,7 +761,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|&x|
i8::try_from(x).ok())),
);
}
@@ -702,7 +776,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|&x|
i16::try_from(x).ok())),
);
}
@@ -717,7 +791,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -732,7 +806,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -747,7 +821,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|x|
from_bytes_to_f16(x))),
);
}
@@ -762,7 +836,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::FLOAT(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -777,7 +851,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::DOUBLE(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -791,7 +865,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
b.append_option(val.map(|x| x.as_ref()));
}
}
@@ -805,7 +879,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
b.append_option(val.map(|x| x.as_ref()));
}
}
@@ -819,7 +893,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
match val {
Some(x) => match
std::str::from_utf8(x.as_ref()) {
Ok(s) => b.append_value(s),
@@ -839,7 +913,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
match val {
Some(x) => match
std::str::from_utf8(x.as_ref()) {
Ok(s) => b.append_value(s),
@@ -855,7 +929,7 @@ macro_rules! get_data_page_statistics {
Ok(Arc::new(b.finish()))
},
DataType::Dictionary(_, value_type) => {
- [<$stat_type_prefix:lower _ page_statistics>](value_type,
chunks.into_iter(), $physical_type)
+ $page_statistics(value_type, chunks.into_iter(),
$physical_type)
},
DataType::Timestamp(unit, timezone) => {
match unit {
@@ -865,7 +939,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -880,7 +954,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -895,7 +969,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -910,7 +984,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -927,7 +1001,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -942,7 +1016,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|&x| (x as i64) *
24 * 60 * 60 * 1000)),
);
}
@@ -957,7 +1031,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -972,25 +1046,25 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.and_then(|&x|
i32::try_from(x).ok())),
);
}
ColumnIndexMetaData::BYTE_ARRAY(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i32(x.as_ref()))),
);
}
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i32(x.as_ref()))),
);
}
@@ -1005,25 +1079,25 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x| *x as i64)),
);
}
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
ColumnIndexMetaData::BYTE_ARRAY(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i64(x.as_ref()))),
);
}
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i64(x.as_ref()))),
);
}
@@ -1038,25 +1112,25 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x| *x as i128)),
);
}
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x| *x as i128)),
);
}
ColumnIndexMetaData::BYTE_ARRAY(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i128(x.as_ref()))),
);
}
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i128(x.as_ref()))),
);
}
@@ -1071,25 +1145,25 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
i256::from_i128(*x as i128))),
);
}
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
i256::from_i128(*x as i128))),
);
}
ColumnIndexMetaData::BYTE_ARRAY(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i256(x.as_ref()))),
);
}
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.map(|x|
from_bytes_to_i256(x.as_ref()))),
);
}
@@ -1106,7 +1180,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -1121,7 +1195,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT32(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -1143,7 +1217,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -1158,7 +1232,7 @@ macro_rules! get_data_page_statistics {
match index {
ColumnIndexMetaData::INT64(index) => {
b.extend_from_iter_option(
- index.[<$stat_type_prefix:lower
_values_iter>]()
+ index.$values_iter()
.map(|val| val.copied()),
);
}
@@ -1177,7 +1251,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
=> {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
match val {
Some(v) => {
if v.len() == *size as usize {
@@ -1200,7 +1274,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
match val {
Some(x) => match
std::str::from_utf8(x.as_ref()) {
Ok(s) => b.append_value(s),
@@ -1222,7 +1296,7 @@ macro_rules! get_data_page_statistics {
for (len, index) in chunks {
match index {
ColumnIndexMetaData::BYTE_ARRAY(index) => {
- for val in index.[<$stat_type_prefix:lower
_values_iter>]() {
+ for val in index.$values_iter() {
match val {
Some(v) => b.append_value(v.as_ref()),
None => b.append_null(),
@@ -1254,8 +1328,7 @@ macro_rules! get_data_page_statistics {
},
}
}
- }
- }
+ };
}
/// Extracts the min statistics from an iterator of [`ParquetStatistics`] to an
/// [`ArrayRef`]
diff --git a/parquet/tests/variant_integration.rs
b/parquet/tests/variant_integration.rs
index b97b3f2254..f4525185f6 100644
--- a/parquet/tests/variant_integration.rs
+++ b/parquet/tests/variant_integration.rs
@@ -41,13 +41,13 @@ type Result<T> = std::result::Result<T, String>;
/// Note the index is zero-based, while the case number is one-based
macro_rules! variant_test_case {
($case_num:literal $(, $expected_error:literal )? ) => {
- paste::paste! {
+ seq_macro::seq!(N in $case_num..=$case_num {
#[test]
$( #[should_panic(expected = $expected_error)] )?
- fn [<test_variant_integration_case_ $case_num>]() {
- all_cases()[$case_num - 1].run()
+ fn test_variant_integration_case_~N() {
+ all_cases()[N - 1].run()
}
- }
+ });
};
}
