This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 209eafa75a Avoid panics and warnings when building avro without
default features (#8576)
209eafa75a is described below
commit 209eafa75ad94ad8b6eda953153c590bd9b9a8a5
Author: Matthijs Brobbel <[email protected]>
AuthorDate: Thu Oct 9 22:46:42 2025 +0200
Avoid panics and warnings when building avro without default features
(#8576)
# Which issue does this PR close?
None.
# Rationale for this change
`cargo -p arrow-avro test --no-default-features` builds with warnings
and panics because of missing features.
# What changes are included in this PR?
Add the required feature gates to avoid problems when building without
default features.
# Are these changes tested?
Tested: `cargo -p arrow-avro test --no-default-features`
# Are there any user-facing changes?
No.
---
arrow-avro/benches/avro_writer.rs | 15 +++++--
arrow-avro/src/compression.rs | 8 ++++
arrow-avro/src/reader/mod.rs | 93 +++++++++++++++++++++++----------------
arrow-avro/src/writer/mod.rs | 31 +++++++++----
4 files changed, 97 insertions(+), 50 deletions(-)
diff --git a/arrow-avro/benches/avro_writer.rs
b/arrow-avro/benches/avro_writer.rs
index 57ab486638..f15f13b989 100644
--- a/arrow-avro/benches/avro_writer.rs
+++ b/arrow-avro/benches/avro_writer.rs
@@ -22,12 +22,13 @@ extern crate criterion;
extern crate once_cell;
use arrow_array::{
- ArrayRef, BinaryArray, BooleanArray, Decimal32Array, Decimal64Array,
Decimal128Array,
- Decimal256Array, FixedSizeBinaryArray, Float32Array, Float64Array,
ListArray, PrimitiveArray,
- RecordBatch, StringArray, StructArray,
+ ArrayRef, BinaryArray, BooleanArray, Decimal128Array, Decimal256Array,
FixedSizeBinaryArray,
+ Float32Array, Float64Array, ListArray, PrimitiveArray, RecordBatch,
StringArray, StructArray,
builder::{ListBuilder, StringBuilder},
types::{Int32Type, Int64Type, IntervalMonthDayNanoType,
TimestampMicrosecondType},
};
+#[cfg(feature = "small_decimals")]
+use arrow_array::{Decimal32Array, Decimal64Array};
use arrow_avro::writer::AvroWriter;
use arrow_buffer::{Buffer, i256};
use arrow_schema::{DataType, Field, IntervalUnit, Schema, TimeUnit,
UnionFields, UnionMode};
@@ -177,11 +178,13 @@ fn make_ts_micros_array_with_tag(n: usize, tag: u64) ->
PrimitiveArray<Timestamp
// === Decimal helpers & generators ===
#[inline]
+#[cfg(feature = "small_decimals")]
fn pow10_i32(p: u8) -> i32 {
(0..p).fold(1i32, |acc, _| acc.saturating_mul(10))
}
#[inline]
+#[cfg(feature = "small_decimals")]
fn pow10_i64(p: u8) -> i64 {
(0..p).fold(1i64, |acc, _| acc.saturating_mul(10))
}
@@ -192,6 +195,7 @@ fn pow10_i128(p: u8) -> i128 {
}
#[inline]
+#[cfg(feature = "small_decimals")]
fn make_decimal32_array_with_tag(n: usize, tag: u64, precision: u8, scale: i8)
-> Decimal32Array {
let mut rng = rng_for(tag, n);
let max = pow10_i32(precision).saturating_sub(1);
@@ -202,6 +206,7 @@ fn make_decimal32_array_with_tag(n: usize, tag: u64,
precision: u8, scale: i8) -
}
#[inline]
+#[cfg(feature = "small_decimals")]
fn make_decimal64_array_with_tag(n: usize, tag: u64, precision: u8, scale: i8)
-> Decimal64Array {
let mut rng = rng_for(tag, n);
let max = pow10_i64(precision).saturating_sub(1);
@@ -539,6 +544,7 @@ static STRUCT_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
.collect()
});
+#[cfg(feature = "small_decimals")]
static DECIMAL32_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
// Choose a representative precision/scale within Decimal32 limits
let precision: u8 = 7;
@@ -554,6 +560,7 @@ static DECIMAL32_DATA: Lazy<Vec<RecordBatch>> =
Lazy::new(|| {
.collect()
});
+#[cfg(feature = "small_decimals")]
static DECIMAL64_DATA: Lazy<Vec<RecordBatch>> = Lazy::new(|| {
let precision: u8 = 13;
let scale: i8 = 3;
@@ -821,7 +828,9 @@ fn criterion_benches(c: &mut Criterion) {
bench_writer_scenario(c, "write-FixedSizeBinary16", &FIXED16_DATA);
bench_writer_scenario(c, "write-UUID(logicalType)", &UUID16_DATA);
bench_writer_scenario(c, "write-IntervalMonthDayNanoDuration",
&INTERVAL_MDN_DATA);
+ #[cfg(feature = "small_decimals")]
bench_writer_scenario(c, "write-Decimal32(bytes)", &DECIMAL32_DATA);
+ #[cfg(feature = "small_decimals")]
bench_writer_scenario(c, "write-Decimal64(bytes)", &DECIMAL64_DATA);
bench_writer_scenario(c, "write-Decimal128(bytes)",
&DECIMAL128_BYTES_DATA);
bench_writer_scenario(c, "write-Decimal128(fixed16)",
&DECIMAL128_FIXED16_DATA);
diff --git a/arrow-avro/src/compression.rs b/arrow-avro/src/compression.rs
index 29d1732b9e..0cb2878a13 100644
--- a/arrow-avro/src/compression.rs
+++ b/arrow-avro/src/compression.rs
@@ -16,6 +16,12 @@
// under the License.
use arrow_schema::ArrowError;
+#[cfg(any(
+ feature = "deflate",
+ feature = "zstd",
+ feature = "bzip2",
+ feature = "xz"
+))]
use std::io::{Read, Write};
/// The metadata key used for storing the JSON encoded [`CompressionCodec`]
@@ -40,6 +46,7 @@ pub enum CompressionCodec {
}
impl CompressionCodec {
+ #[allow(unused_variables)]
pub(crate) fn decompress(&self, block: &[u8]) -> Result<Vec<u8>,
ArrowError> {
match self {
#[cfg(feature = "deflate")]
@@ -112,6 +119,7 @@ impl CompressionCodec {
}
}
+ #[allow(unused_variables)]
pub(crate) fn compress(&self, data: &[u8]) -> Result<Vec<u8>, ArrowError> {
match self {
#[cfg(feature = "deflate")]
diff --git a/arrow-avro/src/reader/mod.rs b/arrow-avro/src/reader/mod.rs
index def0fffe39..2107156e7a 100644
--- a/arrow-avro/src/reader/mod.rs
+++ b/arrow-avro/src/reader/mod.rs
@@ -1273,11 +1273,12 @@ mod test {
};
use crate::test_util::arrow_test_data;
use crate::writer::AvroWriter;
- use arrow::array::ArrayDataBuilder;
use arrow_array::builder::{
- ArrayBuilder, BooleanBuilder, Float32Builder, Float64Builder,
Int32Builder, Int64Builder,
- ListBuilder, MapBuilder, MapFieldNames, StringBuilder, StructBuilder,
+ ArrayBuilder, BooleanBuilder, Float32Builder, Int32Builder,
Int64Builder, ListBuilder,
+ MapBuilder, StringBuilder, StructBuilder,
};
+ #[cfg(feature = "snappy")]
+ use arrow_array::builder::{Float64Builder, MapFieldNames};
use arrow_array::cast::AsArray;
#[cfg(not(feature = "avro_custom_types"))]
use arrow_array::types::Int64Type;
@@ -1288,9 +1289,9 @@ mod test {
};
use arrow_array::types::{Int32Type, IntervalMonthDayNanoType};
use arrow_array::*;
- use arrow_buffer::{
- Buffer, IntervalMonthDayNano, NullBuffer, OffsetBuffer, ScalarBuffer,
i256,
- };
+ #[cfg(feature = "snappy")]
+ use arrow_buffer::{Buffer, NullBuffer};
+ use arrow_buffer::{IntervalMonthDayNano, OffsetBuffer, ScalarBuffer, i256};
#[cfg(feature = "avro_custom_types")]
use arrow_schema::{
ArrowError, DataType, Field, FieldRef, Fields, IntervalUnit, Schema,
TimeUnit, UnionFields,
@@ -1309,6 +1310,23 @@ mod test {
use std::io::{BufReader, Cursor};
use std::sync::Arc;
+ fn files() -> impl Iterator<Item = &'static str> {
+ [
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
+ "avro/alltypes_plain.avro",
+ #[cfg(feature = "snappy")]
+ "avro/alltypes_plain.snappy.avro",
+ #[cfg(feature = "zstd")]
+ "avro/alltypes_plain.zstandard.avro",
+ #[cfg(feature = "bzip2")]
+ "avro/alltypes_plain.bzip2.avro",
+ #[cfg(feature = "xz")]
+ "avro/alltypes_plain.xz.avro",
+ ]
+ .into_iter()
+ }
+
fn read_file(path: &str, batch_size: usize, utf8_view: bool) ->
RecordBatch {
let file = File::open(path).unwrap();
let reader = ReaderBuilder::new()
@@ -1714,14 +1732,7 @@ mod test {
#[test]
fn test_alltypes_schema_promotion_mixed() {
- let files = [
- "avro/alltypes_plain.avro",
- "avro/alltypes_plain.snappy.avro",
- "avro/alltypes_plain.zstandard.avro",
- "avro/alltypes_plain.bzip2.avro",
- "avro/alltypes_plain.xz.avro",
- ];
- for file in files {
+ for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("id", "long");
@@ -1829,14 +1840,7 @@ mod test {
#[test]
fn test_alltypes_schema_promotion_long_to_float_only() {
- let files = [
- "avro/alltypes_plain.avro",
- "avro/alltypes_plain.snappy.avro",
- "avro/alltypes_plain.zstandard.avro",
- "avro/alltypes_plain.bzip2.avro",
- "avro/alltypes_plain.xz.avro",
- ];
- for file in files {
+ for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("bigint_col", "float");
@@ -1933,14 +1937,7 @@ mod test {
#[test]
fn test_alltypes_schema_promotion_bytes_to_string_only() {
- let files = [
- "avro/alltypes_plain.avro",
- "avro/alltypes_plain.snappy.avro",
- "avro/alltypes_plain.zstandard.avro",
- "avro/alltypes_plain.bzip2.avro",
- "avro/alltypes_plain.xz.avro",
- ];
- for file in files {
+ for file in files() {
let file = arrow_test_data(file);
let mut promotions: HashMap<&str, &str> = HashMap::new();
promotions.insert("date_string_col", "string");
@@ -2033,6 +2030,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_alltypes_illegal_promotion_bool_to_double_errors() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let mut promotions: HashMap<&str, &str> = HashMap::new();
@@ -2691,6 +2690,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_alltypes_skip_writer_fields_keep_double_only() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let reader_schema =
@@ -2708,6 +2709,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_alltypes_skip_writer_fields_reorder_and_skip_many() {
let file = arrow_test_data("avro/alltypes_plain.avro");
let reader_schema =
@@ -4470,14 +4473,6 @@ mod test {
#[test]
fn test_alltypes() {
- let files = [
- "avro/alltypes_plain.avro",
- "avro/alltypes_plain.snappy.avro",
- "avro/alltypes_plain.zstandard.avro",
- "avro/alltypes_plain.bzip2.avro",
- "avro/alltypes_plain.xz.avro",
- ];
-
let expected = RecordBatch::try_from_iter_with_nullable([
(
"id",
@@ -4562,7 +4557,7 @@ mod test {
])
.unwrap();
- for file in files {
+ for file in files() {
let file = arrow_test_data(file);
assert_eq!(read_file(&file, 8, false), expected);
@@ -4571,6 +4566,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_alltypes_dictionary() {
let file = "avro/alltypes_dictionary.avro";
let expected = RecordBatch::try_from_iter_with_nullable([
@@ -4693,6 +4690,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_binary() {
let file = arrow_test_data("avro/binary.avro");
let batch = read_file(&file, 8, false);
@@ -4719,6 +4718,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for these files
+ #[cfg(feature = "snappy")]
fn test_decimal() {
// Choose expected Arrow types depending on the `small_decimals`
feature flag.
// With `small_decimals` enabled, Decimal32/Decimal64 are used where
their
@@ -5040,6 +5041,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_dict_pages_offset_zero() {
let file = arrow_test_data("avro/dict-page-offset-zero.avro");
let batch = read_file(&file, 32, false);
@@ -5055,6 +5058,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_list_columns() {
let file = arrow_test_data("avro/list_columns.avro");
let mut int64_list_builder = ListBuilder::new(Int64Builder::new());
@@ -5117,6 +5122,7 @@ mod test {
}
#[test]
+ #[cfg(feature = "snappy")]
fn test_nested_lists() {
use arrow_data::ArrayDataBuilder;
let file = arrow_test_data("avro/nested_lists.snappy.avro");
@@ -5314,6 +5320,7 @@ mod test {
}
#[test]
+ #[cfg(feature = "snappy")]
fn test_single_nan() {
let file = arrow_test_data("avro/single_nan.avro");
let actual = read_file(&file, 1, false);
@@ -5392,6 +5399,7 @@ mod test {
}
#[test]
+ #[cfg(feature = "snappy")]
fn test_datapage_v2() {
let file = arrow_test_data("avro/datapage_v2.snappy.avro");
let batch = read_file(&file, 8, false);
@@ -5653,7 +5661,10 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_repeated_no_annotation() {
+ use arrow_data::ArrayDataBuilder;
let file = arrow_test_data("avro/repeated_no_annotation.avro");
let batch_large = read_file(&file, 8, false);
// id column
@@ -5742,6 +5753,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_nonnullable_impala() {
let file = arrow_test_data("avro/nonnullable.impala.avro");
let id = Int64Array::from(vec![Some(8)]);
@@ -6057,6 +6070,8 @@ mod test {
}
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_nullable_impala() {
let file = arrow_test_data("avro/nullable.impala.avro");
let batch1 = read_file(&file, 3, false);
diff --git a/arrow-avro/src/writer/mod.rs b/arrow-avro/src/writer/mod.rs
index 0b87d6f13f..2e89312cc2 100644
--- a/arrow-avro/src/writer/mod.rs
+++ b/arrow-avro/src/writer/mod.rs
@@ -409,6 +409,23 @@ mod tests {
use std::sync::Arc;
use tempfile::NamedTempFile;
+ fn files() -> impl Iterator<Item = &'static str> {
+ [
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
+ "avro/alltypes_plain.avro",
+ #[cfg(feature = "snappy")]
+ "avro/alltypes_plain.snappy.avro",
+ #[cfg(feature = "zstd")]
+ "avro/alltypes_plain.zstandard.avro",
+ #[cfg(feature = "bzip2")]
+ "avro/alltypes_plain.bzip2.avro",
+ #[cfg(feature = "xz")]
+ "avro/alltypes_plain.xz.avro",
+ ]
+ .into_iter()
+ }
+
fn make_schema() -> Schema {
Schema::new(vec![
Field::new("id", DataType::Int32, false),
@@ -556,14 +573,7 @@ mod tests {
#[test]
fn test_roundtrip_alltypes_roundtrip_writer() -> Result<(), ArrowError> {
- let files = [
- "avro/alltypes_plain.avro",
- "avro/alltypes_plain.snappy.avro",
- "avro/alltypes_plain.zstandard.avro",
- "avro/alltypes_plain.bzip2.avro",
- "avro/alltypes_plain.xz.avro",
- ];
- for rel in files {
+ for rel in files() {
let path = arrow_test_data(rel);
let rdr_file = File::open(&path).expect("open input avro");
let reader = ReaderBuilder::new()
@@ -644,6 +654,7 @@ mod tests {
}
#[test]
+ #[cfg(feature = "snappy")]
fn test_roundtrip_nested_lists_writer() -> Result<(), ArrowError> {
let path = arrow_test_data("avro/nested_lists.snappy.avro");
let rdr_file = File::open(&path).expect("open
nested_lists.snappy.avro");
@@ -837,6 +848,8 @@ mod tests {
// writes it back out with the writer (hitting Map encoding paths), then
reads it
// again and asserts exact Arrow equivalence.
#[test]
+ // TODO: avoid requiring snappy for this file
+ #[cfg(feature = "snappy")]
fn test_nonnullable_impala_roundtrip_writer() -> Result<(), ArrowError> {
// Load source Avro with Map fields
let path = arrow_test_data("avro/nonnullable.impala.avro");
@@ -882,6 +895,8 @@ mod tests {
}
#[test]
+ // TODO: avoid requiring snappy for these files
+ #[cfg(feature = "snappy")]
fn test_roundtrip_decimals_via_writer() -> Result<(), ArrowError> {
// (file, resolve via ARROW_TEST_DATA?)
let files: [(&str, bool); 8] = [
