This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 715318e fix: Allow parquet to be compiled without arrow (fix
--no-default-features) (#731)
715318e is described below
commit 715318eb6f424003b101533abd456706ad1640a4
Author: Markus Westerlind <[email protected]>
AuthorDate: Thu Sep 9 22:00:05 2021 +0200
fix: Allow parquet to be compiled without arrow (fix --no-default-features)
(#731)
* fix: Allow parquet to be compiled without arrow
`--no-default-features` is currently broken in the parquet crate due to
arrow being required. With some small tweaks it can be made entirely
optional.
Added some extra steps to catch when `--no-default-features` does not
work on CI as well.
* Fix CI
* Fix path on CI
* --features test_common is needed for clippy
---
.github/workflows/rust.yml | 7 ++++++-
parquet/Cargo.toml | 3 ++-
parquet/src/data_type.rs | 3 +--
parquet/src/util/bit_util.rs | 9 +++++++++
parquet/src/util/mod.rs | 2 ++
5 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index d76192c..2014fae 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -118,6 +118,11 @@ jobs:
cargo run --example dynamic_types
cargo run --example read_csv
cargo run --example read_csv_infer_schema
+ # Exit arrow directory
+ cd ..
+ (cd parquet && cargo check --no-default-features)
+ (cd arrow && cargo check --no-default-features)
+ (cd arrow-flight && cargo check --no-default-features)
# test the --features "simd" of the arrow crate. This requires nightly.
linux-test-simd:
@@ -234,7 +239,7 @@ jobs:
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
- cargo clippy --all-targets --workspace -- -D warnings -A
clippy::redundant_field_names
+ cargo clippy --features test_common --all-targets --workspace -- -D
warnings -A clippy::redundant_field_names
lint:
name: Lint
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
index 9a3a245..0d1b4d7 100644
--- a/parquet/Cargo.toml
+++ b/parquet/Cargo.toml
@@ -60,6 +60,7 @@ serde_json = { version = "1.0", features = ["preserve_order"]
}
[features]
default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"]
cli = ["serde_json", "base64", "clap"]
+test_common = []
[[ bin ]]
name = "parquet-read"
@@ -79,4 +80,4 @@ harness = false
[[bench]]
name = "arrow_array_reader"
-harness = false
\ No newline at end of file
+harness = false
diff --git a/parquet/src/data_type.rs b/parquet/src/data_type.rs
index dadcba1..8c64e86 100644
--- a/parquet/src/data_type.rs
+++ b/parquet/src/data_type.rs
@@ -585,10 +585,9 @@ impl AsBytes for str {
pub(crate) mod private {
use crate::encodings::decoding::PlainDecoderDetails;
- use crate::util::bit_util::{BitReader, BitWriter};
+ use crate::util::bit_util::{round_upto_power_of_2, BitReader, BitWriter};
use crate::util::memory::ByteBufferPtr;
- use arrow::util::bit_util::round_upto_power_of_2;
use byteorder::ByteOrder;
use std::convert::TryInto;
diff --git a/parquet/src/util/bit_util.rs b/parquet/src/util/bit_util.rs
index 4b34df4..010ed32 100644
--- a/parquet/src/util/bit_util.rs
+++ b/parquet/src/util/bit_util.rs
@@ -680,6 +680,15 @@ impl From<Vec<u8>> for BitReader {
}
}
+/// Returns the nearest multiple of `factor` that is `>=` than `num`. Here
`factor` must
+/// be a power of 2.
+///
+/// Copied from the arrow crate to make arrow optional
+pub fn round_upto_power_of_2(num: usize, factor: usize) -> usize {
+ debug_assert!(factor > 0 && (factor & (factor - 1)) == 0);
+ (num + (factor - 1)) & !(factor - 1)
+}
+
#[cfg(test)]
mod tests {
use super::super::test_common::*;
diff --git a/parquet/src/util/mod.rs b/parquet/src/util/mod.rs
index 8f6d85d..3a69df4 100644
--- a/parquet/src/util/mod.rs
+++ b/parquet/src/util/mod.rs
@@ -22,7 +22,9 @@ pub mod bit_util;
mod bit_packing;
pub mod cursor;
pub mod hash_util;
+#[cfg(any(test, feature = "test_common"))]
pub(crate) mod test_common;
+#[cfg(any(test, feature = "test_common"))]
pub use self::test_common::page_util::{
DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
};
