This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 5302b92 Move more parquet functionality behind experimental feature
flag (#1032) (#1134)
5302b92 is described below
commit 5302b92012e86c5c966378b1564027466a545430
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Mon Jan 10 21:50:31 2022 +0000
Move more parquet functionality behind experimental feature flag (#1032)
(#1134)
* Move more parquet functionality behind experimental feature flag (#1032)
* Fix logical conflicts
---
parquet/src/arrow/array_reader.rs | 4 +--
parquet/src/arrow/arrow_array_reader.rs | 8 +++---
parquet/src/arrow/arrow_writer.rs | 7 +++---
parquet/src/arrow/mod.rs | 8 +++---
parquet/src/compression.rs | 44 ++++++++++++++++++---------------
parquet/src/encodings/encoding.rs | 2 +-
parquet/src/lib.rs | 17 ++++++++-----
7 files changed, 51 insertions(+), 39 deletions(-)
diff --git a/parquet/src/arrow/array_reader.rs
b/parquet/src/arrow/array_reader.rs
index 2190076..752ca4c 100644
--- a/parquet/src/arrow/array_reader.rs
+++ b/parquet/src/arrow/array_reader.rs
@@ -2469,8 +2469,8 @@ mod tests {
#[test]
fn test_complex_array_reader_dict_enc_string() {
- use crate::encoding::{DictEncoder, Encoder};
- use crate::memory::MemTracker;
+ use crate::encodings::encoding::{DictEncoder, Encoder};
+ use crate::util::memory::MemTracker;
// Construct column schema
let message_type = "
message test_schema {
diff --git a/parquet/src/arrow/arrow_array_reader.rs
b/parquet/src/arrow/arrow_array_reader.rs
index cd1e92b..d814f7f 100644
--- a/parquet/src/arrow/arrow_array_reader.rs
+++ b/parquet/src/arrow/arrow_array_reader.rs
@@ -19,12 +19,12 @@ use super::array_reader::ArrayReader;
use crate::arrow::schema::parquet_to_arrow_field;
use crate::basic::Encoding;
use crate::data_type::{ByteArray, ByteArrayType};
-use crate::decoding::{Decoder, DeltaByteArrayDecoder};
+use crate::encodings::decoding::{Decoder, DeltaByteArrayDecoder};
use crate::errors::{ParquetError, Result};
use crate::{
column::page::{Page, PageIterator},
- memory::ByteBufferPtr,
schema::types::{ColumnDescPtr, ColumnDescriptor},
+ util::memory::ByteBufferPtr,
};
use arrow::{
array::{ArrayRef, Int16Array},
@@ -1263,12 +1263,12 @@ mod tests {
use crate::column::writer::ColumnWriter;
use crate::data_type::ByteArray;
use crate::data_type::ByteArrayType;
- use crate::encoding::{DictEncoder, Encoder};
+ use crate::encodings::encoding::{DictEncoder, Encoder};
use crate::file::properties::WriterProperties;
use crate::file::reader::SerializedFileReader;
use crate::file::serialized_reader::SliceableCursor;
use crate::file::writer::{FileWriter, SerializedFileWriter, TryClone};
- use crate::memory::MemTracker;
+ use crate::util::memory::MemTracker;
use crate::schema::parser::parse_message_type;
use crate::schema::types::SchemaDescriptor;
use crate::util::test_common::page_util::{
diff --git a/parquet/src/arrow/arrow_writer.rs
b/parquet/src/arrow/arrow_writer.rs
index 82c6d03..7f255db 100644
--- a/parquet/src/arrow/arrow_writer.rs
+++ b/parquet/src/arrow/arrow_writer.rs
@@ -26,7 +26,8 @@ use arrow_array::Array;
use super::levels::LevelInfo;
use super::schema::{
- add_encoded_arrow_schema_to_metadata, decimal_length_from_precision,
+ add_encoded_arrow_schema_to_metadata, arrow_to_parquet_schema,
+ decimal_length_from_precision,
};
use crate::column::writer::ColumnWriter;
@@ -62,7 +63,7 @@ impl<W: 'static + ParquetWriter> ArrowWriter<W> {
arrow_schema: SchemaRef,
props: Option<WriterProperties>,
) -> Result<Self> {
- let schema = crate::arrow::arrow_to_parquet_schema(&arrow_schema)?;
+ let schema = arrow_to_parquet_schema(&arrow_schema)?;
// add serialized arrow schema
let mut props = props.unwrap_or_else(||
WriterProperties::builder().build());
add_encoded_arrow_schema_to_metadata(&arrow_schema, &mut props);
@@ -470,7 +471,7 @@ macro_rules! def_get_binary_array_fn {
($name:ident, $ty:ty) => {
fn $name(array: &$ty) -> Vec<ByteArray> {
let mut byte_array = ByteArray::new();
- let ptr = crate::memory::ByteBufferPtr::new(
+ let ptr = crate::util::memory::ByteBufferPtr::new(
unsafe { array.value_data().typed_data::<u8>() }.to_vec(),
);
byte_array.set_data(ptr);
diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs
index 7927bf4..57ad5b1 100644
--- a/parquet/src/arrow/mod.rs
+++ b/parquet/src/arrow/mod.rs
@@ -119,17 +119,19 @@
//! ```
experimental_mod!(array_reader);
-pub mod arrow_array_reader;
+experimental_mod!(arrow_array_reader);
pub mod arrow_reader;
pub mod arrow_writer;
-pub mod converter;
+experimental_mod!(converter);
pub(in crate::arrow) mod levels;
pub(in crate::arrow) mod record_reader;
-pub mod schema;
+experimental_mod!(schema);
pub use self::arrow_reader::ArrowReader;
pub use self::arrow_reader::ParquetFileArrowReader;
pub use self::arrow_writer::ArrowWriter;
+
+#[cfg(feature = "experimental")]
pub use self::schema::{
arrow_to_parquet_schema, parquet_to_arrow_schema,
parquet_to_arrow_schema_by_columns,
parquet_to_arrow_schema_by_root_columns,
diff --git a/parquet/src/compression.rs b/parquet/src/compression.rs
index a115597..5d6554d 100644
--- a/parquet/src/compression.rs
+++ b/parquet/src/compression.rs
@@ -20,26 +20,30 @@
//! See [`Compression`](crate::basic::Compression) enum for all available
compression
//! algorithms.
//!
-//! # Example
-//!
-//! ```no_run
-//! use parquet::{basic::Compression, compression::create_codec};
-//!
-//! let mut codec = match create_codec(Compression::SNAPPY) {
-//! Ok(Some(codec)) => codec,
-//! _ => panic!(),
-//! };
-//!
-//! let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
-//! let mut compressed = vec![];
-//! codec.compress(&data[..], &mut compressed).unwrap();
-//!
-//! let mut output = vec![];
-//! codec.decompress(&compressed[..], &mut output).unwrap();
-//!
-//! assert_eq!(output, data);
-//! ```
-
+#[cfg_attr(
+ feature = "experimental",
+ doc = r##"
+# Example
+
+```no_run
+use parquet::{basic::Compression, compression::create_codec};
+
+let mut codec = match create_codec(Compression::SNAPPY) {
+ Ok(Some(codec)) => codec,
+ _ => panic!(),
+};
+
+let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
+let mut compressed = vec![];
+codec.compress(&data[..], &mut compressed).unwrap();
+
+let mut output = vec![];
+codec.decompress(&compressed[..], &mut output).unwrap();
+
+assert_eq!(output, data);
+```
+"##
+)]
use crate::basic::Compression as CodecType;
use crate::errors::{ParquetError, Result};
diff --git a/parquet/src/encodings/encoding.rs
b/parquet/src/encodings/encoding.rs
index f452618..4d67e65 100644
--- a/parquet/src/encodings/encoding.rs
+++ b/parquet/src/encodings/encoding.rs
@@ -922,7 +922,7 @@ mod tests {
use std::sync::Arc;
- use crate::decoding::{get_decoder, Decoder, DictDecoder, PlainDecoder};
+ use crate::encodings::decoding::{get_decoder, Decoder, DictDecoder,
PlainDecoder};
use crate::schema::types::{
ColumnDescPtr, ColumnDescriptor, ColumnPath, Type as SchemaType,
};
diff --git a/parquet/src/lib.rs b/parquet/src/lib.rs
index 6436a88..1c4cafd 100644
--- a/parquet/src/lib.rs
+++ b/parquet/src/lib.rs
@@ -42,11 +42,13 @@
///
/// Experimental modules have no stability guarantees
macro_rules! experimental_mod {
- ($module:ident) => {
+ ($module:ident $(, #[$meta:meta])*) => {
#[cfg(feature = "experimental")]
#[doc(hidden)]
+ $(#[$meta])*
pub mod $module;
#[cfg(not(feature = "experimental"))]
+ $(#[$meta])*
mod $module;
};
}
@@ -54,19 +56,22 @@ macro_rules! experimental_mod {
#[macro_use]
pub mod errors;
pub mod basic;
-#[macro_use]
-pub mod data_type;
+experimental_mod!(data_type, #[macro_use]);
// Exported for external use, such as benchmarks
+#[cfg(feature = "experimental")]
+#[doc(hidden)]
pub use self::encodings::{decoding, encoding};
+
+#[cfg(feature = "experimental")]
+#[doc(hidden)]
pub use self::util::memory;
-#[macro_use]
-pub mod util;
+experimental_mod!(util, #[macro_use]);
#[cfg(any(feature = "arrow", test))]
pub mod arrow;
pub mod column;
-pub mod compression;
+experimental_mod!(compression);
mod encodings;
pub mod file;
pub mod record;
