alamb commented on code in PR #3269: URL: https://github.com/apache/arrow-rs/pull/3269#discussion_r1039924077
########## parquet/src/bin/parquet-layout.rs: ########## @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Binary that prints the physical layout of a parquet file Review Comment: 👍 As a general musing, I think some of these cli helpers are quite nice and maybe eventually we could make them more discoverable / nicer to people who are not working with the parquet source code. I didn't find anything other than https://github.com/apache/arrow-rs/tree/master/parquet/src/bin for documentation Perhaps to `prqs` or something similar 🤔 ########## parquet/src/bin/parquet-layout.rs: ########## @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Binary that prints the physical layout of a parquet file +//! +//! # Install +//! +//! `parquet-layout` can be installed using `cargo`: +//! ``` +//! cargo install parquet --features=cli +//! ``` +//! After this `parquet-layout` should be available: +//! ``` +//! parquet-layout XYZ.parquet +//! ``` +//! +//! The binary can also be built from the source code and run as follows: +//! ``` +//! cargo run --features=cli --bin parquet-layout XYZ.parquet +//! ``` + +use std::fs::File; +use std::io::Read; + +use clap::Parser; +use serde::Serialize; +use thrift::protocol::{TCompactInputProtocol, TSerializable}; + +use parquet::basic::{Compression, Encoding}; +use parquet::errors::Result; +use parquet::file::reader::ChunkReader; +use parquet::format::PageHeader; + +#[derive(Serialize, Debug)] +struct ParquetFile { + row_groups: Vec<RowGroup>, +} + +#[derive(Serialize, Debug)] +struct RowGroup { + columns: Vec<ColumnChunk>, + row_count: i64, +} + +#[derive(Serialize, Debug)] +struct ColumnChunk { + path: String, + has_offset_index: bool, + has_column_index: bool, + has_bloom_filter: bool, + pages: Vec<Page>, +} + +#[derive(Serialize, Debug)] +struct Page { + compression: Option<&'static str>, + encoding: &'static str, + page_type: &'static str, + compressed_bytes: i32, + uncompressed_bytes: i32, + header_bytes: i32, + num_values: i32, +} + +fn do_layout<C: ChunkReader>(reader: &C) -> Result<ParquetFile> { + let metadata = parquet::file::footer::parse_metadata(reader)?; + let schema = metadata.file_metadata().schema_descr(); + + let row_groups = (0..metadata.num_row_groups()) + .map(|row_group_idx| { + let row_group = metadata.row_group(row_group_idx); + let columns = row_group + .columns() + .iter() + .zip(schema.columns()) + .map(|(column, column_schema)| { + let compression = compression(column.compression()); + let mut pages = vec![]; + + let mut start = column + .dictionary_page_offset() + .unwrap_or_else(|| column.data_page_offset()) + as u64; + + let end = start + column.compressed_size() as u64; + while start != end { + let (header_len, header) = read_page_header(reader, start)?; + if let Some(dictionary) = header.dictionary_page_header { + pages.push(Page { + compression, + encoding: encoding(dictionary.encoding), + page_type: "dictionary", + compressed_bytes: header.compressed_page_size, + uncompressed_bytes: header.uncompressed_page_size, + header_bytes: header_len as _, + num_values: dictionary.num_values, + }) + } else if let Some(data_page) = header.data_page_header { + pages.push(Page { + compression, + encoding: encoding(data_page.encoding), + page_type: "data_page_v1", + compressed_bytes: header.compressed_page_size, + uncompressed_bytes: header.uncompressed_page_size, + header_bytes: header_len as _, + num_values: data_page.num_values, + }) + } else if let Some(data_page) = header.data_page_header_v2 { + let is_compressed = data_page.is_compressed.unwrap_or(true); Review Comment: is it really compressed by default? I expected `unwrap_or(false)` ########## parquet/src/bin/parquet-layout.rs: ########## @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Binary that prints the physical layout of a parquet file +//! +//! # Install +//! +//! `parquet-layout` can be installed using `cargo`: +//! ``` +//! cargo install parquet --features=cli +//! ``` +//! After this `parquet-layout` should be available: +//! ``` +//! parquet-layout XYZ.parquet +//! ``` +//! +//! The binary can also be built from the source code and run as follows: +//! ``` +//! cargo run --features=cli --bin parquet-layout XYZ.parquet +//! ``` + +use std::fs::File; +use std::io::Read; + +use clap::Parser; +use serde::Serialize; +use thrift::protocol::{TCompactInputProtocol, TSerializable}; + +use parquet::basic::{Compression, Encoding}; +use parquet::errors::Result; +use parquet::file::reader::ChunkReader; +use parquet::format::PageHeader; + +#[derive(Serialize, Debug)] +struct ParquetFile { + row_groups: Vec<RowGroup>, +} + +#[derive(Serialize, Debug)] +struct RowGroup { + columns: Vec<ColumnChunk>, + row_count: i64, +} + +#[derive(Serialize, Debug)] +struct ColumnChunk { + path: String, + has_offset_index: bool, + has_column_index: bool, + has_bloom_filter: bool, + pages: Vec<Page>, +} + +#[derive(Serialize, Debug)] +struct Page { + compression: Option<&'static str>, + encoding: &'static str, + page_type: &'static str, + compressed_bytes: i32, + uncompressed_bytes: i32, + header_bytes: i32, + num_values: i32, +} + +fn do_layout<C: ChunkReader>(reader: &C) -> Result<ParquetFile> { + let metadata = parquet::file::footer::parse_metadata(reader)?; + let schema = metadata.file_metadata().schema_descr(); + + let row_groups = (0..metadata.num_row_groups()) + .map(|row_group_idx| { + let row_group = metadata.row_group(row_group_idx); + let columns = row_group + .columns() + .iter() + .zip(schema.columns()) + .map(|(column, column_schema)| { + let compression = compression(column.compression()); + let mut pages = vec![]; + + let mut start = column + .dictionary_page_offset() + .unwrap_or_else(|| column.data_page_offset()) + as u64; + + let end = start + column.compressed_size() as u64; + while start != end { + let (header_len, header) = read_page_header(reader, start)?; Review Comment: this is neat -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
