alamb commented on code in PR #8225: URL: https://github.com/apache/arrow-rs/pull/8225#discussion_r2366224349
########## parquet/src/geospatial/mod.rs: ########## @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides functionality for working with geospatial data in Parquet file as defied in the [spec][parquet-geo-spec]. +//! +//! * [`GeospatialStatistics`]: describes the geospatial statistics for a Parquet column. +//! * [`BoundingBox`]: describes the bounding box values for a geospatial column. +//! +//! [`GeospatialStatistics`] describes the geospatial statistics for a Parquet column. +//! * bbox: the [`BoundingBox`] for the geospatial data +//! * geospatial_types: the geospatial types for the geospatial data as specified in [specification][geo-types]. +//! +//! Geospatial bounding box describes the spatial extent of the geospatial data within a Parquet row group. +//! * xmin, xmax: the minimum and maximum longitude values +//! * ymin, ymax: the minimum and maximum latitude values +//! * zmin, zmax: (optional) the minimum and maximum elevation values +//! * mmin, mmax: (optional) the minimum and maximum linear reference values +//! +//! In 2D representation, where x are points: +//! +//! ymax +-----------------------+ +//! | x | +//! | x | +//! | x | +//! | x | +//! ymin +-----------------------+ +//! xmin xmax Review Comment: It might render better like this: ```suggestion //! ```text //! ymax +-----------------------+ //! | x | //! | x | //! | x | //! | x | //! ymin +-----------------------+ //! xmin xmax //! ``` ``` ########## parquet/src/file/metadata/mod.rs: ########## @@ -1969,9 +1986,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let base_expected_size = 2312; + let base_expected_size = 2792; Review Comment: this is a fairly non trivial increase in size (as the parquet metadata size is already a challenge for many users) Can we please reduce this size by `Box`ing the Geo statistics? Something like instead of ```rust geo_statistics: Option<geo_statistics::GeospatialStatistics>, ``` Make it like ```rust geo_statistics: Option<Box<geo_statistics::GeospatialStatistics>>, ``` (that would add just 8-16 bytes for another pointer rather than the size of the inlined structure) ########## parquet/src/geospatial/bounding_box.rs: ########## @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Bounding box for GEOMETRY or GEOGRAPHY type in the representation of min/max +//! value pair of coordinates from each axis. +//! +//! Derived from the parquet format spec: <https://github.com/apache/parquet-format/blob/master/Geospatial.md> +//! +//! +use crate::format as parquet; + +/// A geospatial instance has at least two coordinate dimensions: X and Y for 2D coordinates of each point. +/// X represents longitude/easting and Y represents latitude/northing. A geospatial instance can optionally +/// have Z and/or M values associated with each point. +/// +/// The Z values introduce the third dimension coordinate, typically used to indicate height or elevation. +/// +/// M values allow tracking a value in a fourth dimension. These can represent: +/// - Linear reference values (e.g., highway milepost) +/// - Timestamps +/// - Other values defined by the CRS +/// +/// The bounding box is defined as min/max value pairs of coordinates from each axis. X and Y values are +/// always present, while Z and M are omitted for 2D geospatial instances. +/// +/// When calculating a bounding box: +/// - Null or NaN values in a coordinate dimension are skipped +/// - If a dimension has only null/NaN values, that dimension is omitted +/// - If either X or Y dimension is missing, no bounding box is produced +/// - Example: POINT (1 NaN) contributes to X but not to Y, Z, or M dimensions +/// +/// Special cases: +/// - For X values only, xmin may exceed xmax. In this case, a point matches if x >= xmin OR x <= xmax +/// - This wraparound can occur when the bounding box crosses the antimeridian line. +/// - In geographic terms: xmin=westernmost, xmax=easternmost, ymin=southernmost, ymax=northernmost +/// +/// For GEOGRAPHY types: +/// - X values must be within [-180, 180] (longitude) +/// - Y values must be within [-90, 90] (latitude) +/// +/// Derived from the parquet format [spec][bounding-box-spec] +/// +/// # Examples +/// +/// ``` +/// use parquet::geospatial::bounding_box::BoundingBox; +/// +/// // 2D bounding box +/// let bbox_2d = BoundingBox::new(0.0, 0.0, 100.0, 100.0); +/// +/// // 3D bounding box with elevation +/// let bbox_3d = BoundingBox::new(0.0, 0.0, 100.0, 100.0) +/// .with_zrange(0.0, 1000.0); +/// +/// // 3D bounding box with elevation and measured value +/// let bbox_3d_m = BoundingBox::new(0.0, 0.0, 100.0, 100.0) +/// .with_zrange(0.0, 1000.0) +/// .with_mrange(0.0, 1000.0); +/// ``` +/// +/// [bounding-box-spec]: https://github.com/apache/parquet-format/blob/master/Geospatial.md#bounding-box +#[derive(Clone, Debug, PartialEq)] +pub struct BoundingBox { + /// Minimum X coordinate (longitude or easting) + xmin: f64, + /// Maximum X coordinate (longitude or easting) + xmax: f64, + /// Minimum Y coordinate (latitude or northing) + ymin: f64, + /// Maximum Y coordinate (latitude or northing) + ymax: f64, + /// Minimum Z coordinate (elevation/height), if present + zmin: Option<f64>, + /// Maximum Z coordinate (elevation/height), if present + zmax: Option<f64>, + /// Minimum M coordinate (measured value), if present + mmin: Option<f64>, + /// Maximum M coordinate (measured value), if present + mmax: Option<f64>, +} + +impl BoundingBox { + /// Creates a new bounding box with the specified coordinates. + pub fn new(xmin: f64, xmax: f64, ymin: f64, ymax: f64) -> Self { + Self { + xmin, + xmax, + ymin, + ymax, + zmin: None, + zmax: None, + mmin: None, + mmax: None, + } + } + + /// Updates the bounding box with specified X-coordinate range. + pub fn with_xrange(mut self, xmin: f64, xmax: f64) -> Self { + self.xmin = xmin; + self.xmax = xmax; + self + } + + /// Updates the bounding box with specified Y-coordinate range. + pub fn with_yrange(mut self, ymin: f64, ymax: f64) -> Self { + self.ymin = ymin; + self.ymax = ymax; + self + } + + /// Creates a new bounding box with the specified Z-coordinate range. + pub fn with_zrange(mut self, zmin: f64, zmax: f64) -> Self { + self.zmin = Some(zmin); + self.zmax = Some(zmax); + self + } + + /// Creates a new bounding box with the specified M-coordinate range. + pub fn with_mrange(mut self, mmin: f64, mmax: f64) -> Self { + self.mmin = Some(mmin); + self.mmax = Some(mmax); + self + } + + /// Returns the minimum x-coordinate. + pub fn get_xmin(&self) -> f64 { + self.xmin + } + + /// Returns the maximum x-coordinate. + pub fn get_xmax(&self) -> f64 { + self.xmax + } + + /// Returns the minimum y-coordinate. + pub fn get_ymin(&self) -> f64 { + self.ymin + } + + /// Returns the maximum y-coordinate. + pub fn get_ymax(&self) -> f64 { + self.ymax + } + + /// Returns the minimum z-coordinate, if present. + pub fn get_zmin(&self) -> Option<f64> { + self.zmin + } + + /// Returns the maximum z-coordinate, if present. + pub fn get_zmax(&self) -> Option<f64> { + self.zmax + } + + /// Returns the minimum m-value (measure), if present. + pub fn get_mmin(&self) -> Option<f64> { + self.mmin + } + + /// Returns the maximum m-value (measure), if present. + pub fn get_mmax(&self) -> Option<f64> { + self.mmax + } + + /// Returns `true` if both zmin and zmax are present. + pub fn is_z_valid(&self) -> bool { + self.zmin.is_some() && self.zmax.is_some() + } + + /// Returns `true` if both mmin and mmax are present. + pub fn is_m_valid(&self) -> bool { + self.mmin.is_some() && self.mmax.is_some() + } +} + +impl From<BoundingBox> for parquet::BoundingBox { + /// Converts our internal `BoundingBox` to the Thrift-generated format. + fn from(b: BoundingBox) -> parquet::BoundingBox { + parquet::BoundingBox { + xmin: b.get_xmin().into(), + ymin: b.get_ymin().into(), + xmax: b.get_xmax().into(), + ymax: b.get_ymax().into(), + zmin: b.get_zmin().map(|z| z.into()), + zmax: b.get_zmax().map(|z| z.into()), + mmin: b.get_mmin().map(|m| m.into()), + mmax: b.get_mmax().map(|m| m.into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bounding_box() { + let bbox = BoundingBox::new(0.0, 0.0, 10.0, 10.0); + assert_eq!(bbox.get_xmin(), 0.0); + assert_eq!(bbox.get_xmax(), 0.0); + assert_eq!(bbox.get_ymin(), 10.0); + assert_eq!(bbox.get_ymax(), 10.0); + assert_eq!(bbox.get_zmin(), None); + assert_eq!(bbox.get_zmax(), None); + assert_eq!(bbox.get_mmin(), None); + assert_eq!(bbox.get_mmax(), None); + assert!(!bbox.is_z_valid()); + assert!(!bbox.is_m_valid()); + + // test with zrange + let bbox_z = BoundingBox::new(0.0, 0.0, 10.0, 10.0).with_zrange(5.0, 15.0); + assert_eq!(bbox_z.get_zmin(), Some(5.0)); + assert_eq!(bbox_z.get_zmax(), Some(15.0)); + assert!(bbox_z.is_z_valid()); + assert!(!bbox_z.is_m_valid()); + + // test with mrange + let bbox_m = BoundingBox::new(0.0, 0.0, 10.0, 10.0).with_mrange(10.0, 20.0); + assert_eq!(bbox_m.get_mmin(), Some(10.0)); + assert_eq!(bbox_m.get_mmax(), Some(20.0)); + assert!(!bbox_m.is_z_valid()); + assert!(bbox_m.is_m_valid()); + + // test with zrange and mrange + let bbox_zm = BoundingBox::new(0.0, 0.0, 10.0, 10.0) + .with_zrange(5.0, 15.0) + .with_mrange(10.0, 20.0); + assert_eq!(bbox_zm.get_zmin(), Some(5.0)); + assert_eq!(bbox_zm.get_zmax(), Some(15.0)); + assert_eq!(bbox_zm.get_mmin(), Some(10.0)); + assert_eq!(bbox_zm.get_mmax(), Some(20.0)); + assert!(bbox_zm.is_z_valid()); + assert!(bbox_zm.is_m_valid()); + } + + #[test] + fn test_bounding_box_to_thrift() { Review Comment: Another good set of tests for thrift encoding/decoding would be "roundtrip" tests that create a thrift structure -> rust --> thrift and ensure all fields survive the round trip -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
