paleolimbot commented on code in PR #1043: URL: https://github.com/apache/sedona-db/pull/1043#discussion_r3565562887
########## rust/sedona-geo/src/st_convexhull_agg.rs: ########## @@ -0,0 +1,820 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use arrow_array::builder::{BinaryBuilder, Int32Builder}; +use arrow_array::{Array, ArrayRef, BooleanArray}; +use arrow_schema::{DataType, Field, FieldRef}; +use datafusion_common::{ + cast::{as_binary_array, as_int32_array}, + error::Result, + exec_err, DataFusionError, ScalarValue, +}; +use datafusion_expr::{Accumulator, ColumnarValue, EmitTo, GroupsAccumulator}; +use geo::{algorithm::convex_hull::quick_hull, Coord}; +use geo_traits::{Dimensions, GeometryTrait}; +use sedona_common::sedona_internal_err; +use sedona_expr::{ + aggregate_udf::{SedonaAccumulator, SedonaAccumulatorRef}, + item_crs::ItemCrsSedonaAccumulator, +}; +use sedona_functions::executor::WkbExecutor; +use sedona_geometry::bounds::visit_xy_coords; +use sedona_geometry::wkb_factory::{ + write_wkb_geometrycollection_header, write_wkb_linestring, write_wkb_point, write_wkb_polygon, + WKB_MIN_PROBABLE_BYTES, +}; +use sedona_schema::{ + datatypes::{SedonaType, WKB_GEOMETRY}, + matchers::ArgMatcher, +}; +use wkb::reader::read_wkb; + +/// ST_ConvexHull_Agg() implementation +pub fn st_convexhull_agg_impl() -> Vec<SedonaAccumulatorRef> { + ItemCrsSedonaAccumulator::wrap_impl(STConvexHullAgg {}) +} + +#[derive(Debug)] +struct STConvexHullAgg {} + +impl SedonaAccumulator for STConvexHullAgg { + fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> { + let matcher = ArgMatcher::new(vec![ArgMatcher::is_geometry()], WKB_GEOMETRY); + matcher.match_args(args) + } + + fn accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn Accumulator>> { + Ok(Box::new(ConvexHullAccumulator::new(args[0].clone()))) + } + + fn groups_accumulator_supported(&self, _args: &[SedonaType]) -> bool { + true + } + + fn groups_accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn GroupsAccumulator>> { + Ok(Box::new(ConvexHullGroupsAccumulator::new(args[0].clone()))) + } + + fn state_fields(&self, _args: &[SedonaType]) -> Result<Vec<FieldRef>> { + Ok(vec![ + Arc::new(WKB_GEOMETRY.to_storage_field("hull", true)?), + Arc::new(Field::new("dimension", DataType::Int32, true)), + ]) + } +} + +fn push_hull_coords(geom: impl GeometryTrait<T = f64>, out: &mut Vec<Coord>) -> Result<()> { + visit_xy_coords(geom, false, &mut |x, y| out.push((x, y).into())) + .map_err(|e| DataFusionError::Execution(format!("ST_ConvexHull_Agg(): {e}"))) +} Review Comment: Did you experiment with only pushing the convex hull of each item? My sense is that this would not be faster but might use less memory. There also may be an opportunity to discard points that are definitely contained within the existing convex hull (but again, that requires some compute and might not be faster). ########## rust/sedona-geo/src/st_convexhull_agg.rs: ########## @@ -0,0 +1,820 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use arrow_array::builder::{BinaryBuilder, Int32Builder}; +use arrow_array::{Array, ArrayRef, BooleanArray}; +use arrow_schema::{DataType, Field, FieldRef}; +use datafusion_common::{ + cast::{as_binary_array, as_int32_array}, + error::Result, + exec_err, DataFusionError, ScalarValue, +}; +use datafusion_expr::{Accumulator, ColumnarValue, EmitTo, GroupsAccumulator}; +use geo::{algorithm::convex_hull::quick_hull, Coord}; +use geo_traits::{Dimensions, GeometryTrait}; +use sedona_common::sedona_internal_err; +use sedona_expr::{ + aggregate_udf::{SedonaAccumulator, SedonaAccumulatorRef}, + item_crs::ItemCrsSedonaAccumulator, +}; +use sedona_functions::executor::WkbExecutor; +use sedona_geometry::bounds::visit_xy_coords; +use sedona_geometry::wkb_factory::{ + write_wkb_geometrycollection_header, write_wkb_linestring, write_wkb_point, write_wkb_polygon, + WKB_MIN_PROBABLE_BYTES, +}; +use sedona_schema::{ + datatypes::{SedonaType, WKB_GEOMETRY}, + matchers::ArgMatcher, +}; +use wkb::reader::read_wkb; + +/// ST_ConvexHull_Agg() implementation +pub fn st_convexhull_agg_impl() -> Vec<SedonaAccumulatorRef> { + ItemCrsSedonaAccumulator::wrap_impl(STConvexHullAgg {}) +} + +#[derive(Debug)] +struct STConvexHullAgg {} + +impl SedonaAccumulator for STConvexHullAgg { + fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> { + let matcher = ArgMatcher::new(vec![ArgMatcher::is_geometry()], WKB_GEOMETRY); + matcher.match_args(args) + } + + fn accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn Accumulator>> { + Ok(Box::new(ConvexHullAccumulator::new(args[0].clone()))) + } + + fn groups_accumulator_supported(&self, _args: &[SedonaType]) -> bool { + true + } + + fn groups_accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn GroupsAccumulator>> { + Ok(Box::new(ConvexHullGroupsAccumulator::new(args[0].clone()))) + } + + fn state_fields(&self, _args: &[SedonaType]) -> Result<Vec<FieldRef>> { + Ok(vec![ + Arc::new(WKB_GEOMETRY.to_storage_field("hull", true)?), + Arc::new(Field::new("dimension", DataType::Int32, true)), + ]) + } +} + +fn push_hull_coords(geom: impl GeometryTrait<T = f64>, out: &mut Vec<Coord>) -> Result<()> { + visit_xy_coords(geom, false, &mut |x, y| out.push((x, y).into())) + .map_err(|e| DataFusionError::Execution(format!("ST_ConvexHull_Agg(): {e}"))) +} + +fn dimension_code(dimensions: Dimensions) -> i32 { + match dimensions { + Dimensions::Xy => 0, + Dimensions::Xyz => 1, + Dimensions::Xym => 2, + Dimensions::Xyzm => 3, + Dimensions::Unknown(_) => 4, + } +} + +fn observe_dimension(state: &mut Option<i32>, code: i32) { + // Some(-1) is a sentinel meaning mixed dimensions were observed + *state = match *state { + Some(seen) if seen != code => Some(-1), + _ => Some(code), + }; +} + +fn merge_dimension(state: &mut Option<i32>, other: Option<i32>) { + if let Some(code) = other { + observe_dimension(state, code); + } +} + +fn check_dimension(state: Option<i32>) -> Result<()> { + if state == Some(-1) { + exec_err!("Can't ST_ConvexHull_Agg() mixed dimension geometries") + } else { + Ok(()) + } +} Review Comment: Why is this an error? I think we drop the Z and M coordinates anyway in this implementation (we could keep them in some future like PostGIS). ########## rust/sedona-geo/src/st_convexhull_agg.rs: ########## @@ -0,0 +1,820 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use arrow_array::builder::{BinaryBuilder, Int32Builder}; +use arrow_array::{Array, ArrayRef, BooleanArray}; +use arrow_schema::{DataType, Field, FieldRef}; +use datafusion_common::{ + cast::{as_binary_array, as_int32_array}, + error::Result, + exec_err, DataFusionError, ScalarValue, +}; +use datafusion_expr::{Accumulator, ColumnarValue, EmitTo, GroupsAccumulator}; +use geo::{algorithm::convex_hull::quick_hull, Coord}; +use geo_traits::{Dimensions, GeometryTrait}; +use sedona_common::sedona_internal_err; +use sedona_expr::{ + aggregate_udf::{SedonaAccumulator, SedonaAccumulatorRef}, + item_crs::ItemCrsSedonaAccumulator, +}; +use sedona_functions::executor::WkbExecutor; +use sedona_geometry::bounds::visit_xy_coords; +use sedona_geometry::wkb_factory::{ + write_wkb_geometrycollection_header, write_wkb_linestring, write_wkb_point, write_wkb_polygon, + WKB_MIN_PROBABLE_BYTES, +}; +use sedona_schema::{ + datatypes::{SedonaType, WKB_GEOMETRY}, + matchers::ArgMatcher, +}; +use wkb::reader::read_wkb; + +/// ST_ConvexHull_Agg() implementation +pub fn st_convexhull_agg_impl() -> Vec<SedonaAccumulatorRef> { + ItemCrsSedonaAccumulator::wrap_impl(STConvexHullAgg {}) +} + +#[derive(Debug)] +struct STConvexHullAgg {} + +impl SedonaAccumulator for STConvexHullAgg { + fn return_type(&self, args: &[SedonaType]) -> Result<Option<SedonaType>> { + let matcher = ArgMatcher::new(vec![ArgMatcher::is_geometry()], WKB_GEOMETRY); + matcher.match_args(args) + } + + fn accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn Accumulator>> { + Ok(Box::new(ConvexHullAccumulator::new(args[0].clone()))) + } + + fn groups_accumulator_supported(&self, _args: &[SedonaType]) -> bool { + true + } + + fn groups_accumulator( + &self, + args: &[SedonaType], + _output_type: &SedonaType, + ) -> Result<Box<dyn GroupsAccumulator>> { + Ok(Box::new(ConvexHullGroupsAccumulator::new(args[0].clone()))) + } + + fn state_fields(&self, _args: &[SedonaType]) -> Result<Vec<FieldRef>> { + Ok(vec![ + Arc::new(WKB_GEOMETRY.to_storage_field("hull", true)?), + Arc::new(Field::new("dimension", DataType::Int32, true)), + ]) + } +} + +fn push_hull_coords(geom: impl GeometryTrait<T = f64>, out: &mut Vec<Coord>) -> Result<()> { + visit_xy_coords(geom, false, &mut |x, y| out.push((x, y).into())) + .map_err(|e| DataFusionError::Execution(format!("ST_ConvexHull_Agg(): {e}"))) +} + +fn dimension_code(dimensions: Dimensions) -> i32 { + match dimensions { + Dimensions::Xy => 0, + Dimensions::Xyz => 1, + Dimensions::Xym => 2, + Dimensions::Xyzm => 3, + Dimensions::Unknown(_) => 4, + } +} + +fn observe_dimension(state: &mut Option<i32>, code: i32) { + // Some(-1) is a sentinel meaning mixed dimensions were observed + *state = match *state { + Some(seen) if seen != code => Some(-1), + _ => Some(code), + }; +} + +fn merge_dimension(state: &mut Option<i32>, other: Option<i32>) { + if let Some(code) = other { + observe_dimension(state, code); + } +} + +fn check_dimension(state: Option<i32>) -> Result<()> { + if state == Some(-1) { + exec_err!("Can't ST_ConvexHull_Agg() mixed dimension geometries") + } else { + Ok(()) + } +} + +fn filter_keep(filter: Option<&BooleanArray>, i: usize) -> bool { + filter.is_none_or(|filter| filter.is_valid(i) && filter.value(i)) +} + +fn normalize_zero(v: f64) -> f64 { + if v == 0.0 { + 0.0 + } else { + v + } +} + +fn coord_cmp(a: &Coord, b: &Coord) -> std::cmp::Ordering { + normalize_zero(a.y) + .total_cmp(&normalize_zero(b.y)) + .then(normalize_zero(a.x).total_cmp(&normalize_zero(b.x))) +} + +fn write_hull(coords: &mut [Coord], writer: &mut impl std::io::Write) -> Result<()> { + if coords.is_empty() { + return write_wkb_geometrycollection_header(writer, Dimensions::Xy, 0) + .map_err(|e| DataFusionError::Execution(format!("Failed to write header: {e}"))); + } + + // geo 0.31's quick_hull always returns a closed ring (both the trivial_hull and qhull paths call .close()) + let mut vertices = quick_hull(coords).0; + vertices.pop(); + + match vertices.len() { + 0 => write_wkb_geometrycollection_header(writer, Dimensions::Xy, 0), + 1 => write_wkb_point(writer, (vertices[0].x, vertices[0].y)), + 2 => { + vertices.sort_unstable_by(coord_cmp); + write_wkb_linestring(writer, vertices.iter().map(|c| (c.x, c.y))) + } + _ => { + vertices.reverse(); + let start = vertices + .iter() + .enumerate() + .min_by(|(_, a), (_, b)| coord_cmp(a, b)) + .map(|(i, _)| i) + .unwrap(); + vertices.rotate_left(start); + vertices.push(vertices[0]); + write_wkb_polygon(writer, vertices.iter().map(|c| (c.x, c.y))) + } + } + .map_err(|e| DataFusionError::Execution(format!("Failed to write hull: {e}"))) +} + +fn push_state_coords(hulls: &dyn Array, index: usize, out: &mut Vec<Coord>) -> Result<bool> { + let hulls = as_binary_array(hulls)?; + if hulls.is_null(index) { + return Ok(false); + } + + let hull = read_wkb(hulls.value(index)) + .map_err(|e| DataFusionError::Execution(format!("Failed to read WKB: {e}")))?; + push_hull_coords(&hull, out)?; + Ok(true) +} + +#[derive(Debug)] +struct ConvexHullAccumulator { + input_type: SedonaType, + coords: Vec<Coord>, + has_input: bool, + dimension: Option<i32>, +} + +impl ConvexHullAccumulator { + pub fn new(input_type: SedonaType) -> Self { + Self { + input_type, + coords: Vec::new(), + has_input: false, + dimension: None, + } + } + + fn make_wkb_result(&mut self) -> Result<Option<Vec<u8>>> { + if !self.has_input { + return Ok(None); + } + + check_dimension(self.dimension)?; + let mut wkb = Vec::new(); + write_hull(&mut self.coords, &mut wkb)?; + Ok(Some(wkb)) + } +} + +impl Accumulator for ConvexHullAccumulator { + fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + if values.is_empty() { + return sedona_internal_err!("No input arrays provided to accumulator in update_batch"); + } + + let arg_types = [self.input_type.clone()]; + let args = [ColumnarValue::Array(values[0].clone())]; + let executor = WkbExecutor::new(&arg_types, &args); + executor.execute_wkb_void(|maybe_item| { + if let Some(item) = maybe_item { + self.has_input = true; + observe_dimension(&mut self.dimension, dimension_code(item.dim())); + push_hull_coords(&item, &mut self.coords)?; + } + Ok(()) + })?; + + Ok(()) + } + + fn evaluate(&mut self) -> Result<ScalarValue> { + Ok(ScalarValue::Binary(self.make_wkb_result()?)) + } + + fn state(&mut self) -> Result<Vec<ScalarValue>> { + Ok(vec![ + ScalarValue::Binary(self.make_wkb_result()?), + ScalarValue::Int32(self.dimension), + ]) + } + + fn size(&self) -> usize { + size_of::<ConvexHullAccumulator>() + self.coords.capacity() * size_of::<Coord>() + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { + if states.len() != 2 { + return sedona_internal_err!( + "Unexpected number of state fields for st_convexhull_agg() (expected 2, got {})", + states.len() + ); + } + + let dimensions = as_int32_array(&states[1])?; + for i in 0..states[0].len() { + self.has_input |= push_state_coords(&states[0], i, &mut self.coords)?; + merge_dimension( + &mut self.dimension, + (!dimensions.is_null(i)).then(|| dimensions.value(i)), + ); + } + + Ok(()) + } +} + +#[derive(Debug)] +struct ConvexHullGroupsAccumulator { + input_type: SedonaType, + coords: Vec<Vec<Coord>>, + has_input: Vec<bool>, + dimensions: Vec<Option<i32>>, +} + +impl ConvexHullGroupsAccumulator { + pub fn new(input_type: SedonaType) -> Self { + Self { + input_type, + coords: Vec::new(), + has_input: Vec::new(), + dimensions: Vec::new(), + } + } + + fn execute_update( + &mut self, + values: &[ArrayRef], + group_indices: &[usize], + opt_filter: Option<&BooleanArray>, + total_num_groups: usize, + ) -> Result<()> { + // Check some of our assumptions about how this will be called + debug_assert_eq!(values.len(), 1); + debug_assert_eq!(values[0].len(), group_indices.len()); + if let Some(filter) = opt_filter { + debug_assert_eq!(values[0].len(), filter.len()); + } + + let arg_types = [self.input_type.clone()]; + let args = [ColumnarValue::Array(values[0].clone())]; + let executor = WkbExecutor::new(&arg_types, &args); + self.coords.resize_with(total_num_groups, Default::default); + self.has_input.resize(total_num_groups, false); + self.dimensions.resize(total_num_groups, None); + let mut i = 0; + + executor.execute_wkb_void(|maybe_item| { + let keep = filter_keep(opt_filter, i); + let group_id = group_indices[i]; + i += 1; + + if keep { + if let Some(item) = maybe_item { + self.has_input[group_id] = true; + observe_dimension(&mut self.dimensions[group_id], dimension_code(item.dim())); + push_hull_coords(&item, &mut self.coords[group_id])?; + } + } + + Ok(()) + })?; + + Ok(()) + } + + fn emit_wkb_result(&mut self, emit_to: EmitTo) -> Result<(ArrayRef, ArrayRef)> { + let emit_size = match emit_to { + EmitTo::All => self.coords.len(), + EmitTo::First(n) => n, + }; + + let mut hull_builder = + BinaryBuilder::with_capacity(emit_size, emit_size * WKB_MIN_PROBABLE_BYTES); Review Comment: I am not sure it matters for performance, but you can probably calculate the exact output size here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
