paleolimbot commented on code in PR #699: URL: https://github.com/apache/sedona-db/pull/699#discussion_r2967277267
########## c/sedona-gdal/src/dataset.rs: ########## @@ -0,0 +1,458 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Ported (and contains copied code) from georust/gdal: +//! <https://github.com/georust/gdal/blob/v0.19.0/src/dataset.rs>. +//! Original code is licensed under MIT. + +use std::ffi::{CStr, CString}; +use std::ptr; + +use crate::cpl::CslStringList; +use crate::driver::Driver; +use crate::errors::Result; +use crate::gdal_api::{call_gdal_api, GdalApi}; +use crate::gdal_dyn_bindgen::*; +use crate::raster::rasterband::RasterBand; +use crate::raster::types::{DatasetOptions, GdalDataType as RustGdalDataType}; +use crate::spatial_ref::SpatialRef; +use crate::vector::layer::Layer; + +/// A GDAL dataset. +pub struct Dataset { + api: &'static GdalApi, + c_dataset: GDALDatasetH, + owned: bool, +} Review Comment: Can you expand a little on the ownership here and why we need a non-owned version of this? (I trust that you need this, I have just not seen an `owned` flag that behaves in this way before). ########## c/sedona-gdal/src/dataset.rs: ########## @@ -0,0 +1,458 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Ported (and contains copied code) from georust/gdal: +//! <https://github.com/georust/gdal/blob/v0.19.0/src/dataset.rs>. +//! Original code is licensed under MIT. + +use std::ffi::{CStr, CString}; +use std::ptr; + +use crate::cpl::CslStringList; +use crate::driver::Driver; +use crate::errors::Result; +use crate::gdal_api::{call_gdal_api, GdalApi}; +use crate::gdal_dyn_bindgen::*; +use crate::raster::rasterband::RasterBand; +use crate::raster::types::{DatasetOptions, GdalDataType as RustGdalDataType}; +use crate::spatial_ref::SpatialRef; +use crate::vector::layer::Layer; + +/// A GDAL dataset. +pub struct Dataset { + api: &'static GdalApi, + c_dataset: GDALDatasetH, + owned: bool, +} + +// SAFETY: `Dataset` carries an opaque GDAL dataset handle plus an ownership flag. +// Moving the wrapper across threads only transfers ownership of that handle; it does +// not permit concurrent shared access. The handle is closed at most once on drop when +// `owned` is true, so `Send` is sound while `Sync` remains intentionally unimplemented. +unsafe impl Send for Dataset {} + +impl Drop for Dataset { + fn drop(&mut self) { + if self.owned && !self.c_dataset.is_null() { + unsafe { call_gdal_api!(self.api, GDALClose, self.c_dataset) }; + } + } +} + +impl Dataset { + /// Open a dataset with extended options. + pub fn open_ex( + api: &'static GdalApi, + path: &str, + open_flags: GDALOpenFlags, + allowed_drivers: Option<&[&str]>, + open_options: Option<&[&str]>, + sibling_files: Option<&[&str]>, + ) -> Result<Self> { + let c_path = CString::new(path)?; + + // Build CslStringLists from Option<&[&str]>. + // None → null pointer (use GDAL default). + // Some(&[]) → pointer to [null] (explicitly empty list). + let drivers_csl = allowed_drivers + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + let options_csl = open_options + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + let siblings_csl = sibling_files + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + + let c_dataset = unsafe { + call_gdal_api!( + api, + GDALOpenEx, + c_path.as_ptr(), + open_flags, + drivers_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _), + options_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _), + siblings_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _) + ) + }; + + if c_dataset.is_null() { + return Err(api.last_cpl_err(CE_Failure as u32)); + } + + Ok(Self { + api, + c_dataset, + owned: true, + }) + } + + /// Create a new owned Dataset from a C handle. + pub(crate) fn new_owned(api: &'static GdalApi, c_dataset: GDALDatasetH) -> Self { + Self { + api, + c_dataset, + owned: true, + } + } + + /// Wrap an existing C dataset handle (non-owning). + /// + /// # Safety + /// + /// The caller must ensure the handle is valid and outlives this `Dataset`. + pub unsafe fn from_c_dataset(api: &'static GdalApi, c_dataset: GDALDatasetH) -> Self { + Self { + api, + c_dataset, + owned: false, + } + } + + /// Return the raw C dataset handle. + pub fn c_dataset(&self) -> GDALDatasetH { + self.c_dataset + } + + /// Return raster size as (x_size, y_size). + pub fn raster_size(&self) -> (usize, usize) { + let x = unsafe { call_gdal_api!(self.api, GDALGetRasterXSize, self.c_dataset) }; + let y = unsafe { call_gdal_api!(self.api, GDALGetRasterYSize, self.c_dataset) }; + (x as usize, y as usize) + } + + /// Return the number of raster bands. + pub fn raster_count(&self) -> usize { + unsafe { call_gdal_api!(self.api, GDALGetRasterCount, self.c_dataset) as usize } + } + + /// Fetch a raster band by 1-indexed band number. + /// Band numbers start at 1, as in GDAL. + pub fn rasterband(&self, band_index: usize) -> Result<RasterBand<'_>> { + let band_index_i32 = i32::try_from(band_index)?; + let c_band = + unsafe { call_gdal_api!(self.api, GDALGetRasterBand, self.c_dataset, band_index_i32) }; + if c_band.is_null() { + return Err(self.api.last_null_pointer_err("GDALGetRasterBand")); + } + Ok(RasterBand::new(self.api, c_band, self)) + } + + /// Fetch the dataset geotransform coefficients. + /// Return an error if no geotransform is available. + pub fn geo_transform(&self) -> Result<[f64; 6]> { + let mut gt = [0.0f64; 6]; + let rv = unsafe { + call_gdal_api!( + self.api, + GDALGetGeoTransform, + self.c_dataset, + gt.as_mut_ptr() + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(gt) + } + + /// Set the geo-transform. + pub fn set_geo_transform(&self, gt: &[f64; 6]) -> Result<()> { + let rv = unsafe { + call_gdal_api!( + self.api, + GDALSetGeoTransform, + self.c_dataset, + gt.as_ptr() as *mut f64 + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(()) + } + + /// Fetch the projection definition string for this dataset. + /// Return an empty string if no projection is available. + pub fn projection(&self) -> String { + unsafe { + let ptr = call_gdal_api!(self.api, GDALGetProjectionRef, self.c_dataset); + if ptr.is_null() { + String::new() + } else { + CStr::from_ptr(ptr).to_string_lossy().into_owned() + } + } + } + + /// Set the projection definition string for this dataset. + pub fn set_projection(&self, projection: &str) -> Result<()> { + let c_projection = CString::new(projection)?; + let rv = unsafe { + call_gdal_api!( + self.api, + GDALSetProjection, + self.c_dataset, + c_projection.as_ptr() + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(()) + } Review Comment: Given we have a wrapper for SpatialRef, do we need these? ########## c/sedona-gdal/src/vector/feature.rs: ########## @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Ported (and contains copied code) from georust/gdal: +//! <https://github.com/georust/gdal/blob/v0.19.0/src/vector/feature.rs>. +//! Original code is licensed under MIT. + +use std::ffi::CString; +use std::marker::PhantomData; + +use crate::errors::{GdalError, Result}; +use crate::gdal_api::{call_gdal_api, GdalApi}; +use crate::gdal_dyn_bindgen::*; +use crate::vector::geometry::Envelope; + +/// An OGR feature. +pub struct Feature<'a> { + api: &'static GdalApi, + c_feature: OGRFeatureH, + _lifetime: PhantomData<&'a ()>, +} Review Comment: Unless you need this for raster work I think we can port this separately as well. We would almost certainly use the Arrow interface instead of the feature interface. ########## c/sedona-gdal/src/vector/layer.rs: ########## @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Ported (and contains copied code) from georust/gdal: +//! <https://github.com/georust/gdal/blob/v0.19.0/src/vector/layer.rs>. +//! Original code is licensed under MIT. + +use std::marker::PhantomData; + +use crate::dataset::Dataset; +use crate::errors::{GdalError, Result}; +use crate::gdal_api::{call_gdal_api, GdalApi}; +use crate::gdal_dyn_bindgen::*; +use crate::vector::feature::{Feature, FieldDefn}; + +/// An OGR layer (borrowed from a Dataset). +pub struct Layer<'a> { + api: &'static GdalApi, + c_layer: OGRLayerH, + _dataset: PhantomData<&'a Dataset>, +} Review Comment: Unless you need this for the raster work I think we should remove it for now. The options exposed by georust/gdal are not really sufficient for what we would need to do with them to make use of this (and we need a different ownership model so that we can return `Box<dyn RecordBatchReader>`). I'm happy to port over the vector stuff later. ########## c/sedona-gdal/src/dataset.rs: ########## @@ -0,0 +1,458 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Ported (and contains copied code) from georust/gdal: +//! <https://github.com/georust/gdal/blob/v0.19.0/src/dataset.rs>. +//! Original code is licensed under MIT. + +use std::ffi::{CStr, CString}; +use std::ptr; + +use crate::cpl::CslStringList; +use crate::driver::Driver; +use crate::errors::Result; +use crate::gdal_api::{call_gdal_api, GdalApi}; +use crate::gdal_dyn_bindgen::*; +use crate::raster::rasterband::RasterBand; +use crate::raster::types::{DatasetOptions, GdalDataType as RustGdalDataType}; +use crate::spatial_ref::SpatialRef; +use crate::vector::layer::Layer; + +/// A GDAL dataset. +pub struct Dataset { + api: &'static GdalApi, + c_dataset: GDALDatasetH, + owned: bool, +} + +// SAFETY: `Dataset` carries an opaque GDAL dataset handle plus an ownership flag. +// Moving the wrapper across threads only transfers ownership of that handle; it does +// not permit concurrent shared access. The handle is closed at most once on drop when +// `owned` is true, so `Send` is sound while `Sync` remains intentionally unimplemented. +unsafe impl Send for Dataset {} + +impl Drop for Dataset { + fn drop(&mut self) { + if self.owned && !self.c_dataset.is_null() { + unsafe { call_gdal_api!(self.api, GDALClose, self.c_dataset) }; + } + } +} + +impl Dataset { + /// Open a dataset with extended options. + pub fn open_ex( + api: &'static GdalApi, + path: &str, + open_flags: GDALOpenFlags, + allowed_drivers: Option<&[&str]>, + open_options: Option<&[&str]>, + sibling_files: Option<&[&str]>, + ) -> Result<Self> { + let c_path = CString::new(path)?; + + // Build CslStringLists from Option<&[&str]>. + // None → null pointer (use GDAL default). + // Some(&[]) → pointer to [null] (explicitly empty list). + let drivers_csl = allowed_drivers + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + let options_csl = open_options + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + let siblings_csl = sibling_files + .map(|v| CslStringList::try_from_iter(v.iter().copied())) + .transpose()?; + + let c_dataset = unsafe { + call_gdal_api!( + api, + GDALOpenEx, + c_path.as_ptr(), + open_flags, + drivers_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _), + options_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _), + siblings_csl + .as_ref() + .map_or(ptr::null(), |csl| csl.as_ptr() as *const *const _) + ) + }; + + if c_dataset.is_null() { + return Err(api.last_cpl_err(CE_Failure as u32)); + } + + Ok(Self { + api, + c_dataset, + owned: true, + }) + } + + /// Create a new owned Dataset from a C handle. + pub(crate) fn new_owned(api: &'static GdalApi, c_dataset: GDALDatasetH) -> Self { + Self { + api, + c_dataset, + owned: true, + } + } + + /// Wrap an existing C dataset handle (non-owning). + /// + /// # Safety + /// + /// The caller must ensure the handle is valid and outlives this `Dataset`. + pub unsafe fn from_c_dataset(api: &'static GdalApi, c_dataset: GDALDatasetH) -> Self { + Self { + api, + c_dataset, + owned: false, + } + } + + /// Return the raw C dataset handle. + pub fn c_dataset(&self) -> GDALDatasetH { + self.c_dataset + } + + /// Return raster size as (x_size, y_size). + pub fn raster_size(&self) -> (usize, usize) { + let x = unsafe { call_gdal_api!(self.api, GDALGetRasterXSize, self.c_dataset) }; + let y = unsafe { call_gdal_api!(self.api, GDALGetRasterYSize, self.c_dataset) }; + (x as usize, y as usize) + } + + /// Return the number of raster bands. + pub fn raster_count(&self) -> usize { + unsafe { call_gdal_api!(self.api, GDALGetRasterCount, self.c_dataset) as usize } + } + + /// Fetch a raster band by 1-indexed band number. + /// Band numbers start at 1, as in GDAL. + pub fn rasterband(&self, band_index: usize) -> Result<RasterBand<'_>> { + let band_index_i32 = i32::try_from(band_index)?; + let c_band = + unsafe { call_gdal_api!(self.api, GDALGetRasterBand, self.c_dataset, band_index_i32) }; + if c_band.is_null() { + return Err(self.api.last_null_pointer_err("GDALGetRasterBand")); + } + Ok(RasterBand::new(self.api, c_band, self)) + } + + /// Fetch the dataset geotransform coefficients. + /// Return an error if no geotransform is available. + pub fn geo_transform(&self) -> Result<[f64; 6]> { + let mut gt = [0.0f64; 6]; + let rv = unsafe { + call_gdal_api!( + self.api, + GDALGetGeoTransform, + self.c_dataset, + gt.as_mut_ptr() + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(gt) + } + + /// Set the geo-transform. + pub fn set_geo_transform(&self, gt: &[f64; 6]) -> Result<()> { + let rv = unsafe { + call_gdal_api!( + self.api, + GDALSetGeoTransform, + self.c_dataset, + gt.as_ptr() as *mut f64 + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(()) + } + + /// Fetch the projection definition string for this dataset. + /// Return an empty string if no projection is available. + pub fn projection(&self) -> String { + unsafe { + let ptr = call_gdal_api!(self.api, GDALGetProjectionRef, self.c_dataset); + if ptr.is_null() { + String::new() + } else { + CStr::from_ptr(ptr).to_string_lossy().into_owned() + } + } + } + + /// Set the projection definition string for this dataset. + pub fn set_projection(&self, projection: &str) -> Result<()> { + let c_projection = CString::new(projection)?; + let rv = unsafe { + call_gdal_api!( + self.api, + GDALSetProjection, + self.c_dataset, + c_projection.as_ptr() + ) + }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(()) + } + + /// Fetch the spatial reference for this dataset. + /// GDAL returns a borrowed handle; this method clones it. + pub fn spatial_ref(&self) -> Result<SpatialRef> { + let c_srs = unsafe { call_gdal_api!(self.api, GDALGetSpatialRef, self.c_dataset) }; + if c_srs.is_null() { + return Err(self.api.last_null_pointer_err("GDALGetSpatialRef")); + } + // GDALGetSpatialRef returns a borrowed reference — clone it via OSRClone. + unsafe { SpatialRef::from_c_srs_clone(self.api, c_srs) } + } + + /// Set the spatial reference. + pub fn set_spatial_ref(&self, srs: &SpatialRef) -> Result<()> { + let rv = + unsafe { call_gdal_api!(self.api, GDALSetSpatialRef, self.c_dataset, srs.c_srs()) }; + if rv != CE_None { + return Err(self.api.last_cpl_err(rv as u32)); + } + Ok(()) + } + + /// Create a copy of this dataset to a new file using the given driver. + pub fn create_copy( + &self, + driver: &Driver, + filename: &str, + options: &[&str], + ) -> Result<Dataset> { + let c_filename = CString::new(filename)?; + let csl = CslStringList::try_from_iter(options.iter().copied())?; + + let c_ds = unsafe { + call_gdal_api!( + self.api, + GDALCreateCopy, + driver.c_driver(), + c_filename.as_ptr(), + self.c_dataset, + 0, // bStrict + csl.as_ptr(), + ptr::null_mut(), + ptr::null_mut() + ) + }; + if c_ds.is_null() { + return Err(self.api.last_cpl_err(CE_Failure as u32)); + } + Ok(Dataset { + api: self.api, + c_dataset: c_ds, + owned: true, + }) + } + + /// Create a new vector layer. + pub fn create_layer(&self, options: LayerOptions<'_>) -> Result<Layer<'_>> { + let c_name = CString::new(options.name)?; + let c_srs = options.srs.map_or(ptr::null_mut(), |s| s.c_srs()); + + let csl = CslStringList::try_from_iter(options.options.unwrap_or(&[]).iter().copied())?; + + let c_layer = unsafe { + call_gdal_api!( + self.api, + GDALDatasetCreateLayer, + self.c_dataset, + c_name.as_ptr(), + c_srs, + options.ty, + csl.as_ptr() + ) + }; + if c_layer.is_null() { + return Err(self.api.last_null_pointer_err("GDALDatasetCreateLayer")); + } + Ok(Layer::new(self.api, c_layer, self)) + } + + /// Get the GDAL API reference. + pub fn api(&self) -> &'static GdalApi { + self.api + } + + /// Open a dataset using a `DatasetOptions` struct (georust-compatible convenience). + pub fn open_ex_with_options( + api: &'static GdalApi, + path: &str, + options: DatasetOptions<'_>, + ) -> Result<Self> { + Self::open_ex( + api, + path, + options.open_flags, + options.allowed_drivers, + options.open_options, + options.sibling_files, + ) + } + + /// Add a band backed by an existing memory buffer. + /// Pass `DATAPOINTER`, `PIXELOFFSET`, and `LINEOFFSET` to `GDALAddBand`. + /// + /// # Safety + /// + /// `data_ptr` must point to valid band data that outlives this dataset. + pub unsafe fn add_band_with_data( + &self, + data_type: RustGdalDataType, + data_ptr: *const u8, + pixel_offset: Option<i64>, + line_offset: Option<i64>, + ) -> Result<()> { + let data_pointer = format!("DATAPOINTER={data_ptr:p}"); + + let mut options = CslStringList::with_capacity(3); + options.add_string(&data_pointer)?; + + if let Some(pixel) = pixel_offset { + options.set_name_value("PIXELOFFSET", &pixel.to_string())?; + } + + if let Some(line) = line_offset { + options.set_name_value("LINEOFFSET", &line.to_string())?; + } + + let err = call_gdal_api!( + self.api, + GDALAddBand, + self.c_dataset, + data_type.to_c(), + options.as_ptr() + ); + if err != CE_None { + return Err(self.api.last_cpl_err(err as u32)); + } + Ok(()) + } + + /// Mark this dataset as owning its handle (for `Drop`). + pub fn set_owned(&mut self, owned: bool) { + self.owned = owned; + } +} + +/// Options for creating a vector layer. +pub struct LayerOptions<'a> { + pub name: &'a str, + pub srs: Option<&'a SpatialRef>, + pub ty: OGRwkbGeometryType, + /// Additional driver-specific options, in the form `"name=value"`. + pub options: Option<&'a [&'a str]>, +} + +impl Default for LayerOptions<'_> { + fn default() -> Self { + Self { + name: "", + srs: None, + ty: OGRwkbGeometryType::wkbUnknown, + options: None, + } + } +} + +#[cfg(all(test, feature = "gdal-sys"))] +mod tests { + use crate::driver::DriverManager; + use crate::global::with_global_gdal_api; + Review Comment: These tests don't cover any of the vector layer features. Should those features be removed for now (I can work on GDAL vector layer stuff later so we can use it to back a datasource and add the associated tests then) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
