SteNicholas commented on code in PR #1446:
URL: https://github.com/apache/paimon-rust/pull/1446#discussion_r4610050228
##########
crates/paimon/src/arrow/mod.rs:
##########
@@ -372,105 +362,125 @@ pub fn arrow_fields_to_paimon(fields: &[ArrowField]) ->
crate::Result<Vec<DataFi
/// Build an Arrow [`Schema`](ArrowSchema) from Paimon [`DataField`]s.
pub fn build_target_arrow_schema(fields: &[DataField]) ->
crate::Result<Arc<ArrowSchema>> {
- let arrow_fields: Vec<ArrowField> = fields
- .iter()
- .map(|f| {
- let arrow_type = paimon_type_to_arrow(f.data_type())?;
- Ok(arrow_field_with_paimon_id(
- f.name(),
- arrow_type,
- f.data_type().is_nullable(),
- f.id(),
- ))
- })
- .collect::<crate::Result<Vec<_>>>()?;
- Ok(Arc::new(ArrowSchema::new(arrow_fields)))
+ build_arrow_schema(fields, false)
}
/// Read results can contain NULL after a Java cast overflows or a historical
/// file lacks a requested field, even when the current logical type is NOT
NULL.
-/// Keep those runtime values representable without relaxing the writer schema.
pub(crate) fn paimon_type_to_arrow_for_read(dt: &PaimonDataType) ->
crate::Result<ArrowDataType> {
- Ok(match dt {
- PaimonDataType::Row(row) => ArrowDataType::Struct(
- row.fields()
- .iter()
- .map(|field| {
- Ok(ArrowField::new(
- field.name(),
- paimon_type_to_arrow_for_read(field.data_type())?,
- true,
- ))
- })
- .collect::<crate::Result<Vec<_>>>()?
- .into(),
- ),
- PaimonDataType::Array(array) =>
ArrowDataType::List(Arc::new(ArrowField::new(
- "element",
- paimon_type_to_arrow_for_read(array.element_type())?,
- true,
- ))),
- PaimonDataType::Map(map) => ArrowDataType::Map(
- Arc::new(ArrowField::new(
- "entries",
- ArrowDataType::Struct(
- vec![
- ArrowField::new(
- "key",
- paimon_type_to_arrow_for_read(map.key_type())?,
- false,
- ),
- ArrowField::new(
- "value",
- paimon_type_to_arrow_for_read(map.value_type())?,
- true,
- ),
- ]
- .into(),
- ),
- false,
- )),
- false,
- ),
- PaimonDataType::Multiset(multiset) => ArrowDataType::Map(
- Arc::new(ArrowField::new(
- "entries",
- ArrowDataType::Struct(
- vec![
- ArrowField::new(
- "key",
-
paimon_type_to_arrow_for_read(multiset.element_type())?,
- false,
- ),
- ArrowField::new("value", ArrowDataType::Int32, false),
- ]
- .into(),
- ),
- false,
- )),
- false,
- ),
- _ => paimon_type_to_arrow(dt)?,
- })
+ paimon_arrow_type(dt, true)
}
-/// Build the nullable runtime schema used by table readers. Logical table
-/// constraints and the schema used to validate/write new records are
unchanged.
+/// Build the nullable runtime schema used by table readers.
pub fn build_read_arrow_schema(fields: &[DataField]) ->
crate::Result<Arc<ArrowSchema>> {
+ build_arrow_schema(fields, true)
+}
+
+fn build_arrow_schema(fields: &[DataField], for_read: bool) ->
crate::Result<Arc<ArrowSchema>> {
let fields = fields
.iter()
- .map(|field| {
- Ok(arrow_field_with_paimon_id(
- field.name(),
- paimon_type_to_arrow_for_read(field.data_type())?,
- true,
- field.id(),
- ))
+ .map(|f| {
+ let mut field = paimon_arrow_field(f.name(), f.data_type(),
for_read)?;
+ field
+ .metadata_mut()
+ .insert(PARQUET_FIELD_ID_META_KEY.into(), f.id().to_string());
+ Ok(field)
})
.collect::<crate::Result<Vec<_>>>()?;
Ok(Arc::new(ArrowSchema::new(fields)))
}
+pub(crate) const GEOARROW_WKB: &str = "geoarrow.wkb";
+
+#[derive(serde::Serialize, serde::Deserialize)]
+struct GeoArrowMetadata<C> {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ edges: Option<String>,
+ crs: Option<C>,
+}
+
+fn geoarrow_metadata_error(error: serde_json::Error) -> crate::Error {
+ crate::Error::DataTypeInvalid {
+ message: format!("Invalid geoarrow.wkb metadata: {error}"),
+ }
+}
+
+/// Build an Arrow field, including extension metadata for logical types such
as WKB.
+pub fn paimon_type_to_arrow_field(name: &str, ty: &PaimonDataType) ->
crate::Result<ArrowField> {
+ paimon_arrow_field(name, ty, false)
+}
+
+fn paimon_arrow_field(
+ name: &str,
+ ty: &PaimonDataType,
+ for_read: bool,
+) -> crate::Result<ArrowField> {
+ use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY,
EXTENSION_TYPE_NAME_KEY};
+ let mut field = ArrowField::new(
+ name,
+ paimon_arrow_type(ty, for_read)?,
+ for_read || ty.is_nullable(),
+ );
+ let (crs, edges) = match ty {
+ PaimonDataType::Geometry(g) => (g.crs(), None),
+ PaimonDataType::Geography(g) => (g.crs(),
Some(g.algorithm().to_string())),
+ _ => return Ok(field),
+ };
+ // Java writes the original CRS as a string, including PROJJSON.
+ let metadata = serde_json::to_string(&GeoArrowMetadata {
+ edges,
+ crs: Some(crs),
+ })
+ .map_err(geoarrow_metadata_error)?;
+ field
+ .metadata_mut()
+ .insert(EXTENSION_TYPE_NAME_KEY.into(), GEOARROW_WKB.into());
+ field
+ .metadata_mut()
+ .insert(EXTENSION_TYPE_METADATA_KEY.into(), metadata);
+ Ok(field)
+}
+
+/// Convert an Arrow field, preserving its GeoArrow logical type when present.
+pub fn arrow_field_to_paimon_type(field: &ArrowField) ->
crate::Result<PaimonDataType> {
+ if field.extension_type_name() != Some(GEOARROW_WKB) {
+ return arrow_to_paimon_type(field.data_type(), field.is_nullable());
Review Comment:
**Preserve native Parquet geospatial annotations even when `ARROW:schema` is
absent.** The `parquet` dependency is currently built without its `geospatial`
feature, so a Parquet column annotated as `GEOGRAPHY` with CRS `EPSG:4326` and
algorithm `KARNEY` reaches this fallback as plain Arrow `Binary`. I reproduced
this by writing that native logical type with
`ArrowWriterOptions::with_skip_arrow_metadata(true)` and querying `SELECT g
FROM read_files(..., format => 'parquet')`: the inferred Paimon type is
`VARBINARY(2147483647)` instead of `GEOGRAPHY(EPSG:4326, karney)`. **The CRS
and edge algorithm are silently lost** in the returned schema, although the
file contains both. Please enable native annotation conversion on read (or
translate it explicitly) and add a regression fixture without embedded Arrow
metadata; the current write/read round trips always include that metadata and
therefore miss this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]