This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 519792a5 fix(arrow): convert dictionary and fixed-size-binary Arrow
types (#978)
519792a5 is described below
commit 519792a5e1f5c3698dfd3bd30c7cc3e50a0a5e56
Author: jackylee <[email protected]>
AuthorDate: Fri Oct 2 10:08:38 2026 +0800
fix(arrow): convert dictionary and fixed-size-binary Arrow types (#978)
---
crates/integrations/datafusion/src/sql_context.rs | 23 +++++++
crates/paimon/src/arrow/mod.rs | 78 +++++++++++++++++++++--
2 files changed, 97 insertions(+), 4 deletions(-)
diff --git a/crates/integrations/datafusion/src/sql_context.rs
b/crates/integrations/datafusion/src/sql_context.rs
index 0fb1213f..badbe40f 100644
--- a/crates/integrations/datafusion/src/sql_context.rs
+++ b/crates/integrations/datafusion/src/sql_context.rs
@@ -4938,6 +4938,29 @@ mod tests {
assert!(fields[1].data_type().is_nullable());
}
+ #[tokio::test]
+ async fn persistent_rest_catalog_view_infers_dictionary_column_type() {
+ let catalog = Arc::new(MockCatalog::new());
+ let ctx = make_sql_context(Arc::clone(&catalog)).await;
+
+ // DataFusion emits dictionary-encoded columns for low-cardinality
+ // strings; the view schema should infer the value type (VarChar)
+ // rather than failing as an unsupported Arrow type.
+ ctx.sql(
+ "CREATE VIEW paimon.default.dict_view AS \
+ SELECT arrow_cast('hello', 'Dictionary(Int32, Utf8)') AS label",
+ )
+ .await
+ .unwrap();
+
+ let view = catalog
+ .get_view(&Identifier::new("default", "dict_view"))
+ .await
+ .unwrap();
+ let fields = view.schema().fields();
+ assert!(matches!(fields[0].data_type(), PaimonDataType::VarChar(_)));
+ }
+
#[tokio::test]
async fn
persistent_rest_catalog_view_expands_function_in_owning_database() {
let catalog = Arc::new(MockCatalog::new());
diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs
index 0f65e4a6..286580e5 100644
--- a/crates/paimon/src/arrow/mod.rs
+++ b/crates/paimon/src/arrow/mod.rs
@@ -31,10 +31,10 @@ pub use row_filter::{RowFilter, RowFilterContext,
RowFilterFactory};
pub use variant::variant_get_numeric_fields;
use crate::spec::{
- ArrayType, BigIntType, BooleanType, DataField, DataType as PaimonDataType,
DateType,
- DecimalType, DoubleType, FloatType, IntType, LocalZonedTimestampType,
MapType, RowType,
- SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType,
VarCharType, VariantType,
- VectorType,
+ ArrayType, BigIntType, BinaryType, BooleanType, DataField, DataType as
PaimonDataType,
+ DateType, DecimalType, DoubleType, FloatType, IntType,
LocalZonedTimestampType, MapType,
+ RowType, SmallIntType, TimeType, TimestampType, TinyIntType,
VarBinaryType, VarCharType,
+ VariantType, VectorType,
};
use arrow_schema::DataType as ArrowDataType;
use arrow_schema::{Field as ArrowField, Schema as ArrowSchema, TimeUnit};
@@ -267,6 +267,23 @@ pub fn arrow_to_paimon_type(
nullable, length, element,
)?))
}
+ ArrowDataType::FixedSizeBinary(size) => {
+ // Fixed-length binary maps to Paimon `BINARY(n)`. The size is an
+ // i32; reject non-positive lengths with a clear error rather than
+ // casting a negative into a huge usize.
+ let length = usize::try_from(*size).map_err(|_|
crate::Error::DataTypeInvalid {
+ message: format!("Invalid FixedSizeBinary length: {size}"),
+ })?;
+ Ok(PaimonDataType::Binary(BinaryType::with_nullable(
+ nullable, length,
+ )?))
+ }
+ // A dictionary-encoded column has the logical type of its values; the
+ // key (index) type is a physical encoding detail. DataFusion routinely
+ // produces `Dictionary(Int32, Utf8)` for low-cardinality strings, so
+ // resolving to the value type lets schema inference (e.g. CREATE VIEW)
+ // succeed instead of failing as unsupported.
+ ArrowDataType::Dictionary(_, value_type) =>
arrow_to_paimon_type(value_type, nullable),
_ => Err(crate::Error::Unsupported {
message: format!("Unsupported Arrow type for Paimon conversion:
{arrow_type:?}"),
}),
@@ -437,6 +454,59 @@ mod tests {
}
}
+ #[test]
+ fn test_dictionary_resolves_to_value_type() {
+ // A dictionary-encoded column carries the logical type of its values;
+ // the key/index type is ignored. This is the common DataFusion case
+ // (`arrow_cast(x, 'Dictionary(Int32, Utf8)')`).
+ let varchar = PaimonDataType::VarChar(
+ VarCharType::with_nullable(true, VarCharType::MAX_LENGTH).unwrap(),
+ );
+ assert_arrow_to_paimon(
+ &ArrowDataType::Dictionary(
+ Box::new(ArrowDataType::Int32),
+ Box::new(ArrowDataType::Utf8),
+ ),
+ true,
+ &varchar,
+ );
+
+ // The value type drives the result regardless of the key type.
+ assert_arrow_to_paimon(
+ &ArrowDataType::Dictionary(
+ Box::new(ArrowDataType::Int8),
+ Box::new(ArrowDataType::Int64),
+ ),
+ true,
+ &PaimonDataType::BigInt(BigIntType::with_nullable(true)),
+ );
+
+ // Nullability is carried onto the resolved value type.
+ let non_null_varchar = PaimonDataType::VarChar(
+ VarCharType::with_nullable(false,
VarCharType::MAX_LENGTH).unwrap(),
+ );
+ assert_arrow_to_paimon(
+ &ArrowDataType::Dictionary(
+ Box::new(ArrowDataType::Int32),
+ Box::new(ArrowDataType::Utf8),
+ ),
+ false,
+ &non_null_varchar,
+ );
+ }
+
+ #[test]
+ fn test_fixed_size_binary_maps_to_binary() {
+ assert_arrow_to_paimon(
+ &ArrowDataType::FixedSizeBinary(16),
+ true,
+ &PaimonDataType::Binary(BinaryType::with_nullable(true,
16).unwrap()),
+ );
+
+ // A zero (or negative) length is rejected rather than silently
wrapping.
+ assert!(arrow_to_paimon_type(&ArrowDataType::FixedSizeBinary(0),
true).is_err());
+ }
+
#[test]
fn test_blob_type_maps_one_way_to_arrow_large_binary() {
let blob = PaimonDataType::Blob(BlobType::new());
