This is an automated email from the ASF dual-hosted git repository. quinnj pushed a commit to branch preservation/storage-20261008/julia-dev-Arrow-decimals-f2071e12 in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
commit 8316d5a44d2cee0d149cc475b639ea5add5a4fe4 Author: Jacob Quinn <[email protected]> AuthorDate: Tue Sep 1 09:57:52 2026 -0600 Read Arrow decimal columns as Decimals.Decimal{P,S,T} Every decimal width now materializes as `Decimals.Decimal{P,S,T}`, replacing the raw storage domain (`Int32`/`Int64`, and one heap-allocated `Vector{UInt8}` per element for 128/256). `Decimal{P,S,T}`'s coefficient IS the Arrow slot — the same width, the same two's-complement native-endian bytes — so the whole mapping is one line in `_decimalbasetype`: the storage claim equals the public type, `_postconvert` passes decimals through, and Core's bulk fixed-width copy carries the column with no per-element work at any width. A 100k-row decimal128 column materializes in one 1.6 MB allocation. `Decimal{P,S}` requires `0 <= S <= P`; a descriptor outside that keeps the raw coefficient domain rather than failing the read, since Arrow's scale is a signed Int32 that other producers do use more widely. `_retaineddecimal` follows the public domain so a read-then-rewrite of a decimal column round-trips its descriptor, and `_facadebasetype` now answers for decimals, so a replaced decimal column is refused on its element type like every other retained scalar. NOTE: Decimals.jl is unregistered. It is dev-pinned in the (gitignored) Manifest locally and MUST be registered before this merges or CI can run. Co-Authored-By: Claude Fable 5 <[email protected]> --- Project.toml | 2 ++ src/Arrow.jl | 1 + src/arrowtypes.jl | 4 +-- src/columnconstruction.jl | 73 ++++++++++++++++++++++----------------- src/table.jl | 30 +++++++++++++++- test/Project.toml | 1 + test/facade_tests.jl | 21 +++++++---- test/rewrite_regressions.jl | 5 ++- test/scan_battery.jl | 6 +++- test/support/AcceptanceSupport.jl | 1 + 10 files changed, 99 insertions(+), 45 deletions(-) diff --git a/Project.toml b/Project.toml index b031409..48007b8 100644 --- a/Project.toml +++ b/Project.toml @@ -27,6 +27,7 @@ CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Decimals = "8dc971d1-6768-4187-aca0-432d1dd9b305" EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" @@ -56,6 +57,7 @@ CloudStore = "1.6" CodecLz4 = "0.4" DataAPI = "1" Dates = "1.10" +Decimals = "1" CodecZstd = "0.8" EnumX = "1" Mmap = "1.10" diff --git a/src/Arrow.jl b/src/Arrow.jl index 5c4e05a..756c7cf 100644 --- a/src/Arrow.jl +++ b/src/Arrow.jl @@ -50,6 +50,7 @@ import DataAPI import ArrowStrings import ArrowTypes import Dates +import Decimals import Mmap import CodecLz4 import CodecZstd diff --git a/src/arrowtypes.jl b/src/arrowtypes.jl index 5faaad1..41ac96d 100644 --- a/src/arrowtypes.jl +++ b/src/arrowtypes.jl @@ -185,9 +185,7 @@ function _arrowtypesprimitivebasetype(f::AC.Field) t.unit == AC.MILLISECOND ? Dates.Millisecond : t.unit == AC.MICROSECOND ? Dates.Microsecond : Dates.Nanosecond end - if t isa AC.DecimalType - return t.bits == 32 ? Int32 : t.bits == 64 ? Int64 : Vector{UInt8} - end + t isa AC.DecimalType && return _decimalbasetype(t) if t isa AC.IntervalType return t.unit == AC.YEAR_MONTH ? Int32 : t.unit == AC.DAY_TIME ? NamedTuple{(:days, :millis),Tuple{Int32,Int32}} : diff --git a/src/columnconstruction.jl b/src/columnconstruction.jl index c37cb51..a4ee86a 100644 --- a/src/columnconstruction.jl +++ b/src/columnconstruction.jl @@ -3794,41 +3794,17 @@ function _retainedfixedbytes(f::AC.Field, v::AbstractVector) return _retainedfield(f), d end +# Rebuild a retained decimal column. The materialized public domain is +# `_decimalbasetype(t)`: `Decimal{P,S,T}` for every descriptor Decimals can +# name (its coefficient IS the Arrow slot, so the values vector needs no +# per-element conversion), and Core's raw coefficient domain for the rest. function _retaineddecimal(f::AC.Field, v::AbstractVector) t = f.type::AC.DecimalType present = _retainedvalidity(f, v) - if t.bits == 32 || t.bits == 64 - T = t.bits == 32 ? Int32 : Int64 - values = Vector{T}(undef, length(v)) - for (i, x) in enumerate(v) - x === missing || - x isa T || - throw( - ArgumentError( - "column $(f.name) must contain $T decimal storage values", - ), - ) - values[i] = x === missing ? zero(T) : x - end - databuf = AC._databuffer(values) - else - width = AC.primwidth(t) - values = zeros(UInt8, Base.checked_mul(length(v), width)) - for (i, x) in enumerate(v) - x === missing && continue - x isa AbstractVector{UInt8} || throw( - ArgumentError("column $(f.name) must contain byte-vector decimal values"), - ) - length(x) == width || throw( - ArgumentError( - "column $(f.name) decimal value $i has $(length(x)) bytes; " * - "expected $width", - ), - ) - copyto!(values, (i - 1) * width + 1, x, 1, width) - end - databuf = AC._databuffer(values) - end + D = _decimalbasetype(t) + databuf = + D === Vector{UInt8} ? AC._databuffer(_retaineddecimalbytes(t, f, v)) : + AC._databuffer(_retaineddecimalvalues(D, f, v)) d = AC.ArrayData( t, length(v), @@ -3838,6 +3814,39 @@ function _retaineddecimal(f::AC.Field, v::AbstractVector) return _retainedfield(f), d end +function _retaineddecimalvalues(::Type{D}, f::AC.Field, v::AbstractVector) where {D} + values = Vector{D}(undef, length(v)) + for (i, x) in enumerate(v) + if x === missing + values[i] = zero(D) + elseif x isa D + values[i] = x + else + throw(ArgumentError("column $(f.name) must contain $D decimal values")) + end + end + return values +end + +function _retaineddecimalbytes(t::AC.DecimalType, f::AC.Field, v::AbstractVector) + width = AC.primwidth(t) + values = zeros(UInt8, Base.checked_mul(length(v), width)) + for (i, x) in enumerate(v) + x === missing && continue + x isa AbstractVector{UInt8} || throw( + ArgumentError("column $(f.name) must contain byte-vector decimal values"), + ) + length(x) == width || throw( + ArgumentError( + "column $(f.name) decimal value $i has $(length(x)) bytes; " * + "expected $width", + ), + ) + copyto!(values, (i - 1) * width + 1, x, 1, width) + end + return values +end + function _retainedinterval(f::AC.Field, v::AbstractVector) t = f.type::AC.IntervalType present = _retainedvalidity(f, v) diff --git a/src/table.jl b/src/table.jl index e1f9ecb..87613cd 100644 --- a/src/table.jl +++ b/src/table.jl @@ -668,10 +668,39 @@ end _postconvert(t::AC.DictionaryType, col, budget=nothing) = _postconvert(t.valuetype, col, budget) +# The storage integer and maximum precision behind each Arrow decimal width. +# `Decimals.Decimal`'s own per-storage digit capacities are exactly Arrow's +# per-width precision ceilings, so the widths correspond one for one. +_decimalstorage(bits::Int) = + bits == 32 ? Int32 : bits == 64 ? Int64 : bits == 128 ? Int128 : Decimals.Int256 +_decimalcapacity(bits::Int) = bits == 32 ? 9 : bits == 64 ? 18 : bits == 128 ? 38 : 76 + +""" +The public element type of one Arrow decimal column. + +`Decimal{P,S,T}` holds a two's-complement native-endian coefficient of +exactly `T`'s width — byte for byte an Arrow decimal slot — so the column +materializes through Core's bulk copy with no per-element conversion, and +`_postconvert` passes decimals through untouched. + +`Decimal{P,S}` requires `0 <= S <= P`, but Arrow's `scale` is a signed Int32 +that other producers do use outside that range. Such a column keeps Core's +raw coefficient domain rather than failing the read: a reader must not choke +on a descriptor it merely cannot type. +""" +function _decimalbasetype(t::AC.DecimalType) + raw = t.bits == 32 ? Int32 : t.bits == 64 ? Int64 : Vector{UInt8} + t.bits in (32, 64, 128, 256) || return raw + 1 <= t.precision <= _decimalcapacity(t.bits) || return raw + 0 <= t.scale <= t.precision || return raw + return Decimals.Decimal{t.precision,t.scale,_decimalstorage(t.bits)} +end + # The public element type of the SCALAR layouts the facade converts (Dates) # or passes through; `_declaredbasetype` completes it for every layout and # `_declaredeltype` is the Field-aware rule the facade materializes with. function _facadebasetype(t::AC.ArrowType) + t isa AC.DecimalType && return _decimalbasetype(t) t isa AC.DateType && return t.unit == AC.DAY ? Dates.Date : Dates.DateTime if t isa AC.TimestampType ext = _zonedext(t) @@ -1357,7 +1386,6 @@ _declaredbasetype(t::AC.ArrowType) = t isa AC.StructType ? Vector{Pair{String,Any}} : t isa AC.MapType ? Vector{Pair{Any,Any}} : t isa AC.NullType ? Missing : - t isa AC.DecimalType ? (t.bits == 32 ? Int32 : t.bits == 64 ? Int64 : Vector{UInt8}) : t isa AC.IntervalType ? ( t.unit == AC.YEAR_MONTH ? Int32 : diff --git a/test/Project.toml b/test/Project.toml index ed0b830..23edd5a 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -23,6 +23,7 @@ CloudStore = "3365d9ee-d53b-4a56-812d-5344d5b716d7" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Decimals = "8dc971d1-6768-4187-aca0-432d1dd9b305" ArrowStrings = "c38d8858-22a2-449e-9eca-ef92ec15f353" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" diff --git a/test/facade_tests.jl b/test/facade_tests.jl index cf10461..6d369b3 100644 --- a/test/facade_tests.jl +++ b/test/facade_tests.jl @@ -24,6 +24,7 @@ using Test using Tables using Dates import DataAPI +import Decimals using Arrow using ArrowStrings @@ -1835,8 +1836,13 @@ end # Core _value methods are the authority), including Field-aware # compositions; empty and nonempty columns decide keep/drop alike. @test Arrow._declaredbasetype(Arrow.AC.ListViewType(false)) === Vector{Any} - @test Arrow._declaredbasetype(Arrow.AC.DecimalType(10, 2, 32)) === Int32 - @test Arrow._declaredbasetype(Arrow.AC.DecimalType(38, 2, 128)) === Vector{UInt8} + @test Arrow._declaredbasetype(Arrow.AC.DecimalType(9, 2, 32)) === + Decimals.Decimal32{2} + @test Arrow._declaredbasetype(Arrow.AC.DecimalType(38, 2, 128)) === + Decimals.Decimal128{2} + # A scale Decimal{P,S} cannot name keeps Core's raw coefficient domain. + @test Arrow._declaredbasetype(Arrow.AC.DecimalType(10, -2, 32)) === Int32 + @test Arrow._declaredbasetype(Arrow.AC.DecimalType(38, 40, 128)) === Vector{UInt8} @test Arrow._declaredbasetype(Arrow.AC.IntervalType(Arrow.AC.YEAR_MONTH)) === Int32 @test Arrow._declaredbasetype(Arrow.AC.IntervalType(Arrow.AC.DAY_TIME)) === NamedTuple{(:days, :millis),Tuple{Int32,Int32}} @@ -1950,8 +1956,9 @@ end ) @test isempty(getfield(tuo, :schema).fields) end - # End-to-end: Decimal64 rows are raw Int64 — an => Integer override - # keeps the retained field for empty and nonempty columns alike. + # End-to-end: Decimal64 rows are `Decimal{P,S,Int64} <: Real` — a + # => Real override keeps the retained field for empty and nonempty + # columns alike. for (n, vals) in ((2, Int64[1234, 5678]), (0, Int64[])) dd = Arrow.AC.ArrayData( Arrow.AC.DecimalType(10, 2, 64), @@ -1969,7 +1976,7 @@ end dsch, [Arrow.AC.RecordBatch(dsch, Arrow.AC.ArrayData[dd], n)], ) - td = Arrow.Table(db; scan=Tables.Scan(select=(:dec => Integer,))) + td = Arrow.Table(db; scan=Tables.Scan(select=(:dec => Real,))) dsch2 = getfield(td, :schema) @test length(dsch2.fields) == 1 @test dsch2.fields[1].type isa Arrow.AC.DecimalType @@ -2182,7 +2189,7 @@ end [AC.BufferSlice(), AC._databuffer(Int128[])]; nullcount=0, ), - Vector{UInt8}, + Decimals.Decimal{10,2,Int128}, ), ( "d64", @@ -2193,7 +2200,7 @@ end [AC.BufferSlice(), AC._databuffer(Int64[])]; nullcount=0, ), - Int64, + Decimals.Decimal{10,2,Int64}, ), ( "iym", diff --git a/test/rewrite_regressions.jl b/test/rewrite_regressions.jl index b6b9da6..f30d2bc 100644 --- a/test/rewrite_regressions.jl +++ b/test/rewrite_regressions.jl @@ -24,6 +24,7 @@ using Test using Dates using Tables import DataAPI +import Decimals using Arrow using ArrowStrings @@ -199,7 +200,9 @@ end ) decimalback = Arrow.readstream(_rewrite(decimalbytes)) @test decimalback.schema.fields[1].type == decimaltype - @test Arrow.Table(_rewrite(decimalbytes)).dec == Int64[123, 456] + D = Decimals.Decimal{10,2,Int64} + @test Arrow.Table(_rewrite(decimalbytes)).dec == + D[reinterpret(D, 123), reinterpret(D, 456)] end @testset "retained composites rebuild recursively" begin diff --git a/test/scan_battery.jl b/test/scan_battery.jl index 2430ed7..e43c0e8 100644 --- a/test/scan_battery.jl +++ b/test/scan_battery.jl @@ -416,7 +416,11 @@ function _scan_main() emptyrunends = Field("run_ends", IntType(32, true); nullable=false) routeclaims = ( Field("binary", BinaryType(false); nullable=false) => Vector{UInt8}, - Field("decimal", DecimalType(10, 2, 128); nullable=false) => Vector{UInt8}, + Field("decimal", DecimalType(10, 2, 128); nullable=false) => + Decimals.Decimal{10,2,Int128}, + # A scale `Decimal{P,S}` cannot name keeps the raw byte domain. + Field("decimal_wide_scale", DecimalType(10, 40, 128); nullable=false) => + Vector{UInt8}, Field("null", NullType(); nullable=true) => Any, emptyunionfield => Any, Field( diff --git a/test/support/AcceptanceSupport.jl b/test/support/AcceptanceSupport.jl index e5f9df7..d113edb 100644 --- a/test/support/AcceptanceSupport.jl +++ b/test/support/AcceptanceSupport.jl @@ -18,6 +18,7 @@ module AcceptanceSupport import Arrow +import Decimals using PooledArrays using Tables import Base64
