Poncito commented on issue #281:
URL: https://github.com/apache/arrow-julia/issues/281#issuecomment-1057943353
Hi Jacob,
Sorry for the delay!
This code replicates:
```julia
using Arrow
struct Char8 <: AbstractChar
x::UInt8
end
Char8(x::Integer) = Char8(UInt8(x))
Base.codepoint(c::Char8) = UInt32(c.x)
ArrowTypes.ArrowKind(::Type{Char8}) = ArrowTypes.PrimitiveKind()
ArrowTypes.ArrowType(::Type{Char8}) = UInt8
ArrowTypes.arrowname(::Type{Char8}) = Symbol("JuliaLang.Char8")
ArrowTypes.toarrow(x::Char8) = x.x
table = (col1=[(Char8(1),Char8(2))],)
Arrow.write(IOBuffer(), table;compress=:zstd)
```
The stack is:
```
ERROR: LoadError: MethodError: Cannot `convert` an object of type
Arrow.Compressed{Arrow.Flatbuf.CompressionTypeModule.ZSTD,
Arrow.Primitive{UInt8, ArrowTypes.ToArrow{UInt8, Arrow.ToFixedSizeList{Char8,
2, Vector{Tuple{Char8, Char8}}}}}} to an object of type Arrow.CompressedBuffer
Closest candidates are:
convert(::Type{T}, ::T) where T at essentials.jl:205
Arrow.CompressedBuffer(::Any, ::Any) at
/home/romain/.julia/packages/Arrow/x6smw/src/arraytypes/compressed.jl:18
Stacktrace:
[1] push!(a::Vector{Arrow.CompressedBuffer},
item::Arrow.Compressed{Arrow.Flatbuf.CompressionTypeModule.ZSTD,
Arrow.Primitive{UInt8, ArrowTypes.ToArrow{UInt8, Arrow.ToFixedSizeList{Char8,
2, Vector{Tuple{Char8, Char8}}}}}})
@ Base ./array.jl:928
[2] compress(Z::Arrow.Flatbuf.CompressionTypeModule.CompressionType,
comp::CodecZstd.ZstdCompressor, x::Arrow.FixedSizeList{Tuple{UInt8, UInt8},
Arrow.Primitive{UInt8, ArrowTypes.ToArrow{UInt8, Arrow.ToFixedSizeList{Char8,
2, Vector{Tuple{Char8, Char8}}}}}})
@ Arrow ~/.julia/packages/Arrow/x6smw/src/arraytypes/fixedsizelist.jl:131
[3] toarrowvector(x::Vector{Tuple{Char8, Char8}}, i::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
compression::Vector{CodecZstd.ZstdCompressor}, kw::Base.Iterators.Pairs{Symbol,
Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode,
:dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/x6smw/src/arraytypes/arraytypes.jl:44
[4] (::Arrow.var"#113#114"{Dict{Int64, Any}, Bool,
Vector{CodecZstd.ZstdCompressor}, Bool, Bool, Bool, Int64, Nothing,
Vector{Arrow.DictEncoding}, Vector{Type},
Vector{Any}})(col::Vector{Tuple{Char8, Char8}}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/x6smw/src/write.jl:216
[5] eachcolumn
@ ~/.julia/packages/Tables/OWzlh/src/utils.jl:70 [inlined]
[6] toarrowtable(cols::NamedTuple{(:col1,), Tuple{Vector{Tuple{Char8,
Char8}}}}, dictencodings::Dict{Int64, Any}, largelists::Bool,
compress::Vector{CodecZstd.ZstdCompressor}, denseunions::Bool,
dictencode::Bool, dictencodenested::Bool, maxdepth::Int64, meta::Nothing,
colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/x6smw/src/write.jl:213
[7] macro expansion
@ ~/.julia/packages/Arrow/x6smw/src/write.jl:109 [inlined]
[8] macro expansion
@ ./task.jl:387 [inlined]
[9] write(io::IOBuffer, source::NamedTuple{(:col1,),
Tuple{Vector{Tuple{Char8, Char8}}}}, writetofile::Bool, largelists::Bool,
compress::Symbol, denseunions::Bool, dictencode::Bool, dictencodenested::Bool,
alignment::Int64, maxdepth::Int64, ntasks::Float64, meta::Nothing,
colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/x6smw/src/write.jl:101
[10] #write#102
@ ~/.julia/packages/Arrow/x6smw/src/write.jl:64 [inlined]
[11] top-level scope
@ Untitled-3:15
in expression starting at Untitled-3:15
```
What I understand from the stack is that [compres for a
primitive](https://github.com/apache/arrow-julia/blob/1447cb2b13b728729f9a89760ac07a848e31e599/src/arraytypes/primitive.jl#L73-L79)
returns a `Compressed`, and thus we cannot push directly into `buffers`
[here](https://github.com/apache/arrow-julia/blob/1447cb2b13b728729f9a89760ac07a848e31e599/src/arraytypes/fixedsizelist.jl#L131),
which expects elements of type `CompressedBuffer`.
I believe removing [this
branch](https://github.com/apache/arrow-julia/blob/1447cb2b13b728729f9a89760ac07a848e31e599/src/arraytypes/fixedsizelist.jl#L130-L131)
makes it work.
For my curiosity/understanding, what is the point of that last branch?
What's the point of pushing into `Compressed.buffers` rather than
`Compressed.children`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
