This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
The following commit(s) were added to refs/heads/main by this push:
new 52bfe1f Proposal: change `@scopedenum` to make modules to avoid type
piracy (#267)
52bfe1f is described below
commit 52bfe1f72fa72b0b2d150efeeeb9edacd853f96f
Author: Nathan Daly <[email protected]>
AuthorDate: Fri Jan 21 23:23:18 2022 -0500
Proposal: change `@scopedenum` to make modules to avoid type piracy (#267)
* Try creating module instead of overloading type getproperty
* Apply renaming from `@scopedenum` throughout the Package
Example:
- `Meta.UnionMode.Sparse` => `Meta.UnionModes.Sparse`
* Return the module since it's now meant to be user-visible:
```julia
julia> Arrow.FlatBuffers.@scopedenum MyEnum X=1 Y=2
Main.MyEnums
```
---
src/FlatBuffers/FlatBuffers.jl | 8 +++--
src/arraytypes/arraytypes.jl | 8 ++---
src/arraytypes/unions.jl | 8 ++---
src/eltypes.jl | 70 +++++++++++++++++++++---------------------
src/metadata/File.jl | 4 +--
src/metadata/Message.jl | 6 ++--
src/metadata/Schema.jl | 14 ++++-----
src/table.jl | 12 ++++----
src/write.jl | 8 ++---
test/arrowjson.jl | 28 ++++++++---------
test/runtests.jl | 6 ++--
test/testtables.jl | 14 ++++-----
12 files changed, 95 insertions(+), 91 deletions(-)
diff --git a/src/FlatBuffers/FlatBuffers.jl b/src/FlatBuffers/FlatBuffers.jl
index 187631b..cf1d5fd 100644
--- a/src/FlatBuffers/FlatBuffers.jl
+++ b/src/FlatBuffers/FlatBuffers.jl
@@ -121,7 +121,8 @@ macro scopedenum(T, syms...)
push!(defs.args, :(const $(esc(sym)) = $(esc(typename))($i)))
end
end
- mod = Symbol(typename, "Module")
+ _typename_str = string(typename)
+ mod = Symbol(_typename_str, last(_typename_str) == 's' ? "es" : "s")
syms = Tuple(Base.values(namemap))
blk = quote
module $(esc(mod))
@@ -135,7 +136,6 @@ macro scopedenum(T, syms...)
if isdefined(Base.Enums, :namemap)
Base.Enums.namemap(::Type{$(esc(typename))}) = $(esc(namemap))
end
- Base.getproperty(::Type{$(esc(typename))}, sym::Symbol) = sym in
$syms ? getfield($(esc(mod)), sym) : getfield($(esc(typename)), sym)
Base.typemin(x::Type{$(esc(typename))}) = $(esc(typename))($lo)
Base.typemax(x::Type{$(esc(typename))}) = $(esc(typename))($hi)
let insts = (Any[ $(esc(typename))(v) for v in $values ]...,)
@@ -144,11 +144,15 @@ macro scopedenum(T, syms...)
FlatBuffers.basetype(::$(esc(typename))) = $(basetype)
FlatBuffers.basetype(::Type{$(esc(typename))}) = $(basetype)
$defs
+ #Base.getproperty(::Type{$(esc(typename))}, sym::Symbol) = sym in
$syms ? getfield($(esc(mod)), sym) : getfield($(esc(typename)), sym)
+ export $(syms...)
end
end
push!(blk.args, :nothing)
blk.head = :toplevel
push!(blk.args, :(using .$mod))
+ # Return the newly created module, since it's (now) meant to be user
visible.
+ push!(blk.args, esc(mod))
return blk
end
diff --git a/src/arraytypes/arraytypes.jl b/src/arraytypes/arraytypes.jl
index c6aba0b..3b3273e 100644
--- a/src/arraytypes/arraytypes.jl
+++ b/src/arraytypes/arraytypes.jl
@@ -35,13 +35,13 @@ function toarrowvector(x, i=1, de=Dict{Int64, Any}(),
ded=DictEncoding[], meta=g
@debug 3 x
A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...)
if compression isa LZ4FrameCompressor
- A = compress(Meta.CompressionType.LZ4_FRAME, compression, A)
+ A = compress(Meta.CompressionTypes.LZ4_FRAME, compression, A)
elseif compression isa Vector{LZ4FrameCompressor}
- A = compress(Meta.CompressionType.LZ4_FRAME,
compression[Threads.threadid()], A)
+ A = compress(Meta.CompressionTypes.LZ4_FRAME,
compression[Threads.threadid()], A)
elseif compression isa ZstdCompressor
- A = compress(Meta.CompressionType.ZSTD, compression, A)
+ A = compress(Meta.CompressionTypes.ZSTD, compression, A)
elseif compression isa Vector{ZstdCompressor}
- A = compress(Meta.CompressionType.ZSTD,
compression[Threads.threadid()], A)
+ A = compress(Meta.CompressionTypes.ZSTD,
compression[Threads.threadid()], A)
end
@debug 2 "converted top-level column to arrow format: $(typeof(A))"
@debug 3 A
diff --git a/src/arraytypes/unions.jl b/src/arraytypes/unions.jl
index 045d7ad..72cb7c4 100644
--- a/src/arraytypes/unions.jl
+++ b/src/arraytypes/unions.jl
@@ -17,7 +17,7 @@
# Union arrays
# need a custom representation of Union types since arrow unions
# are ordered, and possibly indirected via separate typeIds array
-# here, T is Meta.UnionMode.Dense or Meta.UnionMode.Sparse,
+# here, T is Meta.UnionModes.Dense or Meta.UnionModes.Sparse,
# typeIds is a NTuple{N, Int32}, and U is a Tuple{...} of the
# unioned types
struct UnionT{T, typeIds, U}
@@ -93,7 +93,7 @@ end
# convenience wrappers for signaling that an array shoudld be written
# as with dense/sparse union arrow buffers
-struct DenseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Dense,
nothing, U}}
+struct DenseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionModes.Dense,
nothing, U}}
itr::T
end
@@ -118,7 +118,7 @@ function todense(::Type{UnionT{T, typeIds, U}}, x) where
{T, typeIds, U}
return types, offsets, data
end
-struct SparseUnionVector{T, U} <: AbstractVector{UnionT{Meta.UnionMode.Sparse,
nothing, U}}
+struct SparseUnionVector{T, U} <:
AbstractVector{UnionT{Meta.UnionModes.Sparse, nothing, U}}
itr::T
end
@@ -214,7 +214,7 @@ arrowvector(::UnionKind, x::Union{DenseUnion, SparseUnion},
i, nl, fi, de, ded,
function arrowvector(::UnionKind, x, i, nl, fi, de, ded, meta; kw...)
UT = eltype(x)
- if unionmode(UT) == Meta.UnionMode.Dense
+ if unionmode(UT) == Meta.UnionModes.Dense
x = x isa DenseUnionVector ? x.itr : x
typeids, offsets, data = todense(UT, x)
data2 = map(y -> arrowvector(y[2], i, nl + 1, y[1], de, ded, nothing;
kw...), enumerate(data))
diff --git a/src/eltypes.jl b/src/eltypes.jl
index 2969bc2..b201aba 100644
--- a/src/eltypes.jl
+++ b/src/eltypes.jl
@@ -109,18 +109,18 @@ end
# primitive types
function juliaeltype(f::Meta.Field, fp::Meta.FloatingPoint, convert)
- if fp.precision == Meta.Precision.HALF
+ if fp.precision == Meta.Precisions.HALF
Float16
- elseif fp.precision == Meta.Precision.SINGLE
+ elseif fp.precision == Meta.Precisions.SINGLE
Float32
- elseif fp.precision == Meta.Precision.DOUBLE
+ elseif fp.precision == Meta.Precisions.DOUBLE
Float64
end
end
function arrowtype(b, ::Type{T}) where {T <: AbstractFloat}
Meta.floatingPointStart(b)
- Meta.floatingPointAddPrecision(b, T === Float16 ? Meta.Precision.HALF : T
=== Float32 ? Meta.Precision.SINGLE : Meta.Precision.DOUBLE)
+ Meta.floatingPointAddPrecision(b, T === Float16 ? Meta.Precisions.HALF : T
=== Float32 ? Meta.Precisions.SINGLE : Meta.Precisions.DOUBLE)
return Meta.FloatingPoint, Meta.floatingPointEnd(b), nothing
end
@@ -175,18 +175,18 @@ struct Date{U, T} <: ArrowTimeType
x::T
end
-const DATE = Date{Meta.DateUnit.DAY, Int32}
+const DATE = Date{Meta.DateUnits.DAY, Int32}
Base.zero(::Type{Date{U, T}}) where {U, T} = Date{U, T}(T(0))
storagetype(::Type{Date{U, T}}) where {U, T} = T
-bitwidth(x::Meta.DateUnit) = x == Meta.DateUnit.DAY ? Int32 : Int64
-Date{Meta.DateUnit.DAY}(days) = DATE(Int32(days))
-Date{Meta.DateUnit.MILLISECOND}(ms) = Date{Meta.DateUnit.MILLISECOND,
Int64}(Int64(ms))
+bitwidth(x::Meta.DateUnit) = x == Meta.DateUnits.DAY ? Int32 : Int64
+Date{Meta.DateUnits.DAY}(days) = DATE(Int32(days))
+Date{Meta.DateUnits.MILLISECOND}(ms) = Date{Meta.DateUnits.MILLISECOND,
Int64}(Int64(ms))
juliaeltype(f::Meta.Field, x::Meta.Date, convert) = Date{x.unit,
bitwidth(x.unit)}
finaljuliatype(::Type{DATE}) = Dates.Date
Base.convert(::Type{Dates.Date}, x::DATE) = Dates.Date(Dates.UTD(Int64(x.x +
UNIX_EPOCH_DATE)))
-finaljuliatype(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}}) = Dates.DateTime
-Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND,
Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME)))
+finaljuliatype(::Type{Date{Meta.DateUnits.MILLISECOND, Int64}}) =
Dates.DateTime
+Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnits.MILLISECOND,
Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME)))
function arrowtype(b, ::Type{Date{U, T}}) where {U, T}
Meta.dateStart(b)
@@ -198,7 +198,7 @@ const UNIX_EPOCH_DATE = Dates.value(Dates.Date(1970))
Base.convert(::Type{DATE}, x::Dates.Date) = DATE(Int32(Dates.value(x) -
UNIX_EPOCH_DATE))
const UNIX_EPOCH_DATETIME = Dates.value(Dates.DateTime(1970))
-Base.convert(::Type{Date{Meta.DateUnit.MILLISECOND, Int64}},
x::Dates.DateTime) = Date{Meta.DateUnit.MILLISECOND,
Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME))
+Base.convert(::Type{Date{Meta.DateUnits.MILLISECOND, Int64}},
x::Dates.DateTime) = Date{Meta.DateUnits.MILLISECOND,
Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME))
ArrowTypes.ArrowType(::Type{Dates.Date}) = DATE
ArrowTypes.toarrow(x::Dates.Date) = convert(DATE, x)
@@ -213,16 +213,16 @@ struct Time{U, T} <: ArrowTimeType
end
Base.zero(::Type{Time{U, T}}) where {U, T} = Time{U, T}(T(0))
-const TIME = Time{Meta.TimeUnit.NANOSECOND, Int64}
+const TIME = Time{Meta.TimeUnits.NANOSECOND, Int64}
-bitwidth(x::Meta.TimeUnit) = x == Meta.TimeUnit.SECOND || x ==
Meta.TimeUnit.MILLISECOND ? Int32 : Int64
+bitwidth(x::Meta.TimeUnit) = x == Meta.TimeUnits.SECOND || x ==
Meta.TimeUnits.MILLISECOND ? Int32 : Int64
Time{U}(x) where {U <: Meta.TimeUnit} = Time{U, bitwidth(U)}(bitwidth(U)(x))
storagetype(::Type{Time{U, T}}) where {U, T} = T
juliaeltype(f::Meta.Field, x::Meta.Time, convert) = Time{x.unit,
bitwidth(x.unit)}
finaljuliatype(::Type{<:Time}) = Dates.Time
-periodtype(U::Meta.TimeUnit) = U === Meta.TimeUnit.SECOND ? Dates.Second :
- U === Meta.TimeUnit.MILLISECOND ?
Dates.Millisecond :
- U === Meta.TimeUnit.MICROSECOND ?
Dates.Microsecond : Dates.Nanosecond
+periodtype(U::Meta.TimeUnit) = U === Meta.TimeUnits.SECOND ? Dates.Second :
+ U === Meta.TimeUnits.MILLISECOND ?
Dates.Millisecond :
+ U === Meta.TimeUnits.MICROSECOND ?
Dates.Microsecond : Dates.Nanosecond
Base.convert(::Type{Dates.Time}, x::Time{U, T}) where {U, T} =
Dates.Time(Dates.Nanosecond(Dates.tons(periodtype(U)(x.x))))
function arrowtype(b, ::Type{Time{U, T}}) where {U, T}
@@ -252,7 +252,7 @@ function juliaeltype(f::Meta.Field, x::Meta.Timestamp,
convert)
return Timestamp{x.unit, x.timezone === nothing ? nothing :
Symbol(x.timezone)}
end
-const DATETIME = Timestamp{Meta.TimeUnit.MILLISECOND, nothing}
+const DATETIME = Timestamp{Meta.TimeUnits.MILLISECOND, nothing}
finaljuliatype(::Type{Timestamp{U, TZ}}) where {U, TZ} = ZonedDateTime
finaljuliatype(::Type{Timestamp{U, nothing}}) where {U} = DateTime
@@ -261,19 +261,19 @@ finaljuliatype(::Type{Timestamp{U, nothing}}) where {U} =
DateTime
@warn "automatically converting Arrow.Timestamp with precision = $U to
`$T` which only supports millisecond precision; conversion may be lossy; to
avoid converting, pass `Arrow.Table(source; convert=false)" maxlog=1
_id=hash((:warntimestamp, U, T))
function Base.convert(::Type{ZonedDateTime}, x::Timestamp{U, TZ}) where {U, TZ}
- (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) &&
warntimestamp(U, ZonedDateTime)
+ (U === Meta.TimeUnits.MICROSECOND || U == Meta.TimeUnits.NANOSECOND) &&
warntimestamp(U, ZonedDateTime)
return
ZonedDateTime(Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) +
UNIX_EPOCH_DATETIME))), TimeZone(String(TZ)))
end
function Base.convert(::Type{DateTime}, x::Timestamp{U, nothing}) where {U}
- (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) &&
warntimestamp(U, DateTime)
+ (U === Meta.TimeUnits.MICROSECOND || U == Meta.TimeUnits.NANOSECOND) &&
warntimestamp(U, DateTime)
return Dates.DateTime(Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) +
UNIX_EPOCH_DATETIME)))
end
-Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}},
x::ZonedDateTime) where {TZ} =
- Timestamp{Meta.TimeUnit.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x)) -
UNIX_EPOCH_DATETIME))
-Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND, nothing}},
x::DateTime) =
- Timestamp{Meta.TimeUnit.MILLISECOND, nothing}(Int64(Dates.value(x) -
UNIX_EPOCH_DATETIME))
+Base.convert(::Type{Timestamp{Meta.TimeUnits.MILLISECOND, TZ}},
x::ZonedDateTime) where {TZ} =
+ Timestamp{Meta.TimeUnits.MILLISECOND, TZ}(Int64(Dates.value(DateTime(x)) -
UNIX_EPOCH_DATETIME))
+Base.convert(::Type{Timestamp{Meta.TimeUnits.MILLISECOND, nothing}},
x::DateTime) =
+ Timestamp{Meta.TimeUnits.MILLISECOND, nothing}(Int64(Dates.value(x) -
UNIX_EPOCH_DATETIME))
function arrowtype(b, ::Type{Timestamp{U, TZ}}) where {U, TZ}
tz = TZ !== nothing ? FlatBuffers.createstring!(b, String(TZ)) :
FlatBuffers.UOffsetT(0)
@@ -289,11 +289,11 @@ const DATETIME_SYMBOL = Symbol("JuliaLang.DateTime")
ArrowTypes.arrowname(::Type{Dates.DateTime}) = DATETIME_SYMBOL
ArrowTypes.JuliaType(::Val{DATETIME_SYMBOL}, S) = Dates.DateTime
ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Timestamp) =
convert(Dates.DateTime, x)
-ArrowTypes.fromarrow(::Type{Dates.DateTime},
x::Date{Meta.DateUnit.MILLISECOND, Int64}) = convert(Dates.DateTime, x)
+ArrowTypes.fromarrow(::Type{Dates.DateTime},
x::Date{Meta.DateUnits.MILLISECOND, Int64}) = convert(Dates.DateTime, x)
ArrowTypes.default(::Type{Dates.DateTime}) = Dates.DateTime(1,1,1,1,1,1)
ArrowTypes.ArrowType(::Type{ZonedDateTime}) = Timestamp
-ArrowTypes.toarrow(x::ZonedDateTime) =
convert(Timestamp{Meta.TimeUnit.MILLISECOND, Symbol(x.timezone)}, x)
+ArrowTypes.toarrow(x::ZonedDateTime) =
convert(Timestamp{Meta.TimeUnits.MILLISECOND, Symbol(x.timezone)}, x)
const ZONEDDATETIME_SYMBOL = Symbol("JuliaLang.ZonedDateTime")
ArrowTypes.arrowname(::Type{ZonedDateTime}) = ZONEDDATETIME_SYMBOL
ArrowTypes.JuliaType(::Val{ZONEDDATETIME_SYMBOL}, S) = ZonedDateTime
@@ -310,15 +310,15 @@ scan each element to check each timezone.
`Arrow.ToTimestamp` provides a "bypass
first element of the `AbstractVector{ZonedDateTime}`, which in turn allows
`Arrow.write` to avoid costly checking/conversion and
can encode the `ZonedDateTime` as `Arrow.Timestamp` directly.
"""
-struct ToTimestamp{A, TZ} <:
AbstractVector{Timestamp{Meta.TimeUnit.MILLISECOND, TZ}}
+struct ToTimestamp{A, TZ} <:
AbstractVector{Timestamp{Meta.TimeUnits.MILLISECOND, TZ}}
data::A # AbstractVector{ZonedDateTime}
end
ToTimestamp(x::A) where {A <: AbstractVector{ZonedDateTime}} = ToTimestamp{A,
Symbol(x[1].timezone)}(x)
Base.IndexStyle(::Type{<:ToTimestamp}) = Base.IndexLinear()
Base.size(x::ToTimestamp) = (length(x.data),)
-Base.eltype(::ToTimestamp{A, TZ}) where {A, TZ} =
Timestamp{Meta.TimeUnit.MILLISECOND, TZ}
-Base.getindex(x::ToTimestamp{A, TZ}, i::Int) where {A, TZ} =
convert(Timestamp{Meta.TimeUnit.MILLISECOND, TZ}, getindex(x.data, i))
+Base.eltype(::ToTimestamp{A, TZ}) where {A, TZ} =
Timestamp{Meta.TimeUnits.MILLISECOND, TZ}
+Base.getindex(x::ToTimestamp{A, TZ}, i::Int) where {A, TZ} =
convert(Timestamp{Meta.TimeUnits.MILLISECOND, TZ}, getindex(x.data, i))
struct Interval{U, T} <: ArrowTimeType
x::T
@@ -326,9 +326,9 @@ end
Base.zero(::Type{Interval{U, T}}) where {U, T} = Interval{U, T}(T(0))
-bitwidth(x::Meta.IntervalUnit) = x == Meta.IntervalUnit.YEAR_MONTH ? Int32 :
Int64
-Interval{Meta.IntervalUnit.YEAR_MONTH}(x) =
Interval{Meta.IntervalUnit.YEAR_MONTH, Int32}(Int32(x))
-Interval{Meta.IntervalUnit.DAY_TIME}(x) = Interval{Meta.IntervalUnit.DAY_TIME,
Int64}(Int64(x))
+bitwidth(x::Meta.IntervalUnit) = x == Meta.IntervalUnits.YEAR_MONTH ? Int32 :
Int64
+Interval{Meta.IntervalUnits.YEAR_MONTH}(x) =
Interval{Meta.IntervalUnits.YEAR_MONTH, Int32}(Int32(x))
+Interval{Meta.IntervalUnits.DAY_TIME}(x) =
Interval{Meta.IntervalUnits.DAY_TIME, Int64}(Int64(x))
function juliaeltype(f::Meta.Field, x::Meta.Interval, convert)
return Interval{x.unit, bitwidth(x.unit)}
@@ -359,10 +359,10 @@ function arrowtype(b, ::Type{Duration{U}}) where {U}
return Meta.Duration, Meta.durationEnd(b), nothing
end
-arrowperiodtype(P) = Meta.TimeUnit.SECOND
-arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnit.MILLISECOND
-arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnit.MICROSECOND
-arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnit.NANOSECOND
+arrowperiodtype(P) = Meta.TimeUnits.SECOND
+arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnits.MILLISECOND
+arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnits.MICROSECOND
+arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnits.NANOSECOND
Base.convert(::Type{Duration{U}}, x::Dates.Period) where {U} =
Duration{U}(Dates.value(periodtype(U)(x)))
diff --git a/src/metadata/File.jl b/src/metadata/File.jl
index 8c1622c..6ad33ba 100644
--- a/src/metadata/File.jl
+++ b/src/metadata/File.jl
@@ -25,7 +25,7 @@ function Base.getproperty(x::Footer, field::Symbol)
if field === :version
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x),
MetadataVersion)
- return MetadataVersion.V1
+ return MetadataVersions.V1
elseif field === :schema
o = FlatBuffers.offset(x, 6)
if o != 0
@@ -87,4 +87,4 @@ function createBlock(b::FlatBuffers.Builder, offset::Int64,
metadatalength::Int3
prepend!(b, metadatalength)
prepend!(b, offset)
return FlatBuffers.offset(b)
-end
\ No newline at end of file
+end
diff --git a/src/metadata/Message.jl b/src/metadata/Message.jl
index 4fe6253..6af3a6e 100644
--- a/src/metadata/Message.jl
+++ b/src/metadata/Message.jl
@@ -54,11 +54,11 @@ function Base.getproperty(x::BodyCompression, field::Symbol)
if field === :codec
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x),
CompressionType)
- return CompressionType.LZ4_FRAME
+ return CompressionTypes.LZ4_FRAME
elseif field === :method
o = FlatBuffers.offset(x, 6)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x),
BodyCompressionMethod)
- return BodyCompressionMethod.BUFFER
+ return BodyCompressionMethods.BUFFER
end
return nothing
end
@@ -199,4 +199,4 @@ messageAddHeader(b::FlatBuffers.Builder,
header::FlatBuffers.UOffsetT) = FlatBuf
messageAddBodyLength(b::FlatBuffers.Builder, bodyLength::Int64) =
FlatBuffers.prependslot!(b, 3, bodyLength, 0)
messageAddCustomMetadata(b::FlatBuffers.Builder, meta::FlatBuffers.UOffsetT) =
FlatBuffers.prependoffsetslot!(b, 4, meta, 0)
messageStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) =
FlatBuffers.startvector!(b, 4, numelems, 4)
-messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
\ No newline at end of file
+messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b)
diff --git a/src/metadata/Schema.jl b/src/metadata/Schema.jl
index 9f83bef..d09b648 100644
--- a/src/metadata/Schema.jl
+++ b/src/metadata/Schema.jl
@@ -108,7 +108,7 @@ function Base.getproperty(x::Union, field::Symbol)
if field === :mode
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), UnionMode)
- return UnionMode.Sparse
+ return UnionModes.Sparse
elseif field === :typeIds
o = FlatBuffers.offset(x, 6)
o != 0 && return FlatBuffers.Array{Int32}(x, o)
@@ -159,7 +159,7 @@ function Base.getproperty(x::FloatingPoint, field::Symbol)
if field === :precision
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Precision)
- return Precision.HALF
+ return Precisions.HALF
end
return nothing
end
@@ -280,7 +280,7 @@ function Base.getproperty(x::Date, field::Symbol)
if field === :unit
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DateUnit)
- return DateUnit.MILLISECOND
+ return DateUnits.MILLISECOND
end
return nothing
end
@@ -302,7 +302,7 @@ function Base.getproperty(x::Time, field::Symbol)
if field === :unit
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
- return TimeUnit.MILLISECOND
+ return TimeUnits.MILLISECOND
elseif field === :bitWidth
o = FlatBuffers.offset(x, 6)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32)
@@ -327,7 +327,7 @@ function Base.getproperty(x::Timestamp, field::Symbol)
if field === :unit
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
- return TimeUnit.SECOND
+ return TimeUnits.SECOND
elseif field === :timezone
o = FlatBuffers.offset(x, 6)
o != 0 && return String(x, o + FlatBuffers.pos(x))
@@ -353,7 +353,7 @@ function Base.getproperty(x::Interval, field::Symbol)
if field === :unit
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x),
IntervalUnit)
- return IntervalUnit.YEAR_MONTH
+ return IntervalUnits.YEAR_MONTH
end
return nothing
end
@@ -373,7 +373,7 @@ function Base.getproperty(x::Duration, field::Symbol)
if field === :unit
o = FlatBuffers.offset(x, 4)
o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit)
- return TimeUnit.MILLISECOND
+ return TimeUnits.MILLISECOND
end
return nothing
end
diff --git a/src/table.jl b/src/table.jl
index d3a3197..88d65b1 100644
--- a/src/table.jl
+++ b/src/table.jl
@@ -173,9 +173,9 @@ function Base.iterate(x::Stream, (pos, id)=(x.pos, 1))
end
if compression !== nothing
- if compression.codec == Flatbuf.CompressionType.ZSTD
+ if compression.codec == Flatbuf.CompressionTypes.ZSTD
x.compression[] = :zstd
- elseif compression.codec == Flatbuf.CompressionType.LZ4_FRAME
+ elseif compression.codec == Flatbuf.CompressionTypes.LZ4_FRAME
x.compression[] = :lz4
else
throw(ArgumentError("unsupported compression codec:
$(compression.codec)"))
@@ -485,9 +485,9 @@ function uncompress(ptr::Ptr{UInt8}, buffer, compression)
len = unsafe_load(convert(Ptr{Int64}, ptr))
ptr += 8 # skip past uncompressed length as Int64
encodedbytes = unsafe_wrap(Array, ptr, buffer.length - 8)
- if compression.codec === Meta.CompressionType.LZ4_FRAME
+ if compression.codec === Meta.CompressionTypes.LZ4_FRAME
decodedbytes = transcode(LZ4FrameDecompressor, encodedbytes)
- elseif compression.codec === Meta.CompressionType.ZSTD
+ elseif compression.codec === Meta.CompressionTypes.ZSTD
decodedbytes = transcode(ZstdDecompressor, encodedbytes)
else
error("unsupported compression type when reading arrow buffers:
$(typeof(compression.codec))")
@@ -590,7 +590,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de,
nodeidx, bufferidx,
buffer = rb.buffers[bufferidx]
bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression)
bufferidx += 1
- if L.mode == Meta.UnionMode.Dense
+ if L.mode == Meta.UnionModes.Dense
buffer = rb.buffers[bufferidx]
bytes2, offsets = reinterp(Int32, batch, buffer, rb.compression)
bufferidx += 1
@@ -605,7 +605,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de,
nodeidx, bufferidx,
meta = buildmetadata(f.custom_metadata)
T = juliaeltype(f, meta, convert)
UT = UnionT(f, convert)
- if L.mode == Meta.UnionMode.Dense
+ if L.mode == Meta.UnionModes.Dense
B = DenseUnion{T, UT, typeof(data)}(bytes, bytes2, typeIds, offsets,
data, meta)
else
B = SparseUnion{T, UT, typeof(data)}(bytes, typeIds, data, meta)
diff --git a/src/write.jl b/src/write.jl
index 5020b32..1fd28aa 100644
--- a/src/write.jl
+++ b/src/write.jl
@@ -164,7 +164,7 @@ function write(io, source, writetofile, largelists,
compress, denseunions, dicte
dicts = FlatBuffers.UOffsetT(0)
end
Meta.footerStart(b)
- Meta.footerAddVersion(b, Meta.MetadataVersion.V4)
+ Meta.footerAddVersion(b, Meta.MetadataVersions.V4)
Meta.footerAddSchema(b, schfoot)
Meta.footerAddDictionaries(b, dicts)
Meta.footerAddRecordBatches(b, recordbatches)
@@ -271,7 +271,7 @@ end
function makemessage(b, headerType, header, columns=nothing, bodylen=0)
# write the message flatbuffer object
Meta.messageStart(b)
- Meta.messageAddVersion(b, Meta.MetadataVersion.V5)
+ Meta.messageAddVersion(b, Meta.MetadataVersions.V5)
Meta.messageAddHeaderType(b, headerType)
Meta.messageAddHeader(b, header)
Meta.messageAddBodyLength(b, Int64(bodylen))
@@ -312,7 +312,7 @@ function makeschema(b, sch::Tables.Schema{names}, columns)
where {names}
end
# write schema object
Meta.schemaStart(b)
- Meta.schemaAddEndianness(b, Meta.Endianness.Little)
+ Meta.schemaAddEndianness(b, Meta.Endiannesses.Little)
Meta.schemaAddFields(b, fields)
Meta.schemaAddCustomMetadata(b, meta)
return Meta.schemaEnd(b)
@@ -443,7 +443,7 @@ function makerecordbatch(b, sch::Tables.Schema{names,
types}, columns, alignment
if compress !== nothing
Meta.bodyCompressionStart(b)
Meta.bodyCompressionAddCodec(b, compress)
- Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethod.BUFFER)
+ Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethods.BUFFER)
compression = Meta.bodyCompressionEnd(b)
else
compression = FlatBuffers.UOffsetT(0)
diff --git a/test/arrowjson.jl b/test/arrowjson.jl
index 586ab93..0c0c8e1 100644
--- a/test/arrowjson.jl
+++ b/test/arrowjson.jl
@@ -80,14 +80,14 @@ mutable struct Timestamp <: Type
end
Timestamp() = Timestamp("", "", nothing)
-unit(U) = U == Arrow.Meta.TimeUnit.SECOND ? "SECOND" :
- U == Arrow.Meta.TimeUnit.MILLISECOND ? "MILLISECOND" :
- U == Arrow.Meta.TimeUnit.MICROSECOND ? "MICROSECOND" : "NANOSECOND"
+unit(U) = U == Arrow.Meta.TimeUnits.SECOND ? "SECOND" :
+ U == Arrow.Meta.TimeUnits.MILLISECOND ? "MILLISECOND" :
+ U == Arrow.Meta.TimeUnits.MICROSECOND ? "MICROSECOND" : "NANOSECOND"
Type(::Base.Type{Arrow.Timestamp{U, TZ}}) where {U, TZ} =
Timestamp("timestamp", unit(U), TZ === nothing ? nothing : String(TZ))
StructTypes.StructType(::Base.Type{Timestamp}) = StructTypes.Mutable()
-unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnit.SECOND :
- u == "MILLISECOND" ? Arrow.Meta.TimeUnit.MILLISECOND :
- u == "MICROSECOND" ? Arrow.Meta.TimeUnit.MICROSECOND :
Arrow.Meta.TimeUnit.NANOSECOND
+unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnits.SECOND :
+ u == "MILLISECOND" ? Arrow.Meta.TimeUnits.MILLISECOND :
+ u == "MICROSECOND" ? Arrow.Meta.TimeUnits.MICROSECOND :
Arrow.Meta.TimeUnits.NANOSECOND
juliatype(f, x::Timestamp) = Arrow.Timestamp{unitT(x.unit), x.timezone ===
nothing ? nothing : Symbol(x.timezone)}
struct Duration <: Type
@@ -104,9 +104,9 @@ struct Date <: Type
unit::String
end
-Type(::Base.Type{Arrow.Date{U, T}}) where {U, T} = Date("date", U ==
Arrow.Meta.DateUnit.DAY ? "DAY" : "MILLISECOND")
+Type(::Base.Type{Arrow.Date{U, T}}) where {U, T} = Date("date", U ==
Arrow.Meta.DateUnits.DAY ? "DAY" : "MILLISECOND")
StructTypes.StructType(::Base.Type{Date}) = StructTypes.Struct()
-juliatype(f, x::Date) = Arrow.Date{x.unit == "DAY" ? Arrow.Meta.DateUnit.DAY :
Arrow.Meta.DateUnit.MILLISECOND, x.unit == "DAY" ? Int32 : Int64}
+juliatype(f, x::Date) = Arrow.Date{x.unit == "DAY" ? Arrow.Meta.DateUnits.DAY
: Arrow.Meta.DateUnits.MILLISECOND, x.unit == "DAY" ? Int32 : Int64}
struct Time <: Type
name::String
@@ -123,9 +123,9 @@ struct Interval <: Type
unit::String
end
-Type(::Base.Type{Arrow.Interval{U, T}}) where {U, T} = Interval("interval", U
== Arrow.Meta.IntervalUnit.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME")
+Type(::Base.Type{Arrow.Interval{U, T}}) where {U, T} = Interval("interval", U
== Arrow.Meta.IntervalUnits.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME")
StructTypes.StructType(::Base.Type{Interval}) = StructTypes.Struct()
-juliatype(f, x::Interval) = Arrow.Interval{x.unit == "YEAR_MONTH" ?
Arrow.Meta.IntervalUnit.YEAR_MONTH : Arrow.Meta.IntervalUnit.DAY_TIME, x.unit
== "YEAR_MONTH" ? Int32 : Int64}
+juliatype(f, x::Interval) = Arrow.Interval{x.unit == "YEAR_MONTH" ?
Arrow.Meta.IntervalUnits.YEAR_MONTH : Arrow.Meta.IntervalUnits.DAY_TIME, x.unit
== "YEAR_MONTH" ? Int32 : Int64}
struct UnionT <: Type
name::String
@@ -133,10 +133,10 @@ struct UnionT <: Type
typIds::Vector{Int64}
end
-Type(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} =
UnionT("union", T == Arrow.Meta.UnionMode.Dense ? "DENSE" : "SPARSE",
collect(typeIds))
+Type(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} =
UnionT("union", T == Arrow.Meta.UnionModes.Dense ? "DENSE" : "SPARSE",
collect(typeIds))
children(::Base.Type{Arrow.UnionT{T, typeIds, U}}) where {T, typeIds, U} =
Field[Field("", fieldtype(U, i), nothing) for i = 1:fieldcount(U)]
StructTypes.StructType(::Base.Type{UnionT}) = StructTypes.Struct()
-juliatype(f, x::UnionT) = Arrow.UnionT{x.mode == "DENSE" ?
Arrow.Meta.UnionMode.DENSE : Arrow.Meta.UnionMode.SPARSE, Tuple(x.typeIds),
Tuple{(juliatype(y) for y in f.children)...}}
+juliatype(f, x::UnionT) = Arrow.UnionT{x.mode == "DENSE" ?
Arrow.Meta.UnionModes.DENSE : Arrow.Meta.UnionModes.SPARSE, Tuple(x.typeIds),
Tuple{(juliatype(y) for y in f.children)...}}
struct List <: Type
name::String
@@ -381,7 +381,7 @@ function FieldData(nm, ::Base.Type{T}, col, dictencodings)
where {T}
U = eltype(S)
tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) :
Arrow.typeids(S)
TYPE_ID = [x === missing ? 0 : tids[Arrow.isatypeid(x, U)] for x
in col]
- if Arrow.unionmode(S) == Arrow.Meta.UnionMode.Dense
+ if Arrow.unionmode(S) == Arrow.Meta.UnionModes.Dense
offs = zeros(Int32, fieldcount(U))
OFFSET = Int32[]
for x in col
@@ -502,7 +502,7 @@ function Base.getindex(x::ArrowArray{T}, i::Base.Int) where
{T}
U = eltype(S)
tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) :
Arrow.typeids(S)
typeid = tids[x.fielddata.TYPE_ID[i]]
- if Arrow.unionmode(S) == Arrow.Meta.UnionMode.DENSE
+ if Arrow.unionmode(S) == Arrow.Meta.UnionModes.DENSE
off = x.fielddata.OFFSET[i]
return ArrowArray(x.field.children[typeid+1],
x.fielddata.children[typeid+1], x.dictionaries)[off]
else
diff --git a/test/runtests.jl b/test/runtests.jl
index 0098aba..51710a7 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -332,7 +332,7 @@ tbl = Arrow.Table(Arrow.tobuffer(t))
# 166
t = (
- col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.NANOSECOND, nothing})],
+ col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.NANOSECOND, nothing})],
)
tbl = Arrow.Table(Arrow.tobuffer(t))
@test_logs (:warn, r"automatically converting Arrow.Timestamp with precision =
NANOSECOND") begin
@@ -342,8 +342,8 @@ end
# 95; Arrow.ToTimestamp
x = [ZonedDateTime(Dates.DateTime(2020), tz"Europe/Paris")]
c = Arrow.ToTimestamp(x)
-@test eltype(c) == Arrow.Timestamp{Arrow.Flatbuf.TimeUnitModule.MILLISECOND,
Symbol("Europe/Paris")}
-@test c[1] == Arrow.Timestamp{Arrow.Flatbuf.TimeUnitModule.MILLISECOND,
Symbol("Europe/Paris")}(1577836800000)
+@test eltype(c) == Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.MILLISECOND,
Symbol("Europe/Paris")}
+@test c[1] == Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.MILLISECOND,
Symbol("Europe/Paris")}(1577836800000)
# 158
# arrow ipc stream generated from pyarrow with no record batches
diff --git a/test/testtables.jl b/test/testtables.jl
index 686f336..4288b36 100644
--- a/test/testtables.jl
+++ b/test/testtables.jl
@@ -54,12 +54,12 @@ testtables = [
"arrow date/time types",
(
col14=[zero(Arrow.Decimal{Int32(2), Int32(2), Int128}),
zero(Arrow.Decimal{Int32(2), Int32(2), Int128}), zero(Arrow.Decimal{Int32(2),
Int32(2), Int128}), missing],
- col15=[zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}),
zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}),
zero(Arrow.Date{Arrow.Meta.DateUnit.DAY, Int32}), missing],
- col16=[zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}),
zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}),
zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND, Int32}), missing],
- col17=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}),
zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}),
zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND, nothing}), missing],
- col18=[zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}),
zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}),
zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH, Int32}), missing],
- col19=[zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}),
zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}),
zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), missing],
- col20=[zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}),
zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}),
zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND, Int64}), missing],
+ col15=[zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}),
zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}),
zero(Arrow.Date{Arrow.Meta.DateUnits.DAY, Int32}), missing],
+ col16=[zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}),
zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}),
zero(Arrow.Time{Arrow.Meta.TimeUnits.SECOND, Int32}), missing],
+ col17=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}),
zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}),
zero(Arrow.Timestamp{Arrow.Meta.TimeUnits.SECOND, nothing}), missing],
+ col18=[zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}),
zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}),
zero(Arrow.Interval{Arrow.Meta.IntervalUnits.YEAR_MONTH, Int32}), missing],
+ col19=[zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}),
zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}),
zero(Arrow.Duration{Arrow.Meta.TimeUnits.SECOND}), missing],
+ col20=[zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}),
zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}),
zero(Arrow.Date{Arrow.Meta.DateUnits.MILLISECOND, Int64}), missing],
),
NamedTuple(),
(convert=false,),
@@ -254,4 +254,4 @@ function testtable(nm, t, writekw, readkw, extratests)
@test length(tt) == length(t)
@test all(isequal.(values(t), values(tt)))
return
-end
\ No newline at end of file
+end
