This is an automated email from the ASF dual-hosted git repository.
quinnj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
The following commit(s) were added to refs/heads/main by this push:
new 0be54a9 replace custom debug handling with LoggingExtras (#355)
0be54a9 is described below
commit 0be54a9799ca3346737d547ee79aa3a2c28a2a96
Author: Ben Baumgold <[email protected]>
AuthorDate: Wed Nov 2 23:03:13 2022 -0500
replace custom debug handling with LoggingExtras (#355)
* remove custom debug handling
* replace @debug with @debugv
Co-authored-by: Ben Baumgold <[email protected]>
---
Project.toml | 2 ++
src/Arrow.jl | 26 +----------------
src/append.jl | 2 +-
src/arraytypes/arraytypes.jl | 12 ++++----
src/arraytypes/bool.jl | 10 +++----
src/arraytypes/compressed.jl | 12 ++++----
src/arraytypes/dictencoding.jl | 12 ++++----
src/arraytypes/fixedsizelist.jl | 12 ++++----
src/arraytypes/map.jl | 16 +++++------
src/arraytypes/primitive.jl | 12 ++++----
src/arraytypes/struct.jl | 8 +++---
src/arraytypes/unions.jl | 14 +++++-----
src/table.jl | 62 ++++++++++++++++++++---------------------
src/write.jl | 20 ++++++-------
14 files changed, 99 insertions(+), 121 deletions(-)
diff --git a/Project.toml b/Project.toml
index 7798081..70b598f 100644
--- a/Project.toml
+++ b/Project.toml
@@ -27,6 +27,7 @@ CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
@@ -41,6 +42,7 @@ BitIntegers = "0.2"
CodecLz4 = "0.4"
CodecZstd = "0.7"
DataAPI = "1"
+LoggingExtras = "0.4"
FilePathsBase = "0.9"
PooledArrays = "0.5, 1.0"
SentinelArrays = "1"
diff --git a/src/Arrow.jl b/src/Arrow.jl
index 6eaa08e..01c7993 100644
--- a/src/Arrow.jl
+++ b/src/Arrow.jl
@@ -43,6 +43,7 @@ module Arrow
using Base.Iterators
using Mmap
+using LoggingExtras
import Dates
using DataAPI, Tables, SentinelArrays, PooledArrays, CodecLz4, CodecZstd,
TimeZones, BitIntegers, WorkerUtilities
@@ -51,31 +52,6 @@ export ArrowTypes
using Base: @propagate_inbounds
import Base: ==
-const DEBUG_LEVEL = Ref(0)
-
-function setdebug!(level::Int)
- DEBUG_LEVEL[] = level
- return
-end
-
-function withdebug(f, level)
- lvl = DEBUG_LEVEL[]
- try
- setdebug!(level)
- f()
- finally
- setdebug!(lvl)
- end
-end
-
-macro debug(level, msg)
- esc(quote
- if DEBUG_LEVEL[] >= $level
- println(string("DEBUG: ", $(QuoteNode(__source__.file)), ":",
$(QuoteNode(__source__.line)), " ", $msg))
- end
- end)
-end
-
const FILE_FORMAT_MAGIC_BYTES = b"ARROW1"
const CONTINUATION_INDICATOR_BYTES = 0xffffffff
diff --git a/src/append.jl b/src/append.jl
index f6716f3..5f521dd 100644
--- a/src/append.jl
+++ b/src/append.jl
@@ -126,7 +126,7 @@ function append(io::IO, source, arrow_schema, compress,
largelists, denseunions,
@error "error writing arrow data on partition = $(errorref[][3])"
exception=(errorref[][1], errorref[][2])
error("fatal error writing arrow data")
end
- @debug 1 "processing table partition i = $i"
+ @debugv 1 "processing table partition i = $i"
tbl_cols = Tables.columns(tbl)
tbl_schema = Tables.schema(tbl_cols)
diff --git a/src/arraytypes/arraytypes.jl b/src/arraytypes/arraytypes.jl
index 32b9491..355a034 100644
--- a/src/arraytypes/arraytypes.jl
+++ b/src/arraytypes/arraytypes.jl
@@ -32,8 +32,8 @@ getmetadata(x::ArrowVector) = x.metadata
Base.deleteat!(x::T, inds) where {T <: ArrowVector} =
throw(ArgumentError("`$T` does not support `deleteat!`; arrow data is by nature
immutable"))
function toarrowvector(x, i=1, de=Dict{Int64, Any}(), ded=DictEncoding[],
meta=getmetadata(x); compression::Union{Nothing, Vector{LZ4FrameCompressor},
LZ4FrameCompressor, Vector{ZstdCompressor}, ZstdCompressor}=nothing, kw...)
- @debug 2 "converting top-level column to arrow format: col = $(typeof(x)),
compression = $compression, kw = $(kw.data)"
- @debug 3 x
+ @debugv 2 "converting top-level column to arrow format: col =
$(typeof(x)), compression = $compression, kw = $(kw.data)"
+ @debugv 3 x
A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...)
if compression isa LZ4FrameCompressor
A = compress(Meta.CompressionTypes.LZ4_FRAME, compression, A)
@@ -44,8 +44,8 @@ function toarrowvector(x, i=1, de=Dict{Int64, Any}(),
ded=DictEncoding[], meta=g
elseif compression isa Vector{ZstdCompressor}
A = compress(Meta.CompressionTypes.ZSTD,
compression[Threads.threadid()], A)
end
- @debug 2 "converted top-level column to arrow format: $(typeof(A))"
- @debug 3 A
+ @debugv 2 "converted top-level column to arrow format: $(typeof(A))"
+ @debugv 3 A
return A
end
@@ -104,7 +104,7 @@ compress(Z::Meta.CompressionType, comp, v::NullVector) =
function makenodesbuffers!(col::NullVector, fieldnodes, fieldbuffers,
bufferoffset, alignment)
push!(fieldnodes, FieldNode(length(col), length(col)))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
return bufferoffset
end
@@ -187,7 +187,7 @@ end
function writebitmap(io, col::ArrowVector, alignment)
v = col.validity
- @debug 1 "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
+ @debugv 1 "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
v.nc == 0 && return 0
n = Base.write(io, view(v.bytes, v.pos:(v.pos + cld(v.ℓ, 8) - 1)))
return n + writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/bool.jl b/src/arraytypes/bool.jl
index c11ceb6..1b1239e 100644
--- a/src/arraytypes/bool.jl
+++ b/src/arraytypes/bool.jl
@@ -89,22 +89,22 @@ function makenodesbuffers!(col::BoolVector, fieldnodes,
fieldbuffers, bufferoffs
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + blen
end
function writebuffer(io, col::BoolVector, alignment)
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
n = Base.write(io, view(col.arrow, col.pos:(col.pos + cld(col.ℓ, 8) - 1)))
return n + writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/compressed.jl b/src/arraytypes/compressed.jl
index 5f8e67e..bbb6680 100644
--- a/src/arraytypes/compressed.jl
+++ b/src/arraytypes/compressed.jl
@@ -53,11 +53,11 @@ compress(Z::Meta.CompressionType, comp, v::ValidityBitmap) =
function makenodesbuffers!(col::Compressed, fieldnodes, fieldbuffers,
bufferoffset, alignment)
push!(fieldnodes, FieldNode(col.len, col.nullcount))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
for buffer in col.buffers
blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
end
for child in col.children
@@ -69,16 +69,16 @@ end
function writearray(io, b::CompressedBuffer)
if length(b.data) > 0
n = Base.write(io, b.uncompressedlength)
- @debug 1 "writing compressed buffer: uncompressedlength =
$(b.uncompressedlength), n = $(length(b.data))"
- @debug 2 b.data
+ @debugv 1 "writing compressed buffer: uncompressedlength =
$(b.uncompressedlength), n = $(length(b.data))"
+ @debugv 2 b.data
return n + Base.write(io, b.data)
end
return 0
end
function writebuffer(io, col::Compressed, alignment)
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
for buffer in col.buffers
n = writearray(io, buffer)
writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/dictencoding.jl b/src/arraytypes/dictencoding.jl
index 3eb03db..e08f1a0 100644
--- a/src/arraytypes/dictencoding.jl
+++ b/src/arraytypes/dictencoding.jl
@@ -289,16 +289,16 @@ function makenodesbuffers!(col::DictEncoded{T, S},
fieldnodes, fieldbuffers, buf
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
# indices
blen = sizeof(S) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
return bufferoffset
end
@@ -308,12 +308,12 @@ DataAPI.refarray(x::DictEncoded{T, S}) where {T, S} =
x.indices .+ one(S)
DataAPI.refpool(x::DictEncoded) = copy(x.encoding.data)
function writebuffer(io, col::DictEncoded, alignment)
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
# write indices
n = writearray(io, col.indices)
- @debug 1 "writing array: col = $(typeof(col.indices)), n = $n, padded =
$(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.indices)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
diff --git a/src/arraytypes/fixedsizelist.jl b/src/arraytypes/fixedsizelist.jl
index 98e5f4b..4246008 100644
--- a/src/arraytypes/fixedsizelist.jl
+++ b/src/arraytypes/fixedsizelist.jl
@@ -139,16 +139,16 @@ function makenodesbuffers!(col::FixedSizeList{T, A},
fieldnodes, fieldbuffers, b
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
if eltype(A) === UInt8
blen =
ArrowTypes.getsize(ArrowTypes.ArrowKind(Base.nonmissingtype(T))) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers,
bufferoffset, alignment)
@@ -157,13 +157,13 @@ function makenodesbuffers!(col::FixedSizeList{T, A},
fieldnodes, fieldbuffers, b
end
function writebuffer(io, col::FixedSizeList{T, A}, alignment) where {T, A}
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
# write values array
if eltype(A) === UInt8
n = writearray(io, UInt8, col.data)
- @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
diff --git a/src/arraytypes/map.jl b/src/arraytypes/map.jl
index fe1409f..67a791c 100644
--- a/src/arraytypes/map.jl
+++ b/src/arraytypes/map.jl
@@ -79,21 +79,21 @@ function makenodesbuffers!(col::Union{Map{T, O, A}, List{T,
O, A}}, fieldnodes,
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make array buffer
bufferoffset += blen
blen = sizeof(O) * (len + 1)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
if eltype(A) == UInt8
blen = length(col.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers,
bufferoffset, alignment)
@@ -102,17 +102,17 @@ function makenodesbuffers!(col::Union{Map{T, O, A},
List{T, O, A}}, fieldnodes,
end
function writebuffer(io, col::Union{Map{T, O, A}, List{T, O, A}}, alignment)
where {T, O, A}
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
# write offsets
n = writearray(io, O, col.offsets.offsets)
- @debug 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n,
padded = $(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n,
padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
# write values array
if eltype(A) == UInt8
n = writearray(io, UInt8, col.data)
- @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
diff --git a/src/arraytypes/primitive.jl b/src/arraytypes/primitive.jl
index 3612b43..aca060b 100644
--- a/src/arraytypes/primitive.jl
+++ b/src/arraytypes/primitive.jl
@@ -82,25 +82,25 @@ function makenodesbuffers!(col::Primitive{T}, fieldnodes,
fieldbuffers, bufferof
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = len * sizeof(Base.nonmissingtype(T))
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + padding(blen, alignment)
end
function writebuffer(io, col::Primitive{T}, alignment) where {T}
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
n = writearray(io, Base.nonmissingtype(T), col.data)
- @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
diff --git a/src/arraytypes/struct.jl b/src/arraytypes/struct.jl
index e32a016..806c58d 100644
--- a/src/arraytypes/struct.jl
+++ b/src/arraytypes/struct.jl
@@ -110,11 +110,11 @@ function makenodesbuffers!(col::Struct{T}, fieldnodes,
fieldbuffers, bufferoffse
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
for child in col.data
bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers,
bufferoffset, alignment)
@@ -123,8 +123,8 @@ function makenodesbuffers!(col::Struct{T}, fieldnodes,
fieldbuffers, bufferoffse
end
function writebuffer(io, col::Struct, alignment)
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
writebitmap(io, col, alignment)
# write values arrays
for child in col.data
diff --git a/src/arraytypes/unions.jl b/src/arraytypes/unions.jl
index 72cb7c4..6b43ec7 100644
--- a/src/arraytypes/unions.jl
+++ b/src/arraytypes/unions.jl
@@ -245,16 +245,16 @@ function makenodesbuffers!(col::Union{DenseUnion,
SparseUnion}, fieldnodes, fiel
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# typeIds buffer
push!(fieldbuffers, Buffer(bufferoffset, len))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(len, alignment)
if col isa DenseUnion
# offsets buffer
blen = sizeof(Int32) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
end
for child in col.data
@@ -264,15 +264,15 @@ function makenodesbuffers!(col::Union{DenseUnion,
SparseUnion}, fieldnodes, fiel
end
function writebuffer(io, col::Union{DenseUnion, SparseUnion}, alignment)
- @debug 1 "writebuffer: col = $(typeof(col))"
- @debug 2 col
+ @debugv 1 "writebuffer: col = $(typeof(col))"
+ @debugv 2 col
# typeIds buffer
n = writearray(io, UInt8, col.typeIds)
- @debug 1 "writing array: col = $(typeof(col.typeIds)), n = $n, padded =
$(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.typeIds)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
if col isa DenseUnion
n = writearray(io, Int32, col.offsets)
- @debug 1 "writing array: col = $(typeof(col.offsets)), n = $n, padded
= $(padding(n, alignment))"
+ @debugv 1 "writing array: col = $(typeof(col.offsets)), n = $n, padded
= $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
end
for child in col.data
diff --git a/src/table.jl b/src/table.jl
index 70da986..799b922 100644
--- a/src/table.jl
+++ b/src/table.jl
@@ -134,7 +134,7 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
push!(x.types, juliaeltype(field,
buildmetadata(field.custom_metadata), x.convert))
# recursively find any dictionaries for any fields
getdictionaries!(x.dictencoded, field)
- @debug 1 "parsed column from schema: field = $field"
+ @debugv 1 "parsed column from schema: field = $field"
end
elseif header != x.schema
throw(ArgumentError("mismatched schemas between different
arrow batches: $(x.schema) != $header"))
@@ -142,7 +142,7 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
elseif header isa Meta.DictionaryBatch
id = header.id
recordbatch = header.data
- @debug 1 "parsing dictionary batch message: id = $id, compression
= $(recordbatch.compression)"
+ @debugv 1 "parsing dictionary batch message: id = $id, compression
= $(recordbatch.compression)"
if recordbatch.compression !== nothing
compression = recordbatch.compression
end
@@ -160,9 +160,9 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
A = ChainedVector([values])
S = field.dictionary.indexType === nothing ? Int32 :
juliaeltype(field, field.dictionary.indexType, false)
x.dictencodings[id] = DictEncoding{eltype(A), S, typeof(A)}(id, A,
field.dictionary.isOrdered, values.metadata)
- @debug 1 "parsed dictionary batch message: id=$id, data=$values\n"
+ @debugv 1 "parsed dictionary batch message: id=$id, data=$values\n"
elseif header isa Meta.RecordBatch
- @debug 1 "parsing record batch message: compression =
$(header.compression)"
+ @debugv 1 "parsing record batch message: compression =
$(header.compression)"
if header.compression !== nothing
compression = header.compression
end
@@ -307,7 +307,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
# store custom_metadata of batch.msg?
header = batch.msg.header
if header isa Meta.Schema
- @debug 1 "parsing schema message"
+ @debugv 1 "parsing schema message"
# assert endianness?
# store custom_metadata?
if sch === nothing
@@ -315,7 +315,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
push!(names(t), Symbol(field.name))
# recursively find any dictionaries for any fields
getdictionaries!(dictencoded, field)
- @debug 1 "parsed column from schema: field = $field"
+ @debugv 1 "parsed column from schema: field = $field"
end
sch = header
schema(t)[] = sch
@@ -325,7 +325,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
elseif header isa Meta.DictionaryBatch
id = header.id
recordbatch = header.data
- @debug 1 "parsing dictionary batch message: id = $id,
compression = $(recordbatch.compression)"
+ @debugv 1 "parsing dictionary batch message: id = $id,
compression = $(recordbatch.compression)"
if haskey(dictencodings, id) && header.isDelta
# delta
field = dictencoded[id]
@@ -346,10 +346,10 @@ function Table(blobs::Vector{ArrowBlob};
convert::Bool=true)
A = values
S = field.dictionary.indexType === nothing ? Int32 :
juliaeltype(field, field.dictionary.indexType, false)
dictencodings[id] = DictEncoding{eltype(A), S, typeof(A)}(id,
A, field.dictionary.isOrdered, values.metadata)
- @debug 1 "parsed dictionary batch message: id=$id,
data=$values\n"
+ @debugv 1 "parsed dictionary batch message: id=$id,
data=$values\n"
elseif header isa Meta.RecordBatch
anyrecordbatches = true
- @debug 1 "parsing record batch message: compression =
$(header.compression)"
+ @debugv 1 "parsing record batch message: compression =
$(header.compression)"
Threads.@spawn begin
cols = collect(VectorIterator(sch, $batch, dictencodings,
convert))
put!(() -> put!(tsks, cols), sync, $(rbi))
@@ -398,36 +398,36 @@ struct Batch
end
function Base.iterate(x::BatchIterator, (pos, id)=(x.startpos, 0))
- @debug 1 "checking for next arrow message: pos = $pos"
+ @debugv 1 "checking for next arrow message: pos = $pos"
if pos + 3 > length(x.bytes)
- @debug 1 "not enough bytes left for another batch message"
+ @debugv 1 "not enough bytes left for another batch message"
return nothing
end
if readbuffer(x.bytes, pos, UInt32) != CONTINUATION_INDICATOR_BYTES
- @debug 1 "didn't find continuation byte to keep parsing messages:
$(readbuffer(x.bytes, pos, UInt32))"
+ @debugv 1 "didn't find continuation byte to keep parsing messages:
$(readbuffer(x.bytes, pos, UInt32))"
return nothing
end
pos += 4
if pos + 3 > length(x.bytes)
- @debug 1 "not enough bytes left to read length of another batch
message"
+ @debugv 1 "not enough bytes left to read length of another batch
message"
return nothing
end
msglen = readbuffer(x.bytes, pos, Int32)
if msglen == 0
- @debug 1 "message has 0 length; terminating message parsing"
+ @debugv 1 "message has 0 length; terminating message parsing"
return nothing
end
pos += 4
if pos + msglen - 1 > length(x.bytes)
- @debug 1 "not enough bytes left to read Meta.Message"
+ @debugv 1 "not enough bytes left to read Meta.Message"
return nothing
end
msg = FlatBuffers.getrootas(Meta.Message, x.bytes, pos-1)
pos += msglen
# pos now points to message body
- @debug 1 "parsing message: pos = $pos, msglen = $msglen, bodyLength =
$(msg.bodyLength)"
+ @debugv 1 "parsing message: pos = $pos, msglen = $msglen, bodyLength =
$(msg.bodyLength)"
if pos + msg.bodyLength - 1 > length(x.bytes)
- @debug 1 "not enough bytes left to read message body"
+ @debugv 1 "not enough bytes left to read message body"
return nothing
end
return Batch(msg, x.bytes, pos, id), (pos + msg.bodyLength, id + 1)
@@ -448,10 +448,10 @@ buildmetadata(x::AbstractDict) = x
function Base.iterate(x::VectorIterator, (columnidx, nodeidx,
bufferidx)=(Int64(1), Int64(1), Int64(1)))
columnidx > length(x.schema.fields) && return nothing
field = x.schema.fields[columnidx]
- @debug 2 "building top-level column: field = $(field), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
+ @debugv 2 "building top-level column: field = $(field), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
A, nodeidx, bufferidx = build(field, x.batch, x.batch.msg.header,
x.dictencodings, nodeidx, bufferidx, x.convert)
- @debug 2 "built top-level column: A = $(typeof(A)), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
- @debug 3 A
+ @debugv 2 "built top-level column: A = $(typeof(A)), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
+ @debugv 3 A
return A, (columnidx + 1, nodeidx, bufferidx)
end
@@ -521,7 +521,7 @@ function reinterp(::Type{T}, batch, buf, compression) where
{T}
end
function build(f::Meta.Field, L::ListTypes, batch, rb, de, nodeidx, bufferidx,
convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
@@ -546,7 +546,7 @@ function build(f::Meta.Field, L::ListTypes, batch, rb, de,
nodeidx, bufferidx, c
end
function build(f::Meta.Field, L::Union{Meta.FixedSizeBinary,
Meta.FixedSizeList}, batch, rb, de, nodeidx, bufferidx, convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
len = rb.nodes[nodeidx].length
@@ -565,7 +565,7 @@ function build(f::Meta.Field,
L::Union{Meta.FixedSizeBinary, Meta.FixedSizeList}
end
function build(f::Meta.Field, L::Meta.Map, batch, rb, de, nodeidx, bufferidx,
convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
@@ -583,7 +583,7 @@ function build(f::Meta.Field, L::Meta.Map, batch, rb, de,
nodeidx, bufferidx, co
end
function build(f::Meta.Field, L::Meta.Struct, batch, rb, de, nodeidx,
bufferidx, convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
len = rb.nodes[nodeidx].length
@@ -600,7 +600,7 @@ function build(f::Meta.Field, L::Meta.Struct, batch, rb,
de, nodeidx, bufferidx,
end
function build(f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx,
bufferidx, convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
buffer = rb.buffers[bufferidx]
bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression)
bufferidx += 1
@@ -628,7 +628,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de,
nodeidx, bufferidx,
end
function build(f::Meta.Field, L::Meta.Null, batch, rb, de, nodeidx, bufferidx,
convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
meta = buildmetadata(f.custom_metadata)
T = juliaeltype(f, meta, convert)
return
NullVector{maybemissing(T)}(MissingVector(rb.nodes[nodeidx].length), meta),
nodeidx + 1, bufferidx
@@ -636,30 +636,30 @@ end
# primitives
function build(f::Meta.Field, ::L, batch, rb, de, nodeidx, bufferidx, convert)
where {L}
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
meta = buildmetadata(f.custom_metadata)
# get storage type (non-converted)
T = juliaeltype(f, nothing, false)
- @debug 2 "storage type for primitive: T = $T"
+ @debugv 2 "storage type for primitive: T = $T"
bytes, A = reinterp(Base.nonmissingtype(T), batch, buffer, rb.compression)
len = rb.nodes[nodeidx].length
T = juliaeltype(f, meta, convert)
- @debug 2 "final julia type for primitive: T = $T"
+ @debugv 2 "final julia type for primitive: T = $T"
return Primitive(T, bytes, validity, A, len, meta), nodeidx + 1, bufferidx
+ 1
end
function build(f::Meta.Field, L::Meta.Bool, batch, rb, de, nodeidx, bufferidx,
convert)
- @debug 2 "building array: L = $L"
+ @debugv 2 "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
meta = buildmetadata(f.custom_metadata)
# get storage type (non-converted)
T = juliaeltype(f, nothing, false)
- @debug 2 "storage type for primitive: T = $T"
+ @debugv 2 "storage type for primitive: T = $T"
buffer = rb.buffers[bufferidx]
voff = batch.pos + buffer.offset
node = rb.nodes[nodeidx]
diff --git a/src/write.jl b/src/write.jl
index 9ee532b..5c965ca 100644
--- a/src/write.jl
+++ b/src/write.jl
@@ -184,11 +184,11 @@ end
function write(writer::Writer, source)
@sync for tbl in Tables.partitions(source)
check_errors(writer)
- @debug 1 "processing table partition $(writer.partition_count)"
+ @debugv 1 "processing table partition $(writer.partition_count)"
tblcols = Tables.columns(tbl)
if !isassigned(writer.firstcols)
if writer.writetofile
- @debug 1 "starting write of arrow formatted file"
+ @debugv 1 "starting write of arrow formatted file"
Base.write(writer.io, "ARROW1\0\0")
end
meta = isnothing(writer.meta) ? getmetadata(source) : writer.meta
@@ -324,7 +324,7 @@ struct ToArrowTable
end
function toarrowtable(cols, dictencodings, largelists, compress, denseunions,
dictencode, dictencodenested, maxdepth, meta, colmeta)
- @debug 1 "converting input table to arrow formatted columns"
+ @debugv 1 "converting input table to arrow formatted columns"
sch = Tables.schema(cols)
types = collect(sch.types)
N = length(types)
@@ -352,7 +352,7 @@ Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i]
function Base.write(io::IO, msg::Message, blocks, sch, alignment)
metalen = padding(length(msg.msgflatbuf), alignment)
- @debug 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen),
isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)"
+ @debugv 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen),
isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)"
if msg.blockmsg
push!(blocks[msg.isrecordbatch ? 1 : 2], Block(position(io), metalen +
8, msg.bodylen))
end
@@ -425,7 +425,7 @@ function makeschema(b, sch::Tables.Schema{names}, columns)
where {names}
end
function makeschemamsg(sch::Tables.Schema, columns)
- @debug 1 "building schema message: sch = $sch"
+ @debugv 1 "building schema message: sch = $sch"
b = FlatBuffers.Builder(1024)
schema = makeschema(b, sch, columns)
return makemessage(b, Meta.Schema, schema)
@@ -481,9 +481,9 @@ function fieldoffset(b, name, col)
end
# build field object
if isdictencoded(col)
- @debug 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type, inttype = $IT, dictionary id = $(getid(col))"
+ @debugv 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type, inttype = $IT, dictionary id = $(getid(col))"
else
- @debug 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type"
+ @debugv 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type"
end
Meta.fieldStart(b)
Meta.fieldAddName(b, nameoff)
@@ -525,7 +525,7 @@ function makerecordbatch(b,
sch::Tables.Schema{names,types}, columns, alignment)
end
bufferoffset = makenodesbuffers!(col, fieldnodes, fieldbuffers,
bufferoffset, alignment)
end
- @debug 1 "building record batch message: nrows = $nrows, sch = $sch,
compress = $compress"
+ @debugv 1 "building record batch message: nrows = $nrows, sch = $sch,
compress = $compress"
# write field nodes objects
FN = length(fieldnodes)
@@ -556,7 +556,7 @@ function makerecordbatch(b,
sch::Tables.Schema{names,types}, columns, alignment)
end
# write record batch object
- @debug 1 "built record batch message: nrows = $nrows, nodes = $fieldnodes,
buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
+ @debugv 1 "built record batch message: nrows = $nrows, nodes =
$fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
Meta.recordBatchStart(b)
Meta.recordBatchAddLength(b, Int64(nrows))
Meta.recordBatchAddNodes(b, nodes)
@@ -566,7 +566,7 @@ function makerecordbatch(b,
sch::Tables.Schema{names,types}, columns, alignment)
end
function makedictionarybatchmsg(sch, columns, id, isdelta, alignment)
- @debug 1 "building dictionary message: id = $id, sch = $sch, isdelta =
$isdelta"
+ @debugv 1 "building dictionary message: id = $id, sch = $sch, isdelta =
$isdelta"
b = FlatBuffers.Builder(1024)
recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
Meta.dictionaryBatchStart(b)
