This is an automated email from the ASF dual-hosted git repository.
quinnj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
The following commit(s) were added to refs/heads/main by this push:
new aa5ad74 Remove usage of LoggingExtras and verbosity logging macros
(#529)
aa5ad74 is described below
commit aa5ad746f417343da0f17e3cb896226251a5ec7a
Author: Jacob Quinn <[email protected]>
AuthorDate: Mon Oct 28 10:52:12 2024 -0600
Remove usage of LoggingExtras and verbosity logging macros (#529)
---
Project.toml | 2 --
src/Arrow.jl | 1 -
src/append.jl | 2 +-
src/arraytypes/arraytypes.jl | 12 ++++----
src/arraytypes/bool.jl | 10 +++----
src/arraytypes/compressed.jl | 12 ++++----
src/arraytypes/dictencoding.jl | 12 ++++----
src/arraytypes/fixedsizelist.jl | 12 ++++----
src/arraytypes/map.jl | 16 +++++------
src/arraytypes/primitive.jl | 12 ++++----
src/arraytypes/struct.jl | 8 +++---
src/arraytypes/unions.jl | 14 +++++-----
src/table.jl | 62 ++++++++++++++++++++---------------------
src/write.jl | 20 ++++++-------
14 files changed, 96 insertions(+), 99 deletions(-)
diff --git a/Project.toml b/Project.toml
index 5b24735..f2dd9f6 100644
--- a/Project.toml
+++ b/Project.toml
@@ -28,7 +28,6 @@ ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb"
DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
-LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
@@ -45,7 +44,6 @@ CodecZstd = "0.7, 0.8"
ConcurrentUtilities = "2"
DataAPI = "1"
EnumX = "1"
-LoggingExtras = "0.4, 1"
PooledArrays = "0.5, 1.0"
SentinelArrays = "1"
Tables = "1.1"
diff --git a/src/Arrow.jl b/src/Arrow.jl
index efef826..e5b5631 100644
--- a/src/Arrow.jl
+++ b/src/Arrow.jl
@@ -43,7 +43,6 @@ module Arrow
using Base.Iterators
using Mmap
-using LoggingExtras
import Dates
using DataAPI,
Tables,
diff --git a/src/append.jl b/src/append.jl
index 64a638b..1a5119e 100644
--- a/src/append.jl
+++ b/src/append.jl
@@ -182,7 +182,7 @@ function append(
(errorref[][1], errorref[][2])
error("fatal error writing arrow data")
end
- @debugv 1 "processing table partition i = $i"
+ @debug "processing table partition i = $i"
tbl_cols = Tables.columns(tbl)
tbl_schema = Tables.schema(tbl_cols)
diff --git a/src/arraytypes/arraytypes.jl b/src/arraytypes/arraytypes.jl
index b417abf..f60101b 100644
--- a/src/arraytypes/arraytypes.jl
+++ b/src/arraytypes/arraytypes.jl
@@ -42,8 +42,8 @@ function toarrowvector(
compression::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor}=nothing,
kw...,
)
- @debugv 2 "converting top-level column to arrow format: col =
$(typeof(x)), compression = $compression, kw = $(values(kw))"
- @debugv 3 x
+ @debug "converting top-level column to arrow format: col = $(typeof(x)),
compression = $compression, kw = $(values(kw))"
+ @debug x
A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...)
if compression isa LZ4FrameCompressor
A = compress(Meta.CompressionType.LZ4_FRAME, compression, A)
@@ -60,8 +60,8 @@ function toarrowvector(
compress(Meta.CompressionType.ZSTD, comp[], A)
end
end
- @debugv 2 "converted top-level column to arrow format: $(typeof(A))"
- @debugv 3 A
+ @debug "converted top-level column to arrow format: $(typeof(A))"
+ @debug A
return A
end
@@ -173,7 +173,7 @@ function makenodesbuffers!(
alignment,
)
push!(fieldnodes, FieldNode(length(col), length(col)))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
return bufferoffset
end
@@ -256,7 +256,7 @@ end
function writebitmap(io, col::ArrowVector, alignment)
v = col.validity
- @debugv 1 "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
+ @debug "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
v.nc == 0 && return 0
n = Base.write(io, view(v.bytes, (v.pos):(v.pos + cld(v.ℓ, 8) - 1)))
return n + writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/bool.jl b/src/arraytypes/bool.jl
index b9e853a..29c1505 100644
--- a/src/arraytypes/bool.jl
+++ b/src/arraytypes/bool.jl
@@ -95,22 +95,22 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + blen
end
function writebuffer(io, col::BoolVector, alignment)
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
n = Base.write(io, view(col.arrow, (col.pos):(col.pos + cld(col.ℓ, 8) -
1)))
return n + writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/compressed.jl b/src/arraytypes/compressed.jl
index 7c0366e..070ca33 100644
--- a/src/arraytypes/compressed.jl
+++ b/src/arraytypes/compressed.jl
@@ -60,11 +60,11 @@ function makenodesbuffers!(
alignment,
)
push!(fieldnodes, FieldNode(col.len, col.nullcount))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
for buffer in col.buffers
blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
end
for child in col.children
@@ -77,16 +77,16 @@ end
function writearray(io, b::CompressedBuffer)
if length(b.data) > 0
n = Base.write(io, b.uncompressedlength)
- @debugv 1 "writing compressed buffer: uncompressedlength =
$(b.uncompressedlength), n = $(length(b.data))"
- @debugv 2 b.data
+ @debug "writing compressed buffer: uncompressedlength =
$(b.uncompressedlength), n = $(length(b.data))"
+ @debug b.data
return n + Base.write(io, b.data)
end
return 0
end
function writebuffer(io, col::Compressed, alignment)
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
for buffer in col.buffers
n = writearray(io, buffer)
writezeros(io, paddinglength(n, alignment))
diff --git a/src/arraytypes/dictencoding.jl b/src/arraytypes/dictencoding.jl
index 9b7f7e0..fe48304 100644
--- a/src/arraytypes/dictencoding.jl
+++ b/src/arraytypes/dictencoding.jl
@@ -402,16 +402,16 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
# indices
blen = sizeof(S) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
return bufferoffset
end
@@ -421,12 +421,12 @@ DataAPI.refarray(x::DictEncoded{T,S}) where {T,S} =
x.indices .+ one(S)
DataAPI.refpool(x::DictEncoded) = copy(x.encoding.data)
function writebuffer(io, col::DictEncoded, alignment)
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
# write indices
n = writearray(io, col.indices)
- @debugv 1 "writing array: col = $(typeof(col.indices)), n = $n, padded =
$(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.indices)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
diff --git a/src/arraytypes/fixedsizelist.jl b/src/arraytypes/fixedsizelist.jl
index e613ec9..2558dd5 100644
--- a/src/arraytypes/fixedsizelist.jl
+++ b/src/arraytypes/fixedsizelist.jl
@@ -169,16 +169,16 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
if eltype(A) === UInt8
blen =
ArrowTypes.getsize(ArrowTypes.ArrowKind(Base.nonmissingtype(T))) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset =
@@ -188,13 +188,13 @@ function makenodesbuffers!(
end
function writebuffer(io, col::FixedSizeList{T,A}, alignment) where {T,A}
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
# write values array
if eltype(A) === UInt8
n = writearray(io, UInt8, col.data)
- @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
diff --git a/src/arraytypes/map.jl b/src/arraytypes/map.jl
index ebd4454..4216073 100644
--- a/src/arraytypes/map.jl
+++ b/src/arraytypes/map.jl
@@ -103,21 +103,21 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make array buffer
bufferoffset += blen
blen = sizeof(O) * (len + 1)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
if liststringtype(col)
blen = length(col.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset =
@@ -127,17 +127,17 @@ function makenodesbuffers!(
end
function writebuffer(io, col::Union{Map{T,O,A},List{T,O,A}}, alignment) where
{T,O,A}
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
# write offsets
n = writearray(io, O, col.offsets.offsets)
- @debugv 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n,
padded = $(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.offsets.offsets)), n = $n,
padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
# write values array
if liststringtype(col)
n = writearray(io, UInt8, col.data)
- @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
diff --git a/src/arraytypes/primitive.jl b/src/arraytypes/primitive.jl
index c187c77..7d86bfe 100644
--- a/src/arraytypes/primitive.jl
+++ b/src/arraytypes/primitive.jl
@@ -88,25 +88,25 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = len * sizeof(Base.nonmissingtype(T))
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + padding(blen, alignment)
end
function writebuffer(io, col::Primitive{T}, alignment) where {T}
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
n = writearray(io, Base.nonmissingtype(T), col.data)
- @debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.data)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
diff --git a/src/arraytypes/struct.jl b/src/arraytypes/struct.jl
index 510d1e4..23a8b64 100644
--- a/src/arraytypes/struct.jl
+++ b/src/arraytypes/struct.jl
@@ -137,11 +137,11 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
for child in col.data
bufferoffset =
@@ -151,8 +151,8 @@ function makenodesbuffers!(
end
function writebuffer(io, col::Struct, alignment)
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
writebitmap(io, col, alignment)
# write values arrays
for child in col.data
diff --git a/src/arraytypes/unions.jl b/src/arraytypes/unions.jl
index a688055..ab4673d 100644
--- a/src/arraytypes/unions.jl
+++ b/src/arraytypes/unions.jl
@@ -307,16 +307,16 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
- @debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
+ @debug "made field node: nodeidx = $(length(fieldnodes)), col =
$(typeof(col)), len = $(fieldnodes[end].length), nc =
$(fieldnodes[end].null_count)"
# typeIds buffer
push!(fieldbuffers, Buffer(bufferoffset, len))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset =
$(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(len, alignment)
if col isa DenseUnion
# offsets buffer
blen = sizeof(Int32) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
- @debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)),
offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded
= $(padding(fieldbuffers[end].length, alignment))"
+ @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset
= $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded =
$(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
end
for child in col.data
@@ -327,15 +327,15 @@ function makenodesbuffers!(
end
function writebuffer(io, col::Union{DenseUnion,SparseUnion}, alignment)
- @debugv 1 "writebuffer: col = $(typeof(col))"
- @debugv 2 col
+ @debug "writebuffer: col = $(typeof(col))"
+ @debug col
# typeIds buffer
n = writearray(io, UInt8, col.typeIds)
- @debugv 1 "writing array: col = $(typeof(col.typeIds)), n = $n, padded =
$(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.typeIds)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
if col isa DenseUnion
n = writearray(io, Int32, col.offsets)
- @debugv 1 "writing array: col = $(typeof(col.offsets)), n = $n, padded
= $(padding(n, alignment))"
+ @debug "writing array: col = $(typeof(col.offsets)), n = $n, padded =
$(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
end
for child in col.data
diff --git a/src/table.jl b/src/table.jl
index ecd8b1d..6dfb9b4 100644
--- a/src/table.jl
+++ b/src/table.jl
@@ -194,7 +194,7 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
)
# recursively find any dictionaries for any fields
getdictionaries!(x.dictencoded, field)
- @debugv 1 "parsed column from schema: field = $field"
+ @debug "parsed column from schema: field = $field"
end
elseif header != x.schema
throw(
@@ -206,7 +206,7 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
elseif header isa Meta.DictionaryBatch
id = header.id
recordbatch = header.data
- @debugv 1 "parsing dictionary batch message: id = $id, compression
= $(recordbatch.compression)"
+ @debug "parsing dictionary batch message: id = $id, compression =
$(recordbatch.compression)"
if recordbatch.compression !== nothing
compression = recordbatch.compression
end
@@ -249,9 +249,9 @@ function Base.iterate(x::Stream, (pos, id)=(1, 0))
field.dictionary.isOrdered,
values.metadata,
)
- @debugv 1 "parsed dictionary batch message: id=$id, data=$values\n"
+ @debug "parsed dictionary batch message: id=$id, data=$values\n"
elseif header isa Meta.RecordBatch
- @debugv 1 "parsing record batch message: compression =
$(header.compression)"
+ @debug "parsing record batch message: compression =
$(header.compression)"
if header.compression !== nothing
compression = header.compression
end
@@ -443,7 +443,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
# store custom_metadata of batch.msg?
header = batch.msg.header
if header isa Meta.Schema
- @debugv 1 "parsing schema message"
+ @debug "parsing schema message"
# assert endianness?
# store custom_metadata?
if sch === nothing
@@ -451,7 +451,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
push!(names(t), Symbol(field.name))
# recursively find any dictionaries for any fields
getdictionaries!(dictencoded, field)
- @debugv 1 "parsed column from schema: field = $field"
+ @debug "parsed column from schema: field = $field"
end
sch = header
schema(t)[] = sch
@@ -465,7 +465,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
elseif header isa Meta.DictionaryBatch
id = header.id
recordbatch = header.data
- @debugv 1 "parsing dictionary batch message: id = $id,
compression = $(recordbatch.compression)"
+ @debug "parsing dictionary batch message: id = $id,
compression = $(recordbatch.compression)"
if haskey(dictencodings, id) && header.isDelta
# delta
field = dictencoded[id]
@@ -518,10 +518,10 @@ function Table(blobs::Vector{ArrowBlob};
convert::Bool=true)
field.dictionary.isOrdered,
values.metadata,
)
- @debugv 1 "parsed dictionary batch message: id=$id,
data=$values\n"
+ @debug "parsed dictionary batch message: id=$id,
data=$values\n"
elseif header isa Meta.RecordBatch
anyrecordbatches = true
- @debugv 1 "parsing record batch message: compression =
$(header.compression)"
+ @debug "parsing record batch message: compression =
$(header.compression)"
@wkspawn begin
cols = collect(VectorIterator(sch, $batch, dictencodings,
convert))
put!(() -> put!(tsks, cols), sync, $(rbi))
@@ -572,36 +572,36 @@ struct Batch
end
function Base.iterate(x::BatchIterator, (pos, id)=(x.startpos, 0))
- @debugv 1 "checking for next arrow message: pos = $pos"
+ @debug "checking for next arrow message: pos = $pos"
if pos + 3 > length(x.bytes)
- @debugv 1 "not enough bytes left for another batch message"
+ @debug "not enough bytes left for another batch message"
return nothing
end
if readbuffer(x.bytes, pos, UInt32) != CONTINUATION_INDICATOR_BYTES
- @debugv 1 "didn't find continuation byte to keep parsing messages:
$(readbuffer(x.bytes, pos, UInt32))"
+ @debug "didn't find continuation byte to keep parsing messages:
$(readbuffer(x.bytes, pos, UInt32))"
return nothing
end
pos += 4
if pos + 3 > length(x.bytes)
- @debugv 1 "not enough bytes left to read length of another batch
message"
+ @debug "not enough bytes left to read length of another batch message"
return nothing
end
msglen = readbuffer(x.bytes, pos, Int32)
if msglen == 0
- @debugv 1 "message has 0 length; terminating message parsing"
+ @debug "message has 0 length; terminating message parsing"
return nothing
end
pos += 4
if pos + msglen - 1 > length(x.bytes)
- @debugv 1 "not enough bytes left to read Meta.Message"
+ @debug "not enough bytes left to read Meta.Message"
return nothing
end
msg = FlatBuffers.getrootas(Meta.Message, x.bytes, pos - 1)
pos += msglen
# pos now points to message body
- @debugv 1 "parsing message: pos = $pos, msglen = $msglen, bodyLength =
$(msg.bodyLength)"
+ @debug "parsing message: pos = $pos, msglen = $msglen, bodyLength =
$(msg.bodyLength)"
if pos + msg.bodyLength - 1 > length(x.bytes)
- @debugv 1 "not enough bytes left to read message body"
+ @debug "not enough bytes left to read message body"
return nothing
end
return Batch(msg, x.bytes, pos, id), (pos + msg.bodyLength, id + 1)
@@ -625,7 +625,7 @@ function Base.iterate(
)
columnidx > length(x.schema.fields) && return nothing
field = x.schema.fields[columnidx]
- @debugv 2 "building top-level column: field = $(field), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
+ @debug "building top-level column: field = $(field), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
A, nodeidx, bufferidx = build(
field,
x.batch,
@@ -635,8 +635,8 @@ function Base.iterate(
bufferidx,
x.convert,
)
- @debugv 2 "built top-level column: A = $(typeof(A)), columnidx =
$columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx"
- @debugv 3 A
+ @debug "built top-level column: A = $(typeof(A)), columnidx = $columnidx,
nodeidx = $nodeidx, bufferidx = $bufferidx"
+ @debug A
return A, (columnidx + 1, nodeidx, bufferidx)
end
@@ -745,7 +745,7 @@ end
const SubVector{T,P} = SubArray{T,1,P,Tuple{UnitRange{Int64}},true}
function build(f::Meta.Field, L::ListTypes, batch, rb, de, nodeidx, bufferidx,
convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
@@ -789,7 +789,7 @@ function build(
bufferidx,
convert,
)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
len = rb.nodes[nodeidx].length
@@ -809,7 +809,7 @@ function build(
end
function build(f::Meta.Field, L::Meta.Map, batch, rb, de, nodeidx, bufferidx,
convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
@@ -827,7 +827,7 @@ function build(f::Meta.Field, L::Meta.Map, batch, rb, de,
nodeidx, bufferidx, co
end
function build(f::Meta.Field, L::Meta.Struct, batch, rb, de, nodeidx,
bufferidx, convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
len = rb.nodes[nodeidx].length
@@ -845,7 +845,7 @@ function build(f::Meta.Field, L::Meta.Struct, batch, rb,
de, nodeidx, bufferidx,
end
function build(f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx,
bufferidx, convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
buffer = rb.buffers[bufferidx]
bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression)
bufferidx += 1
@@ -873,7 +873,7 @@ function build(f::Meta.Field, L::Meta.Union, batch, rb, de,
nodeidx, bufferidx,
end
function build(f::Meta.Field, L::Meta.Null, batch, rb, de, nodeidx, bufferidx,
convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
meta = buildmetadata(f.custom_metadata)
T = juliaeltype(f, meta, convert)
return
NullVector{maybemissing(T)}(MissingVector(rb.nodes[nodeidx].length), meta),
@@ -883,30 +883,30 @@ end
# primitives
function build(f::Meta.Field, ::L, batch, rb, de, nodeidx, bufferidx, convert)
where {L}
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
meta = buildmetadata(f.custom_metadata)
# get storage type (non-converted)
T = juliaeltype(f, nothing, false)
- @debugv 2 "storage type for primitive: T = $T"
+ @debug "storage type for primitive: T = $T"
bytes, A = reinterp(Base.nonmissingtype(T), batch, buffer, rb.compression)
len = rb.nodes[nodeidx].length
T = juliaeltype(f, meta, convert)
- @debugv 2 "final julia type for primitive: T = $T"
+ @debug "final julia type for primitive: T = $T"
return Primitive(T, bytes, validity, A, len, meta), nodeidx + 1, bufferidx
+ 1
end
function build(f::Meta.Field, L::Meta.Bool, batch, rb, de, nodeidx, bufferidx,
convert)
- @debugv 2 "building array: L = $L"
+ @debug "building array: L = $L"
validity = buildbitmap(batch, rb, nodeidx, bufferidx)
bufferidx += 1
buffer = rb.buffers[bufferidx]
meta = buildmetadata(f.custom_metadata)
# get storage type (non-converted)
T = juliaeltype(f, nothing, false)
- @debugv 2 "storage type for primitive: T = $T"
+ @debug "storage type for primitive: T = $T"
buffer = rb.buffers[bufferidx]
voff = batch.pos + buffer.offset
node = rb.nodes[nodeidx]
diff --git a/src/write.jl b/src/write.jl
index 18bdc4e..1f1bfd1 100644
--- a/src/write.jl
+++ b/src/write.jl
@@ -253,11 +253,11 @@ end
function write(writer::Writer, source)
@sync for tbl in Tables.partitions(source)
check_errors(writer)
- @debugv 1 "processing table partition $(writer.partition_count)"
+ @debug "processing table partition $(writer.partition_count)"
tblcols = Tables.columns(tbl)
if !isassigned(writer.firstcols)
if writer.writetofile
- @debugv 1 "starting write of arrow formatted file"
+ @debug "starting write of arrow formatted file"
Base.write(writer.io, FILE_FORMAT_MAGIC_BYTES, b"\0\0")
end
meta = isnothing(writer.meta) ? getmetadata(source) : writer.meta
@@ -517,7 +517,7 @@ function toarrowtable(
meta,
colmeta,
)
- @debugv 1 "converting input table to arrow formatted columns"
+ @debug "converting input table to arrow formatted columns"
sch = Tables.schema(cols)
types = collect(sch.types)
N = length(types)
@@ -563,7 +563,7 @@ Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i]
function Base.write(io::IO, msg::Message, blocks, sch, alignment)
metalen = padding(length(msg.msgflatbuf), alignment)
- @debugv 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen),
isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)"
+ @debug "writing message: metalen = $metalen, bodylen = $(msg.bodylen),
isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)"
if msg.blockmsg
push!(
blocks[msg.isrecordbatch ? 1 : 2],
@@ -647,7 +647,7 @@ function makeschema(b, sch::Tables.Schema, columns)
end
function makeschemamsg(sch::Tables.Schema, columns)
- @debugv 1 "building schema message: sch = $sch"
+ @debug "building schema message: sch = $sch"
b = FlatBuffers.Builder(1024)
schema = makeschema(b, sch, columns)
return makemessage(b, Meta.Schema, schema)
@@ -703,9 +703,9 @@ function fieldoffset(b, name, col)
end
# build field object
if isdictencoded(col)
- @debugv 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type, inttype = $IT, dictionary id = $(getid(col))"
+ @debug "building field: name = $name, nullable = $nullable, T = $T,
type = $type, inttype = $IT, dictionary id = $(getid(col))"
else
- @debugv 1 "building field: name = $name, nullable = $nullable, T = $T,
type = $type"
+ @debug "building field: name = $name, nullable = $nullable, T = $T,
type = $type"
end
Meta.fieldStart(b)
Meta.fieldAddName(b, nameoff)
@@ -757,7 +757,7 @@ function makerecordbatch(
bufferoffset =
makenodesbuffers!(col, fieldnodes, fieldbuffers, bufferoffset,
alignment)
end
- @debugv 1 "building record batch message: nrows = $nrows, sch = $sch,
compress = $compress"
+ @debug "building record batch message: nrows = $nrows, sch = $sch,
compress = $compress"
# write field nodes objects
FN = length(fieldnodes)
@@ -788,7 +788,7 @@ function makerecordbatch(
end
# write record batch object
- @debugv 1 "built record batch message: nrows = $nrows, nodes =
$fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
+ @debug "built record batch message: nrows = $nrows, nodes = $fieldnodes,
buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
Meta.recordBatchStart(b)
Meta.recordBatchAddLength(b, Int64(nrows))
Meta.recordBatchAddNodes(b, nodes)
@@ -798,7 +798,7 @@ function makerecordbatch(
end
function makedictionarybatchmsg(sch, columns, id, isdelta, alignment)
- @debugv 1 "building dictionary message: id = $id, sch = $sch, isdelta =
$isdelta"
+ @debug "building dictionary message: id = $id, sch = $sch, isdelta =
$isdelta"
b = FlatBuffers.Builder(1024)
recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment)
Meta.dictionaryBatchStart(b)
