This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new dd69699564 GH-50390: [Ruby] Add int/float array builder for
red-arrow-format (#50391)
dd69699564 is described below
commit dd6969956484a68c4080252d9cb8af99511dc8b2
Author: Sutou Kouhei <[email protected]>
AuthorDate: Tue Jul 7 09:22:24 2026 +0900
GH-50390: [Ruby] Add int/float array builder for red-arrow-format (#50391)
### Rationale for this change
Int array and float array use similar structure. We can share common code
for builder API for them.
### What changes are included in this PR?
Add `ArrowFormat::PrimitiveArray#build_data`.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
Yes.
* GitHub Issue: #50390
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ruby/red-arrow-format/lib/arrow-format/array.rb | 67 ++++---
.../lib/arrow-format/bitmap-builder.rb | 5 +-
.../lib/arrow-format/integration/json-reader.rb | 33 +++-
ruby/red-arrow-format/lib/arrow-format/type.rb | 200 ++++++++++++---------
ruby/red-arrow-format/test/test-boolean-array.rb | 34 ----
ruby/red-arrow-format/test/test-primitive-array.rb | 94 ++++++++++
6 files changed, 282 insertions(+), 151 deletions(-)
diff --git a/ruby/red-arrow-format/lib/arrow-format/array.rb
b/ruby/red-arrow-format/lib/arrow-format/array.rb
index 5addc730e5..37ed813590 100644
--- a/ruby/red-arrow-format/lib/arrow-format/array.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/array.rb
@@ -21,6 +21,8 @@ require_relative "bitmap"
require_relative "bitmap-builder"
module ArrowFormat
+ using FlatBuffers::AppendAsBytes if
FlatBuffers.const_defined?(:AppendAsBytes)
+
class Array
attr_reader :type
attr_reader :size
@@ -160,7 +162,25 @@ module ArrowFormat
end
class PrimitiveArray < Array
- def initialize(type, size, validity_buffer, values_buffer)
+ include BufferAlignable
+
+ def initialize(*args)
+ n_args = args.size
+ if self.class.respond_to?(:type)
+ type = self.class.type
+ expected_n_args = "1 or 3"
+ else
+ type = args.shift
+ expected_n_args = "2 or 4"
+ end
+ args = build_data(args[0], type) if args.size == 1
+ if args.size != 3
+ message =
+ "wrong number of arguments " +
+ "(given #{n_args}, expected #{expected_n_args})"
+ raise ArgumentError, message
+ end
+ size, validity_buffer, values_buffer = args
super(type, size, validity_buffer)
@values_buffer = values_buffer
end
@@ -185,20 +205,34 @@ module ArrowFormat
def element_size
IO::Buffer.size_of(@type.buffer_type)
end
+
+ def build_data(data, type)
+ n = 0
+ validity_buffer_builder = nil
+ buffer = +"".b
+ pack_template = type.pack_template
+ data.each_with_index do |value, i|
+ if value.nil?
+ validity_buffer_builder ||= SparseBitmapBuilder.new
+ validity_buffer_builder.unset(i)
+ buffer.append_as_bytes([0].pack(pack_template))
+ else
+ buffer.append_as_bytes([value].pack(pack_template))
+ end
+ n += 1
+ end
+ validity_buffer = validity_buffer_builder&.finish(n)
+ pad!(buffer, buffer_padding_size(buffer))
+ buffer.freeze
+ return n, validity_buffer, IO::Buffer.for(buffer)
+ end
end
class BooleanArray < PrimitiveArray
- def initialize(*args)
- if args.size == 1
- args = build_data(args[0])
- end
- n_args = args.size
- if args.size != 3
- message = "wrong number of arguments (given #{n_args}, expected 1 or
3)"
- raise ArgumentError, message
+ class << self
+ def type
+ BooleanType.singleton
end
- size, validity_buffer, values_buffer = args
- super(BooleanType.singleton, size, validity_buffer, values_buffer)
end
def to_a
@@ -222,7 +256,7 @@ module ArrowFormat
@values_bitmap = nil
end
- def build_data(data)
+ def build_data(data, type)
n = 0
validity_buffer_builder = nil
values_buffer_builder = DenseBitmapBuilder.new
@@ -244,9 +278,6 @@ module ArrowFormat
end
class IntArray < PrimitiveArray
- def initialize(size, validity_buffer, values_buffer)
- super(self.class.type, size, validity_buffer, values_buffer)
- end
end
class Int8Array < IntArray
@@ -314,9 +345,6 @@ module ArrowFormat
end
class FloatingPointArray < PrimitiveArray
- def initialize(size, validity_buffer, values_buffer)
- super(self.class.type, size, validity_buffer, values_buffer)
- end
end
class Float32Array < FloatingPointArray
@@ -339,9 +367,6 @@ module ArrowFormat
end
class DateArray < TemporalArray
- def initialize(size, validity_buffer, values_buffer)
- super(self.class.type, size, validity_buffer, values_buffer)
- end
end
class Date32Array < DateArray
diff --git a/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb
b/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb
index 71e82e15d9..d550d058ac 100644
--- a/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb
@@ -18,6 +18,8 @@
require_relative "buffer-alignable"
module ArrowFormat
+ using FlatBuffers::AppendAsBytes if
FlatBuffers.const_defined?(:AppendAsBytes)
+
class DenseBitmapBuilder
include BufferAlignable
@@ -36,8 +38,7 @@ module ArrowFormat
def finish
flush if @n_bits > 0
- padding_size = buffer_padding_size(@buffer)
- @buffer.append_as_bytes(padding(padding_size)) if padding_size > 0
+ pad!(@buffer, buffer_padding_size(@buffer))
@buffer.freeze
IO::Buffer.for(@buffer)
end
diff --git a/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb
b/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb
index b8d84a5a8e..ddc8971c7e 100644
--- a/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb
@@ -207,7 +207,7 @@ module ArrowFormat
def read_bitmap(bitmap)
builder = DenseBitmapBuilder.new
bitmap.each do |bit|
- builder.append(bit == 1 ? true : false)
+ builder.append(bit == 1)
end
builder.finish
end
@@ -248,8 +248,6 @@ module ArrowFormat
def read_values(data, type)
case type
- when BooleanType
- read_bitmap(data.collect {|boolean| boolean ? 1 : 0})
when DayTimeIntervalType
buffer_types = [type.buffer_type] * 2
size = IO::Buffer.size_of(buffer_types)
@@ -314,14 +312,37 @@ module ArrowFormat
end
end
+ def apply_validity(values, validity)
+ values.zip(validity).collect do |value, valid|
+ if valid == 1
+ value
+ else
+ nil
+ end
+ end
+ end
+
def read_array(column, type)
length = column["count"]
case type
when NullType
type.build_array(length)
- when BooleanType,
- NumberType,
- TemporalType,
+ when BooleanType
+ values = apply_validity(column["DATA"], column["VALIDITY"])
+ type.build_array(values)
+ when NumberType
+ values = column["DATA"].collect do |value|
+ if value.is_a?(String)
+ # If the type is 64bit such as `Int64Type`, `value` is a
+ # string not integer to round-trip data through JSON.
+ Integer(value, 10)
+ else
+ value
+ end
+ end
+ values = apply_validity(values, column["VALIDITY"])
+ type.build_array(values)
+ when TemporalType,
FixedSizeBinaryType
validity_buffer = read_bitmap(column["VALIDITY"])
values_buffer = read_values(column["DATA"], type)
diff --git a/ruby/red-arrow-format/lib/arrow-format/type.rb
b/ruby/red-arrow-format/lib/arrow-format/type.rb
index 38523cf00b..affd9db024 100644
--- a/ruby/red-arrow-format/lib/arrow-format/type.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/type.rb
@@ -32,8 +32,8 @@ module ArrowFormat
"Null"
end
- def build_array(size)
- NullArray.new(size)
+ def build_array(...)
+ NullArray.new(...)
end
def to_flatbuffers
@@ -55,8 +55,8 @@ module ArrowFormat
"Boolean"
end
- def build_array(size, validity_buffer, values_buffer)
- BooleanArray.new(size, validity_buffer, values_buffer)
+ def build_array(...)
+ BooleanArray.new(...)
end
def to_flatbuffers
@@ -106,8 +106,12 @@ module ArrowFormat
:S8
end
- def build_array(size, validity_buffer, values_buffer)
- Int8Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "c"
+ end
+
+ def build_array(...)
+ Int8Array.new(...)
end
end
@@ -130,8 +134,12 @@ module ArrowFormat
:U8
end
- def build_array(size, validity_buffer, values_buffer)
- UInt8Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "C"
+ end
+
+ def build_array(...)
+ UInt8Array.new(...)
end
end
@@ -154,8 +162,12 @@ module ArrowFormat
:s16
end
- def build_array(size, validity_buffer, values_buffer)
- Int16Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "s"
+ end
+
+ def build_array(...)
+ Int16Array.new(...)
end
end
@@ -178,8 +190,12 @@ module ArrowFormat
:u16
end
- def build_array(size, validity_buffer, values_buffer)
- UInt16Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "S"
+ end
+
+ def build_array(...)
+ UInt16Array.new(...)
end
end
@@ -202,8 +218,12 @@ module ArrowFormat
:s32
end
- def build_array(size, validity_buffer, values_buffer)
- Int32Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "l"
+ end
+
+ def build_array(...)
+ Int32Array.new(...)
end
end
@@ -226,8 +246,12 @@ module ArrowFormat
:u32
end
- def build_array(size, validity_buffer, values_buffer)
- UInt32Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "L"
+ end
+
+ def build_array(...)
+ UInt32Array.new(...)
end
end
@@ -250,8 +274,12 @@ module ArrowFormat
:s64
end
- def build_array(size, validity_buffer, values_buffer)
- Int64Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "q"
+ end
+
+ def build_array(...)
+ Int64Array.new(...)
end
end
@@ -274,8 +302,12 @@ module ArrowFormat
:u64
end
- def build_array(size, validity_buffer, values_buffer)
- UInt64Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "Q"
+ end
+
+ def build_array(...)
+ UInt64Array.new(...)
end
end
@@ -312,8 +344,12 @@ module ArrowFormat
:f32
end
- def build_array(size, validity_buffer, values_buffer)
- Float32Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "f"
+ end
+
+ def build_array(...)
+ Float32Array.new(...)
end
end
@@ -336,8 +372,12 @@ module ArrowFormat
:f64
end
- def build_array(size, validity_buffer, values_buffer)
- Float64Array.new(size, validity_buffer, values_buffer)
+ def pack_template
+ "d"
+ end
+
+ def build_array(...)
+ Float64Array.new(...)
end
end
@@ -377,8 +417,8 @@ module ArrowFormat
:s32
end
- def build_array(size, validity_buffer, values_buffer)
- Date32Array.new(size, validity_buffer, values_buffer)
+ def build_array(...)
+ Date32Array.new(...)
end
end
@@ -401,8 +441,8 @@ module ArrowFormat
:s64
end
- def build_array(size, validity_buffer, values_buffer)
- Date64Array.new(size, validity_buffer, values_buffer)
+ def build_array(...)
+ Date64Array.new(...)
end
end
@@ -440,8 +480,8 @@ module ArrowFormat
:s32
end
- def build_array(size, validity_buffer, values_buffer)
- Time32Array.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ Time32Array.new(self, ...)
end
end
@@ -458,8 +498,8 @@ module ArrowFormat
:s64
end
- def build_array(size, validity_buffer, values_buffer)
- Time64Array.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ Time64Array.new(self, ...)
end
end
@@ -480,8 +520,8 @@ module ArrowFormat
:s64
end
- def build_array(size, validity_buffer, values_buffer)
- TimestampArray.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ TimestampArray.new(self, ...)
end
def to_s
@@ -531,8 +571,8 @@ module ArrowFormat
:s32
end
- def build_array(size, validity_buffer, values_buffer)
- YearMonthIntervalArray.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ YearMonthIntervalArray.new(self, ...)
end
end
@@ -549,8 +589,8 @@ module ArrowFormat
:s32
end
- def build_array(size, validity_buffer, values_buffer)
- DayTimeIntervalArray.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ DayTimeIntervalArray.new(self, ...)
end
end
@@ -567,11 +607,8 @@ module ArrowFormat
@buffer_types ||= [:s32, :s32, :s64]
end
- def build_array(size, validity_buffer, values_buffer)
- MonthDayNanoIntervalArray.new(self,
- size,
- validity_buffer,
- values_buffer)
+ def build_array(...)
+ MonthDayNanoIntervalArray.new(self, ...)
end
end
@@ -590,8 +627,8 @@ module ArrowFormat
:s64
end
- def build_array(size, validity_buffer, values_buffer)
- DurationArray.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ DurationArray.new(self, ...)
end
def to_s
@@ -627,11 +664,8 @@ module ArrowFormat
Encoding::ASCII_8BIT
end
- def build_array(size, validity_buffer, offsets_buffer, values_buffer)
- BinaryArray.new(size,
- validity_buffer,
- offsets_buffer,
- values_buffer)
+ def build_array(...)
+ BinaryArray.new(...)
end
def to_flatbuffers
@@ -658,11 +692,8 @@ module ArrowFormat
Encoding::ASCII_8BIT
end
- def build_array(size, validity_buffer, offsets_buffer, values_buffer)
- LargeBinaryArray.new(size,
- validity_buffer,
- offsets_buffer,
- values_buffer)
+ def build_array(...)
+ LargeBinaryArray.new(...)
end
def to_flatbuffers
@@ -689,8 +720,8 @@ module ArrowFormat
Encoding::UTF_8
end
- def build_array(size, validity_buffer, offsets_buffer, values_buffer)
- UTF8Array.new(size, validity_buffer, offsets_buffer, values_buffer)
+ def build_array(...)
+ UTF8Array.new(...)
end
def to_flatbuffers
@@ -717,11 +748,8 @@ module ArrowFormat
Encoding::UTF_8
end
- def build_array(size, validity_buffer, offsets_buffer, values_buffer)
- LargeUTF8Array.new(size,
- validity_buffer,
- offsets_buffer,
- values_buffer)
+ def build_array(...)
+ LargeUTF8Array.new(...)
end
def to_flatbuffers
@@ -740,8 +768,8 @@ module ArrowFormat
"FixedSizeBinary"
end
- def build_array(size, validity_buffer, values_buffer)
- FixedSizeBinaryArray.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ FixedSizeBinaryArray.new(self, ...)
end
def to_s
@@ -786,8 +814,8 @@ module ArrowFormat
"Decimal128"
end
- def build_array(size, validity_buffer, values_buffer)
- Decimal128Array.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ Decimal128Array.new(self, ...)
end
end
@@ -800,8 +828,8 @@ module ArrowFormat
"Decimal256"
end
- def build_array(size, validity_buffer, values_buffer)
- Decimal256Array.new(self, size, validity_buffer, values_buffer)
+ def build_array(...)
+ Decimal256Array.new(self, ...)
end
end
@@ -826,8 +854,8 @@ module ArrowFormat
:s32 # TODO: big endian support
end
- def build_array(size, validity_buffer, offsets_buffer, child)
- ListArray.new(self, size, validity_buffer, offsets_buffer, child)
+ def build_array(...)
+ ListArray.new(self, ...)
end
def to_flatbuffers
@@ -844,8 +872,8 @@ module ArrowFormat
:s64 # TODO: big endian support
end
- def build_array(size, validity_buffer, offsets_buffer, child)
- LargeListArray.new(self, size, validity_buffer, offsets_buffer, child)
+ def build_array(...)
+ LargeListArray.new(self, ...)
end
def to_flatbuffers
@@ -870,8 +898,8 @@ module ArrowFormat
"#{super}<#{child.name}: #{child.type}>(#{@size})"
end
- def build_array(size, validity_buffer, child)
- FixedSizeListArray.new(self, size, validity_buffer, child)
+ def build_array(...)
+ FixedSizeListArray.new(self, ...)
end
def to_flatbuffers
@@ -892,8 +920,8 @@ module ArrowFormat
"Struct"
end
- def build_array(size, validity_buffer, children)
- StructArray.new(self, size, validity_buffer, children)
+ def build_array(...)
+ StructArray.new(self, ...)
end
def to_s
@@ -942,8 +970,8 @@ module ArrowFormat
:s32 # TODO: big endian support
end
- def build_array(size, validity_buffer, offsets_buffer, child)
- MapArray.new(self, size, validity_buffer, offsets_buffer, child)
+ def build_array(...)
+ MapArray.new(self, ...)
end
def to_s
@@ -999,8 +1027,8 @@ module ArrowFormat
:s32
end
- def build_array(size, types_buffer, offsets_buffer, children)
- DenseUnionArray.new(self, size, types_buffer, offsets_buffer, children)
+ def build_array(...)
+ DenseUnionArray.new(self, ...)
end
end
@@ -1013,8 +1041,8 @@ module ArrowFormat
"SparseUnion"
end
- def build_array(size, types_buffer, children)
- SparseUnionArray.new(self, size, types_buffer, children)
+ def build_array(...)
+ SparseUnionArray.new(self, ...)
end
end
@@ -1038,12 +1066,8 @@ module ArrowFormat
"Dictionary"
end
- def build_array(size, validity_buffer, indices_buffer, dictionaries)
- DictionaryArray.new(self,
- size,
- validity_buffer,
- indices_buffer,
- dictionaries)
+ def build_array(...)
+ DictionaryArray.new(self, ...)
end
def build_fb_field(fb_field)
diff --git a/ruby/red-arrow-format/test/test-boolean-array.rb
b/ruby/red-arrow-format/test/test-boolean-array.rb
deleted file mode 100644
index 1d80966ddc..0000000000
--- a/ruby/red-arrow-format/test/test-boolean-array.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestBooleanArray < Test::Unit::TestCase
- def test_mixed
- assert_equal([true, nil, false],
- ArrowFormat::BooleanArray.new([true, nil, false]).to_a)
- end
-
- def test_no_null
- assert_equal([true, false],
- ArrowFormat::BooleanArray.new([true, false]).to_a)
- end
-
- def test_more_8bits
- values = [true] * 8 + [nil, false]
- assert_equal(values,
- ArrowFormat::BooleanArray.new(values).to_a)
- end
-end
diff --git a/ruby/red-arrow-format/test/test-primitive-array.rb
b/ruby/red-arrow-format/test/test-primitive-array.rb
new file mode 100644
index 0000000000..7e09193ebf
--- /dev/null
+++ b/ruby/red-arrow-format/test/test-primitive-array.rb
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestPrimitiveArray < Test::Unit::TestCase
+ def test_no_null
+ assert_equal([true, false],
+ ArrowFormat::BooleanArray.new([true, false]).to_a)
+ end
+
+ def test_null_multiple_bytes
+ values = [true] * 8 + [nil, false]
+ assert_equal(values,
+ ArrowFormat::BooleanArray.new(values).to_a)
+ end
+
+ def test_boolean
+ assert_equal([true, nil, false],
+ ArrowFormat::BooleanArray.new([true, nil, false]).to_a)
+ end
+
+ def test_int8
+ values = [-(2 ** 7), nil, (2 ** 7) - 1]
+ assert_equal(values,
+ ArrowFormat::Int8Array.new(values).to_a)
+ end
+
+ def test_uint8
+ values = [0, nil, (2 ** 8) - 1]
+ assert_equal(values,
+ ArrowFormat::UInt8Array.new(values).to_a)
+ end
+
+ def test_int16
+ values = [-(2 ** 15), nil, (2 ** 15) - 1]
+ assert_equal(values,
+ ArrowFormat::Int16Array.new(values).to_a)
+ end
+
+ def test_uint16
+ values = [0, nil, (2 ** 16) - 1]
+ assert_equal(values,
+ ArrowFormat::UInt16Array.new(values).to_a)
+ end
+
+ def test_int32
+ values = [-(2 ** 31), nil, (2 ** 31) - 1]
+ assert_equal(values,
+ ArrowFormat::Int32Array.new(values).to_a)
+ end
+
+ def test_uint32
+ values = [0, nil, (2 ** 32) - 1]
+ assert_equal(values,
+ ArrowFormat::UInt32Array.new(values).to_a)
+ end
+
+ def test_int64
+ values = [-(2 ** 63), nil, (2 ** 63) - 1]
+ assert_equal(values,
+ ArrowFormat::Int64Array.new(values).to_a)
+ end
+
+ def test_uint64
+ values = [0, nil, (2 ** 64) - 1]
+ assert_equal(values,
+ ArrowFormat::UInt64Array.new(values).to_a)
+ end
+
+ def test_float32
+ values = [-Float::INFINITY, -0.0, nil, +0.0, +Float::INFINITY]
+ assert_equal(values,
+ ArrowFormat::Float32Array.new(values).to_a)
+ end
+
+ def test_float64
+ values = [-Float::INFINITY, -0.0, nil, +0.0, +Float::INFINITY]
+ assert_equal(values,
+ ArrowFormat::Float64Array.new(values).to_a)
+ end
+end
