This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 7ca702cb34 GH-50439: Add
ArrowFormat::{,Large}{Binary,UTF8}Array.new(values) (#50452)
7ca702cb34 is described below
commit 7ca702cb34326d92427cc5854b94c4c70953f69d
Author: Aaditya Srinivasan <[email protected]>
AuthorDate: Fri Jul 10 13:15:20 2026 +0530
GH-50439: Add ArrowFormat::{,Large}{Binary,UTF8}Array.new(values) (#50452)
### Rationale for this change
Building variable-size binary Arrow arrays from Ruby objects is convenient.
### What changes are included in this PR?
Accept:
* `ArrowFormat::BinaryArray.new(values)`
* `ArrowFormat::LargeBinaryArray.new(values)`
* `ArrowFormat::UTF8Array.new(values)`
* `ArrowFormat::LargeUTF8Array.new(values)`
### Are these changes tested?
Yes.
Added tests for:
* initialization
* mixed values with `nil`
* equality
* sliced arrays
### Are there any user-facing changes?
Yes.
* GitHub Issue: #50439
Authored-by: Aaditya Srinivasan <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ruby/red-arrow-format/lib/arrow-format/array.rb | 55 +++++++++++++++++++++-
ruby/red-arrow-format/lib/arrow-format/type.rb | 16 +++++++
ruby/red-arrow-format/test/test-binary-array.rb | 55 ++++++++++++++++++++++
.../test/test-large-binary-array.rb | 55 ++++++++++++++++++++++
.../red-arrow-format/test/test-large-utf8-array.rb | 55 ++++++++++++++++++++++
ruby/red-arrow-format/test/test-utf8-array.rb | 55 ++++++++++++++++++++++
6 files changed, 290 insertions(+), 1 deletion(-)
diff --git a/ruby/red-arrow-format/lib/arrow-format/array.rb
b/ruby/red-arrow-format/lib/arrow-format/array.rb
index 0b42a692ec..b3aceecd8f 100644
--- a/ruby/red-arrow-format/lib/arrow-format/array.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/array.rb
@@ -633,7 +633,18 @@ module ArrowFormat
end
class VariableSizeBinaryArray < Array
- def initialize(size, validity_buffer, offsets_buffer, values_buffer)
+ include BufferAlignable
+
+ def initialize(*args)
+ if args.size == 1
+ args = build_data(args.first)
+ elsif args.size != 4
+ raise ArgumentError,
+ "wrong number of arguments (given #{args.size}, expected 1 or 4)"
+ end
+
+ size, validity_buffer, offsets_buffer, values_buffer = args
+
super(self.class.type, size, validity_buffer)
@offsets_buffer = offsets_buffer
@values_buffer = values_buffer
@@ -673,6 +684,48 @@ module ArrowFormat
end
private
+ def build_data(data)
+ type = self.class.type
+
+ n = 0
+ validity_buffer_builder = nil
+
+ values = +"".b
+ offsets = [0]
+
+ data.each_with_index do |value, i|
+ if value.nil?
+ validity_buffer_builder ||= SparseBitmapBuilder.new
+ validity_buffer_builder.unset(i)
+
+ offsets << values.bytesize
+ else
+ values.append_as_bytes(value)
+ offsets << values.bytesize
+ end
+
+ n += 1
+ end
+
+ validity_buffer = validity_buffer_builder&.finish(n)
+
+ offsets_data = offsets.pack("#{type.offset_pack_template}*")
+ pad!(offsets_data, buffer_padding_size(offsets_data))
+ offsets_data.freeze
+ offsets_buffer = IO::Buffer.for(offsets_data)
+
+ pad!(values, buffer_padding_size(values))
+ values.freeze
+ values_buffer = IO::Buffer.for(values)
+
+ [
+ n,
+ validity_buffer,
+ offsets_buffer,
+ values_buffer,
+ ]
+ end
+
def offset_size
IO::Buffer.size_of(@type.offset_buffer_type)
end
diff --git a/ruby/red-arrow-format/lib/arrow-format/type.rb
b/ruby/red-arrow-format/lib/arrow-format/type.rb
index 4420b44b43..b1d278a110 100644
--- a/ruby/red-arrow-format/lib/arrow-format/type.rb
+++ b/ruby/red-arrow-format/lib/arrow-format/type.rb
@@ -767,6 +767,10 @@ module ArrowFormat
:s32 # TODO: big endian support
end
+ def offset_pack_template
+ "l"
+ end
+
def encoding
Encoding::ASCII_8BIT
end
@@ -795,6 +799,10 @@ module ArrowFormat
:s64 # TODO: big endian support
end
+ def offset_pack_template
+ "q"
+ end
+
def encoding
Encoding::ASCII_8BIT
end
@@ -823,6 +831,10 @@ module ArrowFormat
:s32 # TODO: big endian support
end
+ def offset_pack_template
+ "l"
+ end
+
def encoding
Encoding::UTF_8
end
@@ -851,6 +863,10 @@ module ArrowFormat
:s64 # TODO: big endian support
end
+ def offset_pack_template
+ "q"
+ end
+
def encoding
Encoding::UTF_8
end
diff --git a/ruby/red-arrow-format/test/test-binary-array.rb
b/ruby/red-arrow-format/test/test-binary-array.rb
new file mode 100644
index 0000000000..bcc122ccb0
--- /dev/null
+++ b/ruby/red-arrow-format/test/test-binary-array.rb
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBinaryArray < Test::Unit::TestCase
+ sub_test_case("#initialize") do
+ def test_no_null
+ values = ["\x00\x01".b, "\xff\x10".b]
+ assert_equal(values,
+ ArrowFormat::BinaryArray.new(values).to_a)
+ end
+
+ def test_mixed
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ assert_equal(values,
+ ArrowFormat::BinaryArray.new(values).to_a)
+ end
+ end
+
+ sub_test_case("#==") do
+ def test_no_slice
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::BinaryArray.new(values)
+ array2 = ArrowFormat::BinaryArray.new(values)
+ assert_equal(array1, array2)
+ end
+
+ def test_sliced
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::BinaryArray.new(values)
+ array2 = ArrowFormat::BinaryArray.new(["".b, *values, "".b])
+ assert_equal(array1, array2.slice(1, 3))
+ end
+
+ def test_sliced_different_content
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::BinaryArray.new(values)
+ array2 = ArrowFormat::BinaryArray.new(["".b, "".b, *values, "".b])
+ assert_not_equal(array1, array2.slice(1, 3))
+ end
+ end
+end
diff --git a/ruby/red-arrow-format/test/test-large-binary-array.rb
b/ruby/red-arrow-format/test/test-large-binary-array.rb
new file mode 100644
index 0000000000..67e55dd1b6
--- /dev/null
+++ b/ruby/red-arrow-format/test/test-large-binary-array.rb
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestLargeBinaryArray < Test::Unit::TestCase
+ sub_test_case("#initialize") do
+ def test_no_null
+ values = ["\x00\x01".b, "\xff\x10".b]
+ assert_equal(values,
+ ArrowFormat::LargeBinaryArray.new(values).to_a)
+ end
+
+ def test_mixed
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ assert_equal(values,
+ ArrowFormat::LargeBinaryArray.new(values).to_a)
+ end
+ end
+
+ sub_test_case("#==") do
+ def test_no_slice
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::LargeBinaryArray.new(values)
+ array2 = ArrowFormat::LargeBinaryArray.new(values)
+ assert_equal(array1, array2)
+ end
+
+ def test_sliced
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::LargeBinaryArray.new(values)
+ array2 = ArrowFormat::LargeBinaryArray.new(["".b, *values, "".b])
+ assert_equal(array1, array2.slice(1, 3))
+ end
+
+ def test_sliced_different_content
+ values = ["\x00\x01".b, nil, "\xff\x10".b]
+ array1 = ArrowFormat::LargeBinaryArray.new(values)
+ array2 = ArrowFormat::LargeBinaryArray.new(["".b, "".b, *values, "".b])
+ assert_not_equal(array1, array2.slice(1, 3))
+ end
+ end
+end
diff --git a/ruby/red-arrow-format/test/test-large-utf8-array.rb
b/ruby/red-arrow-format/test/test-large-utf8-array.rb
new file mode 100644
index 0000000000..1ee4a39945
--- /dev/null
+++ b/ruby/red-arrow-format/test/test-large-utf8-array.rb
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestLargeUTF8Array < Test::Unit::TestCase
+ sub_test_case("#initialize") do
+ def test_no_null
+ values = ["Hello", "World"]
+ assert_equal(values,
+ ArrowFormat::LargeUTF8Array.new(values).to_a)
+ end
+
+ def test_mixed
+ values = ["Hello", nil, "World"]
+ assert_equal(values,
+ ArrowFormat::LargeUTF8Array.new(values).to_a)
+ end
+ end
+
+ sub_test_case("#==") do
+ def test_no_slice
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::LargeUTF8Array.new(values)
+ array2 = ArrowFormat::LargeUTF8Array.new(values)
+ assert_equal(array1, array2)
+ end
+
+ def test_sliced
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::LargeUTF8Array.new(values)
+ array2 = ArrowFormat::LargeUTF8Array.new(["", *values, ""])
+ assert_equal(array1, array2.slice(1, 3))
+ end
+
+ def test_sliced_different_content
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::LargeUTF8Array.new(values)
+ array2 = ArrowFormat::LargeUTF8Array.new(["", "", *values, ""])
+ assert_not_equal(array1, array2.slice(1, 3))
+ end
+ end
+end
diff --git a/ruby/red-arrow-format/test/test-utf8-array.rb
b/ruby/red-arrow-format/test/test-utf8-array.rb
new file mode 100644
index 0000000000..2970d77793
--- /dev/null
+++ b/ruby/red-arrow-format/test/test-utf8-array.rb
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestUTF8Array < Test::Unit::TestCase
+ sub_test_case("#initialize") do
+ def test_no_null
+ values = ["Hello", "World"]
+ assert_equal(values,
+ ArrowFormat::UTF8Array.new(values).to_a)
+ end
+
+ def test_mixed
+ values = ["Hello", nil, "World"]
+ assert_equal(values,
+ ArrowFormat::UTF8Array.new(values).to_a)
+ end
+ end
+
+ sub_test_case("#==") do
+ def test_no_slice
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::UTF8Array.new(values)
+ array2 = ArrowFormat::UTF8Array.new(values)
+ assert_equal(array1, array2)
+ end
+
+ def test_sliced
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::UTF8Array.new(values)
+ array2 = ArrowFormat::UTF8Array.new(["", *values, ""])
+ assert_equal(array1, array2.slice(1, 3))
+ end
+
+ def test_sliced_different_content
+ values = ["Hello", nil, "World"]
+ array1 = ArrowFormat::UTF8Array.new(values)
+ array2 = ArrowFormat::UTF8Array.new(["", "", *values, ""])
+ assert_not_equal(array1, array2.slice(1, 3))
+ end
+ end
+end
