This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 879bbb3677 GH-49400: [Ruby] Add `Arrow::FixedSizeList#values` and
`#raw_records` (#49401)
879bbb3677 is described below
commit 879bbb3677fcde701dfed89c7d9abdf979271fcd
Author: Sutou Kouhei <[email protected]>
AuthorDate: Fri Feb 27 16:07:35 2026 +0900
GH-49400: [Ruby] Add `Arrow::FixedSizeList#values` and `#raw_records`
(#49401)
### Rationale for this change
They are optimized C++ values -> Ruby values converters.
### What changes are included in this PR?
* Add support for `arrow::FixedSizeListArray` in `#values` and
`#raw_records` implementation in C++
* Add tests for `Arrow::FixedSizeList#values` and `#raw_values`
* Add tests for `Arrow::LargeList#values` and `#raw_values` because they
are missing
* This is not so related to `Arrow::FixedSizeList#values` and
`#raw_values`. Sorry.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
Yes.
* GitHub Issue: #49400
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
ruby/red-arrow/ext/arrow/converters.cpp | 5 +
ruby/red-arrow/ext/arrow/converters.hpp | 118 +++++++++++++++++
ruby/red-arrow/ext/arrow/raw-records.cpp | 4 +
ruby/red-arrow/ext/arrow/values.cpp | 1 +
.../lib/arrow/fixed-size-list-data-type.rb | 118 +++++++++++++++++
ruby/red-arrow/lib/arrow/libraries.rb | 1 +
...list-array.rb => test-fixed-size-list-array.rb} | 141 +++++++++++++++++----
...test-list-array.rb => test-large-list-array.rb} | 77 +++++++++--
ruby/red-arrow/test/raw-records/test-list-array.rb | 57 +++++++++
...list-array.rb => test-fixed-size-list-array.rb} | 130 ++++++++++++++++---
...test-list-array.rb => test-large-list-array.rb} | 67 +++++++++-
ruby/red-arrow/test/values/test-list-array.rb | 53 ++++++++
12 files changed, 711 insertions(+), 61 deletions(-)
diff --git a/ruby/red-arrow/ext/arrow/converters.cpp
b/ruby/red-arrow/ext/arrow/converters.cpp
index bbabe60b7d..657665cb29 100644
--- a/ruby/red-arrow/ext/arrow/converters.cpp
+++ b/ruby/red-arrow/ext/arrow/converters.cpp
@@ -30,6 +30,11 @@ namespace red_arrow {
return large_list_array_value_converter_->convert(array, i);
}
+ VALUE ArrayValueConverter::convert(const arrow::FixedSizeListArray& array,
+ const int64_t i) {
+ return fixed_size_list_array_value_converter_->convert(array, i);
+ }
+
VALUE ArrayValueConverter::convert(const arrow::StructArray& array,
const int64_t i) {
return struct_array_value_converter_->convert(array, i);
diff --git a/ruby/red-arrow/ext/arrow/converters.hpp
b/ruby/red-arrow/ext/arrow/converters.hpp
index 099aa91686..37559fb310 100644
--- a/ruby/red-arrow/ext/arrow/converters.hpp
+++ b/ruby/red-arrow/ext/arrow/converters.hpp
@@ -29,6 +29,7 @@
namespace red_arrow {
class ListArrayValueConverter;
class LargeListArrayValueConverter;
+ class FixedSizeListArrayValueConverter;
class StructArrayValueConverter;
class MapArrayValueConverter;
class UnionArrayValueConverter;
@@ -40,6 +41,7 @@ namespace red_arrow {
: decimal_buffer_(),
list_array_value_converter_(nullptr),
large_list_array_value_converter_(nullptr),
+ fixed_size_list_array_value_converter_(nullptr),
struct_array_value_converter_(nullptr),
map_array_value_converter_(nullptr),
union_array_value_converter_(nullptr),
@@ -48,12 +50,14 @@ namespace red_arrow {
inline void set_sub_value_converters(ListArrayValueConverter*
list_array_value_converter,
LargeListArrayValueConverter*
large_list_array_value_converter,
+ FixedSizeListArrayValueConverter*
fixed_size_list_array_value_converter,
StructArrayValueConverter*
struct_array_value_converter,
MapArrayValueConverter*
map_array_value_converter,
UnionArrayValueConverter*
union_array_value_converter,
DictionaryArrayValueConverter*
dictionary_array_value_converter) {
list_array_value_converter_ = list_array_value_converter;
large_list_array_value_converter_ = large_list_array_value_converter;
+ fixed_size_list_array_value_converter_ =
fixed_size_list_array_value_converter;
struct_array_value_converter_ = struct_array_value_converter;
map_array_value_converter_ = map_array_value_converter;
union_array_value_converter_ = union_array_value_converter;
@@ -286,6 +290,9 @@ namespace red_arrow {
VALUE convert(const arrow::LargeListArray& array,
const int64_t i);
+ VALUE convert(const arrow::FixedSizeListArray& array,
+ const int64_t i);
+
VALUE convert(const arrow::StructArray& array,
const int64_t i);
@@ -322,6 +329,7 @@ namespace red_arrow {
std::string decimal_buffer_;
ListArrayValueConverter* list_array_value_converter_;
LargeListArrayValueConverter* large_list_array_value_converter_;
+ FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter_;
StructArrayValueConverter* struct_array_value_converter_;
MapArrayValueConverter* map_array_value_converter_;
UnionArrayValueConverter* union_array_value_converter_;
@@ -385,6 +393,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -485,6 +494,108 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
+ VISIT(Struct)
+ VISIT(Map)
+ VISIT(SparseUnion)
+ VISIT(DenseUnion)
+ VISIT(Dictionary)
+ VISIT(Decimal128)
+ VISIT(Decimal256)
+ // TODO
+ // VISIT(Extension)
+
+#undef VISIT
+
+ private:
+ template <typename ArrayType>
+ inline VALUE convert_value(const ArrayType& array,
+ const int64_t i) {
+ return array_value_converter_->convert(array, i);
+ }
+
+ template <typename ArrayType>
+ arrow::Status visit_value(const ArrayType& array) {
+ if (array.null_count() > 0) {
+ for (int64_t i = 0; i < length_; ++i) {
+ auto value = Qnil;
+ if (!array.IsNull(i + offset_)) {
+ value = convert_value(array, i + offset_);
+ }
+ rb_ary_push(result_, value);
+ }
+ } else {
+ for (int64_t i = 0; i < length_; ++i) {
+ rb_ary_push(result_, convert_value(array, i + offset_));
+ }
+ }
+ return arrow::Status::OK();
+ }
+
+ ArrayValueConverter* array_value_converter_;
+ int32_t offset_;
+ int32_t length_;
+ VALUE result_;
+ };
+
+ class FixedSizeListArrayValueConverter : public arrow::ArrayVisitor {
+ public:
+ explicit FixedSizeListArrayValueConverter(ArrayValueConverter* converter)
+ : array_value_converter_(converter),
+ offset_(0),
+ length_(0),
+ result_(Qnil) {}
+
+ VALUE convert(const arrow::FixedSizeListArray& array, const int64_t index)
{
+ auto values = array.values().get();
+ auto offset_keep = offset_;
+ auto length_keep = length_;
+ offset_ = array.value_offset(index);
+ length_ = array.value_length(index);
+ auto result_keep = result_;
+ result_ = rb_ary_new_capa(length_);
+ check_status(values->Accept(this),
+ "[raw-records][fixed-size-list-array]");
+ offset_ = offset_keep;
+ length_ = length_keep;
+ auto result_return = result_;
+ result_ = result_keep;
+ return result_return;
+ }
+
+#define VISIT(TYPE) \
+ arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
+ return visit_value(array); \
+ }
+
+ VISIT(Null)
+ VISIT(Boolean)
+ VISIT(Int8)
+ VISIT(Int16)
+ VISIT(Int32)
+ VISIT(Int64)
+ VISIT(UInt8)
+ VISIT(UInt16)
+ VISIT(UInt32)
+ VISIT(UInt64)
+ VISIT(HalfFloat)
+ VISIT(Float)
+ VISIT(Double)
+ VISIT(Binary)
+ VISIT(String)
+ VISIT(FixedSizeBinary)
+ VISIT(Date32)
+ VISIT(Date64)
+ VISIT(Time32)
+ VISIT(Time64)
+ VISIT(Timestamp)
+ VISIT(MonthInterval)
+ VISIT(DayTimeInterval)
+ VISIT(MonthDayNanoInterval)
+ VISIT(Duration)
+ VISIT(List)
+ VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -593,6 +704,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -697,6 +809,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -802,6 +915,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -917,6 +1031,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -947,6 +1062,7 @@ namespace red_arrow {
: array_value_converter_(),
list_array_value_converter_(&array_value_converter_),
large_list_array_value_converter_(&array_value_converter_),
+ fixed_size_list_array_value_converter_(&array_value_converter_),
struct_array_value_converter_(&array_value_converter_),
map_array_value_converter_(&array_value_converter_),
union_array_value_converter_(&array_value_converter_),
@@ -954,6 +1070,7 @@ namespace red_arrow {
array_value_converter_.
set_sub_value_converters(&list_array_value_converter_,
&large_list_array_value_converter_,
+ &fixed_size_list_array_value_converter_,
&struct_array_value_converter_,
&map_array_value_converter_,
&union_array_value_converter_,
@@ -969,6 +1086,7 @@ namespace red_arrow {
ArrayValueConverter array_value_converter_;
ListArrayValueConverter list_array_value_converter_;
LargeListArrayValueConverter large_list_array_value_converter_;
+ FixedSizeListArrayValueConverter fixed_size_list_array_value_converter_;
StructArrayValueConverter struct_array_value_converter_;
MapArrayValueConverter map_array_value_converter_;
UnionArrayValueConverter union_array_value_converter_;
diff --git a/ruby/red-arrow/ext/arrow/raw-records.cpp
b/ruby/red-arrow/ext/arrow/raw-records.cpp
index 7f643bad41..9b7b7ad3ad 100644
--- a/ruby/red-arrow/ext/arrow/raw-records.cpp
+++ b/ruby/red-arrow/ext/arrow/raw-records.cpp
@@ -102,6 +102,8 @@ namespace red_arrow {
VISIT(MonthDayNanoInterval)
VISIT(Duration)
VISIT(List)
+ VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
@@ -241,6 +243,8 @@ namespace red_arrow {
VISIT(MonthDayNanoInterval)
VISIT(Duration)
VISIT(List)
+ VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
diff --git a/ruby/red-arrow/ext/arrow/values.cpp
b/ruby/red-arrow/ext/arrow/values.cpp
index 0296f27398..61baa185d8 100644
--- a/ruby/red-arrow/ext/arrow/values.cpp
+++ b/ruby/red-arrow/ext/arrow/values.cpp
@@ -84,6 +84,7 @@ namespace red_arrow {
VISIT(Duration)
VISIT(List)
VISIT(LargeList)
+ VISIT(FixedSizeList)
VISIT(Struct)
VISIT(Map)
VISIT(SparseUnion)
diff --git a/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb
b/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb
new file mode 100644
index 0000000000..36c265b871
--- /dev/null
+++ b/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+ class FixedSizeListDataType
+ include ListFieldResolvable
+
+ alias_method :initialize_raw, :initialize
+ private :initialize_raw
+
+ # Creates a new {Arrow::FixedSizeListDataType}.
+ #
+ # @overload initialize(field, size)
+ #
+ # @param field [Arrow::Field, Hash] The field of the fixed size
+ # list data type. You can also specify field description by
+ # `Hash`.
+ #
+ # See {Arrow::Field.new} how to specify field description.
+ #
+ # @param size [Integer] The number of values in each element.
+ #
+ # @example Create a fixed size list data type with {Arrow::Field}
+ # visible_field = Arrow::Field.new("visible", :boolean)
+ # size = 2
+ # Arrow::FixedSizeListDataType.new(visible_field, size)
+ #
+ # @example Create a list data type with field description
+ # description = {name: "visible", type: :boolean}
+ # size = 2
+ # Arrow::FixedSizeListDataType.new(description, size)
+ #
+ # @overload initialize(description)
+ #
+ # @param description [Hash] The description of the fixed size
+ # list data type. It must have `:field` value and `:size`
+ # value.
+ #
+ # @option description [Arrow::Field, Hash] :field The field of
+ # the list data type. You can also specify field description
+ # by `Hash`.
+ #
+ # See {Arrow::Field.new} how to specify field description.
+ #
+ # @option description [Integer] :size The number of values of
+ # each element of the fixed size list data type.
+ #
+ # @example Create a fixed size list data type with {Arrow::Field}
+ # visible_field = Arrow::Field.new("visible", :boolean)
+ # Arrow::FixedSizeListDataType.new(field: visible_field, size: 2)
+ #
+ # @example Create a fixed size list data type with field description
+ # Arrow::FixedSizeListDataType.new(field: {
+ # name: "visible",
+ # type: :boolean,
+ # },
+ # size: 2)
+ #
+ # @overload initialize(data_type, size)
+ #
+ # @param data_type [Arrow::DataType, String, Symbol,
+ # ::Array<String>, ::Array<Symbol>, Hash] The element data
+ # type of the fixed size list data type. A field is created
+ # with the default name `"item"` from the data type
+ # automatically.
+ #
+ # See {Arrow::DataType.resolve} how to specify data type.
+ #
+ # @param size [Integer] The number of values in each
+ # element.
+ #
+ # @example Create a fixed size list data type with {Arrow::DataType}
+ # size = 2
+ # Arrow::FixedSizeListDataType.new(Arrow::BooleanDataType.new,
+ # size)
+ #
+ # @example Create a fixed size list data type with data type name as
String
+ # size = 2
+ # Arrow::FixedSizeListDataType.new("boolean", size)
+ #
+ # @example Create a fixed size list data type with data type name as
Symbol
+ # size = 2
+ # Arrow::FixedSizeListDataType.new(:boolean, size)
+ #
+ # @example Create a fixed size list data type with data type as Array
+ # size = 2
+ # Arrow::FixedSizeListDataType.new([:time32, :milli], size)
+ def initialize(*args)
+ n_args = args.size
+ case n_args
+ when 1
+ description = args[0]
+ size = description.delete(:size)
+ initialize_raw(resolve_field(description), size)
+ when 2
+ field, size = args
+ initialize_raw(resolve_field(field), size)
+ else
+ message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+ raise ArgumentError, message
+ end
+ end
+ end
+end
diff --git a/ruby/red-arrow/lib/arrow/libraries.rb
b/ruby/red-arrow/lib/arrow/libraries.rb
index a29a5588bb..f588883d66 100644
--- a/ruby/red-arrow/lib/arrow/libraries.rb
+++ b/ruby/red-arrow/lib/arrow/libraries.rb
@@ -71,6 +71,7 @@ require_relative "file-system"
require_relative "fixed-size-binary-array"
require_relative "fixed-size-binary-array-builder"
require_relative "fixed-size-list-array-builder"
+require_relative "fixed-size-list-data-type"
require_relative "function"
require_relative "group"
require_relative "half-float"
diff --git a/ruby/red-arrow/test/raw-records/test-list-array.rb
b/ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb
similarity index 81%
copy from ruby/red-arrow/test/raw-records/test-list-array.rb
copy to ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb
index d19b707d0f..168bdce595 100644
--- a/ruby/red-arrow/test/raw-records/test-list-array.rb
+++ b/ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-module RawRecordsListArrayTests
+module RawRecordsFixedSizeListArrayTests
def build_schema(type)
field_description = {
name: :element,
@@ -27,15 +27,16 @@ module RawRecordsListArrayTests
end
{
column: {
- type: :list,
+ type: :fixed_size_list,
field: field_description,
+ size: 4,
},
}
end
def test_null
records = [
- [[nil, nil, nil]],
+ [[nil, nil, nil, nil]],
[nil],
]
target = build(:null, records)
@@ -44,7 +45,7 @@ module RawRecordsListArrayTests
def test_boolean
records = [
- [[true, nil, false]],
+ [[true, nil, false, true]],
[nil],
]
target = build(:boolean, records)
@@ -53,7 +54,7 @@ module RawRecordsListArrayTests
def test_int8
records = [
- [[-(2 ** 7), nil, (2 ** 7) - 1]],
+ [[-(2 ** 7), nil, (2 ** 7) - 1, 0]],
[nil],
]
target = build(:int8, records)
@@ -62,7 +63,7 @@ module RawRecordsListArrayTests
def test_uint8
records = [
- [[0, nil, (2 ** 8) - 1]],
+ [[0, nil, (2 ** 8) - 1, 0]],
[nil],
]
target = build(:uint8, records)
@@ -71,7 +72,7 @@ module RawRecordsListArrayTests
def test_int16
records = [
- [[-(2 ** 15), nil, (2 ** 15) - 1]],
+ [[-(2 ** 15), nil, (2 ** 15) - 1, 0]],
[nil],
]
target = build(:int16, records)
@@ -80,7 +81,7 @@ module RawRecordsListArrayTests
def test_uint16
records = [
- [[0, nil, (2 ** 16) - 1]],
+ [[0, nil, (2 ** 16) - 1, 0]],
[nil],
]
target = build(:uint16, records)
@@ -89,7 +90,7 @@ module RawRecordsListArrayTests
def test_int32
records = [
- [[-(2 ** 31), nil, (2 ** 31) - 1]],
+ [[-(2 ** 31), nil, (2 ** 31) - 1, 0]],
[nil],
]
target = build(:int32, records)
@@ -98,7 +99,7 @@ module RawRecordsListArrayTests
def test_uint32
records = [
- [[0, nil, (2 ** 32) - 1]],
+ [[0, nil, (2 ** 32) - 1, 0]],
[nil],
]
target = build(:uint32, records)
@@ -107,7 +108,7 @@ module RawRecordsListArrayTests
def test_int64
records = [
- [[-(2 ** 63), nil, (2 ** 63) - 1]],
+ [[-(2 ** 63), nil, (2 ** 63) - 1, 0]],
[nil],
]
target = build(:int64, records)
@@ -116,7 +117,7 @@ module RawRecordsListArrayTests
def test_uint64
records = [
- [[0, nil, (2 ** 64) - 1]],
+ [[0, nil, (2 ** 64) - 1, 0]],
[nil],
]
target = build(:uint64, records)
@@ -125,7 +126,7 @@ module RawRecordsListArrayTests
def test_float
records = [
- [[-1.0, nil, 1.0]],
+ [[-1.0, nil, 1.0, 0.0]],
[nil],
]
target = build(:float, records)
@@ -134,7 +135,7 @@ module RawRecordsListArrayTests
def test_double
records = [
- [[-1.0, nil, 1.0]],
+ [[-1.0, nil, 1.0, 0.0]],
[nil],
]
target = build(:double, records)
@@ -143,7 +144,7 @@ module RawRecordsListArrayTests
def test_binary
records = [
- [["\x00".b, nil, "\xff".b]],
+ [["\x00".b, nil, "\xff".b, "".b]],
[nil],
]
target = build(:binary, records)
@@ -157,6 +158,7 @@ module RawRecordsListArrayTests
"Ruby",
nil,
"\u3042", # U+3042 HIRAGANA LETTER A
+ "",
],
],
[nil],
@@ -172,6 +174,7 @@ module RawRecordsListArrayTests
Date.new(1960, 1, 1),
nil,
Date.new(2017, 8, 23),
+ Date.new(1970, 1, 1),
],
],
[nil],
@@ -187,6 +190,7 @@ module RawRecordsListArrayTests
DateTime.new(1960, 1, 1, 2, 9, 30),
nil,
DateTime.new(2017, 8, 23, 14, 57, 2),
+ DateTime.new(1970, 1, 1, 0, 0, 0),
],
],
[nil],
@@ -202,6 +206,7 @@ module RawRecordsListArrayTests
Time.parse("1960-01-01T02:09:30Z"),
nil,
Time.parse("2017-08-23T14:57:02Z"),
+ Time.parse("1970-01-01T00:00:00Z"),
],
],
[nil],
@@ -221,6 +226,7 @@ module RawRecordsListArrayTests
Time.parse("1960-01-01T02:09:30.123Z"),
nil,
Time.parse("2017-08-23T14:57:02.987Z"),
+ Time.parse("1970-01-01T00:00:00.000Z"),
],
],
[nil],
@@ -240,6 +246,7 @@ module RawRecordsListArrayTests
Time.parse("1960-01-01T02:09:30.123456Z"),
nil,
Time.parse("2017-08-23T14:57:02.987654Z"),
+ Time.parse("1970-01-01T00:00:00.000000Z"),
],
],
[nil],
@@ -259,6 +266,7 @@ module RawRecordsListArrayTests
Time.parse("1960-01-01T02:09:30.123456789Z"),
nil,
Time.parse("2017-08-23T14:57:02.987654321Z"),
+ Time.parse("1970-01-01T00:00:00.000000000Z"),
],
],
[nil],
@@ -281,6 +289,8 @@ module RawRecordsListArrayTests
nil,
# 02:00:09
Arrow::Time.new(unit, 60 * 60 * 2 + 9),
+ # 00:00:00
+ Arrow::Time.new(unit, 0),
],
],
[nil],
@@ -303,6 +313,8 @@ module RawRecordsListArrayTests
nil,
# 02:00:09.987
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987),
+ # 00:00:00.000
+ Arrow::Time.new(unit, 0),
],
],
[nil],
@@ -325,6 +337,8 @@ module RawRecordsListArrayTests
nil,
# 02:00:09.987654
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654),
+ # 00:00:00.000000
+ Arrow::Time.new(unit, 0),
],
],
[nil],
@@ -347,6 +361,8 @@ module RawRecordsListArrayTests
nil,
# 02:00:09.987654321
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 +
987_654_321),
+ # 00:00:00.000000000
+ Arrow::Time.new(unit, 0),
],
],
[nil],
@@ -366,6 +382,7 @@ module RawRecordsListArrayTests
BigDecimal("92.92"),
nil,
BigDecimal("29.29"),
+ BigDecimal("00.00"),
],
],
[nil],
@@ -386,6 +403,7 @@ module RawRecordsListArrayTests
BigDecimal("92.92"),
nil,
BigDecimal("29.29"),
+ BigDecimal("00.00"),
],
],
[nil],
@@ -401,7 +419,7 @@ module RawRecordsListArrayTests
def test_month_interval
records = [
- [[1, nil, 12]],
+ [[1, nil, 12, 0]],
[nil],
]
target = build(:month_interval, records)
@@ -415,6 +433,7 @@ module RawRecordsListArrayTests
{day: 1, millisecond: 100},
nil,
{day: 2, millisecond: 300},
+ {day: 0, millisecond: 0},
]
],
[nil],
@@ -430,6 +449,7 @@ module RawRecordsListArrayTests
{month: 1, day: 1, nanosecond: 100},
nil,
{month: 2, day: 3, nanosecond: 400},
+ {month: 0, day: 0, nanosecond: 0},
]
],
[nil],
@@ -451,6 +471,10 @@ module RawRecordsListArrayTests
nil,
false,
],
+ [
+ true,
+ false,
+ ],
],
],
[nil],
@@ -466,6 +490,71 @@ module RawRecordsListArrayTests
assert_equal(records, actual_records(target))
end
+ def test_large_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ [
+ true,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
+ def test_fixed_size_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ [
+ true,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
def test_struct
records = [
[
@@ -473,6 +562,7 @@ module RawRecordsListArrayTests
{"field" => true},
nil,
{"field" => nil},
+ {"field" => false},
],
],
[nil],
@@ -496,6 +586,8 @@ module RawRecordsListArrayTests
[
{"key1" => true, "key2" => nil},
nil,
+ {"key1" => false},
+ {"key3" => nil},
],
],
[nil],
@@ -596,6 +688,7 @@ module RawRecordsListArrayTests
"Ruby",
nil,
"GLib",
+ "Ruby",
],
],
[nil],
@@ -611,8 +704,8 @@ module RawRecordsListArrayTests
end
end
-class EachRawRecordRecordBatchListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class EachRawRecordRecordBatchFixedSizeListArrayTest < Test::Unit::TestCase
+ include RawRecordsFixedSizeListArrayTests
def build(type, records)
Arrow::RecordBatch.new(build_schema(type), records)
@@ -623,8 +716,8 @@ class EachRawRecordRecordBatchListArrayTest <
Test::Unit::TestCase
end
end
-class EachRawRecordTableListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class EachRawRecordTableFixedSizeListArrayTest < Test::Unit::TestCase
+ include RawRecordsFixedSizeListArrayTests
def build(type, records)
record_batch = Arrow::RecordBatch.new(build_schema(type), records)
@@ -643,8 +736,8 @@ class EachRawRecordTableListArrayTest < Test::Unit::TestCase
end
-class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class RawRecordsRecordBatchFixedSizeListArrayTest < Test::Unit::TestCase
+ include RawRecordsFixedSizeListArrayTests
def build(type, records)
Arrow::RecordBatch.new(build_schema(type), records)
@@ -655,8 +748,8 @@ class RawRecordsRecordBatchListArrayTest <
Test::Unit::TestCase
end
end
-class RawRecordsTableListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class RawRecordsTableFixedSizeListArrayTest < Test::Unit::TestCase
+ include RawRecordsFixedSizeListArrayTests
def build(type, records)
Arrow::Table.new(build_schema(type), records)
diff --git a/ruby/red-arrow/test/raw-records/test-list-array.rb
b/ruby/red-arrow/test/raw-records/test-large-list-array.rb
similarity index 89%
copy from ruby/red-arrow/test/raw-records/test-list-array.rb
copy to ruby/red-arrow/test/raw-records/test-large-list-array.rb
index d19b707d0f..4557decaa6 100644
--- a/ruby/red-arrow/test/raw-records/test-list-array.rb
+++ b/ruby/red-arrow/test/raw-records/test-large-list-array.rb
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-module RawRecordsListArrayTests
+module RawRecordsLargeListArrayTests
def build_schema(type)
field_description = {
name: :element,
@@ -27,7 +27,7 @@ module RawRecordsListArrayTests
end
{
column: {
- type: :list,
+ type: :large_list,
field: field_description,
},
}
@@ -466,6 +466,63 @@ module RawRecordsListArrayTests
assert_equal(records, actual_records(target))
end
+ def test_large_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
+ def test_fixed_size_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
def test_struct
records = [
[
@@ -611,8 +668,8 @@ module RawRecordsListArrayTests
end
end
-class EachRawRecordRecordBatchListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class EachRawRecordRecordBatchLargeListArrayTest < Test::Unit::TestCase
+ include RawRecordsLargeListArrayTests
def build(type, records)
Arrow::RecordBatch.new(build_schema(type), records)
@@ -623,8 +680,8 @@ class EachRawRecordRecordBatchListArrayTest <
Test::Unit::TestCase
end
end
-class EachRawRecordTableListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class EachRawRecordTableLargeListArrayTest < Test::Unit::TestCase
+ include RawRecordsLargeListArrayTests
def build(type, records)
record_batch = Arrow::RecordBatch.new(build_schema(type), records)
@@ -643,8 +700,8 @@ class EachRawRecordTableListArrayTest < Test::Unit::TestCase
end
-class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class RawRecordsRecordBatchLargeListArrayTest < Test::Unit::TestCase
+ include RawRecordsLargeListArrayTests
def build(type, records)
Arrow::RecordBatch.new(build_schema(type), records)
@@ -655,8 +712,8 @@ class RawRecordsRecordBatchListArrayTest <
Test::Unit::TestCase
end
end
-class RawRecordsTableListArrayTest < Test::Unit::TestCase
- include RawRecordsListArrayTests
+class RawRecordsTableLargeListArrayTest < Test::Unit::TestCase
+ include RawRecordsLargeListArrayTests
def build(type, records)
Arrow::Table.new(build_schema(type), records)
diff --git a/ruby/red-arrow/test/raw-records/test-list-array.rb
b/ruby/red-arrow/test/raw-records/test-list-array.rb
index d19b707d0f..857be3be89 100644
--- a/ruby/red-arrow/test/raw-records/test-list-array.rb
+++ b/ruby/red-arrow/test/raw-records/test-list-array.rb
@@ -466,6 +466,63 @@ module RawRecordsListArrayTests
assert_equal(records, actual_records(target))
end
+ def test_large_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
+ def test_fixed_size_list
+ records = [
+ [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ ],
+ [nil],
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ records)
+ assert_equal(records, actual_records(target))
+ end
+
def test_struct
records = [
[
diff --git a/ruby/red-arrow/test/values/test-list-array.rb
b/ruby/red-arrow/test/values/test-fixed-size-list-array.rb
similarity index 80%
copy from ruby/red-arrow/test/values/test-list-array.rb
copy to ruby/red-arrow/test/values/test-fixed-size-list-array.rb
index 8c1bae9f9f..4e844b646c 100644
--- a/ruby/red-arrow/test/values/test-list-array.rb
+++ b/ruby/red-arrow/test/values/test-fixed-size-list-array.rb
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-module ValuesListArrayTests
+module ValuesFixedSizeListArrayTests
def build_data_type(type)
field_description = {
name: :element,
@@ -25,16 +25,17 @@ module ValuesListArrayTests
else
field_description[:type] = type
end
- Arrow::ListDataType.new(field: field_description)
+ Arrow::FixedSizeListDataType.new(field: field_description,
+ size: 4)
end
def build_array(type, values)
- Arrow::ListArray.new(build_data_type(type), values)
+ Arrow::FixedSizeListArray.new(build_data_type(type), values)
end
def test_null
values = [
- [nil, nil, nil],
+ [nil, nil, nil, nil],
nil,
]
target = build(:null, values)
@@ -43,7 +44,7 @@ module ValuesListArrayTests
def test_boolean
values = [
- [true, nil, false],
+ [true, nil, false, true],
nil,
]
target = build(:boolean, values)
@@ -52,7 +53,7 @@ module ValuesListArrayTests
def test_int8
values = [
- [-(2 ** 7), nil, (2 ** 7) - 1],
+ [-(2 ** 7), nil, (2 ** 7) - 1, 0],
nil,
]
target = build(:int8, values)
@@ -61,7 +62,7 @@ module ValuesListArrayTests
def test_uint8
values = [
- [0, nil, (2 ** 8) - 1],
+ [0, nil, (2 ** 8) - 1, 0],
nil,
]
target = build(:uint8, values)
@@ -70,7 +71,7 @@ module ValuesListArrayTests
def test_int16
values = [
- [-(2 ** 15), nil, (2 ** 15) - 1],
+ [-(2 ** 15), nil, (2 ** 15) - 1, 0],
nil,
]
target = build(:int16, values)
@@ -79,7 +80,7 @@ module ValuesListArrayTests
def test_uint16
values = [
- [0, nil, (2 ** 16) - 1],
+ [0, nil, (2 ** 16) - 1, 0],
nil,
]
target = build(:uint16, values)
@@ -88,7 +89,7 @@ module ValuesListArrayTests
def test_int32
values = [
- [-(2 ** 31), nil, (2 ** 31) - 1],
+ [-(2 ** 31), nil, (2 ** 31) - 1, 0],
nil,
]
target = build(:int32, values)
@@ -97,7 +98,7 @@ module ValuesListArrayTests
def test_uint32
values = [
- [0, nil, (2 ** 32) - 1],
+ [0, nil, (2 ** 32) - 1, 0],
nil,
]
target = build(:uint32, values)
@@ -106,7 +107,7 @@ module ValuesListArrayTests
def test_int64
values = [
- [-(2 ** 63), nil, (2 ** 63) - 1],
+ [-(2 ** 63), nil, (2 ** 63) - 1, 0],
nil,
]
target = build(:int64, values)
@@ -115,7 +116,7 @@ module ValuesListArrayTests
def test_uint64
values = [
- [0, nil, (2 ** 64) - 1],
+ [0, nil, (2 ** 64) - 1, 0],
nil,
]
target = build(:uint64, values)
@@ -124,7 +125,7 @@ module ValuesListArrayTests
def test_float
values = [
- [-1.0, nil, 1.0],
+ [-1.0, nil, 1.0, 0.0],
nil,
]
target = build(:float, values)
@@ -133,7 +134,7 @@ module ValuesListArrayTests
def test_double
values = [
- [-1.0, nil, 1.0],
+ [-1.0, nil, 1.0, 0.0],
nil,
]
target = build(:double, values)
@@ -142,7 +143,7 @@ module ValuesListArrayTests
def test_binary
values = [
- ["\x00".b, nil, "\xff".b],
+ ["\x00".b, nil, "\xff".b, "".b],
nil,
]
target = build(:binary, values)
@@ -155,6 +156,7 @@ module ValuesListArrayTests
"Ruby",
nil,
"\u3042", # U+3042 HIRAGANA LETTER A
+ ""
],
nil,
]
@@ -168,6 +170,7 @@ module ValuesListArrayTests
Date.new(1960, 1, 1),
nil,
Date.new(2017, 8, 23),
+ Date.new(1970, 1, 1),
],
nil,
]
@@ -181,6 +184,7 @@ module ValuesListArrayTests
DateTime.new(1960, 1, 1, 2, 9, 30),
nil,
DateTime.new(2017, 8, 23, 14, 57, 2),
+ DateTime.new(1970, 1, 1, 0, 0, 0),
],
nil,
]
@@ -194,6 +198,7 @@ module ValuesListArrayTests
Time.parse("1960-01-01T02:09:30Z"),
nil,
Time.parse("2017-08-23T14:57:02Z"),
+ Time.parse("1970-01-01T00:00:00Z"),
],
nil,
]
@@ -211,6 +216,7 @@ module ValuesListArrayTests
Time.parse("1960-01-01T02:09:30.123Z"),
nil,
Time.parse("2017-08-23T14:57:02.987Z"),
+ Time.parse("1970-01-01T00:00:00.000Z"),
],
nil,
]
@@ -228,6 +234,7 @@ module ValuesListArrayTests
Time.parse("1960-01-01T02:09:30.123456Z"),
nil,
Time.parse("2017-08-23T14:57:02.987654Z"),
+ Time.parse("1970-01-01T00:00:00.000000Z"),
],
nil,
]
@@ -245,6 +252,7 @@ module ValuesListArrayTests
Time.parse("1960-01-01T02:09:30.123456789Z"),
nil,
Time.parse("2017-08-23T14:57:02.987654321Z"),
+ Time.parse("1970-01-01T00:00:00.000000000Z"),
],
nil,
]
@@ -265,6 +273,8 @@ module ValuesListArrayTests
nil,
# 02:00:09
Arrow::Time.new(unit, 60 * 60 * 2 + 9),
+ # 00:00:00
+ Arrow::Time.new(unit, 0),
],
nil,
]
@@ -285,6 +295,8 @@ module ValuesListArrayTests
nil,
# 02:00:09.987
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987),
+ # 00:00:00.000
+ Arrow::Time.new(unit, 0),
],
nil,
]
@@ -305,6 +317,8 @@ module ValuesListArrayTests
nil,
# 02:00:09.987654
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654),
+ # 00:00:00.000000
+ Arrow::Time.new(unit, 0),
],
nil,
]
@@ -325,6 +339,8 @@ module ValuesListArrayTests
nil,
# 02:00:09.987654321
Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321),
+ # 00:00:00.000000000
+ Arrow::Time.new(unit, 0),
],
nil,
]
@@ -342,6 +358,7 @@ module ValuesListArrayTests
BigDecimal("92.92"),
nil,
BigDecimal("29.29"),
+ BigDecimal("00.00"),
],
nil,
]
@@ -360,6 +377,7 @@ module ValuesListArrayTests
BigDecimal("92.92"),
nil,
BigDecimal("29.29"),
+ BigDecimal("00.00"),
],
nil,
]
@@ -378,6 +396,7 @@ module ValuesListArrayTests
1,
nil,
12,
+ 0,
],
nil,
]
@@ -391,6 +410,7 @@ module ValuesListArrayTests
{day: 1, millisecond: 100},
nil,
{day: 2, millisecond: 300},
+ {day: 0, millisecond: 0},
],
nil,
]
@@ -404,6 +424,7 @@ module ValuesListArrayTests
{month: 1, day: 1, nanosecond: 100},
nil,
{month: 2, day: 3, nanosecond: 400},
+ {month: 0, day: 0, nanosecond: 0},
],
nil,
]
@@ -423,6 +444,10 @@ module ValuesListArrayTests
nil,
false,
],
+ [
+ true,
+ false,
+ ],
],
nil,
]
@@ -437,12 +462,74 @@ module ValuesListArrayTests
assert_equal(values, target.values)
end
+ def test_large_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ [
+ true,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
+ def test_fixed_size_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ [
+ true,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
def test_struct
values = [
[
{"field" => true},
nil,
{"field" => nil},
+ {"field" => false},
],
nil,
]
@@ -464,6 +551,8 @@ module ValuesListArrayTests
[
{"key1" => true, "key2" => nil},
nil,
+ {"key1" => false},
+ {"key3" => nil},
],
nil,
]
@@ -556,6 +645,7 @@ module ValuesListArrayTests
"Ruby",
nil,
"GLib",
+ "Ruby",
],
nil,
]
@@ -570,16 +660,16 @@ module ValuesListArrayTests
end
end
-class ValuesArrayListArrayTest < Test::Unit::TestCase
- include ValuesListArrayTests
+class ValuesArrayFixedSizeListArrayTest < Test::Unit::TestCase
+ include ValuesFixedSizeListArrayTests
def build(type, values)
build_array(type, values)
end
end
-class ValuesChunkedArrayListArrayTest < Test::Unit::TestCase
- include ValuesListArrayTests
+class ValuesChunkedArrayFixedSizeListArrayTest < Test::Unit::TestCase
+ include ValuesFixedSizeListArrayTests
def build(type, values)
Arrow::ChunkedArray.new([build_array(type, values)])
diff --git a/ruby/red-arrow/test/values/test-list-array.rb
b/ruby/red-arrow/test/values/test-large-list-array.rb
similarity index 90%
copy from ruby/red-arrow/test/values/test-list-array.rb
copy to ruby/red-arrow/test/values/test-large-list-array.rb
index 8c1bae9f9f..8971e1fadb 100644
--- a/ruby/red-arrow/test/values/test-list-array.rb
+++ b/ruby/red-arrow/test/values/test-large-list-array.rb
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-module ValuesListArrayTests
+module ValuesLargeListArrayTests
def build_data_type(type)
field_description = {
name: :element,
@@ -25,11 +25,11 @@ module ValuesListArrayTests
else
field_description[:type] = type
end
- Arrow::ListDataType.new(field: field_description)
+ Arrow::LargeListDataType.new(field: field_description)
end
def build_array(type, values)
- Arrow::ListArray.new(build_data_type(type), values)
+ Arrow::LargeListArray.new(build_data_type(type), values)
end
def test_null
@@ -437,6 +437,59 @@ module ValuesListArrayTests
assert_equal(values, target.values)
end
+ def test_large_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
+ def test_fixed_size_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
def test_struct
values = [
[
@@ -570,16 +623,16 @@ module ValuesListArrayTests
end
end
-class ValuesArrayListArrayTest < Test::Unit::TestCase
- include ValuesListArrayTests
+class ValuesArrayLargeListArrayTest < Test::Unit::TestCase
+ include ValuesLargeListArrayTests
def build(type, values)
build_array(type, values)
end
end
-class ValuesChunkedArrayListArrayTest < Test::Unit::TestCase
- include ValuesListArrayTests
+class ValuesChunkedArrayLargeListArrayTest < Test::Unit::TestCase
+ include ValuesLargeListArrayTests
def build(type, values)
Arrow::ChunkedArray.new([build_array(type, values)])
diff --git a/ruby/red-arrow/test/values/test-list-array.rb
b/ruby/red-arrow/test/values/test-list-array.rb
index 8c1bae9f9f..c9458b3666 100644
--- a/ruby/red-arrow/test/values/test-list-array.rb
+++ b/ruby/red-arrow/test/values/test-list-array.rb
@@ -437,6 +437,59 @@ module ValuesListArrayTests
assert_equal(values, target.values)
end
+ def test_large_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :large_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
+ def test_fixed_size_list
+ values = [
+ [
+ [
+ true,
+ nil,
+ ],
+ nil,
+ [
+ nil,
+ false,
+ ],
+ ],
+ nil,
+ ]
+ target = build({
+ type: :fixed_size_list,
+ field: {
+ name: :sub_element,
+ type: :boolean,
+ },
+ size: 2,
+ },
+ values)
+ assert_equal(values, target.values)
+ end
+
def test_struct
values = [
[
