This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6b1ad60ab5e GH-50658: [C++][Dev] Add RunEndEncoded support to
gdb_arrow.py (#50768)
6b1ad60ab5e is described below
commit 6b1ad60ab5e58adb32e2f54be61daeb51bc5c1c6
Author: fenfeng9 <[email protected]>
AuthorDate: Tue Oct 6 16:26:41 2026 +0800
GH-50658: [C++][Dev] Add RunEndEncoded support to gdb_arrow.py (#50768)
### Rationale for this change
The Arrow GDB pretty-printers do not support run-end encoded data types.
### What changes are included in this PR?
Add GDB pretty-printer support and tests for `run-end encoded` types,
scalars, and arrays.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #50658
Authored-by: fenfeng9 <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/gdb_arrow.py | 97 +++++++++++++++++++++++++++++++---
python/pyarrow/src/arrow/python/gdb.cc | 29 ++++++++++
python/pyarrow/tests/test_gdb.py | 50 ++++++++++++++++++
3 files changed, 170 insertions(+), 6 deletions(-)
diff --git a/cpp/gdb_arrow.py b/cpp/gdb_arrow.py
index bab1fa8722d..48956117732 100644
--- a/cpp/gdb_arrow.py
+++ b/cpp/gdb_arrow.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from bisect import bisect_right
from collections import namedtuple
from collections.abc import Sequence
import datetime
@@ -95,7 +96,8 @@ def identity(v):
def has_null_bitmap(type_id):
- return type_id not in (Type.NA, Type.SPARSE_UNION, Type.DENSE_UNION)
+ return type_id not in (Type.NA, Type.SPARSE_UNION, Type.DENSE_UNION,
+ Type.RUN_END_ENCODED)
@lru_cache()
@@ -626,11 +628,12 @@ class Buffer:
"""
Return a view over the bytes of this buffer.
"""
- if self.size > 0:
- if length is None:
- length = self.size
+ if length is None:
+ length = self.size - offset
+ # Sliced arrays may share buffers, so only read the requested range.
+ if length > 0:
mem = gdb.selected_inferior().read_memory(
- self.val['data_'] + offset, self.size)
+ self.val['data_'] + offset, length)
else:
mem = memoryview(b"")
# Read individual bytes as unsigned integers rather than
@@ -769,7 +772,8 @@ class Bitmap(Sequence):
def from_buffer(cls, buf, offset, length):
assert isinstance(buf, Buffer)
byte_offset, bit_offset = divmod(offset, 8)
- byte_length = math.ceil(length + offset / 8) - byte_offset
+ # E.g. offset=3, length=6 selects bits 3..8 and needs 2 bytes.
+ byte_length = math.ceil((bit_offset + length) / 8)
return cls(buf.bytes_view(byte_offset, byte_length),
bit_offset, length)
@@ -1069,6 +1073,7 @@ type_reprs = {
'SparseUnionType': 'sparse_union',
'DenseUnionType': 'dense_union',
'DictionaryType': 'dictionary',
+ 'RunEndEncodedType': 'run_end_encoded',
}
@@ -1170,6 +1175,20 @@ class ListTypePrinter(TypePrinter):
return f"{self._format_type()}({child})"
+class RunEndEncodedTypePrinter(TypePrinter):
+ """
+ Pretty-printer for run-end encoded types.
+ """
+
+ def to_string(self):
+ fields = self.fields
+ if len(fields) != 2:
+ return f"{self._format_type()}<uninitialized or corrupt>"
+ run_end_type = fields[0].type
+ value_type = fields[1].type
+ return f"{self._format_type()}({run_end_type}, {value_type})"
+
+
class FixedSizeListTypePrinter(ListTypePrinter):
"""
Pretty-printer for fixed-size list type.
@@ -1465,6 +1484,18 @@ class BaseListScalarPrinter(ScalarPrinter):
return f"{self._format_type()} of value {value}"
+class RunEndEncodedScalarPrinter(ScalarPrinter):
+ """
+ Pretty-printer for arrow::RunEndEncodedScalar.
+ """
+
+ def to_string(self):
+ if not self.is_valid:
+ return self._format_null()
+ value = deref(self.val['value'])
+ return f"{self._format_type()} of value {value}"
+
+
class StructScalarPrinter(ScalarPrinter):
"""
Pretty-printer for arrow::StructScalar.
@@ -1831,6 +1862,51 @@ class BinaryArrayDataPrinter(ArrayDataPrinter):
yield self._null_child(i)
+class RunEndEncodedArrayDataPrinter(ArrayDataPrinter):
+ """
+ ArrayDataPrinter specialization for run-end encoded arrays.
+ """
+
+ def __init__(self, name, val):
+ if self.length == 0:
+ return
+ child_data = StdVector(self.val['child_data'])
+ self._run_ends_printer = ArrayDataPrinter(
+ "arrow::ArrayData", deref(child_data[0]))
+ self._values_printer = ArrayDataPrinter(
+ "arrow::ArrayData", deref(child_data[1]))
+
+ def display_hint(self):
+ return "array"
+
+ def children(self):
+ if self.length == 0:
+ return
+ run_ends = self._run_ends_printer._unpacked_buffer_values(
+ 1, self._run_ends_printer.type_id)
+ values = iter(self._values_printer.children() or ())
+ run_index = bisect_right(run_ends, self.offset)
+ # Advance to the value for the run containing the logical offset.
+ for _ in range(run_index + 1):
+ value = next(values, None)
+ if value is None:
+ return
+
+ logical_index = self.offset
+ logical_end = self.offset + self.length
+ # Expand each run into the logical elements visible in this slice.
+ while logical_index < logical_end:
+ run_end = run_ends[run_index]
+ for i in range(logical_index, min(run_end, logical_end)):
+ yield self._valid_child(i - self.offset, value[1])
+ logical_index = run_end
+ run_index += 1
+ if logical_index < logical_end:
+ value = next(values, None)
+ if value is None:
+ return
+
+
class ArrayPrinter:
"""
Pretty-printer for arrow::Array and subclasses.
@@ -1999,6 +2075,13 @@ class FixedSizeListTypeClass(DataTypeClass):
scalar_printer = BaseListScalarPrinter
+class RunEndEncodedTypeClass(DataTypeClass):
+ is_parametric = True
+ type_printer = RunEndEncodedTypePrinter
+ scalar_printer = RunEndEncodedScalarPrinter
+ array_data_printer = RunEndEncodedArrayDataPrinter
+
+
class MapTypeClass(DataTypeClass):
is_parametric = True
type_printer = MapTypePrinter
@@ -2093,6 +2176,8 @@ type_traits_by_id = {
Type.DICTIONARY: DataTypeTraits(DictionaryTypeClass, 'DictionaryType'),
Type.EXTENSION: DataTypeTraits(ExtensionTypeClass, 'ExtensionType'),
+ Type.RUN_END_ENCODED: DataTypeTraits(RunEndEncodedTypeClass,
+ 'RunEndEncodedType'),
}
diff --git a/python/pyarrow/src/arrow/python/gdb.cc
b/python/pyarrow/src/arrow/python/gdb.cc
index 4442cc43213..b2434b0a324 100644
--- a/python/pyarrow/src/arrow/python/gdb.cc
+++ b/python/pyarrow/src/arrow/python/gdb.cc
@@ -222,6 +222,9 @@ void TestSession() {
FixedSizeListType fixed_size_list_type(float64(), 3);
auto heap_fixed_size_list_type = fixed_size_list(float64(), 3);
+ RunEndEncodedType run_end_encoded_type(int32(), utf8());
+ auto heap_run_end_encoded_type = run_end_encoded(int32(), utf8());
+
DictionaryType dict_type_unordered(int16(), utf8());
DictionaryType dict_type_ordered(int16(), utf8(), /*ordered=*/true);
auto heap_dict_type = dictionary(int16(), utf8());
@@ -389,6 +392,14 @@ void TestSession() {
FixedSizeListScalar fixed_size_list_scalar_null{
list_value_array, fixed_size_list(int32(), 3), /*is_valid=*/false};
+ auto run_end_encoded_scalar_type = run_end_encoded(int32(), utf8());
+ RunEndEncodedScalar run_end_encoded_scalar{MakeScalar("foo"),
+ run_end_encoded_scalar_type};
+ RunEndEncodedScalar run_end_encoded_scalar_null{run_end_encoded_scalar_type};
+ std::shared_ptr<Scalar> heap_run_end_encoded_scalar =
+ std::make_shared<RunEndEncodedScalar>(MakeScalar("foo"),
+ run_end_encoded_scalar_type);
+
auto struct_scalar_type = struct_({field("ints", int32()), field("strs",
utf8())});
StructScalar struct_scalar{
ScalarVector{MakeScalar(int32_t(42)), MakeScalar("some text")},
struct_scalar_type};
@@ -448,6 +459,24 @@ void TestSession() {
auto heap_list_array = SliceArrayFromJSON(list(int64()), "[[1, 2], null,
[]]");
ListArray list_array{heap_list_array->data()};
+ // Encodes ["foo", "foo", null, null, null].
+ auto run_end_encoded_run_ends = SliceArrayFromJSON(int32(), "[2, 5]");
+ auto run_end_encoded_values = SliceArrayFromJSON(utf8(), R"(["foo", null])");
+ std::shared_ptr<Array> heap_run_end_encoded_array =
*RunEndEncodedArray::Make(
+ /*logical_length=*/5, run_end_encoded_run_ends, run_end_encoded_values);
+ RunEndEncodedArray run_end_encoded_array{heap_run_end_encoded_array->data()};
+ auto heap_run_end_encoded_array_sliced =
heap_run_end_encoded_array->Slice(1, 3);
+
+ // Sliced children
+ std::shared_ptr<Array> heap_ree_sliced = *RunEndEncodedArray::Make(
+ /*logical_length=*/3, SliceArrayFromJSON(int32(), "[1, 2, 5, 0]", 1, 2),
+ SliceArrayFromJSON(utf8(), R"(["bar", "baz", "foo", null, "qux"])", 2,
2),
+ /*logical_offset=*/1);
+
+ std::shared_ptr<Array> heap_ree_int64 = *RunEndEncodedArray::Make(
+ /*logical_length=*/3, SliceArrayFromJSON(int64(), "[1, 3]"),
+ SliceArrayFromJSON(int32(), "[42, null]"));
+
const char* json_double_array = "[-1.5, null]";
auto heap_double_array = SliceArrayFromJSON(float64(), json_double_array);
diff --git a/python/pyarrow/tests/test_gdb.py b/python/pyarrow/tests/test_gdb.py
index cd44e686bb8..f6363ef7f45 100644
--- a/python/pyarrow/tests/test_gdb.py
+++ b/python/pyarrow/tests/test_gdb.py
@@ -407,6 +407,9 @@ def test_types_stack(gdb_arrow):
"arrow::large_list(arrow::large_utf8())")
check_stack_repr(gdb_arrow, "fixed_size_list_type",
"arrow::fixed_size_list(arrow::float64(), 3)")
+ check_stack_repr(
+ gdb_arrow, "run_end_encoded_type",
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8())")
check_stack_repr(
gdb_arrow, "map_type_unsorted",
"arrow::map(arrow::utf8(), arrow::binary(), keys_sorted=false)")
@@ -468,6 +471,9 @@ def test_types_heap(gdb_arrow):
"arrow::large_list(arrow::large_utf8())")
check_heap_repr(gdb_arrow, "heap_fixed_size_list_type",
"arrow::fixed_size_list(arrow::float64(), 3)")
+ check_heap_repr(
+ gdb_arrow, "heap_run_end_encoded_type",
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8())")
check_heap_repr(
gdb_arrow, "heap_map_type",
"arrow::map(arrow::utf8(), arrow::binary(), keys_sorted=false)")
@@ -744,6 +750,14 @@ def test_scalars_stack(gdb_arrow):
gdb_arrow, "fixed_size_list_scalar_null",
('arrow::FixedSizeListScalar of type '
'arrow::fixed_size_list(arrow::int32(), 3), null value'))
+ check_stack_repr(
+ gdb_arrow, "run_end_encoded_scalar",
+ ('arrow::RunEndEncodedScalar of value '
+ 'arrow::StringScalar of size 3, value "foo"'))
+ check_stack_repr(
+ gdb_arrow, "run_end_encoded_scalar_null",
+ ('arrow::RunEndEncodedScalar of type '
+ 'arrow::run_end_encoded(arrow::int32(), arrow::utf8()), null value'))
check_stack_repr(
gdb_arrow, "struct_scalar",
@@ -810,6 +824,10 @@ def test_scalars_heap(gdb_arrow):
gdb_arrow, "heap_map_scalar_null",
('arrow::MapScalar of type arrow::map(arrow::utf8(), arrow::int32(), '
'keys_sorted=false), null value'))
+ check_heap_repr(
+ gdb_arrow, "heap_run_end_encoded_scalar",
+ ('arrow::RunEndEncodedScalar of value '
+ 'arrow::StringScalar of size 3, value "foo"'))
def test_array_data(gdb_arrow):
@@ -828,6 +846,13 @@ def test_arrays_stack(gdb_arrow):
gdb_arrow, "list_array",
("arrow::ListArray of type arrow::list(arrow::int64()), "
"length 3, offset 0, null count 1"))
+ check_stack_repr(
+ gdb_arrow, "run_end_encoded_array",
+ ("arrow::RunEndEncodedArray of type "
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8()), "
+ "length 5, offset 0, null count 0 = "
+ "{[0] = \"foo\", [1] = \"foo\", [2] = null, "
+ "[3] = null, [4] = null}"))
def test_arrays_heap(gdb_arrow):
@@ -1081,6 +1106,31 @@ def test_arrays_heap(gdb_arrow):
gdb_arrow, "heap_list_array",
("arrow::ListArray of type arrow::list(arrow::int64()), "
"length 3, offset 0, null count 1"))
+ check_heap_repr(
+ gdb_arrow, "heap_run_end_encoded_array",
+ ("arrow::RunEndEncodedArray of type "
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8()), "
+ "length 5, offset 0, null count 0 = "
+ "{[0] = \"foo\", [1] = \"foo\", [2] = null, "
+ "[3] = null, [4] = null}"))
+ check_heap_repr(
+ gdb_arrow, "heap_run_end_encoded_array_sliced",
+ ("arrow::RunEndEncodedArray of type "
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8()), "
+ "length 3, offset 1, null count 0 = "
+ "{[0] = \"foo\", [1] = null, [2] = null}"))
+ check_heap_repr(
+ gdb_arrow, "heap_ree_sliced",
+ ("arrow::RunEndEncodedArray of type "
+ "arrow::run_end_encoded(arrow::int32(), arrow::utf8()), "
+ "length 3, offset 1, null count 0 = "
+ "{[0] = \"foo\", [1] = null, [2] = null}"))
+ check_heap_repr(
+ gdb_arrow, "heap_ree_int64",
+ ("arrow::RunEndEncodedArray of type "
+ "arrow::run_end_encoded(arrow::int64(), arrow::int32()), "
+ "length 3, offset 0, null count 0 = "
+ "{[0] = 42, [1] = null, [2] = null}"))
def test_schema(gdb_arrow):
