felipecrv commented on code in PR #37468:
URL: https://github.com/apache/arrow/pull/37468#discussion_r1326377951
##########
go/arrow/array/concat.go:
##########
@@ -355,6 +355,110 @@ func concatOffsets(buffers []*memory.Buffer, byteWidth
int, mem memory.Allocator
}
}
+func sumBufferSizes(buffers []*memory.Buffer) int {
+ outSize := 0
+ for _, b := range buffers {
+ outSize += b.Len()
+ }
+ return outSize
+}
+
+func putListViewOffsets32(in *memory.Buffer, displacement int32, out
*memory.Buffer, outOff int) {
+ if in.Len() == 0 {
+ return
+ }
+ src := arrow.Int32Traits.CastFromBytes(in.Bytes())
+ dst := arrow.Int32Traits.CastFromBytes(out.Bytes())
+ for i, offset := range src {
+ dst[outOff+i] = offset + displacement
+ }
+}
+
+func putListViewOffsets64(in *memory.Buffer, displacement int64, out
*memory.Buffer, outOff int) {
+ if in.Len() == 0 {
+ return
+ }
+ src := arrow.Int64Traits.CastFromBytes(in.Bytes())
+ dst := arrow.Int64Traits.CastFromBytes(out.Bytes())
+ for i, offset := range src {
+ dst[outOff+i] = offset + displacement
+ }
+}
+
+// Concatenate buffers holding list-view offsets into a single buffer of
offsets
+//
+// valueRanges contains the relevant ranges of values in the child array
actually
+// referenced to by the views. Most commonly, these ranges will start from 0,
+// but when that is not the case, we need to adjust the displacement of
offsets.
+// The concatenated child array does not contain values from the beginning
+// if they are not referenced to by any view.
+func concatListViewOffsets(buffers []*memory.Buffer, byteWidth int,
valueRanges []rng, mem memory.Allocator) (*memory.Buffer, error) {
+ outSize := sumBufferSizes(buffers)
+ if byteWidth == 4 && outSize > math.MaxInt32 {
+ return nil, fmt.Errorf("%w: offset overflow while concatenating
arrays", arrow.ErrInvalid)
+ }
+ out := memory.NewResizableBuffer(mem)
+ out.Resize(byteWidth * outSize)
+
+ numChildValues := 0
+ elementsLength := 0
+ for i, b := range buffers {
+ displacement := numChildValues - valueRanges[i].offset
+ if byteWidth == 4 {
+ putListViewOffsets32(b, int32(displacement), out,
elementsLength)
+ } else {
+ putListViewOffsets64(b, int64(displacement), out,
elementsLength)
+ }
+ elementsLength += b.Len() / byteWidth
+ numChildValues += valueRanges[i].len
+ }
+ debug.Assert(elementsLength == outSize/byteWidth, "implementation
error")
+
+ return out, nil
+}
+
+func concatListView(data []arrow.ArrayData, offsetType
arrow.FixedWidthDataType, out *Data, mem memory.Allocator) error {
+ var err error
+ dt := out.dtype
+ offsetWidth := dt.Layout().Buffers[1].ByteWidth
Review Comment:
Yep. Cleaning it up now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
