zeroshade commented on code in PR #1330:
URL: https://github.com/apache/arrow-go/pull/1330#discussion_r4189567534
##########
arrow/array/compare.go:
##########
@@ -427,13 +432,129 @@ func arrayEqualListOffsets[T listOffset](leftValues,
rightValues arrow.Array,
}) == nil
}
+func arrayApproxEqualListOffsets[T listOffset](leftValues, rightValues
arrow.Array,
+ leftOffsets, rightOffsets []T, leftOffset, rightOffset, length int,
validBits []byte, opt equalOption) bool {
+ if len(validBits) == 0 {
+ validBits = nil
+ }
+ return bitutils.VisitSetBitRuns(validBits, int64(leftOffset),
int64(length),
+ func(pos, runLength int64) error {
+ leftIndex := leftOffset + int(pos)
+ rightIndex := rightOffset + int(pos)
+ for i := range int(runLength) {
+ leftLength := int64(leftOffsets[leftIndex+i+1])
- int64(leftOffsets[leftIndex+i])
+ rightLength :=
int64(rightOffsets[rightIndex+i+1]) - int64(rightOffsets[rightIndex+i])
+ if leftLength != rightLength {
+ return arrow.ErrInvalid
+ }
+ }
+
+ leftStart := int64(leftOffsets[leftIndex])
+ rightStart := int64(rightOffsets[rightIndex])
+ leftEnd := int64(leftOffsets[leftIndex+int(runLength)])
+ rightEnd :=
int64(rightOffsets[rightIndex+int(runLength)])
+ if !sliceApproxEqual(leftValues, leftStart, leftEnd,
rightValues, rightStart, rightEnd, opt) {
+ return arrow.ErrInvalid
+ }
+ return nil
+ }) == nil
+}
+
+func arrayApproxEqualListScalar(left, right *List, opt equalOption) bool {
Review Comment:
Optional: `arrayApproxEqualListScalar`, `…LargeListScalar` (L482) and
`…FixedSizeListScalar` (L501) are identical apart from the receiver type. One
helper over a small interface (`arrow.Array` plus `newListValue(int)
arrow.Array`) would replace all three.
##########
arrow/array/list_equal_benchmark_test.go:
##########
@@ -64,6 +64,40 @@ func BenchmarkListEqual(b *testing.B) {
}
}
+func BenchmarkListApproxEqual(b *testing.B) {
+ const rows = 65536
+ tests := []struct {
+ listType string
+ child arrow.DataType
+ listSize int
+ validity string
+ }{
+ {"list", arrow.PrimitiveTypes.Int32, 16, "all-valid"},
+ {"list", arrow.BinaryTypes.String, 16, "all-valid"},
+ {"large-list", arrow.PrimitiveTypes.Int32, 16, "all-valid"},
+ {"fixed-size-list", arrow.PrimitiveTypes.Int32, 16,
"all-valid"},
+ {"list", arrow.PrimitiveTypes.Int32, 16, "10pct-null"},
+ {"list", arrow.PrimitiveTypes.Int32, 16,
"clustered-10pct-null"},
+ {"list", arrow.PrimitiveTypes.Int32, 16, "alternating-null"},
+ }
+
+ for _, tc := range tests {
+ name := fmt.Sprintf("%s/%s/size=%d/%s", tc.listType,
tc.child.Name(), tc.listSize, tc.validity)
+ b.Run(name, func(b *testing.B) {
+ left := makeListEqualBenchmarkArray(tc.listType,
tc.child, rows, tc.listSize, tc.validity)
+ right := makeListEqualBenchmarkArray(tc.listType,
tc.child, rows, tc.listSize, tc.validity)
+ defer left.Release()
+ defer right.Release()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
Review Comment:
Nit: consider `for b.Loop()`, which would also drop the `b.ResetTimer()`. It
copies `BenchmarkListEqual`'s loop at L60, so fine to leave.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
