This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 7a7b414d1f GH-35212: [Go] Add ability to show full call stack with
ARROW_CHECKED_MAX_RETAINED_FRAMES (#35215)
7a7b414d1f is described below
commit 7a7b414d1fd0d648a6d248fb1bbe1984e9585480
Author: Herman Schaaf <[email protected]>
AuthorDate: Tue Apr 18 20:45:41 2023 +0100
GH-35212: [Go] Add ability to show full call stack with
ARROW_CHECKED_MAX_RETAINED_FRAMES (#35215)
This adds support for printing the full call stack when a leak is reported
by the checked memory allocator. An `ARROW_CHECKED_MAX_RETAINED_FRAMES`
environment variable controls how many frames are retained. To keep this
completely backwards-compatible, the default right now is zero. In this case,
the reported error is exactly the same as before. When a higher value is given
though, a longer call stack is printed. For example:
Before (same as `ARROW_CHECKED_MAX_RETAINED_FRAMES=0`):
```
checked_allocator.go:160: LEAK of 64 bytes FROM
github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).newData line 2396
```
After (with `ARROW_CHECKED_MAX_RETAINED_FRAMES=100`):
```
checked_allocator.go:160: LEAK of 64 bytes FROM
github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).newData line 2396
github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).NewTimestampArray
line 2386
github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).NewArray line
2380
github.com/apache/arrow/go/v12/arrow/array.(*RecordBuilder).NewRecord
line 346
github.com/cloudquery/plugin-sdk/v2/testdata.GenTestData line 243
github.com/cloudquery/plugin-sdk/v2/plugins/destination.testMigration
line 53
github.com/cloudquery/plugin-sdk/v2/plugins/destination.(*PluginTestSuite).destinationPluginTestMigrate.func5
line 239
testing.tRunner line 1576
runtime.goexit line 1172
```
* Closes: #35212
Authored-by: Herman Schaaf <[email protected]>
Signed-off-by: Matt Topol <[email protected]>
---
go/arrow/memory/checked_allocator.go | 55 ++++++++++++++++++++++++++++--------
1 file changed, 44 insertions(+), 11 deletions(-)
diff --git a/go/arrow/memory/checked_allocator.go
b/go/arrow/memory/checked_allocator.go
index 06be9bda30..12f92912bb 100644
--- a/go/arrow/memory/checked_allocator.go
+++ b/go/arrow/memory/checked_allocator.go
@@ -17,9 +17,11 @@
package memory
import (
+ "fmt"
"os"
"runtime"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"unsafe"
@@ -46,8 +48,11 @@ func (a *CheckedAllocator) Allocate(size int) []byte {
}
ptr := uintptr(unsafe.Pointer(&out[0]))
+ pcs := make([]uintptr, maxRetainedFrames)
+ runtime.Callers(allocFrames, pcs)
+ callersFrames := runtime.CallersFrames(pcs)
if pc, _, l, ok := runtime.Caller(allocFrames); ok {
- a.allocs.Store(ptr, &dalloc{pc: pc, line: l, sz: size})
+ a.allocs.Store(ptr, &dalloc{pc: pc, line: l, sz: size,
callersFrames: callersFrames})
}
return out
}
@@ -63,9 +68,13 @@ func (a *CheckedAllocator) Reallocate(size int, b []byte)
[]byte {
newptr := uintptr(unsafe.Pointer(&out[0]))
a.allocs.Delete(oldptr)
+ pcs := make([]uintptr, maxRetainedFrames)
+ runtime.Callers(reallocFrames, pcs)
+ callersFrames := runtime.CallersFrames(pcs)
if pc, _, l, ok := runtime.Caller(reallocFrames); ok {
- a.allocs.Store(newptr, &dalloc{pc: pc, line: l, sz: size})
+ a.allocs.Store(newptr, &dalloc{pc: pc, line: l, sz: size,
callersFrames: callersFrames})
}
+
return out
}
@@ -86,14 +95,16 @@ func (a *CheckedAllocator) Free(b []byte) {
// of the inner workings of Buffer in order to find the caller that actually
triggered
// the allocation via a call to Resize/Reserve/etc.
const (
- defAllocFrames = 4
- defReallocFrames = 3
+ defAllocFrames = 4
+ defReallocFrames = 3
+ defMaxRetainedFrames = 0
)
// Use the environment variables ARROW_CHECKED_ALLOC_FRAMES and
ARROW_CHECKED_REALLOC_FRAMES
-// to control how many frames up it checks when storing the caller for
allocations/reallocs
-// when using this to find memory leaks.
-var allocFrames, reallocFrames int = defAllocFrames, defReallocFrames
+// to control how many frames it skips when storing the caller for
allocations/reallocs
+// when using this to find memory leaks. Use ARROW_CHECKED_MAX_RETAINED_FRAMES
to control how
+// many frames are retained for printing the stack trace of a leak.
+var allocFrames, reallocFrames, maxRetainedFrames int = defAllocFrames,
defReallocFrames, defMaxRetainedFrames
func init() {
if val, ok := os.LookupEnv("ARROW_CHECKED_ALLOC_FRAMES"); ok {
@@ -107,12 +118,19 @@ func init() {
reallocFrames = f
}
}
+
+ if val, ok := os.LookupEnv("ARROW_CHECKED_MAX_RETAINED_FRAMES"); ok {
+ if f, err := strconv.Atoi(val); err == nil {
+ maxRetainedFrames = f
+ }
+ }
}
type dalloc struct {
- pc uintptr
- line int
- sz int
+ pc uintptr
+ line int
+ sz int
+ callersFrames *runtime.Frames
}
type TestingT interface {
@@ -124,7 +142,22 @@ func (a *CheckedAllocator) AssertSize(t TestingT, sz int) {
a.allocs.Range(func(_, value interface{}) bool {
info := value.(*dalloc)
f := runtime.FuncForPC(info.pc)
- t.Errorf("LEAK of %d bytes FROM %s line %d\n", info.sz,
f.Name(), info.line)
+ frames := info.callersFrames
+ var callersMsg strings.Builder
+ for {
+ frame, more := frames.Next()
+ if frame.Line == 0 {
+ break
+ }
+ callersMsg.WriteString("\t")
+ callersMsg.WriteString(frame.Function)
+ callersMsg.WriteString(fmt.Sprintf(" line %d",
frame.Line))
+ callersMsg.WriteString("\n")
+ if !more {
+ break
+ }
+ }
+ t.Errorf("LEAK of %d bytes FROM %s line %d\n%v", info.sz,
f.Name(), info.line, callersMsg.String())
return true
})
