Repository: incubator-htrace Updated Branches: refs/heads/master cb2351d29 -> c1e8cc5ac
HTRACE-278. htraced: dump thread stacks and GC statistics when SIGQUIT is sent (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/c1e8cc5a Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/c1e8cc5a Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/c1e8cc5a Branch: refs/heads/master Commit: c1e8cc5ac290c2765bfe309f6310d3db8e5d5d1b Parents: cb2351d Author: Colin Patrick Mccabe <[email protected]> Authored: Tue Oct 13 13:22:14 2015 -0700 Committer: Colin Patrick Mccabe <[email protected]> Committed: Tue Oct 13 13:25:45 2015 -0700 ---------------------------------------------------------------------- .../go/src/org/apache/htrace/common/process.go | 47 +++++++- .../org/apache/htrace/common/process_test.go | 116 +++++++++++++++++++ 2 files changed, 158 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/c1e8cc5a/htrace-htraced/go/src/org/apache/htrace/common/process.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process.go b/htrace-htraced/go/src/org/apache/htrace/common/process.go index d138178..419d6fe 100644 --- a/htrace-htraced/go/src/org/apache/htrace/common/process.go +++ b/htrace-htraced/go/src/org/apache/htrace/common/process.go @@ -24,6 +24,8 @@ import ( "org/apache/htrace/conf" "os" "os/signal" + "runtime" + "runtime/debug" "syscall" ) @@ -51,17 +53,52 @@ func InstallSignalHandlers(cnf *conf.Config) { syscall.SIGBUS, syscall.SIGFPE, syscall.SIGILL, - syscall.SIGQUIT, syscall.SIGSEGV, syscall.SIGTERM, } - sigChan := make(chan os.Signal, len(fatalSigs)) - signal.Notify(sigChan, fatalSigs...) - lg := NewLogger("exit", cnf) + fatalSigChan := make(chan os.Signal, 1) + signal.Notify(fatalSigChan, fatalSigs...) + lg := NewLogger("signal", cnf) go func() { - sig := <-sigChan + sig := <-fatalSigChan lg.Errorf("Terminating on signal: %v\n", sig) lg.Close() os.Exit(1) }() + + sigQuitChan := make(chan os.Signal, 1) + signal.Notify(sigQuitChan, syscall.SIGQUIT) + go func() { + bufSize := 1<<20 + buf := make([]byte, bufSize) + for { + <-sigQuitChan + neededBytes := runtime.Stack(buf, true) + if neededBytes > bufSize { + bufSize = neededBytes + buf = make([]byte, bufSize) + runtime.Stack(buf, true) + } + lg.Info("=== received SIGQUIT ===\n") + lg.Info("=== GOROUTINE STACKS ===\n") + lg.Info(string(buf[:neededBytes])) + lg.Info("\n=== END GOROUTINE STACKS ===\n") + gcs := debug.GCStats{} + debug.ReadGCStats(&gcs) + lg.Info("=== GC STATISTICS ===\n") + lg.Infof("LastGC: %s\n", gcs.LastGC.UTC().String()) + lg.Infof("NumGC: %d\n", gcs.NumGC) + lg.Infof("PauseTotal: %v\n", gcs.PauseTotal) + if gcs.Pause != nil { + pauseStr := "" + prefix := "" + for p := range gcs.Pause { + pauseStr += prefix + gcs.Pause[p].String() + prefix = ", " + } + lg.Infof("Pause History: %s\n", pauseStr) + } + lg.Info("=== END GC STATISTICS ===\n") + } + }() } http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/c1e8cc5a/htrace-htraced/go/src/org/apache/htrace/common/process_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process_test.go b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go new file mode 100644 index 0000000..7609133 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "bufio" + "fmt" + "org/apache/htrace/conf" + "os" + "os/exec" + "strings" + "syscall" + "testing" + "time" +) + +const HTRACED_TEST_HELPER_PROCESS = "HTRACED_TEST_HELPER_PROCESS" + +// This test runs a helper process which will install our htraced signal +// handlers. We will send signals to the subprocess and verify that it has +// caught them and responded appropriately. +func TestSignals(t *testing.T) { + if os.Getenv(HTRACED_TEST_HELPER_PROCESS) == "1" { + runHelperProcess() + os.Exit(0) + } + helper := exec.Command(os.Args[0], "-test.run=TestSignals", "--") + helper.Env = []string { HTRACED_TEST_HELPER_PROCESS + "=1" } + stdoutPipe, err := helper.StdoutPipe() + if err != nil { + panic(fmt.Sprintf("Failed to open pipe to process stdout: %s", + err.Error())) + } + stderrPipe, err := helper.StderrPipe() + if err != nil { + panic(fmt.Sprintf("Failed to open pipe to process stderr: %s", + err.Error())) + } + err = helper.Start() + if err != nil { + t.Fatal("Failed to start command %s: %s\n", os.Args[0], err.Error()) + } + t.Logf("Started suprocess...\n") + done := make(chan interface{}) + go func() { + scanner := bufio.NewScanner(stdoutPipe) + for scanner.Scan() { + text := scanner.Text() + if strings.Contains(text, "=== GOROUTINE STACKS ===") { + break + } + } + t.Logf("Saw 'GOROUTINE STACKS on stdout.' Sending SIGINT.\n") + helper.Process.Signal(syscall.SIGINT) + for scanner.Scan() { + text := scanner.Text() + if strings.Contains(text, "Terminating on signal: SIGINT") { + break + } + } + t.Logf("Saw 'Terminating on signal: SIGINT'. " + + "Helper goroutine exiting.\n") + done<-nil + }() + scanner := bufio.NewScanner(stderrPipe) + for scanner.Scan() { + text := scanner.Text() + if strings.Contains(text, "Signal handler installed.") { + break + } + } + t.Logf("Saw 'Signal handler installed.' Sending SIGINT.") + helper.Process.Signal(syscall.SIGQUIT) + t.Logf("Waiting for helper goroutine to exit.\n") + <-done + t.Logf("Waiting for subprocess to exit.\n") + helper.Wait() + t.Logf("Done.") +} + +// Run the helper process which TestSignals spawns. +func runHelperProcess() { + cnfMap := map[string]string { + conf.HTRACE_LOG_LEVEL: "TRACE", + conf.HTRACE_LOG_PATH: "", // log to stdout + } + cnfBld := conf.Builder{Values: cnfMap, Defaults: conf.DEFAULTS} + cnf, err := cnfBld.Build() + if err != nil { + fmt.Printf("Error building configuration: %s\n", err.Error()) + os.Exit(1) + } + InstallSignalHandlers(cnf) + fmt.Fprintf(os.Stderr, "Signal handler installed.\n") + // Wait for a signal to be delivered + for { + time.Sleep(time.Hour * 100) + } +}
