Repository: incubator-htrace Updated Branches: refs/heads/master df7dad6f8 -> 2fdef0929
HTRACE-291. rename bin/htrace to bin/htracedTool (cmccabe) Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/2fdef092 Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/2fdef092 Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/2fdef092 Branch: refs/heads/master Commit: 2fdef092910872b7fb7bea14f502694dc28b4212 Parents: df7dad6 Author: Colin Patrick Mccabe <[email protected]> Authored: Wed Nov 4 16:26:21 2015 -0800 Committer: Colin Patrick Mccabe <[email protected]> Committed: Wed Nov 4 16:28:49 2015 -0800 ---------------------------------------------------------------------- htrace-c/src/CMakeLists.txt | 2 +- htrace-c/src/test/htraced_rcv-unit.c | 2 +- htrace-c/src/test/mini_htraced.c | 4 +- htrace-c/src/test/test_config.h.cmake | 4 +- .../go/src/org/apache/htrace/htrace/cmd.go | 418 ------------------- .../go/src/org/apache/htrace/htrace/file.go | 138 ------ .../src/org/apache/htrace/htrace/file_test.go | 161 ------- .../go/src/org/apache/htrace/htrace/graph.go | 116 ----- .../src/org/apache/htrace/htrace/graph_test.go | 80 ---- .../go/src/org/apache/htrace/htrace/queries.go | 172 -------- .../src/org/apache/htrace/htrace/query_test.go | 88 ---- .../go/src/org/apache/htrace/htracedTool/cmd.go | 418 +++++++++++++++++++ .../src/org/apache/htrace/htracedTool/file.go | 138 ++++++ .../org/apache/htrace/htracedTool/file_test.go | 161 +++++++ .../src/org/apache/htrace/htracedTool/graph.go | 116 +++++ .../org/apache/htrace/htracedTool/graph_test.go | 80 ++++ .../org/apache/htrace/htracedTool/queries.go | 172 ++++++++ .../org/apache/htrace/htracedTool/query_test.go | 88 ++++ 18 files changed, 1179 insertions(+), 1179 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-c/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/htrace-c/src/CMakeLists.txt b/htrace-c/src/CMakeLists.txt index 8c0d235..815962b 100644 --- a/htrace-c/src/CMakeLists.txt +++ b/htrace-c/src/CMakeLists.txt @@ -44,7 +44,7 @@ INCLUDE(CheckCSourceCompiles) CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_IMPROVED_TLS) CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/util/build.h.cmake ${CMAKE_BINARY_DIR}/util/build.h) -get_filename_component(HTRACE_ABSPATH "../../htrace-htraced/go/build/htrace" ABSOLUTE) +get_filename_component(HTRACED_TOOL_ABSPATH "../../htrace-htraced/go/build/htracedTool" ABSOLUTE) get_filename_component(HTRACED_ABSPATH "../../htrace-htraced/go/build/htraced" ABSOLUTE) CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/test/test_config.h.cmake ${CMAKE_BINARY_DIR}/test/test_config.h) http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-c/src/test/htraced_rcv-unit.c ---------------------------------------------------------------------- diff --git a/htrace-c/src/test/htraced_rcv-unit.c b/htrace-c/src/test/htraced_rcv-unit.c index d5c83ec..521a03d 100644 --- a/htrace-c/src/test/htraced_rcv-unit.c +++ b/htrace-c/src/test/htraced_rcv-unit.c @@ -66,7 +66,7 @@ static int htraced_rcv_test(struct rtest *rt) while (1) { int nspans; - // This uses the bin/htrace program to dump the spans to a json file. + // This uses the bin/htracedTool program to dump the spans to a json file. mini_htraced_dump_spans(ht, err, err_len, json_path); EXPECT_STR_EQ("", err); st = span_table_alloc(); http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-c/src/test/mini_htraced.c ---------------------------------------------------------------------- diff --git a/htrace-c/src/test/mini_htraced.c b/htrace-c/src/test/mini_htraced.c index 9a042fe..ba3f7a9 100644 --- a/htrace-c/src/test/mini_htraced.c +++ b/htrace-c/src/test/mini_htraced.c @@ -488,7 +488,7 @@ void mini_htraced_dump_spans(struct mini_htraced *ht, return; } ht->num_htrace_commands_run++; - pid = mini_htraced_launch(ht, HTRACE_ABSPATH, err, err_len, 0, + pid = mini_htraced_launch(ht, HTRACED_TOOL_ABSPATH, err, err_len, 0, addr, log_path, "dumpAll", path, NULL); free(addr); free(log_path); @@ -501,7 +501,7 @@ void mini_htraced_dump_spans(struct mini_htraced *ht, } if (ret != EXIT_SUCCESS) { snprintf(err, err_len, "%s returned non-zero exit status %d\n", - HTRACE_ABSPATH, ret); + HTRACED_TOOL_ABSPATH, ret); return; } } http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-c/src/test/test_config.h.cmake ---------------------------------------------------------------------- diff --git a/htrace-c/src/test/test_config.h.cmake b/htrace-c/src/test/test_config.h.cmake index 62b1744..8cfa4b6 100644 --- a/htrace-c/src/test/test_config.h.cmake +++ b/htrace-c/src/test/test_config.h.cmake @@ -19,8 +19,8 @@ #ifndef APACHE_HTRACE_TEST_TEST_CONFIG_H #define APACHE_HTRACE_TEST_TEST_CONFIG_H -// The absolute path to the htrace binary, for use in unit tests. -#cmakedefine HTRACE_ABSPATH "@HTRACE_ABSPATH@" +// The absolute path to the htracedTool binary, for use in unit tests. +#cmakedefine HTRACED_TOOL_ABSPATH "@HTRACED_TOOL_ABSPATH@" // The absolute path to the htraced binary, for use in unit tests. #cmakedefine HTRACED_ABSPATH "@HTRACED_ABSPATH@" http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go deleted file mode 100644 index 98b1646..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/alecthomas/kingpin" - "io" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "sort" - "strings" - "text/tabwriter" - "time" -) - -var RELEASE_VERSION string -var GIT_VERSION string - -const EXIT_SUCCESS = 0 -const EXIT_FAILURE = 1 - -var verbose bool - -const USAGE = `The Apache HTrace command-line tool. This tool retrieves and modifies settings and -other data on a running htraced daemon. - -If we find an ` + conf.CONFIG_FILE_NAME + ` configuration file in the list of directories -specified in ` + conf.HTRACED_CONF_DIR + `, we will use that configuration; otherwise, -the defaults will be used. -` - -func main() { - // Load htraced configuration - cnf, cnfLog := conf.LoadApplicationConfig("htrace.tool.") - lg := common.NewLogger("conf", cnf) - defer lg.Close() - scanner := bufio.NewScanner(cnfLog) - for scanner.Scan() { - lg.Debugf(scanner.Text() + "\n") - } - - // Parse argv - app := kingpin.New(os.Args[0], USAGE) - app.Flag("Dmy.key", "Set configuration key 'my.key' to 'my.value'. Replace 'my.key' "+ - "with any key you want to set.").Default("my.value").String() - addr := app.Flag("addr", "Server address.").String() - verbose = *app.Flag("verbose", "Verbose.").Default("false").Bool() - version := app.Command("version", "Print the version of this program.") - serverVersion := app.Command("serverVersion", "Print the version of the htraced server.") - serverStats := app.Command("serverStats", "Print statistics retrieved from the htraced server.") - serverStatsJson := serverStats.Flag("json", "Display statistics as raw JSON.").Default("false").Bool() - serverConf := app.Command("serverConf", "Print the server configuration retrieved from the htraced server.") - findSpan := app.Command("findSpan", "Print information about a trace span with a given ID.") - findSpanId := findSpan.Arg("id", "Span ID to find. Example: be305e54-4534-2110-a0b2-e06b9effe112").Required().String() - findChildren := app.Command("findChildren", "Print out the span IDs that are children of a given span ID.") - parentSpanId := findChildren.Arg("id", "Span ID to print children for. Example: be305e54-4534-2110-a0b2-e06b9effe112"). - Required().String() - childLim := findChildren.Flag("lim", "Maximum number of child IDs to print.").Default("20").Int() - loadFile := app.Command("loadFile", "Write whitespace-separated JSON spans from a file to the server.") - loadFilePath := loadFile.Arg("path", - "A file containing whitespace-separated span JSON.").Required().String() - loadJson := app.Command("load", "Write JSON spans from the command-line to the server.") - loadJsonArg := loadJson.Arg("json", "A JSON span to write to the server.").Required().String() - dumpAll := app.Command("dumpAll", "Dump all spans from the htraced daemon.") - dumpAllOutPath := dumpAll.Arg("path", "The path to dump the trace spans to.").Default("-").String() - dumpAllLim := dumpAll.Flag("lim", "The number of spans to transfer from the server at once."). - Default("100").Int() - graph := app.Command("graph", "Visualize span JSON as a graph.") - graphJsonFile := graph.Arg("input", "The JSON file to load").Required().String() - graphDotFile := graph.Flag("output", - "The path to write a GraphViz dotfile to. This file can be used as input to "+ - "GraphViz, in order to generate a pretty picture. See graphviz.org for more "+ - "information about generating pictures of graphs.").Default("-").String() - query := app.Command("query", "Send a query to htraced.") - queryLim := query.Flag("lim", "Maximum number of spans to retrieve.").Default("20").Int() - queryArg := query.Arg("query", "The query string to send. Query strings have the format "+ - "[TYPE] [OPERATOR] [CONST], joined by AND statements.").Required().String() - rawQuery := app.Command("rawQuery", "Send a raw JSON query to htraced.") - rawQueryArg := rawQuery.Arg("json", "The query JSON to send.").Required().String() - cmd := kingpin.MustParse(app.Parse(os.Args[1:])) - - // Add the command-line settings into the configuration. - if *addr != "" { - cnf = cnf.Clone(conf.HTRACE_WEB_ADDRESS, *addr) - } - - // Handle commands that don't require an HTrace client. - switch cmd { - case version.FullCommand(): - os.Exit(printVersion()) - case graph.FullCommand(): - err := jsonSpanFileToDotFile(*graphJsonFile, *graphDotFile) - if err != nil { - fmt.Printf("graphing error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - } - - // Create HTrace client - hcl, err := htrace.NewClient(cnf) - if err != nil { - fmt.Printf("Failed to create HTrace client: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - - // Handle commands that require an HTrace client. - switch cmd { - case version.FullCommand(): - os.Exit(printVersion()) - case serverVersion.FullCommand(): - os.Exit(printServerVersion(hcl)) - case serverStats.FullCommand(): - if *serverStatsJson { - os.Exit(printServerStatsJson(hcl)) - } else { - os.Exit(printServerStats(hcl)) - } - case serverConf.FullCommand(): - os.Exit(printServerConfJson(hcl)) - case findSpan.FullCommand(): - var id *common.SpanId - id.FromString(*findSpanId) - os.Exit(doFindSpan(hcl, *id)) - case findChildren.FullCommand(): - var id *common.SpanId - id.FromString(*parentSpanId) - os.Exit(doFindChildren(hcl, *id, *childLim)) - case loadJson.FullCommand(): - os.Exit(doLoadSpanJson(hcl, *loadJsonArg)) - case loadFile.FullCommand(): - os.Exit(doLoadSpanJsonFile(hcl, *loadFilePath)) - case dumpAll.FullCommand(): - err := doDumpAll(hcl, *dumpAllOutPath, *dumpAllLim) - if err != nil { - fmt.Printf("dumpAll error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - case query.FullCommand(): - err := doQueryFromString(hcl, *queryArg, *queryLim) - if err != nil { - fmt.Printf("query error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - case rawQuery.FullCommand(): - err := doRawQuery(hcl, *rawQueryArg) - if err != nil { - fmt.Printf("raw query error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - } - - app.UsageErrorf(os.Stderr, "You must supply a command to run.") -} - -// Print the version of the htrace binary. -func printVersion() int { - fmt.Printf("Running htrace command version %s.\n", RELEASE_VERSION) - return EXIT_SUCCESS -} - -// Print information retrieved from an htraced server via /server/info -func printServerVersion(hcl *htrace.Client) int { - ver, err := hcl.GetServerVersion() - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - fmt.Printf("HTraced server version %s (%s)\n", ver.ReleaseVersion, ver.GitVersion) - return EXIT_SUCCESS -} - -// Print information retrieved from an htraced server via /server/info -func printServerStats(hcl *htrace.Client) int { - stats, err := hcl.GetServerStats() - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - w := new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(w, "HTRACED SERVER STATS\n") - fmt.Fprintf(w, "Datastore Start\t%s\n", - common.UnixMsToTime(stats.LastStartMs).Format(time.RFC3339)) - fmt.Fprintf(w, "Server Time\t%s\n", - common.UnixMsToTime(stats.CurMs).Format(time.RFC3339)) - fmt.Fprintf(w, "Spans reaped\t%d\n", stats.ReapedSpans) - fmt.Fprintf(w, "Number of leveldb directories\t%d\n", len(stats.Dirs)) - w.Flush() - fmt.Println("") - for i := range stats.Dirs { - dir := stats.Dirs[i] - fmt.Printf("==== %s ===\n", dir.Path) - fmt.Printf("Approximate number of spans: %d\n", dir.ApproxNumSpans) - stats := strings.Replace(dir.LevelDbStats, "\\n", "\n", -1) - fmt.Printf("%s\n", stats) - } - w = new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(w, "HOST SPAN METRICS\n") - mtxMap := stats.HostSpanMetrics - keys := make(sort.StringSlice, len(mtxMap)) - i := 0 - for k, _ := range mtxMap { - keys[i] = k - i++ - } - sort.Sort(keys) - for k := range keys { - mtx := mtxMap[keys[k]] - fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\tclient dropped: %d\n", - keys[k], mtx.Written, mtx.ServerDropped, mtx.ClientDropped) - } - w.Flush() - return EXIT_SUCCESS -} - -// Print information retrieved from an htraced server via /server/info as JSON -func printServerStatsJson(hcl *htrace.Client) int { - stats, err := hcl.GetServerStats() - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - buf, err := json.MarshalIndent(stats, "", " ") - if err != nil { - fmt.Printf("Error marshalling server stats: %s", err.Error()) - return EXIT_FAILURE - } - fmt.Printf("%s\n", string(buf)) - return EXIT_SUCCESS -} - -// Print information retrieved from an htraced server via /server/conf as JSON -func printServerConfJson(hcl *htrace.Client) int { - cnf, err := hcl.GetServerConf() - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - buf, err := json.MarshalIndent(cnf, "", " ") - if err != nil { - fmt.Printf("Error marshalling server conf: %s", err.Error()) - return EXIT_FAILURE - } - fmt.Printf("%s\n", string(buf)) - return EXIT_SUCCESS -} - -// Print information about a trace span. -func doFindSpan(hcl *htrace.Client, sid common.SpanId) int { - span, err := hcl.FindSpan(sid) - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - if span == nil { - fmt.Printf("Span ID not found.\n") - return EXIT_FAILURE - } - pbuf, err := json.MarshalIndent(span, "", " ") - if err != nil { - fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error()) - return EXIT_FAILURE - } - fmt.Printf("%s\n", string(pbuf)) - return EXIT_SUCCESS -} - -func doLoadSpanJsonFile(hcl *htrace.Client, spanFile string) int { - if spanFile == "" { - fmt.Printf("You must specify the json file to load.\n") - return EXIT_FAILURE - } - file, err := OpenInputFile(spanFile) - if err != nil { - fmt.Printf("Failed to open %s: %s\n", spanFile, err.Error()) - return EXIT_FAILURE - } - defer file.Close() - return doLoadSpans(hcl, bufio.NewReader(file)) -} - -func doLoadSpanJson(hcl *htrace.Client, spanJson string) int { - return doLoadSpans(hcl, bytes.NewBufferString(spanJson)) -} - -func doLoadSpans(hcl *htrace.Client, reader io.Reader) int { - dec := json.NewDecoder(reader) - spans := make([]*common.Span, 0, 32) - var err error - for { - var span common.Span - if err = dec.Decode(&span); err != nil { - if err == io.EOF { - break - } - fmt.Printf("Failed to decode JSON: %s\n", err.Error()) - return EXIT_FAILURE - } - spans = append(spans, &span) - } - if verbose { - fmt.Printf("Writing ") - prefix := "" - for i := range spans { - fmt.Printf("%s%s", prefix, spans[i].ToJson()) - prefix = ", " - } - fmt.Printf("\n") - } - err = hcl.WriteSpans(&common.WriteSpansReq{ - Spans: spans, - }) - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - return EXIT_SUCCESS -} - -// Find information about the children of a span. -func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int { - spanIds, err := hcl.FindChildren(sid, lim) - if err != nil { - fmt.Printf("%s\n", err.Error()) - return EXIT_FAILURE - } - pbuf, err := json.MarshalIndent(spanIds, "", " ") - if err != nil { - fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error()) - return 1 - } - fmt.Printf("%s\n", string(pbuf)) - return 0 -} - -// Dump all spans from the htraced daemon. -func doDumpAll(hcl *htrace.Client, outPath string, lim int) error { - file, err := CreateOutputFile(outPath) - if err != nil { - return err - } - w := bufio.NewWriter(file) - defer func() { - if file != nil { - w.Flush() - file.Close() - } - }() - out := make(chan *common.Span, 50) - var dumpErr error - go func() { - dumpErr = hcl.DumpAll(lim, out) - }() - var numSpans int64 - nextLogTime := time.Now().Add(time.Second * 5) - for { - span, channelOpen := <-out - if !channelOpen { - break - } - if err == nil { - _, err = fmt.Fprintf(w, "%s\n", span.ToJson()) - } - if verbose { - numSpans++ - now := time.Now() - if !now.Before(nextLogTime) { - nextLogTime = now.Add(time.Second * 5) - fmt.Printf("received %d span(s)...\n", numSpans) - } - } - } - if err != nil { - return errors.New(fmt.Sprintf("Write error %s", err.Error())) - } - if dumpErr != nil { - return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error())) - } - err = w.Flush() - if err != nil { - return err - } - err = file.Close() - file = nil - if err != nil { - return err - } - return nil -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/file.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/file.go b/htrace-htraced/go/src/org/apache/htrace/htrace/file.go deleted file mode 100644 index ea214be..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/file.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "encoding/json" - "errors" - "fmt" - "io" - "org/apache/htrace/common" - "os" -) - -// A file used for input. -// Transparently supports using stdin for input. -type InputFile struct { - *os.File - path string -} - -// Open an input file. Stdin will be used when path is - -func OpenInputFile(path string) (*InputFile, error) { - if path == "-" { - return &InputFile{File: os.Stdin, path: path}, nil - } - file, err := os.Open(path) - if err != nil { - return nil, err - } - return &InputFile{File: file, path: path}, nil -} - -func (file *InputFile) Close() { - if file.path != "-" { - file.File.Close() - } -} - -// A file used for output. -// Transparently supports using stdout for output. -type OutputFile struct { - *os.File - path string -} - -// Create an output file. Stdout will be used when path is - -func CreateOutputFile(path string) (*OutputFile, error) { - if path == "-" { - return &OutputFile{File: os.Stdout, path: path}, nil - } - file, err := os.Create(path) - if err != nil { - return nil, err - } - return &OutputFile{File: file, path: path}, nil -} - -func (file *OutputFile) Close() error { - if file.path != "-" { - return file.File.Close() - } - return nil -} - -// FailureDeferringWriter is a writer which allows us to call Printf multiple -// times and then check if all the printfs succeeded at the very end, rather -// than checking after each call. We will not attempt to write more data -// after the first write failure. -type FailureDeferringWriter struct { - io.Writer - err error -} - -func NewFailureDeferringWriter(writer io.Writer) *FailureDeferringWriter { - return &FailureDeferringWriter{writer, nil} -} - -func (w *FailureDeferringWriter) Printf(format string, v ...interface{}) { - if w.err != nil { - return - } - str := fmt.Sprintf(format, v...) - _, err := w.Writer.Write([]byte(str)) - if err != nil { - w.err = err - } -} - -func (w *FailureDeferringWriter) Error() error { - return w.err -} - -// Read a file full of whitespace-separated span JSON into a slice of spans. -func readSpansFile(path string) (common.SpanSlice, error) { - file, err := OpenInputFile(path) - if err != nil { - return nil, err - } - defer file.Close() - return readSpans(bufio.NewReader(file)) -} - -// Read whitespace-separated span JSON into a slice of spans. -func readSpans(reader io.Reader) (common.SpanSlice, error) { - spans := make(common.SpanSlice, 0) - dec := json.NewDecoder(reader) - for { - var span common.Span - err := dec.Decode(&span) - if err != nil { - if err != io.EOF { - return nil, errors.New(fmt.Sprintf("Decode error after decoding %d "+ - "span(s): %s", len(spans), err.Error())) - } - break - } - spans = append(spans, &span) - } - return spans, nil -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go b/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go deleted file mode 100644 index 98e5e6c..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "errors" - "io" - "io/ioutil" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "strings" - "testing" -) - -func TestInputFileAndOutputFile(t *testing.T) { - tdir, err := ioutil.TempDir(os.TempDir(), "TestInputFileAndOutputFile") - if err != nil { - t.Fatalf("failed to create TempDir: %s\n", err.Error()) - } - defer os.RemoveAll(tdir) - tpath := tdir + conf.PATH_SEP + "test" - var ofile *OutputFile - ofile, err = CreateOutputFile(tpath) - if err != nil { - t.Fatalf("failed to create OutputFile at %s: %s\n", tpath, err.Error()) - } - defer func() { - if ofile != nil { - ofile.Close() - } - }() - w := NewFailureDeferringWriter(ofile) - w.Printf("Hello, world!\n") - w.Printf("2 + 2 = %d\n", 4) - if w.Error() != nil { - t.Fatalf("got unexpected error writing to %s: %s\n", tpath, w.Error().Error()) - } - err = ofile.Close() - ofile = nil - if err != nil { - t.Fatalf("error on closing OutputFile for %s: %s\n", tpath, err.Error()) - } - var ifile *InputFile - ifile, err = OpenInputFile(tpath) - defer ifile.Close() - expected := "Hello, world!\n2 + 2 = 4\n" - buf := make([]byte, len(expected)) - _, err = io.ReadAtLeast(ifile, buf, len(buf)) - if err != nil { - t.Fatalf("unexpected error on reading %s: %s\n", tpath, err.Error()) - } - str := string(buf) - if str != expected { - t.Fatalf("Could not read back what we wrote to %s.\n"+ - "Got:\n%s\nExpected:\n%s\n", tpath, str, expected) - } -} - -type LimitedBufferWriter struct { - buf []byte - off int -} - -const LIMITED_BUFFER_MESSAGE = "There isn't enough buffer to go around!" - -func (w *LimitedBufferWriter) Write(p []byte) (int, error) { - var nwritten int - for i := range p { - if w.off >= len(w.buf) { - return nwritten, errors.New(LIMITED_BUFFER_MESSAGE) - } - w.buf[w.off] = p[i] - w.off = w.off + 1 - nwritten++ - } - return nwritten, nil -} - -func TestFailureDeferringWriter(t *testing.T) { - lw := LimitedBufferWriter{buf: make([]byte, 20), off: 0} - w := NewFailureDeferringWriter(&lw) - w.Printf("Zippity do dah #%d\n", 1) - w.Printf("Zippity do dah #%d\n", 2) - if w.Error() == nil { - t.Fatalf("expected FailureDeferringWriter to experience a failure due to " + - "limited buffer size, but it did not.") - } - if w.Error().Error() != LIMITED_BUFFER_MESSAGE { - t.Fatalf("expected FailureDeferringWriter to have the error message %s, but "+ - "the message was %s\n", LIMITED_BUFFER_MESSAGE, w.Error().Error()) - } - expected := "Zippity do dah #1\nZi" - if string(lw.buf) != expected { - t.Fatalf("expected LimitedBufferWriter to contain %s, but it contained %s "+ - "instead.\n", expected, string(lw.buf)) - } -} - -func TestReadSpans(t *testing.T) { - SPAN_TEST_STR := `{"a":"b9f2a1e07b6e4f16b0c2b27303b20e79",` + - `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + - `"r":"FsShell","p":["3afebdc0a13f4feb811cc5c0e42d30b1"]} -{"a":"3afebdc0a13f4feb811cc5c0e42d30b1","b":1424736224969,` + - `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} -` - r := strings.NewReader(SPAN_TEST_STR) - spans, err := readSpans(r) - if err != nil { - t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) - } - SPAN_TEST_EXPECTED := common.SpanSlice{ - &common.Span{ - Id: common.TestId("b9f2a1e07b6e4f16b0c2b27303b20e79"), - SpanData: common.SpanData{ - Begin: 1424736225037, - End: 1424736225901, - Description: "ClientNamenodeProtocol#getFileInfo", - TracerId: "FsShell", - Parents: []common.SpanId{common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1")}, - }, - }, - &common.Span{ - Id: common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1"), - SpanData: common.SpanData{ - Begin: 1424736224969, - End: 1424736225960, - Description: "getFileInfo", - TracerId: "FsShell", - Parents: []common.SpanId{}, - Info: common.TraceInfoMap{ - "path": "/", - }, - }, - }, - } - if len(spans) != len(SPAN_TEST_EXPECTED) { - t.Fatalf("Expected %d spans, but got %d\n", - len(SPAN_TEST_EXPECTED), len(spans)) - } - for i := range SPAN_TEST_EXPECTED { - common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go b/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go deleted file mode 100644 index 024d973..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "errors" - "fmt" - "io" - "org/apache/htrace/common" - "os" - "sort" -) - -// Create a dotfile from a json file. -func jsonSpanFileToDotFile(jsonFile string, dotFile string) error { - spans, err := readSpansFile(jsonFile) - if err != nil { - return errors.New(fmt.Sprintf("error reading %s: %s", - jsonFile, err.Error())) - } - var file *OutputFile - file, err = CreateOutputFile(dotFile) - if err != nil { - return errors.New(fmt.Sprintf("error opening %s for write: %s", - dotFile, err.Error())) - } - defer func() { - if file != nil { - file.Close() - } - }() - writer := bufio.NewWriter(file) - err = spansToDot(spans, writer) - if err != nil { - return err - } - err = writer.Flush() - if err != nil { - return err - } - err = file.Close() - file = nil - return err -} - -// Create output in dotfile format from a set of spans. -func spansToDot(spans common.SpanSlice, writer io.Writer) error { - sort.Sort(spans) - idMap := make(map[[16]byte]*common.Span) - for i := range spans { - span := spans[i] - if idMap[span.Id.ToArray()] != nil { - fmt.Fprintf(os.Stderr, "There were multiple spans listed which "+ - "had ID %s.\nFirst:%s\nOther:%s\n", span.Id.String(), - idMap[span.Id.ToArray()].ToJson(), span.ToJson()) - } else { - idMap[span.Id.ToArray()] = span - } - } - childMap := make(map[[16]byte]common.SpanSlice) - for i := range spans { - child := spans[i] - for j := range child.Parents { - parent := idMap[child.Parents[j].ToArray()] - if parent == nil { - fmt.Fprintf(os.Stderr, "Can't find parent id %s for %s\n", - child.Parents[j].String(), child.ToJson()) - } else { - children := childMap[parent.Id.ToArray()] - if children == nil { - children = make(common.SpanSlice, 0) - } - children = append(children, child) - childMap[parent.Id.ToArray()] = children - } - } - } - w := NewFailureDeferringWriter(writer) - w.Printf("digraph spans {\n") - // Write out the nodes with their descriptions. - for i := range spans { - w.Printf(fmt.Sprintf(` "%s" [label="%s"];`+"\n", - spans[i].Id.String(), spans[i].Description)) - } - // Write out the edges between nodes... the parent/children relationships - for i := range spans { - children := childMap[spans[i].Id.ToArray()] - sort.Sort(children) - if children != nil { - for c := range children { - w.Printf(fmt.Sprintf(` "%s" -> "%s";`+"\n", - spans[i].Id.String(), children[c].Id)) - } - } - } - w.Printf("}\n") - return w.Error() -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go b/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go deleted file mode 100644 index 621b3dc..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bytes" - "org/apache/htrace/common" - "testing" -) - -func TestSpansToDot(t *testing.T) { - TEST_SPANS := common.SpanSlice{ - &common.Span{ - Id: common.TestId("814c8ee0e7984be3a8af00ac64adccb6"), - SpanData: common.SpanData{ - Begin: 1424813349020, - End: 1424813349134, - Description: "newDFSInputStream", - TracerId: "FsShell", - Parents: []common.SpanId{}, - Info: common.TraceInfoMap{ - "path": "/", - }, - }, - }, - &common.Span{ - Id: common.TestId("cf2d5de696454548bc055d1e6024054c"), - SpanData: common.SpanData{ - Begin: 1424813349025, - End: 1424813349133, - Description: "getBlockLocations", - TracerId: "FsShell", - Parents: []common.SpanId{common.TestId("814c8ee0e7984be3a8af00ac64adccb6")}, - }, - }, - &common.Span{ - Id: common.TestId("37623806f9c64483b834b8ea5d6b4827"), - SpanData: common.SpanData{ - Begin: 1424813349027, - End: 1424813349073, - Description: "ClientNamenodeProtocol#getBlockLocations", - TracerId: "FsShell", - Parents: []common.SpanId{common.TestId("cf2d5de696454548bc055d1e6024054c")}, - }, - }, - } - w := bytes.NewBuffer(make([]byte, 0, 2048)) - err := spansToDot(TEST_SPANS, w) - if err != nil { - t.Fatalf("spansToDot failed: error %s\n", err.Error()) - } - EXPECTED_STR := `digraph spans { - "37623806f9c64483b834b8ea5d6b4827" [label="ClientNamenodeProtocol#getBlockLocations"]; - "814c8ee0e7984be3a8af00ac64adccb6" [label="newDFSInputStream"]; - "cf2d5de696454548bc055d1e6024054c" [label="getBlockLocations"]; - "814c8ee0e7984be3a8af00ac64adccb6" -> "cf2d5de696454548bc055d1e6024054c"; - "cf2d5de696454548bc055d1e6024054c" -> "37623806f9c64483b834b8ea5d6b4827"; -} -` - if w.String() != EXPECTED_STR { - t.Fatalf("Expected to get:\n%s\nGot:\n%s\n", EXPECTED_STR, w.String()) - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go b/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go deleted file mode 100644 index 442df4f..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "encoding/json" - "errors" - "fmt" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "strings" - "unicode" -) - -// Convert a string into a whitespace-separated sequence of strings. -func tokenize(str string) []string { - prevQuote := rune(0) - f := func(c rune) bool { - switch { - case c == prevQuote: - prevQuote = rune(0) - return true - case prevQuote != rune(0): - return false - case unicode.In(c, unicode.Quotation_Mark): - prevQuote = c - return true - default: - return unicode.IsSpace(c) - } - } - return strings.FieldsFunc(str, f) -} - -// Parses a query string in the format of a series of -// [TYPE] [OPERATOR] [CONST] tuples, joined by AND statements. -type predicateParser struct { - tokens []string - curToken int -} - -func (ps *predicateParser) Parse() (*common.Predicate, error) { - if ps.curToken >= len(ps.tokens) { - return nil, nil - } - if ps.curToken > 0 { - if strings.ToLower(ps.tokens[ps.curToken]) != "and" { - return nil, errors.New(fmt.Sprintf("Error parsing on token %d: "+ - "expected predicates to be joined by 'and', but found '%s'", - ps.curToken, ps.tokens[ps.curToken])) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after 'and' at "+ - "token %d", ps.curToken)) - } - } - field := common.Field(strings.ToLower(ps.tokens[ps.curToken])) - if !field.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid field specifier at token %d. "+ - "Can't understand %s. Valid field specifiers are %v", ps.curToken, - ps.tokens[ps.curToken], common.ValidFields())) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ - "at token %d", ps.curToken)) - } - op := common.Op(strings.ToLower(ps.tokens[ps.curToken])) - if !op.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid operation specifier at token %d. "+ - "Can't understand %s. Valid operation specifiers are %v", ps.curToken, - ps.tokens[ps.curToken], common.ValidOps())) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ - "at token %d", ps.curToken)) - } - val := ps.tokens[ps.curToken] - ps.curToken++ - return &common.Predicate{Op: op, Field: field, Val: val}, nil -} - -func parseQueryString(str string) ([]common.Predicate, error) { - ps := predicateParser{tokens: tokenize(str)} - if verbose { - fmt.Printf("Running query [ ") - prefix := "" - for tokenIdx := range(ps.tokens) { - fmt.Printf("%s'%s'", prefix, ps.tokens[tokenIdx]) - prefix = ", " - } - fmt.Printf(" ]\n") - } - preds := make([]common.Predicate, 0) - for { - pred, err := ps.Parse() - if err != nil { - return nil, err - } - if pred == nil { - break - } - preds = append(preds, *pred) - } - if len(preds) == 0 { - return nil, errors.New("Empty query string") - } - return preds, nil -} - -// Send a query from a query string. -func doQueryFromString(hcl *htrace.Client, str string, lim int) error { - query := &common.Query{Lim: lim} - var err error - query.Predicates, err = parseQueryString(str) - if err != nil { - return err - } - return doQuery(hcl, query) -} - -// Send a query from a raw JSON string. -func doRawQuery(hcl *htrace.Client, str string) error { - jsonBytes := []byte(str) - var query common.Query - err := json.Unmarshal(jsonBytes, &query) - if err != nil { - return errors.New(fmt.Sprintf("Error parsing provided JSON: %s\n", err.Error())) - } - return doQuery(hcl, &query) -} - -// Send a query. -func doQuery(hcl *htrace.Client, query *common.Query) error { - if verbose { - qbytes, err := json.Marshal(*query) - if err != nil { - qbytes = []byte("marshaling error: " + err.Error()) - } - fmt.Printf("Sending query: %s\n", string(qbytes)) - } - spans, err := hcl.Query(query) - if err != nil { - return err - } - if verbose { - fmt.Printf("%d results...\n", len(spans)) - } - for i := range spans { - fmt.Printf("%s\n", spans[i].ToJson()) - } - return nil -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htrace/query_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/query_test.go b/htrace-htraced/go/src/org/apache/htrace/htrace/query_test.go deleted file mode 100644 index cab1e92..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/htrace/query_test.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "encoding/json" - "org/apache/htrace/common" - "reflect" - "testing" -) - -func predsToStr(preds []common.Predicate) string { - b, err := json.MarshalIndent(preds, "", " ") - if err != nil { - return "JSON marshaling error: " + err.Error() - } - return string(b) -} - -func checkParseQueryString(t *testing.T, str string, epreds []common.Predicate) { - preds, err := parseQueryString(str) - if err != nil { - t.Fatalf("got unexpected parseQueryString error: %s\n", err.Error()) - } - if !reflect.DeepEqual(preds, epreds) { - t.Fatalf("Unexpected result from parseQueryString. " + - "Expected: %s, got: %s\n", predsToStr(epreds), predsToStr(preds)) - } -} - -func TestParseQueryString(t *testing.T) { - verbose = testing.Verbose() - checkParseQueryString(t, "description eq ls", []common.Predicate { - common.Predicate { - Op: common.EQUALS, - Field: common.DESCRIPTION, - Val: "ls", - }, - }) - checkParseQueryString(t, "begin gt 123 and end le 456", []common.Predicate { - common.Predicate { - Op: common.GREATER_THAN, - Field: common.BEGIN_TIME, - Val: "123", - }, - common.Predicate { - Op: common.LESS_THAN_OR_EQUALS, - Field: common.END_TIME, - Val: "456", - }, - }) - checkParseQueryString(t, `DESCRIPTION cn "Foo Bar" and ` + - `BEGIN ge "999" and SPANID eq "4565d8abc4f70ac1216a3f1834c6860b"`, - []common.Predicate { - common.Predicate { - Op: common.CONTAINS, - Field: common.DESCRIPTION, - Val: "Foo Bar", - }, - common.Predicate { - Op: common.GREATER_THAN_OR_EQUALS, - Field: common.BEGIN_TIME, - Val: "999", - }, - common.Predicate { - Op: common.EQUALS, - Field: common.SPAN_ID, - Val: "4565d8abc4f70ac1216a3f1834c6860b", - }, - }) -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go new file mode 100644 index 0000000..98b1646 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/alecthomas/kingpin" + "io" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "sort" + "strings" + "text/tabwriter" + "time" +) + +var RELEASE_VERSION string +var GIT_VERSION string + +const EXIT_SUCCESS = 0 +const EXIT_FAILURE = 1 + +var verbose bool + +const USAGE = `The Apache HTrace command-line tool. This tool retrieves and modifies settings and +other data on a running htraced daemon. + +If we find an ` + conf.CONFIG_FILE_NAME + ` configuration file in the list of directories +specified in ` + conf.HTRACED_CONF_DIR + `, we will use that configuration; otherwise, +the defaults will be used. +` + +func main() { + // Load htraced configuration + cnf, cnfLog := conf.LoadApplicationConfig("htrace.tool.") + lg := common.NewLogger("conf", cnf) + defer lg.Close() + scanner := bufio.NewScanner(cnfLog) + for scanner.Scan() { + lg.Debugf(scanner.Text() + "\n") + } + + // Parse argv + app := kingpin.New(os.Args[0], USAGE) + app.Flag("Dmy.key", "Set configuration key 'my.key' to 'my.value'. Replace 'my.key' "+ + "with any key you want to set.").Default("my.value").String() + addr := app.Flag("addr", "Server address.").String() + verbose = *app.Flag("verbose", "Verbose.").Default("false").Bool() + version := app.Command("version", "Print the version of this program.") + serverVersion := app.Command("serverVersion", "Print the version of the htraced server.") + serverStats := app.Command("serverStats", "Print statistics retrieved from the htraced server.") + serverStatsJson := serverStats.Flag("json", "Display statistics as raw JSON.").Default("false").Bool() + serverConf := app.Command("serverConf", "Print the server configuration retrieved from the htraced server.") + findSpan := app.Command("findSpan", "Print information about a trace span with a given ID.") + findSpanId := findSpan.Arg("id", "Span ID to find. Example: be305e54-4534-2110-a0b2-e06b9effe112").Required().String() + findChildren := app.Command("findChildren", "Print out the span IDs that are children of a given span ID.") + parentSpanId := findChildren.Arg("id", "Span ID to print children for. Example: be305e54-4534-2110-a0b2-e06b9effe112"). + Required().String() + childLim := findChildren.Flag("lim", "Maximum number of child IDs to print.").Default("20").Int() + loadFile := app.Command("loadFile", "Write whitespace-separated JSON spans from a file to the server.") + loadFilePath := loadFile.Arg("path", + "A file containing whitespace-separated span JSON.").Required().String() + loadJson := app.Command("load", "Write JSON spans from the command-line to the server.") + loadJsonArg := loadJson.Arg("json", "A JSON span to write to the server.").Required().String() + dumpAll := app.Command("dumpAll", "Dump all spans from the htraced daemon.") + dumpAllOutPath := dumpAll.Arg("path", "The path to dump the trace spans to.").Default("-").String() + dumpAllLim := dumpAll.Flag("lim", "The number of spans to transfer from the server at once."). + Default("100").Int() + graph := app.Command("graph", "Visualize span JSON as a graph.") + graphJsonFile := graph.Arg("input", "The JSON file to load").Required().String() + graphDotFile := graph.Flag("output", + "The path to write a GraphViz dotfile to. This file can be used as input to "+ + "GraphViz, in order to generate a pretty picture. See graphviz.org for more "+ + "information about generating pictures of graphs.").Default("-").String() + query := app.Command("query", "Send a query to htraced.") + queryLim := query.Flag("lim", "Maximum number of spans to retrieve.").Default("20").Int() + queryArg := query.Arg("query", "The query string to send. Query strings have the format "+ + "[TYPE] [OPERATOR] [CONST], joined by AND statements.").Required().String() + rawQuery := app.Command("rawQuery", "Send a raw JSON query to htraced.") + rawQueryArg := rawQuery.Arg("json", "The query JSON to send.").Required().String() + cmd := kingpin.MustParse(app.Parse(os.Args[1:])) + + // Add the command-line settings into the configuration. + if *addr != "" { + cnf = cnf.Clone(conf.HTRACE_WEB_ADDRESS, *addr) + } + + // Handle commands that don't require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case graph.FullCommand(): + err := jsonSpanFileToDotFile(*graphJsonFile, *graphDotFile) + if err != nil { + fmt.Printf("graphing error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + // Create HTrace client + hcl, err := htrace.NewClient(cnf) + if err != nil { + fmt.Printf("Failed to create HTrace client: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + + // Handle commands that require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case serverVersion.FullCommand(): + os.Exit(printServerVersion(hcl)) + case serverStats.FullCommand(): + if *serverStatsJson { + os.Exit(printServerStatsJson(hcl)) + } else { + os.Exit(printServerStats(hcl)) + } + case serverConf.FullCommand(): + os.Exit(printServerConfJson(hcl)) + case findSpan.FullCommand(): + var id *common.SpanId + id.FromString(*findSpanId) + os.Exit(doFindSpan(hcl, *id)) + case findChildren.FullCommand(): + var id *common.SpanId + id.FromString(*parentSpanId) + os.Exit(doFindChildren(hcl, *id, *childLim)) + case loadJson.FullCommand(): + os.Exit(doLoadSpanJson(hcl, *loadJsonArg)) + case loadFile.FullCommand(): + os.Exit(doLoadSpanJsonFile(hcl, *loadFilePath)) + case dumpAll.FullCommand(): + err := doDumpAll(hcl, *dumpAllOutPath, *dumpAllLim) + if err != nil { + fmt.Printf("dumpAll error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case query.FullCommand(): + err := doQueryFromString(hcl, *queryArg, *queryLim) + if err != nil { + fmt.Printf("query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case rawQuery.FullCommand(): + err := doRawQuery(hcl, *rawQueryArg) + if err != nil { + fmt.Printf("raw query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + app.UsageErrorf(os.Stderr, "You must supply a command to run.") +} + +// Print the version of the htrace binary. +func printVersion() int { + fmt.Printf("Running htrace command version %s.\n", RELEASE_VERSION) + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/info +func printServerVersion(hcl *htrace.Client) int { + ver, err := hcl.GetServerVersion() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + fmt.Printf("HTraced server version %s (%s)\n", ver.ReleaseVersion, ver.GitVersion) + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/info +func printServerStats(hcl *htrace.Client) int { + stats, err := hcl.GetServerStats() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintf(w, "HTRACED SERVER STATS\n") + fmt.Fprintf(w, "Datastore Start\t%s\n", + common.UnixMsToTime(stats.LastStartMs).Format(time.RFC3339)) + fmt.Fprintf(w, "Server Time\t%s\n", + common.UnixMsToTime(stats.CurMs).Format(time.RFC3339)) + fmt.Fprintf(w, "Spans reaped\t%d\n", stats.ReapedSpans) + fmt.Fprintf(w, "Number of leveldb directories\t%d\n", len(stats.Dirs)) + w.Flush() + fmt.Println("") + for i := range stats.Dirs { + dir := stats.Dirs[i] + fmt.Printf("==== %s ===\n", dir.Path) + fmt.Printf("Approximate number of spans: %d\n", dir.ApproxNumSpans) + stats := strings.Replace(dir.LevelDbStats, "\\n", "\n", -1) + fmt.Printf("%s\n", stats) + } + w = new(tabwriter.Writer) + w.Init(os.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintf(w, "HOST SPAN METRICS\n") + mtxMap := stats.HostSpanMetrics + keys := make(sort.StringSlice, len(mtxMap)) + i := 0 + for k, _ := range mtxMap { + keys[i] = k + i++ + } + sort.Sort(keys) + for k := range keys { + mtx := mtxMap[keys[k]] + fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\tclient dropped: %d\n", + keys[k], mtx.Written, mtx.ServerDropped, mtx.ClientDropped) + } + w.Flush() + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/info as JSON +func printServerStatsJson(hcl *htrace.Client) int { + stats, err := hcl.GetServerStats() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + buf, err := json.MarshalIndent(stats, "", " ") + if err != nil { + fmt.Printf("Error marshalling server stats: %s", err.Error()) + return EXIT_FAILURE + } + fmt.Printf("%s\n", string(buf)) + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/conf as JSON +func printServerConfJson(hcl *htrace.Client) int { + cnf, err := hcl.GetServerConf() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + buf, err := json.MarshalIndent(cnf, "", " ") + if err != nil { + fmt.Printf("Error marshalling server conf: %s", err.Error()) + return EXIT_FAILURE + } + fmt.Printf("%s\n", string(buf)) + return EXIT_SUCCESS +} + +// Print information about a trace span. +func doFindSpan(hcl *htrace.Client, sid common.SpanId) int { + span, err := hcl.FindSpan(sid) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + if span == nil { + fmt.Printf("Span ID not found.\n") + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(span, "", " ") + if err != nil { + fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + fmt.Printf("%s\n", string(pbuf)) + return EXIT_SUCCESS +} + +func doLoadSpanJsonFile(hcl *htrace.Client, spanFile string) int { + if spanFile == "" { + fmt.Printf("You must specify the json file to load.\n") + return EXIT_FAILURE + } + file, err := OpenInputFile(spanFile) + if err != nil { + fmt.Printf("Failed to open %s: %s\n", spanFile, err.Error()) + return EXIT_FAILURE + } + defer file.Close() + return doLoadSpans(hcl, bufio.NewReader(file)) +} + +func doLoadSpanJson(hcl *htrace.Client, spanJson string) int { + return doLoadSpans(hcl, bytes.NewBufferString(spanJson)) +} + +func doLoadSpans(hcl *htrace.Client, reader io.Reader) int { + dec := json.NewDecoder(reader) + spans := make([]*common.Span, 0, 32) + var err error + for { + var span common.Span + if err = dec.Decode(&span); err != nil { + if err == io.EOF { + break + } + fmt.Printf("Failed to decode JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + spans = append(spans, &span) + } + if verbose { + fmt.Printf("Writing ") + prefix := "" + for i := range spans { + fmt.Printf("%s%s", prefix, spans[i].ToJson()) + prefix = ", " + } + fmt.Printf("\n") + } + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: spans, + }) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + return EXIT_SUCCESS +} + +// Find information about the children of a span. +func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int { + spanIds, err := hcl.FindChildren(sid, lim) + if err != nil { + fmt.Printf("%s\n", err.Error()) + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(spanIds, "", " ") + if err != nil { + fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error()) + return 1 + } + fmt.Printf("%s\n", string(pbuf)) + return 0 +} + +// Dump all spans from the htraced daemon. +func doDumpAll(hcl *htrace.Client, outPath string, lim int) error { + file, err := CreateOutputFile(outPath) + if err != nil { + return err + } + w := bufio.NewWriter(file) + defer func() { + if file != nil { + w.Flush() + file.Close() + } + }() + out := make(chan *common.Span, 50) + var dumpErr error + go func() { + dumpErr = hcl.DumpAll(lim, out) + }() + var numSpans int64 + nextLogTime := time.Now().Add(time.Second * 5) + for { + span, channelOpen := <-out + if !channelOpen { + break + } + if err == nil { + _, err = fmt.Fprintf(w, "%s\n", span.ToJson()) + } + if verbose { + numSpans++ + now := time.Now() + if !now.Before(nextLogTime) { + nextLogTime = now.Add(time.Second * 5) + fmt.Printf("received %d span(s)...\n", numSpans) + } + } + } + if err != nil { + return errors.New(fmt.Sprintf("Write error %s", err.Error())) + } + if dumpErr != nil { + return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error())) + } + err = w.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + if err != nil { + return err + } + return nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/file.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/file.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/file.go new file mode 100644 index 0000000..ea214be --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/file.go @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "org/apache/htrace/common" + "os" +) + +// A file used for input. +// Transparently supports using stdin for input. +type InputFile struct { + *os.File + path string +} + +// Open an input file. Stdin will be used when path is - +func OpenInputFile(path string) (*InputFile, error) { + if path == "-" { + return &InputFile{File: os.Stdin, path: path}, nil + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + return &InputFile{File: file, path: path}, nil +} + +func (file *InputFile) Close() { + if file.path != "-" { + file.File.Close() + } +} + +// A file used for output. +// Transparently supports using stdout for output. +type OutputFile struct { + *os.File + path string +} + +// Create an output file. Stdout will be used when path is - +func CreateOutputFile(path string) (*OutputFile, error) { + if path == "-" { + return &OutputFile{File: os.Stdout, path: path}, nil + } + file, err := os.Create(path) + if err != nil { + return nil, err + } + return &OutputFile{File: file, path: path}, nil +} + +func (file *OutputFile) Close() error { + if file.path != "-" { + return file.File.Close() + } + return nil +} + +// FailureDeferringWriter is a writer which allows us to call Printf multiple +// times and then check if all the printfs succeeded at the very end, rather +// than checking after each call. We will not attempt to write more data +// after the first write failure. +type FailureDeferringWriter struct { + io.Writer + err error +} + +func NewFailureDeferringWriter(writer io.Writer) *FailureDeferringWriter { + return &FailureDeferringWriter{writer, nil} +} + +func (w *FailureDeferringWriter) Printf(format string, v ...interface{}) { + if w.err != nil { + return + } + str := fmt.Sprintf(format, v...) + _, err := w.Writer.Write([]byte(str)) + if err != nil { + w.err = err + } +} + +func (w *FailureDeferringWriter) Error() error { + return w.err +} + +// Read a file full of whitespace-separated span JSON into a slice of spans. +func readSpansFile(path string) (common.SpanSlice, error) { + file, err := OpenInputFile(path) + if err != nil { + return nil, err + } + defer file.Close() + return readSpans(bufio.NewReader(file)) +} + +// Read whitespace-separated span JSON into a slice of spans. +func readSpans(reader io.Reader) (common.SpanSlice, error) { + spans := make(common.SpanSlice, 0) + dec := json.NewDecoder(reader) + for { + var span common.Span + err := dec.Decode(&span) + if err != nil { + if err != io.EOF { + return nil, errors.New(fmt.Sprintf("Decode error after decoding %d "+ + "span(s): %s", len(spans), err.Error())) + } + break + } + spans = append(spans, &span) + } + return spans, nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/file_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/file_test.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/file_test.go new file mode 100644 index 0000000..98e5e6c --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/file_test.go @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "errors" + "io" + "io/ioutil" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "strings" + "testing" +) + +func TestInputFileAndOutputFile(t *testing.T) { + tdir, err := ioutil.TempDir(os.TempDir(), "TestInputFileAndOutputFile") + if err != nil { + t.Fatalf("failed to create TempDir: %s\n", err.Error()) + } + defer os.RemoveAll(tdir) + tpath := tdir + conf.PATH_SEP + "test" + var ofile *OutputFile + ofile, err = CreateOutputFile(tpath) + if err != nil { + t.Fatalf("failed to create OutputFile at %s: %s\n", tpath, err.Error()) + } + defer func() { + if ofile != nil { + ofile.Close() + } + }() + w := NewFailureDeferringWriter(ofile) + w.Printf("Hello, world!\n") + w.Printf("2 + 2 = %d\n", 4) + if w.Error() != nil { + t.Fatalf("got unexpected error writing to %s: %s\n", tpath, w.Error().Error()) + } + err = ofile.Close() + ofile = nil + if err != nil { + t.Fatalf("error on closing OutputFile for %s: %s\n", tpath, err.Error()) + } + var ifile *InputFile + ifile, err = OpenInputFile(tpath) + defer ifile.Close() + expected := "Hello, world!\n2 + 2 = 4\n" + buf := make([]byte, len(expected)) + _, err = io.ReadAtLeast(ifile, buf, len(buf)) + if err != nil { + t.Fatalf("unexpected error on reading %s: %s\n", tpath, err.Error()) + } + str := string(buf) + if str != expected { + t.Fatalf("Could not read back what we wrote to %s.\n"+ + "Got:\n%s\nExpected:\n%s\n", tpath, str, expected) + } +} + +type LimitedBufferWriter struct { + buf []byte + off int +} + +const LIMITED_BUFFER_MESSAGE = "There isn't enough buffer to go around!" + +func (w *LimitedBufferWriter) Write(p []byte) (int, error) { + var nwritten int + for i := range p { + if w.off >= len(w.buf) { + return nwritten, errors.New(LIMITED_BUFFER_MESSAGE) + } + w.buf[w.off] = p[i] + w.off = w.off + 1 + nwritten++ + } + return nwritten, nil +} + +func TestFailureDeferringWriter(t *testing.T) { + lw := LimitedBufferWriter{buf: make([]byte, 20), off: 0} + w := NewFailureDeferringWriter(&lw) + w.Printf("Zippity do dah #%d\n", 1) + w.Printf("Zippity do dah #%d\n", 2) + if w.Error() == nil { + t.Fatalf("expected FailureDeferringWriter to experience a failure due to " + + "limited buffer size, but it did not.") + } + if w.Error().Error() != LIMITED_BUFFER_MESSAGE { + t.Fatalf("expected FailureDeferringWriter to have the error message %s, but "+ + "the message was %s\n", LIMITED_BUFFER_MESSAGE, w.Error().Error()) + } + expected := "Zippity do dah #1\nZi" + if string(lw.buf) != expected { + t.Fatalf("expected LimitedBufferWriter to contain %s, but it contained %s "+ + "instead.\n", expected, string(lw.buf)) + } +} + +func TestReadSpans(t *testing.T) { + SPAN_TEST_STR := `{"a":"b9f2a1e07b6e4f16b0c2b27303b20e79",` + + `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + + `"r":"FsShell","p":["3afebdc0a13f4feb811cc5c0e42d30b1"]} +{"a":"3afebdc0a13f4feb811cc5c0e42d30b1","b":1424736224969,` + + `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} +` + r := strings.NewReader(SPAN_TEST_STR) + spans, err := readSpans(r) + if err != nil { + t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) + } + SPAN_TEST_EXPECTED := common.SpanSlice{ + &common.Span{ + Id: common.TestId("b9f2a1e07b6e4f16b0c2b27303b20e79"), + SpanData: common.SpanData{ + Begin: 1424736225037, + End: 1424736225901, + Description: "ClientNamenodeProtocol#getFileInfo", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1")}, + }, + }, + &common.Span{ + Id: common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1"), + SpanData: common.SpanData{ + Begin: 1424736224969, + End: 1424736225960, + Description: "getFileInfo", + TracerId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + } + if len(spans) != len(SPAN_TEST_EXPECTED) { + t.Fatalf("Expected %d spans, but got %d\n", + len(SPAN_TEST_EXPECTED), len(spans)) + } + for i := range SPAN_TEST_EXPECTED { + common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph.go new file mode 100644 index 0000000..024d973 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph.go @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "org/apache/htrace/common" + "os" + "sort" +) + +// Create a dotfile from a json file. +func jsonSpanFileToDotFile(jsonFile string, dotFile string) error { + spans, err := readSpansFile(jsonFile) + if err != nil { + return errors.New(fmt.Sprintf("error reading %s: %s", + jsonFile, err.Error())) + } + var file *OutputFile + file, err = CreateOutputFile(dotFile) + if err != nil { + return errors.New(fmt.Sprintf("error opening %s for write: %s", + dotFile, err.Error())) + } + defer func() { + if file != nil { + file.Close() + } + }() + writer := bufio.NewWriter(file) + err = spansToDot(spans, writer) + if err != nil { + return err + } + err = writer.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + return err +} + +// Create output in dotfile format from a set of spans. +func spansToDot(spans common.SpanSlice, writer io.Writer) error { + sort.Sort(spans) + idMap := make(map[[16]byte]*common.Span) + for i := range spans { + span := spans[i] + if idMap[span.Id.ToArray()] != nil { + fmt.Fprintf(os.Stderr, "There were multiple spans listed which "+ + "had ID %s.\nFirst:%s\nOther:%s\n", span.Id.String(), + idMap[span.Id.ToArray()].ToJson(), span.ToJson()) + } else { + idMap[span.Id.ToArray()] = span + } + } + childMap := make(map[[16]byte]common.SpanSlice) + for i := range spans { + child := spans[i] + for j := range child.Parents { + parent := idMap[child.Parents[j].ToArray()] + if parent == nil { + fmt.Fprintf(os.Stderr, "Can't find parent id %s for %s\n", + child.Parents[j].String(), child.ToJson()) + } else { + children := childMap[parent.Id.ToArray()] + if children == nil { + children = make(common.SpanSlice, 0) + } + children = append(children, child) + childMap[parent.Id.ToArray()] = children + } + } + } + w := NewFailureDeferringWriter(writer) + w.Printf("digraph spans {\n") + // Write out the nodes with their descriptions. + for i := range spans { + w.Printf(fmt.Sprintf(` "%s" [label="%s"];`+"\n", + spans[i].Id.String(), spans[i].Description)) + } + // Write out the edges between nodes... the parent/children relationships + for i := range spans { + children := childMap[spans[i].Id.ToArray()] + sort.Sort(children) + if children != nil { + for c := range children { + w.Printf(fmt.Sprintf(` "%s" -> "%s";`+"\n", + spans[i].Id.String(), children[c].Id)) + } + } + } + w.Printf("}\n") + return w.Error() +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph_test.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph_test.go new file mode 100644 index 0000000..621b3dc --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/graph_test.go @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "org/apache/htrace/common" + "testing" +) + +func TestSpansToDot(t *testing.T) { + TEST_SPANS := common.SpanSlice{ + &common.Span{ + Id: common.TestId("814c8ee0e7984be3a8af00ac64adccb6"), + SpanData: common.SpanData{ + Begin: 1424813349020, + End: 1424813349134, + Description: "newDFSInputStream", + TracerId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + &common.Span{ + Id: common.TestId("cf2d5de696454548bc055d1e6024054c"), + SpanData: common.SpanData{ + Begin: 1424813349025, + End: 1424813349133, + Description: "getBlockLocations", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("814c8ee0e7984be3a8af00ac64adccb6")}, + }, + }, + &common.Span{ + Id: common.TestId("37623806f9c64483b834b8ea5d6b4827"), + SpanData: common.SpanData{ + Begin: 1424813349027, + End: 1424813349073, + Description: "ClientNamenodeProtocol#getBlockLocations", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("cf2d5de696454548bc055d1e6024054c")}, + }, + }, + } + w := bytes.NewBuffer(make([]byte, 0, 2048)) + err := spansToDot(TEST_SPANS, w) + if err != nil { + t.Fatalf("spansToDot failed: error %s\n", err.Error()) + } + EXPECTED_STR := `digraph spans { + "37623806f9c64483b834b8ea5d6b4827" [label="ClientNamenodeProtocol#getBlockLocations"]; + "814c8ee0e7984be3a8af00ac64adccb6" [label="newDFSInputStream"]; + "cf2d5de696454548bc055d1e6024054c" [label="getBlockLocations"]; + "814c8ee0e7984be3a8af00ac64adccb6" -> "cf2d5de696454548bc055d1e6024054c"; + "cf2d5de696454548bc055d1e6024054c" -> "37623806f9c64483b834b8ea5d6b4827"; +} +` + if w.String() != EXPECTED_STR { + t.Fatalf("Expected to get:\n%s\nGot:\n%s\n", EXPECTED_STR, w.String()) + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/queries.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/queries.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/queries.go new file mode 100644 index 0000000..442df4f --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/queries.go @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "strings" + "unicode" +) + +// Convert a string into a whitespace-separated sequence of strings. +func tokenize(str string) []string { + prevQuote := rune(0) + f := func(c rune) bool { + switch { + case c == prevQuote: + prevQuote = rune(0) + return true + case prevQuote != rune(0): + return false + case unicode.In(c, unicode.Quotation_Mark): + prevQuote = c + return true + default: + return unicode.IsSpace(c) + } + } + return strings.FieldsFunc(str, f) +} + +// Parses a query string in the format of a series of +// [TYPE] [OPERATOR] [CONST] tuples, joined by AND statements. +type predicateParser struct { + tokens []string + curToken int +} + +func (ps *predicateParser) Parse() (*common.Predicate, error) { + if ps.curToken >= len(ps.tokens) { + return nil, nil + } + if ps.curToken > 0 { + if strings.ToLower(ps.tokens[ps.curToken]) != "and" { + return nil, errors.New(fmt.Sprintf("Error parsing on token %d: "+ + "expected predicates to be joined by 'and', but found '%s'", + ps.curToken, ps.tokens[ps.curToken])) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after 'and' at "+ + "token %d", ps.curToken)) + } + } + field := common.Field(strings.ToLower(ps.tokens[ps.curToken])) + if !field.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid field specifier at token %d. "+ + "Can't understand %s. Valid field specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidFields())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + op := common.Op(strings.ToLower(ps.tokens[ps.curToken])) + if !op.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid operation specifier at token %d. "+ + "Can't understand %s. Valid operation specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidOps())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + val := ps.tokens[ps.curToken] + ps.curToken++ + return &common.Predicate{Op: op, Field: field, Val: val}, nil +} + +func parseQueryString(str string) ([]common.Predicate, error) { + ps := predicateParser{tokens: tokenize(str)} + if verbose { + fmt.Printf("Running query [ ") + prefix := "" + for tokenIdx := range(ps.tokens) { + fmt.Printf("%s'%s'", prefix, ps.tokens[tokenIdx]) + prefix = ", " + } + fmt.Printf(" ]\n") + } + preds := make([]common.Predicate, 0) + for { + pred, err := ps.Parse() + if err != nil { + return nil, err + } + if pred == nil { + break + } + preds = append(preds, *pred) + } + if len(preds) == 0 { + return nil, errors.New("Empty query string") + } + return preds, nil +} + +// Send a query from a query string. +func doQueryFromString(hcl *htrace.Client, str string, lim int) error { + query := &common.Query{Lim: lim} + var err error + query.Predicates, err = parseQueryString(str) + if err != nil { + return err + } + return doQuery(hcl, query) +} + +// Send a query from a raw JSON string. +func doRawQuery(hcl *htrace.Client, str string) error { + jsonBytes := []byte(str) + var query common.Query + err := json.Unmarshal(jsonBytes, &query) + if err != nil { + return errors.New(fmt.Sprintf("Error parsing provided JSON: %s\n", err.Error())) + } + return doQuery(hcl, &query) +} + +// Send a query. +func doQuery(hcl *htrace.Client, query *common.Query) error { + if verbose { + qbytes, err := json.Marshal(*query) + if err != nil { + qbytes = []byte("marshaling error: " + err.Error()) + } + fmt.Printf("Sending query: %s\n", string(qbytes)) + } + spans, err := hcl.Query(query) + if err != nil { + return err + } + if verbose { + fmt.Printf("%d results...\n", len(spans)) + } + for i := range spans { + fmt.Printf("%s\n", spans[i].ToJson()) + } + return nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/2fdef092/htrace-htraced/go/src/org/apache/htrace/htracedTool/query_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/query_test.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/query_test.go new file mode 100644 index 0000000..cab1e92 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/query_test.go @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "org/apache/htrace/common" + "reflect" + "testing" +) + +func predsToStr(preds []common.Predicate) string { + b, err := json.MarshalIndent(preds, "", " ") + if err != nil { + return "JSON marshaling error: " + err.Error() + } + return string(b) +} + +func checkParseQueryString(t *testing.T, str string, epreds []common.Predicate) { + preds, err := parseQueryString(str) + if err != nil { + t.Fatalf("got unexpected parseQueryString error: %s\n", err.Error()) + } + if !reflect.DeepEqual(preds, epreds) { + t.Fatalf("Unexpected result from parseQueryString. " + + "Expected: %s, got: %s\n", predsToStr(epreds), predsToStr(preds)) + } +} + +func TestParseQueryString(t *testing.T) { + verbose = testing.Verbose() + checkParseQueryString(t, "description eq ls", []common.Predicate { + common.Predicate { + Op: common.EQUALS, + Field: common.DESCRIPTION, + Val: "ls", + }, + }) + checkParseQueryString(t, "begin gt 123 and end le 456", []common.Predicate { + common.Predicate { + Op: common.GREATER_THAN, + Field: common.BEGIN_TIME, + Val: "123", + }, + common.Predicate { + Op: common.LESS_THAN_OR_EQUALS, + Field: common.END_TIME, + Val: "456", + }, + }) + checkParseQueryString(t, `DESCRIPTION cn "Foo Bar" and ` + + `BEGIN ge "999" and SPANID eq "4565d8abc4f70ac1216a3f1834c6860b"`, + []common.Predicate { + common.Predicate { + Op: common.CONTAINS, + Field: common.DESCRIPTION, + Val: "Foo Bar", + }, + common.Predicate { + Op: common.GREATER_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "999", + }, + common.Predicate { + Op: common.EQUALS, + Field: common.SPAN_ID, + Val: "4565d8abc4f70ac1216a3f1834c6860b", + }, + }) +}
