http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/htracedTool/file_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/htracedTool/file_test.go b/htrace-htraced/go/src/htrace/htracedTool/file_test.go new file mode 100644 index 0000000..f955add --- /dev/null +++ b/htrace-htraced/go/src/htrace/htracedTool/file_test.go @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "errors" + "htrace/common" + "htrace/conf" + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +func TestInputFileAndOutputFile(t *testing.T) { + tdir, err := ioutil.TempDir(os.TempDir(), "TestInputFileAndOutputFile") + if err != nil { + t.Fatalf("failed to create TempDir: %s\n", err.Error()) + } + defer os.RemoveAll(tdir) + tpath := tdir + conf.PATH_SEP + "test" + var ofile *OutputFile + ofile, err = CreateOutputFile(tpath) + if err != nil { + t.Fatalf("failed to create OutputFile at %s: %s\n", tpath, err.Error()) + } + defer func() { + if ofile != nil { + ofile.Close() + } + }() + w := NewFailureDeferringWriter(ofile) + w.Printf("Hello, world!\n") + w.Printf("2 + 2 = %d\n", 4) + if w.Error() != nil { + t.Fatalf("got unexpected error writing to %s: %s\n", tpath, w.Error().Error()) + } + err = ofile.Close() + ofile = nil + if err != nil { + t.Fatalf("error on closing OutputFile for %s: %s\n", tpath, err.Error()) + } + var ifile *InputFile + ifile, err = OpenInputFile(tpath) + defer ifile.Close() + expected := "Hello, world!\n2 + 2 = 4\n" + buf := make([]byte, len(expected)) + _, err = io.ReadAtLeast(ifile, buf, len(buf)) + if err != nil { + t.Fatalf("unexpected error on reading %s: %s\n", tpath, err.Error()) + } + str := string(buf) + if str != expected { + t.Fatalf("Could not read back what we wrote to %s.\n"+ + "Got:\n%s\nExpected:\n%s\n", tpath, str, expected) + } +} + +type LimitedBufferWriter struct { + buf []byte + off int +} + +const LIMITED_BUFFER_MESSAGE = "There isn't enough buffer to go around!" + +func (w *LimitedBufferWriter) Write(p []byte) (int, error) { + var nwritten int + for i := range p { + if w.off >= len(w.buf) { + return nwritten, errors.New(LIMITED_BUFFER_MESSAGE) + } + w.buf[w.off] = p[i] + w.off = w.off + 1 + nwritten++ + } + return nwritten, nil +} + +func TestFailureDeferringWriter(t *testing.T) { + lw := LimitedBufferWriter{buf: make([]byte, 20), off: 0} + w := NewFailureDeferringWriter(&lw) + w.Printf("Zippity do dah #%d\n", 1) + w.Printf("Zippity do dah #%d\n", 2) + if w.Error() == nil { + t.Fatalf("expected FailureDeferringWriter to experience a failure due to " + + "limited buffer size, but it did not.") + } + if w.Error().Error() != LIMITED_BUFFER_MESSAGE { + t.Fatalf("expected FailureDeferringWriter to have the error message %s, but "+ + "the message was %s\n", LIMITED_BUFFER_MESSAGE, w.Error().Error()) + } + expected := "Zippity do dah #1\nZi" + if string(lw.buf) != expected { + t.Fatalf("expected LimitedBufferWriter to contain %s, but it contained %s "+ + "instead.\n", expected, string(lw.buf)) + } +} + +func TestReadSpans(t *testing.T) { + SPAN_TEST_STR := `{"a":"b9f2a1e07b6e4f16b0c2b27303b20e79",` + + `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + + `"r":"FsShell","p":["3afebdc0a13f4feb811cc5c0e42d30b1"]} +{"a":"3afebdc0a13f4feb811cc5c0e42d30b1","b":1424736224969,` + + `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} +` + r := strings.NewReader(SPAN_TEST_STR) + spans, err := readSpans(r) + if err != nil { + t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) + } + SPAN_TEST_EXPECTED := common.SpanSlice{ + &common.Span{ + Id: common.TestId("b9f2a1e07b6e4f16b0c2b27303b20e79"), + SpanData: common.SpanData{ + Begin: 1424736225037, + End: 1424736225901, + Description: "ClientNamenodeProtocol#getFileInfo", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1")}, + }, + }, + &common.Span{ + Id: common.TestId("3afebdc0a13f4feb811cc5c0e42d30b1"), + SpanData: common.SpanData{ + Begin: 1424736224969, + End: 1424736225960, + Description: "getFileInfo", + TracerId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + } + if len(spans) != len(SPAN_TEST_EXPECTED) { + t.Fatalf("Expected %d spans, but got %d\n", + len(SPAN_TEST_EXPECTED), len(spans)) + } + for i := range SPAN_TEST_EXPECTED { + common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) + } +}
http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/htracedTool/graph.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/htracedTool/graph.go b/htrace-htraced/go/src/htrace/htracedTool/graph.go new file mode 100644 index 0000000..b238f11 --- /dev/null +++ b/htrace-htraced/go/src/htrace/htracedTool/graph.go @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "errors" + "fmt" + "htrace/common" + "io" + "os" + "sort" +) + +// Create a dotfile from a json file. +func jsonSpanFileToDotFile(jsonFile string, dotFile string) error { + spans, err := readSpansFile(jsonFile) + if err != nil { + return errors.New(fmt.Sprintf("error reading %s: %s", + jsonFile, err.Error())) + } + var file *OutputFile + file, err = CreateOutputFile(dotFile) + if err != nil { + return errors.New(fmt.Sprintf("error opening %s for write: %s", + dotFile, err.Error())) + } + defer func() { + if file != nil { + file.Close() + } + }() + writer := bufio.NewWriter(file) + err = spansToDot(spans, writer) + if err != nil { + return err + } + err = writer.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + return err +} + +// Create output in dotfile format from a set of spans. +func spansToDot(spans common.SpanSlice, writer io.Writer) error { + sort.Sort(spans) + idMap := make(map[[16]byte]*common.Span) + for i := range spans { + span := spans[i] + if idMap[span.Id.ToArray()] != nil { + fmt.Fprintf(os.Stderr, "There were multiple spans listed which "+ + "had ID %s.\nFirst:%s\nOther:%s\n", span.Id.String(), + idMap[span.Id.ToArray()].ToJson(), span.ToJson()) + } else { + idMap[span.Id.ToArray()] = span + } + } + childMap := make(map[[16]byte]common.SpanSlice) + for i := range spans { + child := spans[i] + for j := range child.Parents { + parent := idMap[child.Parents[j].ToArray()] + if parent == nil { + fmt.Fprintf(os.Stderr, "Can't find parent id %s for %s\n", + child.Parents[j].String(), child.ToJson()) + } else { + children := childMap[parent.Id.ToArray()] + if children == nil { + children = make(common.SpanSlice, 0) + } + children = append(children, child) + childMap[parent.Id.ToArray()] = children + } + } + } + w := NewFailureDeferringWriter(writer) + w.Printf("digraph spans {\n") + // Write out the nodes with their descriptions. + for i := range spans { + w.Printf(fmt.Sprintf(` "%s" [label="%s"];`+"\n", + spans[i].Id.String(), spans[i].Description)) + } + // Write out the edges between nodes... the parent/children relationships + for i := range spans { + children := childMap[spans[i].Id.ToArray()] + sort.Sort(children) + if children != nil { + for c := range children { + w.Printf(fmt.Sprintf(` "%s" -> "%s";`+"\n", + spans[i].Id.String(), children[c].Id)) + } + } + } + w.Printf("}\n") + return w.Error() +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/htracedTool/graph_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/htracedTool/graph_test.go b/htrace-htraced/go/src/htrace/htracedTool/graph_test.go new file mode 100644 index 0000000..af6d262 --- /dev/null +++ b/htrace-htraced/go/src/htrace/htracedTool/graph_test.go @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "htrace/common" + "testing" +) + +func TestSpansToDot(t *testing.T) { + TEST_SPANS := common.SpanSlice{ + &common.Span{ + Id: common.TestId("814c8ee0e7984be3a8af00ac64adccb6"), + SpanData: common.SpanData{ + Begin: 1424813349020, + End: 1424813349134, + Description: "newDFSInputStream", + TracerId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + &common.Span{ + Id: common.TestId("cf2d5de696454548bc055d1e6024054c"), + SpanData: common.SpanData{ + Begin: 1424813349025, + End: 1424813349133, + Description: "getBlockLocations", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("814c8ee0e7984be3a8af00ac64adccb6")}, + }, + }, + &common.Span{ + Id: common.TestId("37623806f9c64483b834b8ea5d6b4827"), + SpanData: common.SpanData{ + Begin: 1424813349027, + End: 1424813349073, + Description: "ClientNamenodeProtocol#getBlockLocations", + TracerId: "FsShell", + Parents: []common.SpanId{common.TestId("cf2d5de696454548bc055d1e6024054c")}, + }, + }, + } + w := bytes.NewBuffer(make([]byte, 0, 2048)) + err := spansToDot(TEST_SPANS, w) + if err != nil { + t.Fatalf("spansToDot failed: error %s\n", err.Error()) + } + EXPECTED_STR := `digraph spans { + "37623806f9c64483b834b8ea5d6b4827" [label="ClientNamenodeProtocol#getBlockLocations"]; + "814c8ee0e7984be3a8af00ac64adccb6" [label="newDFSInputStream"]; + "cf2d5de696454548bc055d1e6024054c" [label="getBlockLocations"]; + "814c8ee0e7984be3a8af00ac64adccb6" -> "cf2d5de696454548bc055d1e6024054c"; + "cf2d5de696454548bc055d1e6024054c" -> "37623806f9c64483b834b8ea5d6b4827"; +} +` + if w.String() != EXPECTED_STR { + t.Fatalf("Expected to get:\n%s\nGot:\n%s\n", EXPECTED_STR, w.String()) + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/htracedTool/queries.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/htracedTool/queries.go b/htrace-htraced/go/src/htrace/htracedTool/queries.go new file mode 100644 index 0000000..3111c62 --- /dev/null +++ b/htrace-htraced/go/src/htrace/htracedTool/queries.go @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + htrace "htrace/client" + "htrace/common" + "strings" + "unicode" +) + +// Convert a string into a whitespace-separated sequence of strings. +func tokenize(str string) []string { + prevQuote := rune(0) + f := func(c rune) bool { + switch { + case c == prevQuote: + prevQuote = rune(0) + return true + case prevQuote != rune(0): + return false + case unicode.In(c, unicode.Quotation_Mark): + prevQuote = c + return true + default: + return unicode.IsSpace(c) + } + } + return strings.FieldsFunc(str, f) +} + +// Parses a query string in the format of a series of +// [TYPE] [OPERATOR] [CONST] tuples, joined by AND statements. +type predicateParser struct { + tokens []string + curToken int +} + +func (ps *predicateParser) Parse() (*common.Predicate, error) { + if ps.curToken >= len(ps.tokens) { + return nil, nil + } + if ps.curToken > 0 { + if strings.ToLower(ps.tokens[ps.curToken]) != "and" { + return nil, errors.New(fmt.Sprintf("Error parsing on token %d: "+ + "expected predicates to be joined by 'and', but found '%s'", + ps.curToken, ps.tokens[ps.curToken])) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after 'and' at "+ + "token %d", ps.curToken)) + } + } + field := common.Field(strings.ToLower(ps.tokens[ps.curToken])) + if !field.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid field specifier at token %d. "+ + "Can't understand %s. Valid field specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidFields())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + op := common.Op(strings.ToLower(ps.tokens[ps.curToken])) + if !op.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid operation specifier at token %d. "+ + "Can't understand %s. Valid operation specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidOps())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + val := ps.tokens[ps.curToken] + ps.curToken++ + return &common.Predicate{Op: op, Field: field, Val: val}, nil +} + +func parseQueryString(str string) ([]common.Predicate, error) { + ps := predicateParser{tokens: tokenize(str)} + if verbose { + fmt.Printf("Running query [ ") + prefix := "" + for tokenIdx := range ps.tokens { + fmt.Printf("%s'%s'", prefix, ps.tokens[tokenIdx]) + prefix = ", " + } + fmt.Printf(" ]\n") + } + preds := make([]common.Predicate, 0) + for { + pred, err := ps.Parse() + if err != nil { + return nil, err + } + if pred == nil { + break + } + preds = append(preds, *pred) + } + if len(preds) == 0 { + return nil, errors.New("Empty query string") + } + return preds, nil +} + +// Send a query from a query string. +func doQueryFromString(hcl *htrace.Client, str string, lim int) error { + query := &common.Query{Lim: lim} + var err error + query.Predicates, err = parseQueryString(str) + if err != nil { + return err + } + return doQuery(hcl, query) +} + +// Send a query from a raw JSON string. +func doRawQuery(hcl *htrace.Client, str string) error { + jsonBytes := []byte(str) + var query common.Query + err := json.Unmarshal(jsonBytes, &query) + if err != nil { + return errors.New(fmt.Sprintf("Error parsing provided JSON: %s\n", err.Error())) + } + return doQuery(hcl, &query) +} + +// Send a query. +func doQuery(hcl *htrace.Client, query *common.Query) error { + if verbose { + qbytes, err := json.Marshal(*query) + if err != nil { + qbytes = []byte("marshaling error: " + err.Error()) + } + fmt.Printf("Sending query: %s\n", string(qbytes)) + } + spans, err := hcl.Query(query) + if err != nil { + return err + } + if verbose { + fmt.Printf("%d results...\n", len(spans)) + } + for i := range spans { + fmt.Printf("%s\n", spans[i].ToJson()) + } + return nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/htracedTool/query_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/htracedTool/query_test.go b/htrace-htraced/go/src/htrace/htracedTool/query_test.go new file mode 100644 index 0000000..55aff91 --- /dev/null +++ b/htrace-htraced/go/src/htrace/htracedTool/query_test.go @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "htrace/common" + "reflect" + "testing" +) + +func predsToStr(preds []common.Predicate) string { + b, err := json.MarshalIndent(preds, "", " ") + if err != nil { + return "JSON marshaling error: " + err.Error() + } + return string(b) +} + +func checkParseQueryString(t *testing.T, str string, epreds []common.Predicate) { + preds, err := parseQueryString(str) + if err != nil { + t.Fatalf("got unexpected parseQueryString error: %s\n", err.Error()) + } + if !reflect.DeepEqual(preds, epreds) { + t.Fatalf("Unexpected result from parseQueryString. "+ + "Expected: %s, got: %s\n", predsToStr(epreds), predsToStr(preds)) + } +} + +func TestParseQueryString(t *testing.T) { + verbose = testing.Verbose() + checkParseQueryString(t, "description eq ls", []common.Predicate{ + common.Predicate{ + Op: common.EQUALS, + Field: common.DESCRIPTION, + Val: "ls", + }, + }) + checkParseQueryString(t, "begin gt 123 and end le 456", []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN, + Field: common.BEGIN_TIME, + Val: "123", + }, + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.END_TIME, + Val: "456", + }, + }) + checkParseQueryString(t, `DESCRIPTION cn "Foo Bar" and `+ + `BEGIN ge "999" and SPANID eq "4565d8abc4f70ac1216a3f1834c6860b"`, + []common.Predicate{ + common.Predicate{ + Op: common.CONTAINS, + Field: common.DESCRIPTION, + Val: "Foo Bar", + }, + common.Predicate{ + Op: common.GREATER_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "999", + }, + common.Predicate{ + Op: common.EQUALS, + Field: common.SPAN_ID, + Val: "4565d8abc4f70ac1216a3f1834c6860b", + }, + }) +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/htrace/test/random.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/htrace/test/random.go b/htrace-htraced/go/src/htrace/test/random.go new file mode 100644 index 0000000..ad3104c --- /dev/null +++ b/htrace-htraced/go/src/htrace/test/random.go @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package test + +import ( + "fmt" + "htrace/common" + "math/rand" +) + +func NonZeroRand64(rnd *rand.Rand) int64 { + for { + r := rnd.Int63() + if r == 0 { + continue + } + if rnd.Intn(1) != 0 { + return -r + } + return r + } +} + +func NonZeroRandSpanId(rnd *rand.Rand) common.SpanId { + var id common.SpanId + id = make([]byte, 16) + for i := 0; i < len(id); i++ { + id[i] = byte(rnd.Intn(0x100)) + } + return id +} + +func NonZeroRand32(rnd *rand.Rand) int32 { + for { + r := rnd.Int31() + if r == 0 { + continue + } + if rnd.Intn(1) != 0 { + return -r + } + return r + } +} + +// Create a random span. +func NewRandomSpan(rnd *rand.Rand, potentialParents []*common.Span) *common.Span { + parents := []common.SpanId{} + if potentialParents != nil { + parentIdx := rnd.Intn(len(potentialParents) + 1) + if parentIdx < len(potentialParents) { + parents = []common.SpanId{potentialParents[parentIdx].Id} + } + } + return &common.Span{Id: NonZeroRandSpanId(rnd), + SpanData: common.SpanData{ + Begin: NonZeroRand64(rnd), + End: NonZeroRand64(rnd), + Description: "getFileDescriptors", + Parents: parents, + TracerId: fmt.Sprintf("tracer%d", NonZeroRand32(rnd)), + }} +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/client/client.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/client/client.go b/htrace-htraced/go/src/org/apache/htrace/client/client.go deleted file mode 100644 index a2a6f8b..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/client/client.go +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package client - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "org/apache/htrace/common" - "org/apache/htrace/conf" -) - -// A golang client for htraced. -// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff -func NewClient(cnf *conf.Config, testHooks *TestHooks) (*Client, error) { - hcl := Client{testHooks: testHooks} - hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS) - if testHooks != nil && testHooks.HrpcDisabled { - hcl.hrpcAddr = "" - } else { - hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS) - } - return &hcl, nil -} - -type TestHooks struct { - // If true, HRPC is disabled. - HrpcDisabled bool - - // A function which gets called after we connect to the server and send the - // message frame, but before we write the message body. - HandleWriteRequestBody func() -} - -type Client struct { - // REST address of the htraced server. - restAddr string - - // HRPC address of the htraced server. - hrpcAddr string - - // The test hooks to use, or nil if test hooks are not enabled. - testHooks *TestHooks -} - -// Get the htraced server version information. -func (hcl *Client) GetServerVersion() (*common.ServerVersion, error) { - buf, _, err := hcl.makeGetRequest("server/info") - if err != nil { - return nil, err - } - var info common.ServerVersion - err = json.Unmarshal(buf, &info) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &info, nil -} - -// Get the htraced server debug information. -func (hcl *Client) GetServerDebugInfo() (*common.ServerDebugInfo, error) { - buf, _, err := hcl.makeGetRequest("server/debugInfo") - if err != nil { - return nil, err - } - var debugInfo common.ServerDebugInfo - err = json.Unmarshal(buf, &debugInfo) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &debugInfo, nil -} - -// Get the htraced server statistics. -func (hcl *Client) GetServerStats() (*common.ServerStats, error) { - buf, _, err := hcl.makeGetRequest("server/stats") - if err != nil { - return nil, err - } - var stats common.ServerStats - err = json.Unmarshal(buf, &stats) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &stats, nil -} - -// Get the htraced server statistics. -func (hcl *Client) GetServerConf() (map[string]string, error) { - buf, _, err := hcl.makeGetRequest("server/conf") - if err != nil { - return nil, err - } - cnf := make(map[string]string) - err = json.Unmarshal(buf, &cnf) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return cnf, nil -} - -// Get information about a trace span. Returns nil, nil if the span was not found. -func (hcl *Client) FindSpan(sid common.SpanId) (*common.Span, error) { - buf, rc, err := hcl.makeGetRequest(fmt.Sprintf("span/%s", sid.String())) - if err != nil { - if rc == http.StatusNoContent { - return nil, nil - } - return nil, err - } - var span common.Span - err = json.Unmarshal(buf, &span) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &span, nil -} - -func (hcl *Client) WriteSpans(spans []*common.Span) error { - if hcl.hrpcAddr == "" { - return hcl.writeSpansHttp(spans) - } - hcr, err := newHClient(hcl.hrpcAddr, hcl.testHooks) - if err != nil { - return err - } - defer hcr.Close() - return hcr.writeSpans(spans) -} - -func (hcl *Client) writeSpansHttp(spans []*common.Span) error { - req := common.WriteSpansReq { - NumSpans: len(spans), - } - var w bytes.Buffer - enc := json.NewEncoder(&w) - err := enc.Encode(req) - if err != nil { - return errors.New(fmt.Sprintf("Error serializing WriteSpansReq: %s", - err.Error())) - } - for spanIdx := range(spans) { - err := enc.Encode(spans[spanIdx]) - if err != nil { - return errors.New(fmt.Sprintf("Error serializing span %d out " + - "of %d: %s", spanIdx, len(spans), err.Error())) - } - } - _, _, err = hcl.makeRestRequest("POST", "writeSpans", &w) - if err != nil { - return err - } - return nil -} - -// Find the child IDs of a given span ID. -func (hcl *Client) FindChildren(sid common.SpanId, lim int) ([]common.SpanId, error) { - buf, _, err := hcl.makeGetRequest(fmt.Sprintf("span/%s/children?lim=%d", - sid.String(), lim)) - if err != nil { - return nil, err - } - var spanIds []common.SpanId - err = json.Unmarshal(buf, &spanIds) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return spanIds, nil -} - -// Make a query -func (hcl *Client) Query(query *common.Query) ([]common.Span, error) { - in, err := json.Marshal(query) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error marshalling query: %s", err.Error())) - } - var out []byte - var url = fmt.Sprintf("query?query=%s", in) - out, _, err = hcl.makeGetRequest(url) - if err != nil { - return nil, err - } - var spans []common.Span - err = json.Unmarshal(out, &spans) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error unmarshalling results: %s", err.Error())) - } - return spans, nil -} - -func (hcl *Client) makeGetRequest(reqName string) ([]byte, int, error) { - return hcl.makeRestRequest("GET", reqName, nil) -} - -// Make a general JSON REST request. -// Returns the request body, the response code, and the error. -// Note: if the response code is non-zero, the error will also be non-zero. -func (hcl *Client) makeRestRequest(reqType string, reqName string, - reqBody io.Reader) ([]byte, int, error) { - url := fmt.Sprintf("http://%s/%s";, - hcl.restAddr, reqName) - req, err := http.NewRequest(reqType, url, reqBody) - req.Header.Set("Content-Type", "application/json") - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, -1, errors.New(fmt.Sprintf("Error: error making http request to %s: %s\n", url, - err.Error())) - } - defer resp.Body.Close() - body, err2 := ioutil.ReadAll(resp.Body) - if err2 != nil { - return nil, -1, errors.New(fmt.Sprintf("Error: error reading response body: %s\n", err2.Error())) - } - if resp.StatusCode != http.StatusOK { - return nil, resp.StatusCode, - errors.New(fmt.Sprintf("Error: got bad response status from " + - "%s: %s\n%s\n", url, resp.Status, body)) - } - return body, 0, nil -} - -// Dump all spans from the htraced daemon. -func (hcl *Client) DumpAll(lim int, out chan *common.Span) error { - defer func() { - close(out) - }() - searchId := common.INVALID_SPAN_ID - for { - q := common.Query{ - Lim: lim, - Predicates: []common.Predicate{ - common.Predicate{ - Op: "ge", - Field: "spanid", - Val: searchId.String(), - }, - }, - } - spans, err := hcl.Query(&q) - if err != nil { - return errors.New(fmt.Sprintf("Error querying spans with IDs at or after "+ - "%s: %s", searchId.String(), err.Error())) - } - if len(spans) == 0 { - return nil - } - for i := range spans { - out <- &spans[i] - } - searchId = spans[len(spans)-1].Id.Next() - } -} - -func (hcl *Client) Close() { - hcl.restAddr = "" - hcl.hrpcAddr = "" -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/client/hclient.go b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go deleted file mode 100644 index 43f0c6c..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/client/hclient.go +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package client - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "github.com/ugorji/go/codec" - "io" - "net" - "net/rpc" - "org/apache/htrace/common" -) - -type hClient struct { - rpcClient *rpc.Client -} - -type HrpcClientCodec struct { - rwc io.ReadWriteCloser - length uint32 - testHooks *TestHooks -} - -func (cdc *HrpcClientCodec) WriteRequest(rr *rpc.Request, msg interface{}) error { - methodId := common.HrpcMethodNameToId(rr.ServiceMethod) - if methodId == common.METHOD_ID_NONE { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s", - rr.ServiceMethod)) - } - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - w := bytes.NewBuffer(make([]byte, 0, 2048)) - - var err error - enc := codec.NewEncoder(w, mh) - if methodId == common.METHOD_ID_WRITE_SPANS { - spans := msg.([]*common.Span) - req := &common.WriteSpansReq { - NumSpans: len(spans), - } - err = enc.Encode(req) - if err != nil { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ - "message as msgpack: %s", err.Error())) - } - for spanIdx := range(spans) { - err = enc.Encode(spans[spanIdx]) - if err != nil { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ - "span %d out of %d as msgpack: %s", spanIdx, len(spans), err.Error())) - } - } - } else { - err = enc.Encode(msg) - if err != nil { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ - "message as msgpack: %s", err.Error())) - } - } - buf := w.Bytes() - if len(buf) > common.MAX_HRPC_BODY_LENGTH { - return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+ - "bytes, but the maximum message size is %d bytes.", - len(buf), common.MAX_HRPC_BODY_LENGTH)) - } - hdr := common.HrpcRequestHeader{ - Magic: common.HRPC_MAGIC, - MethodId: methodId, - Seq: rr.Seq, - Length: uint32(len(buf)), - } - err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr) - if err != nil { - return errors.New(fmt.Sprintf("Error writing header bytes: %s", - err.Error())) - } - if cdc.testHooks != nil && cdc.testHooks.HandleWriteRequestBody != nil { - cdc.testHooks.HandleWriteRequestBody() - } - _, err = cdc.rwc.Write(buf) - if err != nil { - return errors.New(fmt.Sprintf("Error writing body bytes: %s", - err.Error())) - } - return nil -} - -func (cdc *HrpcClientCodec) ReadResponseHeader(resp *rpc.Response) error { - hdr := common.HrpcResponseHeader{} - err := binary.Read(cdc.rwc, binary.LittleEndian, &hdr) - if err != nil { - return errors.New(fmt.Sprintf("Error reading response header "+ - "bytes: %s", err.Error())) - } - resp.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) - if resp.ServiceMethod == "" { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "invalid method ID %d.", hdr.MethodId)) - } - resp.Seq = hdr.Seq - if hdr.ErrLength > 0 { - if hdr.ErrLength > common.MAX_HRPC_ERROR_LENGTH { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "error message was %d bytes long, but "+ - "MAX_HRPC_ERROR_LENGTH is %d.", - hdr.ErrLength, common.MAX_HRPC_ERROR_LENGTH)) - } - buf := make([]byte, hdr.ErrLength) - var nread int - nread, err = cdc.rwc.Read(buf) - if uint32(nread) != hdr.ErrLength { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "failed to read %d bytes of error message.", nread)) - } - if err != nil { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "failed to read %d bytes of error message: %s", - nread, err.Error())) - } - resp.Error = string(buf) - } else { - resp.Error = "" - } - cdc.length = hdr.Length - return nil -} - -func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error { - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh) - err := dec.Decode(body) - if err != nil { - return errors.New(fmt.Sprintf("Failed to read response body: %s", - err.Error())) - } - return nil -} - -func (cdc *HrpcClientCodec) Close() error { - return cdc.rwc.Close() -} - -func newHClient(hrpcAddr string, testHooks *TestHooks) (*hClient, error) { - hcr := hClient{} - conn, err := net.Dial("tcp", hrpcAddr) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error contacting the HRPC server "+ - "at %s: %s", hrpcAddr, err.Error())) - } - hcr.rpcClient = rpc.NewClientWithCodec(&HrpcClientCodec{ - rwc: conn, - testHooks: testHooks, - }) - return &hcr, nil -} - -func (hcr *hClient) writeSpans(spans []*common.Span) error { - resp := common.WriteSpansResp{} - return hcr.rpcClient.Call(common.METHOD_NAME_WRITE_SPANS, spans, &resp) -} - -func (hcr *hClient) Close() { - hcr.rpcClient.Close() -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/log.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log.go b/htrace-htraced/go/src/org/apache/htrace/common/log.go deleted file mode 100644 index 8cb3953..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/log.go +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "errors" - "fmt" - "log" - "org/apache/htrace/conf" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" -) - -// A logSink is a place logs can be written to. -type logSink struct { - path logPath - file *os.File - lock sync.Mutex - refCount int // protected by logFilesLock -} - -// Write to the logSink. -func (sink *logSink) write(str string) { - sink.lock.Lock() - defer sink.lock.Unlock() - _, err := sink.file.Write([]byte(str)) - if err != nil { - fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error()) - } -} - -// Unreference the logSink. If there are no more references, and the logSink is -// closeable, then we will close it here. -func (sink *logSink) Unref() { - logFilesLock.Lock() - defer logFilesLock.Unlock() - sink.refCount-- - if sink.refCount <= 0 { - if sink.path.IsCloseable() { - err := sink.file.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n", - sink.path, err.Error()) - } - } - logSinks[sink.path] = nil - } -} - -type logPath string - -// An empty LogPath represents "stdout." -const STDOUT_LOG_PATH = "" - -// Convert a path to a logPath. -func logPathFromString(path string) logPath { - if path == STDOUT_LOG_PATH { - return logPath("") - } - absPath, err := filepath.Abs(path) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n", - path, err.Error()) - return logPath(path) - } - return logPath(absPath) -} - -// Convert the path to a human-readable string. -func (path logPath) String() string { - if path == "" { - return "(stdout)" - } else { - return string(path) - } -} - -// Return true if the path is closeable. stdout is not closeable. -func (path logPath) IsCloseable() bool { - return path != STDOUT_LOG_PATH -} - -func (path logPath) Open() *logSink { - if path == STDOUT_LOG_PATH { - return &logSink{path: path, file: os.Stdout} - } - file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout} - fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n", - path, err.Error()) - return sink - } - return &logSink{path: path, file: file} -} - -var logFilesLock sync.Mutex - -var logSinks map[logPath]*logSink = make(map[logPath]*logSink) - -func getOrCreateLogSink(pathStr string) *logSink { - path := logPathFromString(pathStr) - logFilesLock.Lock() - defer logFilesLock.Unlock() - sink := logSinks[path] - if sink == nil { - sink = path.Open() - logSinks[path] = sink - } - sink.refCount++ - return sink -} - -type Level int - -const ( - TRACE Level = iota - DEBUG - INFO - WARN - ERROR -) - -var levelToString map[Level]string = map[Level]string{ - TRACE: "TRACE", - DEBUG: "DEBUG", - INFO: "INFO", - WARN: "WARN", - ERROR: "ERROR", -} - -func (level Level) String() string { - return levelToString[level] -} - -func (level Level) LogString() string { - return level.String()[0:1] -} - -func LevelFromString(str string) (Level, error) { - for k, v := range levelToString { - if strings.ToLower(v) == strings.ToLower(str) { - return k, nil - } - } - var levelNames sort.StringSlice - levelNames = make([]string, len(levelToString)) - var i int - for _, v := range levelToString { - levelNames[i] = v - i++ - } - sort.Sort(levelNames) - return TRACE, errors.New(fmt.Sprintf("No such level as '%s'. Valid "+ - "levels are '%v'\n", str, levelNames)) -} - -type Logger struct { - sink *logSink - Level Level -} - -func NewLogger(faculty string, cnf *conf.Config) *Logger { - path, level := parseConf(faculty, cnf) - sink := getOrCreateLogSink(path) - return &Logger{sink: sink, Level: level} -} - -func parseConf(faculty string, cnf *conf.Config) (string, Level) { - facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH - var facultyLogPath string - if cnf.Contains(facultyLogPathKey) { - facultyLogPath = cnf.Get(facultyLogPathKey) - } else { - facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH) - } - facultyLogLevelKey := faculty + "." + conf.HTRACE_LOG_LEVEL - var facultyLogLevelStr string - if cnf.Contains(facultyLogLevelKey) { - facultyLogLevelStr = cnf.Get(facultyLogLevelKey) - } else { - facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL) - } - level, err := LevelFromString(facultyLogLevelStr) - if err != nil { - fmt.Fprintf(os.Stderr, "Error configuring log level: %s. Using TRACE.\n") - level = TRACE - } - return facultyLogPath, level -} - -func (lg *Logger) Trace(str string) { - lg.Write(TRACE, str) -} - -func (lg *Logger) Tracef(format string, v ...interface{}) { - lg.Write(TRACE, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Debug(str string) { - lg.Write(DEBUG, str) -} - -func (lg *Logger) Debugf(format string, v ...interface{}) { - lg.Write(DEBUG, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Info(str string) { - lg.Write(INFO, str) -} - -func (lg *Logger) Infof(format string, v ...interface{}) { - lg.Write(INFO, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Warn(str string) error { - lg.Write(WARN, str) - return errors.New(str) -} - -func (lg *Logger) Warnf(format string, v ...interface{}) error { - str := fmt.Sprintf(format, v...) - lg.Write(WARN, str) - return errors.New(str) -} - -func (lg *Logger) Error(str string) error { - lg.Write(ERROR, str) - return errors.New(str) -} - -func (lg *Logger) Errorf(format string, v ...interface{}) error { - str := fmt.Sprintf(format, v...) - lg.Write(ERROR, str) - return errors.New(str) -} - -func (lg *Logger) Write(level Level, str string) { - if level >= lg.Level { - lg.sink.write(time.Now().UTC().Format(time.RFC3339) + " " + - level.LogString() + ": " + str) - } -} - -// -// A few functions which can be used to determine if a certain level of tracing -// is enabled. These are useful in situations when evaluating the parameters -// of a logging function is expensive. (Note, however, that we don't pay the -// cost of string concatenation and manipulation when a log message doesn't -// trigger.) -// - -func (lg *Logger) TraceEnabled() bool { - return lg.Level <= TRACE -} - -func (lg *Logger) DebugEnabled() bool { - return lg.Level <= DEBUG -} - -func (lg *Logger) InfoEnabled() bool { - return lg.Level <= INFO -} - -func (lg *Logger) WarnEnabled() bool { - return lg.Level <= WARN -} - -func (lg *Logger) ErrorEnabled() bool { - return lg.Level <= ERROR -} - -func (lg *Logger) LevelEnabled(level Level) bool { - return lg.Level <= level -} - -func (lg *Logger) Close() { - lg.sink.Unref() - lg.sink = nil -} - -// Wraps an htrace logger in a golang standard logger. -// -// This is a bit messy because of the difference in interfaces between the -// golang standard logger and the htrace logger. The golang standard logger -// doesn't support log levels directly, so you must choose up front what htrace -// log level all messages should be treated as. Golang standard loggers expect -// to be able to write to an io.Writer, but make no guarantees about whether -// they will break messages into multiple Write() calls (although this does -// not seem to be a major problem in practice.) -// -// Despite these limitations, it's still useful to have this method to be able -// to log things that come out of the go HTTP server and other standard library -// systems. -type WrappedLogger struct { - lg *Logger - level Level -} - -func (lg *Logger) Wrap(prefix string, level Level) *log.Logger { - wlg := &WrappedLogger{ - lg: lg, - level: level, - } - return log.New(wlg, prefix, 0) -} - -func (wlg *WrappedLogger) Write(p []byte) (int, error) { - wlg.lg.Write(wlg.level, string(p)) - return len(p), nil -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log_test.go b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go deleted file mode 100644 index f0b1cde..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/log_test.go +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bufio" - "fmt" - "io" - "io/ioutil" - "org/apache/htrace/conf" - "os" - "strings" - "testing" -) - -func newLogger(faculty string, args ...string) *Logger { - cnfBld := conf.Builder{Defaults: conf.DEFAULTS} - cnf, err := cnfBld.Build() - if err != nil { - panic(fmt.Sprintf("failed to create conf: %s", err.Error())) - } - cnf2 := cnf.Clone(args...) - lg := NewLogger(faculty, cnf2) - return lg -} - -func TestNewLogger(t *testing.T) { - lg := newLogger("foo", "log.level", "TRACE") - lg.Close() -} - -func verifyLines(t *testing.T, rdr io.Reader, lines []string) { - scanner := bufio.NewScanner(rdr) - lineIdx := 0 - for scanner.Scan() { - line := scanner.Text() - if !strings.Contains(line, lines[lineIdx]) { - t.Fatalf("Error on line %d: didn't find substring '%s' in line '%s'\n", - (lineIdx + 1), lines[lineIdx], line) - } - lineIdx++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err.Error()) - } -} - -func TestFileLogs(t *testing.T) { - tempDir, err := ioutil.TempDir(os.TempDir(), "testFileLogs") - if err != nil { - panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) - } - defer os.RemoveAll(tempDir) - logPath := tempDir + conf.PATH_SEP + "log" - lg := newLogger("foo", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - lg.Tracef("Non-important stuff, ignore this.\n") - lg.Infof("problem with the foobar\n") - lg.Tracef("More non-important stuff, also ignore this.\n") - lg.Infof("and another problem with the foobar\n") - logFile, err := os.Open(logPath) - if err != nil { - t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) - } - verifyLines(t, logFile, []string{ - "problem with the foobar", - "and another problem with the foobar", - }) - logFile.Close() - lg.Close() -} - -func TestMultipleFileLogs(t *testing.T) { - tempDir, err := ioutil.TempDir(os.TempDir(), "testMultipleFileLogs") - if err != nil { - panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) - } - defer os.RemoveAll(tempDir) - logPath := tempDir + conf.PATH_SEP + "log" - fooLg := newLogger("foo", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - fooLg.Infof("The foo needs maintenance.\n") - barLg := newLogger("bar", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - barLg.Debugf("The bar is open\n") - fooLg.Errorf("Fizz buzz\n") - logFile, err := os.Open(logPath) - if err != nil { - t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) - } - fooLg.Tracef("Fizz buzz2\n") - barLg.Tracef("Fizz buzz3\n") - verifyLines(t, logFile, []string{ - "The foo needs maintenance.", - "The bar is open", - "Fizz buzz", - "Fizz buzz3", - }) - logFile.Close() - fooLg.Close() - barLg.Close() -} - -func TestLogLevelEnabled(t *testing.T) { - tempDir, err := ioutil.TempDir(os.TempDir(), "TestLogLevelEnabled") - if err != nil { - panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) - } - defer os.RemoveAll(tempDir) - // set log level to DEBUG for facility "foo" - logPath := tempDir + conf.PATH_SEP + "log" - lg := newLogger("foo", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - if lg.TraceEnabled() { - t.Fatalf("foo logger has TraceEnabled") - } - if lg.DebugEnabled() { - t.Fatalf("foo logger have DebugEnabled") - } - if !lg.InfoEnabled() { - t.Fatalf("foo logger does not have InfoEnabled") - } - if !lg.WarnEnabled() { - t.Fatalf("foo logger does not have WarnEnabled") - } - if !lg.ErrorEnabled() { - t.Fatalf("foo logger does not have ErrorEnabled") - } - lg.Close() - lg = newLogger("foo", "log.level", "WARN", - "foo.log.level", "INFO", - "log.path", logPath) - if lg.TraceEnabled() { - t.Fatalf("foo logger has TraceEnabled") - } - if lg.DebugEnabled() { - t.Fatalf("foo logger has DebugEnabled") - } - if !lg.InfoEnabled() { - t.Fatalf("foo logger does not have InfoEnabled") - } - if !lg.WarnEnabled() { - t.Fatalf("foo logger does not have WarnEnabled") - } - if !lg.ErrorEnabled() { - t.Fatalf("foo logger does not have ErrorEnabled") - } - lg.Close() -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/process.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process.go b/htrace-htraced/go/src/org/apache/htrace/common/process.go deleted file mode 100644 index ce32067..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/process.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bytes" - "fmt" - "org/apache/htrace/conf" - "os" - "os/signal" - "runtime" - "runtime/debug" - "syscall" -) - -func InstallSignalHandlers(cnf *conf.Config) { - fatalSigs := []os.Signal{ - os.Interrupt, - os.Kill, - syscall.SIGINT, - syscall.SIGABRT, - syscall.SIGALRM, - syscall.SIGBUS, - syscall.SIGFPE, - syscall.SIGILL, - syscall.SIGSEGV, - syscall.SIGTERM, - } - fatalSigChan := make(chan os.Signal, 1) - signal.Notify(fatalSigChan, fatalSigs...) - lg := NewLogger("signal", cnf) - go func() { - sig := <-fatalSigChan - lg.Errorf("Terminating on signal: %v\n", sig) - lg.Close() - os.Exit(1) - }() - - sigQuitChan := make(chan os.Signal, 1) - signal.Notify(sigQuitChan, syscall.SIGQUIT) - go func() { - stackTraceBuf := make([]byte, 1<<20) - for { - <-sigQuitChan - GetStackTraces(&stackTraceBuf) - lg.Info("=== received SIGQUIT ===\n") - lg.Info("=== GOROUTINE STACKS ===\n") - lg.Info(string(stackTraceBuf)) - lg.Info("\n=== END GOROUTINE STACKS ===\n") - lg.Info("=== GC STATISTICS ===\n") - lg.Info(GetGCStats()) - lg.Info("=== END GC STATISTICS ===\n") - } - }() -} - -func GetStackTraces(buf *[]byte) { - *buf = (*buf)[0:cap(*buf)] - neededBytes := runtime.Stack(*buf, true) - for neededBytes > len(*buf) { - *buf = make([]byte, neededBytes) - runtime.Stack(*buf, true) - } - *buf = (*buf)[0:neededBytes] -} - -func GetGCStats() string { - gcs := debug.GCStats{} - debug.ReadGCStats(&gcs) - var buf bytes.Buffer - buf.WriteString(fmt.Sprintf("LastGC: %s\n", gcs.LastGC.UTC().String())) - buf.WriteString(fmt.Sprintf("NumGC: %d\n", gcs.NumGC)) - buf.WriteString(fmt.Sprintf("PauseTotal: %v\n", gcs.PauseTotal)) - if gcs.Pause != nil { - pauseStr := "" - prefix := "" - for p := range gcs.Pause { - pauseStr += prefix + gcs.Pause[p].String() - prefix = ", " - } - buf.WriteString(fmt.Sprintf("Pause History: %s\n", pauseStr)) - } - return buf.String() -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process_test.go b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go deleted file mode 100644 index d3f5a56..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/process_test.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bufio" - "fmt" - "org/apache/htrace/conf" - "os" - "os/exec" - "strings" - "syscall" - "testing" - "time" -) - -const HTRACED_TEST_HELPER_PROCESS = "HTRACED_TEST_HELPER_PROCESS" - -// This test runs a helper process which will install our htraced signal -// handlers. We will send signals to the subprocess and verify that it has -// caught them and responded appropriately. -func TestSignals(t *testing.T) { - if os.Getenv(HTRACED_TEST_HELPER_PROCESS) == "1" { - runHelperProcess() - os.Exit(0) - } - helper := exec.Command(os.Args[0], "-test.run=TestSignals", "--") - helper.Env = []string{HTRACED_TEST_HELPER_PROCESS + "=1"} - stdoutPipe, err := helper.StdoutPipe() - if err != nil { - panic(fmt.Sprintf("Failed to open pipe to process stdout: %s", - err.Error())) - } - stderrPipe, err := helper.StderrPipe() - if err != nil { - panic(fmt.Sprintf("Failed to open pipe to process stderr: %s", - err.Error())) - } - err = helper.Start() - if err != nil { - t.Fatal("Failed to start command %s: %s\n", os.Args[0], err.Error()) - } - t.Logf("Started suprocess...\n") - done := make(chan interface{}) - go func() { - scanner := bufio.NewScanner(stdoutPipe) - for scanner.Scan() { - text := scanner.Text() - if strings.Contains(text, "=== GOROUTINE STACKS ===") { - break - } - } - t.Logf("Saw 'GOROUTINE STACKS on stdout.' Sending SIGINT.\n") - helper.Process.Signal(syscall.SIGINT) - for scanner.Scan() { - text := scanner.Text() - if strings.Contains(text, "Terminating on signal: SIGINT") { - break - } - } - t.Logf("Saw 'Terminating on signal: SIGINT'. " + - "Helper goroutine exiting.\n") - done <- nil - }() - scanner := bufio.NewScanner(stderrPipe) - for scanner.Scan() { - text := scanner.Text() - if strings.Contains(text, "Signal handler installed.") { - break - } - } - t.Logf("Saw 'Signal handler installed.' Sending SIGINT.") - helper.Process.Signal(syscall.SIGQUIT) - t.Logf("Waiting for helper goroutine to exit.\n") - <-done - t.Logf("Waiting for subprocess to exit.\n") - helper.Wait() - t.Logf("Done.") -} - -// Run the helper process which TestSignals spawns. -func runHelperProcess() { - cnfMap := map[string]string{ - conf.HTRACE_LOG_LEVEL: "TRACE", - conf.HTRACE_LOG_PATH: "", // log to stdout - } - cnfBld := conf.Builder{Values: cnfMap, Defaults: conf.DEFAULTS} - cnf, err := cnfBld.Build() - if err != nil { - fmt.Printf("Error building configuration: %s\n", err.Error()) - os.Exit(1) - } - InstallSignalHandlers(cnf) - fmt.Fprintf(os.Stderr, "Signal handler installed.\n") - // Wait for a signal to be delivered - for { - time.Sleep(time.Hour * 100) - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/query.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query.go b/htrace-htraced/go/src/org/apache/htrace/common/query.go deleted file mode 100644 index 7a9e523..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/query.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "encoding/json" -) - -// -// Represents queries that can be sent to htraced. -// -// Each query consists of set of predicates that will be 'AND'ed together to -// return a set of spans. Predicates contain an operation, a field, and a -// value. -// -// For example, a query might be "return the first 100 spans between 5:00pm -// and 5:01pm" This query would have two predicates: time greater than or -// equal to 5:00pm, and time less than or equal to 5:01pm. -// In HTrace, times are always expressed in milliseconds since the Epoch. -// So this would become: -// { "lim" : 100, "pred" : [ -// { "op" : "ge", "field" : "begin", "val" : 1234 }, -// { "op" : "le", "field" : "begin", "val" : 5678 }, -// ] } -// -// Where '1234' and '5678' were replaced by times since the epoch in -// milliseconds. -// - -type Op string - -const ( - CONTAINS Op = "cn" - EQUALS Op = "eq" - LESS_THAN_OR_EQUALS Op = "le" - GREATER_THAN_OR_EQUALS Op = "ge" - GREATER_THAN Op = "gt" -) - -func (op Op) IsDescending() bool { - return op == LESS_THAN_OR_EQUALS -} - -func (op Op) IsValid() bool { - ops := ValidOps() - for i := range ops { - if ops[i] == op { - return true - } - } - return false -} - -func ValidOps() []Op { - return []Op{CONTAINS, EQUALS, LESS_THAN_OR_EQUALS, GREATER_THAN_OR_EQUALS, - GREATER_THAN} -} - -type Field string - -const ( - SPAN_ID Field = "spanid" - DESCRIPTION Field = "description" - BEGIN_TIME Field = "begin" - END_TIME Field = "end" - DURATION Field = "duration" - TRACER_ID Field = "tracerid" -) - -func (field Field) IsValid() bool { - fields := ValidFields() - for i := range fields { - if fields[i] == field { - return true - } - } - return false -} - -func ValidFields() []Field { - return []Field{SPAN_ID, DESCRIPTION, BEGIN_TIME, END_TIME, - DURATION, TRACER_ID} -} - -type Predicate struct { - Op Op `json:"op"` - Field Field `json:"field"` - Val string `val:"val"` -} - -func (pred *Predicate) String() string { - buf, err := json.Marshal(pred) - if err != nil { - panic(err) - } - return string(buf) -} - -type Query struct { - Predicates []Predicate `json:"pred"` - Lim int `json:"lim"` - Prev *Span `json:"prev"` -} - -func (query *Query) String() string { - buf, err := json.Marshal(query) - if err != nil { - panic(err) - } - return string(buf) -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query_test.go b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go deleted file mode 100644 index 2697d9c..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/query_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "testing" -) - -func TestValidOps(t *testing.T) { - for i := range ValidOps() { - op := ValidOps()[i] - if !op.IsValid() { - t.Fatalf("op %s was in ValidOps, but IsValid returned false.\n", op) - } - } - invalidOp := Op("completelybogus") - if invalidOp.IsValid() { - t.Fatalf("op %s was invalid, but IsValid returned true.\n", invalidOp) - } -} - -func TestValidFields(t *testing.T) { - for i := range ValidFields() { - field := ValidFields()[i] - if !field.IsValid() { - t.Fatalf("field %s was in ValidFields, but IsValid returned false.\n", field) - } - } - invalidField := Field("completelybogus") - if invalidField.IsValid() { - t.Fatalf("field %s was invalid, but IsValid returned true.\n", invalidField) - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go deleted file mode 100644 index 5f02db6..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -// The 4-byte magic number which is sent first in the HRPC header -const HRPC_MAGIC = 0x43525448 - -// Method ID codes. Do not reorder these. -const ( - METHOD_ID_NONE = 0 - METHOD_ID_WRITE_SPANS = iota -) - -const METHOD_NAME_WRITE_SPANS = "HrpcHandler.WriteSpans" - -// Maximum length of the error message passed in an HRPC response -const MAX_HRPC_ERROR_LENGTH = 4 * 1024 * 1024 - -// Maximum length of HRPC message body -const MAX_HRPC_BODY_LENGTH = 32 * 1024 * 1024 - -// A request to write spans to htraced. -// This request is followed by a sequence of spans. -type WriteSpansReq struct { - DefaultTrid string `json:",omitempty"` - NumSpans int -} - -// Info returned by /server/version -type ServerVersion struct { - // The server release version. - ReleaseVersion string - - // The git hash that this software was built with. - GitVersion string -} - -// A response to a WriteSpansReq -type WriteSpansResp struct { -} - -// The header which is sent over the wire for HRPC -type HrpcRequestHeader struct { - Magic uint32 - MethodId uint32 - Seq uint64 - Length uint32 -} - -// The response which is sent over the wire for HRPC -type HrpcResponseHeader struct { - Seq uint64 - MethodId uint32 - ErrLength uint32 - Length uint32 -} - -func HrpcMethodIdToMethodName(id uint32) string { - switch id { - case METHOD_ID_WRITE_SPANS: - return METHOD_NAME_WRITE_SPANS - default: - return "" - } -} - -func HrpcMethodNameToId(name string) uint32 { - switch name { - case METHOD_NAME_WRITE_SPANS: - return METHOD_ID_WRITE_SPANS - default: - return METHOD_ID_NONE - } -} - -type SpanMetrics struct { - // The total number of spans written to HTraced. - Written uint64 - - // The total number of spans dropped by the server. - ServerDropped uint64 -} - -// A map from network address strings to SpanMetrics structures. -type SpanMetricsMap map[string]*SpanMetrics - -// Info returned by /server/stats -type ServerStats struct { - // Statistics for each shard (directory) - Dirs []StorageDirectoryStats - - // Per-host Span Metrics - HostSpanMetrics SpanMetricsMap - - // The time (in UTC milliseconds since the epoch) when the - // datastore was last started. - LastStartMs int64 - - // The current time (in UTC milliseconds since the epoch) on the server. - CurMs int64 - - // The total number of spans which have been reaped. - ReapedSpans uint64 - - // The total number of spans which have been ingested since the server started, by WriteSpans - // requests. This number counts spans that didn't get written to persistent storage as well as - // those that did. - IngestedSpans uint64 - - // The total number of spans which have been written to leveldb since the server started. - WrittenSpans uint64 - - // The total number of spans dropped by the server since the server started. - ServerDroppedSpans uint64 - - // The maximum latency of a writeSpans request, in milliseconds. - MaxWriteSpansLatencyMs uint32 - - // The average latency of a writeSpans request, in milliseconds. - AverageWriteSpansLatencyMs uint32 -} - -type StorageDirectoryStats struct { - Path string - - // The approximate number of bytes on disk present in this shard. - ApproximateBytes uint64 - - // leveldb.stats information - LevelDbStats string -} - -type ServerDebugInfoReq struct { -} - -type ServerDebugInfo struct { - // Stack traces from all goroutines - StackTraces string - - // Garbage collection statistics - GCStats string -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/semaphore.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/semaphore.go b/htrace-htraced/go/src/org/apache/htrace/common/semaphore.go deleted file mode 100644 index 1acde76..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/semaphore.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "sync" -) - -// A simple lock-and-condition-variable based semaphore implementation. -type Semaphore struct { - lock sync.Mutex - cond *sync.Cond - count int64 -} - -func NewSemaphore(count int64) *Semaphore { - sem := &Semaphore{ - count: int64(count), - } - sem.cond = &sync.Cond{ - L: &sem.lock, - } - return sem -} - -func (sem *Semaphore) Post() { - sem.lock.Lock() - sem.count++ - if sem.count > 0 { - sem.cond.Broadcast() - } - sem.lock.Unlock() -} - -func (sem *Semaphore) Posts(amt int64) { - sem.lock.Lock() - sem.count += amt - if sem.count > 0 { - sem.cond.Broadcast() - } - sem.lock.Unlock() -} - -func (sem *Semaphore) Wait() { - sem.lock.Lock() - for { - if sem.count > 0 { - sem.count-- - sem.lock.Unlock() - return - } - sem.cond.Wait() - } -} - -func (sem *Semaphore) Waits(amt int64) { - var i int64 - for i = 0; i < amt; i++ { - sem.Wait() - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/semaphore_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/semaphore_test.go b/htrace-htraced/go/src/org/apache/htrace/common/semaphore_test.go deleted file mode 100644 index 089c51b..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/semaphore_test.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "sync/atomic" - "testing" - "time" -) - -func TestSemaphoreWake(t *testing.T) { - var done uint32 - sem := NewSemaphore(0) - go func() { - time.Sleep(10 * time.Nanosecond) - atomic.AddUint32(&done, 1) - sem.Post() - }() - sem.Wait() - doneVal := atomic.LoadUint32(&done) - if doneVal != 1 { - t.Fatalf("sem.Wait did not wait for sem.Post") - } -} - -func TestSemaphoreCount(t *testing.T) { - sem := NewSemaphore(1) - sem.Post() - sem.Wait() - sem.Wait() - - sem = NewSemaphore(-1) - sem.Post() - sem.Post() - sem.Wait() -} - -func TestSemaphoreMultipleGoroutines(t *testing.T) { - var done uint32 - sem := NewSemaphore(0) - sem2 := NewSemaphore(0) - go func() { - sem.Wait() - atomic.AddUint32(&done, 1) - sem2.Post() - }() - go func() { - time.Sleep(10 * time.Nanosecond) - atomic.AddUint32(&done, 1) - sem.Post() - }() - go func() { - time.Sleep(20 * time.Nanosecond) - atomic.AddUint32(&done, 1) - sem.Post() - }() - sem.Wait() - go func() { - time.Sleep(10 * time.Nanosecond) - atomic.AddUint32(&done, 1) - sem.Post() - }() - sem.Wait() - sem2.Wait() - doneVal := atomic.LoadUint32(&done) - if doneVal != 4 { - t.Fatalf("sem.Wait did not wait for sem.Posts") - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/span.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span.go b/htrace-htraced/go/src/org/apache/htrace/common/span.go deleted file mode 100644 index 1716c5a..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/span.go +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "hash/fnv" -) - -// -// Represents a trace span. -// -// Compatibility notes: -// When converting to JSON, we store the 64-bit numbers as hexadecimal strings rather than as -// integers. This is because JavaScript lacks the ability to handle 64-bit integers. Numbers above -// about 55 bits will be rounded by Javascript. Since the Javascript UI is a primary consumer of -// this JSON data, we have to simply pass it as a string. -// - -type TraceInfoMap map[string]string - -type TimelineAnnotation struct { - Time int64 `json:"t"` - Msg string `json:"m"` -} - -type SpanId []byte - -var INVALID_SPAN_ID SpanId = make([]byte, 16) // all zeroes - -func (id SpanId) String() string { - return fmt.Sprintf("%02x%02x%02x%02x"+ - "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", - id[0], id[1], id[2], id[3], id[4], id[5], id[6], id[7], id[8], - id[9], id[10], id[11], id[12], id[13], id[14], id[15]) -} - -func (id SpanId) Val() []byte { - return []byte(id) -} - -func (id SpanId) FindProblem() string { - if id == nil { - return "The span ID is nil" - } - if len(id) != 16 { - return "The span ID is not exactly 16 bytes." - } - if bytes.Equal(id.Val(), INVALID_SPAN_ID.Val()) { - return "The span ID is all zeros." - } - return "" -} - -func (id SpanId) ToArray() [16]byte { - var ret [16]byte - copy(ret[:], id.Val()[:]) - return ret -} - -// Return the next ID in lexicographical order. For the maximum ID, -// returns the minimum. -func (id SpanId) Next() SpanId { - next := make([]byte, 16) - copy(next, id) - for i := len(next) - 1; i >= 0; i-- { - if next[i] == 0xff { - next[i] = 0 - } else { - next[i] = next[i] + 1 - break - } - } - return next -} - -// Return the previous ID in lexicographical order. For the minimum ID, -// returns the maximum ID. -func (id SpanId) Prev() SpanId { - prev := make([]byte, 16) - copy(prev, id) - for i := len(prev) - 1; i >= 0; i-- { - if prev[i] == 0x00 { - prev[i] = 0xff - } else { - prev[i] = prev[i] - 1 - break - } - } - return prev -} - -func (id SpanId) MarshalJSON() ([]byte, error) { - return []byte(`"` + id.String() + `"`), nil -} - -func (id SpanId) Compare(other SpanId) int { - return bytes.Compare(id.Val(), other.Val()) -} - -func (id SpanId) Equal(other SpanId) bool { - return bytes.Equal(id.Val(), other.Val()) -} - -func (id SpanId) Hash32() uint32 { - h := fnv.New32a() - h.Write(id.Val()) - return h.Sum32() -} - -type SpanSlice []*Span - -func (s SpanSlice) Len() int { - return len(s) -} - -func (s SpanSlice) Less(i, j int) bool { - return s[i].Id.Compare(s[j].Id) < 0 -} - -func (s SpanSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -type SpanIdSlice []SpanId - -func (s SpanIdSlice) Len() int { - return len(s) -} - -func (s SpanIdSlice) Less(i, j int) bool { - return s[i].Compare(s[j]) < 0 -} - -func (s SpanIdSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -const DOUBLE_QUOTE = 0x22 - -func (id *SpanId) UnmarshalJSON(b []byte) error { - if b[0] != DOUBLE_QUOTE { - return errors.New("Expected spanID to start with a string quote.") - } - if b[len(b)-1] != DOUBLE_QUOTE { - return errors.New("Expected spanID to end with a string quote.") - } - return id.FromString(string(b[1 : len(b)-1])) -} - -func (id *SpanId) FromString(str string) error { - i := SpanId(make([]byte, 16)) - n, err := fmt.Sscanf(str, "%02x%02x%02x%02x"+ - "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", - &i[0], &i[1], &i[2], &i[3], &i[4], &i[5], &i[6], &i[7], &i[8], - &i[9], &i[10], &i[11], &i[12], &i[13], &i[14], &i[15]) - if err != nil { - return err - } - if n != 16 { - return errors.New("Failed to find 16 hex digits in the SpanId") - } - *id = i - return nil -} - -type SpanData struct { - Begin int64 `json:"b"` - End int64 `json:"e"` - Description string `json:"d"` - Parents []SpanId `json:"p"` - Info TraceInfoMap `json:"n,omitempty"` - TracerId string `json:"r"` - TimelineAnnotations []TimelineAnnotation `json:"t,omitempty"` -} - -type Span struct { - Id SpanId `json:"a"` - SpanData -} - -func (span *Span) ToJson() []byte { - jbytes, err := json.Marshal(*span) - if err != nil { - panic(err) - } - return jbytes -} - -func (span *Span) String() string { - return string(span.ToJson()) -} - -// Compute the span duration. We ignore overflow since we never deal with negative times. -func (span *Span) Duration() int64 { - return span.End - span.Begin -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span_test.go b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go deleted file mode 100644 index 7fb128d..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/span_test.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bytes" - "encoding/hex" - "fmt" - "github.com/ugorji/go/codec" - "testing" -) - -func TestSpanToJson(t *testing.T) { - t.Parallel() - span := Span{Id: TestId("33f25a1a750a471db5bafa59309d7d6f"), - SpanData: SpanData{ - Begin: 123, - End: 456, - Description: "getFileDescriptors", - Parents: []SpanId{}, - TracerId: "testTracerId", - }} - ExpectStrEqual(t, - `{"a":"33f25a1a750a471db5bafa59309d7d6f","b":123,"e":456,"d":"getFileDescriptors","p":[],"r":"testTracerId"}`, - string(span.ToJson())) -} - -func TestAnnotatedSpanToJson(t *testing.T) { - t.Parallel() - span := Span{Id: TestId("11eace42e6404b40a7644214cb779a08"), - SpanData: SpanData{ - Begin: 1234, - End: 4567, - Description: "getFileDescriptors2", - Parents: []SpanId{}, - TracerId: "testAnnotatedTracerId", - TimelineAnnotations: []TimelineAnnotation{ - TimelineAnnotation{ - Time: 7777, - Msg: "contactedServer", - }, - TimelineAnnotation{ - Time: 8888, - Msg: "passedFd", - }, - }, - }} - ExpectStrEqual(t, - `{"a":"11eace42e6404b40a7644214cb779a08","b":1234,"e":4567,"d":"getFileDescriptors2","p":[],"r":"testAnnotatedTracerId","t":[{"t":7777,"m":"contactedServer"},{"t":8888,"m":"passedFd"}]}`, - string(span.ToJson())) -} - -func TestSpanNext(t *testing.T) { - ExpectStrEqual(t, TestId("00000000000000000000000000000001").String(), - TestId("00000000000000000000000000000000").Next().String()) - ExpectStrEqual(t, TestId("00000000000000000000000000f00000").String(), - TestId("00000000000000000000000000efffff").Next().String()) - ExpectStrEqual(t, TestId("00000000000000000000000000000000").String(), - TestId("ffffffffffffffffffffffffffffffff").Next().String()) -} - -func TestSpanPrev(t *testing.T) { - ExpectStrEqual(t, TestId("00000000000000000000000000000000").String(), - TestId("00000000000000000000000000000001").Prev().String()) - ExpectStrEqual(t, TestId("00000000000000000000000000efffff").String(), - TestId("00000000000000000000000000f00000").Prev().String()) - ExpectStrEqual(t, TestId("ffffffffffffffffffffffffffffffff").String(), - TestId("00000000000000000000000000000000").Prev().String()) -} - -func TestSpanMsgPack(t *testing.T) { - span := Span{Id: TestId("33f25a1a750a471db5bafa59309d7d6f"), - SpanData: SpanData{ - Begin: 1234, - End: 5678, - Description: "getFileDescriptors", - Parents: []SpanId{}, - TracerId: "testTracerId", - }} - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - w := bytes.NewBuffer(make([]byte, 0, 2048)) - enc := codec.NewEncoder(w, mh) - err := enc.Encode(span) - if err != nil { - t.Fatal("Error encoding span as msgpack: " + err.Error()) - } - buf := w.Bytes() - fmt.Printf("span: %s\n", hex.EncodeToString(buf)) - mh = new(codec.MsgpackHandle) - mh.WriteExt = true - dec := codec.NewDecoder(bytes.NewReader(buf), mh) - var span2 Span - err = dec.Decode(&span2) - if err != nil { - t.Fatal("Failed to reverse msgpack encoding for " + span.String()) - } - ExpectSpansEqual(t, &span, &span2) -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/test_util.go b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go deleted file mode 100644 index a761525..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/test_util.go +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "fmt" - "testing" - "strings" - "time" -) - -type Int64Slice []int64 - -func (p Int64Slice) Len() int { return len(p) } -func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -type SupplierFun func() bool - -// -// Wait for a configurable amount of time for a precondition to become true. -// -// Example: -// WaitFor(time.Minute * 1, time.Millisecond * 1, func() bool { -// return ht.Store.GetStatistics().NumSpansWritten >= 3 -// }) -// -func WaitFor(dur time.Duration, poll time.Duration, fun SupplierFun) { - if poll == 0 { - poll = dur / 10 - } - if poll <= 0 { - panic("Can't have a polling time less than zero.") - } - endTime := time.Now().Add(dur) - for { - if fun() { - return - } - if !time.Now().Before(endTime) { - break - } - time.Sleep(poll) - } - panic(fmt.Sprintf("Timed out after %s", dur)) -} - -// Trigger a test failure if two strings are not equal. -func ExpectStrEqual(t *testing.T, expect string, actual string) { - if expect != actual { - t.Fatalf("Expected:\n%s\nGot:\n%s\n", expect, actual) - } -} - -// Trigger a test failure if the JSON representation of two spans are not equals. -func ExpectSpansEqual(t *testing.T, spanA *Span, spanB *Span) { - ExpectStrEqual(t, string(spanA.ToJson()), string(spanB.ToJson())) -} - -func TestId(str string) SpanId { - var spanId SpanId - err := spanId.FromString(str) - if err != nil { - panic(err.Error()) - } - return spanId -} - -func AssertErrContains(t *testing.T, err error, str string) { - if !strings.Contains(err.Error(), str) { - t.Fatalf("expected the error to contain %s, but it was %s\n", - str, err.Error()) - } -} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/5737e65b/htrace-htraced/go/src/org/apache/htrace/common/time.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/time.go b/htrace-htraced/go/src/org/apache/htrace/common/time.go deleted file mode 100644 index 8b4b6b8..0000000 --- a/htrace-htraced/go/src/org/apache/htrace/common/time.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "time" -) - -func TimeToUnixMs(t time.Time) int64 { - return t.UnixNano() / 1000000 -} - -func UnixMsToTime(u int64) time.Time { - secs := u / 1000 - nanos := u - (secs * 1000) - return time.Unix(secs, nanos) -}
