HTRACE-183 Have go build into the target dir rather than under src Move go from src/go to go.
Change where htraced looks for the web dir from ../../web to ../web. Have gobuild.py create a symlink to the web dir at src/web beside the go dir so can do in-situ development. Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/42b2f6a2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/42b2f6a2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/42b2f6a2 Branch: refs/heads/master Commit: 42b2f6a2c22046fe5da348c301d49521f1501bf9 Parents: a560a55 Author: stack <[email protected]> Authored: Wed Jun 17 14:51:42 2015 -0700 Committer: stack <[email protected]> Committed: Wed Jun 17 15:22:43 2015 -0700 ---------------------------------------------------------------------- htrace-htraced/go/Godeps/Godeps.json | 30 + htrace-htraced/go/format.sh | 40 + htrace-htraced/go/gobuild.sh | 119 +++ .../go/src/org/apache/htrace/client/client.go | 245 +++++ .../go/src/org/apache/htrace/client/hclient.go | 157 +++ .../go/src/org/apache/htrace/common/log.go | 296 ++++++ .../go/src/org/apache/htrace/common/log_test.go | 122 +++ .../go/src/org/apache/htrace/common/process.go | 67 ++ .../go/src/org/apache/htrace/common/query.go | 126 +++ .../src/org/apache/htrace/common/query_test.go | 50 + .../go/src/org/apache/htrace/common/rest.go | 29 + .../go/src/org/apache/htrace/common/rpc.go | 81 ++ .../go/src/org/apache/htrace/common/span.go | 140 +++ .../src/org/apache/htrace/common/span_test.go | 66 ++ .../src/org/apache/htrace/common/test_util.go | 74 ++ .../go/src/org/apache/htrace/conf/config.go | 266 +++++ .../src/org/apache/htrace/conf/config_keys.go | 86 ++ .../src/org/apache/htrace/conf/config_test.go | 138 +++ .../go/src/org/apache/htrace/conf/xml.go | 61 ++ .../go/src/org/apache/htrace/htrace/cmd.go | 317 ++++++ .../go/src/org/apache/htrace/htrace/file.go | 138 +++ .../src/org/apache/htrace/htrace/file_test.go | 164 +++ .../go/src/org/apache/htrace/htrace/graph.go | 116 +++ .../src/org/apache/htrace/htrace/graph_test.go | 84 ++ .../go/src/org/apache/htrace/htrace/queries.go | 161 +++ .../org/apache/htrace/htraced/client_test.go | 206 ++++ .../src/org/apache/htrace/htraced/datastore.go | 999 +++++++++++++++++++ .../org/apache/htrace/htraced/datastore_test.go | 514 ++++++++++ .../go/src/org/apache/htrace/htraced/hrpc.go | 251 +++++ .../go/src/org/apache/htrace/htraced/htraced.go | 140 +++ .../org/apache/htrace/htraced/mini_htraced.go | 165 +++ .../go/src/org/apache/htrace/htraced/rest.go | 308 ++++++ .../go/src/org/apache/htrace/test/random.go | 72 ++ .../go/src/org/apache/htrace/test/util.go | 33 + htrace-htraced/pom.xml | 16 +- htrace-htraced/src/go/Godeps/Godeps.json | 30 - htrace-htraced/src/go/format.sh | 40 - htrace-htraced/src/go/gobuild.sh | 116 --- .../go/src/org/apache/htrace/client/client.go | 245 ----- .../go/src/org/apache/htrace/client/hclient.go | 157 --- .../src/go/src/org/apache/htrace/common/log.go | 296 ------ .../go/src/org/apache/htrace/common/log_test.go | 122 --- .../go/src/org/apache/htrace/common/process.go | 67 -- .../go/src/org/apache/htrace/common/query.go | 126 --- .../src/org/apache/htrace/common/query_test.go | 50 - .../src/go/src/org/apache/htrace/common/rest.go | 29 - .../src/go/src/org/apache/htrace/common/rpc.go | 81 -- .../src/go/src/org/apache/htrace/common/span.go | 140 --- .../src/org/apache/htrace/common/span_test.go | 66 -- .../src/org/apache/htrace/common/test_util.go | 74 -- .../src/go/src/org/apache/htrace/conf/config.go | 266 ----- .../src/org/apache/htrace/conf/config_keys.go | 86 -- .../src/org/apache/htrace/conf/config_test.go | 138 --- .../src/go/src/org/apache/htrace/conf/xml.go | 61 -- .../src/go/src/org/apache/htrace/htrace/cmd.go | 317 ------ .../src/go/src/org/apache/htrace/htrace/file.go | 138 --- .../src/org/apache/htrace/htrace/file_test.go | 164 --- .../go/src/org/apache/htrace/htrace/graph.go | 116 --- .../src/org/apache/htrace/htrace/graph_test.go | 84 -- .../go/src/org/apache/htrace/htrace/queries.go | 161 --- .../org/apache/htrace/htraced/client_test.go | 206 ---- .../src/org/apache/htrace/htraced/datastore.go | 999 ------------------- .../org/apache/htrace/htraced/datastore_test.go | 514 ---------- .../go/src/org/apache/htrace/htraced/hrpc.go | 251 ----- .../go/src/org/apache/htrace/htraced/htraced.go | 140 --- .../org/apache/htrace/htraced/mini_htraced.go | 165 --- .../go/src/org/apache/htrace/htraced/rest.go | 308 ------ .../src/go/src/org/apache/htrace/test/random.go | 72 -- .../src/go/src/org/apache/htrace/test/util.go | 33 - pom.xml | 4 +- 70 files changed, 5871 insertions(+), 5868 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/Godeps/Godeps.json ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/Godeps/Godeps.json b/htrace-htraced/go/Godeps/Godeps.json new file mode 100644 index 0000000..47aa90e --- /dev/null +++ b/htrace-htraced/go/Godeps/Godeps.json @@ -0,0 +1,30 @@ +{ + "ImportPath": "git-wip-us.apache.org/repos/asf/incubator-htrace.git", + "GoVersion": "go1.3.1", + "Deps": [ + { + "ImportPath": "github.com/alecthomas/kingpin", + "Rev": "afafa8aab106d31c9dc8f5e562b3f30f6246c3d4" + }, + { + "ImportPath": "github.com/alecthomas/units", + "Rev": "6b4e7dc5e3143b85ea77909c72caf89416fc2915" + }, + { + "ImportPath": "github.com/gorilla/context", + "Rev": "215affda49addc4c8ef7e2534915df2c8c35c6cd" + }, + { + "ImportPath": "github.com/gorilla/mux", + "Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf" + }, + { + "ImportPath": "github.com/jmhodges/levigo", + "Rev": "2c43dde93d0e056173706534afd514fcbc1dd578" + }, + { + "ImportPath": "github.com/ugorji/go/codec", + "Rev": "08bbe4aa39b9f189f4e294b5c8408b5fa5787bb2" + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/format.sh ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/format.sh b/htrace-htraced/go/format.sh new file mode 100755 index 0000000..46aa5b1 --- /dev/null +++ b/htrace-htraced/go/format.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Reformats the HTrace code. +# +# ./format.sh Reformats all code. +# + +die() { + echo $@ + exit 1 +} + +# Check for gofmt. It should be installed whenever the go developement tools +# are installed. +which gofmt &> /dev/null +[ $? -ne 0 ] && die "You must install the gofmt code reformatting formatting tool." + +# Find go sources. We assume no newlines or whitespace in file names. +SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" +find "${SCRIPT_DIR}/src" -noleaf -xdev -name '*.go' | xargs -L 1 gofmt -w http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/gobuild.sh ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/gobuild.sh b/htrace-htraced/go/gobuild.sh new file mode 100755 index 0000000..259e6a7 --- /dev/null +++ b/htrace-htraced/go/gobuild.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Builds the HTrace server code. +# +# ./gobuild.sh Builds the code. +# ./gobuild.sh test Builds and runs all unit tests. +# ./gobuild.sh bench Builds and runs all benchmarks +# + +die() { + echo $@ + exit 1 +} + +ACTION=install +if [ $# -gt 0 ]; then + ACTION="${1}" + shift +fi +RELEASE_VERSION=${RELEASE_VERSION:-unknown} + +# Set up directories. The build/ directory is where build dependencies and +# build binaries should go. +SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" +export GOBIN="${SCRIPT_DIR}/build" +mkdir -p "${GOBIN}" || die "failed to mkdir -p ${GOBIN}" +cd "${GOBIN}" || die "failed to cd to ${SCRIPT_DIR}" +export GOPATH="${GOBIN}:${SCRIPT_DIR}" + +# Use the unsafe package when possible to get greater speed. For example, +# go-codec can bypass the overhead of converting between []byte and string in +# some cases when using unsafe. +TAGS="-tags unsafe" + +# Check for go +which go &> /dev/null +if [ $? -ne 0 ]; then + cat <<EOF +You must install the Golang programming language. + +If you are using Debian, try "apt-get install golang". +For Red Hat, try "yum install go". +For other distributions and operating systems use your packaging tool. +EOF +exit 1 +fi + +# Check for libleveldb.so +if [ -x "/sbin/ldconfig" ]; then + # Suse requires ldconfig to be run via the absolute path + ldconfig=/sbin/ldconfig +else + which ldconfig &> /dev/null + [ $? -eq 0 ] && ldconfig=ldconfig +fi +if [ -n "${ldconfig}" ]; then + if "${ldconfig}" -p | grep -q libleveldb; then + : + else + echo "You must install the leveldb-devel package (or distro-specific equivalent.)" + exit 1 + fi +fi + +case $ACTION in +clean) + rm -rf -- "${GOBIN}" ${SCRIPT_DIR}/pkg + ;; +install) + # Ensure that we have the godep program. + PATH="${PATH}:${GOBIN}" + which godep &> /dev/null + if [ $? -ne 0 ]; then + echo "Installing godep..." + go get github.com/tools/godep || die "failed to get godep" + fi + + # Download dependencies into the build directory. + echo "godep restore..." + godep restore || die "failed to set up dependencies" + + # Discover the git version + GIT_VERSION=$(git rev-parse HEAD) + [ $? -eq 0 ] || GIT_VERSION="unknown" + + # Inject the release and git version into the htraced ldflags. + FLAGS="-X main.RELEASE_VERSION ${RELEASE_VERSION} -X main.GIT_VERSION ${GIT_VERSION}" + go install ${TAGS} -ldflags "${FLAGS}" -v org/apache/htrace/... "$@" + # Make a symlink to web src dir so can do development in-situ out + # of build dir. This is ugly but blame go build. + ln -fs "../src/web" "${GOBIN}/../" + ;; +bench) + go test org/apache/htrace/... ${TAGS} -test.bench=. "$@" + ;; +*) + go ${ACTION} org/apache/htrace/... ${TAGS} "$@" + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/client/client.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/client/client.go b/htrace-htraced/go/src/org/apache/htrace/client/client.go new file mode 100644 index 0000000..6a62e81 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/client/client.go @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package client + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "org/apache/htrace/common" + "org/apache/htrace/conf" +) + +// A golang client for htraced. +// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff + +func NewClient(cnf *conf.Config) (*Client, error) { + hcl := Client{} + hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS) + hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS) + return &hcl, nil +} + +type Client struct { + // REST address of the htraced server. + restAddr string + + // HRPC address of the htraced server. + hrpcAddr string + + // The HRPC client, or null if it is not enabled. + hcr *hClient +} + +// Get the htraced server information. +func (hcl *Client) GetServerInfo() (*common.ServerInfo, error) { + buf, _, err := hcl.makeGetRequest("server/info") + if err != nil { + return nil, err + } + var info common.ServerInfo + err = json.Unmarshal(buf, &info) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return &info, nil +} + +// Get information about a trace span. Returns nil, nil if the span was not found. +func (hcl *Client) FindSpan(sid common.SpanId) (*common.Span, error) { + buf, rc, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x", uint64(sid))) + if err != nil { + if rc == http.StatusNoContent { + return nil, nil + } + return nil, err + } + var span common.Span + err = json.Unmarshal(buf, &span) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return &span, nil +} + +func (hcl *Client) WriteSpans(req *common.WriteSpansReq) error { + if hcl.hrpcAddr == "" { + return hcl.writeSpansHttp(req) + } + hcr, err := newHClient(hcl.hrpcAddr) + if err != nil { + return err + } + defer hcr.Close() + return hcr.writeSpans(req) +} + +func (hcl *Client) writeSpansHttp(req *common.WriteSpansReq) error { + var w bytes.Buffer + var err error + for i := range req.Spans { + var buf []byte + buf, err = json.Marshal(req.Spans[i]) + if err != nil { + return errors.New(fmt.Sprintf("Error serializing span: %s", + err.Error())) + } + _, err = w.Write(buf) + if err != nil { + return errors.New(fmt.Sprintf("Error writing span: %s", + err.Error())) + } + _, err = w.Write([]byte{'\n'}) + //err = io.WriteString(&w, "\n") + if err != nil { + return errors.New(fmt.Sprintf("Error writing: %s", + err.Error())) + } + } + customHeaders := make(map[string]string) + if req.DefaultPid != "" { + customHeaders["htrace-pid"] = req.DefaultPid + } + _, _, err = hcl.makeRestRequest("POST", "writeSpans", + &w, customHeaders) + if err != nil { + return err + } + return nil +} + +// Find the child IDs of a given span ID. +func (hcl *Client) FindChildren(sid common.SpanId, lim int) ([]common.SpanId, error) { + buf, _, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x/children?lim=%d", + uint64(sid), lim)) + if err != nil { + return nil, err + } + var spanIds []common.SpanId + err = json.Unmarshal(buf, &spanIds) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return spanIds, nil +} + +// Make a query +func (hcl *Client) Query(query *common.Query) ([]common.Span, error) { + in, err := json.Marshal(query) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error marshalling query: %s", err.Error())) + } + var out []byte + var url = fmt.Sprintf("query?query=%s", in) + out, _, err = hcl.makeGetRequest(url) + if err != nil { + return nil, err + } + var spans []common.Span + err = json.Unmarshal(out, &spans) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error unmarshalling results: %s", err.Error())) + } + return spans, nil +} + +var EMPTY = make(map[string]string) + +func (hcl *Client) makeGetRequest(reqName string) ([]byte, int, error) { + return hcl.makeRestRequest("GET", reqName, nil, EMPTY) +} + +// Make a general JSON REST request. +// Returns the request body, the response code, and the error. +// Note: if the response code is non-zero, the error will also be non-zero. +func (hcl *Client) makeRestRequest(reqType string, reqName string, reqBody io.Reader, + customHeaders map[string]string) ([]byte, int, error) { + url := fmt.Sprintf("http://%s/%s";, + hcl.restAddr, reqName) + req, err := http.NewRequest(reqType, url, reqBody) + req.Header.Set("Content-Type", "application/json") + for k, v := range customHeaders { + req.Header.Set(k, v) + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, -1, errors.New(fmt.Sprintf("Error: error making http request to %s: %s\n", url, + err.Error())) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, + errors.New(fmt.Sprintf("Error: got bad response status from %s: %s\n", url, resp.Status)) + } + var body []byte + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return nil, -1, errors.New(fmt.Sprintf("Error: error reading response body: %s\n", err.Error())) + } + return body, 0, nil +} + +// Dump all spans from the htraced daemon. +func (hcl *Client) DumpAll(lim int, out chan *common.Span) error { + defer func() { + close(out) + }() + searchId := common.SpanId(0) + for { + q := common.Query{ + Lim: lim, + Predicates: []common.Predicate{ + common.Predicate{ + Op: "ge", + Field: "spanid", + Val: searchId.String(), + }, + }, + } + spans, err := hcl.Query(&q) + if err != nil { + return errors.New(fmt.Sprintf("Error querying spans with IDs at or after "+ + "%s: %s", searchId.String(), err.Error())) + } + if len(spans) == 0 { + return nil + } + for i := range spans { + out <- &spans[i] + } + searchId = spans[len(spans)-1].Id + 1 + } +} + +func (hcl *Client) Close() { + if hcl.hcr != nil { + hcl.hcr.Close() + } + hcl.restAddr = "" + hcl.hcr = nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/client/hclient.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/client/hclient.go b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go new file mode 100644 index 0000000..608dd59 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package client + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "github.com/ugorji/go/codec" + "io" + "net" + "net/rpc" + "org/apache/htrace/common" +) + +type hClient struct { + rpcClient *rpc.Client +} + +type HrpcClientCodec struct { + rwc io.ReadWriteCloser + length uint32 +} + +func (cdc *HrpcClientCodec) WriteRequest(req *rpc.Request, msg interface{}) error { + methodId := common.HrpcMethodNameToId(req.ServiceMethod) + if methodId == common.METHOD_ID_NONE { + return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s", + req.ServiceMethod)) + } + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + w := bytes.NewBuffer(make([]byte, 0, 2048)) + enc := codec.NewEncoder(w, mh) + err := enc.Encode(msg) + if err != nil { + return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ + "message as msgpack: %s", err.Error())) + } + buf := w.Bytes() + if len(buf) > common.MAX_HRPC_BODY_LENGTH { + return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+ + "bytes, but the maximum message size is %d bytes.", + len(buf), common.MAX_HRPC_BODY_LENGTH)) + } + hdr := common.HrpcRequestHeader{ + Magic: common.HRPC_MAGIC, + MethodId: methodId, + Seq: req.Seq, + Length: uint32(len(buf)), + } + err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr) + if err != nil { + return errors.New(fmt.Sprintf("Error writing header bytes: %s", + err.Error())) + } + _, err = cdc.rwc.Write(buf) + if err != nil { + return errors.New(fmt.Sprintf("Error writing body bytes: %s", + err.Error())) + } + return nil +} + +func (cdc *HrpcClientCodec) ReadResponseHeader(resp *rpc.Response) error { + hdr := common.HrpcResponseHeader{} + err := binary.Read(cdc.rwc, binary.LittleEndian, &hdr) + if err != nil { + return errors.New(fmt.Sprintf("Error reading response header "+ + "bytes: %s", err.Error())) + } + resp.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) + if resp.ServiceMethod == "" { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "invalid method ID %d.", hdr.MethodId)) + } + resp.Seq = hdr.Seq + if hdr.ErrLength > 0 { + if hdr.ErrLength > common.MAX_HRPC_ERROR_LENGTH { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "error message was %d bytes long, but "+ + "MAX_HRPC_ERROR_LENGTH is %d.", + hdr.ErrLength, common.MAX_HRPC_ERROR_LENGTH)) + } + buf := make([]byte, hdr.ErrLength) + var nread int + nread, err = cdc.rwc.Read(buf) + if uint32(nread) != hdr.ErrLength { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "failed to read %d bytes of error message.", nread)) + } + if err != nil { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "failed to read %d bytes of error message: %s", + nread, err.Error())) + } + resp.Error = string(buf) + } else { + resp.Error = "" + } + cdc.length = hdr.Length + return nil +} + +func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error { + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh) + err := dec.Decode(body) + if err != nil { + return errors.New(fmt.Sprintf("Failed to read response body: %s", + err.Error())) + } + return nil +} + +func (cdc *HrpcClientCodec) Close() error { + return cdc.rwc.Close() +} + +func newHClient(hrpcAddr string) (*hClient, error) { + hcr := hClient{} + conn, err := net.Dial("tcp", hrpcAddr) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error contacting the HRPC server "+ + "at %s: %s", hrpcAddr, err.Error())) + } + hcr.rpcClient = rpc.NewClientWithCodec(&HrpcClientCodec{rwc: conn}) + return &hcr, nil +} + +func (hcr *hClient) writeSpans(req *common.WriteSpansReq) error { + resp := common.WriteSpansResp{} + return hcr.rpcClient.Call(common.METHOD_NAME_WRITE_SPANS, req, &resp) +} + +func (hcr *hClient) Close() { + hcr.rpcClient.Close() +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/log.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log.go b/htrace-htraced/go/src/org/apache/htrace/common/log.go new file mode 100644 index 0000000..305ecf3 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/log.go @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "errors" + "fmt" + "org/apache/htrace/conf" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// A logSink is a place logs can be written to. +type logSink struct { + path logPath + file *os.File + lock sync.Mutex + refCount int // protected by logFilesLock +} + +// Write to the logSink. +func (sink *logSink) write(str string) { + sink.lock.Lock() + defer sink.lock.Unlock() + _, err := sink.file.Write([]byte(str)) + if err != nil { + fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error()) + } +} + +// Unreference the logSink. If there are no more references, and the logSink is +// closeable, then we will close it here. +func (sink *logSink) Unref() { + logFilesLock.Lock() + defer logFilesLock.Unlock() + sink.refCount-- + if sink.refCount <= 0 { + if sink.path.IsCloseable() { + err := sink.file.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n", + sink.path, err.Error()) + } + } + logSinks[sink.path] = nil + } +} + +type logPath string + +// An empty LogPath represents "stdout." +const STDOUT_LOG_PATH = "" + +// Convert a path to a logPath. +func logPathFromString(path string) logPath { + if path == STDOUT_LOG_PATH { + return logPath("") + } + absPath, err := filepath.Abs(path) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n", + path, err.Error()) + return logPath(path) + } + return logPath(absPath) +} + +// Convert the path to a human-readable string. +func (path logPath) String() string { + if path == "" { + return "(stdout)" + } else { + return string(path) + } +} + +// Return true if the path is closeable. stdout is not closeable. +func (path logPath) IsCloseable() bool { + return path != STDOUT_LOG_PATH +} + +func (path logPath) Open() *logSink { + if path == STDOUT_LOG_PATH { + return &logSink{path: path, file: os.Stdout} + } + file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout} + fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n", + path, err.Error()) + return sink + } + return &logSink{path: path, file: file} +} + +var logFilesLock sync.Mutex + +var logSinks map[logPath]*logSink = make(map[logPath]*logSink) + +func getOrCreateLogSink(pathStr string) *logSink { + path := logPathFromString(pathStr) + logFilesLock.Lock() + defer logFilesLock.Unlock() + sink := logSinks[path] + if sink == nil { + sink = path.Open() + logSinks[path] = sink + } + sink.refCount++ + return sink +} + +type Level int + +const ( + TRACE Level = iota + DEBUG + INFO + WARN + ERROR +) + +var levelToString map[Level]string = map[Level]string{ + TRACE: "TRACE", + DEBUG: "DEBUG", + INFO: "INFO", + WARN: "WARN", + ERROR: "ERROR", +} + +func (level Level) String() string { + return levelToString[level] +} + +func (level Level) LogString() string { + return level.String()[0:1] +} + +func LevelFromString(str string) (Level, error) { + for k, v := range levelToString { + if strings.ToLower(v) == strings.ToLower(str) { + return k, nil + } + } + var levelNames sort.StringSlice + levelNames = make([]string, len(levelToString)) + var i int + for _, v := range levelToString { + levelNames[i] = v + i++ + } + sort.Sort(levelNames) + return TRACE, errors.New(fmt.Sprintf("No such level as '%s'. Valid "+ + "levels are '%v'\n", str, levelNames)) +} + +type Logger struct { + sink *logSink + Level Level +} + +func NewLogger(faculty string, cnf *conf.Config) *Logger { + path, level := parseConf(faculty, cnf) + sink := getOrCreateLogSink(path) + return &Logger{sink: sink, Level: level} +} + +func parseConf(faculty string, cnf *conf.Config) (string, Level) { + facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH + var facultyLogPath string + if cnf.Contains(facultyLogPathKey) { + facultyLogPath = cnf.Get(facultyLogPathKey) + } else { + facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH) + } + facultyLogLevelKey := faculty + conf.HTRACE_LOG_LEVEL + var facultyLogLevelStr string + if cnf.Contains(facultyLogLevelKey) { + facultyLogLevelStr = cnf.Get(facultyLogLevelKey) + } else { + facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL) + } + level, err := LevelFromString(facultyLogLevelStr) + if err != nil { + fmt.Fprintf(os.Stderr, "Error configuring log level: %s. Using TRACE.\n") + level = TRACE + } + return facultyLogPath, level +} + +func (lg *Logger) Trace(str string) { + lg.Write(TRACE, str) +} + +func (lg *Logger) Tracef(format string, v ...interface{}) { + lg.Write(TRACE, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Debug(str string) { + lg.Write(DEBUG, str) +} + +func (lg *Logger) Debugf(format string, v ...interface{}) { + lg.Write(DEBUG, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Info(str string) { + lg.Write(INFO, str) +} + +func (lg *Logger) Infof(format string, v ...interface{}) { + lg.Write(INFO, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Warn(str string) error { + lg.Write(WARN, str) + return errors.New(str) +} + +func (lg *Logger) Warnf(format string, v ...interface{}) error { + str := fmt.Sprintf(format, v...) + lg.Write(WARN, str) + return errors.New(str) +} + +func (lg *Logger) Error(str string) error { + lg.Write(ERROR, str) + return errors.New(str) +} + +func (lg *Logger) Errorf(format string, v ...interface{}) error { + str := fmt.Sprintf(format, v...) + lg.Write(ERROR, str) + return errors.New(str) +} + +func (lg *Logger) Write(level Level, str string) { + if level >= lg.Level { + lg.sink.write(time.Now().Format(time.RFC3339) + " " + + level.LogString() + ": " + str) + } +} + +// +// A few functions which can be used to determine if a certain level of tracing +// is enabled. These are useful in situations when evaluating the parameters +// of a logging function is expensive. (Note, however, that we don't pay the +// cost of string concatenation and manipulation when a log message doesn't +// trigger.) +// + +func (lg *Logger) TraceEnabled() bool { + return lg.Level >= TRACE +} + +func (lg *Logger) DebugEnabled() bool { + return lg.Level >= DEBUG +} + +func (lg *Logger) InfoEnabled() bool { + return lg.Level >= INFO +} + +func (lg *Logger) WarnEnabled() bool { + return lg.Level >= WARN +} + +func (lg *Logger) ErrorEnabled() bool { + return lg.Level >= ERROR +} + +func (lg *Logger) Close() { + lg.sink.Unref() + lg.sink = nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/log_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log_test.go b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go new file mode 100644 index 0000000..b415ce2 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "org/apache/htrace/conf" + "os" + "strings" + "testing" +) + +func newLogger(faculty string, args ...string) *Logger { + cnfBld := conf.Builder{Defaults: conf.DEFAULTS} + cnf, err := cnfBld.Build() + if err != nil { + panic(fmt.Sprintf("failed to create conf: %s", err.Error())) + } + cnf2 := cnf.Clone(args...) + lg := NewLogger(faculty, cnf2) + return lg +} + +func TestNewLogger(t *testing.T) { + lg := newLogger("foo", "log.level", "TRACE") + lg.Close() +} + +func verifyLines(t *testing.T, rdr io.Reader, lines []string) { + scanner := bufio.NewScanner(rdr) + lineIdx := 0 + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, lines[lineIdx]) { + t.Fatalf("Error on line %d: didn't find substring '%s' in line '%s'\n", + (lineIdx + 1), lines[lineIdx], line) + } + lineIdx++ + } + if err := scanner.Err(); err != nil { + t.Fatal(err.Error()) + } +} + +func TestFileLogs(t *testing.T) { + tempDir, err := ioutil.TempDir(os.TempDir(), "testFileLogs") + if err != nil { + panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) + } + defer os.RemoveAll(tempDir) + logPath := tempDir + conf.PATH_SEP + "log" + lg := newLogger("foo", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + lg.Tracef("Non-important stuff, ignore this.\n") + lg.Infof("problem with the foobar\n") + lg.Tracef("More non-important stuff, also ignore this.\n") + lg.Infof("and another problem with the foobar\n") + logFile, err := os.Open(logPath) + if err != nil { + t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) + } + verifyLines(t, logFile, []string{ + "problem with the foobar", + "and another problem with the foobar", + }) + logFile.Close() + lg.Close() +} + +func TestMultipleFileLogs(t *testing.T) { + tempDir, err := ioutil.TempDir(os.TempDir(), "testMultipleFileLogs") + if err != nil { + panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) + } + defer os.RemoveAll(tempDir) + logPath := tempDir + conf.PATH_SEP + "log" + fooLg := newLogger("foo", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + fooLg.Infof("The foo needs maintenance.\n") + barLg := newLogger("bar", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + barLg.Debugf("The bar is open\n") + fooLg.Errorf("Fizz buzz\n") + logFile, err := os.Open(logPath) + if err != nil { + t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) + } + fooLg.Tracef("Fizz buzz2\n") + barLg.Tracef("Fizz buzz3\n") + verifyLines(t, logFile, []string{ + "The foo needs maintenance.", + "The bar is open", + "Fizz buzz", + "Fizz buzz3", + }) + logFile.Close() + fooLg.Close() + barLg.Close() +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/process.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process.go b/htrace-htraced/go/src/org/apache/htrace/common/process.go new file mode 100644 index 0000000..d138178 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/process.go @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "bufio" + "org/apache/htrace/conf" + "os" + "os/signal" + "syscall" +) + +func LoadApplicationConfig() *conf.Config { + cnf, dlog := conf.LoadApplicationConfig() + lg := NewLogger("conf", cnf) + defer lg.Close() + if lg.Level <= DEBUG { + // Print out the debug information from loading the configuration. + scanner := bufio.NewScanner(dlog) + for scanner.Scan() { + lg.Debugf(scanner.Text() + "\n") + } + } + return cnf +} + +func InstallSignalHandlers(cnf *conf.Config) { + fatalSigs := []os.Signal{ + os.Interrupt, + os.Kill, + syscall.SIGINT, + syscall.SIGABRT, + syscall.SIGALRM, + syscall.SIGBUS, + syscall.SIGFPE, + syscall.SIGILL, + syscall.SIGQUIT, + syscall.SIGSEGV, + syscall.SIGTERM, + } + sigChan := make(chan os.Signal, len(fatalSigs)) + signal.Notify(sigChan, fatalSigs...) + lg := NewLogger("exit", cnf) + go func() { + sig := <-sigChan + lg.Errorf("Terminating on signal: %v\n", sig) + lg.Close() + os.Exit(1) + }() +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/query.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query.go b/htrace-htraced/go/src/org/apache/htrace/common/query.go new file mode 100644 index 0000000..8c9128f --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/query.go @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "encoding/json" +) + +// +// Represents queries that can be sent to htraced. +// +// Each query consists of set of predicates that will be 'AND'ed together to +// return a set of spans. Predicates contain an operation, a field, and a +// value. +// +// For example, a query might be "return the first 100 spans between 5:00pm +// and 5:01pm" This query would have two predicates: time greater than or +// equal to 5:00pm, and time less than or equal to 5:01pm. +// In HTrace, times are always expressed in milliseconds since the Epoch. +// So this would become: +// { "lim" : 100, "pred" : [ +// { "op" : "ge", "field" : "begin", "val" : 1234 }, +// { "op" : "le", "field" : "begin", "val" : 5678 }, +// ] } +// +// Where '1234' and '5678' were replaced by times since the epoch in +// milliseconds. +// + +type Op string + +const ( + CONTAINS Op = "cn" + EQUALS Op = "eq" + LESS_THAN_OR_EQUALS Op = "le" + GREATER_THAN_OR_EQUALS Op = "ge" + GREATER_THAN Op = "gt" +) + +func (op Op) IsDescending() bool { + return op == LESS_THAN_OR_EQUALS +} + +func (op Op) IsValid() bool { + ops := ValidOps() + for i := range ops { + if ops[i] == op { + return true + } + } + return false +} + +func ValidOps() []Op { + return []Op{CONTAINS, EQUALS, LESS_THAN_OR_EQUALS, GREATER_THAN_OR_EQUALS, + GREATER_THAN} +} + +type Field string + +const ( + SPAN_ID Field = "spanid" + DESCRIPTION Field = "description" + BEGIN_TIME Field = "begin" + END_TIME Field = "end" + DURATION Field = "duration" +) + +func (field Field) IsValid() bool { + fields := ValidFields() + for i := range fields { + if fields[i] == field { + return true + } + } + return false +} + +func ValidFields() []Field { + return []Field{SPAN_ID, DESCRIPTION, BEGIN_TIME, END_TIME, DURATION} +} + +type Predicate struct { + Op Op `json:"op"` + Field Field `json:"field"` + Val string `val:"val"` +} + +func (pred *Predicate) String() string { + buf, err := json.Marshal(pred) + if err != nil { + panic(err) + } + return string(buf) +} + +type Query struct { + Predicates []Predicate `json:"pred"` + Lim int `json:"lim"` + Prev *Span `json:"prev"` +} + +func (query *Query) String() string { + buf, err := json.Marshal(query) + if err != nil { + panic(err) + } + return string(buf) +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/query_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query_test.go b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go new file mode 100644 index 0000000..2697d9c --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "testing" +) + +func TestValidOps(t *testing.T) { + for i := range ValidOps() { + op := ValidOps()[i] + if !op.IsValid() { + t.Fatalf("op %s was in ValidOps, but IsValid returned false.\n", op) + } + } + invalidOp := Op("completelybogus") + if invalidOp.IsValid() { + t.Fatalf("op %s was invalid, but IsValid returned true.\n", invalidOp) + } +} + +func TestValidFields(t *testing.T) { + for i := range ValidFields() { + field := ValidFields()[i] + if !field.IsValid() { + t.Fatalf("field %s was in ValidFields, but IsValid returned false.\n", field) + } + } + invalidField := Field("completelybogus") + if invalidField.IsValid() { + t.Fatalf("field %s was invalid, but IsValid returned true.\n", invalidField) + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/rest.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rest.go b/htrace-htraced/go/src/org/apache/htrace/common/rest.go new file mode 100644 index 0000000..b898ca4 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/rest.go @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +// Info returned by /server/info +type ServerInfo struct { + // The server release version. + ReleaseVersion string + + // The git hash that this software was built with. + GitVersion string +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/rpc.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go new file mode 100644 index 0000000..fe50a44 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +// The 4-byte magic number which is sent first in the HRPC header +const HRPC_MAGIC = 0x43525448 + +// Method ID codes. Do not reorder these. +const ( + METHOD_ID_NONE = 0 + METHOD_ID_WRITE_SPANS = iota +) + +const METHOD_NAME_WRITE_SPANS = "HrpcHandler.WriteSpans" + +// Maximum length of the error message passed in an HRPC response +const MAX_HRPC_ERROR_LENGTH = 4 * 1024 * 1024 + +// Maximum length of HRPC message body +const MAX_HRPC_BODY_LENGTH = 64 * 1024 * 1024 + +// A request to write spans to htraced. +type WriteSpansReq struct { + DefaultPid string + Spans []*Span +} + +// A response to a WriteSpansReq +type WriteSpansResp struct { +} + +// The header which is sent over the wire for HRPC +type HrpcRequestHeader struct { + Magic uint32 + MethodId uint32 + Seq uint64 + Length uint32 +} + +// The response which is sent over the wire for HRPC +type HrpcResponseHeader struct { + Seq uint64 + MethodId uint32 + ErrLength uint32 + Length uint32 +} + +func HrpcMethodIdToMethodName(id uint32) string { + switch id { + case METHOD_ID_WRITE_SPANS: + return METHOD_NAME_WRITE_SPANS + default: + return "" + } +} + +func HrpcMethodNameToId(name string) uint32 { + switch name { + case METHOD_NAME_WRITE_SPANS: + return METHOD_ID_WRITE_SPANS + default: + return METHOD_ID_NONE + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/span.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span.go b/htrace-htraced/go/src/org/apache/htrace/common/span.go new file mode 100644 index 0000000..b276844 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/span.go @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" +) + +// +// Represents a trace span. +// +// Compatibility notes: +// When converting to JSON, we store the 64-bit numbers as hexadecimal strings rather than as +// integers. This is because JavaScript lacks the ability to handle 64-bit integers. Numbers above +// about 55 bits will be rounded by Javascript. Since the Javascript UI is a primary consumer of +// this JSON data, we have to simply pass it as a string. +// + +type TraceInfoMap map[string]string + +type TimelineAnnotation struct { + Time int64 `json:"t"` + Msg string `json:"m"` +} + +type SpanId uint64 + +func (id SpanId) String() string { + return fmt.Sprintf("%016x", uint64(id)) +} + +func (id SpanId) Val() uint64 { + return uint64(id) +} + +func (id SpanId) MarshalJSON() ([]byte, error) { + return []byte(`"` + fmt.Sprintf("%016x", uint64(id)) + `"`), nil +} + +type SpanSlice []*Span + +func (s SpanSlice) Len() int { + return len(s) +} + +func (s SpanSlice) Less(i, j int) bool { + return s[i].Id < s[j].Id +} + +func (s SpanSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +type SpanIdSlice []SpanId + +func (s SpanIdSlice) Len() int { + return len(s) +} + +func (s SpanIdSlice) Less(i, j int) bool { + return s[i] < s[j] +} + +func (s SpanIdSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +const DOUBLE_QUOTE = 0x22 + +func (id *SpanId) UnmarshalJSON(b []byte) error { + if b[0] != DOUBLE_QUOTE { + return errors.New("Expected spanID to start with a string quote.") + } + if b[len(b)-1] != DOUBLE_QUOTE { + return errors.New("Expected spanID to end with a string quote.") + } + return id.FromString(string(b[1 : len(b)-1])) +} + +func (id *SpanId) FromString(str string) error { + v, err := strconv.ParseUint(str, 16, 64) + if err != nil { + return err + } + *id = SpanId(v) + return nil +} + +type SpanData struct { + Begin int64 `json:"b"` + End int64 `json:"e"` + Description string `json:"d"` + TraceId SpanId `json:"i"` + Parents []SpanId `json:"p"` + Info TraceInfoMap `json:"n,omitempty"` + ProcessId string `json:"r"` + TimelineAnnotations []TimelineAnnotation `json:"t,omitempty"` +} + +type Span struct { + Id SpanId `json:"s"` + SpanData +} + +func (span *Span) ToJson() []byte { + jbytes, err := json.Marshal(*span) + if err != nil { + panic(err) + } + return jbytes +} + +func (span *Span) String() string { + return string(span.ToJson()) +} + +// Compute the span duration. We ignore overflow since we never deal with negative times. +func (span *Span) Duration() int64 { + return span.End - span.Begin +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/span_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span_test.go b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go new file mode 100644 index 0000000..f218b3a --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "testing" +) + +func TestSpanToJson(t *testing.T) { + t.Parallel() + span := Span{Id: 2305843009213693952, + SpanData: SpanData{ + Begin: 123, + End: 456, + Description: "getFileDescriptors", + TraceId: 999, + Parents: []SpanId{}, + ProcessId: "testProcessId", + }} + ExpectStrEqual(t, + `{"s":"2000000000000000","b":123,"e":456,"d":"getFileDescriptors","i":"00000000000003e7","p":[],"r":"testProcessId"}`, + string(span.ToJson())) +} + +func TestAnnotatedSpanToJson(t *testing.T) { + t.Parallel() + span := Span{Id: 1305813009213693952, + SpanData: SpanData{ + Begin: 1234, + End: 4567, + Description: "getFileDescriptors2", + TraceId: 999, + Parents: []SpanId{}, + ProcessId: "testAnnotatedProcessId", + TimelineAnnotations: []TimelineAnnotation{ + TimelineAnnotation{ + Time: 7777, + Msg: "contactedServer", + }, + TimelineAnnotation{ + Time: 8888, + Msg: "passedFd", + }, + }, + }} + ExpectStrEqual(t, + `{"s":"121f2e036d442000","b":1234,"e":4567,"d":"getFileDescriptors2","i":"00000000000003e7","p":[],"r":"testAnnotatedProcessId","t":[{"t":7777,"m":"contactedServer"},{"t":8888,"m":"passedFd"}]}`, + string(span.ToJson())) +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/common/test_util.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/common/test_util.go b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go new file mode 100644 index 0000000..871c847 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "fmt" + "testing" + "time" +) + +type Int64Slice []int64 + +func (p Int64Slice) Len() int { return len(p) } +func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type SupplierFun func() bool + +// +// Wait for a configurable amount of time for a precondition to become true. +// +// Example: +// WaitFor(time.Minute * 1, time.Millisecond * 1, func() bool { +// return ht.Store.GetStatistics().NumSpansWritten >= 3 +// }) +// +func WaitFor(dur time.Duration, poll time.Duration, fun SupplierFun) { + if poll == 0 { + poll = dur / 10 + } + if poll <= 0 { + panic("Can't have a polling time less than zero.") + } + endTime := time.Now().Add(dur) + for { + if fun() { + return + } + if !time.Now().Before(endTime) { + break + } + time.Sleep(poll) + } + panic(fmt.Sprintf("Timed out after %s", dur)) +} + +// Trigger a test failure if two strings are not equal. +func ExpectStrEqual(t *testing.T, expect string, actual string) { + if expect != actual { + t.Fatalf("Expected:\n%s\nGot:\n%s\n", expect, actual) + } +} + +// Trigger a test failure if the JSON representation of two spans are not equals. +func ExpectSpansEqual(t *testing.T, spanA *Span, spanB *Span) { + ExpectStrEqual(t, string(spanA.ToJson()), string(spanB.ToJson())) +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/conf/config.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config.go b/htrace-htraced/go/src/org/apache/htrace/conf/config.go new file mode 100644 index 0000000..6093649 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config.go @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" +) + +// +// The configuration code for HTraced. +// +// HTraced can be configured via Hadoop-style XML configuration files, or by passing -Dkey=value +// command line arguments. Command-line arguments without an equals sign, such as "-Dkey", will be +// treated as setting the key to "true". +// +// Configuration key constants should be defined in config_keys.go. Each key should have a default, +// which will be used if the user supplies no value, or supplies an invalid value. +// For that reason, it is not necessary for the Get, GetInt, etc. functions to take a default value +// argument. +// +// Configuration objects are immutable. However, you can make a copy of a configuration which adds +// some changes using Configuration#Clone(). +// + +type Config struct { + settings map[string]string + defaults map[string]string +} + +type Builder struct { + // If non-nil, the XML configuration file to read. + Reader io.Reader + + // If non-nil, the configuration values to use. + Values map[string]string + + // If non-nil, the default configuration values to use. + Defaults map[string]string + + // If non-nil, the command-line arguments to use. + Argv []string +} + +func getHTracedConfDirs(dlog io.Writer) []string { + confDir := os.Getenv("HTRACED_CONF_DIR") + io.WriteString(dlog, fmt.Sprintf("HTRACED_CONF_DIR=%s\n", confDir)) + paths := filepath.SplitList(confDir) + if len(paths) < 1 { + return []string{"."} + } + return paths +} + +// Load a configuration from the application's argv, configuration file, and the standard +// defaults. +func LoadApplicationConfig() (*Config, io.Reader) { + dlog := new(bytes.Buffer) + reader := openFile(CONFIG_FILE_NAME, getHTracedConfDirs(dlog), dlog) + bld := Builder{} + if reader != nil { + defer reader.Close() + bld.Reader = bufio.NewReader(reader) + } + bld.Argv = os.Args[1:] + bld.Defaults = DEFAULTS + cnf, err := bld.Build() + if err != nil { + log.Fatal("Error building configuration: " + err.Error()) + } + os.Args = append(os.Args[0:1], bld.Argv...) + keys := make(sort.StringSlice, 0, 20) + for k, _ := range cnf.settings { + keys = append(keys, k) + } + sort.Sort(keys) + for i := range keys { + io.WriteString(dlog, fmt.Sprintf("%s = %s\n", + keys[i], cnf.settings[keys[i]])) + } + return cnf, dlog +} + +// Attempt to open a configuration file somewhere on the provided list of paths. +func openFile(cnfName string, paths []string, dlog io.Writer) io.ReadCloser { + for p := range paths { + path := fmt.Sprintf("%s%c%s", paths[p], os.PathSeparator, cnfName) + file, err := os.Open(path) + if err == nil { + io.WriteString(dlog, fmt.Sprintf("Reading configuration from %s.\n", path)) + return file + } + if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT { + continue + } + io.WriteString(dlog, fmt.Sprintf("Error opening %s for read: %s\n", path, err.Error())) + } + return nil +} + +// Try to parse a command-line element as a key=value pair. +func parseAsConfigFlag(flag string) (string, string) { + var confPart string + if strings.HasPrefix(flag, "-D") { + confPart = flag[2:] + } else if strings.HasPrefix(flag, "--D") { + confPart = flag[3:] + } else { + return "", "" + } + if len(confPart) == 0 { + return "", "" + } + idx := strings.Index(confPart, "=") + if idx == -1 { + return confPart, "true" + } + return confPart[0:idx], confPart[idx+1:] +} + +// Build a new configuration object from the provided conf.Builder. +func (bld *Builder) Build() (*Config, error) { + // Load values and defaults + cnf := Config{} + cnf.settings = make(map[string]string) + if bld.Values != nil { + for k, v := range bld.Values { + cnf.settings[k] = v + } + } + cnf.defaults = make(map[string]string) + if bld.Defaults != nil { + for k, v := range bld.Defaults { + cnf.defaults[k] = v + } + } + + // Process the configuration file, if we have one + if bld.Reader != nil { + parseXml(bld.Reader, cnf.settings) + } + + // Process command line arguments + var i int + for i < len(bld.Argv) { + str := bld.Argv[i] + key, val := parseAsConfigFlag(str) + if key != "" { + if val == "" { + cnf.settings[key] = "true" + } else { + cnf.settings[key] = val + } + bld.Argv = append(bld.Argv[:i], bld.Argv[i+1:]...) + } else { + i++ + } + } + return &cnf, nil +} + +// Returns true if the configuration has a non-default value for the given key. +func (cnf *Config) Contains(key string) bool { + _, ok := cnf.settings[key] + return ok +} + +// Get a string configuration key. +func (cnf *Config) Get(key string) string { + ret := cnf.settings[key] + if ret != "" { + return ret + } + return cnf.defaults[key] +} + +// Get a boolean configuration key. +func (cnf *Config) GetBool(key string) bool { + str := cnf.settings[key] + ret, err := strconv.ParseBool(str) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.ParseBool(str) + if err == nil { + return ret + } + return false +} + +// Get an integer configuration key. +func (cnf *Config) GetInt(key string) int { + str := cnf.settings[key] + ret, err := strconv.Atoi(str) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.Atoi(str) + if err == nil { + return ret + } + return 0 +} + +// Get an int64 configuration key. +func (cnf *Config) GetInt64(key string) int64 { + str := cnf.settings[key] + ret, err := strconv.ParseInt(str, 10, 64) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.ParseInt(str, 10, 64) + if err == nil { + return ret + } + return 0 +} + +// Make a deep copy of the given configuration. +// Optionally, you can specify particular key/value pairs to change. +// Example: +// cnf2 := cnf.Copy("my.changed.key", "my.new.value") +func (cnf *Config) Clone(args ...string) *Config { + if len(args)%2 != 0 { + panic("The arguments to Config#copy are key1, value1, " + + "key2, value2, and so on. You must specify an even number of arguments.") + } + ncnf := &Config{defaults: cnf.defaults} + ncnf.settings = make(map[string]string) + for k, v := range cnf.settings { + ncnf.settings[k] = v + } + for i := 0; i < len(args); i += 2 { + ncnf.settings[args[i]] = args[i+1] + } + return ncnf +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go new file mode 100644 index 0000000..ccb09e0 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "fmt" + "os" +) + +// +// Configuration keys for HTrace. +// + +// The platform-specific path separator. Usually slash. +var PATH_SEP string = fmt.Sprintf("%c", os.PathSeparator) + +// The platform-specific path list separator. Usually colon. +var PATH_LIST_SEP string = fmt.Sprintf("%c", os.PathListSeparator) + +// The name of the XML configuration file to look for. +const CONFIG_FILE_NAME = "htraced-conf.xml" + +// An environment variable containing a list of paths to search for the +// configuration file in. +const HTRACED_CONF_DIR = "HTRACED_CONF_DIR" + +// The web address to start the REST server on. +const HTRACE_WEB_ADDRESS = "web.address" + +// The default port for the Htrace web address. +const HTRACE_WEB_ADDRESS_DEFAULT_PORT = 9095 + +// The web address to start the REST server on. +const HTRACE_HRPC_ADDRESS = "hrpc.address" + +// The default port for the Htrace HRPC address. +const HTRACE_HRPC_ADDRESS_DEFAULT_PORT = 9075 + +// The directories to put the data store into. Separated by PATH_LIST_SEP. +const HTRACE_DATA_STORE_DIRECTORIES = "data.store.directories" + +// Boolean key which indicates whether we should clear data on startup. +const HTRACE_DATA_STORE_CLEAR = "data.store.clear" + +// How many writes to buffer before applying backpressure to span senders. +const HTRACE_DATA_STORE_SPAN_BUFFER_SIZE = "data.store.span.buffer.size" + +// Path to put the logs from htrace, or the empty string to use stdout. +const HTRACE_LOG_PATH = "log.path" + +// The log level to use for the logs in htrace. +const HTRACE_LOG_LEVEL = "log.level" + +// A host:port pair to send information to on startup. This is used in unit +// tests to determine the (random) port of the htraced process that has been +// started. +const HTRACE_STARTUP_NOTIFICATION_ADDRESS = "startup.notification.address" + +// Default values for HTrace configuration keys. +var DEFAULTS = map[string]string{ + HTRACE_WEB_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_WEB_ADDRESS_DEFAULT_PORT), + HTRACE_HRPC_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_HRPC_ADDRESS_DEFAULT_PORT), + HTRACE_DATA_STORE_DIRECTORIES: PATH_SEP + "tmp" + PATH_SEP + "htrace1" + + PATH_LIST_SEP + PATH_SEP + "tmp" + PATH_SEP + "htrace2", + HTRACE_DATA_STORE_CLEAR: "false", + HTRACE_DATA_STORE_SPAN_BUFFER_SIZE: "100", + HTRACE_LOG_PATH: "", + HTRACE_LOG_LEVEL: "INFO", +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go new file mode 100644 index 0000000..42c1c71 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "bytes" + "os" + "strings" + "testing" +) + +// Test that parsing command-line arguments of the form -Dfoo=bar works. +func TestParseArgV(t *testing.T) { + t.Parallel() + argv := []string{"-Dfoo=bar", "-Dbaz=123", "-DsillyMode"} + bld := &Builder{Argv: argv} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + if "bar" != cnf.Get("foo") { + t.Fatal() + } + if 123 != cnf.GetInt("baz") { + t.Fatal() + } + if !cnf.GetBool("sillyMode") { + t.Fatal() + } + if cnf.GetBool("otherSillyMode") { + t.Fatal() + } +} + +// Test that default values work. +// Defaults are used only when the configuration option is not present or can't be parsed. +func TestDefaults(t *testing.T) { + t.Parallel() + argv := []string{"-Dfoo=bar", "-Dbaz=invalidNumber"} + defaults := map[string]string{ + "foo": "notbar", + "baz": "456", + "foo2": "4611686018427387904", + } + bld := &Builder{Argv: argv, Defaults: defaults} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + if "bar" != cnf.Get("foo") { + t.Fatal() + } + if 456 != cnf.GetInt("baz") { + t.Fatal() + } + if 4611686018427387904 != cnf.GetInt64("foo2") { + t.Fatal() + } +} + +// Test that we can parse our XML configuration file. +func TestXmlConfigurationFile(t *testing.T) { + t.Parallel() + xml := ` +<?xml version="1.0"?> +<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?> +<configuration> + <property> + <name>foo.bar</name> + <value>123</value> + </property> + <property> + <name>foo.baz</name> + <value>xmlValue</value> + </property> + <!--<property> + <name>commented.out</name> + <value>stuff</value> + </property>--> +</configuration> +` + xmlReader := strings.NewReader(xml) + argv := []string{"-Dfoo.bar=456"} + defaults := map[string]string{ + "foo.bar": "789", + "cmdline.opt": "4611686018427387904", + } + bld := &Builder{Argv: argv, Defaults: defaults, Reader: xmlReader} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + // The command-line argument takes precedence over the XML and the defaults. + if 456 != cnf.GetInt("foo.bar") { + t.Fatal() + } + if "xmlValue" != cnf.Get("foo.baz") { + t.Fatalf("foo.baz = %s", cnf.Get("foo.baz")) + } + if "" != cnf.Get("commented.out") { + t.Fatal() + } + if 4611686018427387904 != cnf.GetInt64("cmdline.opt") { + t.Fatal() + } +} + +// Test our handling of the HTRACE_CONF_DIR environment variable. +func TestGetHTracedConfDirs(t *testing.T) { + os.Setenv("HTRACED_CONF_DIR", "") + dlog := new(bytes.Buffer) + dirs := getHTracedConfDirs(dlog) + if len(dirs) != 1 || dirs[0] != "." { + t.Fatal() + } + os.Setenv("HTRACED_CONF_DIR", "/foo/bar:/baz") + dirs = getHTracedConfDirs(dlog) + if len(dirs) != 2 || dirs[0] != "/foo/bar" || dirs[1] != "/baz" { + t.Fatal() + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/conf/xml.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/xml.go b/htrace-htraced/go/src/org/apache/htrace/conf/xml.go new file mode 100644 index 0000000..de14bc5 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/xml.go @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "encoding/xml" + "io" + "log" +) + +type configuration struct { + Properties []propertyXml `xml:"property"` +} + +type propertyXml struct { + Name string `xml:"name"` + Value string `xml:"value"` +} + +// Parse an XML configuration file. +func parseXml(reader io.Reader, m map[string]string) error { + dec := xml.NewDecoder(reader) + configurationXml := configuration{} + err := dec.Decode(&configurationXml) + if err != nil { + return err + } + props := configurationXml.Properties + for p := range props { + key := props[p].Name + value := props[p].Value + if key == "" { + log.Println("Warning: ignoring element with missing or empty <name>.") + continue + } + if value == "" { + log.Println("Warning: ignoring element with key " + key + " with missing or empty <value>.") + continue + } + //log.Printf("setting %s to %s\n", key, value) + m[key] = value + } + return nil +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/42b2f6a2/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go new file mode 100644 index 0000000..38cdb58 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/alecthomas/kingpin" + "io" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "time" +) + +var RELEASE_VERSION string +var GIT_VERSION string + +const EXIT_SUCCESS = 0 +const EXIT_FAILURE = 1 + +var verbose *bool + +const USAGE = `The Apache HTrace command-line tool. This tool retrieves and modifies settings and +other data on a running htraced daemon. + +If we find an ` + conf.CONFIG_FILE_NAME + ` configuration file in the list of directories +specified in ` + conf.HTRACED_CONF_DIR + `, we will use that configuration; otherwise, +the defaults will be used. +` + +func main() { + // Load htraced configuration + cnf := common.LoadApplicationConfig() + + // Parse argv + app := kingpin.New(os.Args[0], USAGE) + app.Flag("Dmy.key", "Set configuration key 'my.key' to 'my.value'. Replace 'my.key' "+ + "with any key you want to set.").Default("my.value").String() + addr := app.Flag("addr", "Server address.").String() + verbose = app.Flag("verbose", "Verbose.").Default("false").Bool() + version := app.Command("version", "Print the version of this program.") + serverInfo := app.Command("serverInfo", "Print information retrieved from an htraced server.") + findSpan := app.Command("findSpan", "Print information about a trace span with a given ID.") + findSpanId := findSpan.Arg("id", "Span ID to find. Example: 0x123456789abcdef").Required().Uint64() + findChildren := app.Command("findChildren", "Print out the span IDs that are children of a given span ID.") + parentSpanId := findChildren.Arg("id", "Span ID to print children for. Example: 0x123456789abcdef"). + Required().Uint64() + childLim := findChildren.Flag("lim", "Maximum number of child IDs to print.").Default("20").Int() + loadFile := app.Command("loadFile", "Write whitespace-separated JSON spans from a file to the server.") + loadFilePath := loadFile.Arg("path", + "A file containing whitespace-separated span JSON.").Required().String() + loadJson := app.Command("load", "Write JSON spans from the command-line to the server.") + loadJsonArg := loadJson.Arg("json", "A JSON span to write to the server.").Required().String() + dumpAll := app.Command("dumpAll", "Dump all spans from the htraced daemon.") + dumpAllOutPath := dumpAll.Arg("path", "The path to dump the trace spans to.").Default("-").String() + dumpAllLim := dumpAll.Flag("lim", "The number of spans to transfer from the server at once."). + Default("100").Int() + graph := app.Command("graph", "Visualize span JSON as a graph.") + graphJsonFile := graph.Arg("input", "The JSON file to load").Required().String() + graphDotFile := graph.Flag("output", + "The path to write a GraphViz dotfile to. This file can be used as input to "+ + "GraphViz, in order to generate a pretty picture. See graphviz.org for more "+ + "information about generating pictures of graphs.").Default("-").String() + query := app.Command("query", "Send a query to htraced.") + queryLim := query.Flag("lim", "Maximum number of spans to retrieve.").Default("20").Int() + queryArg := query.Arg("query", "The query string to send. Query strings have the format "+ + "[TYPE] [OPERATOR] [CONST], joined by AND statements.").Required().String() + rawQuery := app.Command("rawQuery", "Send a raw JSON query to htraced.") + rawQueryArg := query.Arg("json", "The query JSON to send.").Required().String() + cmd := kingpin.MustParse(app.Parse(os.Args[1:])) + + // Add the command-line settings into the configuration. + if *addr != "" { + cnf = cnf.Clone(conf.HTRACE_WEB_ADDRESS, *addr) + } + + // Handle commands that don't require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case graph.FullCommand(): + err := jsonSpanFileToDotFile(*graphJsonFile, *graphDotFile) + if err != nil { + fmt.Printf("graphing error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + // Create HTrace client + hcl, err := htrace.NewClient(cnf) + if err != nil { + fmt.Printf("Failed to create HTrace client: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + + // Handle commands that require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case serverInfo.FullCommand(): + os.Exit(printServerInfo(hcl)) + case findSpan.FullCommand(): + os.Exit(doFindSpan(hcl, common.SpanId(*findSpanId))) + case findChildren.FullCommand(): + os.Exit(doFindChildren(hcl, common.SpanId(*parentSpanId), *childLim)) + case loadJson.FullCommand(): + os.Exit(doLoadSpanJson(hcl, *loadJsonArg)) + case loadFile.FullCommand(): + os.Exit(doLoadSpanJsonFile(hcl, *loadFilePath)) + case dumpAll.FullCommand(): + err := doDumpAll(hcl, *dumpAllOutPath, *dumpAllLim) + if err != nil { + fmt.Printf("dumpAll error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case query.FullCommand(): + err := doQueryFromString(hcl, *queryArg, *queryLim) + if err != nil { + fmt.Printf("query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case rawQuery.FullCommand(): + err := doRawQuery(hcl, *rawQueryArg) + if err != nil { + fmt.Printf("raw query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + app.UsageErrorf(os.Stderr, "You must supply a command to run.") +} + +// Print the version of the htrace binary. +func printVersion() int { + fmt.Printf("Running htrace command version %s.\n", RELEASE_VERSION) + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/info +func printServerInfo(hcl *htrace.Client) int { + info, err := hcl.GetServerInfo() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + fmt.Printf("HTraced server version %s (%s)\n", info.ReleaseVersion, info.GitVersion) + return EXIT_SUCCESS +} + +// Print information about a trace span. +func doFindSpan(hcl *htrace.Client, sid common.SpanId) int { + span, err := hcl.FindSpan(sid) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + if span == nil { + fmt.Printf("Span ID not found.\n") + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(span, "", " ") + if err != nil { + fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + fmt.Printf("%s\n", string(pbuf)) + return EXIT_SUCCESS +} + +func doLoadSpanJsonFile(hcl *htrace.Client, spanFile string) int { + if spanFile == "" { + fmt.Printf("You must specify the json file to load.\n") + return EXIT_FAILURE + } + file, err := OpenInputFile(spanFile) + if err != nil { + fmt.Printf("Failed to open %s: %s\n", spanFile, err.Error()) + return EXIT_FAILURE + } + defer file.Close() + return doLoadSpans(hcl, bufio.NewReader(file)) +} + +func doLoadSpanJson(hcl *htrace.Client, spanJson string) int { + return doLoadSpans(hcl, bytes.NewBufferString(spanJson)) +} + +func doLoadSpans(hcl *htrace.Client, reader io.Reader) int { + dec := json.NewDecoder(reader) + spans := make([]*common.Span, 0, 32) + var err error + for { + var span common.Span + if err = dec.Decode(&span); err != nil { + if err == io.EOF { + break + } + fmt.Printf("Failed to decode JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + spans = append(spans, &span) + } + if *verbose { + fmt.Printf("Writing ") + prefix := "" + for i := range spans { + fmt.Printf("%s%s", prefix, spans[i].ToJson()) + prefix = ", " + } + fmt.Printf("\n") + } + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: spans, + }) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + return EXIT_SUCCESS +} + +// Find information about the children of a span. +func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int { + spanIds, err := hcl.FindChildren(sid, lim) + if err != nil { + fmt.Printf("%s\n", err.Error()) + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(spanIds, "", " ") + if err != nil { + fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error()) + return 1 + } + fmt.Printf("%s\n", string(pbuf)) + return 0 +} + +// Dump all spans from the htraced daemon. +func doDumpAll(hcl *htrace.Client, outPath string, lim int) error { + file, err := CreateOutputFile(outPath) + if err != nil { + return err + } + w := bufio.NewWriter(file) + defer func() { + if file != nil { + w.Flush() + file.Close() + } + }() + out := make(chan *common.Span, 50) + var dumpErr error + go func() { + dumpErr = hcl.DumpAll(lim, out) + }() + var numSpans int64 + nextLogTime := time.Now().Add(time.Second * 5) + for { + span, channelOpen := <-out + if !channelOpen { + break + } + if err == nil { + _, err = fmt.Fprintf(w, "%s\n", span.ToJson()) + } + if *verbose { + numSpans++ + now := time.Now() + if !now.Before(nextLogTime) { + nextLogTime = now.Add(time.Second * 5) + fmt.Printf("received %d span(s)...\n", numSpans) + } + } + } + if err != nil { + return errors.New(fmt.Sprintf("Write error %s", err.Error())) + } + if dumpErr != nil { + return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error())) + } + err = w.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + if err != nil { + return err + } + return nil +}
