Add TM2 validator for DsStats delivery services
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/095def7c Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/095def7c Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/095def7c Branch: refs/heads/master Commit: 095def7c03974f74b08e180476528ebbb6b32a90 Parents: 0dd73c1 Author: Robert Butts <[email protected]> Authored: Thu Mar 30 10:45:50 2017 -0600 Committer: Dave Neuman <[email protected]> Committed: Wed Apr 12 15:43:31 2017 -0600 ---------------------------------------------------------------------- .../traffic_monitor/tmcheck/dsstats.go | 137 +++++++++++++++++++ .../traffic_monitor/tmcheck/tmcheck.go | 20 +++ .../tools/nagios-validate-deliveryservices.go | 69 ++++++++++ .../traffic_monitor/tools/validator-service.go | 11 +- 4 files changed, 235 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/095def7c/traffic_monitor_golang/traffic_monitor/tmcheck/dsstats.go ---------------------------------------------------------------------- diff --git a/traffic_monitor_golang/traffic_monitor/tmcheck/dsstats.go b/traffic_monitor_golang/traffic_monitor/tmcheck/dsstats.go new file mode 100644 index 0000000..a35a3a7 --- /dev/null +++ b/traffic_monitor_golang/traffic_monitor/tmcheck/dsstats.go @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package tmcheck + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig" + dsdata "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/deliveryservicedata" + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/enum" + to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" +) + +// ValidateDSStates validates that all Delivery Services in the CRConfig exist in given Traffic Monitor's DSStats. +// Existence in DSStats is useful to verify, because "Available: false" in CRStates +func ValidateDSStats(tmURI string, toClient *to.Session) error { + cdn, err := GetCDN(tmURI) + if err != nil { + return fmt.Errorf("getting CDN from Traffic Monitor: %v", err) + } + return ValidateDSStatsWithCDN(tmURI, cdn, toClient) +} + +// ValidateOfflineStatesWithCDN validates per ValidateOfflineStates, but saves an additional query if the Traffic Monitor's CDN is known. +func ValidateDSStatsWithCDN(tmURI string, tmCDN string, toClient *to.Session) error { + crConfigBytes, err := toClient.CRConfigRaw(tmCDN) + if err != nil { + return fmt.Errorf("getting CRConfig: %v", err) + } + + crConfig := crconfig.CRConfig{} + if err := json.Unmarshal(crConfigBytes, &crConfig); err != nil { + return fmt.Errorf("unmarshalling CRConfig JSON: %v", err) + } + + return ValidateDSStatsWithCRConfig(tmURI, &crConfig, toClient) +} + +// ValidateOfflineStatesWithCRConfig validates per ValidateOfflineStates, but saves querying the CRconfig if it's already fetched. +func ValidateDSStatsWithCRConfig(tmURI string, crConfig *crconfig.CRConfig, toClient *to.Session) error { + dsStats, err := GetDSStats(tmURI + TrafficMonitorDSStatsPath) + if err != nil { + return fmt.Errorf("getting DSStats: %v", err) + } + + return ValidateDSStatsData(dsStats, crConfig) +} + +func hasCaches(dsName string, crconfig *crconfig.CRConfig) bool { + for _, server := range crconfig.ContentServers { + if _, ok := server.DeliveryServices[dsName]; ok { + return true + } + } + return false +} + +// ValidateDSStatsData validates that all delivery services in the given CRConfig with caches assigned exist in the given DSStats. +func ValidateDSStatsData(dsStats *dsdata.StatsOld, crconfig *crconfig.CRConfig) error { + for dsName, _ := range crconfig.DeliveryServices { + if !hasCaches(dsName, crconfig) { + continue + } + if _, ok := dsStats.DeliveryService[enum.DeliveryServiceName(dsName)]; !ok { + return fmt.Errorf("Delivery Service %v in CRConfig but not DSStats", dsName) + } + } + return nil +} + +// DSStatsValidator is designed to be run as a goroutine, and does not return. It continously validates every `interval`, and calls `onErr` on failure, `onResumeSuccess` when a failure ceases, and `onCheck` on every poll. +func DSStatsValidator( + tmURI string, + toClient *to.Session, + interval time.Duration, + grace time.Duration, + onErr func(error), + onResumeSuccess func(), + onCheck func(error), +) { + Validator(tmURI, toClient, interval, grace, onErr, onResumeSuccess, onCheck, ValidateDSStats) +} + +// AllMonitorsDSStatsValidator is designed to be run as a goroutine, and does not return. It continously validates every `interval`, and calls `onErr` on failure, `onResumeSuccess` when a failure ceases, and `onCheck` on every poll. Note the error passed to `onErr` may be a general validation error not associated with any monitor, in which case the passed `enum.TrafficMonitorName` will be empty. +func AllMonitorsDSStatsValidator( + toClient *to.Session, + interval time.Duration, + includeOffline bool, + grace time.Duration, + onErr func(enum.TrafficMonitorName, error), + onResumeSuccess func(enum.TrafficMonitorName), + onCheck func(enum.TrafficMonitorName, error), +) { + AllValidator(toClient, interval, includeOffline, grace, onErr, onResumeSuccess, onCheck, ValidateAllMonitorsDSStats) +} + +// ValidateAllMonitorDSStats validates, for all monitors in the given Traffic Ops, DSStats contains all Delivery Services in the CRConfig. +func ValidateAllMonitorsDSStats(toClient *to.Session, includeOffline bool) (map[enum.TrafficMonitorName]error, error) { + servers, err := GetMonitors(toClient, includeOffline) + if err != nil { + return nil, err + } + + crConfigs := GetCRConfigs(GetCDNs(servers), toClient) + + errs := map[enum.TrafficMonitorName]error{} + for _, server := range servers { + crConfig := crConfigs[enum.CDNName(server.CDNName)] + if err := crConfig.Err; err != nil { + errs[enum.TrafficMonitorName(server.HostName)] = fmt.Errorf("getting CRConfig: %v", err) + continue + } + + uri := fmt.Sprintf("http://%s.%s";, server.HostName, server.DomainName) + errs[enum.TrafficMonitorName(server.HostName)] = ValidateDSStatsWithCRConfig(uri, crConfig.CRConfig, toClient) + } + return errs, nil +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/095def7c/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go ---------------------------------------------------------------------- diff --git a/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go b/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go index 9c6016b..1ef2532 100644 --- a/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go +++ b/traffic_monitor_golang/traffic_monitor/tmcheck/tmcheck.go @@ -29,6 +29,7 @@ import ( "time" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig" + dsdata "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/deliveryservicedata" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/enum" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/peer" to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" @@ -37,6 +38,7 @@ import ( const RequestTimeout = time.Second * time.Duration(30) const TrafficMonitorCRStatesPath = "/publish/CrStates" +const TrafficMonitorDSStatsPath = "/publish/DsStats" const TrafficMonitorConfigDocPath = "/publish/ConfigDoc" func getClient() *http.Client { @@ -87,6 +89,24 @@ func GetCRStates(uri string) (*peer.Crstates, error) { return &states, nil } +// GetCRStates gets the CRStates from the given Traffic Monitor. +func GetDSStats(uri string) (*dsdata.StatsOld, error) { + resp, err := getClient().Get(uri) + if err != nil { + return nil, fmt.Errorf("reading reply from %v: %v\n", uri, err) + } + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading reply from %v: %v\n", uri, err) + } + + dsStats := dsdata.StatsOld{} + if err := json.Unmarshal(respBytes, &dsStats); err != nil { + return nil, fmt.Errorf("unmarshalling: %v", err) + } + return &dsStats, nil +} + type ValidatorFunc func( tmURI string, toClient *to.Session, http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/095def7c/traffic_monitor_golang/traffic_monitor/tools/nagios-validate-deliveryservices.go ---------------------------------------------------------------------- diff --git a/traffic_monitor_golang/traffic_monitor/tools/nagios-validate-deliveryservices.go b/traffic_monitor_golang/traffic_monitor/tools/nagios-validate-deliveryservices.go new file mode 100644 index 0000000..3afdb72 --- /dev/null +++ b/traffic_monitor_golang/traffic_monitor/tools/nagios-validate-deliveryservices.go @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "flag" + "fmt" + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/nagios" + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/tmcheck" + to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" +) + +const UserAgent = "tm-offline-validator/0.1" + +func main() { + toURI := flag.String("to", "", "The Traffic Ops URI, whose CRConfig to validate") + toUser := flag.String("touser", "", "The Traffic Ops user") + toPass := flag.String("topass", "", "The Traffic Ops password") + includeOffline := flag.Bool("includeOffline", false, "Whether to include Offline Monitors") + help := flag.Bool("help", false, "Usage info") + helpBrief := flag.Bool("h", false, "Usage info") + flag.Parse() + if *help || *helpBrief || *toURI == "" { + fmt.Printf("Usage: ./nagios-validate-offline -to https://traffic-ops.example.net -touser bill -topass thelizard -includeOffline true\n") + return + } + + toClient, err := to.LoginWithAgent(*toURI, *toUser, *toPass, true, UserAgent, false, tmcheck.RequestTimeout) + if err != nil { + fmt.Printf("Error logging in to Traffic Ops: %v\n", err) + return + } + + monitorErrs, err := tmcheck.ValidateAllMonitorsDSStats(toClient, *includeOffline) + + if err != nil { + nagios.Exit(nagios.Critical, fmt.Sprintf("Error validating monitor delivery services: %v", err)) + } + + errStr := "" + for monitor, err := range monitorErrs { + if err != nil { + errStr += fmt.Sprintf("error validating delivery services for monitor %v : %v\n", monitor, err.Error()) + } + } + + if errStr != "" { + nagios.Exit(nagios.Critical, errStr) + } + + nagios.Exit(nagios.Ok, "") +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/095def7c/traffic_monitor_golang/traffic_monitor/tools/validator-service.go ---------------------------------------------------------------------- diff --git a/traffic_monitor_golang/traffic_monitor/tools/validator-service.go b/traffic_monitor_golang/traffic_monitor/tools/validator-service.go index 22295dd..5dc1e04 100644 --- a/traffic_monitor_golang/traffic_monitor/tools/validator-service.go +++ b/traffic_monitor_golang/traffic_monitor/tools/validator-service.go @@ -157,8 +157,9 @@ func main() { crStatesOfflineLogs := startValidator(tmcheck.AllMonitorsCRStatesOfflineValidator, toClient, *interval, *includeOffline, *grace) peerPollerLogs := startValidator(tmcheck.PeerPollersAllValidator, toClient, *interval, *includeOffline, *grace) + dsStatsLogs := startValidator(tmcheck.AllMonitorsDSStatsValidator, toClient, *interval, *includeOffline, *grace) - if err := serve(*toURI, crStatesOfflineLogs, peerPollerLogs); err != nil { + if err := serve(*toURI, crStatesOfflineLogs, peerPollerLogs, dsStatsLogs); err != nil { fmt.Printf("Serve error: %v\n", err) } } @@ -198,7 +199,7 @@ func printLogs(logs Logs, w io.Writer) { fmt.Fprintf(w, `</table>`) } -func serve(toURI string, crStatesOfflineLogs Logs, peerPollerLogs Logs) error { +func serve(toURI string, crStatesOfflineLogs Logs, peerPollerLogs Logs, dsStatsLogs Logs) error { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "text/html") @@ -212,13 +213,19 @@ func serve(toURI string, crStatesOfflineLogs Logs, peerPollerLogs Logs) error { fmt.Fprintf(w, `<h1>Traffic Monitor Validator</h1>`) fmt.Fprintf(w, `<p>%s`, toURI) + fmt.Fprintf(w, `<p>%s`, time.Now()) fmt.Fprintf(w, `<h2>CRStates Offline</h2>`) + fmt.Fprintf(w, `<h3>validates all OFFLINE and ADMIN_DOWN caches in the CRConfig are Unavailable</h3>`) printLogs(crStatesOfflineLogs, w) fmt.Fprintf(w, `<h2>Peer Poller</h2>`) + fmt.Fprintf(w, `<h3>validates all peers in the CRConfig have been polled within the last %v</h3>`, tmcheck.PeerPollMax) printLogs(peerPollerLogs, w) + fmt.Fprintf(w, `<h2>Delivery Services</h2>`) + fmt.Fprintf(w, `<h3>validates all Delivery Services in the CRConfig exist in DsStats</h3>`) + printLogs(dsStatsLogs, w) }) return http.ListenAndServe(":80", nil) }
