rob05c commented on a change in pull request #2306: Add TO Go cdns/capacity URL: https://github.com/apache/trafficcontrol/pull/2306#discussion_r316795931
########## File path: traffic_ops/traffic_ops_golang/cdn/capacity.go ########## @@ -0,0 +1,325 @@ +package cdn + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" +) + +func GetCapacity(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + api.RespWriter(w, r, inf.Tx.Tx)(getCapacity(inf.Tx.Tx)) +} + +const MonitorProxyParameter = "tm.traffic_mon_fwd_proxy" +const MonitorRequestTimeout = time.Second * 10 +const MonitorOnlineStatus = "ONLINE" + +// CRStates contains the Monitor CRStates members needed for health. It is NOT the full object served by the Monitor, but only the data required by this endpoint. +type CRStates struct { + Caches map[tc.CacheName]Available `json:"caches"` +} + +type Available struct { + IsAvailable bool `json:"isAvailable"` +} + +// CRConfig contains the Monitor CRConfig members needed for health. It is NOT the full object served by the Monitor, but only the data required by this endpoint. +type CRConfig struct { + ContentServers map[tc.CacheName]CRConfigServer `json:"contentServers"` +} + +type CRConfigServer struct { + CacheGroup tc.CacheGroupName `json:"locationId"` + Status tc.CacheStatus `json:"status"` + Type tc.CacheType `json:"type"` + Profile string `json:"profile"` +} + +func getCapacity(tx *sql.Tx) (CapacityResp, error) { + monitors, err := getCDNMonitorFQDNs(tx) + if err != nil { + return CapacityResp{}, errors.New("getting monitors: " + err.Error()) + } + return getMonitorsCapacity(tx, monitors) +} + +type CapacityResp struct { + AvailablePercent float64 `json:"availablePercent"` + UnavailablePercent float64 `json:"unavailablePercent"` + UtilizedPercent float64 `json:utilizedPercent"` + MaintenancePercent float64 `json:maintenancePercent"` +} + +type CapData struct { + Available float64 + Unavailable float64 + Utilized float64 + Maintenance float64 + Capacity float64 +} + +func getMonitorsCapacity(tx *sql.Tx, monitors map[tc.CDNName]string) (CapacityResp, error) { + monitorForwardProxy, monitorForwardProxyExists, err := getGlobalParam(tx, MonitorProxyParameter) + if err != nil { + return CapacityResp{}, errors.New("getting global monitor proxy parameter: " + err.Error()) + } + client := &http.Client{Timeout: MonitorRequestTimeout} + if monitorForwardProxyExists { + proxyURI, err := url.Parse(monitorForwardProxy) + if err != nil { + return CapacityResp{}, errors.New("monitor forward proxy '" + monitorForwardProxy + "' in parameter '" + MonitorProxyParameter + "' not a URI: " + err.Error()) + } + client = &http.Client{Timeout: MonitorRequestTimeout, Transport: &http.Transport{Proxy: http.ProxyURL(proxyURI)}} + } + + thresholds, err := getEdgeProfileHealthThresholdBandwidth(tx) + if err != nil { + return CapacityResp{}, errors.New("getting profile thresholds: " + err.Error()) + } + + cap := CapData{} + for cdn, monitorFQDN := range monitors { + crStates, err := getCRStates(monitorFQDN, client) + // TODO on err, try another online monitor + if err != nil { + return CapacityResp{}, errors.New("getting CRStates for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + crConfig, err := getCRConfig(monitorFQDN, client) + // TODO on err, try another online monitor + if err != nil { + return CapacityResp{}, errors.New("getting CRConfig for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + cacheStats := CacheStats{} + // TODO on err, try another online monitor + if err := getCacheStats(monitorFQDN, client, []string{"kbps", "maxKbps"}, &cacheStats); err != nil { + return CapacityResp{}, errors.New("getting cache stats for CDN '" + string(cdn) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + + cap = addCapacity(cap, cacheStats, crStates, crConfig, thresholds) + } + if cap.Capacity == 0 { + return CapacityResp{}, errors.New("capacity was zero!") // avoid divide-by-zero below. + } + resp := CapacityResp{} + resp.UtilizedPercent = (cap.Available * 100) / cap.Capacity + resp.UnavailablePercent = (cap.Unavailable * 100) / cap.Capacity + resp.MaintenancePercent = (cap.Maintenance * 100) / cap.Capacity + resp.AvailablePercent = ((cap.Capacity - cap.Unavailable - cap.Maintenance - cap.Available) * 100) / cap.Capacity + return resp, nil +} + +func addCapacity(cap CapData, cacheStats CacheStats, crStates CRStates, crConfig CRConfig, thresholds map[string]float64) CapData { + for cacheName, stats := range cacheStats.Caches { + cache, ok := crConfig.ContentServers[cacheName] + if !ok { + continue + } + if !strings.HasPrefix(string(cache.Type), string(tc.CacheTypeEdge)) { + continue + } + if len(stats.KBPS) < 1 || len(stats.MaxKBPS) < 1 { + continue + } + if cache.Status == "REPORTED" || cache.Status == "ONLINE" { + if crStates.Caches[cacheName].IsAvailable { + cap.Available += float64(stats.KBPS[0].Value) + } else { + cap.Unavailable += float64(stats.KBPS[0].Value) + } + } else if cache.Status == "ADMIN_DOWN" { + cap.Maintenance += float64(stats.KBPS[0].Value) + } else { + continue // don't add capacity for OFFLINE or other statuses + } + cap.Capacity += float64(stats.MaxKBPS[0].Value) - thresholds[cache.Profile] + } + return cap +} + +func getEdgeProfileHealthThresholdBandwidth(tx *sql.Tx) (map[string]float64, error) { + rows, err := tx.Query(` +SELECT pr.name as profile, pa.name, pa.config_file, pa.value +FROM parameter as pa +JOIN profile_parameter as pp ON pp.parameter = pa.id +JOIN profile as pr ON pp.profile = pr.id +JOIN server as s ON s.profile = pr.id +JOIN cdn as c ON c.id = s.cdn_id +JOIN type as t ON s.type = t.id +WHERE t.name LIKE 'EDGE%' +AND pa.config_file = 'rascal-config.txt' +AND pa.name = 'health.threshold.availableBandwidthInKbps' +`) + if err != nil { + return nil, errors.New("querying thresholds: " + err.Error()) + } + defer rows.Close() + profileThresholds := map[string]float64{} + for rows.Next() { + profile := "" + threshStr := "" + if err := rows.Scan(&profile, &threshStr); err != nil { + return nil, errors.New("scanning thresholds: " + err.Error()) + } + threshStr = strings.TrimPrefix(threshStr, ">") + thresh, err := strconv.ParseFloat(threshStr, 64) + if err != nil { + return nil, errors.New("profile '" + profile + "' health.threshold.availableBandwidthInKbps is not a number") + } + profileThresholds[profile] = thresh + } + return profileThresholds, nil +} + +func getCRStates(monitorFQDN string, client *http.Client) (CRStates, error) { + path := `/publish/CrStates` + resp, err := client.Get("http://"; + monitorFQDN + path) + if err != nil { + return CRStates{}, errors.New("getting CRStates from Monitor '" + monitorFQDN + "': " + err.Error()) + } + defer resp.Body.Close() + + crs := CRStates{} + if err := json.NewDecoder(resp.Body).Decode(&crs); err != nil { + return CRStates{}, errors.New("decoding CRStates from monitor '" + monitorFQDN + "': " + err.Error()) + } + return crs, nil +} + +func getCRConfig(monitorFQDN string, client *http.Client) (CRConfig, error) { + path := `/publish/CrConfig` + resp, err := client.Get("http://"; + monitorFQDN + path) + if err != nil { + return CRConfig{}, errors.New("getting CRConfig from Monitor '" + monitorFQDN + "': " + err.Error()) + } + defer resp.Body.Close() + crs := CRConfig{} + if err := json.NewDecoder(resp.Body).Decode(&crs); err != nil { + return CRConfig{}, errors.New("decoding CRConfig from monitor '" + monitorFQDN + "': " + err.Error()) + } + return crs, nil +} + +// CacheStats contains the Monitor CacheStats needed by Cachedata. It is NOT the full object served by the Monitor, but only the data required by the caches stats endpoint. +type CacheStats struct { + Caches map[tc.CacheName]CacheStat `json:"caches"` +} + +type CacheStat struct { + KBPS []CacheStatData `json:"kbps"` + MaxKBPS []CacheStatData `json:"maxKbps"` +} + +type CacheStatData struct { + Value float64 `json:"value,string"` +} + +// getCacheStats gets the cache stats from the given monitor. It takes stats, a slice of stat names; and cacheStats, an object to deserialize stats into. The cacheStats type must be of the form struct {caches map[tc.CacheName]struct{statName []struct{value float64}}} with the desired stats, with appropriate member names or tags. +func getCacheStats(monitorFQDN string, client *http.Client, stats []string, cacheStats interface{}) error { + path := `/publish/CacheStats` + if len(stats) > 0 { + path += `?stats=` + strings.Join(stats, `,`) + } + resp, err := client.Get("http://"; + monitorFQDN + path) + if err != nil { + return errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error()) + } + defer resp.Body.Close() + if err := json.NewDecoder(resp.Body).Decode(cacheStats); err != nil { + return errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error()) + } + return nil +} + +func getMonitorForwardProxy(tx *sql.Tx) (string, error) { Review comment: It wasn't, removed. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
