Move TO Golang microservice out of experimental
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/04ed9f14 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/04ed9f14 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/04ed9f14 Branch: refs/heads/master Commit: 04ed9f141da0022678571f343752124e46a18969 Parents: aa90195 Author: Robert Butts <robert.o.bu...@gmail.com> Authored: Sun Jul 9 09:22:28 2017 -0600 Committer: Dewayne Richardson <dewr...@apache.org> Committed: Thu Aug 10 09:46:02 2017 -0600 ---------------------------------------------------------------------- traffic_ops/experimental/goto/config.go | 80 ---- traffic_ops/experimental/goto/config.json | 13 - traffic_ops/experimental/goto/goto.go | 56 --- traffic_ops/experimental/goto/monitoring.go | 398 ------------------- traffic_ops/experimental/goto/routes.go | 95 ----- traffic_ops/experimental/goto/wrappers.go | 55 --- traffic_ops/traffic_ops_golang/config.go | 80 ++++ traffic_ops/traffic_ops_golang/monitoring.go | 398 +++++++++++++++++++ traffic_ops/traffic_ops_golang/routes.go | 95 +++++ .../traffic_ops_golang.config | 13 + .../traffic_ops_golang/traffic_ops_golang.go | 56 +++ traffic_ops/traffic_ops_golang/wrappers.go | 55 +++ 12 files changed, 697 insertions(+), 697 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/config.go ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/config.go b/traffic_ops/experimental/goto/config.go deleted file mode 100644 index a0f3401..0000000 --- a/traffic_ops/experimental/goto/config.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/url" -) - -type Config struct { - HTTPPort string `json:"port"` - DBUser string `json:"db_user"` - DBPass string `json:"db_pass"` - DBServer string `json:"db_server"` - DBDB string `json:"db_name"` - DBSSL bool `json:"db_ssl"` - TOSecret string `json:"to_secret"` - TOURLStr string `json:"to_url"` - TOURL *url.URL `json:"-"` - NoAuth bool `json:"no_auth"` - CertPath string `json:"cert_path"` - KeyPath string `json:"key_path"` -} - -func LoadConfig(fileName string) (Config, error) { - if fileName == "" { - return Config{}, fmt.Errorf("no filename") - } - - configBytes, err := ioutil.ReadFile(fileName) - if err != nil { - return Config{}, err - } - - cfg := Config{} - if err := json.Unmarshal(configBytes, &cfg); err != nil { - return Config{}, err - } - - if cfg, err = ParseConfig(cfg); err != nil { - return Config{}, err - } - - return cfg, nil -} - -// ParseConfig validates required fields, and parses non-JSON types -func ParseConfig(cfg Config) (Config, error) { - if cfg.HTTPPort == "" { - return Config{}, fmt.Errorf("missing port") - } - if cfg.DBUser == "" { - return Config{}, fmt.Errorf("missing database user") - } - if cfg.DBPass == "" { - return Config{}, fmt.Errorf("missing database password") - } - if cfg.DBServer == "" { - return Config{}, fmt.Errorf("missing database server") - } - if cfg.DBDB == "" { - return Config{}, fmt.Errorf("missing database name") - } - if cfg.TOSecret == "" { - return Config{}, fmt.Errorf("missing secret") - } - if cfg.CertPath == "" { - return Config{}, fmt.Errorf("missing certificate path") - } - if cfg.KeyPath == "" { - return Config{}, fmt.Errorf("missing certificate key path") - } - - var err error - if cfg.TOURL, err = url.Parse(cfg.TOURLStr); err != nil { - return Config{}, fmt.Errorf("Invalid Traffic Ops URL '%v': err", cfg.TOURL, err) - } - - return cfg, nil -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/config.json ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/config.json b/traffic_ops/experimental/goto/config.json deleted file mode 100644 index 19d2599..0000000 --- a/traffic_ops/experimental/goto/config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "port": "443", - "db_user": "bill", - "db_pass": "thelizard", - "db_server": "db.trafficops.example.net", - "db_name": "trafficops", - "db_ssl": true, - "to_secret": "walrus", - "to_url": "https://trafficops.example.net:60443", - "no_auth": false, - "cert_path": "/opt/traffic_ops/cert", - "key_path": "/opt/traffic_ops/key" -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/goto.go ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/goto.go b/traffic_ops/experimental/goto/goto.go deleted file mode 100644 index 48d3c4e..0000000 --- a/traffic_ops/experimental/goto/goto.go +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "database/sql" - "flag" - "fmt" - _ "github.com/lib/pq" - "net/http" -) - -const DefaultConfigPath = "/etc/goto/config.json" - -func main() { - configFileName := flag.String("cfg", "", "The config file path") - flag.Parse() - if *configFileName == "" { - *configFileName = DefaultConfigPath - } - - cfg, err := LoadConfig(*configFileName) - if err != nil { - fmt.Println("Error loading config '" + *configFileName + "': " + err.Error()) - return - } - - sslStr := "require" - if !cfg.DBSSL { - sslStr = "disable" - } - - db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", cfg.DBUser, cfg.DBPass, cfg.DBServer, cfg.DBDB, sslStr)) - if err != nil { - fmt.Printf("Error opening database: %v\n", err) - return - } - defer db.Close() - - RegisterRoutes(ServerData{DB: db, Config: cfg}) - fmt.Println("Listening on " + cfg.HTTPPort) - if err := http.ListenAndServeTLS(":"+cfg.HTTPPort, cfg.CertPath, cfg.KeyPath, nil); err != nil { - fmt.Printf("Error stopping server: %v\n", err) - return - } -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/monitoring.go ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/monitoring.go b/traffic_ops/experimental/goto/monitoring.go deleted file mode 100644 index b353fb5..0000000 --- a/traffic_ops/experimental/goto/monitoring.go +++ /dev/null @@ -1,398 +0,0 @@ -package main - -import ( - "database/sql" - "encoding/json" - "fmt" - "github.com/lib/pq" - "net/http" - "strings" - "time" -) - -const CacheMonitorConfigFile = "rascal.properties" -const MonitorType = "RASCAL" -const RouterType = "CCR" -const MonitorProfilePrefix = "RASCAL" -const MonitorConfigFile = "rascal-config.txt" -const KilobitsPerMegabit = 1000 -const DeliveryServiceStatus = "REPORTED" - -type BasicServer struct { - Profile string `json:"profile"` - Status string `json:"status"` - IP string `json:"ip"` - IP6 string `json:"ip6"` - Port int `json:"port"` - Cachegroup string `json:"cachegroup"` - HostName string `json:"hostName"` - FQDN string `json:"fqdn"` -} - -type Monitor struct { - BasicServer -} - -type Cache struct { - BasicServer - InterfaceName string `json:"interfaceName"` - Type string `json:"type"` - HashID string `json:"hashId"` -} - -type Cachegroup struct { - Name string `json:"name"` - Coordinates Coordinates `json:"coordinates"` -} - -type Coordinates struct { - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` -} - -type Profile struct { - Name string `json:"name"` - Type string `json:"type"` - Parameters map[string]string `json:"parameters"` -} - -type Monitoring struct { - TrafficServers []Cache `json:"trafficServers"` - TrafficMonitors []Monitor `json:"trafficMonitors"` - Cachegroups []Cachegroup `json:"cacheGroups"` - Profiles []Profile `json:"profiles"` - DeliveryServices []DeliveryService `json:"deliveryServices"` - Config map[string]string `json:"config"` -} - -type MonitoringResponse struct { - Response Monitoring `json:"response"` -} - -type Router struct { - Type string - Profile string -} - -type DeliveryService struct { - XMLID string `json:"xmlId"` - TotalTPSThreshold float64 `json:"totalTpsThreshold"` - Status string `json:"status"` - TotalKBPSThreshold float64 `json:"totalKbpsThreshold"` -} - -// TODO change to use the ParamMap, instead of parsing the URL -func monitoringHandler(db *sql.DB) RegexHandlerFunc { - return func(w http.ResponseWriter, r *http.Request, p ParamMap) { - handleErr := func(err error, status int) { - fmt.Printf("%v %v error %v\n", time.Now(), r.RemoteAddr, err) - w.WriteHeader(status) - fmt.Fprintf(w, http.StatusText(status)) - } - - cdnName := p["cdn"] - - resp, err := getMonitoringJson(cdnName, db) - if err != nil { - handleErr(err, http.StatusInternalServerError) - return - } - - respBts, err := json.Marshal(resp) - if err != nil { - handleErr(err, http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - fmt.Fprintf(w, "%s", respBts) - } -} - -func getServers(db *sql.DB, cdn string) ([]Monitor, []Cache, []Router, error) { - query := `SELECT -me.host_name as hostName, -CONCAT(me.host_name, '.', me.domain_name) as fqdn, -status.name as status, -cachegroup.name as cachegroup, -me.tcp_port as port, -me.ip_address as ip, -me.ip6_address as ip6, -profile.name as profile, -me.interface_name as interfaceName, -type.name as type, -me.xmpp_id as hashId -FROM server me -JOIN type type ON type.id = me.type -JOIN status status ON status.id = me.status -JOIN cachegroup cachegroup ON cachegroup.id = me.cachegroup -JOIN profile profile ON profile.id = me.profile -JOIN cdn cdn ON cdn.id = me.cdn_id -WHERE cdn.name = $1` - - rows, err := db.Query(query, cdn) - if err != nil { - return nil, nil, nil, err - } - defer rows.Close() - - monitors := []Monitor{} - caches := []Cache{} - routers := []Router{} - - for rows.Next() { - var hostName sql.NullString - var fqdn sql.NullString - var status sql.NullString - var cachegroup sql.NullString - var port sql.NullInt64 - var ip sql.NullString - var ip6 sql.NullString - var profile sql.NullString - var interfaceName sql.NullString - var ttype sql.NullString - var hashId sql.NullString - - if err := rows.Scan(&hostName, &fqdn, &status, &cachegroup, &port, &ip, &ip6, &profile, &interfaceName, &ttype, &hashId); err != nil { - return nil, nil, nil, err - } - - if ttype.String == MonitorType { - monitors = append(monitors, Monitor{ - BasicServer: BasicServer{ - Profile: profile.String, - Status: status.String, - IP: ip.String, - IP6: ip6.String, - Port: int(port.Int64), - Cachegroup: cachegroup.String, - HostName: hostName.String, - FQDN: fqdn.String, - }, - }) - } else if strings.HasPrefix(ttype.String, "EDGE") || strings.HasPrefix(ttype.String, "MID") { - caches = append(caches, Cache{ - BasicServer: BasicServer{ - Profile: profile.String, - Status: status.String, - IP: ip.String, - IP6: ip6.String, - Port: int(port.Int64), - Cachegroup: cachegroup.String, - HostName: hostName.String, - FQDN: fqdn.String, - }, - InterfaceName: interfaceName.String, - Type: ttype.String, - HashID: hashId.String, - }) - } else if ttype.String == RouterType { - routers = append(routers, Router{ - Type: ttype.String, - Profile: profile.String, - }) - } - } - return monitors, caches, routers, nil -} - -func getCachegroups(db *sql.DB, cdn string) ([]Cachegroup, error) { - query := ` -SELECT name, latitude, longitude -FROM cachegroup -WHERE id IN - (SELECT cachegroup FROM server WHERE server.cdn_id = - (SELECT id FROM cdn WHERE name = $1));` - - rows, err := db.Query(query, cdn) - if err != nil { - return nil, err - } - defer rows.Close() - - cachegroups := []Cachegroup{} - - for rows.Next() { - var name sql.NullString - var lat sql.NullFloat64 - var lon sql.NullFloat64 - if err := rows.Scan(&name, &lat, &lon); err != nil { - return nil, err - } - cachegroups = append(cachegroups, Cachegroup{ - Name: name.String, - Coordinates: Coordinates{ - Latitude: lat.Float64, - Longitude: lon.Float64, - }, - }) - } - return cachegroups, nil -} - -func getProfiles(db *sql.DB, caches []Cache, routers []Router) ([]Profile, error) { - cacheProfileTypes := map[string]string{} - profiles := map[string]Profile{} - profileNames := []string{} - for _, router := range routers { - profiles[router.Profile] = Profile{ - Name: router.Profile, - Type: router.Type, - } - } - - for _, cache := range caches { - if _, ok := cacheProfileTypes[cache.Profile]; !ok { - cacheProfileTypes[cache.Profile] = cache.Type - profiles[cache.Profile] = Profile{ - Name: cache.Profile, - Type: cache.Type, - } - profileNames = append(profileNames, cache.Profile) - } - } - - query := ` -SELECT p.name as profile, pr.name, pr.value -FROM parameter pr -JOIN profile p ON p.name = ANY($1) -JOIN profile_parameter pp ON pp.profile = p.id and pp.parameter = pr.id -WHERE pr.config_file = $2; -` - rows, err := db.Query(query, pq.Array(profileNames), CacheMonitorConfigFile) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var profileName sql.NullString - var name sql.NullString - var value sql.NullString - if err := rows.Scan(&profileName, &name, &value); err != nil { - return nil, err - } - if name.String == "" { - return nil, fmt.Errorf("null name") // TODO continue and warn? - } - profile := profiles[profileName.String] - if profile.Parameters == nil { - profile.Parameters = map[string]string{} - } - profile.Parameters[name.String] = value.String - profiles[profileName.String] = profile - - } - - profilesArr := []Profile{} // TODO make for efficiency? - for _, profile := range profiles { - profilesArr = append(profilesArr, profile) - } - return profilesArr, nil -} - -func getDeliveryServices(db *sql.DB, routers []Router) ([]DeliveryService, error) { - profileNames := []string{} - for _, router := range routers { - profileNames = append(profileNames, router.Profile) - } - - query := ` -SELECT ds.xml_id, ds.global_max_tps, ds.global_max_mbps -FROM deliveryservice ds -JOIN profile profile ON profile.id = ds.profile -WHERE profile.name = ANY($1) -AND ds.active = true -` - rows, err := db.Query(query, pq.Array(profileNames)) - if err != nil { - return nil, err - } - defer rows.Close() - - dses := []DeliveryService{} - - for rows.Next() { - var xmlid sql.NullString - var tps sql.NullFloat64 - var mbps sql.NullFloat64 - if err := rows.Scan(&xmlid, &tps, &mbps); err != nil { - return nil, err - } - dses = append(dses, DeliveryService{ - XMLID: xmlid.String, - TotalTPSThreshold: tps.Float64, - Status: DeliveryServiceStatus, - TotalKBPSThreshold: mbps.Float64 * KilobitsPerMegabit, - }) - } - return dses, nil -} - -func getConfig(db *sql.DB) (map[string]string, error) { - // TODO remove 'like' in query? Slow? - query := fmt.Sprintf(` -SELECT pr.name, pr.value -FROM parameter pr -JOIN profile p ON p.name LIKE '%s%%' -JOIN profile_parameter pp ON pp.profile = p.id and pp.parameter = pr.id -WHERE pr.config_file = '%s' -`, MonitorProfilePrefix, MonitorConfigFile) - - rows, err := db.Query(query) - if err != nil { - return nil, err - } - defer rows.Close() - - cfg := map[string]string{} - - for rows.Next() { - var name sql.NullString - var val sql.NullString - if err := rows.Scan(&name, &val); err != nil { - return nil, err - } - cfg[name.String] = val.String - } - return cfg, nil -} - -func getMonitoringJson(cdnName string, db *sql.DB) (*MonitoringResponse, error) { - monitors, caches, routers, err := getServers(db, cdnName) - if err != nil { - return nil, fmt.Errorf("error getting servers: %v", err) - } - - cachegroups, err := getCachegroups(db, cdnName) - if err != nil { - return nil, fmt.Errorf("error getting cachegroups: %v", err) - } - - profiles, err := getProfiles(db, caches, routers) - if err != nil { - return nil, fmt.Errorf("error getting profiles: %v", err) - } - - deliveryServices, err := getDeliveryServices(db, routers) - if err != nil { - return nil, fmt.Errorf("error getting deliveryservices: %v", err) - } - - config, err := getConfig(db) - if err != nil { - return nil, fmt.Errorf("error getting config: %v", err) - } - - resp := MonitoringResponse{ - Response: Monitoring{ - TrafficServers: caches, - TrafficMonitors: monitors, - Cachegroups: cachegroups, - Profiles: profiles, - DeliveryServices: deliveryServices, - Config: config, - }, - } - return &resp, nil -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/routes.go ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/routes.go b/traffic_ops/experimental/goto/routes.go deleted file mode 100644 index cc14450..0000000 --- a/traffic_ops/experimental/goto/routes.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "crypto/tls" - "database/sql" - "net/http" - "net/http/httputil" - "regexp" - "strings" -) - -type ServerData struct { - Config - DB *sql.DB -} - -type ParamMap map[string]string - -type RegexHandlerFunc func(w http.ResponseWriter, r *http.Request, params ParamMap) - -func getMonitoringRoute(d ServerData) RegexHandlerFunc { - return wrapLogTime(wrapAuth(monitoringHandler(d.DB), d.NoAuth, d.TOSecret)) -} - -// getRootHandler returns the / handler for the service, which reverse-proxies the old Perl Traffic Ops -func getRootHandler(d ServerData) http.Handler { - // debug - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - rp := httputil.NewSingleHostReverseProxy(d.TOURL) - rp.Transport = tr - return rp -} - -// GetRoutes returns the map of regex routes, and a catchall route if no regex matches. -func GetRoutes(d ServerData) (map[string]RegexHandlerFunc, http.Handler) { - return map[string]RegexHandlerFunc{ - "api/1.2/cdns/{cdn}/configs/monitoring.json": getMonitoringRoute(d), - }, getRootHandler(d) -} - -type CompiledRoute struct { - Handler RegexHandlerFunc - Regex *regexp.Regexp - Params []string -} - -func CompileRoutes(routes *map[string]RegexHandlerFunc) map[string]CompiledRoute { - compiledRoutes := map[string]CompiledRoute{} - for route, handler := range *routes { - originalRoute := route - var params []string - for open := strings.Index(route, "{"); open > 0; open = strings.Index(route, "{") { - close := strings.Index(route, "}") - if close < 0 { - panic("malformed route") - } - param := route[open+1 : close] - - params = append(params, param) - route = route[:open] + `(.+)` + route[close+1:] - } - regex := regexp.MustCompile(route) - compiledRoutes[originalRoute] = CompiledRoute{Handler: handler, Regex: regex, Params: params} - } - return compiledRoutes -} - -func Handler(routes map[string]CompiledRoute, catchall http.Handler, w http.ResponseWriter, r *http.Request) { - requested := r.URL.Path[1:] - - for _, compiledRoute := range routes { - match := compiledRoute.Regex.FindStringSubmatch(requested) - if len(match) == 0 { - continue - } - - params := map[string]string{} - for i, v := range compiledRoute.Params { - params[v] = match[i+1] - } - compiledRoute.Handler(w, r, params) - return - } - catchall.ServeHTTP(w, r) -} - -func RegisterRoutes(d ServerData) { - routes, catchall := GetRoutes(d) - compiledRoutes := CompileRoutes(&routes) - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - Handler(compiledRoutes, catchall, w, r) - }) -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/experimental/goto/wrappers.go ---------------------------------------------------------------------- diff --git a/traffic_ops/experimental/goto/wrappers.go b/traffic_ops/experimental/goto/wrappers.go deleted file mode 100644 index daf5e34..0000000 --- a/traffic_ops/experimental/goto/wrappers.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "fmt" - "github.com/apache/incubator-trafficcontrol/traffic_ops/experimental/tocookie" - "log" // TODO change to traffic_monitor_golang/common/log - "net/http" - "time" -) - -func wrapAuth(h RegexHandlerFunc, noAuth bool, secret string) RegexHandlerFunc { - if noAuth { - return h - } - return func(w http.ResponseWriter, r *http.Request, p ParamMap) { - handleUnauthorized := func(reason string) { - log.Printf("%v %v %v sent 401 - %v\n", time.Now(), r.RemoteAddr, r.URL.Path, reason) - status := http.StatusUnauthorized - w.WriteHeader(status) - fmt.Fprintf(w, http.StatusText(status)) - } - - cookie, err := r.Cookie(tocookie.Name) - if err != nil { - handleUnauthorized("error getting cookie: " + err.Error()) - return - } - - if cookie == nil { - handleUnauthorized("no auth cookie") - return - } - - oldCookie, err := tocookie.Parse(secret, cookie.Value) - if err != nil { - handleUnauthorized("cookie error: " + err.Error()) - return - } - - newCookieVal := tocookie.Refresh(oldCookie, secret) - http.SetCookie(w, &http.Cookie{Name: tocookie.Name, Value: newCookieVal}) - h(w, r, p) - } -} - -func wrapLogTime(h RegexHandlerFunc) RegexHandlerFunc { - return func(w http.ResponseWriter, r *http.Request, p ParamMap) { - start := time.Now() - defer func() { - now := time.Now() - log.Printf("%v %v served %v in %v\n", now, r.RemoteAddr, r.URL.Path, now.Sub(start)) - }() - h(w, r, p) - } -} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/config.go ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/config.go b/traffic_ops/traffic_ops_golang/config.go new file mode 100644 index 0000000..a0f3401 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/config.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/url" +) + +type Config struct { + HTTPPort string `json:"port"` + DBUser string `json:"db_user"` + DBPass string `json:"db_pass"` + DBServer string `json:"db_server"` + DBDB string `json:"db_name"` + DBSSL bool `json:"db_ssl"` + TOSecret string `json:"to_secret"` + TOURLStr string `json:"to_url"` + TOURL *url.URL `json:"-"` + NoAuth bool `json:"no_auth"` + CertPath string `json:"cert_path"` + KeyPath string `json:"key_path"` +} + +func LoadConfig(fileName string) (Config, error) { + if fileName == "" { + return Config{}, fmt.Errorf("no filename") + } + + configBytes, err := ioutil.ReadFile(fileName) + if err != nil { + return Config{}, err + } + + cfg := Config{} + if err := json.Unmarshal(configBytes, &cfg); err != nil { + return Config{}, err + } + + if cfg, err = ParseConfig(cfg); err != nil { + return Config{}, err + } + + return cfg, nil +} + +// ParseConfig validates required fields, and parses non-JSON types +func ParseConfig(cfg Config) (Config, error) { + if cfg.HTTPPort == "" { + return Config{}, fmt.Errorf("missing port") + } + if cfg.DBUser == "" { + return Config{}, fmt.Errorf("missing database user") + } + if cfg.DBPass == "" { + return Config{}, fmt.Errorf("missing database password") + } + if cfg.DBServer == "" { + return Config{}, fmt.Errorf("missing database server") + } + if cfg.DBDB == "" { + return Config{}, fmt.Errorf("missing database name") + } + if cfg.TOSecret == "" { + return Config{}, fmt.Errorf("missing secret") + } + if cfg.CertPath == "" { + return Config{}, fmt.Errorf("missing certificate path") + } + if cfg.KeyPath == "" { + return Config{}, fmt.Errorf("missing certificate key path") + } + + var err error + if cfg.TOURL, err = url.Parse(cfg.TOURLStr); err != nil { + return Config{}, fmt.Errorf("Invalid Traffic Ops URL '%v': err", cfg.TOURL, err) + } + + return cfg, nil +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/monitoring.go ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/monitoring.go b/traffic_ops/traffic_ops_golang/monitoring.go new file mode 100644 index 0000000..b353fb5 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/monitoring.go @@ -0,0 +1,398 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "github.com/lib/pq" + "net/http" + "strings" + "time" +) + +const CacheMonitorConfigFile = "rascal.properties" +const MonitorType = "RASCAL" +const RouterType = "CCR" +const MonitorProfilePrefix = "RASCAL" +const MonitorConfigFile = "rascal-config.txt" +const KilobitsPerMegabit = 1000 +const DeliveryServiceStatus = "REPORTED" + +type BasicServer struct { + Profile string `json:"profile"` + Status string `json:"status"` + IP string `json:"ip"` + IP6 string `json:"ip6"` + Port int `json:"port"` + Cachegroup string `json:"cachegroup"` + HostName string `json:"hostName"` + FQDN string `json:"fqdn"` +} + +type Monitor struct { + BasicServer +} + +type Cache struct { + BasicServer + InterfaceName string `json:"interfaceName"` + Type string `json:"type"` + HashID string `json:"hashId"` +} + +type Cachegroup struct { + Name string `json:"name"` + Coordinates Coordinates `json:"coordinates"` +} + +type Coordinates struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} + +type Profile struct { + Name string `json:"name"` + Type string `json:"type"` + Parameters map[string]string `json:"parameters"` +} + +type Monitoring struct { + TrafficServers []Cache `json:"trafficServers"` + TrafficMonitors []Monitor `json:"trafficMonitors"` + Cachegroups []Cachegroup `json:"cacheGroups"` + Profiles []Profile `json:"profiles"` + DeliveryServices []DeliveryService `json:"deliveryServices"` + Config map[string]string `json:"config"` +} + +type MonitoringResponse struct { + Response Monitoring `json:"response"` +} + +type Router struct { + Type string + Profile string +} + +type DeliveryService struct { + XMLID string `json:"xmlId"` + TotalTPSThreshold float64 `json:"totalTpsThreshold"` + Status string `json:"status"` + TotalKBPSThreshold float64 `json:"totalKbpsThreshold"` +} + +// TODO change to use the ParamMap, instead of parsing the URL +func monitoringHandler(db *sql.DB) RegexHandlerFunc { + return func(w http.ResponseWriter, r *http.Request, p ParamMap) { + handleErr := func(err error, status int) { + fmt.Printf("%v %v error %v\n", time.Now(), r.RemoteAddr, err) + w.WriteHeader(status) + fmt.Fprintf(w, http.StatusText(status)) + } + + cdnName := p["cdn"] + + resp, err := getMonitoringJson(cdnName, db) + if err != nil { + handleErr(err, http.StatusInternalServerError) + return + } + + respBts, err := json.Marshal(resp) + if err != nil { + handleErr(err, http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, "%s", respBts) + } +} + +func getServers(db *sql.DB, cdn string) ([]Monitor, []Cache, []Router, error) { + query := `SELECT +me.host_name as hostName, +CONCAT(me.host_name, '.', me.domain_name) as fqdn, +status.name as status, +cachegroup.name as cachegroup, +me.tcp_port as port, +me.ip_address as ip, +me.ip6_address as ip6, +profile.name as profile, +me.interface_name as interfaceName, +type.name as type, +me.xmpp_id as hashId +FROM server me +JOIN type type ON type.id = me.type +JOIN status status ON status.id = me.status +JOIN cachegroup cachegroup ON cachegroup.id = me.cachegroup +JOIN profile profile ON profile.id = me.profile +JOIN cdn cdn ON cdn.id = me.cdn_id +WHERE cdn.name = $1` + + rows, err := db.Query(query, cdn) + if err != nil { + return nil, nil, nil, err + } + defer rows.Close() + + monitors := []Monitor{} + caches := []Cache{} + routers := []Router{} + + for rows.Next() { + var hostName sql.NullString + var fqdn sql.NullString + var status sql.NullString + var cachegroup sql.NullString + var port sql.NullInt64 + var ip sql.NullString + var ip6 sql.NullString + var profile sql.NullString + var interfaceName sql.NullString + var ttype sql.NullString + var hashId sql.NullString + + if err := rows.Scan(&hostName, &fqdn, &status, &cachegroup, &port, &ip, &ip6, &profile, &interfaceName, &ttype, &hashId); err != nil { + return nil, nil, nil, err + } + + if ttype.String == MonitorType { + monitors = append(monitors, Monitor{ + BasicServer: BasicServer{ + Profile: profile.String, + Status: status.String, + IP: ip.String, + IP6: ip6.String, + Port: int(port.Int64), + Cachegroup: cachegroup.String, + HostName: hostName.String, + FQDN: fqdn.String, + }, + }) + } else if strings.HasPrefix(ttype.String, "EDGE") || strings.HasPrefix(ttype.String, "MID") { + caches = append(caches, Cache{ + BasicServer: BasicServer{ + Profile: profile.String, + Status: status.String, + IP: ip.String, + IP6: ip6.String, + Port: int(port.Int64), + Cachegroup: cachegroup.String, + HostName: hostName.String, + FQDN: fqdn.String, + }, + InterfaceName: interfaceName.String, + Type: ttype.String, + HashID: hashId.String, + }) + } else if ttype.String == RouterType { + routers = append(routers, Router{ + Type: ttype.String, + Profile: profile.String, + }) + } + } + return monitors, caches, routers, nil +} + +func getCachegroups(db *sql.DB, cdn string) ([]Cachegroup, error) { + query := ` +SELECT name, latitude, longitude +FROM cachegroup +WHERE id IN + (SELECT cachegroup FROM server WHERE server.cdn_id = + (SELECT id FROM cdn WHERE name = $1));` + + rows, err := db.Query(query, cdn) + if err != nil { + return nil, err + } + defer rows.Close() + + cachegroups := []Cachegroup{} + + for rows.Next() { + var name sql.NullString + var lat sql.NullFloat64 + var lon sql.NullFloat64 + if err := rows.Scan(&name, &lat, &lon); err != nil { + return nil, err + } + cachegroups = append(cachegroups, Cachegroup{ + Name: name.String, + Coordinates: Coordinates{ + Latitude: lat.Float64, + Longitude: lon.Float64, + }, + }) + } + return cachegroups, nil +} + +func getProfiles(db *sql.DB, caches []Cache, routers []Router) ([]Profile, error) { + cacheProfileTypes := map[string]string{} + profiles := map[string]Profile{} + profileNames := []string{} + for _, router := range routers { + profiles[router.Profile] = Profile{ + Name: router.Profile, + Type: router.Type, + } + } + + for _, cache := range caches { + if _, ok := cacheProfileTypes[cache.Profile]; !ok { + cacheProfileTypes[cache.Profile] = cache.Type + profiles[cache.Profile] = Profile{ + Name: cache.Profile, + Type: cache.Type, + } + profileNames = append(profileNames, cache.Profile) + } + } + + query := ` +SELECT p.name as profile, pr.name, pr.value +FROM parameter pr +JOIN profile p ON p.name = ANY($1) +JOIN profile_parameter pp ON pp.profile = p.id and pp.parameter = pr.id +WHERE pr.config_file = $2; +` + rows, err := db.Query(query, pq.Array(profileNames), CacheMonitorConfigFile) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var profileName sql.NullString + var name sql.NullString + var value sql.NullString + if err := rows.Scan(&profileName, &name, &value); err != nil { + return nil, err + } + if name.String == "" { + return nil, fmt.Errorf("null name") // TODO continue and warn? + } + profile := profiles[profileName.String] + if profile.Parameters == nil { + profile.Parameters = map[string]string{} + } + profile.Parameters[name.String] = value.String + profiles[profileName.String] = profile + + } + + profilesArr := []Profile{} // TODO make for efficiency? + for _, profile := range profiles { + profilesArr = append(profilesArr, profile) + } + return profilesArr, nil +} + +func getDeliveryServices(db *sql.DB, routers []Router) ([]DeliveryService, error) { + profileNames := []string{} + for _, router := range routers { + profileNames = append(profileNames, router.Profile) + } + + query := ` +SELECT ds.xml_id, ds.global_max_tps, ds.global_max_mbps +FROM deliveryservice ds +JOIN profile profile ON profile.id = ds.profile +WHERE profile.name = ANY($1) +AND ds.active = true +` + rows, err := db.Query(query, pq.Array(profileNames)) + if err != nil { + return nil, err + } + defer rows.Close() + + dses := []DeliveryService{} + + for rows.Next() { + var xmlid sql.NullString + var tps sql.NullFloat64 + var mbps sql.NullFloat64 + if err := rows.Scan(&xmlid, &tps, &mbps); err != nil { + return nil, err + } + dses = append(dses, DeliveryService{ + XMLID: xmlid.String, + TotalTPSThreshold: tps.Float64, + Status: DeliveryServiceStatus, + TotalKBPSThreshold: mbps.Float64 * KilobitsPerMegabit, + }) + } + return dses, nil +} + +func getConfig(db *sql.DB) (map[string]string, error) { + // TODO remove 'like' in query? Slow? + query := fmt.Sprintf(` +SELECT pr.name, pr.value +FROM parameter pr +JOIN profile p ON p.name LIKE '%s%%' +JOIN profile_parameter pp ON pp.profile = p.id and pp.parameter = pr.id +WHERE pr.config_file = '%s' +`, MonitorProfilePrefix, MonitorConfigFile) + + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + cfg := map[string]string{} + + for rows.Next() { + var name sql.NullString + var val sql.NullString + if err := rows.Scan(&name, &val); err != nil { + return nil, err + } + cfg[name.String] = val.String + } + return cfg, nil +} + +func getMonitoringJson(cdnName string, db *sql.DB) (*MonitoringResponse, error) { + monitors, caches, routers, err := getServers(db, cdnName) + if err != nil { + return nil, fmt.Errorf("error getting servers: %v", err) + } + + cachegroups, err := getCachegroups(db, cdnName) + if err != nil { + return nil, fmt.Errorf("error getting cachegroups: %v", err) + } + + profiles, err := getProfiles(db, caches, routers) + if err != nil { + return nil, fmt.Errorf("error getting profiles: %v", err) + } + + deliveryServices, err := getDeliveryServices(db, routers) + if err != nil { + return nil, fmt.Errorf("error getting deliveryservices: %v", err) + } + + config, err := getConfig(db) + if err != nil { + return nil, fmt.Errorf("error getting config: %v", err) + } + + resp := MonitoringResponse{ + Response: Monitoring{ + TrafficServers: caches, + TrafficMonitors: monitors, + Cachegroups: cachegroups, + Profiles: profiles, + DeliveryServices: deliveryServices, + Config: config, + }, + } + return &resp, nil +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/routes.go ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/routes.go b/traffic_ops/traffic_ops_golang/routes.go new file mode 100644 index 0000000..cc14450 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/routes.go @@ -0,0 +1,95 @@ +package main + +import ( + "crypto/tls" + "database/sql" + "net/http" + "net/http/httputil" + "regexp" + "strings" +) + +type ServerData struct { + Config + DB *sql.DB +} + +type ParamMap map[string]string + +type RegexHandlerFunc func(w http.ResponseWriter, r *http.Request, params ParamMap) + +func getMonitoringRoute(d ServerData) RegexHandlerFunc { + return wrapLogTime(wrapAuth(monitoringHandler(d.DB), d.NoAuth, d.TOSecret)) +} + +// getRootHandler returns the / handler for the service, which reverse-proxies the old Perl Traffic Ops +func getRootHandler(d ServerData) http.Handler { + // debug + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + rp := httputil.NewSingleHostReverseProxy(d.TOURL) + rp.Transport = tr + return rp +} + +// GetRoutes returns the map of regex routes, and a catchall route if no regex matches. +func GetRoutes(d ServerData) (map[string]RegexHandlerFunc, http.Handler) { + return map[string]RegexHandlerFunc{ + "api/1.2/cdns/{cdn}/configs/monitoring.json": getMonitoringRoute(d), + }, getRootHandler(d) +} + +type CompiledRoute struct { + Handler RegexHandlerFunc + Regex *regexp.Regexp + Params []string +} + +func CompileRoutes(routes *map[string]RegexHandlerFunc) map[string]CompiledRoute { + compiledRoutes := map[string]CompiledRoute{} + for route, handler := range *routes { + originalRoute := route + var params []string + for open := strings.Index(route, "{"); open > 0; open = strings.Index(route, "{") { + close := strings.Index(route, "}") + if close < 0 { + panic("malformed route") + } + param := route[open+1 : close] + + params = append(params, param) + route = route[:open] + `(.+)` + route[close+1:] + } + regex := regexp.MustCompile(route) + compiledRoutes[originalRoute] = CompiledRoute{Handler: handler, Regex: regex, Params: params} + } + return compiledRoutes +} + +func Handler(routes map[string]CompiledRoute, catchall http.Handler, w http.ResponseWriter, r *http.Request) { + requested := r.URL.Path[1:] + + for _, compiledRoute := range routes { + match := compiledRoute.Regex.FindStringSubmatch(requested) + if len(match) == 0 { + continue + } + + params := map[string]string{} + for i, v := range compiledRoute.Params { + params[v] = match[i+1] + } + compiledRoute.Handler(w, r, params) + return + } + catchall.ServeHTTP(w, r) +} + +func RegisterRoutes(d ServerData) { + routes, catchall := GetRoutes(d) + compiledRoutes := CompileRoutes(&routes) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + Handler(compiledRoutes, catchall, w, r) + }) +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/traffic_ops_golang.config ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/traffic_ops_golang.config b/traffic_ops/traffic_ops_golang/traffic_ops_golang.config new file mode 100644 index 0000000..19d2599 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/traffic_ops_golang.config @@ -0,0 +1,13 @@ +{ + "port": "443", + "db_user": "bill", + "db_pass": "thelizard", + "db_server": "db.trafficops.example.net", + "db_name": "trafficops", + "db_ssl": true, + "to_secret": "walrus", + "to_url": "https://trafficops.example.net:60443", + "no_auth": false, + "cert_path": "/opt/traffic_ops/cert", + "key_path": "/opt/traffic_ops/key" +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/traffic_ops_golang.go ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/traffic_ops_golang.go b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go new file mode 100644 index 0000000..48d3c4e --- /dev/null +++ b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go @@ -0,0 +1,56 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "database/sql" + "flag" + "fmt" + _ "github.com/lib/pq" + "net/http" +) + +const DefaultConfigPath = "/etc/goto/config.json" + +func main() { + configFileName := flag.String("cfg", "", "The config file path") + flag.Parse() + if *configFileName == "" { + *configFileName = DefaultConfigPath + } + + cfg, err := LoadConfig(*configFileName) + if err != nil { + fmt.Println("Error loading config '" + *configFileName + "': " + err.Error()) + return + } + + sslStr := "require" + if !cfg.DBSSL { + sslStr = "disable" + } + + db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", cfg.DBUser, cfg.DBPass, cfg.DBServer, cfg.DBDB, sslStr)) + if err != nil { + fmt.Printf("Error opening database: %v\n", err) + return + } + defer db.Close() + + RegisterRoutes(ServerData{DB: db, Config: cfg}) + fmt.Println("Listening on " + cfg.HTTPPort) + if err := http.ListenAndServeTLS(":"+cfg.HTTPPort, cfg.CertPath, cfg.KeyPath, nil); err != nil { + fmt.Printf("Error stopping server: %v\n", err) + return + } +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/04ed9f14/traffic_ops/traffic_ops_golang/wrappers.go ---------------------------------------------------------------------- diff --git a/traffic_ops/traffic_ops_golang/wrappers.go b/traffic_ops/traffic_ops_golang/wrappers.go new file mode 100644 index 0000000..daf5e34 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/wrappers.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "github.com/apache/incubator-trafficcontrol/traffic_ops/experimental/tocookie" + "log" // TODO change to traffic_monitor_golang/common/log + "net/http" + "time" +) + +func wrapAuth(h RegexHandlerFunc, noAuth bool, secret string) RegexHandlerFunc { + if noAuth { + return h + } + return func(w http.ResponseWriter, r *http.Request, p ParamMap) { + handleUnauthorized := func(reason string) { + log.Printf("%v %v %v sent 401 - %v\n", time.Now(), r.RemoteAddr, r.URL.Path, reason) + status := http.StatusUnauthorized + w.WriteHeader(status) + fmt.Fprintf(w, http.StatusText(status)) + } + + cookie, err := r.Cookie(tocookie.Name) + if err != nil { + handleUnauthorized("error getting cookie: " + err.Error()) + return + } + + if cookie == nil { + handleUnauthorized("no auth cookie") + return + } + + oldCookie, err := tocookie.Parse(secret, cookie.Value) + if err != nil { + handleUnauthorized("cookie error: " + err.Error()) + return + } + + newCookieVal := tocookie.Refresh(oldCookie, secret) + http.SetCookie(w, &http.Cookie{Name: tocookie.Name, Value: newCookieVal}) + h(w, r, p) + } +} + +func wrapLogTime(h RegexHandlerFunc) RegexHandlerFunc { + return func(w http.ResponseWriter, r *http.Request, p ParamMap) { + start := time.Now() + defer func() { + now := time.Now() + log.Printf("%v %v served %v in %v\n", now, r.RemoteAddr, r.URL.Path, now.Sub(start)) + }() + h(w, r, p) + } +}