elsloo closed pull request #2914: Add TO Go /api/1.4/cdns/dnssec/refresh URL: https://github.com/apache/trafficcontrol/pull/2914
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbde00db..44170450c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed - Issue 2821: Fixed "Traffic Router may choose wrong certificate when SNI names overlap" - +- traffic_ops/app/bin/checks/ToDnssecRefresh.pl now requires "user" and "pass" parameters of an operations-level user! Update your scripts accordingly! This was necessary to move to an API endpoint with proper authentication, which may be safely exposed. + ## [3.0.0] - 2018-10-30 ### Added - Removed MySQL-to-Postgres migration tools. This tool is supported for 1.x to 2.x upgrades only and should not be used with 3.x. diff --git a/traffic_ops/app/bin/checks/ToDnssecRefresh.pl b/traffic_ops/app/bin/checks/ToDnssecRefresh.pl index fad097d5c..d3ac64581 100755 --- a/traffic_ops/app/bin/checks/ToDnssecRefresh.pl +++ b/traffic_ops/app/bin/checks/ToDnssecRefresh.pl @@ -62,11 +62,37 @@ TRACE Dumper($jconf); my $b_url = $jconf->{base_url}; +my $to_user= $jconf->{user}; +my $to_pass= $jconf->{pass}; + +if ( !defined($to_user) || $to_user eq '' ) { + ERROR "Config missing \"user\" key, this script now requires \"user\" and \"pass\" keys, as the endpoint used by this script now requires admin-level authentication."; + exit(1); +} + +if ( !defined($to_pass) || $to_pass eq '' ) { + ERROR "Config missing \"pass\" key, this script now requires \"user\" and \"pass\" keys, as the endpoint used by this script now requires admin-level authentication."; + exit(1); +} + my $ua = LWP::UserAgent->new; $ua->timeout(30); $ua->ssl_opts(verify_hostname => 0); +$ua->cookie_jar( {} ); + +my $login_url = "$b_url/api/1.2/user/login"; +TRACE "posting $login_url"; +my $req = HTTP::Request->new( 'POST', $login_url ); +$req->header( 'Content-Type' => 'application/json' ); + +$req->content( "{\"u\": \"$to_user\",\"p\": \"$to_pass\"}" ); +my $login_response = $ua->request($req); +if ( ! $login_response->is_success ) { + ERROR "Error trying to update keys, login failed, response was " . $login_response->status_line; + exit(1); +} -my $url = "$b_url/internal/api/1.2/cdns/dnsseckeys/refresh.json"; +my $url = "$b_url/api/1.4/cdns/dnsseckeys/refresh"; TRACE "getting $url"; my $response = $ua->get($url); if ( $response->is_success ) { diff --git a/traffic_ops/traffic_ops_golang/api/api.go b/traffic_ops/traffic_ops_golang/api/api.go index d46df5f8c..75254ee0a 100644 --- a/traffic_ops/traffic_ops_golang/api/api.go +++ b/traffic_ops/traffic_ops_golang/api/api.go @@ -291,11 +291,11 @@ type APIInfo struct { // } // func NewInfo(r *http.Request, requiredParams []string, intParamNames []string) (*APIInfo, error, error, int) { - db, err := getDB(r.Context()) + db, err := GetDB(r.Context()) if err != nil { return &APIInfo{Tx: &sqlx.Tx{}}, errors.New("getting db: " + err.Error()), nil, http.StatusInternalServerError } - cfg, err := getConfig(r.Context()) + cfg, err := GetConfig(r.Context()) if err != nil { return &APIInfo{Tx: &sqlx.Tx{}}, errors.New("getting config: " + err.Error()), nil, http.StatusInternalServerError } @@ -336,7 +336,8 @@ func (inf *APIInfo) Close() { } } -func getDB(ctx context.Context) (*sqlx.DB, error) { +// GetDB returns the database from the context. This should very rarely be needed, rather `NewInfo` should always be used to get a transaction, except in extenuating circumstances. +func GetDB(ctx context.Context) (*sqlx.DB, error) { val := ctx.Value(DBContextKey) if val != nil { switch v := val.(type) { @@ -349,7 +350,7 @@ func getDB(ctx context.Context) (*sqlx.DB, error) { return nil, errors.New("No db found in Context") } -func getConfig(ctx context.Context) (*config.Config, error) { +func GetConfig(ctx context.Context) (*config.Config, error) { val := ctx.Value(ConfigContextKey) if val != nil { switch v := val.(type) { @@ -362,10 +363,6 @@ func getConfig(ctx context.Context) (*config.Config, error) { return nil, errors.New("No config found in Context") } -func GetConfig(ctx context.Context) (*config.Config, error) { - return getConfig(ctx) -} - func getReqID(ctx context.Context) (uint64, error) { val := ctx.Value(ReqIDContextKey) if val != nil { diff --git a/traffic_ops/traffic_ops_golang/cdn/dnssecrefresh.go b/traffic_ops/traffic_ops_golang/cdn/dnssecrefresh.go new file mode 100644 index 000000000..f97fca798 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/cdn/dnssecrefresh.go @@ -0,0 +1,405 @@ +package cdn + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + // "context" + "database/sql" + "errors" + "net/http" + "strconv" + "sync/atomic" + "time" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/lib/go-util" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/config" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/riaksvc" + + "github.com/lib/pq" +) + +func RefreshDNSSECKeys(w http.ResponseWriter, r *http.Request) { + if setInDNSSECKeyRefresh() { + db, err := api.GetDB(r.Context()) + noTx := (*sql.Tx)(nil) // make a variable instead of passing nil directly, to reduce copy-paste errors + if err != nil { + api.HandleErr(w, r, noTx, http.StatusInternalServerError, nil, errors.New("RefresHDNSSECKeys getting db from context: "+err.Error())) + return + } + cfg, err := api.GetConfig(r.Context()) + if err != nil { + api.HandleErr(w, r, noTx, http.StatusInternalServerError, nil, errors.New("RefresHDNSSECKeys getting config from context: "+err.Error())) + return + } + + tx, err := db.Begin() + if err != nil { + api.HandleErr(w, r, noTx, http.StatusInternalServerError, nil, errors.New("RefresHDNSSECKeys beginning tx: "+err.Error())) + } + go doDNSSECKeyRefresh(tx, cfg) // doDNSSECKeyRefresh takes ownership of tx and MUST close it. + } else { + log.Infoln("RefreshDNSSECKeys called, while server was concurrently executing a refresh, doing nothing") + } + + api.WriteResp(w, r, "Checking DNSSEC keys for refresh in the background") +} + +const DNSSECKeyRefreshDefaultTTL = time.Duration(60) * time.Second +const DNSSECKeyRefreshDefaultGenerationMultiplier = uint64(10) +const DNSSECKeyRefreshDefaultEffectiveMultiplier = uint64(10) +const DNSSECKeyRefreshDefaultKSKExpiration = time.Duration(365) * time.Hour * 24 +const DNSSECKeyRefreshDefaultZSKExpiration = time.Duration(30) * time.Hour * 24 + +// doDNSSECKeyRefresh refreshes the CDN's DNSSEC keys, as necessary. +// This takes ownership of tx, and MUST call `tx.Close()`. +// This SHOULD only be called if setInDNSSECKeyRefresh() returned true, in which case this MUST call unsetInDNSSECKeyRefresh() before returning. +func doDNSSECKeyRefresh(tx *sql.Tx, cfg *config.Config) { + doCommit := true + defer func() { + if doCommit { + tx.Commit() + } else { + tx.Rollback() + } + }() + defer unsetInDNSSECKeyRefresh() + + updatedAny := false + + cdnDNSSECKeyParams, err := getDNSSECKeyRefreshParams(tx) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: getting cdn parameters: " + err.Error()) + doCommit = false + return + } + cdns := []string{} + for _, inf := range cdnDNSSECKeyParams { + if inf.DNSSECEnabled { + cdns = append(cdns, string(inf.CDNName)) + } + } + // TODO change to return a slice, map is slow and unnecessary + dsInfo, err := getDNSSECKeyRefreshDSInfo(tx, cdns) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: getting ds info: " + err.Error()) + doCommit = false + return + } + dses := []string{} + for ds, _ := range dsInfo { + dses = append(dses, string(ds)) + } + + dsMatchlists, err := deliveryservice.GetDeliveryServicesMatchLists(dses, tx) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: getting ds matchlists: " + err.Error()) + doCommit = false + return + } + exampleURLs := map[tc.DeliveryServiceName][]string{} + for ds, inf := range dsInfo { + exampleURLs[ds] = deliveryservice.MakeExampleURLs(inf.Protocol, inf.Type, inf.RoutingName, dsMatchlists[string(ds)], inf.CDNDomain) + } + + for _, cdnInf := range cdnDNSSECKeyParams { + keys, ok, err := riaksvc.GetDNSSECKeys(string(cdnInf.CDNName), tx, cfg.RiakAuthOptions) // TODO get all in a map beforehand + if err != nil { + log.Warnln("refreshing DNSSEC Keys: getting cdn '" + string(cdnInf.CDNName) + "' keys from Riak, skipping: " + err.Error()) + continue + } + if !ok { + log.Warnln("refreshing DNSSEC Keys: cdn '" + string(cdnInf.CDNName) + "' has no keys in Riak, skipping") + continue + } + + ttl := DNSSECKeyRefreshDefaultTTL + if cdnInf.TLDTTLsDNSKEY != nil { + ttl = time.Duration(*cdnInf.TLDTTLsDNSKEY) * time.Second + } + + genMultiplier := DNSSECKeyRefreshDefaultGenerationMultiplier + if cdnInf.DNSKEYGenerationMultiplier != nil { + genMultiplier = *cdnInf.DNSKEYGenerationMultiplier + } + + effectiveMultiplier := DNSSECKeyRefreshDefaultEffectiveMultiplier + if cdnInf.DNSKEYEffectiveMultiplier != nil { + effectiveMultiplier = *cdnInf.DNSKEYEffectiveMultiplier + } + + nowPlusTTL := time.Now().Add(ttl * time.Duration(genMultiplier)) // "key_expiration" in the Perl this was transliterated from + + defaultKSKExpiration := DNSSECKeyRefreshDefaultKSKExpiration + for _, key := range keys[string(cdnInf.CDNName)].KSK { + if key.Status != tc.DNSSECKeyStatusNew { + continue + } + defaultKSKExpiration = time.Unix(key.ExpirationDateUnix, 0).Sub(time.Unix(key.InceptionDateUnix, 0)) + break + } + + defaultZSKExpiration := DNSSECKeyRefreshDefaultZSKExpiration + for _, key := range keys[string(cdnInf.CDNName)].ZSK { + if key.Status != tc.DNSSECKeyStatusNew { + continue + } + expiration := time.Unix(key.ExpirationDateUnix, 0) + inception := time.Unix(key.InceptionDateUnix, 0) + defaultZSKExpiration = expiration.Sub(inception) + + if expiration.After(nowPlusTTL) { + continue + } + log.Infoln("The ZSK keys for '" + string(cdnInf.CDNName) + "' are expired!") + effectiveDate := expiration.Add(ttl * time.Duration(effectiveMultiplier) * -1) // -1 to subtract + isKSK := false + newKeys, err := regenExpiredKeys(isKSK, keys[string(cdnInf.CDNName)], effectiveDate, false, false) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: regenerating expired ZSK keys: " + err.Error()) + } else { + keys[string(cdnInf.CDNName)] = newKeys + updatedAny = true + } + } + + for _, ds := range dsInfo { + if ds.CDNName != cdnInf.CDNName { + continue + } + if t := ds.Type; !t.IsHTTP() && !t.IsSteering() && !t.IsDNS() { + continue + } + + dsKeys, dsKeysExist := keys[string(ds.DSName)] + if !dsKeysExist { + log.Infoln("Keys do not exist for ds '" + string(ds.DSName) + "'") + + cdnKeys, ok := keys[string(ds.CDNName)] + if !ok { + log.Errorln("refreshing DNSSEC Keys: cdn has no keys, cannot create ds keys: " + err.Error()) + continue + } + + overrideTTL := false + dsKeys, err := deliveryservice.CreateDNSSECKeys(tx, cfg, string(ds.DSName), exampleURLs[ds.DSName], cdnKeys, defaultKSKExpiration, defaultZSKExpiration, ttl, overrideTTL) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: creating missing ds keys: " + err.Error()) + } + keys[string(ds.DSName)] = dsKeys + updatedAny = true + continue + } + + for _, key := range dsKeys.KSK { + if key.Status != tc.DNSSECKeyStatusNew { + continue + } + expiration := time.Unix(key.ExpirationDateUnix, 0) + if expiration.After(nowPlusTTL) { + continue + } + log.Infoln("The KSK keys for '" + ds.DSName + "' are expired!") + effectiveDate := expiration.Add(ttl * time.Duration(effectiveMultiplier) * -1) // -1 to subtract + isKSK := true + newKeys, err := regenExpiredKeys(isKSK, dsKeys, effectiveDate, false, false) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: regenerating expired KSK keys for ds '" + string(ds.DSName) + "': " + err.Error()) + } else { + keys[string(ds.DSName)] = newKeys + updatedAny = true + } + } + + for _, key := range dsKeys.ZSK { + if key.Status != tc.DNSSECKeyStatusNew { + continue + } + expiration := time.Unix(key.ExpirationDateUnix, 0) + if expiration.After(nowPlusTTL) { + continue + } + log.Infoln("The ZSK keys for '" + ds.DSName + "' are expired!") + effectiveDate := expiration.Add(ttl * time.Duration(effectiveMultiplier) * -1) // -1 to subtract + isKSK := false + newKeys, err := regenExpiredKeys(isKSK, dsKeys, effectiveDate, false, false) + if err != nil { + log.Errorln("refreshing DNSSEC Keys: regenerating expired ZSK keys for ds '" + string(ds.DSName) + "': " + err.Error()) + } else { + keys[string(ds.DSName)] = newKeys + updatedAny = true + } + } + } + if updatedAny { + if err := riaksvc.PutDNSSECKeys(keys, string(cdnInf.CDNName), tx, cfg.RiakAuthOptions); err != nil { + log.Errorln("refreshing DNSSEC Keys: putting keys into Riak for cdn '" + string(cdnInf.CDNName) + "': " + err.Error()) + } + } + } + log.Infoln("Done refreshing DNSSEC keys") +} + +type DNSSECKeyRefreshCDNInfo struct { + CDNName tc.CDNName + DNSSECEnabled bool + TLDTTLsDNSKEY *uint64 + DNSKEYEffectiveMultiplier *uint64 + DNSKEYGenerationMultiplier *uint64 +} + +// getDNSSECKeyRefreshParams returns returns the CDN's profile's tld.ttls.DNSKEY, DNSKEY.effective.multiplier, and DNSKEY.generation.multiplier parameters. If either parameter doesn't exist, nil is returned. +// If a CDN exists, but has no parameters, it is returned as a key in the map with a nil value. +func getDNSSECKeyRefreshParams(tx *sql.Tx) (map[tc.CDNName]DNSSECKeyRefreshCDNInfo, error) { + qry := ` +WITH cdn_profile_ids AS ( + SELECT + DISTINCT(c.name) as cdn_name, + c.dnssec_enabled as cdn_dnssec_enabled, + MAX(p.id) as profile_id -- We only want 1 profile, so get the probably-newest if there's more than one. + FROM + cdn c + LEFT JOIN profile p ON c.id = p.cdn AND (p.name like 'CCR%' OR p.name like 'TR%') + GROUP BY c.name, c.dnssec_enabled +) +SELECT + DISTINCT(pi.cdn_name), + pi.cdn_dnssec_enabled, + MAX(pa.name) as parameter_name, + MAX(pa.value) as parameter_value +FROM + cdn_profile_ids pi + LEFT JOIN profile pr ON pi.profile_id = pr.id + LEFT JOIN profile_parameter pp ON pr.id = pp.profile + LEFT JOIN parameter pa ON pp.parameter = pa.id AND ( + pa.name = 'tld.ttls.DNSKEY' + OR pa.name = 'DNSKEY.effective.multiplier' + OR pa.name = 'DNSKEY.generation.multiplier' + ) +GROUP BY pi.cdn_name, pi.cdn_dnssec_enabled +` + rows, err := tx.Query(qry) + if err != nil { + return nil, errors.New("getting cdn dnssec key refresh parameters: " + err.Error()) + } + defer rows.Close() + + params := map[tc.CDNName]DNSSECKeyRefreshCDNInfo{} + for rows.Next() { + cdnName := tc.CDNName("") + dnssecEnabled := false + name := util.StrPtr("") + valStr := util.StrPtr("") + if err := rows.Scan(&cdnName, &dnssecEnabled, &name, &valStr); err != nil { + return nil, errors.New("scanning cdn dnssec key refresh parameters: " + err.Error()) + } + + inf := params[cdnName] + inf.CDNName = cdnName + inf.DNSSECEnabled = dnssecEnabled + + if name == nil || valStr == nil { + // no DNSKEY parameters, but the CDN still exists. + params[cdnName] = inf + continue + } + + val, err := strconv.ParseUint(*valStr, 10, 64) + if err != nil { + log.Warnln("getting CDN dnssec refresh parameters: parameter '" + *name + "' value '" + *valStr + "' is not a number, skipping") + params[cdnName] = inf + continue + } + + switch *name { + case "tld.ttls.DNSKEY": + inf.TLDTTLsDNSKEY = &val + case "DNSKEY.effective.multiplier": + inf.DNSKEYEffectiveMultiplier = &val + case "DNSKEY.generation.multiplier": + inf.DNSKEYGenerationMultiplier = &val + default: + log.Warnln("getDNSSECKeyRefreshParams got unknown parameter '" + *name + "', skipping") + continue + } + params[cdnName] = inf + } + return params, nil +} + +type DNSSECKeyRefreshDSInfo struct { + DSName tc.DeliveryServiceName + Type tc.DSType + Protocol *int + CDNName tc.CDNName + CDNDomain string + RoutingName string +} + +func getDNSSECKeyRefreshDSInfo(tx *sql.Tx, cdns []string) (map[tc.DeliveryServiceName]DNSSECKeyRefreshDSInfo, error) { + qry := ` +SELECT + ds.xml_id, + tp.name as type, + ds.protocol, + c.name as cdn_name, + c.domain_name as cdn_domain, + ds.routing_name +FROM + deliveryservice ds + JOIN type tp ON tp.id = ds.type + JOIN cdn c ON c.id = ds.cdn_id +WHERE + c.name = ANY($1) +` + rows, err := tx.Query(qry, pq.Array(cdns)) + if err != nil { + return nil, errors.New("getting cdn dnssec key refresh ds info: " + err.Error()) + } + defer rows.Close() + + dsInf := map[tc.DeliveryServiceName]DNSSECKeyRefreshDSInfo{} + for rows.Next() { + i := DNSSECKeyRefreshDSInfo{} + if err := rows.Scan(&i.DSName, &i.Type, &i.Protocol, &i.CDNName, &i.CDNDomain, &i.RoutingName); err != nil { + return nil, errors.New("scanning cdn dnssec key refresh ds info: " + err.Error()) + } + dsInf[i.DSName] = i + } + return dsInf, nil +} + +// inDNSSECKeyRefresh is whether the server is currently processing a refresh in the background. +// This is used to only perform 1 refresh at a time. +// This MUST NOT be changed outside of atomic operations. +// This MUST NOT be changed to a boolean, or set without atomics. Atomic semantics involve more than just setting a memory location. +var inDNSSECKeyRefresh = uint64(0) + +// setInDNSSECKeyRefresh attempts to set whether the server is currently executing a DNSSEC key refresh operation. +// Returns false if a refresh operation is already executing. +// If this returns true, the caller MUST call unsetInDNSSECKeyRefresh(). +func setInDNSSECKeyRefresh() bool { return atomic.CompareAndSwapUint64(&inDNSSECKeyRefresh, 0, 1) } + +// unsetInDNSSECKeyRefresh sets the flag indicating that the server is currently executing a DNSSEC key refresh operation to false. +// This MUST NOT be called, unless setInDNSSECKeyRefresh() was previously called and returned true. +func unsetInDNSSECKeyRefresh() { atomic.StoreUint64(&inDNSSECKeyRefresh, 0) } diff --git a/traffic_ops/traffic_ops_golang/deliveryservice/dnssec.go b/traffic_ops/traffic_ops_golang/deliveryservice/dnssec.go index c1ea6bd33..269c8af37 100644 --- a/traffic_ops/traffic_ops_golang/deliveryservice/dnssec.go +++ b/traffic_ops/traffic_ops_golang/deliveryservice/dnssec.go @@ -111,7 +111,7 @@ func GetDNSSECKeys(keyType string, dsName string, ttl time.Duration, inception t func genKeys(dsName string, ksk bool, ttl time.Duration, tld bool) (string, string, *tc.DNSSECKeyDSRecord, error) { bits := 1024 flags := 256 - algorithm := dns.RSASHA1 // 5 - http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml + algorithm := dns.RSASHA256 // 8 - http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml protocol := 3 if ksk { diff --git a/traffic_ops/traffic_ops_golang/routes.go b/traffic_ops/traffic_ops_golang/routes.go index f7d4715ae..9b4104735 100644 --- a/traffic_ops/traffic_ops_golang/routes.go +++ b/traffic_ops/traffic_ops_golang/routes.go @@ -137,6 +137,8 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) { {1.1, http.MethodPost, `cdns/dnsseckeys/generate(\.json)?$`, cdn.CreateDNSSECKeys, auth.PrivLevelAdmin, Authenticated, nil}, {1.1, http.MethodGet, `cdns/name/{name}/dnsseckeys/delete/?(\.json)?$`, cdn.DeleteDNSSECKeys, auth.PrivLevelAdmin, Authenticated, nil}, + {1.4, http.MethodGet, `cdns/dnsseckeys/refresh/?(\.json)?$`, cdn.RefreshDNSSECKeys, auth.PrivLevelOperations, Authenticated, nil}, + //CDN: Monitoring: Traffic Monitor {1.1, http.MethodGet, `cdns/{cdn}/configs/monitoring(\.json)?$`, monitoring.Get, auth.PrivLevelReadOnly, Authenticated, nil}, ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
