This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new fe27e7e Make `t3c` request less unnecessary DSS and jobs data (#6186)
fe27e7e is described below
commit fe27e7e10c3496083b0b5b19425ff4cff9c61197
Author: Rawlin Peters <[email protected]>
AuthorDate: Fri Sep 10 13:19:22 2021 -0600
Make `t3c` request less unnecessary DSS and jobs data (#6186)
Use the new query parameters supported by Traffic Ops to reduce the
amount of unnecessary data requested from the /deliveryserviceserver and
/jobs APIs.
Closes: #5674
Closes: #6034
---
CHANGELOG.md | 3 ++
cache-config/t3cutil/getdatacfg.go | 52 +++++++++++-----------
cache-config/t3cutil/toreq/clientfuncs.go | 15 ++++---
cache-config/t3cutil/toreq/clienthlp.go | 4 +-
.../cdn-in-a-box/cache/traffic_ops_ort.crontab | 2 +-
5 files changed, 43 insertions(+), 33 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfaca06..2a37b5a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,9 @@ The format is based on [Keep a
Changelog](http://keepachangelog.com/en/1.0.0/).
- [#5674](https://github.com/apache/trafficcontrol/issues/5674) Added new
query parameters `cdn` and `maxRevalDurationDays` to the `GET /api/x/jobs`
Traffic Ops API to filter by CDN name and within the start_time window defined
by the `maxRevalDurationDays` GLOBAL profile parameter, respectively.
- [#6034](https://github.com/apache/trafficcontrol/issues/6034) Added new
query parameter `cdn` to the `GET /api/x/deliveryserviceserver` Traffic Ops API
to filter by CDN name
+### Changed
+- Updated `t3c` to request less unnecessary deliveryservice-server assignment
and invalidation jobs data via new query params supported by Traffic Ops
+
## unreleased
### Added
- [#4982](https://github.com/apache/trafficcontrol/issues/4982) Added the
ability to support queueing updates by server type and profile
diff --git a/cache-config/t3cutil/getdatacfg.go
b/cache-config/t3cutil/getdatacfg.go
index 9f02149..84e6fdf 100644
--- a/cache-config/t3cutil/getdatacfg.go
+++ b/cache-config/t3cutil/getdatacfg.go
@@ -332,7 +332,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy
bool, cacheHostName st
if oldCfg != nil {
reqHdr =
MakeReqHdr(oldCfg.MetaData.DeliveryServiceServers)
}
- dss, reqInf, err :=
toClient.GetDeliveryServiceServers(nil, nil, reqHdr)
+ dss, reqInf, err :=
toClient.GetDeliveryServiceServers(nil, nil, *server.CDNName, reqHdr)
if err != nil {
return errors.New("getting
delivery service servers: " + err.Error())
}
@@ -515,7 +515,30 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy
bool, cacheHostName st
}
return nil
}
- fs := []func() error{dsF, serverParamsF, cdnF, profileF}
+ jobsF := func() error {
+ defer func(start time.Time) { log.Infof("jobsF took
%v\n", time.Since(start)) }(time.Now())
+ {
+ reqHdr := (http.Header)(nil)
+ if oldCfg != nil {
+ reqHdr =
MakeReqHdr(oldCfg.MetaData.Jobs)
+ }
+ jobs, reqInf, err := toClient.GetJobs(reqHdr,
*server.CDNName)
+ if err != nil {
+ return errors.New("getting jobs: " +
err.Error())
+ }
+ if reqInf.StatusCode == http.StatusNotModified {
+ log.Infof("Getting config: %v not
modified, using old config", "Jobs")
+ toData.Jobs = oldCfg.Jobs
+ } else {
+ log.Infof("Getting config: %v is
modified, using new response", "Jobs")
+ toData.Jobs = jobs
+ }
+ toData.MetaData.Jobs =
MakeReqMetaData(reqInf.RespHeaders)
+ toIPs.Store(reqInf.RemoteAddr, nil)
+ }
+ return nil
+ }
+ fs := []func() error{dsF, serverParamsF, cdnF, profileF, jobsF}
if !revalOnly {
fs = append([]func() error{sslF}, fs...) // skip ssl
keys for reval only, which doesn't need them
}
@@ -545,29 +568,6 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy
bool, cacheHostName st
}
return nil
}
- jobsF := func() error {
- defer func(start time.Time) { log.Infof("jobsF took %v\n",
time.Since(start)) }(time.Now())
- {
- reqHdr := (http.Header)(nil)
- if oldCfg != nil {
- reqHdr = MakeReqHdr(oldCfg.MetaData.Jobs)
- }
- jobs, reqInf, err := toClient.GetJobs(reqHdr) // TODO
add cdn query param to jobs endpoint
- if err != nil {
- return errors.New("getting jobs: " +
err.Error())
- }
- if reqInf.StatusCode == http.StatusNotModified {
- log.Infof("Getting config: %v not modified,
using old config", "Jobs")
- toData.Jobs = oldCfg.Jobs
- } else {
- log.Infof("Getting config: %v is modified,
using new response", "Jobs")
- toData.Jobs = jobs
- }
- toData.MetaData.Jobs =
MakeReqMetaData(reqInf.RespHeaders)
- toIPs.Store(reqInf.RemoteAddr, nil)
- }
- return nil
- }
capsF := func() error {
defer func(start time.Time) { log.Infof("capsF took %v\n",
time.Since(start)) }(time.Now())
{
@@ -739,7 +739,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy
bool, cacheHostName st
return nil
}
- fs := []func() error{serversF, cgF, jobsF}
+ fs := []func() error{serversF, cgF}
if !revalOnly {
// skip data not needed for reval, if we're reval-only
fs = append([]func() error{dsrF, cacheKeyConfigParamsF,
remapConfigParamsF, parentConfigParamsF, capsF, dsCapsF, topologiesF}, fs...)
diff --git a/cache-config/t3cutil/toreq/clientfuncs.go
b/cache-config/t3cutil/toreq/clientfuncs.go
index decc7b7..a28476a 100644
--- a/cache-config/t3cutil/toreq/clientfuncs.go
+++ b/cache-config/t3cutil/toreq/clientfuncs.go
@@ -177,7 +177,7 @@ func (cl *TOClient) GetCacheGroups(reqHdr http.Header)
([]tc.CacheGroupNullable,
// If your use case is more efficient to only get the needed objects, for
example if you're frequently requesting one file, set this false to get and
cache the specific needed delivery services and servers.
const DeliveryServiceServersAlwaysGetAll = true
-func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int,
reqHdr http.Header) ([]tc.DeliveryServiceServer, toclientlib.ReqInf, error) {
+func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int,
cdnName string, reqHdr http.Header) ([]tc.DeliveryServiceServer,
toclientlib.ReqInf, error) {
if cl.c == nil {
return cl.old.GetDeliveryServiceServers(dsIDs, serverIDs)
}
@@ -200,7 +200,7 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int,
serverIDs []int, reqH
}
dsServers := []tc.DeliveryServiceServer{}
- err := torequtil.GetRetry(cl.NumRetries,
"deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr, &dsServers, func(obj
interface{}) error {
+ err := torequtil.GetRetry(cl.NumRetries,
"deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr+"_cdn_"+cdnName,
&dsServers, func(obj interface{}) error {
dsIDStrs := []string{}
for _, dsID := range dsIDsToFetch {
@@ -214,6 +214,8 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int,
serverIDs []int, reqH
queryParams := url.Values{}
queryParams.Set("limit", "999999") // TODO add "no limit" param
to DSS endpoint
+ queryParams.Set("cdn", cdnName)
+ queryParams.Set("orderby", "") // prevent unnecessary sorting
of the response
if len(dsIDsToFetch) > 0 {
queryParams.Set("deliveryserviceids",
strings.Join(dsIDStrs, ","))
}
@@ -513,7 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr
http.Header) ([]tc.Delivery
return regexes, reqInf, nil
}
-func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJob,
toclientlib.ReqInf, error) {
+func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string)
([]tc.InvalidationJob, toclientlib.ReqInf, error) {
if cl.c == nil {
oldJobs, inf, err := cl.old.GetJobs()
jobs, err := atscfg.JobsToInvalidationJobs(oldJobs)
@@ -525,8 +527,11 @@ func (cl *TOClient) GetJobs(reqHdr http.Header)
([]tc.InvalidationJob, toclientl
jobs := []tc.InvalidationJob{}
reqInf := toclientlib.ReqInf{}
- err := torequtil.GetRetry(cl.NumRetries, "jobs", &jobs, func(obj
interface{}) error {
- toJobs, toReqInf, err :=
cl.c.GetInvalidationJobs(*ReqOpts(reqHdr))
+ err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs,
func(obj interface{}) error {
+ opts := *ReqOpts(reqHdr)
+ opts.QueryParameters.Set("maxRevalDurationDays", "") // only
get jobs with a start time within the window defined by the GLOBAL parameter
'maxRevalDurationDays'
+ opts.QueryParameters.Set("cdn", cdnName) // only
get jobs for delivery services in this server's CDN
+ toJobs, toReqInf, err := cl.c.GetInvalidationJobs(opts)
if err != nil {
return errors.New("getting jobs from Traffic Ops '" +
torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error())
}
diff --git a/cache-config/t3cutil/toreq/clienthlp.go
b/cache-config/t3cutil/toreq/clienthlp.go
index e356d1e..035932f 100644
--- a/cache-config/t3cutil/toreq/clienthlp.go
+++ b/cache-config/t3cutil/toreq/clienthlp.go
@@ -119,5 +119,7 @@ func GetDeliveryServiceURLSigKeys(toClient
*toclient.Session, dsName string, opt
// ReqOpts takes an http.Header and returns a
traffic_ops/v4-client.RequestOptions with that header.
// This is a helper function, for brevity.
func ReqOpts(hdr http.Header) *toclient.RequestOptions {
- return &toclient.RequestOptions{Header: hdr}
+ opts := toclient.NewRequestOptions()
+ opts.Header = hdr
+ return &opts
}
diff --git a/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab
b/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab
index 9caebac..14964b6 100644
--- a/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab
+++ b/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-*/1 * * * * t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL
--traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv
2>&1 >> /var/log/ort.log
+*/1 * * * * root t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL
--traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv
--cache-host-name=$(hostname -s) >> /var/log/ort.log 2>&1
