This is an automated email from the ASF dual-hosted git repository.
rawlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 0ed2846 Make t3c jobs work with previous TO 4.0 (#6355)
0ed2846 is described below
commit 0ed284603bf7e5d6e5295ee74e7080e4f431d498
Author: Robert O Butts <[email protected]>
AuthorDate: Thu Nov 18 12:46:48 2021 -0700
Make t3c jobs work with previous TO 4.0 (#6355)
* Make t3c jobs work with previous TO 4.0
* Add t3c comments per PR Review.
---
cache-config/t3c-request/config/config.go | 6 +-
cache-config/t3cutil/toreq/clientfuncs.go | 4 +-
cache-config/t3cutil/toreq/conversions.go | 105 ++++++++++++++++++++++++
cache-config/testing/ort-tests/t3c-jobs_test.go | 96 ++++++++++++++++++++++
lib/go-atscfg/atscfg.go | 16 +++-
5 files changed, 223 insertions(+), 4 deletions(-)
diff --git a/cache-config/t3c-request/config/config.go
b/cache-config/t3c-request/config/config.go
index 68da602..6a1d9e8 100644
--- a/cache-config/t3c-request/config/config.go
+++ b/cache-config/t3c-request/config/config.go
@@ -170,9 +170,11 @@ func InitConfig() (Cfg, error) {
// load old config after initializing the loggers, because we want to
log how long it takes
oldCfg, err := LoadOldCfg(*oldCfgPtr)
if err != nil {
- return Cfg{}, errors.New("loading old config: " + err.Error())
+ log.Warnf("loading old config failed, old config will not be
used! Error: %v\n", err)
+ } else {
+ log.Infof("using old config for IMS requests")
+ cfg.OldCfg = oldCfg
}
- cfg.OldCfg = oldCfg
return cfg, nil
}
diff --git a/cache-config/t3cutil/toreq/clientfuncs.go
b/cache-config/t3cutil/toreq/clientfuncs.go
index 7785e48..cec1cbe 100644
--- a/cache-config/t3cutil/toreq/clientfuncs.go
+++ b/cache-config/t3cutil/toreq/clientfuncs.go
@@ -531,7 +531,9 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName
string) ([]atscfg.Invali
opts := *ReqOpts(reqHdr)
opts.QueryParameters.Set("maxRevalDurationDays", "") // only
get jobs with a start time within the window defined by the GLOBAL parameter
'maxRevalDurationDays'
opts.QueryParameters.Set("cdn", cdnName) // only
get jobs for delivery services in this server's CDN
- toJobs, toReqInf, err := cl.c.GetInvalidationJobs(opts)
+ // GetJobsCompat can be changed back to
'cl.c.GetInvalidationJobs' when backwards compatibility
+ // with Traffic Ops from previous ATS 'master' changesets is no
longer desired, presumably after the next major ATC release.
+ toJobs, toReqInf, err := cl.GetJobsCompat(opts)
if err != nil {
return errors.New("getting jobs from Traffic Ops '" +
torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error())
}
diff --git a/cache-config/t3cutil/toreq/conversions.go
b/cache-config/t3cutil/toreq/conversions.go
index 3cd3687..94d870c 100644
--- a/cache-config/t3cutil/toreq/conversions.go
+++ b/cache-config/t3cutil/toreq/conversions.go
@@ -20,8 +20,16 @@ package toreq
*/
import (
+ "errors"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
"github.com/apache/trafficcontrol/lib/go-atscfg"
"github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/traffic_ops/toclientlib"
+ toclient "github.com/apache/trafficcontrol/traffic_ops/v4-client"
)
func serversToLatest(svs tc.ServersV4Response) ([]atscfg.Server, error) {
@@ -40,3 +48,100 @@ func dsesToLatest(dses []tc.DeliveryServiceV40)
[]atscfg.DeliveryService {
func jobsToLatest(jobs []tc.InvalidationJobV4) []atscfg.InvalidationJob {
return atscfg.ToInvalidationJobs(jobs)
}
+
+// GetJobsCompat gets jobs from any Traffic Ops built from the ATC `master`
branch, and converts the different formats to the latest.
+// This makes t3c work with old or new Traffic Ops deployed from `master`,
+// though it doesn't make a version of t3c older than this work with a new TO,
+// which isn't logically possible from the client.
+func (cl *TOClient) GetJobsCompat(opts toclient.RequestOptions)
(tc.InvalidationJobsResponseV4, toclientlib.ReqInf, error) {
+ path := "/jobs"
+
+ objs := struct {
+ Response []InvalidationJobV4PlusLegacy `json:"response"`
+ tc.Alerts
+ }{}
+
+ if len(opts.QueryParameters) > 0 {
+ path += "?" + opts.QueryParameters.Encode()
+ }
+ reqInf, err := cl.c.TOClient.Req(http.MethodGet, path, nil,
opts.Header, &objs)
+ if err != nil {
+ return tc.InvalidationJobsResponseV4{}, reqInf,
errors.New("request: " + err.Error())
+ }
+
+ resp := tc.InvalidationJobsResponseV4{Alerts: objs.Alerts}
+ for _, job := range objs.Response {
+ newJob, err := InvalidationJobV4FromLegacy(job) //
(InvalidationJobV4, error) {
+ if err != nil {
+ return tc.InvalidationJobsResponseV4{}, reqInf,
errors.New("converting job from possible legacy format: " + err.Error())
+ }
+ resp.Response = append(resp.Response, newJob)
+ }
+ return resp, reqInf, nil
+}
+
+// InvalidationJobV4ForLegacy is a type alias to prevent MarshalJSON recursion.
+type InvalidationJobV4ForLegacy tc.InvalidationJobV4
+
+// InvalidationJobV4PlusLegacy has the data to deserialize both the latest and
older versions that Traffic Ops could return.
+type InvalidationJobV4PlusLegacy struct {
+ // StartTime overrides the StartTime in InvalidationJobV4 in order to
unmarshal any string format.
+ //
+ // A json.Unmarshal will place a 'startTime' value in this field,
+ // rather than the anonymous embedded InvalidationJobV4ForLegacy
(tc.InvalidationJobV4).
+ //
+ // InvalidationJobV4FromLegacy will then parse multiple time formats
that different Traffic Ops servers may return,
+ // and put the parsed time in tc.InvalidationJobV4.StartTime.
+ StartTime *string `json:"startTime"`
+ InvalidationJobV4ForLegacy
+ InvalidationJobV4Legacy
+}
+
+type InvalidationJobV4Legacy struct {
+ Keyword *string `json:"keyword"`
+ Parameters *string `json:"parameters"`
+}
+
+func InvalidationJobV4FromLegacy(job InvalidationJobV4PlusLegacy)
(tc.InvalidationJobV4, error) {
+ if job.StartTime != nil {
+ err := error(nil)
+ job.InvalidationJobV4ForLegacy.StartTime, err =
time.Parse(atscfg.JobV4TimeFormat, *job.StartTime)
+ if err != nil {
+ job.InvalidationJobV4ForLegacy.StartTime, err =
time.Parse(atscfg.JobLegacyTimeFormat, *job.StartTime)
+ if err != nil {
+ return tc.InvalidationJobV4{},
errors.New("malformed startTime")
+ }
+ }
+ }
+
+ if job.TTLHours == 0 && job.Parameters != nil {
+ params := *job.Parameters
+ params = strings.TrimSpace(params)
+ params = strings.ToLower(params)
+ params = strings.Replace(params, " ", "", -1)
+
+ paramPrefix := strings.ToLower(atscfg.JobLegacyParamPrefix)
+ paramSuffix := strings.ToLower(atscfg.JobLegacyParamSuffix)
+ if !strings.HasPrefix(params, paramPrefix) ||
!strings.HasSuffix(params, paramSuffix) {
+ return tc.InvalidationJobV4{}, errors.New("legacy
job.Parameters was not nil, but unexpected format '" + params + "'")
+ }
+
+ hoursStr := params[len(paramPrefix) :
len(params)-len(paramSuffix)]
+ hours, err := strconv.Atoi(hoursStr)
+ if err != nil {
+ return tc.InvalidationJobV4{}, errors.New("legacy
job.Parameters was not nil, but hours not an integer: '" + params + "'")
+ }
+ job.TTLHours = uint(hours)
+ }
+
+ if job.InvalidationType == "" && job.Parameters != nil {
+ job.InvalidationType = tc.REFRESH
+ }
+ if strings.HasSuffix(job.AssetURL, atscfg.JobLegacyRefetchSuffix) {
+ job.InvalidationType = tc.REFETCH
+ }
+ job.AssetURL = strings.TrimSuffix(job.AssetURL,
atscfg.JobLegacyRefreshSuffix)
+ job.AssetURL = strings.TrimSuffix(job.AssetURL,
atscfg.JobLegacyRefetchSuffix)
+
+ return tc.InvalidationJobV4(job.InvalidationJobV4ForLegacy), nil
+}
diff --git a/cache-config/testing/ort-tests/t3c-jobs_test.go
b/cache-config/testing/ort-tests/t3c-jobs_test.go
new file mode 100644
index 0000000..9ace24a
--- /dev/null
+++ b/cache-config/testing/ort-tests/t3c-jobs_test.go
@@ -0,0 +1,96 @@
+package orttest
+
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import (
+ "io/ioutil"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/apache/trafficcontrol/cache-config/testing/ort-tests/tcdata"
+ "github.com/apache/trafficcontrol/lib/go-tc"
+ "github.com/apache/trafficcontrol/lib/go-util"
+)
+
+func TestT3CJobs(t *testing.T) {
+ t.Logf("------------- Starting TestT3CJobs ---------------")
+ tcd.WithObjs(t, []tcdata.TCObj{
+ tcdata.CDNs, tcdata.Types, tcdata.Tenants, tcdata.Parameters,
+ tcdata.Profiles, tcdata.ProfileParameters, tcdata.Statuses,
+ tcdata.Divisions, tcdata.Regions, tcdata.PhysLocations,
+ tcdata.CacheGroups, tcdata.Servers, tcdata.Topologies,
+ tcdata.DeliveryServices}, func() {
+
+ now := tc.Time{Time: time.Now().Add(time.Minute)}
+ dsi := (interface{})("ds1")
+ ttli := (interface{})(72.0)
+ job := tc.InvalidationJobInput{
+ DeliveryService: &dsi,
+ Regex:
util.StrPtr(`/refetch-test\.png##REFETCH##`),
+ StartTime: &now,
+ TTL: &ttli,
+ }
+ if _, _, err := tcdata.TOSession.CreateInvalidationJob(job);
err != nil {
+ t.Fatalf("ERROR: create refetch job failed: %v\n", err)
+ }
+ job.Regex = util.StrPtr(`/refresh-test\.png`)
+ if _, _, err := tcdata.TOSession.CreateInvalidationJob(job);
err != nil {
+ t.Fatalf("ERROR: create refresh job failed: %v\n", err)
+ }
+
+ out, err := t3cUpdateWaitForParents("atlanta-edge-03",
"badass", util.StrPtr("false"))
+ if err != nil {
+ t.Fatalf("ERROR: t3c badass failed: %v\n", err)
+ }
+ t.Log("TestT3CJobs t3c output: '''" + out + "'''")
+
+ fileName := filepath.Join(test_config_dir,
"regex_revalidate.config")
+ revalFileBts, err := ioutil.ReadFile(fileName)
+ if err != nil {
+ t.Fatalf("reading %v: %v\n", fileName, err)
+ }
+ revalFile := string(revalFileBts)
+ lines := strings.Split(revalFile, "\n")
+ sawRefresh := false
+ sawRefetch := false
+ for _, line := range lines {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue // blank lines are ok
+ }
+ if strings.Contains(line, "refresh-test") {
+ sawRefresh = true
+ if !strings.HasSuffix(line, "STALE") {
+ t.Errorf("expected
regex_revalidate.config refresh-test line to contain 'STALE', actual '''%v'''",
revalFile)
+ }
+ }
+ if strings.Contains(line, "refetch-test") {
+ sawRefetch = true
+ if !strings.HasSuffix(line, "MISS") {
+ t.Errorf("expected
regex_revalidate.config refetch-test line to contain 'MISS', actual '''%v'''",
revalFile)
+ }
+ }
+ }
+ if !sawRefresh {
+ t.Errorf("expected regex_revalidate.config to contain
refresh-test, actual '''%v'''", revalFile)
+ }
+ if !sawRefetch {
+ t.Errorf("expected regex_revalidate.config to contain
refetch-test, actual '''%v'''", revalFile)
+ }
+ })
+ t.Logf("------------- End of TestT3CJobs ---------------")
+}
diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go
index ca3dc3b..c055d32 100644
--- a/lib/go-atscfg/atscfg.go
+++ b/lib/go-atscfg/atscfg.go
@@ -27,6 +27,7 @@ import (
"sort"
"strconv"
"strings"
+ "time"
"github.com/apache/trafficcontrol/lib/go-tc"
)
@@ -695,6 +696,14 @@ func JobsToInvalidationJobs(oldJobs []tc.Job)
([]InvalidationJob, error) {
return jobs, nil
}
+const JobV4TimeFormat = time.RFC3339Nano
+const JobLegacyTimeFormat = "2006-01-02 15:04:05-07"
+const JobLegacyRefetchSuffix = `##REFETCH##`
+const JobLegacyRefreshSuffix = `##REFRESH##`
+const JobLegacyParamPrefix = "TTL:"
+const JobLegacyParamSuffix = "h"
+const JobLegacyKeyword = "PURGE"
+
func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) {
startTime := tc.Time{}
if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err
!= nil {
@@ -704,13 +713,18 @@ func JobToInvalidationJob(jb tc.Job) (InvalidationJob,
error) {
if err != nil {
return InvalidationJob{}, errors.New("unmarshalling ttl: " +
err.Error())
}
+ invalType := tc.REFRESH
+ if strings.HasSuffix(jb.AssetURL, JobLegacyRefetchSuffix) {
+ invalType = tc.REFETCH
+ }
+
return InvalidationJob{
AssetURL: jb.AssetURL,
CreatedBy: jb.CreatedBy,
DeliveryService: jb.DeliveryService,
ID: uint64(jb.ID),
TTLHours: uint(ttl),
- InvalidationType: tc.REFRESH,
+ InvalidationType: invalType,
StartTime: startTime.Time,
}, nil
}
