This is an automated email from the ASF dual-hosted git repository.
friede pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 6f527a10b0 Configure uncacheable delivery services in Varnish (#7771)
6f527a10b0 is described below
commit 6f527a10b09d9b043c881d0998a40b8315ce83f6
Author: AbdelrahmanElawady
<[email protected]>
AuthorDate: Tue Sep 12 19:45:12 2023 +0300
Configure uncacheable delivery services in Varnish (#7771)
---
lib/go-atscfg/cachedotconfig.go | 121 ++++++++++++++++++++---------------
lib/varnishcfg/cache_control.go | 57 +++++++++++++++++
lib/varnishcfg/cache_control_test.go | 90 ++++++++++++++++++++++++++
lib/varnishcfg/vclbuilder.go | 10 +++
4 files changed, 227 insertions(+), 51 deletions(-)
diff --git a/lib/go-atscfg/cachedotconfig.go b/lib/go-atscfg/cachedotconfig.go
index 949bc4c27c..9418d462e3 100644
--- a/lib/go-atscfg/cachedotconfig.go
+++ b/lib/go-atscfg/cachedotconfig.go
@@ -20,6 +20,7 @@ package atscfg
*/
import (
+ "fmt"
"sort"
"strings"
@@ -71,6 +72,62 @@ func makeCacheDotConfigEdge(
return Cfg{}, makeErr(warnings, "server missing profiles")
}
+ profileDSes, dsWarnings := GetProfileDSes(server, servers,
deliveryServices, deliveryServiceServers)
+ warnings = append(warnings, dsWarnings...)
+
+ lines := map[string]struct{}{} // use a "set" for lines, to avoid
duplicates, since we're looking up by profile
+ for _, ds := range profileDSes {
+ if ds.Type != tc.DSTypeHTTPNoCache {
+ continue
+ }
+ if ds.OriginFQDN == "" {
+ warnings = append(warnings,
fmt.Sprintf("profileCacheDotConfig ds %s has no origin fqdn, skipping!",
ds.Name))
+ continue
+ }
+ originFQDN, originPort := GetHostPortFromURI(ds.OriginFQDN)
+ if originPort != "" {
+ l := "dest_domain=" + originFQDN + " port=" +
originPort + " scheme=http action=never-cache\n"
+ lines[l] = struct{}{}
+ } else {
+ l := "dest_domain=" + originFQDN + " scheme=http
action=never-cache\n"
+ lines[l] = struct{}{}
+ }
+ }
+
+ linesArr := []string{}
+ for line, _ := range lines {
+ linesArr = append(linesArr, line)
+ }
+ sort.Strings(linesArr)
+ text := strings.Join(linesArr, "")
+ if text == "" {
+ text = "\n" // If no params exist, don't send "not found," but
an empty file. We know the profile exists.
+ }
+ hdr := makeHdrComment(opt.HdrComment)
+ text = hdr + text
+
+ return Cfg{
+ Text: text,
+ ContentType: ContentTypeCacheDotConfig,
+ LineComment: LineCommentCacheDotConfig,
+ Warnings: warnings,
+ }, nil
+}
+
+// ProfileDS struct for filtered delivery services.
+type ProfileDS struct {
+ Name string
+ Type tc.DSType
+ OriginFQDN string
+}
+
+// GetProfileDSes filters delivery services and return delivery services with
valid type and non-empty FQDN.
+func GetProfileDSes(server *Server,
+ servers []Server,
+ deliveryServices []DeliveryService,
+ deliveryServiceServers []DeliveryServiceServer,
+) ([]ProfileDS, []string) {
+ warnings := make([]string, 0)
profileServerIDsMap := map[int]struct{}{}
for _, sv := range servers {
if len(sv.ProfileNames) == 0 {
@@ -97,7 +154,7 @@ func makeCacheDotConfigEdge(
dsIDs[dss.DeliveryService] = struct{}{}
}
- profileDSes := []profileDS{}
+ profileDSes := []ProfileDS{}
for _, ds := range deliveryServices {
if ds.ID == nil {
warnings = append(warnings, "deliveryservices had ds
with nil id, skipping!")
@@ -110,6 +167,10 @@ func makeCacheDotConfigEdge(
if ds.OrgServerFQDN == nil {
continue // this is normal for steering and anymap dses
}
+ if ds.XMLID == nil || *ds.XMLID == "" {
+ warnings = append(warnings, "got ds with missing XMLID,
skipping!")
+ continue
+ }
if *ds.Type == tc.DSTypeInvalid {
warnings = append(warnings, "deliveryservices had ds
with invalid type, skipping!")
continue
@@ -121,73 +182,31 @@ func makeCacheDotConfigEdge(
if _, ok := dsIDs[*ds.ID]; !ok && ds.Topology == nil {
continue
}
- origin := *ds.OrgServerFQDN
- profileDSes = append(profileDSes, profileDS{Type: *ds.Type,
OriginFQDN: &origin})
+ profileDSes = append(profileDSes, ProfileDS{Name: *ds.XMLID,
Type: *ds.Type, OriginFQDN: *ds.OrgServerFQDN})
}
- lines := map[string]struct{}{} // use a "set" for lines, to avoid
duplicates, since we're looking up by profile
- for _, ds := range profileDSes {
- if ds.Type != tc.DSTypeHTTPNoCache {
- continue
- }
- if ds.OriginFQDN == nil || *ds.OriginFQDN == "" {
- warnings = append(warnings, "profileCacheDotConfig ds
has no origin fqdn, skipping!") // TODO add ds name to data loaded, to put it
in the error here?
- continue
- }
- originFQDN, originPort := getHostPortFromURI(*ds.OriginFQDN)
- if originPort != "" {
- l := "dest_domain=" + originFQDN + " port=" +
originPort + " scheme=http action=never-cache\n"
- lines[l] = struct{}{}
- } else {
- l := "dest_domain=" + originFQDN + " scheme=http
action=never-cache\n"
- lines[l] = struct{}{}
- }
- }
-
- linesArr := []string{}
- for line, _ := range lines {
- linesArr = append(linesArr, line)
- }
- sort.Strings(linesArr)
- text := strings.Join(linesArr, "")
- if text == "" {
- text = "\n" // If no params exist, don't send "not found," but
an empty file. We know the profile exists.
- }
- hdr := makeHdrComment(opt.HdrComment)
- text = hdr + text
-
- return Cfg{
- Text: text,
- ContentType: ContentTypeCacheDotConfig,
- LineComment: LineCommentCacheDotConfig,
- Warnings: warnings,
- }, nil
-}
-
-type profileDS struct {
- Type tc.DSType
- OriginFQDN *string
+ return profileDSes, warnings
}
// dsesToProfileDSes is a helper function to convert a
[]tc.DeliveryServiceNullable to []ProfileDS.
// Note this does not check for nil values. If any DeliveryService's Type or
OrgServerFQDN may be nil, the returned ProfileDS should be checked for
DSTypeInvalid and nil, respectively.
-func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []profileDS {
- pdses := []profileDS{}
+func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []ProfileDS {
+ pdses := []ProfileDS{}
for _, ds := range dses {
- pds := profileDS{}
+ pds := ProfileDS{}
if ds.Type != nil {
pds.Type = *ds.Type
}
if ds.OrgServerFQDN != nil && *ds.OrgServerFQDN != "" {
- org := *ds.OrgServerFQDN
- pds.OriginFQDN = &org
+ pds.OriginFQDN = *ds.OrgServerFQDN
}
pdses = append(pdses, pds)
}
return pdses
}
-func getHostPortFromURI(uriStr string) (string, string) {
+// GetHostPortFromURI strips HTTP(s) scheme and path and return host with port
(if found).
+func GetHostPortFromURI(uriStr string) (string, string) {
originFQDN := uriStr
originFQDN = strings.TrimPrefix(originFQDN, "http://";)
originFQDN = strings.TrimPrefix(originFQDN, "https://";)
diff --git a/lib/varnishcfg/cache_control.go b/lib/varnishcfg/cache_control.go
new file mode 100644
index 0000000000..851c0ff919
--- /dev/null
+++ b/lib/varnishcfg/cache_control.go
@@ -0,0 +1,57 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/apache/trafficcontrol/lib/go-atscfg"
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func (v VCLBuilder) configureUncacheableDSes(vclFile *vclFile, profileDSes
[]atscfg.ProfileDS) []string {
+ warnings := make([]string, 0)
+
+ uncacheableHostsConditions := make([]string, 0)
+ for _, ds := range profileDSes {
+ if ds.Type != tc.DSTypeHTTPNoCache {
+ continue
+ }
+
+ if ds.OriginFQDN == "" {
+ warnings = append(warnings, fmt.Sprintf("ds %s has no
origin fqdn, skipping!", ds.Name))
+ continue
+ }
+
+ host, _ := atscfg.GetHostPortFromURI(ds.OriginFQDN)
+ uncacheableHostsConditions = append(uncacheableHostsConditions,
fmt.Sprintf(`bereq.http.host == "%s"`, host))
+ }
+ if len(uncacheableHostsConditions) == 0 {
+ return warnings
+ }
+ berespLines := make([]string, 0)
+ berespLines = append(berespLines, fmt.Sprintf(`if (%s) {`,
strings.Join(uncacheableHostsConditions, " || ")))
+ berespLines = append(berespLines, fmt.Sprint(` set beresp.uncacheable
= true;`))
+ berespLines = append(berespLines, fmt.Sprint(`}`))
+ vclFile.subroutines["vcl_backend_response"] =
append(vclFile.subroutines["vcl_backend_response"], berespLines...)
+
+ return warnings
+}
diff --git a/lib/varnishcfg/cache_control_test.go
b/lib/varnishcfg/cache_control_test.go
new file mode 100644
index 0000000000..beb57d6cb4
--- /dev/null
+++ b/lib/varnishcfg/cache_control_test.go
@@ -0,0 +1,90 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/apache/trafficcontrol/cache-config/t3cutil"
+ "github.com/apache/trafficcontrol/lib/go-atscfg"
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func TestConfigureUncacheableDSes(t *testing.T) {
+ testCases := []struct {
+ name string
+ dses []atscfg.ProfileDS
+ expectedSubroutines map[string][]string
+ }{
+ {
+ name: "no uncacheable DSes",
+ dses: []atscfg.ProfileDS{
+ {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN:
"ds1.example.com"},
+ {Name: "ds2", Type: tc.DSTypeHTTP, OriginFQDN:
"ds2.example.com"},
+ },
+ expectedSubroutines: make(map[string][]string),
+ },
+ {
+ name: "single uncacheable DS",
+ dses: []atscfg.ProfileDS{
+ {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN:
"ds1.example.com"},
+ {Name: "ds2", Type: tc.DSTypeHTTPNoCache,
OriginFQDN: "ds2.example.com"},
+ },
+ expectedSubroutines: map[string][]string{
+ "vcl_backend_response": {
+ `if (bereq.http.host ==
"ds2.example.com") {`,
+ ` set beresp.uncacheable = true;`,
+ `}`,
+ },
+ },
+ },
+ {
+ name: "multiple uncacheable DS",
+ dses: []atscfg.ProfileDS{
+ {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN:
"ds1.example.com"},
+ {Name: "ds2", Type: tc.DSTypeHTTPNoCache,
OriginFQDN: "ds2.example.com"},
+ {Name: "ds3", Type: tc.DSTypeHTTPNoCache,
OriginFQDN: "ds3.example.com"},
+ {Name: "ds4", Type: tc.DSTypeHTTPNoCache,
OriginFQDN: "ds4.example.com"},
+ {Name: "ds5", Type: tc.DSTypeDNS, OriginFQDN:
"ds5.example.com"},
+ },
+ expectedSubroutines: map[string][]string{
+ "vcl_backend_response": {
+ `if (bereq.http.host ==
"ds2.example.com" || bereq.http.host == "ds3.example.com" || bereq.http.host ==
"ds4.example.com") {`,
+ ` set beresp.uncacheable = true;`,
+ `}`,
+ },
+ },
+ },
+ }
+ for _, tC := range testCases {
+ t.Run(tC.name, func(t *testing.T) {
+ vb := NewVCLBuilder(&t3cutil.ConfigData{})
+ vclFile := newVCLFile(defaultVCLVersion)
+ warnings := vb.configureUncacheableDSes(&vclFile,
tC.dses)
+ if len(warnings) != 0 {
+ t.Errorf("got unexpected warnings %v", warnings)
+ }
+ if !reflect.DeepEqual(vclFile.subroutines,
tC.expectedSubroutines) {
+ t.Errorf("got %v want %v", vclFile.subroutines,
tC.expectedSubroutines)
+ }
+ })
+ }
+}
diff --git a/lib/varnishcfg/vclbuilder.go b/lib/varnishcfg/vclbuilder.go
index 70c89de9c7..480aaa686e 100644
--- a/lib/varnishcfg/vclbuilder.go
+++ b/lib/varnishcfg/vclbuilder.go
@@ -75,8 +75,18 @@ func (vb *VCLBuilder) BuildVCLFile() (string, []string,
error) {
if err != nil {
return "", nil, fmt.Errorf("(warnings: %s) %w",
strings.Join(warnings, ", "), err)
}
+ profileDSes, dsWarnings := atscfg.GetProfileDSes(
+ vb.toData.Server,
+ vb.toData.Servers,
+ vb.toData.DeliveryServices,
+ vb.toData.DeliveryServiceServers,
+ )
+ warnings = append(warnings, dsWarnings...)
+ cacheWarnings := vb.configureUncacheableDSes(&v, profileDSes)
+ warnings = append(warnings, cacheWarnings...)
dirWarnings, err := vb.configureDirectors(&v, parents)
warnings = append(warnings, dirWarnings...)
+
return fmt.Sprint(v), warnings, err
}
