ocket8888 commented on a change in pull request #5247:
URL: https://github.com/apache/trafficcontrol/pull/5247#discussion_r525449058
##########
File path: lib/go-atscfg/meta.go
##########
@@ -20,102 +20,148 @@ package atscfg
*/
import (
- "encoding/json"
"errors"
"path/filepath"
"strings"
- "github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
)
-type ConfigProfileParams struct {
- FileNameOnDisk string
- Location string
- URL string
+type CfgMeta struct {
+ Name string
+ Path string
}
-// APIVersion is the Traffic Ops API version for config fiels.
-// This is used to generate the meta config, which has API paths.
-// Note the version in the meta config is not used by the atstccfg generator,
which isn't actually an API.
-// TODO change the config system to not use old API paths, and remove this.
-const APIVersion = "2.0"
+// MakeMetaObj returns the list of config files, any warnings, and any errors.
+func MakeConfigFilesList(
+ configDir string,
+ server *Server,
+ serverParams []tc.Parameter,
+ deliveryServices []DeliveryService,
+ deliveryServiceServers []tc.DeliveryServiceServer,
+ globalParams []tc.Parameter,
+ cacheGroupArr []tc.CacheGroupNullable,
+ topologies []tc.Topology,
+) ([]CfgMeta, []string, error) {
+ warnings := []string{}
-// requiredFiles is a constant (because Go doesn't allow const slices).
-// Note these are not exhaustive. This is only used to error if these are
missing.
-// The presence of these is no guarantee the location Parameters are complete
and correct.
-func requiredFiles() []string {
- return []string{
- "cache.config",
- "hosting.config",
- "ip_allow.config",
- "parent.config",
- "plugin.config",
- "records.config",
- "remap.config",
- "storage.config",
- "volume.config",
+ if server.Cachegroup == nil {
+ return nil, warnings, errors.New("this server missing
Cachegroup")
+ } else if server.CachegroupID == nil {
+ return nil, warnings, errors.New("this server missing
CachegroupID")
+ } else if server.ProfileID == nil {
+ return nil, warnings, errors.New("server missing ProfileID")
+ } else if server.TCPPort == nil {
+ return nil, warnings, errors.New("server missing TCPPort")
+ } else if server.HostName == nil {
+ return nil, warnings, errors.New("server missing HostName")
+ } else if server.CDNID == nil {
+ return nil, warnings, errors.New("server missing CDNID")
+ } else if server.CDNName == nil {
+ return nil, warnings, errors.New("server missing CDNName")
+ } else if server.ID == nil {
+ return nil, warnings, errors.New("server missing ID")
+ } else if server.Profile == nil {
+ return nil, warnings, errors.New("server missing Profile")
}
-}
-func MakeMetaConfig(
- server *tc.ServerNullable,
- tmURL string, // global tm.url Parameter
- tmReverseProxyURL string, // global tm.rev_proxy.url Parameter
- locationParams map[string]ConfigProfileParams, //
map[configFile]params; 'location' and 'URL' Parameters on serverHostName's
Profile
- uriSignedDSes []tc.DeliveryServiceName,
- scopeParams map[string]string, // map[configFileName]scopeParam
- dses map[tc.DeliveryServiceName]tc.DeliveryServiceNullableV30,
- cacheGroupArr []tc.CacheGroupNullable,
- topologies []tc.Topology,
-) string {
- configDir := "" // this should only be used for Traffic Ops, which
doesn't have a local ATS install config directory (and thus will fail if any
location Parameters are missing or relative).
- return MetaObjToMetaConfig(MakeMetaObj(server, tmURL,
tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses,
cacheGroupArr, topologies, configDir))
-}
+ tmURL, tmReverseProxyURL := getTOURLAndReverseProxy(globalParams)
+ if tmURL == "" {
+ warnings = append(warnings, "global tm.url parameter missing or
empty! Setting empty in meta config!")
+ }
-func MetaObjToMetaConfig(atsData tc.ATSConfigMetaData, err error) string {
- if err != nil {
- return "error creating meta config: " + err.Error()
+ dses, dsWarns := filterConfigFileDSes(server, deliveryServices,
deliveryServiceServers)
+ warnings = append(warnings, dsWarns...)
+
+ locationParams := getLocationParams(serverParams)
+
+ uriSignedDSes, signDSWarns := getURISignedDSes(dses)
+ warnings = append(warnings, signDSWarns...)
+
+ configFiles := []CfgMeta{}
+
+ if locationParams["remap.config"].Path != "" {
+ configLocation := locationParams["remap.config"].Path
+ for _, ds := range uriSignedDSes {
+ cfgName := "uri_signing_" + string(ds) + ".config"
+ // If there's already a parameter for it, don't clobber
it. The user may wish to override the location.
+ if _, ok := locationParams[cfgName]; !ok {
+ p := locationParams[cfgName]
+ p.Name = cfgName
+ p.Path = configLocation
+ locationParams[cfgName] = p
+ }
+ }
}
- bts, err := json.Marshal(atsData)
- if err != nil {
- // should never happen
- log.Errorln("marshalling meta config: " + err.Error())
- bts = []byte("error encoding to json, see log for details")
+
+locationParamsFor:
Review comment:
I don't see a `goto` anywhere; what are these labels for?
##########
File path: lib/go-atscfg/remapdotconfig.go
##########
@@ -35,81 +34,104 @@ const CacheKeyParameterConfigFile = "cachekey.config"
const ContentTypeRemapDotConfig = ContentTypeTextASCII
const LineCommentRemapDotConfig = LineCommentHash
+const RemapConfigRangeDirective = `__RANGE_DIRECTIVE__`
+
func MakeRemapDotConfig(
- server *tc.ServerNullable,
- unfilteredDSes []tc.DeliveryServiceNullableV30,
+ server *Server,
+ unfilteredDSes []DeliveryService,
dss []tc.DeliveryServiceServer,
dsRegexArr []tc.DeliveryServiceRegexes,
serverParams []tc.Parameter,
cdn *tc.CDN,
- toToolName string, // tm.toolname global parameter (TODO: cache itself?)
- toURL string, // tm.url global parameter (TODO: cache itself?)
cacheKeyParams []tc.Parameter,
topologies []tc.Topology,
cacheGroupArr []tc.CacheGroupNullable,
serverCapabilities map[int]map[ServerCapability]struct{},
dsRequiredCapabilities map[int]map[ServerCapability]struct{},
-) string {
+ hdrComment string,
+) (Cfg, error) {
+ warnings := []string{}
if server.HostName == nil {
- return "ERROR: server HostName missing"
+ return Cfg{}, makeErr(warnings, "server HostName missing")
} else if server.ID == nil {
- return "ERROR: server ID missing"
+ return Cfg{}, makeErr(warnings, "server ID missing")
} else if server.Cachegroup == nil {
- return "ERROR: server Cachegroup missing"
+ return Cfg{}, makeErr(warnings, "server Cachegroup missing")
} else if server.DomainName == nil {
- return "ERROR: server DomainName missing"
+ return Cfg{}, makeErr(warnings, "server DomainName missing")
}
cdnDomain := cdn.DomainName
dsRegexes := makeDSRegexMap(dsRegexArr)
// Returned DSes are guaranteed to have a non-nil XMLID, Type, DSCP,
ID, and Active.
- dses := remapFilterDSes(server, dss, unfilteredDSes, cacheKeyParams)
+ dses, dsWarns := remapFilterDSes(server, dss, unfilteredDSes,
cacheKeyParams)
+ warnings = append(warnings, dsWarns...)
- dsProfilesCacheKeyConfigParams, err :=
makeDSProfilesCacheKeyConfigParams(server, dses, cacheKeyParams)
+ dsProfilesCacheKeyConfigParams, paramWarns, err :=
makeDSProfilesCacheKeyConfigParams(server, dses, cacheKeyParams)
+ warnings = append(warnings, paramWarns...)
if err != nil {
- log.Errorln("Error making Delivery Service Cache Key Params,
cache key will be missing! : " + err.Error())
+ warnings = append(warnings, "making Delivery Service Cache Key
Params, cache key will be missing! : "+err.Error())
}
- atsMajorVersion := getATSMajorVersion(serverParams)
- serverPackageParamData := makeServerPackageParamData(server,
serverParams)
- cacheURLConfigParams := ParamsToMap(FilterParams(serverParams,
CacheURLParameterConfigFile, "", "", ""))
- cacheGroups, err := MakeCGMap(cacheGroupArr)
+ atsMajorVersion, verWarns := getATSMajorVersion(serverParams)
+ warnings = append(warnings, verWarns...)
+ serverPackageParamData, paramWarns :=
makeServerPackageParamData(server, serverParams)
+ warnings = append(warnings, paramWarns...)
+ cacheURLConfigParams, paramWarns :=
paramsToMap(filterParams(serverParams, CacheURLParameterConfigFile, "", "", ""))
+ warnings = append(warnings, paramWarns...)
+ cacheGroups, err := makeCGMap(cacheGroupArr)
if err != nil {
- log.Errorln("making remap.config, config will be malformed! : "
+ err.Error())
+ return Cfg{}, makeErr(warnings, "making remap.config, config
will be malformed! : "+err.Error())
}
- nameTopologies := MakeTopologyNameMap(topologies)
+ nameTopologies := makeTopologyNameMap(topologies)
- hdr := GenericHeaderComment(*server.HostName, toToolName, toURL)
+ hdr := makeHdrComment(hdrComment)
+ txt := ""
+ typeWarns := []string{}
if tc.CacheTypeFromString(server.Type) == tc.CacheTypeMid {
- return GetServerConfigRemapDotConfigForMid(atsMajorVersion,
dsProfilesCacheKeyConfigParams, dses, dsRegexes, hdr, server, nameTopologies,
cacheGroups, serverCapabilities, dsRequiredCapabilities)
+ txt, typeWarns, err =
getServerConfigRemapDotConfigForMid(atsMajorVersion,
dsProfilesCacheKeyConfigParams, dses, dsRegexes, hdr, server, nameTopologies,
cacheGroups, serverCapabilities, dsRequiredCapabilities)
+ } else {
+ txt, typeWarns, err =
getServerConfigRemapDotConfigForEdge(cacheURLConfigParams,
dsProfilesCacheKeyConfigParams, serverPackageParamData, dses, dsRegexes,
atsMajorVersion, hdr, server, nameTopologies, cacheGroups, serverCapabilities,
dsRequiredCapabilities, cdnDomain)
}
- return GetServerConfigRemapDotConfigForEdge(cacheURLConfigParams,
dsProfilesCacheKeyConfigParams, serverPackageParamData, dses, dsRegexes,
atsMajorVersion, hdr, server, nameTopologies, cacheGroups, serverCapabilities,
dsRequiredCapabilities, cdnDomain)
+ warnings = append(warnings, typeWarns...)
+ if err != nil {
+ return Cfg{}, makeErr(warnings, err.Error()) // the GetFor
funcs include error context
+ }
+
+ return Cfg{
+ Text: txt,
+ ContentType: ContentTypeRemapDotConfig,
+ LineComment: LineCommentRemapDotConfig,
+ Warnings: warnings,
+ }, nil
}
-func GetServerConfigRemapDotConfigForMid(
+// GetServerConfigRemapDotConfigForMid returns the remap lines, any warnings,
and any error.
Review comment:
Same as above RE: GoDoc
##########
File path: lib/go-atscfg/parentdotconfig.go
##########
@@ -1001,62 +1030,66 @@ func GetTopologyParents(
continue
}
- if !HasRequiredCapabilities(serverCapabilities[*sv.ID],
dsRequiredCapabilities[*ds.ID]) {
+ if !hasRequiredCapabilities(serverCapabilities[*sv.ID],
dsRequiredCapabilities[*ds.ID]) {
continue
}
if *sv.Cachegroup == parentCG {
- parentStr, err := serverParentStr(&sv.ServerNullable,
sv.Params)
+ parentStr, err := serverParentStr(&sv.Server, sv.Params)
if err != nil {
- return nil, nil, errors.New("getting server
parent string: " + err.Error())
+ return nil, nil, warnings, errors.New("getting
server parent string: " + err.Error())
}
if parentStr != "" { // will be empty if server is
not_a_parent (possibly other reasons)
parentStrs = append(parentStrs, parentStr)
}
}
if *sv.Cachegroup == secondaryParentCG {
- parentStr, err := serverParentStr(&sv.ServerNullable,
sv.Params)
+ parentStr, err := serverParentStr(&sv.Server, sv.Params)
if err != nil {
- return nil, nil, errors.New("getting server
parent string: " + err.Error())
+ return nil, nil, warnings, errors.New("getting
server parent string: " + err.Error())
}
secondaryParentStrs = append(secondaryParentStrs,
parentStr)
}
}
- return parentStrs, secondaryParentStrs, nil
+ return parentStrs, secondaryParentStrs, warnings, nil
}
-func GetOriginURI(fqdn string) (*url.URL, error) {
+// GetOriginURI returns the URL, any warnings, and any error.
Review comment:
Nit: GoDoc should start with the name of the documented symbol.
##########
File path: lib/go-atscfg/packages.go
##########
@@ -32,39 +32,48 @@ const PackagesParamConfigFile = `package`
const ContentTypePackages = ContentTypeTextASCII
Review comment:
I know this didn't change in this PR, but isn't this actually a
JSON-encoded output?
##########
File path: lib/go-atscfg/cachedotconfig.go
##########
@@ -23,33 +23,106 @@ import (
"sort"
"strings"
- "github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
)
const ContentTypeCacheDotConfig = ContentTypeTextASCII
const LineCommentCacheDotConfig = LineCommentHash
-type ProfileDS struct {
- Type tc.DSType
- OriginFQDN *string
+func MakeCacheDotConfig(
+ server *Server,
+ servers []Server,
+ deliveryServices []DeliveryService,
+ deliveryServiceServers []tc.DeliveryServiceServer,
+ hdrComment string,
+) (Cfg, error) {
+ if tc.CacheTypeFromString(server.Type) == tc.CacheTypeMid {
+ return makeCacheDotConfigMid(server, deliveryServices,
hdrComment)
+ } else {
+ return makeCacheDotConfigEdge(server, servers,
deliveryServices, deliveryServiceServers, hdrComment)
+ }
}
// MakeCacheDotConfig makes the ATS cache.config config file.
// profileDSes must be the list of delivery services, which are assigned to
severs, for which this profile is assigned. It MUST NOT contain any other
delivery services. Note DSesToProfileDSes may be helpful if you have a
[]tc.DeliveryServiceNullable, for example from traffic_ops/client.
Review comment:
I think you wanna move this GoDoc up to the `MakeCacheDotConfig`
function.
##########
File path: lib/go-atscfg/parentdotconfig.go
##########
@@ -1408,32 +1442,33 @@ func GetParentConfigProfileParams(
}
parentConfigServerCacheProfileParams[*cgServer.Profile] =
profileCache
}
- return parentConfigServerCacheProfileParams
+ return parentConfigServerCacheProfileParams, warnings
}
-// GetDSOrigins takes a map[deliveryServiceID]DeliveryService, and returns a
map[DeliveryServiceID]OriginURI.
-func GetDSOrigins(dses map[int]tc.DeliveryServiceNullableV30)
(map[int]*OriginURI, error) {
- dsOrigins := map[int]*OriginURI{}
+// GetDSOrigins takes a map[deliveryServiceID]DeliveryService, and returns a
map[DeliveryServiceID]OriginURI, any warnings, and any error.
Review comment:
same as above RE: GoDoc
##########
File path: lib/go-atscfg/atscfg.go
##########
@@ -22,56 +22,90 @@ package atscfg
import (
"encoding/json"
"errors"
+ "fmt"
"net"
"sort"
"strconv"
"strings"
- "time"
- "github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
)
const InvalidID = -1
-
const DefaultATSVersion = "5" // TODO Emulates Perl; change to 6? ATC no
longer officially supports ATS 5.
-
const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006"
-
const ContentTypeTextASCII = `text/plain; charset=us-ascii`
-
const LineCommentHash = "#"
+const ConfigSuffix = ".config"
+
+type DeliveryServiceID int
+type ProfileID int
+type ServerID int
type TopologyName string
type CacheGroupType string
type ServerCapability string
-type ServerInfo struct {
- CacheGroupID int
- CacheGroupName string
- CDN tc.CDNName
- CDNID int
- DomainName string
- HostName string
- HTTPSPort int
- ID int
- IP string
- ParentCacheGroupID int
- ParentCacheGroupType string
- ProfileID ProfileID
- ProfileName string
- Port int
- SecondaryParentCacheGroupID int
- SecondaryParentCacheGroupType string
- Type string
-}
-
-func (s *ServerInfo) IsTopLevelCache() bool {
- return (s.ParentCacheGroupType == tc.CacheGroupOriginTypeName ||
s.ParentCacheGroupID == InvalidID) &&
- (s.SecondaryParentCacheGroupType == tc.CacheGroupOriginTypeName
|| s.SecondaryParentCacheGroupID == InvalidID)
-}
-
-func MakeCGMap(cgs []tc.CacheGroupNullable)
(map[tc.CacheGroupName]tc.CacheGroupNullable, error) {
+// Server is a tc.Server for the latest lib/go-tc and traffic_ops/vx-client
type.
+// This allows atscfg to not have to change the type everywhere it's used,
every time ATC changes the base type,
+// but to only have to change it here, and the places where breaking symbol
changes were made.
+type Server tc.ServerV30
+
+// DeliveryService is a tc.DeliveryService for the latest lib/go-tc and
traffic_ops/vx-client type.
+// This allows atscfg to not have to change the type everywhere it's used,
every time ATC changes the base type,
+// but to only have to change it here, and the places where breaking symbol
changes were made.
+type DeliveryService tc.DeliveryServiceNullableV30
+
+// ToDeliveryServices converts a slice of the latest lib/go-tc and
traffic_ops/vx-client type to the local alias.
+func ToDeliveryServices(dses []tc.DeliveryServiceNullableV30)
[]DeliveryService {
+ ad := []DeliveryService{}
+ for _, ds := range dses {
+ ad = append(ad, DeliveryService(ds))
+ }
+ return ad
+}
+
+// OldToDeliveryServices converts a slice of the old traffic_ops/client type
to the local alias.
+func OldToDeliveryServices(dses []tc.DeliveryServiceNullable)
[]DeliveryService {
+ ad := []DeliveryService{}
+ for _, ds := range dses {
+ upgradedDS :=
tc.DeliveryServiceNullableV30{DeliveryServiceNullableV15:
tc.DeliveryServiceNullableV15(ds)}
+ ad = append(ad, DeliveryService(upgradedDS))
+ }
+ return ad
+}
+
+// ToServers converts a slice of the latest lib/go-tc and
traffic_ops/vx-client type to the local alias.
+func ToServers(servers []tc.ServerV30) []Server {
+ as := []Server{}
+ for _, sv := range servers {
+ as = append(as, Server(sv))
+ }
+ return as
+}
Review comment:
I know you don't like it, but since Servers and Delivery Services are
ATC's most populous objects (in implementations of which I'm aware) I think
there could be a somewhat significant performance gain by just using `make`
instead of re-allocating multiple times on these long loops.
##########
File path: lib/go-atscfg/sslmulticertdotconfig.go
##########
@@ -95,11 +112,11 @@ func GetSSLMultiCertDotConfigCertAndKeyName(dsName
tc.DeliveryServiceName, ds SS
return cerName, keyName
}
-// GetSSLMultiCertDotConfigDeliveryServices takes a list of delivery services,
and returns the delivery services which will be inserted into the config by
MakeSSLMultiCertDotConfig.
+// getSSLMultiCertDotConfigDeliveryServices takes a list of delivery services,
and returns the delivery services which will be inserted into the config by
MakeSSLMultiCertDotConfig.
Review comment:
Same as above RE: GoDoc
##########
File path: lib/go-atscfg/parentdotconfig.go
##########
@@ -1219,46 +1257,38 @@ func unavailableServerRetryResponsesValid(s string)
bool {
return re.MatchString(s)
}
-// HasRequiredCapabilities returns whether the given caps has all the required
capabilities in the given reqCaps.
-func HasRequiredCapabilities(caps map[ServerCapability]struct{}, reqCaps
map[ServerCapability]struct{}) bool {
- for reqCap, _ := range reqCaps {
- if _, ok := caps[reqCap]; !ok {
- return false
- }
- }
- return true
-}
-
-func GetOriginServersAndProfileCaches(
- cgServers map[int]tc.ServerNullable,
+// GetOriginServersAndProfileCaches returns the origin servers, ProfileCaches,
any warnings, and any error.
Review comment:
same as above RE: GoDoc
##########
File path: lib/go-atscfg/servercachedotconfig.go
##########
@@ -27,18 +27,36 @@ import (
"github.com/apache/trafficcontrol/lib/go-tc"
)
-type ServerCacheConfigDS struct {
- OrgServerFQDN string
- Type tc.DSType
-}
+const ServerCacheDotConfigIncludeInactiveDSes = false // TODO move to
lib/go-atscfg
Review comment:
More like... TODONE
##########
File path: lib/go-atscfg/remapdotconfig.go
##########
@@ -192,37 +218,39 @@ func GetServerConfigRemapDotConfigForMid(
text := header
text += strings.Join(textLines, "")
- return text
+ return text, warnings, nil
}
-func GetServerConfigRemapDotConfigForEdge(
+// GetServerConfigRemapDotConfigForEdge returns the remap lines, any warnings,
and any error
Review comment:
Same as above RE: GoDoc
##########
File path: lib/go-atscfg/remapdotconfig.go
##########
@@ -408,22 +448,22 @@ func MakeDSTopologyHeaderRewriteTxt(ds
tc.DeliveryServiceNullableV30, cg tc.Cach
if placement.IsLastCacheTier && ds.LastHeaderRewrite != nil &&
*ds.LastHeaderRewrite != "" {
txt += pluginTxt + LastHeaderRewriteConfigFileName(*ds.XMLID) +
` `
}
- return txt
+ return txt, nil
}
-type RemapLine struct {
+type remapLine struct {
From string
To string
}
// MakeEdgeDSDataRemapLines returns the remap lines for the given server and
delivery service.
Review comment:
Same as above RE: GoDoc
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
