[GitHub] [trafficcontrol] alficles commented on a change in pull request #3688: Add TO Go remap.config
alficles commented on a change in pull request #3688: Add TO Go remap.config URL: https://github.com/apache/trafficcontrol/pull/3688#discussion_r321574341 ## File path: lib/go-atscfg/remapdotconfig.go ## @@ -0,0 +1,369 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "sort" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" +) + +type RemapConfigDSData struct { + ID int + Type tc.DSType + OriginFQDN *string + MidHeaderRewrite *string + CacheURL *string + RangeRequestHandling *int + CacheKeyConfigParams map[string]string + RemapText*string + EdgeHeaderRewrite*string + SigningAlgorithm *string + Name string + QStringIgnore*int + RegexRemap *string + FQPacingRate *int + DSCP int + RoutingName *string + MultiSiteOrigin *string + Pattern *string + RegexType*string + Domain *string + RegexSetNumber *string + OriginShield *string + ProfileID*int + Protocol *int + AnonymousBlockingEnabled *bool + Active bool +} + +func MakeRemapDotConfig( + serverName tc.CacheName, + toToolName string, // tm.toolname global parameter (TODO: cache itself?) + toURL string, // tm.url global parameter (TODO: cache itself?) + atsMajorVersion int, + cacheURLConfigParams map[string]string, // map[name]value for this server's profile and config file 'cacheurl.config' + dsProfilesCacheKeyConfigParams map[int]map[string]string, // map[profileID][paramName]paramVal for this server's profile, config file 'cachekey.config' + serverPackageParamData map[string]string, // map[paramName]paramVal for this server, config file 'package' + serverInfo *ServerInfo, // ServerInfo for this server + remapDSData []RemapConfigDSData, +) string { + hdr := GenericHeaderComment(string(serverName), toToolName, toURL) + text := "" + if tc.CacheTypeFromString(serverInfo.Type) == tc.CacheTypeMid { + text = GetServerConfigRemapDotConfigForMid(atsMajorVersion, dsProfilesCacheKeyConfigParams, serverInfo, remapDSData, hdr) + } else { + text = GetServerConfigRemapDotConfigForEdge(cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData, atsMajorVersion, hdr) + } + return text +} + +func GetServerConfigRemapDotConfigForMid( + atsMajorVersion int, + profilesCacheKeyConfigParams map[int]map[string]string, + server *ServerInfo, + dses []RemapConfigDSData, + header string, +) string { + midRemaps := map[string]string{} + for _, ds := range dses { + if ds.Type.IsLive() && !ds.Type.IsNational() { + continue // Live local delivery services skip mids + } + + if ds.OriginFQDN == nil || *ds.OriginFQDN == "" { + log.Warnf("GetServerConfigRemapDotConfigForMid ds '" + ds.Name + "' has no origin fqdn, skipping!") // TODO confirm - Perl uses without checking! + continue + } + + if midRemaps[*ds.OriginFQDN] != "" { + continue // skip remap rules from extra HOST_REGEXP entries + } + + midRemap := "" + if ds.MidHeaderRewrite != nil && *ds.MidHeaderRewrite != "" { + midRemap += ` @plugin=header_rewrite.so @pparam=` + MidHeaderRewriteConfigFileName(ds.Name) + } + if ds.QStringIgnore != nil && *ds.QStringIgnore == tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp { + midRemap +=
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3688: Add TO Go remap.config
alficles commented on a change in pull request #3688: Add TO Go remap.config URL: https://github.com/apache/trafficcontrol/pull/3688#discussion_r321573672 ## File path: lib/go-atscfg/remapdotconfig.go ## @@ -0,0 +1,369 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "sort" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" +) + +type RemapConfigDSData struct { + ID int + Type tc.DSType + OriginFQDN *string + MidHeaderRewrite *string + CacheURL *string + RangeRequestHandling *int + CacheKeyConfigParams map[string]string + RemapText*string + EdgeHeaderRewrite*string + SigningAlgorithm *string + Name string + QStringIgnore*int + RegexRemap *string + FQPacingRate *int + DSCP int + RoutingName *string + MultiSiteOrigin *string + Pattern *string + RegexType*string + Domain *string + RegexSetNumber *string + OriginShield *string + ProfileID*int + Protocol *int + AnonymousBlockingEnabled *bool + Active bool +} + +func MakeRemapDotConfig( + serverName tc.CacheName, + toToolName string, // tm.toolname global parameter (TODO: cache itself?) + toURL string, // tm.url global parameter (TODO: cache itself?) + atsMajorVersion int, + cacheURLConfigParams map[string]string, // map[name]value for this server's profile and config file 'cacheurl.config' + dsProfilesCacheKeyConfigParams map[int]map[string]string, // map[profileID][paramName]paramVal for this server's profile, config file 'cachekey.config' + serverPackageParamData map[string]string, // map[paramName]paramVal for this server, config file 'package' + serverInfo *ServerInfo, // ServerInfo for this server + remapDSData []RemapConfigDSData, +) string { + hdr := GenericHeaderComment(string(serverName), toToolName, toURL) + text := "" + if tc.CacheTypeFromString(serverInfo.Type) == tc.CacheTypeMid { + text = GetServerConfigRemapDotConfigForMid(atsMajorVersion, dsProfilesCacheKeyConfigParams, serverInfo, remapDSData, hdr) + } else { + text = GetServerConfigRemapDotConfigForEdge(cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData, atsMajorVersion, hdr) + } + return text +} + +func GetServerConfigRemapDotConfigForMid( + atsMajorVersion int, + profilesCacheKeyConfigParams map[int]map[string]string, + server *ServerInfo, + dses []RemapConfigDSData, + header string, +) string { + midRemaps := map[string]string{} + for _, ds := range dses { + if ds.Type.IsLive() && !ds.Type.IsNational() { + continue // Live local delivery services skip mids + } + + if ds.OriginFQDN == nil || *ds.OriginFQDN == "" { + log.Warnf("GetServerConfigRemapDotConfigForMid ds '" + ds.Name + "' has no origin fqdn, skipping!") // TODO confirm - Perl uses without checking! + continue + } + + if midRemaps[*ds.OriginFQDN] != "" { + continue // skip remap rules from extra HOST_REGEXP entries + } + + midRemap := "" + if ds.MidHeaderRewrite != nil && *ds.MidHeaderRewrite != "" { + midRemap += ` @plugin=header_rewrite.so @pparam=` + MidHeaderRewriteConfigFileName(ds.Name) + } + if ds.QStringIgnore != nil && *ds.QStringIgnore == tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp { + midRemap +=
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3899: Add TO Go server cache configs
alficles commented on a change in pull request #3899: Add TO Go server cache configs URL: https://github.com/apache/trafficcontrol/pull/3899#discussion_r321566988 ## File path: lib/go-atscfg/servercachedotconfig.go ## @@ -0,0 +1,91 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-tc" +) + +type ServerCacheConfigDS struct { + OrgServerFQDN string + Type tc.DSType +} + +func MakeServerCacheDotConfig( + serverName tc.CacheName, + toToolName string, // tm.toolname global parameter (TODO: cache itself?) + toURL string, // tm.url global parameter (TODO: cache itself?) + dses map[tc.DeliveryServiceName]ServerCacheConfigDS, +) string { + text := GenericHeaderComment(string(serverName), toToolName, toURL) + + seenOrigins := map[string]struct{}{} + for _, ds := range dses { + if ds.Type != tc.DSTypeHTTPNoCache { + continue + } + if _, ok := seenOrigins[ds.OrgServerFQDN]; ok { + continue + } + seenOrigins[ds.OrgServerFQDN] = struct{}{} + + originFQDN, originPort := GetOriginFQDNAndPort(ds.OrgServerFQDN) + if originPort != nil { + text += `dest_domain=` + originFQDN + ` port=` + strconv.Itoa(*originPort) + ` scheme=http action=never-cache` + "\n" + } else { + text += `dest_domain=` + originFQDN + ` scheme=http action=never-cache` + "\n" + } + } + return text +} + +// TODO unit test +func GetOriginFQDNAndPort(origin string) (string, *int) { + origin = strings.TrimSpace(origin) + origin = strings.Replace(origin, `https://`, ``, -1) + origin = strings.Replace(origin, `http://`, ``, -1) + + // if the origin includes a path, strip it + + slashI := strings.Index(origin, `/`) + if slashI != -1 { + origin = origin[:slashI] + } + + hostName := origin + + colonI := strings.Index(origin, ":") + if colonI == -1 { + return hostName, nil // no :, the origin must not include a port + } + portStr := origin[colonI+1:] + port, err := strconv.Atoi(portStr) + if err != nil { + // either the port isn't an integer, or the : we found was something else. + // Return the origin, as if it didn't contain a port. + return hostName, nil + } + + hostName = origin[:colonI] + return hostName, Review comment: I can't help but wonder if the go standard library would do a better job of parsing the URI here. This will probably be fine, and it's probably just like what the perl did. But it seems that we're re-inventing a wheel that can be surprisingly hard to get right at times. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3899: Add TO Go server cache configs
alficles commented on a change in pull request #3899: Add TO Go server cache configs URL: https://github.com/apache/trafficcontrol/pull/3899#discussion_r321566036 ## File path: lib/go-atscfg/ipallowdotconfig.go ## @@ -0,0 +1,253 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "net" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/lib/go-util" +) + +const IPAllowConfigFileName = `ip_allow.config` + +type IPAllowData struct { + Srcstring + Action string + Method string +} + +const ParamPurgeAllowIP = "purge_allow_ip" +const ParamCoalesceMaskLenV4 = "coalesce_masklen_v4" +const ParamCoalesceNumberV4 = "coalesce_number_v4" +const ParamCoalesceMaskLenV6 = "coalesce_masklen_v6" +const ParamCoalesceNumberV6 = "coalesce_number_v6" + +type IPAllowServer struct { + IPAddress string + IP6Address string +} + +const DefaultCoalesceMaskLenV4 = 24 +const DefaultCoalesceNumberV4 = 5 +const DefaultCoalesceMaskLenV6 = 48 +const DefaultCoalesceNumberV6 = 5 + +// MakeIPAllowDotConfig creates the ip_allow.config ATS config file. +// The childServers is a list of servers which are children for this Mid-tier server. This should be empty for Edge servers. +// More specifically, it should be the list of edges whose cachegroup's parent_cachegroup or secondary_parent_cachegroup is the cachegroup of this Mid server. +func MakeIPAllowDotConfig( + serverName tc.CacheName, + serverType tc.CacheType, + toToolName string, // tm.toolname global parameter (TODO: cache itself?) + toURL string, // tm.url global parameter (TODO: cache itself?) + params map[string][]string, // map[name]value - config file should always be ip_allow.config + childServers map[tc.CacheName]IPAllowServer, +) string { + ipAllowData := []IPAllowData{} + const ActionAllow = "ip_allow" + const ActionDeny = "ip_deny" + const MethodAll = "ALL" + + // localhost is trusted. + ipAllowData = append(ipAllowData, IPAllowData{ + Src:`127.0.0.1`, + Action: ActionAllow, + Method: MethodAll, + }) + ipAllowData = append(ipAllowData, IPAllowData{ + Src:`::1`, + Action: ActionAllow, + Method: MethodAll, + }) + + // default for coalesce_ipv4 = 24, 5 and for ipv6 48, 5; override with the parameters in the server profile. + coalesceMaskLenV4 := DefaultCoalesceMaskLenV4 + coalesceNumberV4 := DefaultCoalesceNumberV4 + coalesceMaskLenV6 := DefaultCoalesceMaskLenV6 + coalesceNumberV6 := DefaultCoalesceNumberV6 + + for name, vals := range params { + for _, val := range vals { + switch name { + case "purge_allow_ip": + ipAllowData = append(ipAllowData, IPAllowData{ + Src:val, + Action: ActionAllow, + Method: MethodAll, + }) + case ParamCoalesceMaskLenV4: + if vi, err := strconv.Atoi(val); err != nil { + log.Warnln("MakeIPAllowDotConfig got param '" + name + "' val '" + val + "' not a number, ignoring!") + } else if coalesceMaskLenV4 != DefaultCoalesceMaskLenV4 { + log.Warnln("MakeIPAllowDotConfig got multiple param '" + name + "' - ignoring val '" + val + "'!") + } else { + coalesceMaskLenV4 = vi + } + case ParamCoalesceNumberV4: + if vi, err := strconv.Atoi(val); err != nil { + log.Warnln("MakeIPAllowDotConfig got param '" + name + "' val '" + val + "' not a number, ignoring!") +
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs
alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs URL: https://github.com/apache/trafficcontrol/pull/3875#discussion_r321472119 ## File path: lib/go-atscfg/atscfg.go ## @@ -0,0 +1,114 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "strconv" + "strings" + "time" + + "github.com/apache/trafficcontrol/lib/go-tc" +) + +// TODO move in config gen PR +const InvalidID = -1 + +const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006" + +type ServerInfo struct { + CacheGroupID int + CDN tc.CDNName + CDNID int + DomainNamestring + HostName string + IDint + IPstring + ParentCacheGroupIDint + ParentCacheGroupType string + ProfileID ProfileID + ProfileName string + Port int + SecondaryParentCacheGroupID int + SecondaryParentCacheGroupType string + Type string +} + +func (s *ServerInfo) IsTopLevelCache() bool { + return (s.ParentCacheGroupType == tc.CacheGroupOriginTypeName || s.ParentCacheGroupID == InvalidID) && + (s.SecondaryParentCacheGroupType == tc.CacheGroupOriginTypeName || s.SecondaryParentCacheGroupID == InvalidID) +} + +func HeaderCommentWithTOVersionStr(name string, nameVersionStr string) string { + return "# DO NOT EDIT - Generated for " + name + " by " + nameVersionStr + " on " + time.Now().Format(HeaderCommentDateFormat) + "\n" +} + +func GetNameVersionStringFromToolNameAndURL(toolName string, url string) string { + return toolName + " (" + url + ")" +} + +func GenericHeaderComment(name string, toolName string, url string) string { + return HeaderCommentWithTOVersionStr(name, GetNameVersionStringFromToolNameAndURL(toolName, url)) +} + +// GetATSMajorVersionFromATSVersion returns the major version of the given profile's package trafficserver parameter. +// The atsVersion is typically a Parameter on the Server's Profile, with the configFile "package" name "trafficserver". +// Returns an error if atsVersion is empty or not a number. +func GetATSMajorVersionFromATSVersion(atsVersion string) (int, error) { + if len(atsVersion) == 0 { + return 0, errors.New("ats version missing") + } + atsMajorVer, err := strconv.Atoi(atsVersion[:1]) Review comment: I am totally aware that this is exactly how perl does it, because I objected when I saw the perl version as well. But this would be a delightful opportunity to not re-write the bug that is going to happen when ATS 10 is released... This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs
alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs URL: https://github.com/apache/trafficcontrol/pull/3875#discussion_r321563447 ## File path: traffic_ops/traffic_ops_golang/ats/atscdn/atscdn.go ## @@ -0,0 +1,152 @@ +package atscdn + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "errors" + "net/http" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/ats" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/config" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" +) + +// GenericProfileConfig generates a generic profile config text, from the profile's parameters with the given config file name. +// This does not include a header comment, because a generic config may not use a number sign as a comment. +// If you need a header comment, it can be added manually via ats.HeaderComment, or automatically with WithProfileDataHdr. +func GenericProfileConfig(tx *sql.Tx, profile ats.ProfileData, fileName string, separator string) (string, error) { + profileParamData, err := ats.GetProfileParamData(tx, profile.ID, fileName) + if err != nil { + return "", errors.New("getting profile param data: " + err.Error()) + } + text := "" + for name, val := range profileParamData { + name = trimParamUnderscoreNumSuffix(name) + text += name + separator + val + "\n" + } + return text, nil +} + +// trimParamUnderscoreNumSuffix removes any trailing "__[0-9]+" and returns the trimmed string. +func trimParamUnderscoreNumSuffix(paramName string) string { + underscorePos := strings.LastIndex(paramName, `__`) + if underscorePos == -1 { + return paramName + } + if _, err := strconv.ParseFloat(paramName[underscorePos+2:], 64); err != nil { Review comment: That's fine, then. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs
alficles commented on a change in pull request #3875: Add TO Go ATS CDN configs URL: https://github.com/apache/trafficcontrol/pull/3875#discussion_r321562245 ## File path: lib/go-atscfg/atscfg.go ## @@ -0,0 +1,114 @@ +package atscfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "strconv" + "strings" + "time" + + "github.com/apache/trafficcontrol/lib/go-tc" +) + +// TODO move in config gen PR +const InvalidID = -1 + +const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006" + +type ServerInfo struct { + CacheGroupID int + CDN tc.CDNName + CDNID int + DomainNamestring + HostName string + IDint + IPstring + ParentCacheGroupIDint + ParentCacheGroupType string + ProfileID ProfileID + ProfileName string + Port int + SecondaryParentCacheGroupID int + SecondaryParentCacheGroupType string + Type string +} + +func (s *ServerInfo) IsTopLevelCache() bool { + return (s.ParentCacheGroupType == tc.CacheGroupOriginTypeName || s.ParentCacheGroupID == InvalidID) && + (s.SecondaryParentCacheGroupType == tc.CacheGroupOriginTypeName || s.SecondaryParentCacheGroupID == InvalidID) +} + +func HeaderCommentWithTOVersionStr(name string, nameVersionStr string) string { + return "# DO NOT EDIT - Generated for " + name + " by " + nameVersionStr + " on " + time.Now().Format(HeaderCommentDateFormat) + "\n" +} + +func GetNameVersionStringFromToolNameAndURL(toolName string, url string) string { + return toolName + " (" + url + ")" +} + +func GenericHeaderComment(name string, toolName string, url string) string { + return HeaderCommentWithTOVersionStr(name, GetNameVersionStringFromToolNameAndURL(toolName, url)) +} + +// GetATSMajorVersionFromATSVersion returns the major version of the given profile's package trafficserver parameter. +// The atsVersion is typically a Parameter on the Server's Profile, with the configFile "package" name "trafficserver". +// Returns an error if atsVersion is empty or not a number. +func GetATSMajorVersionFromATSVersion(atsVersion string) (int, error) { + if len(atsVersion) == 0 { + return 0, errors.New("ats version missing") + } + atsMajorVer, err := strconv.Atoi(atsVersion[:1]) + if err != nil { + return 0, errors.New("ats version parameter '" + atsVersion + "' is not a number") + } + return atsMajorVer, nil +} + +type DeliveryServiceID int +type ProfileID int +type ServerID int + +// GenericProfileConfig generates a generic profile config text, from the profile's parameters with the given config file name. +// This does not include a header comment, because a generic config may not use a number sign as a comment. +// If you need a header comment, it can be added manually via ats.HeaderComment, or automatically with WithProfileDataHdr. +func GenericProfileConfig( + paramData map[string]string, // GetProfileParamData(tx, profileID, fileName) + separator string, +) string { + text := "" + for name, val := range paramData { + name = trimParamUnderscoreNumSuffix(name) + text += name + separator + val + "\n" + } + return text +} + +// trimParamUnderscoreNumSuffix removes any trailing "__[0-9]+" and returns the trimmed string. +func trimParamUnderscoreNumSuffix(paramName string) string { + underscorePos := strings.LastIndex(paramName, `__`) + if underscorePos == -1 { + return paramName + } + if _, err := strconv.ParseFloat(paramName[underscorePos+2:], 64); err != nil { Review comment: I think changing it would be better. If someone _did_ start using 1.1, they might be confused as to why 1.1.4 didn't work the way they expected. And why `name__03E7` works but `name__03D7` doesn't work
[GitHub] [trafficcontrol] asf-ci commented on issue #3905: Rewrite /cachegroups/{id}/unassigned_parameters to golang
asf-ci commented on issue #3905: Rewrite /cachegroups/{id}/unassigned_parameters to golang URL: https://github.com/apache/trafficcontrol/pull/3905#issuecomment-528648428 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4206/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] mhoppa opened a new pull request #3905: Rewrite /cachegroups/{id}/unassigned_parameters to golang
mhoppa opened a new pull request #3905: Rewrite /cachegroups/{id}/unassigned_parameters to golang URL: https://github.com/apache/trafficcontrol/pull/3905 ## What does this PR (Pull Request) do? This PR rewrites the /cachegroups/{id}/unassigned_parameters endpoint to Go - [x] This PR fixes #3807 ## Which Traffic Control components are affected by this PR? - Documentation - Traffic Ops ## What is the best way to verify this PR? Bring up TO locally and hit the endpoint to ensure parity between perl implementation and newly written Golang implementation The API tests and/or unit tests for TO can be run to verify the PR. ## If this is a bug fix, what versions of Traffic Control are affected? ## The following criteria are ALL met by this PR - [x] This PR includes tests - [x] This PR includes documentation - [x] This PR includes an update to CHANGELOG.md - [x] This PR includes any and all required license headers - [x] This PR does not include a database migration - [x] This PR **DOES NOT FIX A SERIOUS SECURITY VULNERABILITY** (see [the Apache Software Foundation's security guidelines](https://www.apache.org/security/) for details) This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] asf-ci commented on issue #3903: Rewrote /user/logout to Go
asf-ci commented on issue #3903: Rewrote /user/logout to Go URL: https://github.com/apache/trafficcontrol/pull/3903#issuecomment-528490839 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4205/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] lbathina opened a new issue #3904: Revalidate should be specific to the DS for which the invalidation request for submitted.
lbathina opened a new issue #3904: Revalidate should be specific to the DS for which the invalidation request for submitted. URL: https://github.com/apache/trafficcontrol/issues/3904 ## I'm submitting a ... - [ ] bug report - [ ] new feature / enhancement request - [X] improvement request (usability, performance, tech debt, etc.) - [ ] other ## Traffic Control components affected ... - [ ] CDN in a Box - [ ] Documentation - [ ] Grove - [ ] Traffic Control Client - [ ] Traffic Monitor - [X] Traffic Ops - [X] Traffic Ops ORT - [ ] Traffic Portal - [ ] Traffic Router - [ ] Traffic Stats - [ ] Traffic Vault - [ ] unknown ## Current behavior: When a invalidation request is submitted for a specific DS, currently the request is processed on all the caches at CDN level ## Expected / new behavior: we want the request to be scoped to the DS level. That is, execute the invalidate request for caches on DS that is in the request ## Minimal reproduction of the problem with instructions: ## Anything else: This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 opened a new pull request #3903: Rewrote /user/logout to Go
ocket opened a new pull request #3903: Rewrote /user/logout to Go URL: https://github.com/apache/trafficcontrol/pull/3903 ## What does this PR (Pull Request) do? - [x] This PR fixes #3841 Rewrote /user/logout to Go ## Which Traffic Control components are affected by this PR? - Traffic Ops functionality is not changing, so no docs ## What is the best way to verify this PR? Authenticate as normal, then just logout. Normally, this would be done in Traffic Portal through the button on the sidebar. Then just observe that you're not authenticated - no endpoints (except `/ping` ) should respond with anything other than an "Unauthenticated" error ## The following criteria are ALL met by this PR - [x] This PR includes tests - [x] I have explained why documentation is unnecessary - [x] This PR includes an update to CHANGELOG.md - [x] This PR includes any and all required license headers - [x] This PR does not include a database migration - [x] This PR **DOES NOT FIX A SERIOUS SECURITY VULNERABILITY** This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] smalenfant commented on issue #3764: Specify TLS version support in Traffic Router server.xml
smalenfant commented on issue #3764: Specify TLS version support in Traffic Router server.xml URL: https://github.com/apache/trafficcontrol/issues/3764#issuecomment-528457421 We were able to change TLS protocols in Traffic Router 2.2 and below. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] lbathina opened a new issue #3902: TP: internal server error on dashboard page
lbathina opened a new issue #3902: TP: internal server error on dashboard page URL: https://github.com/apache/trafficcontrol/issues/3902 ## I'm submitting a ... - [X] bug report - [ ] new feature / enhancement request - [ ] improvement request (usability, performance, tech debt, etc.) - [ ] other ## Traffic Control components affected ... - [ ] CDN in a Box - [ ] Documentation - [ ] Grove - [ ] Traffic Control Client - [ ] Traffic Monitor - [X] Traffic Ops - [ ] Traffic Ops ORT - [X] Traffic Portal - [ ] Traffic Router - [ ] Traffic Stats - [ ] Traffic Vault - [ ] unknown ## Current behavior: On login or navigation to Dashboard page. Internal server error is displayed. end point api/1.4/cdns/capacity returns 500 Internal server error ## Expected / new behavior: No error should be displayed. ## Minimal reproduction of the problem with instructions: ## Anything else: This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] asf-ci commented on issue #3534: WIP - TP Delivery Service Generate SSL update, new letsencrypt generate and…
asf-ci commented on issue #3534: WIP - TP Delivery Service Generate SSL update, new letsencrypt generate and… URL: https://github.com/apache/trafficcontrol/pull/3534#issuecomment-528440029 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4204/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] asf-ci commented on issue #3534: WIP - TP Delivery Service Generate SSL update, new letsencrypt generate and…
asf-ci commented on issue #3534: WIP - TP Delivery Service Generate SSL update, new letsencrypt generate and… URL: https://github.com/apache/trafficcontrol/pull/3534#issuecomment-528425594 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4203/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 merged pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go
ocket merged pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go URL: https://github.com/apache/trafficcontrol/pull/3900 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 closed issue #3806: Rewrite /cachegroups/{{id}}/parameters to Go
ocket closed issue #3806: Rewrite /cachegroups/{{id}}/parameters to Go URL: https://github.com/apache/trafficcontrol/issues/3806 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go
ocket commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go URL: https://github.com/apache/trafficcontrol/pull/3900#discussion_r321331054 ## File path: traffic_ops/traffic_ops_golang/cachegroup/parameters.go ## @@ -0,0 +1,111 @@ +package cachegroup + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "net/http" + "strconv" + + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/lib/go-util" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/parameter" +) + +const ( + CacheGroupIDQueryParam = "id" + ParameterIDQueryParam = "parameterId" +) + +//we need a type alias to define functions on +type TOCacheGroupParameter struct { + api.APIInfoImpl `json:"-"` + tc.CacheGroupParameterNullable +} + +func (cgparam *TOCacheGroupParameter) ParamColumns() map[string]dbhelpers.WhereColumnInfo { + return map[string]dbhelpers.WhereColumnInfo{ + CacheGroupIDQueryParam: dbhelpers.WhereColumnInfo{"cgp.cachegroup", api.IsInt}, + ParameterIDQueryParam: dbhelpers.WhereColumnInfo{"p.id", api.IsInt}, + } +} + +func (cgparam *TOCacheGroupParameter) GetType() string { + return "cachegroup_params" +} + +func (cgparam *TOCacheGroupParameter) Read() ([]interface{}, error, error, int) { + queryParamsToQueryCols := cgparam.ParamColumns() + parameters := cgparam.APIInfo().Params + where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(parameters, queryParamsToQueryCols) + if len(errs) > 0 { + return nil, util.JoinErrs(errs), nil, http.StatusBadRequest + } + + cgID, err := strconv.Atoi(parameters[CacheGroupIDQueryParam]) Review comment: You're right, I just assumed it was doing that because it was parsing it. That's actually happening above in `dbhelpers.BuildWhereAndOrderByAndPagination`, which doesn't appear to store the converted value anywhere... It's a real shame we can't just do that parse once. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] asf-ci commented on issue #3901: Fixed runtime panic when creating roles, added docs
asf-ci commented on issue #3901: Fixed runtime panic when creating roles, added docs URL: https://github.com/apache/trafficcontrol/pull/3901#issuecomment-528409520 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4202/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] mhoppa commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go
mhoppa commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go URL: https://github.com/apache/trafficcontrol/pull/3900#discussion_r321302732 ## File path: traffic_ops/traffic_ops_golang/cachegroup/parameters.go ## @@ -0,0 +1,111 @@ +package cachegroup + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "net/http" + "strconv" + + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/lib/go-util" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/parameter" +) + +const ( + CacheGroupIDQueryParam = "id" + ParameterIDQueryParam = "parameterId" +) + +//we need a type alias to define functions on +type TOCacheGroupParameter struct { + api.APIInfoImpl `json:"-"` + tc.CacheGroupParameterNullable +} + +func (cgparam *TOCacheGroupParameter) ParamColumns() map[string]dbhelpers.WhereColumnInfo { + return map[string]dbhelpers.WhereColumnInfo{ + CacheGroupIDQueryParam: dbhelpers.WhereColumnInfo{"cgp.cachegroup", api.IsInt}, + ParameterIDQueryParam: dbhelpers.WhereColumnInfo{"p.id", api.IsInt}, + } +} + +func (cgparam *TOCacheGroupParameter) GetType() string { + return "cachegroup_params" +} + +func (cgparam *TOCacheGroupParameter) Read() ([]interface{}, error, error, int) { + queryParamsToQueryCols := cgparam.ParamColumns() + parameters := cgparam.APIInfo().Params + where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(parameters, queryParamsToQueryCols) + if len(errs) > 0 { + return nil, util.JoinErrs(errs), nil, http.StatusBadRequest + } + + cgID, err := strconv.Atoi(parameters[CacheGroupIDQueryParam]) Review comment: Not in the generic case when using ReadHandler -> https://github.com/apache/trafficcontrol/blob/master/traffic_ops/traffic_ops_golang/api/shared_handlers.go#L122 it passes in nil into NewAPI for int params This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] asf-ci commented on issue #3901: Fixed runtime panic when creating roles, added docs
asf-ci commented on issue #3901: Fixed runtime panic when creating roles, added docs URL: https://github.com/apache/trafficcontrol/pull/3901#issuecomment-528382522 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/trafficcontrol-PR/4201/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go
ocket commented on a change in pull request #3900: Rewrite /cachegroups/{{id}}/parameters to Go URL: https://github.com/apache/trafficcontrol/pull/3900#discussion_r321275087 ## File path: traffic_ops/traffic_ops_golang/cachegroup/parameters.go ## @@ -0,0 +1,111 @@ +package cachegroup + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "errors" + "net/http" + "strconv" + + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/apache/trafficcontrol/lib/go-util" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/parameter" +) + +const ( + CacheGroupIDQueryParam = "id" + ParameterIDQueryParam = "parameterId" +) + +//we need a type alias to define functions on +type TOCacheGroupParameter struct { + api.APIInfoImpl `json:"-"` + tc.CacheGroupParameterNullable +} + +func (cgparam *TOCacheGroupParameter) ParamColumns() map[string]dbhelpers.WhereColumnInfo { + return map[string]dbhelpers.WhereColumnInfo{ + CacheGroupIDQueryParam: dbhelpers.WhereColumnInfo{"cgp.cachegroup", api.IsInt}, + ParameterIDQueryParam: dbhelpers.WhereColumnInfo{"p.id", api.IsInt}, + } +} + +func (cgparam *TOCacheGroupParameter) GetType() string { + return "cachegroup_params" +} + +func (cgparam *TOCacheGroupParameter) Read() ([]interface{}, error, error, int) { + queryParamsToQueryCols := cgparam.ParamColumns() + parameters := cgparam.APIInfo().Params + where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(parameters, queryParamsToQueryCols) + if len(errs) > 0 { + return nil, util.JoinErrs(errs), nil, http.StatusBadRequest + } + + cgID, err := strconv.Atoi(parameters[CacheGroupIDQueryParam]) Review comment: This is actually handled for you - the "id" in the path is parsed into the `APIInfo` structure's `IntParams` map. You can test that by just requesting `/cachegroups/notanumber/parameters`, which gives back the error "id cannot parse to integer"; it never hits this error handling. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [trafficcontrol] ocket8888 opened a new pull request #3901: Fixed runtime panic when creating roles, added docs
ocket opened a new pull request #3901: Fixed runtime panic when creating roles, added docs URL: https://github.com/apache/trafficcontrol/pull/3901 ## What does this PR (Pull Request) do? - [x] This PR fixes #3891 ## Which Traffic Control components are affected by this PR? - Documentation - Traffic Ops ## What is the best way to verify this PR? Verify existing tests pass. Build and run Traffic Ops/DB/Portal (CDN-in-a-Box recommended) and verify that admin users can create roles of privilege levels less than or equal to 30 (admin's privilege level) successfully, and no erroneous error message is reported. Build and read the documentation to ensure that it is accurate, comprehensible, and free of guideline violations and/or spelling/grammar mistakes. ## If this is a bug fix, what versions of Traffic Control are affected? - master (c18d535) - 3.0.1 - 3.0.2 (RC2) - 3.1.0 (pre-release candidate) ## The following criteria are ALL met by this PR - [x] This PR adds no additional tests, but tests for the endpoint already exist - [x] This PR includes documentation - [x] An update to CHANGELOG.md is not necessary - [x] This PR includes any and all required license headers - [x] This PR does not include a database migration - [x] This PR **DOES NOT FIX A SERIOUS SECURITY VULNERABILITY** This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services