Vijay-1 closed pull request #1908: Changes for Backup Edge Cache Group URL: https://github.com/apache/incubator-trafficcontrol/pull/1908
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f77b2990..cd65a168c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,8 +31,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - /api/1.3/statuses (R) - /api/1.3/system/info - /api/1.3/types (R) +- Backup Edge Cache group: Backup Edge group for a particular cache group can be configured using coverage zone files. With this, users will be able control traffic within portions of their network, thereby avoiding choosing fallback cache groups using geo coordinates(getClosestAvailableCachegroup). This can be controlled using "backupZones" which contains configuration for backup cache groups parsed from the Coverage Zone File as explained [here](http://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_ops/using.html#the-coverage-zone-file-and-asn-table) ### Changed - Reformatted this CHANGELOG file to the keep-a-changelog format -[Unreleased]: https://github.com/apache/incubator-trafficcontrol/compare/RELEASE-2.1.0...HEAD +[Unreleased]: https://github.com/apache/incubator-trafficcontrol/compare/RELEASE-2.1.0...HEAD + diff --git a/docs/source/admin/traffic_ops/using.rst b/docs/source/admin/traffic_ops/using.rst index b5849753e..18c5de75d 100644 --- a/docs/source/admin/traffic_ops/using.rst +++ b/docs/source/admin/traffic_ops/using.rst @@ -873,6 +873,24 @@ The Coverage Zone File (CZF) should contain a cachegroup name to network prefix "192.168.4.0/24", "192.168.5.0/24" ] + }, + "cache-group-03": { + "backupZones":{ + "list": ["cache-group-02"], + "fallbackToClosestGroup": true + }, + "coordinates": { + "latitude": 3.3, + "longitude": 4.4 + }, + "network6": [ + "1234:566a::/64", + "1234:566b::/64" + ], + "network": [ + "192.168.2.0/24", + "192.168.3.0/24" + ] } } } @@ -881,7 +899,7 @@ The CZF is an input to the Traffic Control CDN, and as such does not get generat The script that generates the CZF file is not part of Traffic Control, since it is different for each situation. -.. note:: The ``"coordinates"`` section is optional and may be used by Traffic Router for localization in the case of a CZF "hit" where the zone name does not map to a Cache Group name in Traffic Ops (i.e. Traffic Router will route to the closest Cache Group(s) geographically). +.. note:: The ``"coordinates"`` section is optional and may be used by Traffic Router for localization in the case of a CZF "hit" where the zone name does not map to a Cache Group name in Traffic Ops (i.e. Traffic Router will route to the closest Cache Group(s) geographically). The ``"backupZones"`` section is optional and is used by Traffic Router for localization in the case of a CZF "hit" when there are no caches available for that DS in the matched cache group. This backup "list" contains an ordered list of backup cache groups to choose from if the matched cache group has no caches available for a requested DS. If an available cache cannot be found in any of the backup groups either, the "fallbackToClosestGroup" flag determines the Traffic Router's following behavior. If true, Traffic Router will find the next closest cache group with available caches. If false (the default), Traffic Router will bypass the request (if configured) or reject it. .. _rl-deep-czf: diff --git a/docs/source/admin/traffic_router.rst b/docs/source/admin/traffic_router.rst index c1f33c8a4..8b9de6619 100644 --- a/docs/source/admin/traffic_router.rst +++ b/docs/source/admin/traffic_router.rst @@ -246,6 +246,8 @@ Fields Always Present +--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ |GEO_NO_CACHE_FOUND |Traffic Router could not find a resource via geolocation data based on the requesting client's geolocation | +--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +|DS_CZ_BACKUP_CG |Traffic Router found a cache from backup cache groups configured via backupZones / coordinates in Coverage zone data | ++--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ --------------- diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/NetworkNode.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/NetworkNode.java index 9d292aeb2..191f8fa1f 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/NetworkNode.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/NetworkNode.java @@ -49,6 +49,8 @@ private String loc; private CacheLocation cacheLocation = null; private Geolocation geolocation = null; + private List<String> backupCacheGroups = null; + private boolean useClosestOnBackupFailure = false; protected Map<NetworkNode,NetworkNode> children; private Set<String> deepCacheNames; @@ -106,7 +108,11 @@ public static NetworkNode generateTree(final JsonNode json, final boolean verify final String loc = czIter.next(); final JsonNode locData = JsonUtils.getJsonNode(coverageZones, loc); final JsonNode coordinates = locData.get("coordinates"); + final JsonNode backupConfigJson = locData.get("backupZones"); + boolean useClosestOnBackupFailure = true; + Geolocation geolocation = null; + List<String> backupCacheGroups = null; if (coordinates != null && coordinates.has("latitude") && coordinates.has("longitude")) { final double latitude = coordinates.get("latitude").asDouble(); @@ -114,7 +120,17 @@ public static NetworkNode generateTree(final JsonNode json, final boolean verify geolocation = new Geolocation(latitude, longitude); } - if (!addNetworkNodesToRoot(root, loc, locData, geolocation, useDeep)) { + if (backupConfigJson != null) { + if (backupConfigJson.has("list")) { + backupCacheGroups = new ArrayList<>(); + for (final JsonNode cacheGroup : JsonUtils.getJsonNode(backupConfigJson, "list")) { + backupCacheGroups.add(cacheGroup.asText()); + } + } + useClosestOnBackupFailure = JsonUtils.optBoolean(backupConfigJson, "fallbackToClosestGroup", false); + } + + if (!addNetworkNodesToRoot(root, loc, locData, geolocation, backupCacheGroups, useDeep, useClosestOnBackupFailure)) { return null; } } @@ -137,8 +153,10 @@ public static NetworkNode generateTree(final JsonNode json, final boolean verify return null; } + + @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"}) private static boolean addNetworkNodesToRoot(final SuperNode root, final String loc, final JsonNode locData, - final Geolocation geolocation, final boolean useDeep) { + final Geolocation geolocation, final List<String> backupCacheGroups, final boolean useDeep, final boolean useClosestOnBackupFailure) { final CacheLocation deepLoc = new CacheLocation( "deep." + loc, geolocation != null ? geolocation : new Geolocation(0.0, 0.0)); // TODO JvD final Set<String> cacheNames = parseDeepCacheNames(locData); @@ -146,9 +164,8 @@ private static boolean addNetworkNodesToRoot(final SuperNode root, final String try { for (final JsonNode network : JsonUtils.getJsonNode(locData, key)) { final String ip = network.asText(); - try { - final NetworkNode nn = new NetworkNode(ip, loc, geolocation); + final NetworkNode nn = new NetworkNode(ip, loc, geolocation, backupCacheGroups, useClosestOnBackupFailure); if (useDeep) { // For a deep NetworkNode, we set the CacheLocation here without any Caches. // The deep Caches will be lazily loaded in getCoverageZoneCacheLocation() where we have @@ -198,12 +215,14 @@ public NetworkNode(final String str) throws NetworkNodeException { } public NetworkNode(final String str, final String loc) throws NetworkNodeException { - this(str, loc, null); + this(str, loc, null, null, false); } - public NetworkNode(final String str, final String loc, final Geolocation geolocation) throws NetworkNodeException { + public NetworkNode(final String str, final String loc, final Geolocation geolocation, final List<String> backupCacheGroups, final boolean useClosestOnBackupFailure) throws NetworkNodeException { this.loc = loc; this.geolocation = geolocation; + this.backupCacheGroups = backupCacheGroups; + this.useClosestOnBackupFailure = useClosestOnBackupFailure; cidrAddress = CidrAddress.fromString(str); } @@ -281,6 +300,14 @@ public Geolocation getGeolocation() { return geolocation; } + public List<String> getBackupCacheGroups() { + return backupCacheGroups; + } + + public boolean isUseClosest() { + return useClosestOnBackupFailure; + } + public CacheLocation getCacheLocation() { return cacheLocation; } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java index b444cd82a..60918f832 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java @@ -116,7 +116,7 @@ public void setRegionalAlternateCount(final int regionalAlternateCount) { public enum ResultDetails { NO_DETAILS, DS_NOT_FOUND, DS_TLS_MISMATCH, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CLIENT_GEO_UNSUPPORTED, GEO_NO_CACHE_FOUND, - REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE + REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE, DS_CZ_BACKUP_CG } long time; @@ -130,6 +130,8 @@ public void setRegionalAlternateCount(final int regionalAlternateCount) { boolean isClientGeolocationQueried; RegionalGeoResult regionalGeoResult; + boolean fromBackupCzGroup; + boolean useNextClosest = true; public Track() { start(); @@ -185,6 +187,22 @@ public RegionalGeoResult getRegionalGeoResult() { return regionalGeoResult; } + public void setFromBackupCzGroup(final boolean fromBackupCzGroup) { + this.fromBackupCzGroup = fromBackupCzGroup; + } + + public boolean isFromBackupCzGroup() { + return fromBackupCzGroup; + } + + public void setUseNextClosest(final boolean useNextClosest) { + this.useNextClosest = useNextClosest; + } + + public boolean isUseNextClosest() { + return useNextClosest; + } + public final void start() { time = System.currentTimeMillis(); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java index 9662fd8ca..a5307b97e 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java @@ -265,8 +265,9 @@ public Geolocation getLocation(final String clientIP, final DeliveryService deli protected List<Cache> selectCaches(final HTTPRequest request, final DeliveryService ds, final Track track) throws GeolocationException { CacheLocation cacheLocation; ResultType result = ResultType.CZ; + final boolean useDeep = (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS); - if (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS) { + if (useDeep) { // Deep caching is enabled. See if there are deep caches available cacheLocation = getDeepCoverageZoneCacheLocation(request.getClientIP(), ds); if (cacheLocation != null && cacheLocation.getCaches().size() != 0) { @@ -278,7 +279,7 @@ public Geolocation getLocation(final String clientIP, final DeliveryService deli } } else { // Deep caching not enabled for this Delivery Service; use the regular CZ - cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, track, useDeep); } List<Cache>caches = selectCachesByCZ(ds, cacheLocation, track, result); @@ -295,7 +296,8 @@ public Geolocation getLocation(final String clientIP, final DeliveryService deli track.setResult(ResultType.MISS); track.setResultDetails(ResultDetails.DS_CZ_ONLY); } - } else { + } else if (track.isUseNextClosest()) { + //Even with Geo limit none, TR wont do geo look-up, if fallback is disabled via backupZones configuration caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); } @@ -303,7 +305,6 @@ public Geolocation getLocation(final String clientIP, final DeliveryService deli } public List<Cache> selectCachesByGeo(final String clientIp, final DeliveryService deliveryService, final CacheLocation cacheLocation, final Track track) throws GeolocationException { - Geolocation clientLocation = null; try { @@ -365,7 +366,7 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws return result; } - final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, track, false); List<Cache> caches = selectCachesByCZ(ds, cacheLocation, track); if (caches != null) { @@ -394,7 +395,9 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws LOGGER.error("Bad client address: '" + request.getClientIP() + "'"); } - caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + if (track.isUseNextClosest()) { + caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + } if (caches != null) { track.setResult(ResultType.GEO); @@ -486,6 +489,9 @@ public Geolocation getClientLocation(final String clientIp, final DeliveryServic if (caches != null && track != null) { track.setResult(result); + if (track.isFromBackupCzGroup()) { + track.setResultDetails(ResultDetails.DS_CZ_BACKUP_CG); + } track.setResultLocation(cacheLocation.getGeolocation()); } @@ -655,11 +661,11 @@ protected NetworkNode getNetworkNode(final String ip) { } public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId) { - return getCoverageZoneCacheLocation(ip, deliveryServiceId, false); // default is not deep + return getCoverageZoneCacheLocation(ip, deliveryServiceId, null, false); // default is not deep } @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"}) - public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep) { + public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final Track track, final boolean useDeep) { final NetworkNode networkNode = useDeep ? getDeepNetworkNode(ip) : getNetworkNode(ip); if (networkNode == null) { @@ -695,17 +701,48 @@ public CacheLocation getCoverageZoneCacheLocation(final String ip, final String return cacheLocation; } - // We had a hit in the CZF but the name does not match a known cache location. - // Check whether the CZF entry has a geolocation and use it if so. - return getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + if (networkNode.getBackupCacheGroups() != null) { + for (final String cacheGroup : networkNode.getBackupCacheGroups()) { + final CacheLocation bkCacheLocation = getCacheRegister().getCacheLocationById(cacheGroup); + if (bkCacheLocation != null && !getSupportingCaches(bkCacheLocation.getCaches(), deliveryService).isEmpty()) { + LOGGER.debug("Got backup CZ cache group " + bkCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + } + return bkCacheLocation; + } + } + } + + CacheLocation closestCacheLocation = null; + + if (networkNode.isUseClosest()) { + // We had a hit in the CZF but the name does not match a known cache location. + // Check whether the CZF entry has a geolocation and use it if so. + closestCacheLocation = getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + if (closestCacheLocation != null) { + LOGGER.debug("Got closest CZ cache group " + closestCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + + } + return closestCacheLocation; + } + } else { + if (track != null) { + track.setUseNextClosest(false); + } + } + + return closestCacheLocation; } public CacheLocation getDeepCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { - return getCoverageZoneCacheLocation(ip, deliveryService, true); + return getCoverageZoneCacheLocation(ip, deliveryService, null, true); } - protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep) { - return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep); + protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final Track track, final boolean useDeep) { + return getCoverageZoneCacheLocation(ip, deliveryService.getId(), track, useDeep); } protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java index 0387c89c2..da2f5a4a2 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java @@ -76,7 +76,7 @@ public void before() throws Exception { cacheGroups.add(eastCacheGroup); cacheGroups.add(westCacheGroup); - NetworkNode eastNetworkNode = new NetworkNode("12.23.34.0/24", "east-cache-group", testLocation); + NetworkNode eastNetworkNode = new NetworkNode("12.23.34.0/24", "east-cache-group", testLocation, null, true); CacheRegister cacheRegister = mock(CacheRegister.class); @@ -88,7 +88,7 @@ public void before() throws Exception { trafficRouter = PowerMockito.mock(TrafficRouter.class); Whitebox.setInternalState(trafficRouter, "cacheRegister", cacheRegister); when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1")).thenCallRealMethod(); - when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false)).thenCallRealMethod(); + when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", null, false)).thenCallRealMethod(); when(trafficRouter.getCacheRegister()).thenReturn(cacheRegister); when(trafficRouter.orderCacheLocations(cacheGroups,testLocation)).thenCallRealMethod(); when(trafficRouter.getSupportingCaches(anyListOf(Cache.class), eq(deliveryService))).thenCallRealMethod(); ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
