This is an automated email from the ASF dual-hosted git repository.
rob pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit e829d1447c39ff8465c9f4ac75c6d19ed5205cc0
Author: Jan van Doorn <jan_vando...@comcast.com>
AuthorDate: Tue May 8 15:57:40 2018 -0600
add FreshFor
---
grove/plugin/README_http_cacheinspector.md | 23 ++---
grove/plugin/http_cacheinspector.go | 9 +-
grove/remap/rules.go | 146 ++---------------------------
grove/web/util.go | 144 ++++++++++++++++++++++++++++
4 files changed, 167 insertions(+), 155 deletions(-)
diff --git a/grove/plugin/README_http_cacheinspector.md
b/grove/plugin/README_http_cacheinspector.md
index e008e57..d9b1206 100644
--- a/grove/plugin/README_http_cacheinspector.md
+++ b/grove/plugin/README_http_cacheinspector.md
@@ -29,21 +29,18 @@ Jump to: disk my-disk-cache-two
*** Cache "" ***
- * Size of in use cache: 9.5M
- * Cache capacity: 9.5M
- * Number of elements in LRU: 169
+ * Size of in use cache: 1.5M
+ * Cache capacity: 9.5M
+ * Number of elements in LRU: 54
* Objects in cache sorted by Least Recently Used on top, showing only first
100 and last 100:
- # Code Size Age Key
- 00000 200 34K 16h19m9.135951506s
GET:http://localhost/34k.bin?hah
- 00001 200 35K 16h19m9.103181146s
GET:http://localhost/35k.bin?hah
- 00002 200 36K 16h19m9.079490592s
GET:http://localhost/36k.bin?hah
- 00003 200 37K 16h19m9.050852429s
GET:http://localhost/37k.bin?hah
- 00004 200 38K 16h19m9.020264008s
GET:http://localhost/38k.bin?hah
- 00005 200 39K 16h19m8.989722029s
GET:http://localhost/39k.bin?hah
- 00006 200 40K 16h19m8.967823063s
GET:http://localhost/40k.bin?hah
- 00007 200 41K 16h19m8.939422783s
GET:http://localhost/41k.bin?hah
- 00008 200 42K 16h19m8.913485123s
GET:http://localhost/42k.bin?h
+ # Code Size Age FreshFor
HitCount Key
+ 0 200 15K 600.922ms 50.541023s
1 GET:http://localhost/15k.bin?
+ 1 200 16K 627.929ms 50.540856s
1 GET:http://localhost/16k.bin?
+ 2 200 17K 650.813ms 50.540698s
1 GET:http://localhost/17k.bin?
+ 3 200 18K 671.732ms 50.540547s
1 GET:http://localhost/18k.bin?
+ 4 200 19K 691.981ms 50.540373s
1 GET:http://localhost/19k.bin?
+ 5 200 20K 715.566ms 50.540261s
1 GET:http://localhost/20k.bin?
```
diff --git a/grove/plugin/http_cacheinspector.go
b/grove/plugin/http_cacheinspector.go
index ee792b4..2cdb158 100644
--- a/grove/plugin/http_cacheinspector.go
+++ b/grove/plugin/http_cacheinspector.go
@@ -170,16 +170,17 @@ func cacheinspect(icfg interface{}, d OnRequestData) bool
{
w.Write([]byte(fmt.Sprintf("showing only first
%d and last %d:\n\n", head, tail)))
}
- w.Write([]byte(fmt.Sprintf("<b> # Code
Size Age HitCount Key</b>\n")))
+ w.Write([]byte(fmt.Sprintf("<b> # Code
Size Age FreshFor HitCount
Key</b>\n")))
for i, key := range keys {
- if (doSearch && !strings.Contains(key,
searchArr[0])) || !doSearch && (i > tail && i < len(keys)-head) {
+ if (doSearch && !strings.Contains(key,
searchArr[0])) || !doSearch && (i >= tail && i < len(keys)-head) {
continue
}
cacheObject, _ := d.Stats.CachePeek(key, cName)
age := time.Now().Sub(cacheObject.ReqRespTime)
- w.Write([]byte(fmt.Sprintf("
%8d%8d%10s%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\";>%s</a>\n",
- i, cacheObject.Code,
bytefmt.ByteSize(cacheObject.Size), age, cacheObject.HitCount, req.Host,
CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
+ freshFor :=
web.FreshFor(cacheObject.RespHeaders, cacheObject.RespCacheControl,
cacheObject.ReqRespTime, cacheObject.RespRespTime)
+ w.Write([]byte(fmt.Sprintf("
%8d%8d%10s%22v%22v%12d <a href=\"http://%s%s?key=%s&cache=%s\";>%s</a>\n",
+ i, cacheObject.Code,
bytefmt.ByteSize(cacheObject.Size), age, freshFor, cacheObject.HitCount,
req.Host, CacheStatsEndpoint, url.QueryEscape(key), cName, key)))
}
}
diff --git a/grove/remap/rules.go b/grove/remap/rules.go
index d98a04a..31ba715 100644
--- a/grove/remap/rules.go
+++ b/grove/remap/rules.go
@@ -15,9 +15,7 @@ package remap
*/
import (
- "math"
"net/http"
- "strconv"
"strings"
"time"
@@ -268,149 +266,21 @@ func fresh(
respReqTime time.Time,
respRespTime time.Time,
) bool {
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders,
respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
log.Debugf("Fresh: freshnesslifetime %v currentAge %v\n",
freshnessLifetime, currentAge)
fresh := freshnessLifetime > currentAge
return fresh
}
-// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds
from the given map (which is typically a `http.Header` or `CacheControl`.
Returns false if the given key doesn't exist in the map, or if the value isn't
a valid Delta Seconds per RFC2616§3.3.2.
-func getHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration,
bool) {
- maybeSeconds, ok := m[key]
- if !ok {
- return 0, false
- }
- if len(maybeSeconds) == 0 {
- return 0, false
- }
- maybeSec := maybeSeconds[0]
-
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// getHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds
from the given map (which is typically a `http.Header` or `CacheControl`.
Returns false if the given key doesn't exist in the map, or if the value isn't
a valid Delta Seconds per RFC2616§3.3.2.
-func getHTTPDeltaSecondsCacheControl(m map[string]string, key string)
(time.Duration, bool) {
- maybeSec, ok := m[key]
- if !ok {
- return 0, false
- }
- seconds, err := strconv.ParseUint(maybeSec, 10, 64)
- if err != nil {
- return 0, false
- }
- return time.Duration(seconds) * time.Second, true
-}
-
-// getFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
-func getFreshnessLifetime(respHeaders http.Header, respCacheControl
web.CacheControl) time.Duration {
- if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl,
"s-maxage"); ok {
- return s
- }
- if s, ok := getHTTPDeltaSecondsCacheControl(respCacheControl,
"max-age"); ok {
- return s
- }
-
- getExpires := func() (time.Duration, bool) {
- expires, ok := web.GetHTTPDate(respHeaders, "Expires")
- if !ok {
- return 0, false
- }
- date, ok := web.GetHTTPDate(respHeaders, "Date")
- if !ok {
- return 0, false
- }
- return expires.Sub(date), true
- }
- if s, ok := getExpires(); ok {
- return s
- }
- return heuristicFreshness(respHeaders)
-}
-
-const Day = time.Hour * time.Duration(24)
-
-// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns
the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they
exist, and 24 hours if they don't.
-// TODO: smarter and configurable heuristics
-func heuristicFreshness(respHeaders http.Header) time.Duration {
- sinceLastModified, ok := sinceLastModified(respHeaders)
- if !ok {
- return Day
- }
- freshness := time.Duration(math.Min(float64(Day),
float64(sinceLastModified)))
- return freshness
-}
-
-func sinceLastModified(headers http.Header) (time.Duration, bool) {
- lastModified, ok := web.GetHTTPDate(headers, "last-modified")
- if !ok {
- return 0, false
- }
- date, ok := web.GetHTTPDate(headers, "date")
- if !ok {
- return 0, false
- }
- return date.Sub(lastModified), true
-}
-
-// ageValue is used to calculate current_age per RFC7234§4.2.3
-func ageValue(respHeaders http.Header) time.Duration {
- s, ok := getHTTPDeltaSeconds(respHeaders, "age")
- if !ok {
- return 0
- }
- return s
-}
-
-// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns
time, or false if the response had no Date header (in violation of HTTP/1.1).
-func dateValue(respHeaders http.Header) (time.Time, bool) {
- return web.GetHTTPDate(respHeaders, "date")
-}
-
-func apparentAge(respHeaders http.Header, respRespTime time.Time)
time.Duration {
- dateValue, ok := dateValue(respHeaders)
- if !ok {
- return 0 // TODO log warning?
- }
- rawAge := respRespTime.Sub(dateValue)
- return time.Duration(math.Max(0.0, float64(rawAge)))
-}
-
-func responseDelay(respReqTime time.Time, respRespTime time.Time)
time.Duration {
- return respRespTime.Sub(respReqTime)
-}
-
-func correctedAgeValue(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
- return ageValue(respHeaders) + responseDelay(respReqTime, respRespTime)
-}
-
-func correctedInitialAge(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
- return time.Duration(math.Max(float64(apparentAge(respHeaders,
respRespTime)), float64(correctedAgeValue(respHeaders, respReqTime,
respRespTime))))
-}
-
-func residentTime(respRespTime time.Time) time.Duration {
- return time.Now().Sub(respRespTime)
-}
-
-func getCurrentAge(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
- correctedInitial := correctedInitialAge(respHeaders, respReqTime,
respRespTime)
- resident := residentTime(respRespTime)
- log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n",
correctedInitial, resident)
- return correctedInitial + resident
-}
-
// inMinFresh returns whether the given response is within the `min-fresh`
request directive. If no `min-fresh` directive exists in the request, `true` is
returned.
func inMinFresh(respHeaders http.Header, reqCacheControl web.CacheControl,
respCacheControl web.CacheControl, respReqTime time.Time, respRespTime
time.Time) bool {
- minFresh, ok := getHTTPDeltaSecondsCacheControl(reqCacheControl,
"min-fresh")
+ minFresh, ok := web.GetHTTPDeltaSecondsCacheControl(reqCacheControl,
"min-fresh")
if !ok {
return true // no min-fresh => within min-fresh
}
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders,
respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
inMinFresh := minFresh < (freshnessLifetime - currentAge)
log.Debugf("inMinFresh minFresh %v freshnessLifetime %v currentAge %v
=> %v < (%v - %v) = %v\n", minFresh, freshnessLifetime, currentAge, minFresh,
freshnessLifetime, currentAge, inMinFresh)
return inMinFresh
@@ -452,13 +322,13 @@ func allowedStale(respHeaders http.Header,
reqCacheControl web.CacheControl, res
// InMaxStale returns whether the given response is within the `max-stale`
request directive. If no `max-stale` directive exists in the request, `true` is
returned.
func inMaxStale(respHeaders http.Header, respCacheControl web.CacheControl,
respReqTime time.Time, respRespTime time.Time) bool {
- maxStale, ok := getHTTPDeltaSecondsCacheControl(respCacheControl,
"max-stale")
+ maxStale, ok := web.GetHTTPDeltaSecondsCacheControl(respCacheControl,
"max-stale")
if !ok {
// maxStale = 5 // debug
return true // no max-stale => within max-stale
}
- freshnessLifetime := getFreshnessLifetime(respHeaders, respCacheControl)
- currentAge := getCurrentAge(respHeaders, respReqTime, respRespTime)
+ freshnessLifetime := web.GetFreshnessLifetime(respHeaders,
respCacheControl)
+ currentAge := web.GetCurrentAge(respHeaders, respReqTime, respRespTime)
log.Errorf("DEBUGR InMaxStale maxStale %v freshnessLifetime %v
currentAge %v => %v > (%v, %v)\n", maxStale, freshnessLifetime, currentAge,
maxStale, currentAge, freshnessLifetime) // DEBUG
inMaxStale := maxStale > (currentAge - freshnessLifetime)
return inMaxStale
diff --git a/grove/web/util.go b/grove/web/util.go
index 4d36e08..23c7922 100644
--- a/grove/web/util.go
+++ b/grove/web/util.go
@@ -23,6 +23,8 @@ import (
"time"
"github.com/apache/incubator-trafficcontrol/lib/go-log"
+ "math"
+ "strconv"
)
type Hdr struct {
@@ -198,3 +200,145 @@ func ParseHTTPDate(d string) (time.Time, bool) {
// RemapTextKey is the plugin shared data key inserted by grovetccfg for the
Remap Line of the Delivery Service in Traffic Control, Traffic Ops.
const RemapTextKey = "remap_text"
+
+const Day = time.Hour * time.Duration(24)
+
+// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds
from the given map (which is typically a `http.Header` or `CacheControl`.
Returns false if the given key doesn't exist in the map, or if the value isn't
a valid Delta Seconds per RFC2616§3.3.2.
+func GetHTTPDeltaSeconds(m map[string][]string, key string) (time.Duration,
bool) {
+ maybeSeconds, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ if len(maybeSeconds) == 0 {
+ return 0, false
+ }
+ maybeSec := maybeSeconds[0]
+
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// GetHTTPDeltaSeconds is a helper function which gets an HTTP Delta Seconds
from the given map (which is typically a `http.Header` or `CacheControl`.
Returns false if the given key doesn't exist in the map, or if the value isn't
a valid Delta Seconds per RFC2616§3.3.2.
+func GetHTTPDeltaSecondsCacheControl(m map[string]string, key string)
(time.Duration, bool) {
+ maybeSec, ok := m[key]
+ if !ok {
+ return 0, false
+ }
+ seconds, err := strconv.ParseUint(maybeSec, 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return time.Duration(seconds) * time.Second, true
+}
+
+// HeuristicFreshness follows the recommendation of RFC7234§4.2.2 and returns
the min of 10% of the (Date - Last-Modified) headers and 24 hours, if they
exist, and 24 hours if they don't.
+// TODO: smarter and configurable heuristics
+func HeuristicFreshness(respHeaders http.Header) time.Duration {
+ sinceLastModified, ok := sinceLastModified(respHeaders)
+ if !ok {
+ return Day
+ }
+ freshness := time.Duration(math.Min(float64(Day),
float64(sinceLastModified)))
+ return freshness
+}
+
+func sinceLastModified(headers http.Header) (time.Duration, bool) {
+ lastModified, ok := GetHTTPDate(headers, "last-modified")
+ if !ok {
+ return 0, false
+ }
+ date, ok := GetHTTPDate(headers, "date")
+ if !ok {
+ return 0, false
+ }
+ return date.Sub(lastModified), true
+}
+
+// GetFreshnessLifetime calculates the freshness_lifetime per RFC7234§4.2.1
+func GetFreshnessLifetime(respHeaders http.Header, respCacheControl
CacheControl) time.Duration {
+ if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl,
"s-maxage"); ok {
+ return s
+ }
+ if s, ok := GetHTTPDeltaSecondsCacheControl(respCacheControl,
"max-age"); ok {
+ return s
+ }
+
+ getExpires := func() (time.Duration, bool) {
+ expires, ok := GetHTTPDate(respHeaders, "Expires")
+ if !ok {
+ return 0, false
+ }
+ date, ok := GetHTTPDate(respHeaders, "Date")
+ if !ok {
+ return 0, false
+ }
+ return expires.Sub(date), true
+ }
+ if s, ok := getExpires(); ok {
+ return s
+ }
+ return HeuristicFreshness(respHeaders)
+}
+
+// t6AgeValue is used to calculate current_age per RFC7234§4.2.3
+func AgeValue(respHeaders http.Header) time.Duration {
+ s, ok := GetHTTPDeltaSeconds(respHeaders, "age")
+ if !ok {
+ return 0
+ }
+ return s
+}
+
+func GetCurrentAge(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
+ correctedInitial := CorrectedInitialAge(respHeaders, respReqTime,
respRespTime)
+ resident := residentTime(respRespTime)
+ log.Debugf("getCurrentAge: correctedInitialAge %v residentTime %v\n",
correctedInitial, resident)
+ return correctedInitial + resident
+}
+
+func CorrectedInitialAge(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
+ return time.Duration(math.Max(float64(ApparentAge(respHeaders,
respRespTime)), float64(CorrectedAgeValue(respHeaders, respReqTime,
respRespTime))))
+}
+
+func CorrectedAgeValue(respHeaders http.Header, respReqTime time.Time,
respRespTime time.Time) time.Duration {
+ return AgeValue(respHeaders) + responseDelay(respReqTime, respRespTime)
+}
+
+func responseDelay(respReqTime time.Time, respRespTime time.Time)
time.Duration {
+ return respRespTime.Sub(respReqTime)
+}
+
+func residentTime(respRespTime time.Time) time.Duration {
+ return time.Now().Sub(respRespTime)
+}
+
+func ApparentAge(respHeaders http.Header, respRespTime time.Time)
time.Duration {
+ dateValue, ok := dateValue(respHeaders)
+ if !ok {
+ return 0 // TODO log warning?
+ }
+ rawAge := respRespTime.Sub(dateValue)
+ return time.Duration(math.Max(0.0, float64(rawAge)))
+}
+
+// dateValue is used to calculate current_age per RFC7234§4.2.3. It returns
time, or false if the response had no Date header (in violation of HTTP/1.1).
+func dateValue(respHeaders http.Header) (time.Time, bool) {
+ return GetHTTPDate(respHeaders, "date")
+}
+
+// FreshFor checks returns how long this object is still good for
+func FreshFor(
+ respHeaders http.Header,
+ respCacheControl CacheControl,
+ respReqTime time.Time,
+ respRespTime time.Time,
+) time.Duration {
+ freshnessLifetime := GetFreshnessLifetime(respHeaders, respCacheControl)
+ currentAge := GetCurrentAge(respHeaders, respReqTime, respRespTime)
+ log.Debugf("FreshFor: freshnesslifetime %v currentAge %v\n",
freshnessLifetime, currentAge)
+ //fresh := freshnessLifetime > currentAge
+ return freshnessLifetime - currentAge
+}
--
To stop receiving notification emails like this one, please contact
r...@apache.org.
