This is an automated email from the ASF dual-hosted git repository. rob pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-trafficcontrol.git
commit 2418ed7c4beff63529544abf17da7987d90e2b5e Author: Jan van Doorn <j...@knutsel.com> AuthorDate: Wed May 23 12:43:10 2018 -0600 RR tests --- grove/integration_test/Dockerfile | 26 ++++ grove/integration_test/compare_gets | Bin 0 -> 5782484 bytes grove/integration_test/compare_gets.go | 160 +++++++++++++++++++++ grove/integration_test/grove.cfg | 32 +++++ grove/integration_test/remap-base-test.json | 152 ++++++++++++++++++++ grove/integration_test/setup-and-run.sh | 45 ++++++ .../tests/plugins/range_req_handler/remap.json | 152 ++++++++++++++++++++ .../tests/plugins/range_req_handler/test.sh | 42 ++++++ 8 files changed, 609 insertions(+) diff --git a/grove/integration_test/Dockerfile b/grove/integration_test/Dockerfile new file mode 100644 index 0000000..84d43cb --- /dev/null +++ b/grove/integration_test/Dockerfile @@ -0,0 +1,26 @@ +FROM centos:7 +MAINTAINER The CentOS Project <cloud-...@centos.org> +LABEL Vendor="CentOS" \ + License=GPLv2 \ + Version=2.4.6-40 + + +RUN yum -y --setopt=tsflags=nodocs update && \ + yum -y --setopt=tsflags=nodocs install httpd && \ + yum -y --setopt=tsflags=nodocs install perl && \ + yum -y --setopt=tsflags=nodocs install git && \ + yum -y --setopt=tsflags=nodocs install golang && \ + yum -y --setopt=tsflags=nodocs install openssl && \ + yum clean all + +#EXPOSE 80 + +# Simple startup script to avoid some issues observed with container restart +ADD setup-and-run.sh setup-and-run.sh / +RUN chmod -v +x /setup-and-run.sh +ADD remap-base-test.json /remap-base-test.json +ADD grove.cfg /grove.cfg +ADD tests /tests +ADD compare_gets.go /compare_gets.go + +CMD ["/setup-and-run.sh"] diff --git a/grove/integration_test/compare_gets b/grove/integration_test/compare_gets new file mode 100755 index 0000000..c2a86a6 Binary files /dev/null and b/grove/integration_test/compare_gets differ diff --git a/grove/integration_test/compare_gets.go b/grove/integration_test/compare_gets.go new file mode 100644 index 0000000..385df26 --- /dev/null +++ b/grove/integration_test/compare_gets.go @@ -0,0 +1,160 @@ +package main + +import ( + "flag" + "fmt" + "github.com/apache/incubator-trafficcontrol/grove/web" + "io/ioutil" + "log" + "net/http" + "os" + "strings" +) + +type responseType struct { + Headers http.Header + Body []byte +} + +func httpGet(URL, headers string) responseType { + client := &http.Client{} + req, err := http.NewRequest("GET", URL, nil) + if err != nil { + fmt.Println("ERROR in httpGet") + } + //log.Printf(">>>%v<<< %v\n", headers, len(strings.Split(headers, "."))) + for _, hdrString := range strings.Split(headers, " ") { + //log.Println(">>> ", hdrString) + if hdrString == "" { + continue + } + parts := strings.Split(hdrString, ":") + if parts[0] == "Host" { + req.Host = parts[1] + } else { + //log.Println("> ", parts) + req.Header.Set(parts[0], parts[1]) + } + } + //log.Printf(">>>> %v", req) + resp, err := client.Do(req) + if err != nil { + fmt.Println("ERROR in httpGet") + } + defer resp.Body.Close() + var response responseType + response.Headers = web.CopyHeader(resp.Header) + response.Body, err = ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("ERROR in httpGet (readall)") + } + return response +} + +func equalBodies(a, b []byte) bool { + if a == nil || b == nil { + return false + } + + if a == nil && b == nil { + return true + } + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func equalStringSlices(a, b []string) bool { + if a == nil || b == nil { + return false + } + + if a == nil && b == nil { + return true + } + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func inStringSlice(str string, arr []string) bool { + for _, strEnt := range arr { + if strEnt == str { + return true + } + } + return false +} + +func compareResponses(response1 responseType, response2 responseType, ignoreHdrs []string, ignoreMPB bool) bool { + + if ignoreMPB { + contentTypeHdr := response1.Headers.Get("Content-type") + fmt.Println("ignoreing", contentTypeHdr, response1) + if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") { + parts := strings.Split(contentTypeHdr, "=") + MPBoundary := parts[1] + //log.Println("+++") + //log.Printf("%s\n", string(response1.Body)) + response1.Body = []byte(strings.Replace(string(response1.Body), MPBoundary, "", -1)) + //log.Printf("%s\n", response1.Body) + } + contentTypeHdr = response2.Headers.Get("Content-type") + if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") { + parts := strings.Split(contentTypeHdr, "=") + MPBoundary := parts[1] + response2.Body = []byte(strings.Replace(string(response2.Body), MPBoundary, "", -1)) + } + } + if !equalBodies(response1.Body, response2.Body) { + return false + } + for hdrKey, _ := range response1.Headers { + if inStringSlice(hdrKey, ignoreHdrs) { + continue + } + if !equalStringSlices(response1.Headers[hdrKey], response2.Headers[hdrKey]) { + log.Printf("ERROR hdr %v doesn't match: \"%v\" != \"%v\"\n", hdrKey, response1.Headers[hdrKey], response2.Headers[hdrKey]) + return false + } + //fmt.Printf(">>>>> %v\n", hdrKey) + } + + return true +} +func main() { + originURL := flag.String("org", "http://localhost", "The origin URL (default: \"http://localhost\")") + cacheURL := flag.String("cache", "http://localhost:8080", "The cache URL (default: \"http://localhost:8080\")") + path := flag.String("path", "", "The path to GET") + orgHdrs := flag.String("ohdrs", "", "Comma seperated list of headers to add to origin request") + cacheHdrs := flag.String("chdrs", "", "Comma separated list of headers to add to cache request") + ignoreHdrs := flag.String("ignorehdrs", "Server,Date", "Comma separated list of headers to ignore in the compare") + ignoreMultiPartBoundary := flag.Bool("ignorembp", true, "Ignore multi part boundary in body comparison.") + flag.Parse() + + resp := httpGet(*originURL+"/"+*path, *orgHdrs) + cresp := httpGet(*cacheURL+"/"+*path, *cacheHdrs) + if !compareResponses(resp, cresp, strings.Split(*ignoreHdrs, ","), *ignoreMultiPartBoundary) { + fmt.Println("FAIL: Body bytes don't match \n%s\n != \n%s\n", string(resp.Body), string(cresp.Body)) + os.Exit(1) + + } + fmt.Println("PASS") + os.Exit(0) +} diff --git a/grove/integration_test/grove.cfg b/grove/integration_test/grove.cfg new file mode 100644 index 0000000..b5af0f0 --- /dev/null +++ b/grove/integration_test/grove.cfg @@ -0,0 +1,32 @@ +{ + "rfc_compliant":false, + "port":8080, + "https_port":8443, + "cache_size_bytes":10000000, + "cache_files":{ + "disk":[ + { + "path":"/diskcachefile0.db", + "size_bytes":100000000 + }, + { + "path":"/diskcachefile1.db", + "size_bytes":100000000 + } + ], + "my-disk-cache-two":[ + { + "path":"/singlefilecache.db", + "size_bytes":10000000 + } + ] + }, + "remap_rules_file":"./remap.json", + "log_location_error":"./error.log", + "log_location_warning":"./error.log", + "log_location_info":"./error.log", + "log_location_debug":"./error.log", + "log_location_event":"./custom_ats_2.log", + "cert_file":"./cert.pem", + "key_file":"./key.pem" +} diff --git a/grove/integration_test/remap-base-test.json b/grove/integration_test/remap-base-test.json new file mode 100644 index 0000000..4cf49e5 --- /dev/null +++ b/grove/integration_test/remap-base-test.json @@ -0,0 +1,152 @@ +{ + "parent_selection": "consistent-hash", + "plugins": { + "modify_response_headers_global": { + "set": [ + { + "name": "Server", + "value": "Grove/0.39999999" + } + ] + } + }, + "retry_codes": null, + "retry_num": null, + "rules": [ + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk-test.cdn.kabletown.net", + "name": "jvd-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + }, + "range_req_handler": { + "mode": "get_full_serve_range" + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk1-test.cdn.kabletown.net", + "name": "jvd1-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://mem-test.cdn.kabletown.net", + "name": "cim-linear-hds.http.http.cim-linear-hds", + "parent_selection": "consistent-hash", + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000, + "plugins": { + "range_req_handler": { + "mode": "store_ranges" + } + }, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + } + ], + "stats": { + "allow": [ + "127.0.0.1/32", + "::1/128" + ], + "deny": null + }, + "timeout_ms": 5000 +} diff --git a/grove/integration_test/setup-and-run.sh b/grove/integration_test/setup-and-run.sh new file mode 100644 index 0000000..c9abf6b --- /dev/null +++ b/grove/integration_test/setup-and-run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/bash +echo Configuring Grove Integration Test Environment. + +#perl -v +#ls -l /var/www/html +cd /var/www/html +echo Generating origin test files... +perl -e 'foreach $i ( 0 ... 1024*1024-1 ) { printf "%09d\n", $i*10 }' > 10Mb.txt +for i in {1..1000} ; do dd if=/dev/urandom of=${i}k.bin bs=${i}k count=1 > /dev/null 2>&1 ; done +httpd + +cd / +echo Setting up go enviroment... +export GOPATH=~/go +go get golang.org/x/text +go get golang.org/x/sys/unix +go get golang.org/x/net/http2 +go get golang.org/x/net/ipv4 +go get golang.org/x/net/ipv6 + +mkdir -p $GOPATH/src/github.com/apache/ +cd $GOPATH/src/github.com/apache/ +#git clone https://github.com/apache/incubator-trafficcontrol +git clone $REPO +cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove +git checkout $BRANCH +go build + +cd / +openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key.pem -out cert.pem -subj "/C=US/ST=CO/L=Denver/O=.../OU=.../CN=.../emailAddress=..." + +cp /remap-base-test.json /remap.json +ls -l +${GOPATH}/src/github.com/apache/incubator-trafficcontrol/grove/grove -cfg grove.cfg & + + +sleep 3 +curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt + +#cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test +go build compare_gets.go + + + + diff --git a/grove/integration_test/tests/plugins/range_req_handler/remap.json b/grove/integration_test/tests/plugins/range_req_handler/remap.json new file mode 100644 index 0000000..4cf49e5 --- /dev/null +++ b/grove/integration_test/tests/plugins/range_req_handler/remap.json @@ -0,0 +1,152 @@ +{ + "parent_selection": "consistent-hash", + "plugins": { + "modify_response_headers_global": { + "set": [ + { + "name": "Server", + "value": "Grove/0.39999999" + } + ] + } + }, + "retry_codes": null, + "retry_num": null, + "rules": [ + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk-test.cdn.kabletown.net", + "name": "jvd-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + }, + "range_req_handler": { + "mode": "get_full_serve_range" + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk1-test.cdn.kabletown.net", + "name": "jvd1-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://mem-test.cdn.kabletown.net", + "name": "cim-linear-hds.http.http.cim-linear-hds", + "parent_selection": "consistent-hash", + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000, + "plugins": { + "range_req_handler": { + "mode": "store_ranges" + } + }, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost", + "weight": 1 + } + ] + } + ], + "stats": { + "allow": [ + "127.0.0.1/32", + "::1/128" + ], + "deny": null + }, + "timeout_ms": 5000 +} diff --git a/grove/integration_test/tests/plugins/range_req_handler/test.sh b/grove/integration_test/tests/plugins/range_req_handler/test.sh new file mode 100644 index 0000000..4654318 --- /dev/null +++ b/grove/integration_test/tests/plugins/range_req_handler/test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash -x + +#curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt +originurl="http://localhost/" +host="mem-test.cdn.kabletown.net" +cacheurl="http://localhost:8080/" +file="10Mb.txt" + +result=0 +testno=0 + +#curl -s -r 50000-50009 ${originurl}${file} > /tmp/out1 && echo FAIL test ${testno} +#result=$(($result+$?)) +#testno=$(($testno+1)) +# +#curl -s -r 50000-50009 -H"Host: ${host}" ${cacheurl}/${file} > /tmp/out2 && echo FAIL test ${testno} +# +#result=$(($result+$?)) +#testno=$(($testno+1)) +# +#diff /tmp/out1 /tmp/out2 && echo FAIL test ${testno} +#result=$(($result+$?)) +#testno=$(($testno+1)) + +for host in "mem-test.cdn.kabletown.net", "disk1-test.cdn.kabletown.net" +do + for r in "0-0", "0-100", "5000-", "-100", "0-0,10-15", "0-100,200-210", "33-99,66-88" "-" + do + test="/compare_gets --chdrs \"Host:$host,Range:bytes=\r${r}\" --ohdrs \"Range:bytes=${r}\" --path \"10Mb.txt\" --ignorehdrs \"Server,Date\"" + testno=$(($testno+1)) + echo -n "Test $testno ($test): " + + /compare_gets --chdrs "Host:$host,Range:bytes=${r}" --ohdrs "Range:bytes=${r}" --path "10Mb.txt" --ignorehdrs "Server,Date" + + result=$(($result+$?)) + done +done + + +echo "$testno tests done, $result failed." + +exit $result -- To stop receiving notification emails like this one, please contact r...@apache.org.