rob05c closed pull request #2320: Start of integration tests for Grove / some fixes to range_req_handler URL: https://github.com/apache/incubator-trafficcontrol/pull/2320
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/grove/integration_test/Dockerfile b/grove/integration_test/Dockerfile new file mode 100644 index 000000000..2ac69879f --- /dev/null +++ b/grove/integration_test/Dockerfile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM centos:7 + +RUN yum -y --setopt=tsflags=nodocs update && \ + yum -y --setopt=tsflags=nodocs install httpd && \ + yum -y --setopt=tsflags=nodocs install perl && \ + yum -y --setopt=tsflags=nodocs install git && \ + yum -y --setopt=tsflags=nodocs install golang && \ + yum -y --setopt=tsflags=nodocs install openssl && \ + yum clean all + +#EXPOSE 80 + +# Simple startup script to avoid some issues observed with container restart +ADD setup-and-run.sh setup-and-run.sh / +RUN chmod -v +x /setup-and-run.sh +ADD remap-base-test.json /remap-base-test.json +ADD grove.cfg /grove.cfg +ADD tests /tests +ADD compare_gets.go /compare_gets.go + +CMD ["/setup-and-run.sh"] diff --git a/grove/integration_test/compare_gets.go b/grove/integration_test/compare_gets.go new file mode 100644 index 000000000..b88cb69d0 --- /dev/null +++ b/grove/integration_test/compare_gets.go @@ -0,0 +1,176 @@ +package main + +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ( + "flag" + "fmt" + "github.com/apache/incubator-trafficcontrol/grove/web" + "io/ioutil" + "log" + "net/http" + "os" + "strings" +) + +type responseType struct { + Headers http.Header + Body []byte +} + +func httpGet(URL, headers string) responseType { + client := &http.Client{} + req, err := http.NewRequest("GET", URL, nil) + if err != nil { + fmt.Println("ERROR in httpGet") + } + //log.Printf(">>>%v<<< %v\n", headers, len(strings.Split(headers, "."))) + for _, hdrString := range strings.Split(headers, " ") { + //log.Println(">>> ", hdrString) + if hdrString == "" { + continue + } + parts := strings.Split(hdrString, ":") + if parts[0] == "Host" { + req.Host = parts[1] + } else { + //log.Println("> ", parts) + req.Header.Set(parts[0], parts[1]) + } + } + //log.Printf(">>>> %v", req) + resp, err := client.Do(req) + if err != nil { + fmt.Println("ERROR in httpGet") + } + defer resp.Body.Close() + var response responseType + response.Headers = web.CopyHeader(resp.Header) + response.Body, err = ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("ERROR in httpGet (readall)") + } + return response +} + +func equalBodies(a, b []byte) bool { + if a == nil || b == nil { + return false + } + + if a == nil && b == nil { + return true + } + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func equalStringSlices(a, b []string) bool { + if a == nil || b == nil { + return false + } + + if a == nil && b == nil { + return true + } + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func inStringSlice(str string, arr []string) bool { + for _, strEnt := range arr { + if strEnt == str { + return true + } + } + return false +} + +func compareResponses(response1 responseType, response2 responseType, ignoreHdrs []string, ignoreMPB bool) bool { + + if ignoreMPB { + contentTypeHdr := response1.Headers.Get("Content-type") + ignoreHdrs = append(ignoreHdrs, "Content-Type") // the boundary will be different + ignoreHdrs = append(ignoreHdrs, "Content-Length") // the boundary will be different + //fmt.Println("ignoreing", contentTypeHdr, response1) + if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") { + parts := strings.Split(contentTypeHdr, "=") + MPBoundary := parts[1] + //log.Println("+++") + //log.Printf("%s\n", string(response1.Body)) + response1.Body = []byte(strings.Replace(string(response1.Body), MPBoundary, "", -1)) + //log.Printf("%s\n", response1.Body) + } + contentTypeHdr = response2.Headers.Get("Content-type") + if strings.HasPrefix(contentTypeHdr, "multipart/byteranges") { + parts := strings.Split(contentTypeHdr, "=") + MPBoundary := parts[1] + response2.Body = []byte(strings.Replace(string(response2.Body), MPBoundary, "", -1)) + } + } + if !equalBodies(response1.Body, response2.Body) { + return false + } + for hdrKey, _ := range response1.Headers { + if inStringSlice(hdrKey, ignoreHdrs) { + continue + } + if !equalStringSlices(response1.Headers[hdrKey], response2.Headers[hdrKey]) { + log.Printf("ERROR hdr %v doesn't match: \"%v\" != \"%v\"\n", hdrKey, response1.Headers[hdrKey], response2.Headers[hdrKey]) + return false + } + //fmt.Printf(">>>>> %v\n", hdrKey) + } + + return true +} +func main() { + originURL := flag.String("org", "http://localhost";, "The origin URL (default: \"http://localhost\";)") + cacheURL := flag.String("cache", "http://localhost:8080";, "The cache URL (default: \"http://localhost:8080\";)") + path := flag.String("path", "", "The path to GET") + orgHdrs := flag.String("ohdrs", "", "Comma seperated list of headers to add to origin request") + cacheHdrs := flag.String("chdrs", "", "Comma separated list of headers to add to cache request") + ignoreHdrs := flag.String("ignorehdrs", "Server,Date", "Comma separated list of headers to ignore in the compare") + ignoreMultiPartBoundary := flag.Bool("ignorempb", true, "Ignore multi part boundary in body comparison.") + flag.Parse() + + resp := httpGet(*originURL+"/"+*path, *orgHdrs) + cresp := httpGet(*cacheURL+"/"+*path, *cacheHdrs) + if !compareResponses(resp, cresp, strings.Split(*ignoreHdrs, ","), *ignoreMultiPartBoundary) { + fmt.Printf("FAIL: Body bytes don't match \n%s\n != \n%s\n", string(resp.Body), string(cresp.Body)) + os.Exit(1) + + } + fmt.Println("PASS") + os.Exit(0) +} diff --git a/grove/integration_test/grove.cfg b/grove/integration_test/grove.cfg new file mode 100644 index 000000000..b5af0f021 --- /dev/null +++ b/grove/integration_test/grove.cfg @@ -0,0 +1,32 @@ +{ + "rfc_compliant":false, + "port":8080, + "https_port":8443, + "cache_size_bytes":10000000, + "cache_files":{ + "disk":[ + { + "path":"/diskcachefile0.db", + "size_bytes":100000000 + }, + { + "path":"/diskcachefile1.db", + "size_bytes":100000000 + } + ], + "my-disk-cache-two":[ + { + "path":"/singlefilecache.db", + "size_bytes":10000000 + } + ] + }, + "remap_rules_file":"./remap.json", + "log_location_error":"./error.log", + "log_location_warning":"./error.log", + "log_location_info":"./error.log", + "log_location_debug":"./error.log", + "log_location_event":"./custom_ats_2.log", + "cert_file":"./cert.pem", + "key_file":"./key.pem" +} diff --git a/grove/integration_test/remap-base-test.json b/grove/integration_test/remap-base-test.json new file mode 100644 index 000000000..4cf49e5d5 --- /dev/null +++ b/grove/integration_test/remap-base-test.json @@ -0,0 +1,152 @@ +{ + "parent_selection": "consistent-hash", + "plugins": { + "modify_response_headers_global": { + "set": [ + { + "name": "Server", + "value": "Grove/0.39999999" + } + ] + } + }, + "retry_codes": null, + "retry_num": null, + "rules": [ + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk-test.cdn.kabletown.net";, + "name": "jvd-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + }, + "range_req_handler": { + "mode": "get_full_serve_range" + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk1-test.cdn.kabletown.net";, + "name": "jvd1-test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "JvD-Grove" + } + ] + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://mem-test.cdn.kabletown.net";, + "name": "cim-linear-hds.http.http.cim-linear-hds", + "parent_selection": "consistent-hash", + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000, + "plugins": { + "range_req_handler": { + "mode": "store_ranges" + } + }, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + } + ], + "stats": { + "allow": [ + "127.0.0.1/32", + "::1/128" + ], + "deny": null + }, + "timeout_ms": 5000 +} diff --git a/grove/integration_test/setup-and-run.sh b/grove/integration_test/setup-and-run.sh new file mode 100644 index 000000000..f16612f05 --- /dev/null +++ b/grove/integration_test/setup-and-run.sh @@ -0,0 +1,102 @@ +#!/usr/bin/bash + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo Configuring Grove Integration Test Environment. + +#perl -v +#ls -l /var/www/html +cd /var/www/html +echo Generating origin test files... +perl -e 'foreach $i ( 0 ... 1024*1024-1 ) { printf "%09d\n", $i*10 }' > 10Mb.txt +for i in {1..1000} ; do dd if=/dev/urandom of=${i}k.bin bs=${i}k count=1 > /dev/null 2>&1 ; done +httpd + +cd / +echo Setting up go enviroment... +export GOPATH=~/go +go get golang.org/x/text +go get golang.org/x/sys/unix +go get golang.org/x/net/http2 +go get golang.org/x/net/ipv4 +go get golang.org/x/net/ipv6 + +mkdir -p $GOPATH/src/github.com/apache/ +cd $GOPATH/src/github.com/apache/ +#git clone https://github.com/apache/incubator-trafficcontrol +git clone $REPO +cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove +git checkout $BRANCH +go build + +cd / +openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key.pem -out cert.pem -subj "/C=US/ST=CO/L=Denver/O=.../OU=.../CN=.../emailAddress=..." + +cp /remap-base-test.json /remap.json +ls -l +${GOPATH}/src/github.com/apache/incubator-trafficcontrol/grove/grove -cfg grove.cfg & + + +sleep 3 +curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt + +#cd $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test +go build compare_gets.go + +#function run_test () { +# command=${1} +# output=${2:-/dev/null} +# echo -n "Test ${testno} (${command}): " +# ${command} > ${output} 2>&1 +# thisresult=$? +# +# result=$(($result+thisresult)) +# if [ $thisresult -eq 0 ] +# then +# echo "PASS" +# else +# echo "FAIL" +# fi +# +# testno=$(($testno+1)) +#} + + +function run_test () { + "$@" > /tmp/run_test.out 2>&1 + thisresult=$? + + echo -n "Test ${testno} ($@): " + result=$(($result+thisresult)) + if [ $thisresult -eq 0 ] + then + echo "PASS" + else + echo "FAIL" + fi + + testno=$(($testno+1)) +} +export -f run_test + +cp $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test/tests/plugins/modify_headers/remap.json /remap.json +pkill -HUP grove +bash $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test/tests/plugins/modify_headers/test.sh + +cp $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test/tests/plugins/range_req_handler/remap.json /remap.json +pkill -HUP grove +bash $GOPATH/src/github.com/apache/incubator-trafficcontrol/grove/integration_test/tests/plugins/range_req_handler/test.sh + + diff --git a/grove/integration_test/tests/plugins/modify_headers/remap.json b/grove/integration_test/tests/plugins/modify_headers/remap.json new file mode 100644 index 000000000..80739e441 --- /dev/null +++ b/grove/integration_test/tests/plugins/modify_headers/remap.json @@ -0,0 +1,152 @@ +{ + "parent_selection": "consistent-hash", + "plugins": { + "modify_response_headers_global": { + "set": [ + { + "name": "Server", + "value": "Grove/0.39999999" + } + ] + } + }, + "retry_codes": null, + "retry_num": null, + "rules": [ + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk-test.cdn.kabletown.net";, + "name": "test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "GroverCDN" + } + ] + }, + "range_req_handler": { + "mode": "get_full_serve_range" + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk1-test.cdn.kabletown.net";, + "name": "test1", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "GroverCDN" + } + ] + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://mem-test.cdn.kabletown.net";, + "name": "test2", + "parent_selection": "consistent-hash", + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000, + "plugins": { + "range_req_handler": { + "mode": "store_ranges" + } + }, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + } + ], + "stats": { + "allow": [ + "127.0.0.1/32", + "::1/128" + ], + "deny": null + }, + "timeout_ms": 5000 +} diff --git a/grove/integration_test/tests/plugins/modify_headers/test.sh b/grove/integration_test/tests/plugins/modify_headers/test.sh new file mode 100644 index 000000000..0e507fe07 --- /dev/null +++ b/grove/integration_test/tests/plugins/modify_headers/test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash -x + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +originurl="http://localhost/"; +host="mem-test.cdn.kabletown.net" +cacheurl="http://localhost:8080/"; +file="10Mb.txt" + +result=0 +testno=0 + + +# test global header set +run_test curl -Lkvs -HHost:mem-test.cdn.kabletown.net -o /tmp/10k.bin http://localhost:8080/10k.bin +cp /tmp/run_test.out /tmp/hdrs.out +run_test diff /tmp/10k.bin /var/www/html/10k.bin +run_test grep "< Server: Grove/0.39999999" /tmp/hdrs.out + +# test per remap header set up and down +run_test curl -XTRACE -H'Host: disk-test.cdn.kabletown.net' http://localhost:8080/10Mb.txt -Lkvs -o /tmp/out +cp /tmp/run_test.out /tmp/hdrs.out +run_test grep "X-From-Cdn: Traffic-Control" /tmp/out +run_test grep "< X-Cdn-Name: GroverCDN" /tmp/hdrs.out + +echo "plugin/modify_headers: $testno tests done, $result failed." + +exit $result diff --git a/grove/integration_test/tests/plugins/range_req_handler/remap.json b/grove/integration_test/tests/plugins/range_req_handler/remap.json new file mode 100644 index 000000000..80739e441 --- /dev/null +++ b/grove/integration_test/tests/plugins/range_req_handler/remap.json @@ -0,0 +1,152 @@ +{ + "parent_selection": "consistent-hash", + "plugins": { + "modify_response_headers_global": { + "set": [ + { + "name": "Server", + "value": "Grove/0.39999999" + } + ] + } + }, + "retry_codes": null, + "retry_num": null, + "rules": [ + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk-test.cdn.kabletown.net";, + "name": "test", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "GroverCDN" + } + ] + }, + "range_req_handler": { + "mode": "get_full_serve_range" + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://disk1-test.cdn.kabletown.net";, + "name": "test1", + "parent_selection": "consistent-hash", + "cache_name": "disk", + "plugins": { + "modify_parent_request_headers": { + "set": [ + { + "name": "X-From-CDN", + "value": "Traffic-Control" + } + ] + }, + "modify_headers": { + "set": [ + { + "name": "X-CDN-name", + "value": "GroverCDN" + } + ] + } + }, + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000000, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + }, + { + "allow": null, + "certificate-file": "", + "certificate-key-file": "", + "concurrent_rule_requests": 0, + "connection-close": false, + "deny": null, + "from": "http://mem-test.cdn.kabletown.net";, + "name": "test2", + "parent_selection": "consistent-hash", + "query-string": { + "cache": true, + "remap": true + }, + "retry_codes": [], + "retry_num": 5, + "timeout_ms": 5000, + "plugins": { + "range_req_handler": { + "mode": "store_ranges" + } + }, + "to": [ + { + "retry_codes": [], + "retry_num": 0, + "timeout_ms": 5000000, + "url": "http://localhost";, + "weight": 1 + } + ] + } + ], + "stats": { + "allow": [ + "127.0.0.1/32", + "::1/128" + ], + "deny": null + }, + "timeout_ms": 5000 +} diff --git a/grove/integration_test/tests/plugins/range_req_handler/test.sh b/grove/integration_test/tests/plugins/range_req_handler/test.sh new file mode 100644 index 000000000..0926ded78 --- /dev/null +++ b/grove/integration_test/tests/plugins/range_req_handler/test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash -x + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +CMP_TOOL="${CMP_TOOL:-/compare_gets}" + +#curl -H'Host: mem-test.cdn.kabletown.net' -Lsv -r 50000-50009 http://localhost:8080/10Mb.txt +originurl="http://localhost/"; +host="mem-test.cdn.kabletown.net" +cacheurl="http://localhost:8080/"; +file="10Mb.txt" + +result=0 +testno=0 + +for host in "mem-test.cdn.kabletown.net" "disk-test.cdn.kabletown.net" +do + for r in "0-0" "0-100" "5000-" "-100" "8-10,9-15,100-200" "0-300,200-250" "-33,66-99,50-150" + do + test="${CMP_TOOL} --chdrs \"Host:$host Range:bytes=${r}\" --ohdrs \"Range:bytes=${r}\" --path \"10Mb.txt\" --ignorehdrs \"Server,Date\"" + testno=$(($testno+1)) + echo -n "Test $testno ($test): " + + ${CMP_TOOL} --chdrs "Host:$host Range:bytes=${r}" --ohdrs "Range:bytes=${r}" --path "10Mb.txt" --ignorehdrs "Server,Date" + + result=$(($result+$?)) + done +done + +# multipart +for host in "mem-test.cdn.kabletown.net" "disk-test.cdn.kabletown.net" +do + for r in "0-0" "0-100" "5000-" "-100" "0-0,10-15" "0-100,200-210" "33-99,101-188" "8-10,9-15,100-200" "0-300,200-250" "-33,66-99,50-150" "300-304,500-,600-700" + do + test="${CMP_TOOL} --chdrs \"Host:$host Range:bytes=${r}\" --ohdrs \"Range:bytes=${r}\" --path \"10Mb.txt\" --ignorehdrs \"Server,Date\" --ignorempb" + testno=$(($testno+1)) + echo -n "Test $testno ($test): " + + ${CMP_TOOL} --chdrs "Host:$host Range:bytes=${r}" --ohdrs "Range:bytes=${r}" --path "10Mb.txt" --ignorehdrs "Server,Date" --ignorempb + + result=$(($result+$?)) + done +done + +echo "plugin/range_req_handler: $testno tests done, $result failed." + +exit $result diff --git a/grove/plugin/range_req_handler.go b/grove/plugin/range_req_handler.go index 0bd8e0cec..2bc477483 100644 --- a/grove/plugin/range_req_handler.go +++ b/grove/plugin/range_req_handler.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net/http" + "sort" "strconv" "strings" @@ -31,6 +32,8 @@ type byteRange struct { End int64 } +const MAXINT64 = 1<<63 - 1 + type rangeRequestConfig struct { Mode string `json:"mode"` MultiPartBoundary string // not in the json @@ -163,26 +166,30 @@ func rangeReqHandleBeforeRespond(icfg interface{}, d BeforeRespondData) { } body := make([]byte, 0) for _, thisRange := range ctx { - if thisRange.End == -1 || thisRange.End >= totalContentLength { // if the end range is "", or too large serve until the end + if thisRange.End == MAXINT64 || thisRange.End >= totalContentLength { // if the end range is "", or too large serve until the end + thisRange.End = totalContentLength - 1 + } + if thisRange.Start == -1 { + thisRange.Start = totalContentLength - thisRange.End thisRange.End = totalContentLength - 1 } - rangeString := "bytes " + strconv.FormatInt(thisRange.Start, 10) + "-" + strconv.FormatInt(thisRange.End, 10) + "\r\n\r\n" + rangeString := "bytes " + strconv.FormatInt(thisRange.Start, 10) + "-" + strconv.FormatInt(thisRange.End, 10) log.Debugf("range:%d-%d\n", thisRange.Start, thisRange.End) if multipart { body = append(body, []byte("\r\n--"+multipartBoundaryString+"\r\n")...) - body = append(body, []byte("Content-type: "+originalContentType+"%s\r\n")...) - body = append(body, []byte("Content-range: "+rangeString)...) + body = append(body, []byte("Content-type: "+originalContentType+"\r\n")...) + body = append(body, []byte("Content-range: "+rangeString+"/"+strconv.FormatInt(totalContentLength, 10)+"\r\n\r\n")...) } else { d.Hdr.Add("Content-Range", rangeString+"/"+strconv.FormatInt(totalContentLength, 10)) } bSlice := (*d.Body)[thisRange.Start : thisRange.End+1] body = append(body, bSlice...) } - if multipartBoundaryString != "" { - body = append(body, []byte(fmt.Sprintf("\r\n--%s--\r\n", multipartBoundaryString))...) + if multipart { + body = append(body, []byte("\r\n--"+multipartBoundaryString+"--\r\n")...) } - d.Hdr.Set("Content-Length", fmt.Sprintf("%d", len(body))) + d.Hdr.Set("Content-Length", strconv.Itoa(len(body))) *d.Body = body *d.Code = http.StatusPartialContent return @@ -193,7 +200,7 @@ func parseRange(rangeString string) (byteRange, error) { var bRange byteRange if parts[0] == "" { - bRange.Start = 0 + bRange.Start = -1 // -1 means from the end } else { start, err := strconv.ParseInt(parts[0], 10, 64) if err != nil { @@ -203,7 +210,7 @@ func parseRange(rangeString string) (byteRange, error) { bRange.Start = start } if parts[1] == "" { - bRange.End = -1 // -1 means till the end + bRange.End = MAXINT64 // MAXINT64 means till the end } else { end, err := strconv.ParseInt(parts[1], 10, 64) if err != nil { @@ -229,5 +236,38 @@ func parseRangeHeader(rHdrVal string) []byteRange { } byteRanges = append(byteRanges, thisRange) } - return byteRanges + + // if there is just one range, return, and don't incur the overhead of determining overlaps + if len(byteRanges) <= 1 { + return byteRanges + } + + // Collapse overlapping byte range requests, first sort the array by Start + sort.Slice(byteRanges, func(i, j int) bool { + return byteRanges[i].Start < byteRanges[j].Start + }) + + // Then, copy ranges into collapsedRanges if applicable, collapse as needed + collapsedRanges := make([]byteRange, 0) + j := 0 + collapsedRanges = append(collapsedRanges, byteRanges[j]) + for i := 1; i < len(byteRanges); i++ { + if collapsedRanges[j].End < byteRanges[i].Start { + // Most normal case, the ranges are not overlapping; add the range to the collapsedRanges array + collapsedRanges = append(collapsedRanges, byteRanges[i]) + j++ + continue + } + if collapsedRanges[j].Start <= byteRanges[i].Start && collapsedRanges[j].End >= byteRanges[i].End { + // Don't add the entry at i, it is part of the entry at i-1 + continue + } + if collapsedRanges[j].Start <= byteRanges[i].Start && collapsedRanges[j].End < byteRanges[i].End { + // Overlapping ranges, combine into one + collapsedRanges[j].End = byteRanges[i].End + continue + } + } + + return collapsedRanges } ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
