[go-nuts] HTTP client request rate

[Valentin Vidic]

Hi,

Can someone help me figure out why the performance (request rate)
of this program goes down when I add more gorutines?

$ ./gb -duration 15s -parallel 5 http://nginx
Running 5 parallel clients for 15s...
Requests: 217998
Rate: 14533.2/s
Bytes: 133414776
Code[200]: 217998

$ ./gb -duration 15s -parallel 20 http://nginx
Running 20 parallel clients for 15s...
Requests: 155150
Rate: 10343.3/s
Bytes: 94951800
Code[200]: 155150

-- 
Valentin

-- 
You received this message because you are subscribed to the Google Groups 
"golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to golang-nuts+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

package main

import (
        "flag"
        "fmt"
        "io"
        "net"
        "net/http"
        "os"
        "runtime"
        "runtime/pprof"
        "time"
)

type stats struct {
        req   int
        err   int
        rerr  int
        bytes int
        code  map[int]int
}

func bench(url string, compress bool, timeout time.Duration, done chan 
struct{}, result chan stats) {
        s := stats{}
        s.code = make(map[int]int)

        transport := &http.Transport{
                DisableCompression:  !compress,
                TLSHandshakeTimeout: timeout,
                DialContext: (&net.Dialer{
                        Timeout:   timeout,
                        DualStack: true,
                }).DialContext,
        }
        client := &http.Client{
                Transport: transport,
                Timeout:   timeout,
        }

        written := 0
        buf := make([]byte, 10*1024)

LOOP:
        for {
                s.req++
                resp, err := client.Get(url)
                if err != nil {
                        fmt.Println(err)
                        s.err++
                } else {
                        s.code[resp.StatusCode]++

                        for {
                                written, err = resp.Body.Read(buf)
                                s.bytes += written
                                if err != nil {
                                        break
                                }
                        }

                        if err != nil && err != io.EOF {
                                fmt.Println(err)
                                s.rerr++
                        }

                        resp.Body.Close()
                }

                select {
                case <-done:
                        break LOOP
                default:
                }
        }

        result <- s
}

func main() {
        var compression = flag.Bool("compression", true, "use HTTP compression")
        var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to 
file")
        var duration = flag.Duration("duration", 15*time.Second, "test 
duration")
        var memprofile = flag.String("memprofile", "", "write memory profile to 
file")
        var parallel = flag.Int("parallel", 20, "number of parallel client 
connections")
        var timeout = flag.Duration("timeout", 10*time.Second, "request 
timeout")

        flag.Parse()
        url := flag.Arg(0)
        if url == "" {
                fmt.Println("No url given")
                os.Exit(1)
        }

        if *cpuprofile != "" {
                f, err := os.Create(*cpuprofile)
                if err != nil {
                        fmt.Println("Could not create cpu profile:", err)
                        os.Exit(1)
                }
                pprof.StartCPUProfile(f)
                defer pprof.StopCPUProfile()
        }

        done := make(chan struct{})
        result := make(chan stats)

        fmt.Printf("Running %d parallel clients for %v...\n", *parallel, 
*duration)
        for i := 0; i < *parallel; i++ {
                go bench(url, *compression, *timeout, done, result)
        }

        time.Sleep(*duration)
        close(done)

        total := stats{}
        total.code = make(map[int]int)
        for i := 0; i < *parallel; i++ {
                s := <-result
                total.req += s.req
                total.err += s.err
                total.rerr += s.rerr
                total.bytes += s.bytes
                for k, v := range s.code {
                        total.code[k] += v
                }
        }

        fmt.Println("Requests:", total.req)
        fmt.Printf("Rate: %.1f/s\n", float64(total.req)/duration.Seconds())
        fmt.Println("Bytes:", total.bytes)
        if total.err > 0 {
                fmt.Println("Errors:", total.err)
        }
        if total.rerr > 0 {
                fmt.Println("Read errors:", total.rerr)
        }
        for k, v := range total.code {
                fmt.Printf("Code[%d]: %d\n", k, v)
        }

        if *memprofile != "" {
                f, err := os.Create(*memprofile)
                if err != nil {
                        fmt.Println("Could not create memory profile: ", err)
                }
                runtime.GC() // get up-to-date statistics
                if err := pprof.WriteHeapProfile(f); err != nil {
                        fmt.Println("could not write memory profile: ", err)
                }
                f.Close()
        }
}