The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/3779
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === - add support for CSV reporting in `lxd-benchmark`, conforming to the JMeter CSV format (understood by the Jenkins performance plugin) - add a `perf.sh` script that runs performance tests (create/start/stop/delete containers) To test, run something like ``` sudo -E LXD_TMPFS=1 LXD_TEST_IMAGE=images:alpine/3.6 ./perf.sh ``` Closes #3670
From 2e0c0e191da1f1c8229906ed38ff73e04edf308d Mon Sep 17 00:00:00 2001 From: Alberto Donato <[email protected]> Date: Wed, 6 Sep 2017 18:08:36 +0200 Subject: [PATCH 1/4] benchmark: add csv reporting Signed-off-by: Alberto Donato <[email protected]> --- lxd-benchmark/benchmark/report.go | 95 +++++++++++++++++++++++++++++++++++++++ lxd-benchmark/main.go | 54 ++++++++++++++++++---- 2 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 lxd-benchmark/benchmark/report.go diff --git a/lxd-benchmark/benchmark/report.go b/lxd-benchmark/benchmark/report.go new file mode 100644 index 000000000..60ab02f6d --- /dev/null +++ b/lxd-benchmark/benchmark/report.go @@ -0,0 +1,95 @@ +package benchmark + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "time" +) + +// Subset of JMeter CSV log format that are required by Jenkins performance +// plugin +// (see http://jmeter.apache.org/usermanual/listeners.html#csvlogformat) +var csvFields = []string{ + "timeStamp", // in milliseconds since 1/1/1970 + "elapsed", // in milliseconds + "label", + "responseCode", + "success", // "true" or "false" +} + +// CSVReport reads/writes a CSV report file. +type CSVReport struct { + Filename string + + records [][]string +} + +// Load reads current content of the filename and loads records. +func (r *CSVReport) Load() error { + file, err := os.Open(r.Filename) + if err != nil { + return err + } + defer file.Close() + + reader := csv.NewReader(file) + for line := 1; err != io.EOF; line++ { + record, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } + + err = r.addRecord(record) + if err != nil { + return err + } + } + logf("Loaded report file %s", r.Filename) + return nil +} + +// Write writes current records to file. +func (r *CSVReport) Write() error { + file, err := os.OpenFile(r.Filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640) + if err != nil { + return err + } + defer file.Close() + + writer := csv.NewWriter(file) + err = writer.WriteAll(r.records) + if err != nil { + return err + } + + logf("Written report file %s", r.Filename) + return nil +} + +// AddRecord adds a record to the report. +func (r *CSVReport) AddRecord(label string, elapsed time.Duration) error { + if len(r.records) == 0 { + r.addRecord(csvFields) + } + + record := []string{ + fmt.Sprintf("%d", time.Now().UnixNano()/int64(time.Millisecond)), // timestamp + fmt.Sprintf("%d", elapsed/time.Millisecond), + label, + "", // responseCode is not used + "true", // success" + } + return r.addRecord(record) +} + +func (r *CSVReport) addRecord(record []string) error { + if len(record) != len(csvFields) { + return fmt.Errorf("Invalid number of fields : %q", record) + } + r.records = append(r.records, record) + return nil +} diff --git a/lxd-benchmark/main.go b/lxd-benchmark/main.go index f9c28d791..76bc9e2fd 100644 --- a/lxd-benchmark/main.go +++ b/lxd-benchmark/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "time" "github.com/lxc/lxd/client" "github.com/lxc/lxd/lxd-benchmark/benchmark" @@ -17,6 +18,8 @@ var argImage = gnuflag.String("image", "ubuntu:", "Image to use for the test") var argPrivileged = gnuflag.Bool("privileged", false, "Use privileged containers") var argStart = gnuflag.Bool("start", true, "Start the container after creation") var argFreeze = gnuflag.Bool("freeze", false, "Freeze the container right after start") +var argReportFile = gnuflag.String("report-file", "", "A CSV file to write test file to. If the file is present, it will be appended to.") +var argReportLabel = gnuflag.String("report-label", "", "A label for the report entry. By default, the action is used.") func main() { err := run(os.Args) @@ -66,32 +69,65 @@ func run(args []string) error { benchmark.PrintServerInfo(c) - switch os.Args[1] { + var report *benchmark.CSVReport + if *argReportFile != "" { + report = &benchmark.CSVReport{Filename: *argReportFile} + if shared.PathExists(*argReportFile) { + err := report.Load() + if err != nil { + return err + } + } + } + + action := os.Args[1] + var duration time.Duration + switch action { case "spawn": - _, err = benchmark.SpawnContainers(c, *argCount, *argParallel, *argImage, *argPrivileged, *argStart, *argFreeze) - return err + duration, err = benchmark.SpawnContainers( + c, *argCount, *argParallel, *argImage, *argPrivileged, *argStart, *argFreeze) + if err != nil { + return err + } case "start": containers, err := benchmark.GetContainers(c) if err != nil { return err } - _, err = benchmark.StartContainers(c, containers, *argParallel) - return err + duration, err = benchmark.StartContainers(c, containers, *argParallel) + if err != nil { + return err + } case "stop": containers, err := benchmark.GetContainers(c) if err != nil { return err } - _, err = benchmark.StopContainers(c, containers, *argParallel) - return err + duration, err = benchmark.StopContainers(c, containers, *argParallel) + if err != nil { + return err + } case "delete": containers, err := benchmark.GetContainers(c) if err != nil { return err } - _, err = benchmark.DeleteContainers(c, containers, *argParallel) - return err + duration, err = benchmark.DeleteContainers(c, containers, *argParallel) + if err != nil { + return err + } } + if report != nil { + label := action + if *argReportLabel != "" { + label = *argReportLabel + } + report.AddRecord(label, duration) + err := report.Write() + if err != nil { + return err + } + } return nil } From 564795f4e8f62c3c3489c6bd5e00c235ca36008d Mon Sep 17 00:00:00 2001 From: Alberto Donato <[email protected]> Date: Thu, 7 Sep 2017 10:51:05 +0200 Subject: [PATCH 2/4] benchmark: fix ensureImage when a local alias is passed Signed-off-by: Alberto Donato <[email protected]> --- lxd-benchmark/benchmark/benchmark.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lxd-benchmark/benchmark/benchmark.go b/lxd-benchmark/benchmark/benchmark.go index 1f4bf82e7..aca077cb4 100644 --- a/lxd-benchmark/benchmark/benchmark.go +++ b/lxd-benchmark/benchmark/benchmark.go @@ -275,12 +275,22 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) { logf("Failed to import image: %s", err) return "", err } - } else { - logf("Found image in local store: %s", fingerprint) } } else { + alias, _, err := c.GetImageAlias(image) + if err == nil { + fingerprint = alias.Target + } else { + _, _, err = c.GetImage(image) + } + + if err != nil { + logf("Image not found in local store: %s", image) + return "", err + } fingerprint = image - logf("Found image in local store: %s", fingerprint) } + + logf("Found image in local store: %s", fingerprint) return fingerprint, nil } From ef08b624130621f4c38528b03dfa304614b7e15f Mon Sep 17 00:00:00 2001 From: Alberto Donato <[email protected]> Date: Thu, 7 Sep 2017 11:00:11 +0200 Subject: [PATCH 3/4] benchmark: more cleanups Signed-off-by: Alberto Donato <[email protected]> --- lxd-benchmark/benchmark/benchmark.go | 58 ++++++++++-------------------------- lxd-benchmark/benchmark/operation.go | 18 +++++++++++ 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/lxd-benchmark/benchmark/benchmark.go b/lxd-benchmark/benchmark/benchmark.go index aca077cb4..1d25968e7 100644 --- a/lxd-benchmark/benchmark/benchmark.go +++ b/lxd-benchmark/benchmark/benchmark.go @@ -55,23 +55,19 @@ func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image strin defer wg.Done() name := getContainerName(count, index) - - err := createContainer(c, fingerprint, name, privileged) - if err != nil { + if err := createContainer(c, fingerprint, name, privileged); err != nil { logf("Failed to spawn container '%s': %s", name, err) return } if start { - err := startContainer(c, name) - if err != nil { + if err := startContainer(c, name); err != nil { logf("Failed to start container '%s': %s", name, err) return } if freeze { - err := freezeContainer(c, name) - if err != nil { + if err := freezeContainer(c, name); err != nil { logf("Failed to freeze container '%s': %s", name, err) return } @@ -96,9 +92,7 @@ func CreateContainers(c lxd.ContainerServer, count int, parallel int, fingerprin defer wg.Done() name := getContainerName(count, index) - - err := createContainer(c, fingerprint, name, privileged) - if err != nil { + if err := createContainer(c, fingerprint, name, privileged); err != nil { logf("Failed to spawn container '%s': %s", name, err) return } @@ -144,8 +138,7 @@ func StartContainers(c lxd.ContainerServer, containers []api.Container, parallel container := containers[index] if !container.IsActive() { - err := startContainer(c, container.Name) - if err != nil { + if err := startContainer(c, container.Name); err != nil { logf("Failed to start container '%s': %s", container.Name, err) return } @@ -173,8 +166,7 @@ func StopContainers(c lxd.ContainerServer, containers []api.Container, parallel container := containers[index] if container.IsActive() { - err := stopContainer(c, container.Name) - if err != nil { + if err := stopContainer(c, container.Name); err != nil { logf("Failed to stop container '%s': %s", container.Name, err) return } @@ -200,26 +192,17 @@ func DeleteContainers(c lxd.ContainerServer, containers []api.Container, paralle deleteContainer := func(index int, wg *sync.WaitGroup) { defer wg.Done() - ct := containers[index] - - if ct.IsActive() { - err := stopContainer(c, ct.Name) - if err != nil { - logf("Failed to stop container '%s': %s", ct.Name, err) + container := containers[index] + name := container.Name + if container.IsActive() { + if err := stopContainer(c, name); err != nil { + logf("Failed to stop container '%s': %s", name, err) return } } - // Delete - op, err := c.DeleteContainer(ct.Name) - if err != nil { - logf("Failed to delete container: %s", ct.Name) - return - } - - err = op.Wait() - if err != nil { - logf("Failed to delete container: %s", ct.Name) + if err := deleteContainer(c, name); err != nil { + logf("Failed to delete container: %s", name) return } } @@ -250,13 +233,11 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) { fingerprint = "default" } - alias, _, err := imageServer.GetImageAlias(fingerprint) - if err == nil { + if alias, _, err := imageServer.GetImageAlias(fingerprint); err == nil { fingerprint = alias.Target } - _, _, err = c.GetImage(fingerprint) - if err != nil { + if _, _, err = c.GetImage(fingerprint); err != nil { logf("Importing image into local store: %s", fingerprint) image, _, err := imageServer.GetImage(fingerprint) if err != nil { @@ -264,14 +245,7 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) { return "", err } - op, err := c.CopyImage(imageServer, *image, nil) - if err != nil { - logf("Failed to import image: %s", err) - return "", err - } - - err = op.Wait() - if err != nil { + if err := copyImage(c, imageServer, *image); err != nil { logf("Failed to import image: %s", err) return "", err } diff --git a/lxd-benchmark/benchmark/operation.go b/lxd-benchmark/benchmark/operation.go index e01e5aaf3..b4f4f3cea 100644 --- a/lxd-benchmark/benchmark/operation.go +++ b/lxd-benchmark/benchmark/operation.go @@ -58,3 +58,21 @@ func freezeContainer(c lxd.ContainerServer, name string) error { return op.Wait() } + +func deleteContainer(c lxd.ContainerServer, name string) error { + op, err := c.DeleteContainer(name) + if err != nil { + return err + } + + return op.Wait() +} + +func copyImage(c lxd.ContainerServer, s lxd.ImageServer, image api.Image) error { + op, err := c.CopyImage(s, image, nil) + if err != nil { + return err + } + + return op.Wait() +} From d212d23031cdaaa2055a9db0838c5f94587644af Mon Sep 17 00:00:00 2001 From: Alberto Donato <[email protected]> Date: Thu, 17 Aug 2017 16:32:28 +0200 Subject: [PATCH 4/4] add performance regression tests Signed-off-by: Alberto Donato <[email protected]> --- test/perf.sh | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 test/perf.sh diff --git a/test/perf.sh b/test/perf.sh new file mode 100755 index 000000000..ab9b76551 --- /dev/null +++ b/test/perf.sh @@ -0,0 +1,82 @@ +#!/bin/sh -eu +# +# Performance tests runner +# + +[ -n "${GOPATH:-}" ] && export "PATH=${GOPATH}/bin:${PATH}" + +PERF_LOG_CSV="perf.csv" + +import_subdir_files() { + test "$1" + # shellcheck disable=SC2039 + local file + for file in "$1"/*.sh; do + # shellcheck disable=SC1090 + . "$file" + done +} + +import_subdir_files includes + +log_message() { + echo "==>" "$@" +} + +run_benchmark() { + # shellcheck disable=SC2039 + local label description opts + label="$1" + description="$2" + shift 2 + + log_message "Benchmark start: $label - $description" + lxd_benchmark "$@" --report-file "$PERF_LOG_CSV" --report-label "$label" + log_message "Benchmark completed: $label" +} + +lxd_benchmark() { + # shellcheck disable=SC2039 + local opts + [ "${LXD_TEST_IMAGE:-}" ] && opts="--image $LXD_TEST_IMAGE" || opts="" + lxd-benchmark "$@" $opts +} + +cleanup() { + if [ "$TEST_RESULT" != "success" ]; then + rm -f "$PERF_LOG_CSV" + fi + lxd_benchmark delete # ensure all test containers have been deleted + kill_lxd "$LXD_DIR" + cleanup_lxds "$TEST_DIR" + log_message "Performance tests result: $TEST_RESULT" +} + +trap cleanup EXIT HUP INT TERM + +# Setup test directories +TEST_DIR=$(mktemp -d -p "$(pwd)" tmp.XXX) +LXD_DIR=$(mktemp -d -p "${TEST_DIR}" XXX) +export LXD_DIR +chmod +x "${TEST_DIR}" "${LXD_DIR}" + +if [ -z "${LXD_BACKEND:-}" ]; then + LXD_BACKEND="dir" +fi + +import_storage_backends + +spawn_lxd "${LXD_DIR}" true + +# shellcheck disable=SC2034 +TEST_RESULT=failure + +run_benchmark "create-one" "create 1 container" spawn --count 1 --start=false +run_benchmark "start-one" "start 1 container" start +run_benchmark "stop-one" "stop 1 container" stop +run_benchmark "delete-one" "delete 1 container" delete +run_benchmark "create-128" "create 128 containers" spawn --count 128 --start=false +run_benchmark "delete-128" "delete 128 containers" delete + +# shellcheck disable=SC2034 +TEST_RESULT=success
_______________________________________________ lxc-devel mailing list [email protected] http://lists.linuxcontainers.org/listinfo/lxc-devel
