[PATCH 2/2] test: initial performance testing infrastructure
From: David Bremner
This is not near as fancy as as the unit tests, on the theory that
the code should typically be crashing when performance tuning.
Nonetheless, there is plenty of room for improvement. Several more of
the pieces of the test infrastructure (e.g. the option parsing) could
be factored out into test/test-lib-common.sh
---
Makefile |3 +-
performance-test/.gitignore|1 +
performance-test/Makefile |7 +++
performance-test/Makefile.local| 24 +
performance-test/README| 36 +
performance-test/basic | 15 ++
performance-test/download/.gitignore |1 +
.../download/notmuch-email-corpus-0.1.tar.gz.asc |9
performance-test/notmuch-perf-test | 25 +
performance-test/perf-test-lib.sh | 57
10 files changed, 177 insertions(+), 1 deletion(-)
create mode 100644 performance-test/.gitignore
create mode 100644 performance-test/Makefile
create mode 100644 performance-test/Makefile.local
create mode 100644 performance-test/README
create mode 100755 performance-test/basic
create mode 100644 performance-test/download/.gitignore
create mode 100644
performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc
create mode 100755 performance-test/notmuch-perf-test
create mode 100644 performance-test/perf-test-lib.sh
diff --git a/Makefile b/Makefile
index bb9c316..5decbea 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,8 @@
all:
# List all subdirectories here. Each contains its own Makefile.local
-subdirs = compat completion emacs lib man parse-time-string util test
+subdirs := compat completion emacs lib man parse-time-string
+subdirs := $(subdirs) performance-test util test
# We make all targets depend on the Makefiles themselves.
global_deps = Makefile Makefile.config Makefile.local \
diff --git a/performance-test/.gitignore b/performance-test/.gitignore
new file mode 100644
index 000..53f2697
--- /dev/null
+++ b/performance-test/.gitignore
@@ -0,0 +1 @@
+tmp.*/
diff --git a/performance-test/Makefile b/performance-test/Makefile
new file mode 100644
index 000..de492a7
--- /dev/null
+++ b/performance-test/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+ $(MAKE) -C .. all
+
+.DEFAULT:
+ $(MAKE) -C .. $@
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
new file mode 100644
index 000..d20457e
--- /dev/null
+++ b/performance-test/Makefile.local
@@ -0,0 +1,24 @@
+# -*- makefile -*-
+
+dir := performance-test
+
+PERFTEST_VERSION := 0.1
+
+TGZFILE := $(dir)/download/notmuch-email-corpus-$(PERFTEST_VERSION).tar.gz
+SIGFILE := $(TGZFILE).asc
+TEST_SCRIPT := $(dir)/notmuch-perf-test
+
+perf-test: setup-perf-test all
+ $(TEST_SCRIPT) $(OPTIONS)
+
+setup-perf-test: $(TGZFILE)
+ gpg --verify $(SIGFILE)
+
+$(TGZFILE):
+ @echo
+ @echo Please download ${TGZFILE}
+ @echo See http://notmuchmail.org/corpus for download locations
+ @echo
+ @false
+
+CLEAN := $(CLEAN) $(dir)/tmp.*
diff --git a/performance-test/README b/performance-test/README
new file mode 100644
index 000..1f98337
--- /dev/null
+++ b/performance-test/README
@@ -0,0 +1,36 @@
+Pre-requisites
+--
+
+In addition to having notmuch, you need:
+
+- gpg
+- gnu tar
+- gnu time
+
+Getting set up to run tests:
+
+
+First, you need to get the corpus.
+
+It should work to run
+
+ % cd download
+ % wget http://notmuchmail.org/releases/notmuch-email-corpus-${V}.tar.gz
+
+Where $V is the current version.
+
+In case that fails or is too slow, check
+
+ http://notmuchmail.org/corpus
+
+for a list of mirrors.
+
+Running tests
+-
+
+The easiest way to run performance tests is to say "make perf-test", (or
+simply run the notmuch-perf-test script). Either command will run all
+available performance tests.
+
+Alternately, you can run a specific subset of tests by simply invoking
+one of the executable scripts in this directory, (such as ./basic).
diff --git a/performance-test/basic b/performance-test/basic
new file mode 100755
index 000..9d015ee
--- /dev/null
+++ b/performance-test/basic
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+. ./perf-test-lib.sh
+
+add_email_corpus
+
+print_header
+
+time_run 'initial notmuch new' 'notmuch new'
+time_run 'second notmuch new' 'notmuch new'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+
+time_done
diff --git a/performance-test/download/.gitignore
b/performance-test/download/.gitignore
new file mode 100644
index 000..335ec95
--- /dev/null
+++ b/performance-test/download/.gitignore
@@ -0,0 +1 @@
+*.tar.gz
diff --git a
[PATCH 2/2] test: initial performance testing infrastructure
From: David Bremner
This is not near as fancy as as the unit tests, on the theory that
the code should typically be crashing when performance tuning.
Nonetheless, there is plenty of room for improvement. Several more of
the pieces of the test infrastructure (e.g. the option parsing) could
be factored out into test/test-lib-common.sh
---
Makefile |3 +-
performance-test/.gitignore|1 +
performance-test/Makefile |7 +++
performance-test/Makefile.local| 24 +
performance-test/README| 36 +
performance-test/basic | 15 ++
performance-test/download/.gitignore |1 +
.../download/notmuch-email-corpus-0.1.tar.gz.asc |9
performance-test/notmuch-perf-test | 25 +
performance-test/perf-test-lib.sh | 57
10 files changed, 177 insertions(+), 1 deletion(-)
create mode 100644 performance-test/.gitignore
create mode 100644 performance-test/Makefile
create mode 100644 performance-test/Makefile.local
create mode 100644 performance-test/README
create mode 100755 performance-test/basic
create mode 100644 performance-test/download/.gitignore
create mode 100644
performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc
create mode 100755 performance-test/notmuch-perf-test
create mode 100644 performance-test/perf-test-lib.sh
diff --git a/Makefile b/Makefile
index bb9c316..5decbea 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,8 @@
all:
# List all subdirectories here. Each contains its own Makefile.local
-subdirs = compat completion emacs lib man parse-time-string util test
+subdirs := compat completion emacs lib man parse-time-string
+subdirs := $(subdirs) performance-test util test
# We make all targets depend on the Makefiles themselves.
global_deps = Makefile Makefile.config Makefile.local \
diff --git a/performance-test/.gitignore b/performance-test/.gitignore
new file mode 100644
index 000..53f2697
--- /dev/null
+++ b/performance-test/.gitignore
@@ -0,0 +1 @@
+tmp.*/
diff --git a/performance-test/Makefile b/performance-test/Makefile
new file mode 100644
index 000..de492a7
--- /dev/null
+++ b/performance-test/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+ $(MAKE) -C .. all
+
+.DEFAULT:
+ $(MAKE) -C .. $@
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
new file mode 100644
index 000..d20457e
--- /dev/null
+++ b/performance-test/Makefile.local
@@ -0,0 +1,24 @@
+# -*- makefile -*-
+
+dir := performance-test
+
+PERFTEST_VERSION := 0.1
+
+TGZFILE := $(dir)/download/notmuch-email-corpus-$(PERFTEST_VERSION).tar.gz
+SIGFILE := $(TGZFILE).asc
+TEST_SCRIPT := $(dir)/notmuch-perf-test
+
+perf-test: setup-perf-test all
+ $(TEST_SCRIPT) $(OPTIONS)
+
+setup-perf-test: $(TGZFILE)
+ gpg --verify $(SIGFILE)
+
+$(TGZFILE):
+ @echo
+ @echo Please download ${TGZFILE}
+ @echo See http://notmuchmail.org/corpus for download locations
+ @echo
+ @false
+
+CLEAN := $(CLEAN) $(dir)/tmp.*
diff --git a/performance-test/README b/performance-test/README
new file mode 100644
index 000..1f98337
--- /dev/null
+++ b/performance-test/README
@@ -0,0 +1,36 @@
+Pre-requisites
+--
+
+In addition to having notmuch, you need:
+
+- gpg
+- gnu tar
+- gnu time
+
+Getting set up to run tests:
+
+
+First, you need to get the corpus.
+
+It should work to run
+
+ % cd download
+ % wget http://notmuchmail.org/releases/notmuch-email-corpus-${V}.tar.gz
+
+Where $V is the current version.
+
+In case that fails or is too slow, check
+
+ http://notmuchmail.org/corpus
+
+for a list of mirrors.
+
+Running tests
+-
+
+The easiest way to run performance tests is to say "make perf-test", (or
+simply run the notmuch-perf-test script). Either command will run all
+available performance tests.
+
+Alternately, you can run a specific subset of tests by simply invoking
+one of the executable scripts in this directory, (such as ./basic).
diff --git a/performance-test/basic b/performance-test/basic
new file mode 100755
index 000..9d015ee
--- /dev/null
+++ b/performance-test/basic
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+. ./perf-test-lib.sh
+
+add_email_corpus
+
+print_header
+
+time_run 'initial notmuch new' 'notmuch new'
+time_run 'second notmuch new' 'notmuch new'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+
+time_done
diff --git a/performance-test/download/.gitignore
b/performance-test/download/.gitignore
new file mode 100644
index 000..335ec95
--- /dev/null
+++ b/performance-test/download/.gitignore
@@ -0,0 +1 @@
+*.tar.gz
diff --git
[PATCH 2/2] test: initial performance testing infrastructure.
From: David Bremner This is not near as fancy as as the unit tests, on the theory that the code should typically be crashing when performance tuning. Nonetheless, there is plenty of room for improvement. Several more of the pieces of the test infrastructure (e.g. the option parsing) could be factored out into test/test-lib-common.sh --- Makefile |3 +- performance-test/.gitignore|2 ++ performance-test/Makefile |7 performance-test/Makefile.local| 28 +++ performance-test/README| 26 ++ performance-test/basic | 12 +++ performance-test/download/.gitignore |2 ++ .../download/mail-corpus-0.1.mbox.sha256 |1 + performance-test/notmuch-perf-test | 25 ++ performance-test/perf-test-lib.sh | 36 10 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 performance-test/.gitignore create mode 100644 performance-test/Makefile create mode 100644 performance-test/Makefile.local create mode 100644 performance-test/README create mode 100755 performance-test/basic create mode 100644 performance-test/download/.gitignore create mode 100644 performance-test/download/mail-corpus-0.1.mbox.sha256 create mode 100755 performance-test/notmuch-perf-test create mode 100644 performance-test/perf-test-lib.sh diff --git a/Makefile b/Makefile index bb9c316..5decbea 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ all: # List all subdirectories here. Each contains its own Makefile.local -subdirs = compat completion emacs lib man parse-time-string util test +subdirs := compat completion emacs lib man parse-time-string +subdirs := $(subdirs) performance-test util test # We make all targets depend on the Makefiles themselves. global_deps = Makefile Makefile.config Makefile.local \ diff --git a/performance-test/.gitignore b/performance-test/.gitignore new file mode 100644 index 000..e757e8d --- /dev/null +++ b/performance-test/.gitignore @@ -0,0 +1,2 @@ +corpus/* +tmp.*/ diff --git a/performance-test/Makefile b/performance-test/Makefile new file mode 100644 index 000..de492a7 --- /dev/null +++ b/performance-test/Makefile @@ -0,0 +1,7 @@ +# See Makefile.local for the list of files to be compiled in this +# directory. +all: + $(MAKE) -C .. all + +.DEFAULT: + $(MAKE) -C .. $@ diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local new file mode 100644 index 000..b38bbbd --- /dev/null +++ b/performance-test/Makefile.local @@ -0,0 +1,28 @@ +# -*- makefile -*- + +dir := performance-test + +PERFTEST_VERSION := 0.1 + +MBNAME := mail-corpus-$(PERFTEST_VERSION).mbox +GZFILE := $(CURDIR)/$(dir)/download/$(MBNAME).gz +MBFILE := $(CURDIR)/$(dir)/download/$(MBNAME) +MBDIR := $(CURDIR)/$(dir)/corpus +SHA256FILE := $(dir)/download/mail-corpus-$(PERFTEST_VERSION).sha256 +TEST_SCRIPT := $(dir)/notmuch-perf-test + +perf-test: setup-perf-test all + $(TEST_SCRIPT) $(OPTIONS) + +setup-perf-test: $(MBFILE) + mb2md -s $(MBFILE) -d $(MBDIR) + +$(GZFILE): + wget -O $@ http://notmuchmail.org/releases/$(MBNAME).gz + +$(MBFILE): $(GZFILE) + gunzip -c $(GZFILE) > $(MBFILE) + sha256sum -c $(MBFILE).sha256 + +CLEAN := $(CLEAN) $(dir)/tmp.* +DISTCLEAN := $(DISTCLEAN) $(GZFILE) $(MBFILE) $(dir)/corpus diff --git a/performance-test/README b/performance-test/README new file mode 100644 index 000..d22438a --- /dev/null +++ b/performance-test/README @@ -0,0 +1,26 @@ +Pre-requisites +-- + +In addition to having notmuch, you need: + +- mb2md +- gnu time +- wget +- sha256sum + +Getting set up to run tests: + + +% make setup-perf-test + +This requires a net connection to download the test corpus. + +Running tests +- + +The easiest way to run performance tests is to say "make perf-test", (or +simply run the notmuch-perf-test script). Either command will run all +available performance tests. + +Alternately, you can run a specific subset of tests by simply invoking +one of the executable scripts in this directory, (such as ./basic). diff --git a/performance-test/basic b/performance-test/basic new file mode 100755 index 000..16b7d03 --- /dev/null +++ b/performance-test/basic @@ -0,0 +1,12 @@ + +. ./perf-test-lib.sh + +add_email_corpus + +time_run 'initial notmuch new' 'notmuch new' +time_run 'second notmuch new' 'notmuch new' +time_run 'dump *' 'notmuch dump > tags.out' +time_run 'restore *' 'notmuch restore < tags.out' +time_run 'tag * +new_tag' "notmuch tag +new_tag '*'" + +time_done diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore new file mode 100644 index 000..b08aedd --- /dev/null +++ b/performance-test/download/.gitignore @@ -0,0 +1,2 @@ +*.mbox.g
[PATCH 2/2] test: initial performance testing infrastructure.
From: David Bremner This is not near as fancy as as the unit tests, on the theory that the code should typically be crashing when performance tuning. Nonetheless, there is plenty of room for improvement. Several more of the pieces of the test infrastructure (e.g. the option parsing) could be factored out into test/test-lib-common.sh --- Makefile |3 +- performance-test/.gitignore|2 ++ performance-test/Makefile |7 performance-test/Makefile.local| 28 +++ performance-test/README| 26 ++ performance-test/basic | 12 +++ performance-test/download/.gitignore |2 ++ .../download/mail-corpus-0.1.mbox.sha256 |1 + performance-test/notmuch-perf-test | 25 ++ performance-test/perf-test-lib.sh | 36 10 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 performance-test/.gitignore create mode 100644 performance-test/Makefile create mode 100644 performance-test/Makefile.local create mode 100644 performance-test/README create mode 100755 performance-test/basic create mode 100644 performance-test/download/.gitignore create mode 100644 performance-test/download/mail-corpus-0.1.mbox.sha256 create mode 100755 performance-test/notmuch-perf-test create mode 100644 performance-test/perf-test-lib.sh diff --git a/Makefile b/Makefile index bb9c316..5decbea 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ all: # List all subdirectories here. Each contains its own Makefile.local -subdirs = compat completion emacs lib man parse-time-string util test +subdirs := compat completion emacs lib man parse-time-string +subdirs := $(subdirs) performance-test util test # We make all targets depend on the Makefiles themselves. global_deps = Makefile Makefile.config Makefile.local \ diff --git a/performance-test/.gitignore b/performance-test/.gitignore new file mode 100644 index 000..e757e8d --- /dev/null +++ b/performance-test/.gitignore @@ -0,0 +1,2 @@ +corpus/* +tmp.*/ diff --git a/performance-test/Makefile b/performance-test/Makefile new file mode 100644 index 000..de492a7 --- /dev/null +++ b/performance-test/Makefile @@ -0,0 +1,7 @@ +# See Makefile.local for the list of files to be compiled in this +# directory. +all: + $(MAKE) -C .. all + +.DEFAULT: + $(MAKE) -C .. $@ diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local new file mode 100644 index 000..b38bbbd --- /dev/null +++ b/performance-test/Makefile.local @@ -0,0 +1,28 @@ +# -*- makefile -*- + +dir := performance-test + +PERFTEST_VERSION := 0.1 + +MBNAME := mail-corpus-$(PERFTEST_VERSION).mbox +GZFILE := $(CURDIR)/$(dir)/download/$(MBNAME).gz +MBFILE := $(CURDIR)/$(dir)/download/$(MBNAME) +MBDIR := $(CURDIR)/$(dir)/corpus +SHA256FILE := $(dir)/download/mail-corpus-$(PERFTEST_VERSION).sha256 +TEST_SCRIPT := $(dir)/notmuch-perf-test + +perf-test: setup-perf-test all + $(TEST_SCRIPT) $(OPTIONS) + +setup-perf-test: $(MBFILE) + mb2md -s $(MBFILE) -d $(MBDIR) + +$(GZFILE): + wget -O $@ http://notmuchmail.org/releases/$(MBNAME).gz + +$(MBFILE): $(GZFILE) + gunzip -c $(GZFILE) > $(MBFILE) + sha256sum -c $(MBFILE).sha256 + +CLEAN := $(CLEAN) $(dir)/tmp.* +DISTCLEAN := $(DISTCLEAN) $(GZFILE) $(MBFILE) $(dir)/corpus diff --git a/performance-test/README b/performance-test/README new file mode 100644 index 000..d22438a --- /dev/null +++ b/performance-test/README @@ -0,0 +1,26 @@ +Pre-requisites +-- + +In addition to having notmuch, you need: + +- mb2md +- gnu time +- wget +- sha256sum + +Getting set up to run tests: + + +% make setup-perf-test + +This requires a net connection to download the test corpus. + +Running tests +- + +The easiest way to run performance tests is to say "make perf-test", (or +simply run the notmuch-perf-test script). Either command will run all +available performance tests. + +Alternately, you can run a specific subset of tests by simply invoking +one of the executable scripts in this directory, (such as ./basic). diff --git a/performance-test/basic b/performance-test/basic new file mode 100755 index 000..16b7d03 --- /dev/null +++ b/performance-test/basic @@ -0,0 +1,12 @@ + +. ./perf-test-lib.sh + +add_email_corpus + +time_run 'initial notmuch new' 'notmuch new' +time_run 'second notmuch new' 'notmuch new' +time_run 'dump *' 'notmuch dump > tags.out' +time_run 'restore *' 'notmuch restore < tags.out' +time_run 'tag * +new_tag' "notmuch tag +new_tag '*'" + +time_done diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore new file mode 100644 index 000..b08aedd --- /dev/null +++ b/performance-test/download/.gitignore @@ -0,0 +1,2 @@ +*.mbox
