[PATCH] notmuch dump: default to batch-tag format.

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

Although we didn't formally deprecate the old format, the new one has
been available for a year.
---
 NEWS   | 10 ++
 notmuch-dump.c |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 28788d8..bdb248e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,13 @@
+Notmuch 0.18 (2014-xx-xx)
+=
+
+Command-Line Interface
+--
+
+`notmuch dump` now defaults to `batch-tag` format.
+
+  The old format is still available with `--format=sup`.
+
 Notmuch 0.17 (2013-12-30)
 =

diff --git a/notmuch-dump.c b/notmuch-dump.c
index f8edda7..158142f 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -40,7 +40,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, 
char *argv[])
 char *output_file_name = NULL;
 int opt_index;

-int output_format = DUMP_FORMAT_SUP;
+int output_format = DUMP_FORMAT_BATCH_TAG;

 notmuch_opt_desc_t options[] = {
{ NOTMUCH_OPT_KEYWORD, _format, "format", 'f',
-- 
1.8.5.2

[PATCH v2] util: detect byte order

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

Unfortunately old versions of GCC and clang do not provide byte order
macros, so we re-invent them.

If UTIL_BYTE_ORDER is not defined or defined to 0, we fall back to
macros supported by recent versions of GCC and clang
---

I think I got all of Tomi's comments, including the most serious one
about the test in endian-utils.h (LITTLE twice instead of BIG,
LITTLE).

 configure  | 24 ++--
 lib/libsha1.c  | 21 +++--
 util/endian-util.h | 38 ++
 3 files changed, 67 insertions(+), 16 deletions(-)
 create mode 100644 util/endian-util.h

diff --git a/configure b/configure
index 1a8e939..a0c6771 100755
--- a/configure
+++ b/configure
@@ -441,6 +441,19 @@ else
 EOF
 fi

+printf "Checking byte order... "
+cat> _byteorder.c <
+#include 
+uint32_t test = 0x34333231;
+int main() { printf("%.4s\n", (const char*)); return 0; }
+EOF
+${CC} ${CFLAGS} _byteorder.c -o _byteorder > /dev/null 2>&1
+util_byte_order=$(./_byteorder)
+echo $util_byte_order
+
+#rm -f _byteorder _byteorder.c
+
 if [ $errors -gt 0 ]; then
 cat < /* for memcpy() etc.*/
-
+#include "endian-util.h"
 #include "libsha1.h"

 #if defined(__cplusplus)
@@ -49,20 +49,13 @@ extern "C"

 #define bswap_32(x) ((rotr32((x), 24) & 0x00ff00ff) | (rotr32((x), 8) & 
0xff00ff00))

-/* The macros __BYTE_ORDER__ and __ORDER_*_ENDIAN__ are GNU C
- * extensions. They are also supported by clang as of v3.2 */
-
-#ifdef __BYTE_ORDER__
-#  if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#define bsw_32(p,n) \
-   { int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
-#  elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-#define bsw_32(p,n)
-#  else
-#error "unknown byte order"
-#  endif
+#if (UTIL_BYTE_ORDER == UTIL_ORDER_LITTLE_ENDIAN)
+#  define bsw_32(p,n) \
+ { int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
+#elif (UTIL_BYTE_ORDER == UTIL_ORDER_BIG_ENDIAN)
+#  define bsw_32(p,n)
 #else
-#error "macro __BYTE_ORDER__ is not defined"
+#  error "Unsupported byte order"
 #endif

 #define SHA1_MASK   (SHA1_BLOCK_SIZE - 1)
diff --git a/util/endian-util.h b/util/endian-util.h
new file mode 100644
index 000..bc80c40
--- /dev/null
+++ b/util/endian-util.h
@@ -0,0 +1,38 @@
+/* this file mimics the macros present in recent GCC and CLANG */
+
+#ifndef _ENDIAN_UTIL_H
+#define _ENDIAN_UTIL_H
+
+/* This are prefixed with UTIL to avoid collisions
+ *
+ * You can use something like the following to define UTIL_BYTE_ORDER
+ * in a configure script.
+ */
+#if 0
+#include 
+#include 
+uint32_t test = 0x34333231;
+int main() { printf("%.4s\n", (const char*)); return 0; }
+#endif
+
+#define UTIL_ORDER_BIG_ENDIAN4321
+#define UTIL_ORDER_LITTLE_ENDIAN  1234
+
+
+#if !defined(UTIL_BYTE_ORDER) || ((UTIL_BYTE_ORDER != UTIL_ORDER_BIG_ENDIAN) 
&& \
+ (UTIL_BYTE_ORDER != UTIL_ORDER_LITTLE_ENDIAN))
+#undef UTIL_BYTE_ORDER
+#ifdef __BYTE_ORDER__
+#  if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define UTIL_BYTE_ORDER UTIL_ORDER_LITTLE_ENDIAN
+#  elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define UTIL_BYTE_ORDER UTIL_ORDER_BIG_ENDIAN
+#  else
+#error "Unsupported __BYTE_ORDER__"
+#  endif
+#else
+#  error "UTIL_BYTE_ORDER not correctly defined and __BYTE_ORDER__ not 
defined."
+#endif
+#endif
+
+#endif
-- 
1.8.4.2

[PATCH] util: detect byte order

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

Unfortunately old versions of GCC and clang do not provide byte order
macros, so we re-invent them.

If UTIL_BYTE_ORDER is not defined, we fall back to macros supported by
recent versions of GCC and clang
---

I pushed the series
id:1385328583-24602-1-git-send-email-david at tethera.net; unfortunately
that broke compilation on old versions of GCC, in particular on our
buildbot. Here is a proposed fix for the fix.

 configure  | 26 --
 lib/libsha1.c  | 19 +--
 util/endian-util.h | 39 +++
 3 files changed, 68 insertions(+), 16 deletions(-)
 create mode 100644 util/endian-util.h

diff --git a/configure b/configure
index 1a8e939..02d6396 100755
--- a/configure
+++ b/configure
@@ -441,6 +441,21 @@ else
 EOF
 fi

+printf "Checking byte order... "
+cat> _byteorder.c <
+#include 
+#include 
+uint32_t test = 0x31323334;
+char buf[5];
+int main() { memcpy(buf, , 4); buf[4] = '\0'; printf("%s\n", buf); return 
0; }
+EOF
+${CC} ${CFLAGS} _byteorder.c -o _byteorder > /dev/null 2>&1
+util_byte_order=$(./_byteorder)
+echo $util_byte_order
+
+#rm -f _byteorder _byteorder.c
+
 if [ $errors -gt 0 ]; then
 cat < /* for memcpy() etc.*/
-
+#include "endian-util.h"
 #include "libsha1.h"

 #if defined(__cplusplus)
@@ -49,20 +49,11 @@ extern "C"

 #define bswap_32(x) ((rotr32((x), 24) & 0x00ff00ff) | (rotr32((x), 8) & 
0xff00ff00))

-/* The macros __BYTE_ORDER__ and __ORDER_*_ENDIAN__ are GNU C
- * extensions. They are also supported by clang as of v3.2 */
-
-#ifdef __BYTE_ORDER__
-#  if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
-#define bsw_32(p,n) \
-   { int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
-#  elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
-#define bsw_32(p,n)
-#  else
-#error "unknown byte order"
-#  endif
+#if (UTIL_BYTE_ORDER == UTIL_ORDER_LITTLE_ENDIAN)
+#  define bsw_32(p,n) \
+ { int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
 #else
-#error "macro __BYTE_ORDER__ is not defined"
+#  define bsw_32(p,n)
 #endif

 #define SHA1_MASK   (SHA1_BLOCK_SIZE - 1)
diff --git a/util/endian-util.h b/util/endian-util.h
new file mode 100644
index 000..cbecf66
--- /dev/null
+++ b/util/endian-util.h
@@ -0,0 +1,39 @@
+/* this file mimics the macros present in recent GCC and CLANG */
+
+#ifndef _ENDIAN_UTIL_H
+#define _ENDIAN_UTIL_H
+
+/* This are prefixed with UTIL to avoid collisions
+ *
+ * You can use something like the following to define UTIL_BYTE_ORDER
+ * in a configure script.
+ */
+#if 0
+#include 
+#include 
+#include 
+uint32_t test = 0x31323334;
+char buf[5];
+int main() { memcpy(buf, , 4); buf[4] = '\0'; printf("%s\n", buf); return 
0; }
+#endif
+
+#define UTIL_ORDER_LITTLE_ENDIAN 4321
+#define UTIL_ORDER_BIG_ENDIAN1234
+
+
+#if !defined(UTIL_BYTE_ORDER) || ((UTIL_BYTE_ORDER != 
UTIL_ORDER_LITTLE_ENDIAN) && \
+  (UTIL_BYTE_ORDER != 
UTIL_ORDER_LITTLE_ENDIAN))
+#ifdef __BYTE_ORDER__
+#  if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define UTIL_BYTE_ORDER UTIL_ORDER_LITTLE_ENDIAN
+#  elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define UTIL_BYTE_ORDER UTIL_ORDER_BIG_ENDIAN
+#  else
+#error "Unsupported __BYTE_ORDER__"
+#  endif
+#else
+#  error "UTIL_BYTE_ORDER not correctly defined and __BYTE_ORDER__ not 
defined."
+#endif
+#endif
+
+#endif
-- 
1.8.4.2

[PATCH 2/2] NEWS: News for big endian sha1 bug fix.

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

We could give more details about how to migrate tags, but I'm not sure
that it's a practical problem, or just a theoretical one.
---
 NEWS | 14 +-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 3383ecf..31c6284 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,18 @@
-Notmuch 0.17~rc1 (2013-11-20)
+Notmuch 0.17~rc2 (2013-xx-yy)
 =

+Incompatible change in SHA1 computation
+---
+
+Previously on big endian architectures like sparc and powerpc the
+computation of SHA1 hashes was incorrect. This meant that messages
+with overlong or missing message-ids were given different computed
+message-ids than on more common little endian architectures like i386
+and amd64.  If you use notmuch on a big endian architecture, you are
+strongly advised to make a backup of your tags using `notmuch dump`
+before this upgrade. It should be possible to migrate the tags using a
+script.
+
 Command-Line Interface
 --

-- 
1.8.4.2

[PATCH 1/2] lib: fix byte order test in libsha1.c

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

Previously PLATFORM_BYTE_ORDER and IS_LITTLE_ENDIAN were not defined,
so the little endian code was always compiled in.

This will have the effect that the "SHA1s" on big endian architectures
will change (i.e. become actual sha1s). So someone re-indexing their
database could conceivable lose tags on messages without a message-id
header.
---
 lib/libsha1.c | 14 ++
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/libsha1.c b/lib/libsha1.c
index 5d16f6a..794854b 100644
--- a/lib/libsha1.c
+++ b/lib/libsha1.c
@@ -49,11 +49,17 @@ extern "C"

 #define bswap_32(x) ((rotr32((x), 24) & 0x00ff00ff) | (rotr32((x), 8) & 
0xff00ff00))

-#if (PLATFORM_BYTE_ORDER == IS_LITTLE_ENDIAN)
-#define bsw_32(p,n) \
-{ int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
+#ifdef __BYTE_ORDER__
+#  if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define bsw_32(p,n) \
+   { int _i = (n); while(_i--) ((uint32_t*)p)[_i] = 
bswap_32(((uint32_t*)p)[_i]); }
+#  elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define bsw_32(p,n)
+#  else
+#error "unknown byte order"
+#  endif
 #else
-#define bsw_32(p,n)
+#error "macro __BYTE_ORDER__ is not defined"
 #endif

 #define SHA1_MASK   (SHA1_BLOCK_SIZE - 1)
-- 
1.8.4.2

[PATCH v2] test: replace $PWD with YYY in emacs & emacs-show tests

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

When executed command line is written to *Notmuch errors* buffer,
shell-quote-argument will backslash-escape any char that is not in
"POSIX filename characters" (i.e. matching "[^-0-9a-zA-Z_./\n]").

Currently in two emacs tests shell has expanded $PWD as part of
emacs variable, which will later be fed to #'shell-quote-argument
and finally written to ERROR file. If $PWD contained non-POSIX
filename characters, data in ERROR file will not match $PWD when
later comparing in shell. Therefore, in these two particular cases
the escaped $PWD is replaced with YYY in ERROR file and expected
content is adjusted accordingly.
---

In this second version, pass the command name to filter as a parameter

 test/emacs   | 15 ---
 test/emacs-show  | 10 +-
 test/test-lib.sh | 11 +++
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/test/emacs b/test/emacs
index 3b3b14d..456435c 100755
--- a/test/emacs
+++ b/test/emacs
@@ -881,15 +881,16 @@ test_emacs "(let ((notmuch-command \"$PWD/notmuch_fail\"))
   (with-current-buffer \"*Notmuch errors*\"
  (test-output \"ERROR\"))
   (test-output))"
-sed -i -e 's/^\[.*\]$/[XXX]/' ERROR
-test_expect_equal "$(cat OUTPUT; echo ---; cat MESSAGES; echo ---; cat ERROR)" 
"\
+
+test_expect_equal "$(notmuch_emacs_error_sanitize notmuch_fail OUTPUT MESSAGES 
ERROR)" "\
+=== OUTPUT ===
 End of search results.

-$PWD/notmuch_fail exited with status 1 (see *Notmuch errors* for more details)

+=== MESSAGES ===
+YYY/notmuch_fail exited with status 1 (see *Notmuch errors* for more details)
+=== ERROR ===
 [XXX]
-$PWD/notmuch_fail exited with status 1
-command: $PWD/notmuch_fail search --format\=sexp --format-version\=2 
--sort\=newest-first tag\:inbox
+YYY/notmuch_fail exited with status 1
+command: YYY/notmuch_fail search --format\=sexp --format-version\=2 
--sort\=newest-first tag\:inbox
 exit status: 1"

 test_begin_subtest "Search handles subprocess warnings"
diff --git a/test/emacs-show b/test/emacs-show
index fb23db4..ec86333 100755
--- a/test/emacs-show
+++ b/test/emacs-show
@@ -181,14 +181,14 @@ test_emacs "(let ((notmuch-command \"$PWD/notmuch_fail\"))
   (with-current-buffer \"*Notmuch errors*\"
  (test-output \"ERROR\"))
   (test-output))"
-sed -i -e 's/^\[.*\]$/[XXX]/' ERROR
-test_expect_equal "$(cat OUTPUT; echo ---; cat MESSAGES; echo ---; cat ERROR)" 
"\

+test_expect_equal "$(notmuch_emacs_error_sanitize notmuch_fail OUTPUT MESSAGES 
ERROR)" "\
+=== OUTPUT ===
+=== MESSAGES ===
 This is an error (see *Notmuch errors* for more details)

+=== ERROR ===
 [XXX]
 This is an error
-command: $PWD/notmuch_fail show --format\\=sexp --format-version\\=1 
--exclude\\=false \\' \\* \\'
+command: YYY/notmuch_fail show --format\\=sexp --format-version\\=1 
--exclude\\=false \\' \\* \\'
 exit status: 1
 stderr:
 This is an error
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 2aa4dfc..8611ba5 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -628,6 +628,17 @@ notmuch_json_show_sanitize ()
-e 's|"filename": "/[^"]*",|"filename": "Y",|g'
 }

+notmuch_emacs_error_sanitize ()
+{
+local command=$1
+shift
+for file in "$@"; do
+   echo "=== $file ==="
+   cat "$file"
+done | sed  \
+   -e 's/^\[.*\]$/[XXX]/' \
+   -e "s|^\(command: \)\{0,1\}/.*/$command|\1YYY/$command|"
+}
 # End of notmuch helper functions

 # Use test_set_prereq to tell that a particular prerequisite is available.
-- 
1.8.4.2

[PATCH] test: replace $PWD with YYY in emacs & emacs-show tests

[da...@tethera.net]

From: David Bremner <da...@tethera.net>

When executed command line is written to *Notmuch errors* buffer,
shell-quote-argument will backslash-escape any char that is not in
"POSIX filename characters" (i.e. matching "[^-0-9a-zA-Z_./\n]").

Currently in two emacs tests shell has expanded $PWD as part of
emacs variable, which will later be fed to #'shell-quote-argument
and finally written to ERROR file. If $PWD contained non-POSIX
filename characters, data in ERROR file will not match $PWD when
later comparing in shell. Therefore, in these two particular cases
the escaped $PWD is replaced with YYY in ERROR file and expected
content is adjusted accordingly.
---

The commit message is shamelessly boosted from Tomi's patch.  To be
honest I didn't notice that Tomi had already sent a patch when I
started on this, and then I was too stubborn to stop. I'm not sure
which is better. It depends whether you think the cosmetic stuff is an
improvement, or just gratuitous. For what it's worth (not that much as
Tomi points out) this should work ok with spaces in the path. Better
to light a candle than to curse the darkness, etc...

 test/emacs   | 15 ---
 test/emacs-show  | 10 +-
 test/test-lib.sh |  9 +
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/test/emacs b/test/emacs
index 3b3b14d..dd544f5e 100755
--- a/test/emacs
+++ b/test/emacs
@@ -881,15 +881,16 @@ test_emacs "(let ((notmuch-command \"$PWD/notmuch_fail\"))
   (with-current-buffer \"*Notmuch errors*\"
  (test-output \"ERROR\"))
   (test-output))"
-sed -i -e 's/^\[.*\]$/[XXX]/' ERROR
-test_expect_equal "$(cat OUTPUT; echo ---; cat MESSAGES; echo ---; cat ERROR)" 
"\
+
+test_expect_equal "$(notmuch_emacs_error_sanitize OUTPUT MESSAGES ERROR)" "\
+=== OUTPUT ===
 End of search results.

-$PWD/notmuch_fail exited with status 1 (see *Notmuch errors* for more details)

+=== MESSAGES ===
+YYY/notmuch_fail exited with status 1 (see *Notmuch errors* for more details)
+=== ERROR ===
 [XXX]
-$PWD/notmuch_fail exited with status 1
-command: $PWD/notmuch_fail search --format\=sexp --format-version\=2 
--sort\=newest-first tag\:inbox
+YYY/notmuch_fail exited with status 1
+command: YYY/notmuch_fail search --format\=sexp --format-version\=2 
--sort\=newest-first tag\:inbox
 exit status: 1"

 test_begin_subtest "Search handles subprocess warnings"
diff --git a/test/emacs-show b/test/emacs-show
index fb23db4..21f1c68 100755
--- a/test/emacs-show
+++ b/test/emacs-show
@@ -181,14 +181,14 @@ test_emacs "(let ((notmuch-command \"$PWD/notmuch_fail\"))
   (with-current-buffer \"*Notmuch errors*\"
  (test-output \"ERROR\"))
   (test-output))"
-sed -i -e 's/^\[.*\]$/[XXX]/' ERROR
-test_expect_equal "$(cat OUTPUT; echo ---; cat MESSAGES; echo ---; cat ERROR)" 
"\

+test_expect_equal "$(notmuch_emacs_error_sanitize OUTPUT MESSAGES ERROR)" "\
+=== OUTPUT ===
+=== MESSAGES ===
 This is an error (see *Notmuch errors* for more details)

+=== ERROR ===
 [XXX]
 This is an error
-command: $PWD/notmuch_fail show --format\\=sexp --format-version\\=1 
--exclude\\=false \\' \\* \\'
+command: YYY/notmuch_fail show --format\\=sexp --format-version\\=1 
--exclude\\=false \\' \\* \\'
 exit status: 1
 stderr:
 This is an error
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 2aa4dfc..611aee3 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -628,6 +628,15 @@ notmuch_json_show_sanitize ()
-e 's|"filename": "/[^"]*",|"filename": "Y",|g'
 }

+notmuch_emacs_error_sanitize ()
+{
+for file in "$@"; do
+   echo "=== $file ==="
+   cat "$file"
+done | sed  \
+   -e 's/^\[.*\]$/[XXX]/' \
+   -e 's|^\(command: \)\{0,1\}/.*/notmuch_fail|\1YYY/notmuch_fail|'
+}
 # End of notmuch helper functions

 # Use test_set_prereq to tell that a particular prerequisite is available.
-- 
1.8.4.2

[PATCH] lib: update documentation of callback functions for database_compact and database_upgrade.

[da...@tethera.net]

From: David Bremner 

Compact was missing callback documentation entirely, and upgrade did not 
discuss the
closure parameter.
---
This patch depends on 

 id:2a58adbdc1257f16579692544b4bcbadca3d3045.1383315568.git.jani at 
nikula.org

BTW, I didn't completely understand the remark about SONAME bumps;
since we're providing new symbols, it doesn't really matter what the
signature is?

 lib/notmuch.h | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index cd301a4..82fd599 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -227,6 +227,9 @@ typedef void (*notmuch_compact_status_cb_t)(const char 
*message, void *closure);
  * The database will be opened with NOTMUCH_DATABASE_MODE_READ_WRITE
  * during the compaction process to ensure no writes are made.
  *
+ * If the optional callback function 'status_cb' is non-NULL, it will
+ * be called with diagnostic and informational messages. The argument
+ * 'closure' is passed verbatim to any callback invoked.
  */
 notmuch_status_t
 notmuch_database_compact (const char* path,
@@ -270,7 +273,8 @@ notmuch_database_needs_upgrade (notmuch_database_t 
*database);
  * provide progress indication to the user. If non-NULL it will be
  * called periodically with 'progress' as a floating-point value in
  * the range of [0.0 .. 1.0] indicating the progress made so far in
- * the upgrade process.
+ * the upgrade process.  The argument 'closure' is passed verbatim to
+ * any callback invoked.
  */
 notmuch_status_t
 notmuch_database_upgrade (notmuch_database_t *database,
-- 
1.8.4.rc3

[PATCH] test: update insert tests for new maildir synchronization rules

[da...@tethera.net]

From: David Bremner 

As of id:1355952747-27350-4-git-send-email-sojkam1 at fel.cvut.cz
we are more conservative about moving messages from ./new to ./cur.
This updates the insert tests to match
---

Hopefully this makes sense, and is not just papering over a real
problem that should be fixed.

 test/insert | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/insert b/test/insert
index 021edb6..1718120 100755
--- a/test/insert
+++ b/test/insert
@@ -82,7 +82,7 @@ gen_insert_msg
 notmuch insert --folder=Drafts < "$gen_msg_filename"
 output=$(notmuch search --output=files folder:Drafts)
 dirname=$(dirname "$output")
-test_expect_equal "$dirname" "$MAIL_DIR/Drafts/cur"
+test_expect_equal "$dirname" "$MAIL_DIR/Drafts/new"

 test_begin_subtest "Insert message into folder, add/remove tags"
 gen_insert_msg
@@ -99,14 +99,14 @@ gen_insert_msg
 notmuch insert --folder=F --create-folder +folder < "$gen_msg_filename"
 output=$(notmuch search --output=files folder:F tag:folder)
 basename=$(basename "$output")
-test_expect_equal_file "$gen_msg_filename" "$MAIL_DIR/F/cur/${basename}"
+test_expect_equal_file "$gen_msg_filename" "$MAIL_DIR/F/new/${basename}"

 test_begin_subtest "Insert message, create subfolder"
 gen_insert_msg
 notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
 output=$(notmuch search --output=files folder:F/G/H/I/J tag:folder)
 basename=$(basename "$output")
-test_expect_equal_file "$gen_msg_filename" 
"${MAIL_DIR}/F/G/H/I/J/cur/${basename}"
+test_expect_equal_file "$gen_msg_filename" 
"${MAIL_DIR}/F/G/H/I/J/new/${basename}"

 test_begin_subtest "Insert message, create existing subfolder"
 gen_insert_msg
-- 
1.8.3.2

[PATCH] emacs: replace setq + let with let*

[da...@tethera.net]

From: David Bremner 

I found several places where a setq is immediately followed by a let
or a let*. This seems to be the pessimal combination, with the
implicit scope of the setq combined with the extra indentation of the let.
I combined these cases into a single let* which I think is easier to read.
In two places I turned a single clause let into a let*.
---
 emacs/notmuch-hello.el | 4 ++--
 emacs/notmuch-show.el  | 4 ++--
 emacs/notmuch.el   | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index c1c6f4b..15e3614 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -260,8 +260,8 @@ afterwards.")
 (defun notmuch-hello-search ( search)
   (interactive)
   (unless (null search)
-(setq search (notmuch-hello-trim search))
-(let ((history-delete-duplicates t))
+(let* ((search (notmuch-hello-trim search))
+  (history-delete-duplicates t))
   (add-to-history 'notmuch-search-history search)))
   (notmuch-search search notmuch-search-oldest-first nil nil
  #'notmuch-hello-search-continuation))
diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 18b4671..e8c8343 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -1734,8 +1734,8 @@ TAG-CHANGES is a list of tag operations for 
`notmuch-tag'."

 See `notmuch-tag' for information on the format of TAG-CHANGES."
   (interactive)
-  (setq tag-changes (notmuch-tag (notmuch-show-get-message-id) tag-changes))
-  (let* ((current-tags (notmuch-show-get-tags))
+  (let* ((tag-changes (notmuch-tag (notmuch-show-get-message-id) tag-changes))
+(current-tags (notmuch-show-get-tags))
 (new-tags (notmuch-update-tags current-tags tag-changes)))
 (unless (equal current-tags new-tags)
   (notmuch-show-set-tags new-tags
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index af107e2..edb5a1c 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -904,9 +904,8 @@ Other optional parameters are used as follows:
   target-line: The line number to move to if the target thread does not
appear in the search results."
   (interactive)
-  (if (null query)
-  (setq query (notmuch-read-query "Notmuch search: ")))
-  (let ((buffer (get-buffer-create (notmuch-search-buffer-title query
+  (let* ((query (or query (notmuch-read-query "Notmuch search: ")))
+(buffer (get-buffer-create (notmuch-search-buffer-title query
 (switch-to-buffer buffer)
 (notmuch-search-mode)
 ;; Don't track undo information for this buffer
-- 
1.8.2.rc2

[PATCH] emacs: add `notmuch-archive-tags' cross references in docstrings

[da...@tethera.net]

From: David Bremner 

Several function docstrings refer to behaviour in docstrings that is
really controlled by notmuch-archive-tags. Add cross references, and
replace hardcoding.
---
 emacs/notmuch-show.el |  4 ++--
 emacs/notmuch.el  | 27 ---
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 600e802..c281f05 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -1529,8 +1529,8 @@ This command is intended to be one of the simplest ways to
 process a thread of email. It works exactly like
 notmuch-show-advance, in that it scrolls through messages in a
 show buffer, except that when it gets to the end of the buffer it
-archives the entire current thread, (remove the \"inbox\" tag
-from each message), kills the buffer, and displays the next
+archives the entire current thread, (apply changes in
+`notmuch-archive-tags'), kills the buffer, and displays the next
 thread from the search from which this thread was originally
 shown."
   (interactive)
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 5a8c957..e78334d 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -386,17 +386,22 @@ number of matched messages and total messages in the 
thread,
 participants in the thread, a representative subject line, and
 any tags).

-Pressing \\[notmuch-search-show-thread] on any line displays that thread. The 
'\\[notmuch-search-add-tag]' and '\\[notmuch-search-remove-tag]'
-keys can be used to add or remove tags from a thread. The 
'\\[notmuch-search-archive-thread]' key
-is a convenience for archiving a thread (removing the \"inbox\"
-tag). The '\\[notmuch-search-tag-all]' key can be used to add and/or remove 
tags from all
-messages (as opposed to threads) that match the current query.  Use with 
caution, as this
-will also tag matching messages that arrived *after* constructing the buffer.
-
-Other useful commands are '\\[notmuch-search-filter]' for filtering the 
current search
-based on an additional query string, '\\[notmuch-search-filter-by-tag]' for 
filtering to include
-only messages with a given tag, and '\\[notmuch-search]' to execute a new, 
global
-search.
+Pressing \\[notmuch-search-show-thread] on any line displays that
+thread. The '\\[notmuch-search-add-tag]' and
+'\\[notmuch-search-remove-tag]' keys can be used to add or remove
+tags from a thread. The '\\[notmuch-search-archive-thread]' key
+is a convenience for archiving a thread (applying changes in
+`notmuch-archive-tags'). The '\\[notmuch-search-tag-all]' key can
+be used to add and/or remove tags from all messages (as opposed
+to threads) that match the current query.  Use with caution, as
+this will also tag matching messages that arrived *after*
+constructing the buffer.
+
+Other useful commands are '\\[notmuch-search-filter]' for
+filtering the current search based on an additional query string,
+'\\[notmuch-search-filter-by-tag]' for filtering to include only
+messages with a given tag, and '\\[notmuch-search]' to execute a
+new, global search.

 Complete list of currently available key bindings:

-- 
1.8.2.rc2

[PATCH] emacs: remove hardcoded defaults values from docstrings

[da...@tethera.net]

From: David Bremner 

These functions refer to default values of variables, but it seems
less confusing and less likely to get out of date to just allow the
user to follow the help cross-reference links.
---
 emacs/notmuch-show.el | 14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)

another proposed change from our notmuch-pick discussions.

diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 613e666..600e802 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -1835,10 +1835,9 @@ search results instead."
   "Archive each message in thread.

 Archive each message currently shown by applying the tag changes
-in `notmuch-archive-tags' to each (remove the \"inbox\" tag by
-default). If a prefix argument is given, the messages will be
-\"unarchived\", i.e. the tag changes in `notmuch-archive-tags'
-will be reversed.
+in `notmuch-archive-tags' to each. If a prefix argument is given,
+the messages will be \"unarchived\", i.e. the tag changes in
+`notmuch-archive-tags' will be reversed.

 Note: This command is safe from any race condition of new messages
 being delivered to the same thread. It does not archive the
@@ -1865,10 +1864,9 @@ buffer."
   "Archive the current message.

 Archive the current message by applying the tag changes in
-`notmuch-archive-tags' to it (remove the \"inbox\" tag by
-default). If a prefix argument is given, the message will be
-\"unarchived\", i.e. the tag changes in `notmuch-archive-tags'
-will be reversed."
+`notmuch-archive-tags' to it. If a prefix argument is given, the
+message will be \"unarchived\", i.e. the tag changes in
+`notmuch-archive-tags' will be reversed."
   (interactive "P")
   (when notmuch-archive-tags
 (apply 'notmuch-show-tag-message
-- 
1.8.2.rc2

[PATCH 4/4] debian: update .gitignore

[da...@tethera.net]

From: David Bremner 

Several packages have been added or renamed, but their build detritus
not yet ignored.
---
 debian/.gitignore | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/debian/.gitignore b/debian/.gitignore
index 9f09f22..31aa2bb 100644
--- a/debian/.gitignore
+++ b/debian/.gitignore
@@ -1,10 +1,12 @@
 tmp/
 libnotmuch-dev/
-libnotmuch2/
+libnotmuch*/
 notmuch-emacs/
-notmuch-vim/
 notmuch/
-python-notmuch/
+notmuch-mutt/
+notmuch-ruby/
+python*-notmuch/
 *.debhelper
 *.debhelper.log
 *.substvars
+files
-- 
1.8.2.rc2

[PATCH 3/4] bindings/go: ignore downloaded source from github.com

[da...@tethera.net]

From: David Bremner 

Start a .gitignore for go bindings.
---
 bindings/go/.gitignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 bindings/go/.gitignore

diff --git a/bindings/go/.gitignore b/bindings/go/.gitignore
new file mode 100644
index 000..a2670c1
--- /dev/null
+++ b/bindings/go/.gitignore
@@ -0,0 +1 @@
+src/github.com/
-- 
1.8.2.rc2

[PATCH 2/4] emacs: update .gitignore

[da...@tethera.net]

From: David Bremner 

Start a seperate .gitignore for emacs stuff, move .elc rule there.
---
 .gitignore   | 1 -
 emacs/.gitignore | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 emacs/.gitignore

diff --git a/.gitignore b/.gitignore
index d428290..ef4f074 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,5 +12,4 @@ libnotmuch*.dylib
 *.[ao]
 *~
 .*.swp
-*.elc
 releases
diff --git a/emacs/.gitignore b/emacs/.gitignore
new file mode 100644
index 000..5421301
--- /dev/null
+++ b/emacs/.gitignore
@@ -0,0 +1,2 @@
+.eldeps*
+*.elc
-- 
1.8.2.rc2

[PATCH 1/4] bindings/python: ignore build subdirectory.

[da...@tethera.net]

From: David Bremner 

Update .gitignore to make git status less noisy
---
 bindings/python/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore
index 1fbea8a..da0732e 100644
--- a/bindings/python/.gitignore
+++ b/bindings/python/.gitignore
@@ -1,3 +1,4 @@
 *.py[co]
 /docs/build
 /docs/html
+build/
-- 
1.8.2.rc2

[PATCH] emacs: replace (funcall 'foo ...) with (foo ...)

[da...@tethera.net]

From: David Bremner 

I can't see any benefit to the funcall, and it looks like the result
of cut-and-paste from some code that actually used a variable for the
function to call.
---

Mark and I were discussing some style issues in the context of pick,
and it seems to me that some of the elisp style in the current code
could be improved. I don't claim to be an elisp style expert by any
stretch of the imagination. It would be great if some experts could
add a section to devel/STYLE about elisp.


 emacs/notmuch-message.el | 2 +-
 emacs/notmuch-show.el| 6 +++---
 emacs/notmuch.el | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/emacs/notmuch-message.el b/emacs/notmuch-message.el
index 4dc4883..914bdd1 100644
--- a/emacs/notmuch-message.el
+++ b/emacs/notmuch-message.el
@@ -40,7 +40,7 @@ the \"inbox\" and \"todo\" tags, you would set:
   ;; get the in-reply-to header and parse it for the message id.
   (let ((rep (mail-header-parse-addresses (message-field-value 
"In-Reply-To"
 (when (and notmuch-message-replied-tags rep)
-  (funcall 'notmuch-tag (notmuch-id-to-query (car (car rep)))
+  (notmuch-tag (notmuch-id-to-query (car (car rep)))
   (notmuch-tag-change-list notmuch-message-replied-tags)

 (add-hook 'message-send-hook 'notmuch-message-mark-replied)
diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 613e666..18b4671 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -1726,7 +1726,7 @@ TAG-CHANGES is a list of tag operations for 
`notmuch-tag'."
   (let* ((current-tags (notmuch-show-get-tags))
 (new-tags (notmuch-update-tags current-tags tag-changes)))
 (unless (equal current-tags new-tags)
-  (funcall 'notmuch-tag (notmuch-show-get-message-id) tag-changes)
+  (notmuch-tag (notmuch-show-get-message-id) tag-changes)
   (notmuch-show-set-tags new-tags

 (defun notmuch-show-tag ( tag-changes)
@@ -1734,7 +1734,7 @@ TAG-CHANGES is a list of tag operations for 
`notmuch-tag'."

 See `notmuch-tag' for information on the format of TAG-CHANGES."
   (interactive)
-  (setq tag-changes (funcall 'notmuch-tag (notmuch-show-get-message-id) 
tag-changes))
+  (setq tag-changes (notmuch-tag (notmuch-show-get-message-id) tag-changes))
   (let* ((current-tags (notmuch-show-get-tags))
 (new-tags (notmuch-update-tags current-tags tag-changes)))
 (unless (equal current-tags new-tags)
@@ -1745,7 +1745,7 @@ See `notmuch-tag' for information on the format of 
TAG-CHANGES."

 See `notmuch-tag' for information on the format of TAG-CHANGES."
   (interactive)
-  (setq tag-changes (funcall 'notmuch-tag 
(notmuch-show-get-messages-ids-search) tag-changes))
+  (setq tag-changes (notmuch-tag (notmuch-show-get-messages-ids-search) 
tag-changes))
   (notmuch-show-mapc
(lambda ()
  (let* ((current-tags (notmuch-show-get-tags))
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 5a8c957..af107e2 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -562,7 +562,7 @@ will be signaled."
 (defun notmuch-search-tag-region (beg end  tag-changes)
   "Change tags for threads in the given region."
   (let ((search-string (notmuch-search-find-thread-id-region-search beg end)))
-(setq tag-changes (funcall 'notmuch-tag search-string tag-changes))
+(setq tag-changes (notmuch-tag search-string tag-changes))
 (notmuch-search-foreach-result beg end
   (lambda (pos)
(notmuch-search-set-tags
@@ -576,7 +576,7 @@ See `notmuch-tag' for information on the format of 
TAG-CHANGES."
   (interactive)
   (let* ((beg (if (region-active-p) (region-beginning) (point)))
 (end (if (region-active-p) (region-end) (point
-(funcall 'notmuch-search-tag-region beg end tag-changes)))
+(notmuch-search-tag-region beg end tag-changes)))

 (defun notmuch-search-add-tag ()
   "Same as `notmuch-search-tag' but sets initial input to '+'."
-- 
1.8.2.rc2

[PATCH 3/3] build: pass CPPFLAGS to C and C++ compilers

[da...@tethera.net]

From: David Bremner 

This is used in particular by hardening flags.
---
 Makefile.local |8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile.local b/Makefile.local
index c274f07..644623f 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -39,8 +39,8 @@ GPG_FILE=$(SHA1_FILE).asc
 PV_FILE=bindings/python/notmuch/version.py

 # Smash together user's values with our extra values
-FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CFLAGS) $(WARN_CFLAGS) 
$(CONFIGURE_CFLAGS) $(extra_cflags)
-FINAL_CXXFLAGS = $(CXXFLAGS) $(WARN_CXXFLAGS) $(CONFIGURE_CXXFLAGS) 
$(extra_cflags) $(extra_cxxflags)
+FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) 
$(WARN_CFLAGS) $(CONFIGURE_CFLAGS) $(extra_cflags)
+FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) 
$(CONFIGURE_CXXFLAGS) $(extra_cflags) $(extra_cxxflags)
 FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch 
$(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)
 FINAL_NOTMUCH_LINKER = CC
 ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
@@ -237,11 +237,11 @@ quiet ?= $($(shell echo $1 | sed -e s'/ .*//'))

 %.o: %.cc $(global_deps)
@mkdir -p .deps/$(@D)
-   $(call quiet,CXX $(CXXFLAGS)) -c $(FINAL_CXXFLAGS) $< -o $@ -MD -MP -MF 
.deps/$*.d
+   $(call quiet,CXX $(CPPFLAGS) $(CXXFLAGS)) -c $(FINAL_CXXFLAGS) $< -o $@ 
-MD -MP -MF .deps/$*.d

 %.o: %.c $(global_deps)
@mkdir -p .deps/$(@D)
-   $(call quiet,CC $(CFLAGS)) -c $(FINAL_CFLAGS) $< -o $@ -MD -MP -MF 
.deps/$*.d
+   $(call quiet,CC $(CPPFLAGS) $(CFLAGS)) -c $(FINAL_CFLAGS) $< -o $@ -MD 
-MP -MF .deps/$*.d

 .PHONY : clean
 clean:
-- 
1.7.10.4

[PATCH 2/3] configure: grab CPPFLAGS from the environment.

[da...@tethera.net]

From: David Bremner 

This is needed in particular for hardening flags.
---
 configure |5 +
 1 file changed, 5 insertions(+)

diff --git a/configure b/configure
index 460fcfc..3ba1ec3 100755
--- a/configure
+++ b/configure
@@ -43,6 +43,7 @@ fi
 CC=${CC:-gcc}
 CXX=${CXX:-g++}
 CFLAGS=${CFLAGS:--O2}
+CPPFLAGS=${CPPFLAGS:-}
 CXXFLAGS=${CXXFLAGS:-\$(CFLAGS)}
 LDFLAGS=${LDFLAGS:-}
 XAPIAN_CONFIG=${XAPIAN_CONFIG:-xapian-config}
@@ -91,6 +92,7 @@ First, some common variables can specified via environment 
variables:

CC  The C compiler to use
CFLAGS  Flags to pass to the C compiler
+CPPFLAGS   Flags to pass to the C preprocessor
CXX The C++ compiler to use
CXXFLAGSFlags to pass to the C compiler
LDFLAGS Flags to pass when linking
@@ -615,6 +617,9 @@ EMACS = emacs --quick
 # Default FLAGS for C compiler (can be overridden by user such as "make 
CFLAGS=-g")
 CFLAGS = ${CFLAGS}

+# Default FLAGS for C preprocessor (can be overridden by user such as "make 
CPPFLAGS=-I/usr/local/include")
+CPPFLAGS = ${CPPFLAGS}
+
 # Default FLAGS for C++ compiler (can be overridden by user such as "make 
CXXFLAGS=-g")
 CXXFLAGS = ${CXXFLAGS}

-- 
1.7.10.4

[PATCH 1/3] debian: compile with V=1

[da...@tethera.net]

From: David Bremner 

The idea is to allow hardening verification tools (in particular blhc)
to scan the logs. Actually fixing the problem will require modifying
the notmuch configure script to propagate CPPFLAGS.
---
 debian/rules |2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/rules b/debian/rules
index 71a5602..4257c5a 100755
--- a/debian/rules
+++ b/debian/rules
@@ -9,7 +9,7 @@ override_dh_auto_configure:
dh_auto_configure -- --emacslispdir=/usr/share/emacs/site-lisp/notmuch

 override_dh_auto_build:
-   dh_auto_build
+   dh_auto_build -- V=1
dh_auto_build --sourcedirectory bindings/python
cd bindings/python && $(python3_all) setup.py build
cd bindings/ruby && ruby extconf.rb --vendor && make
-- 
1.7.10.4

handle CPPFLAGS in configure and make

[da...@tethera.net]

I wanted to enable hardening flags in the debian build (I guess other
distros will want to do the same); I realized this is made more
difficult by the fact that we don't handle CPPFLAGS in our build
system. Well, if it makes us feel any better, CMake had (has?) the
same bug.

[PATCH] devel: add dkg's printmimestructure script to notmuch devel scripts

[da...@tethera.net]

From: David Bremner 

I find this script pretty useful when figuring out who to blame for
MIME rendering problems.  It currently isn't very widely known, so it
seems worth distributing with notmuch.
---

I did a small amount whitespace cleanup compared to version in dkg's git repo.

 devel/printmimestructure | 54 
 1 file changed, 54 insertions(+)
 create mode 100755 devel/printmimestructure

diff --git a/devel/printmimestructure b/devel/printmimestructure
new file mode 100755
index 000..b8084a9
--- /dev/null
+++ b/devel/printmimestructure
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Author: Daniel Kahn Gillmor 
+# License: GPLv3+
+
+# Updates: git://lair.fifthhorseman.net/~dkg/printmimestructure
+
+# This script reads a MIME message from stdin and produces a treelike
+# representation on it stdout.
+
+# Example:
+#
+# 0 dkg at alice:~$ printmimestructure < 
'Maildir/cur/1269025522.M338697P12023.monkey,S=6459,W=6963:2,Sa'
+# ???multipart/signed 6546 bytes
+#  ???text/plain inline 895 bytes
+#  ???application/pgp-signature inline [signature.asc] 836 bytes
+# 0 dkg at alice:~$
+
+
+# If you want to number the parts, i suggest piping the output through
+# something like "cat -n"
+
+import email
+import sys
+
+def test(z, prefix=''):
+fname = '' if z.get_filename() is None else ' [' + z.get_filename() + ']'
+cset = '' if z.get_charset() is None else ' (' + z.get_charset() + ')'
+disp = z.get_params(None, header='Content-Disposition')
+if (disp is None):
+disposition = ''
+else:
+disposition = ''
+for d in disp:
+if d[0] in [ 'attachment', 'inline' ]:
+disposition = ' ' + d[0]
+if (z.is_multipart()):
+print prefix + '??' + z.get_content_type() + cset + disposition + 
fname, z.as_string().__len__().__str__() + ' bytes'
+if prefix.endswith('?'):
+prefix = prefix.rpartition('?')[0] + ' '
+if prefix.endswith('?'):
+prefix = prefix.rpartition('?')[0] + '?'
+parts = z.get_payload()
+i = 0
+while (i < parts.__len__()-1):
+test(parts[i], prefix + '?')
+i += 1
+test(parts[i], prefix + '?')
+# FIXME: show epilogue?
+else:
+print prefix + '??'+ z.get_content_type() + cset + disposition + 
fname, z.get_payload().__len__().__str__(), 'bytes'
+
+test(email.message_from_file(sys.stdin), '?')
-- 
1.8.2.rc2

[PATCH 4/4] perf-test: add notmuch-memory-test

[da...@tethera.net]

From: David Bremner 

Somehow this file was not added to the patch set which split the tests
into time and memory tests. Take advantage of the the "new" way of
running tests to avoid listing the explicitly.
---
 performance-test/notmuch-memory-test | 21 +
 1 file changed, 21 insertions(+)
 create mode 100755 performance-test/notmuch-memory-test

diff --git a/performance-test/notmuch-memory-test 
b/performance-test/notmuch-memory-test
new file mode 100755
index 000..3cf28c7
--- /dev/null
+++ b/performance-test/notmuch-memory-test
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Run tests
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+# Adapted from a Makefile to a shell script by Carl Worth (2010)
+
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+echo "Error: The notmuch test suite requires a bash version >= 4.0"
+echo "due to use of associative arrays within the test suite."
+echo "Please try again with a newer bash (or help us fix the"
+echo "test suite to be more portable). Thanks."
+exit 1
+fi
+
+cd $(dirname "$0")
+
+for test in M*.sh; do
+./"$test" "$@"
+done
-- 
1.8.2.rc2

[PATCH 3/4] perf-test: run all appropriately named time tests

[da...@tethera.net]

From: David Bremner 

This avoids hassle with manually adding every test to the master list.
---
 performance-test/notmuch-time-test | 10 ++
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/performance-test/notmuch-time-test 
b/performance-test/notmuch-time-test
index 54a208f..7113efb 100755
--- a/performance-test/notmuch-time-test
+++ b/performance-test/notmuch-time-test
@@ -16,12 +16,6 @@ fi

 cd $(dirname "$0")

-TESTS="
-  T00-new
-  T01-dump-restore
-  T02-tag
-"
-
-for test in $TESTS; do
-./$test "$@"
+for test in T*.sh; do
+./"$test" "$@"
 done
-- 
1.8.2.rc2

[PATCH 2/4] perf-test: rename time tests to have .sh suffix

[da...@tethera.net]

From: David Bremner 

This will simplify notmuch time tests
---
 performance-test/T00-new | 15 ---
 performance-test/T00-new.sh  | 15 +++
 performance-test/T01-dump-restore| 13 -
 performance-test/T01-dump-restore.sh | 13 +
 performance-test/T02-tag | 14 --
 performance-test/T02-tag.sh  | 14 ++
 6 files changed, 42 insertions(+), 42 deletions(-)
 delete mode 100755 performance-test/T00-new
 create mode 100755 performance-test/T00-new.sh
 delete mode 100755 performance-test/T01-dump-restore
 create mode 100755 performance-test/T01-dump-restore.sh
 delete mode 100755 performance-test/T02-tag
 create mode 100755 performance-test/T02-tag.sh

diff --git a/performance-test/T00-new b/performance-test/T00-new
deleted file mode 100755
index 553bb8b..000
--- a/performance-test/T00-new
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-test_description='notmuch new'
-
-. ./perf-test-lib.sh
-
-uncache_database
-
-time_start
-
-for i in $(seq 2 6); do
-time_run "notmuch new #$i" 'notmuch new'
-done
-
-time_done
diff --git a/performance-test/T00-new.sh b/performance-test/T00-new.sh
new file mode 100755
index 000..553bb8b
--- /dev/null
+++ b/performance-test/T00-new.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. ./perf-test-lib.sh
+
+uncache_database
+
+time_start
+
+for i in $(seq 2 6); do
+time_run "notmuch new #$i" 'notmuch new'
+done
+
+time_done
diff --git a/performance-test/T01-dump-restore 
b/performance-test/T01-dump-restore
deleted file mode 100755
index b2ff940..000
--- a/performance-test/T01-dump-restore
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-test_description='dump and restore'
-
-. ./perf-test-lib.sh
-
-time_start
-
-time_run 'load nmbug tags' 'notmuch restore --accumulate < 
corpus.tags/nmbug.sup-dump'
-time_run 'dump *' 'notmuch dump > tags.out'
-time_run 'restore *' 'notmuch restore < tags.out'
-
-time_done
diff --git a/performance-test/T01-dump-restore.sh 
b/performance-test/T01-dump-restore.sh
new file mode 100755
index 000..b2ff940
--- /dev/null
+++ b/performance-test/T01-dump-restore.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='dump and restore'
+
+. ./perf-test-lib.sh
+
+time_start
+
+time_run 'load nmbug tags' 'notmuch restore --accumulate < 
corpus.tags/nmbug.sup-dump'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+
+time_done
diff --git a/performance-test/T02-tag b/performance-test/T02-tag
deleted file mode 100755
index 78ceccc..000
--- a/performance-test/T02-tag
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-test_description='tagging'
-
-. ./perf-test-lib.sh
-
-time_start
-
-time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
-time_run 'tag * +existing_tag' "notmuch tag +new_tag '*'"
-time_run 'tag * -existing_tag' "notmuch tag -new_tag '*'"
-time_run 'tag * -missing_tag' "notmuch tag -new_tag '*'"
-
-time_done
diff --git a/performance-test/T02-tag.sh b/performance-test/T02-tag.sh
new file mode 100755
index 000..78ceccc
--- /dev/null
+++ b/performance-test/T02-tag.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+test_description='tagging'
+
+. ./perf-test-lib.sh
+
+time_start
+
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+time_run 'tag * +existing_tag' "notmuch tag +new_tag '*'"
+time_run 'tag * -existing_tag' "notmuch tag -new_tag '*'"
+time_run 'tag * -missing_tag' "notmuch tag -new_tag '*'"
+
+time_done
-- 
1.8.2.rc2

[PATCH 1/4] perf-test: rename memory tests

[da...@tethera.net]

From: David Bremner 

The common suffix will help running them all automatically.
---
 performance-test/M00-new | 15 ---
 performance-test/M00-new.sh  | 15 +++
 performance-test/M01-dump-restore| 15 ---
 performance-test/M01-dump-restore.sh | 15 +++
 4 files changed, 30 insertions(+), 30 deletions(-)
 delete mode 100755 performance-test/M00-new
 create mode 100755 performance-test/M00-new.sh
 delete mode 100755 performance-test/M01-dump-restore
 create mode 100755 performance-test/M01-dump-restore.sh

diff --git a/performance-test/M00-new b/performance-test/M00-new
deleted file mode 100755
index 99c3f52..000
--- a/performance-test/M00-new
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-test_description='notmuch new'
-
-. ./perf-test-lib.sh
-
-# ensure initial 'notmuch new' is run by memory_start
-uncache_database
-
-memory_start
-
-# run 'notmuch new' a second time, to test different code paths
-memory_run "notmuch new" "notmuch new"
-
-memory_done
diff --git a/performance-test/M00-new.sh b/performance-test/M00-new.sh
new file mode 100755
index 000..99c3f52
--- /dev/null
+++ b/performance-test/M00-new.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. ./perf-test-lib.sh
+
+# ensure initial 'notmuch new' is run by memory_start
+uncache_database
+
+memory_start
+
+# run 'notmuch new' a second time, to test different code paths
+memory_run "notmuch new" "notmuch new"
+
+memory_done
diff --git a/performance-test/M01-dump-restore 
b/performance-test/M01-dump-restore
deleted file mode 100755
index be5894a..000
--- a/performance-test/M01-dump-restore
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-test_description='dump and restore'
-
-. ./perf-test-lib.sh
-
-memory_start
-
-memory_run 'load nmbug tags' 'notmuch restore --accumulate 
--input=corpus.tags/nmbug.sup-dump'
-memory_run 'dump *' 'notmuch dump --output=tags.sup'
-memory_run 'restore *' 'notmuch restore --input=tags.sup'
-memory_run 'dump --format=batch-tag *' 'notmuch dump --format=batch-tag 
--output=tags.bt'
-memory_run 'restore --format=batch-tag *' 'notmuch restore --format=batch-tag 
--input=tags.bt'
-
-memory_done
diff --git a/performance-test/M01-dump-restore.sh 
b/performance-test/M01-dump-restore.sh
new file mode 100755
index 000..be5894a
--- /dev/null
+++ b/performance-test/M01-dump-restore.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='dump and restore'
+
+. ./perf-test-lib.sh
+
+memory_start
+
+memory_run 'load nmbug tags' 'notmuch restore --accumulate 
--input=corpus.tags/nmbug.sup-dump'
+memory_run 'dump *' 'notmuch dump --output=tags.sup'
+memory_run 'restore *' 'notmuch restore --input=tags.sup'
+memory_run 'dump --format=batch-tag *' 'notmuch dump --format=batch-tag 
--output=tags.bt'
+memory_run 'restore --format=batch-tag *' 'notmuch restore --format=batch-tag 
--input=tags.bt'
+
+memory_done
-- 
1.8.2.rc2

simplify and fix performance tests

[da...@tethera.net]

Jani noticed the alleged file notmuch-memory-test was missing.
Austin grumbled about explicitly adding tests to a list.
This series should give them both something else to grumble about.

[RFC Patch v2 3/3] debian: install info files

[da...@tethera.net]

From: David Bremner 

It seems dh_installinfo doesn't understand wildcards or
look in debian/tmp
---
 debian/notmuch-emacs.info | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 debian/notmuch-emacs.info

diff --git a/debian/notmuch-emacs.info b/debian/notmuch-emacs.info
new file mode 100644
index 000..2de46d3
--- /dev/null
+++ b/debian/notmuch-emacs.info
@@ -0,0 +1,3 @@
+info/notmuch.info
+info/notmuch-emacs.info
+info/notmuch-search-terms.info
-- 
1.8.2.rc2

[RFC Patch v2 2/3] man: partial conversion to pod.

[da...@tethera.net]

From: David Bremner 

This allows generation of man page and info document from the same source.
It is also a bit more friendly to edit for most people.

The conversion was done as follows:

 % groff -e -mandoc -Tascii -rHY=0 $* | rman -f POD | sed  -e '/./,/^$/!d' -e 
's/

Some small hand-editing of the .pod may be needed afterwards.
---
 INSTALL |   6 +
 configure   |  12 ++
 info/Makefile.local |  25 +++-
 man/Makefile.local  |  19 ++-
 man/man1/notmuch.1  | 184 ---
 man/man7/notmuch-search-terms.7 | 266 
 pod/notmuch-search-terms.pod| 233 +++
 pod/notmuch.pod | 149 ++
 8 files changed, 440 insertions(+), 454 deletions(-)
 delete mode 100644 man/man1/notmuch.1
 delete mode 100644 man/man7/notmuch-search-terms.7
 create mode 100644 pod/notmuch-search-terms.pod
 create mode 100644 pod/notmuch.pod

diff --git a/INSTALL b/INSTALL
index 451bf05..697b7b2 100644
--- a/INSTALL
+++ b/INSTALL
@@ -60,6 +60,12 @@ Talloc which are each described below:

Talloc is available from http://talloc.samba.org/

+   pod2man
+   ---
+
+   Some of the documentation is built with pod2man. This is part
+   of the standard Perl distribution since Perl 5.6.0
+
texinfo
---

diff --git a/configure b/configure
index 5243f6a..a0c53e2 100755
--- a/configure
+++ b/configure
@@ -371,6 +371,15 @@ else
 have_emacs=0
 fi

+printf "Checking for pod2man... "
+if pod2man --help > /dev/null 2>&1; then
+printf "Yes.\n"
+have_pod2man=1
+else
+printf "No (man page install may fail)\n"
+have_pod2man=0
+fi
+
 printf "Checking for makeinfo... "
 if makeinfo --version > /dev/null 2>&1; then
 printf "Yes.\n"
@@ -687,6 +696,9 @@ HAVE_MAKEINFO = ${have_makeinfo}
 # Whether there's an install-info binary available
 HAVE_INSTALLINFO = ${have_installinfo}

+# Is pod2man in the path?
+HAVE_POD2MAN = ${have_pod2man}
+
 # where to install info files

 INFODIR = ${INFODIR}
diff --git a/info/Makefile.local b/info/Makefile.local
index 55e9740..cca891a 100644
--- a/info/Makefile.local
+++ b/info/Makefile.local
@@ -2,10 +2,14 @@

 dir := info

+man_texi :=  $(dir)/notmuch.texi $(dir)/notmuch-search-terms.texi
+man_info := $(man_texi:.texi=.info)
+man_entry := $(man_texi:.texi=.entry)
+
 texi_sources :=  $(dir)/notmuch-emacs.texi
 emacs_info := $(texi_sources:.texi=.info)

-info := $(emacs_info)
+info := $(emacs_info) $(man_info)

 ifeq ($(HAVE_MAKEINFO),1)
 all: $(info)
@@ -15,11 +19,23 @@ ifeq ($(HAVE_INSTALLINFO),1)
 install: install-info
 endif

-%.info: %.texi
+%.entry: ../pod/%.pod
+   printf "@dircategory Notmuch\n at direntry\n" > $@
+   printf "* %s: (%s). " $(*F) $(*F) >> $@
+   podselect -section Name $< | \
+ perl -n -e  's/notmuch.* - (.*)/\u\L$$1/ && print' >> $@
+   printf "@end direntry\n" >> $@
+
+%.info: %.texi %.entry
makeinfo --no-split -o $@ $<

 $(dir)/notmuch-emacs.info: $(dir)/notmuch-emacs.texi $(dir)/version.texi

+%.texi: ../pod/%.pod
+   # a nasty hack, but the nicer ways seem to have bugs.
+   pod2texi  $< | \
+  sed 's/@node Top/@include $(*F).entry\n at node Top/' > $@
+
 .PHONY: $(dir)/version.texi
 $(dir)/version.texi: version
printf "@set VERSION ${VERSION}\n" > $@
@@ -29,5 +45,8 @@ install-info: ${info}
mkdir -p "$(DESTDIR)$(INFODIR)"
install -m0644 $(info) "$(DESTDIR)$(INFODIR)"
install-info --section=Notmuch --info-dir=${DESTDIR}${INFODIR} 
$(emacs_info)
+   for ifile in $(man_info); do \
+   install-info --info-dir=${DESTDIR}${INFODIR} $${ifile}; \
+   done

-CLEAN := $(CLEAN) $(info)
+CLEAN := $(CLEAN) $(info) $(man_entry) $(dir)/version.texi
diff --git a/man/Makefile.local b/man/Makefile.local
index 72e2a18..ceb5063 100644
--- a/man/Makefile.local
+++ b/man/Makefile.local
@@ -21,6 +21,8 @@ MAN1 := \
 MAN5 := $(dir)/man5/notmuch-hooks.5
 MAN7 := $(dir)/man7/notmuch-search-terms.7

+GENERATED_MAN := $(MAIN_PAGE) $(MAN7)
+
 MAN1_GZ := $(addsuffix .gz,$(MAN1))
 MAN5_GZ := $(addsuffix .gz,$(MAN5))
 MAN7_GZ := $(addsuffix .gz,$(MAN7))
@@ -32,6 +34,21 @@ COMPRESSED_MAN := $(MAN1_GZ) $(MAN5_GZ) $(MAN7_GZ)
 %.gz: %
gzip --stdout $^ > $@

+POD2MAN_RECIPE = mkdir -p $(@D) && \
+pod2man --section=$(subst .,,$(suffix $@)) \
+--center="Notmuch Documentation" --release=${VERSION} 
$< > $@
+
+$(dir)/man1/%.1: $(dir)/../pod/%.pod
+   $(POD2MAN_RECIPE)
+
+$(dir)/man5/%.5: $(dir)/../pod/%.pod
+   $(POD2MAN_RECIPE)
+
+$(dir)/man7/%.7: $(dir)/../pod/%.pod
+   $(POD2MAN_RECIPE)
+
+CLEAN := $(CLEAN) $(NROFF7)
+
 .PHONY: install-man update-man-versions

 install-man: $(COMPRESSED_MAN)
@@ -50,4 +67,4 @@ update-man-versions: $(MAN_SOURCE)
< $$file.bak > $$file; \
done

Second draft of info manual

[da...@tethera.net]

In order to get docs both as man pages and as info docs for reading in
emacs, I decided to try converting a couple man pages to pod.

There turned out to a painful amount of infrastructure needed to get
this working, so I didn't get much done on the notmuch-emacs docs.

I think all the infrastructure is in place at this point. I'm probably
not going to do much more work into the actual docs until I get some
feedback on the basic approach; it's clear that the autoconverted pod
files need a bit of tidying.

As far as build depends, this needs only perl for the basic docs
(pod2man is part of perl since a while). To build the info docs needs
pod2texi and makinfo, which are part of recent texinfo releases. 

[PATCH] man: document NOTMUCH_DEBUG_QUERY

[da...@tethera.net]

From: David Bremner 

This is is really functionality of the library, but CLI users might
not look at library docs even if they existed beyond notmuch.h
---
 man/man1/notmuch.1 |5 +
 1 file changed, 5 insertions(+)

diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
index 923fefe..033cc10 100644
--- a/man/man1/notmuch.1
+++ b/man/man1/notmuch.1
@@ -164,6 +164,11 @@ Location to write a talloc memory usage report. See
 in \fBtalloc\fR(3)
 for more information.

+.TP
+.B NOTMUCH_DEBUG_QUERY
+If set to a non-empty value, the notmuch library will print (to stderr) Xapian
+queries it constructs.
+
 .SH SEE ALSO

 \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[RFC patch] emacs: skeleton of texinfo manual for emacs interface.

[da...@tethera.net]

From: David Bremner 

Currently this only attempts to document the notmuch-hello interface.
---

I have thought for a long time that we should have some unified
documentation for the emacs interface. This is some sketch of a
beginning.  Building such a document turns out to be a fair amount of
work, but I guess it should be possible to have something better than
what we have now (i.e. nothing). I did try to avoid duplication with
both the docstrings and the man pages. Eventually perhaps we should
have some common format to generate the man pages and info pages from,
but I didn't want to hold up the whole effort waiting for that.

One thing that would make this effort more bearable is being able to
re-use material from the wiki. Currently there is no license
information at all on that material; I'm not sure exactly how to
proceed.

To build this, use "makeinfo notmuch.texi"
You can then (perversely) view it without emacs with "info -f notmuch.info"
or in emacs with C-u C-h i notmuch.info.


 emacs/notmuch.texi | 274 +
 emacs/version.texi |   2 +
 2 files changed, 276 insertions(+)
 create mode 100644 emacs/notmuch.texi
 create mode 100644 emacs/version.texi

diff --git a/emacs/notmuch.texi b/emacs/notmuch.texi
new file mode 100644
index 000..d4f7296
--- /dev/null
+++ b/emacs/notmuch.texi
@@ -0,0 +1,274 @@
+\input texinfo   @c -*-texinfo-*-
+ at comment $Id at w{$}
+ at comment %**start of header
+ at setfilename notmuch.info
+ at include version.texi
+ at settitle Notmuch @value{VERSION}
+ at comment %**end of header
+
+ at macro keyindex {NAME}
+ at kindex \NAME\
+ at cindex \NAME\
+ at end macro
+
+ at macro funindex {NAME}
+ at findex \NAME\
+ at cindex \NAME\
+ at end macro
+
+ at macro varindex {NAME}
+ at vindex \NAME\
+ at cindex \NAME\
+ at end macro
+
+
+ at copying
+This manual is for Notmuch (version @value{VERSION}, @value{UPDATED})
+
+Copyright @copyright{} 2013 David Bremner
+
+This manual is distributed under the same terms as notmuch, which are as 
follows.
+ at quotation
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see http://www.gnu.org/licenses/ .
+
+ at end quotation
+ at end copying
+
+ at dircategory Texinfo documentation system
+ at direntry
+* notmuch: (notmuch)Emacs interface to notmuch
+ at end direntry
+
+ at titlepage
+ at title Notmuch
+ at subtitle for version @value{VERSION}, @value{UPDATED}
+ at author David Bremner (@email{david@@tethera.net})
+ at page
+ at vskip 0pt plus 1filll
+ at insertcopying
+ at end titlepage
+
+ at contents
+
+ at ifnottex
+ at node Top
+ at top Notmuch
+
+This manual is for Notmuch (version @value{VERSION}, @value{UPDATED}).
+ at end ifnottex
+
+ at menu
+* About this Manual::
+* notmuch-hello::
+* notmuch-search::
+* Search Syntax::
+* Configuration::
+* Function Index::
+* Variable Index::
+* Index::
+ at end menu
+
+
+ at node About this Manual
+ at unnumbered About this Manual
+
+This manual covers only the emacs interface to notmuch. For
+information on the command line interface, see
+ at url{http://notmuchmail.org/manpages/notmuch-1,the notmuch man page}.
+To save
+typing, we will sometimes use @emph{notmuch} in this manual to refer
+to the Emacs interface to notmuch. If the distinction should every be
+important, we'll refer to the Emacs inteface as @emph{notmuch-emacs}.
+
+Notmuch-emacs is highly customizable via the the Emacs customization
+framework (or just by setting the appropriate variables).  We try to
+point out relevant variables in this manual, but in order to avoid
+duplication of information, but you can usually find the most detailed
+description in the varables docstring.
+
+ at node notmuch-hello
+ at chapter notmuch-hello
+
+ at funindex notmuch-hello
+ at funindex notmuch
+
+ at command{notmuch-hello} is the main entry point for notmuch. You can
+start it with @kbd{M-x notmuch} or @kbd{M-x notmuch-hello}. The
+startup screen looks something like the following. There are some
+hints at the bottom of the screen.  There are three main parts to the
+notmuch-hello screen, discussed below. The @strong{bold} text
+indicates buttons you can click with a mouse or by positioning the
+cursor and pressing @kbd{}
+
+ at example
+ at group
+
+
+   Welcome to @strong{notmuch}. You have 52 messages.
+
+Saved searches: @strong{[edit]}
+
+ 

[PATCH] emacs: introduce notmuch-command-to-string, replace use of shell-command-to-string

[da...@tethera.net]

From: David Bremner 

This has two benefits: unified error handling, and avoiding tramp's
hooking into shell-command-string.

This seems to be a fix for id:874nguxbvq.fsf at tu-dortmund.de
---

Simon: can you check if this fixes your bug?

 emacs/notmuch-lib.el | 22 --
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 270e3dc..556b1a4 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -112,13 +112,25 @@ For example, if you wanted to remove an \"inbox\" tag and 
add an
  (select-window (posn-window (event-start last-input-event)))
  (button-activate button)))

+(defun notmuch-command-to-string ( args)
+  "Synchronously invoke \"notmuch\" with the given list of arguments.
+
+If notmuch exits with a non-zero status, output from the process
+will appear in a buffer named \"*Notmuch errors*\" and an error
+will be signaled.
+
+Otherwise the output will be returned"
+  (with-temp-buffer
+(let* ((status (apply #'call-process notmuch-command nil t nil args))
+  (output (buffer-string)))
+  (notmuch-check-exit-status status (cons notmuch-command args) output)
+  output)))
+
 (defun notmuch-version ()
   "Return a string with the notmuch version number."
   (let ((long-string
 ;; Trim off the trailing newline.
-(substring (shell-command-to-string
-(concat notmuch-command " --version"))
-   0 -1)))
+(substring (notmuch-command-to-string "--version") 0 -1))
 (if (string-match "^notmuch\\( version\\)? \\(.*\\)$"
  long-string)
(match-string 2 long-string)
@@ -127,9 +139,7 @@ For example, if you wanted to remove an \"inbox\" tag and 
add an
 (defun notmuch-config-get (item)
   "Return a value from the notmuch configuration."
   ;; Trim off the trailing newline
-  (substring (shell-command-to-string
- (concat notmuch-command " config get " item))
- 0 -1))
+  (substring (notmuch-command-to-string "config" "get" item) 0 -1))

 (defun notmuch-database-path ()
   "Return the database.path value from the notmuch configuration."
-- 
1.8.2.rc1

[Patch v2 4/4] nmbug: allow empty prefix

[da...@tethera.net]

From: David Bremner 

Current code does not distinguish between an empty string in the
NMBPREFIX environment variable and the variable being undefined. This
makes it impossible to define an empty prefix, if, e.g. somebody wants
to dump all of their tags with nmbug.
---
 devel/nmbug/nmbug |2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devel/nmbug/nmbug b/devel/nmbug/nmbug
index b9c70e4..90d98b6 100755
--- a/devel/nmbug/nmbug
+++ b/devel/nmbug/nmbug
@@ -13,7 +13,7 @@ my $NMBGIT = $ENV{NMBGIT} || $ENV{HOME}.'/.nmbug';

 $NMBGIT .= '/.git' if (-d $NMBGIT.'/.git');

-my $TAGPREFIX = $ENV{NMBPREFIX} || 'notmuch::';
+my $TAGPREFIX = defined($ENV{NMBPREFIX}) ? $ENV{NMBPREFIX} : 'notmuch::';

 # for encoding

-- 
1.7.10.4

[Patch v2 3/4] nmbug: replace hard-coded magic hash with git-hash-object

[da...@tethera.net]

From: David Bremner 

This is at least easier to understand than the magic hash. It may also
be a bit more robust, although it is hard to imagine these numbers
changing without many other changes in git.
---
 devel/nmbug/nmbug |6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devel/nmbug/nmbug b/devel/nmbug/nmbug
index 73d64fe..b9c70e4 100755
--- a/devel/nmbug/nmbug
+++ b/devel/nmbug/nmbug
@@ -15,9 +15,6 @@ $NMBGIT .= '/.git' if (-d $NMBGIT.'/.git');

 my $TAGPREFIX = $ENV{NMBPREFIX} || 'notmuch::';

-# magic hash for git
-my $EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391';
-
 # for encoding

 my $ESCAPE_CHAR =  '%';
@@ -50,6 +47,9 @@ if (!exists $command{$subcommand}) {
   usage ();
 }

+# magic hash for git
+my $EMPTYBLOB = git (qw{hash-object -t blob /dev/null});
+
 &{$command{$subcommand}}(@ARGV);

 sub git_pipe {
-- 
1.7.10.4

[Patch v2 2/4] nmbug: use 'notmuch tag --batch'

[da...@tethera.net]

From: David Bremner 

This should be more robust with respect to tags with whitespace and
and other special characters. It also (hopefully) fixes a remaining
bug handling message-ids with whitespace.  It should also be
noticeably faster for large sets of changes since it does one exec per
change set as opposed to one exec per tag changed.
---
 devel/nmbug/nmbug |   27 ++-
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/devel/nmbug/nmbug b/devel/nmbug/nmbug
index befc3d9..73d64fe 100755
--- a/devel/nmbug/nmbug
+++ b/devel/nmbug/nmbug
@@ -267,6 +267,20 @@ sub do_checkout {
   do_sync (action => 'checkout');
 }

+sub quote_for_xapian {
+  my $str = shift;
+  $str =~ s/"/""/g;
+  return '"' . $str . '"';
+}
+
+sub pair_to_batch_line {
+  my ($action, $pair) = @_;
+
+  # the tag should already be suitably encoded
+
+  return $action . $ENCPREFIX . $pair->{tag} .
+' -- id:' . quote_for_xapian ($pair->{id})."\n";
+}

 sub do_sync {

@@ -283,17 +297,20 @@ sub do_sync {
 $D_action = '-';
   }

-  foreach my $pair (@{$status->{added}}) {
+  my $notmuch = spawn ({}, '|-', qw/notmuch tag --batch/)
+or die 'notmuch tag --batch';

-notmuch ('tag', $A_action.$TAGPREFIX.$pair->{tag},
-'id:'.$pair->{id});
+  foreach my $pair (@{$status->{added}}) {
+print $notmuch pair_to_batch_line ($A_action, $pair);
   }

   foreach my $pair (@{$status->{deleted}}) {
-notmuch ('tag', $D_action.$TAGPREFIX.$pair->{tag},
-'id:'.$pair->{id});
+print $notmuch pair_to_batch_line ($D_action, $pair);
   }

+  unless (close $notmuch) {
+die "'notmuch tag --batch' exited with nonzero value\n";
+  }
 }


-- 
1.7.10.4

[Patch v2 1/4] nmbug: use dump --format=batch-tag

[da...@tethera.net]

From: David Bremner 

This should make nmbug tolerate tags with whitespace and other special
characters it.  At the moment this relies on _not_ passing calls to
notmuch tag through the shell, which is a documented feature of perl's
system function.
---
 devel/nmbug/nmbug |   27 ---
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/devel/nmbug/nmbug b/devel/nmbug/nmbug
index fe103b3..befc3d9 100755
--- a/devel/nmbug/nmbug
+++ b/devel/nmbug/nmbug
@@ -39,6 +39,11 @@ my %command = (
 status => \_status,
 );

+# Convert prefix into form suitable for literal matching against
+# notmuch dump --format=batch-tag output.
+my $ENCPREFIX = encode_for_fs ($TAGPREFIX);
+$ENCPREFIX =~ s/:/%3a/g;
+
 my $subcommand = shift || usage ();

 if (!exists $command{$subcommand}) {
@@ -203,9 +208,9 @@ sub index_tags {

   my $index = $NMBGIT.'/nmbug.index';

-  my $query = join ' ', map ("tag:$_", get_tags ($TAGPREFIX));
+  my $query = join ' ', map ("tag:\"$_\"", get_tags ($TAGPREFIX));

-  my $fh = spawn ('-|', qw/notmuch dump --/, $query)
+  my $fh = spawn ('-|', qw/notmuch dump --format=batch-tag --/, $query)
 or die "notmuch dump: $!";

   git ('read-tree', '--empty');
@@ -214,22 +219,30 @@ sub index_tags {
 or die 'git update-index';

   while (<$fh>) {
-m/ ( [^ ]* ) \s+ \( ([^\)]* ) \) /x || die 'syntax error in dump';
-my ($id,$rest) = ($1,$2);

-#strip prefixes before writing
-my @tags = grep { s/^$TAGPREFIX//; } split (' ', $rest);
+chomp();
+my ($rest,$id) = split(/ -- id:/);
+
+if ($id =~ s/^"(.*)"\s*$/$1/) {
+  # xapian quoted string, dequote.
+  $id =~ s/""/"/g;
+}
+
+#strip prefixes from tags before writing
+my @tags = grep { s/^[+]$ENCPREFIX//; } split (' ', $rest);
 index_tags_for_msg ($git,$id, 'A', @tags);
   }
   unless (close $git) {
 die "'git update-index --index-info' exited with nonzero value\n";
   }
   unless (close $fh) {
-die "'notmuch dump -- $query' exited with nonzero value\n";
+die "'notmuch dump --format=batch-tag -- $query' exited with nonzero 
value\n";
   }
   return $index;
 }

+# update the git index to either create or delete an empty file.
+# Neither argument should be encoded/escaped.
 sub index_tags_for_msg {
   my $fh = shift;
   my $msgid = shift;
-- 
1.7.10.4

Update for nmbug, round 2

[da...@tethera.net]

This obsoletes 

 id:1360374019-20988-1-git-send-email-david at tethera.net

This less broken than the last version ;). I've used these patches for
a few days without ill effects. The first two patches use
batch-tagging, which should have some speedup. The includes fixes for
the issue about quoting that Tomi raised.

The second two patches are a style improvement and a bug fix for a bug
that probably not many people hit.

[PATCH] contrib/nmbug: convert to use batch tagging.

[da...@tethera.net]

From: David Bremner 

In the case of large changes to the database from git, one of main
current bottlenecks is the large number of execs of notmuch tag. This
avoids that by using use the batch tagging facilities as of notmuch
0.15.

We use "spawn" directly rather than inventing a "notmuch_pipe", since
it seems the only place we need to pipe to notmuch so far.
---

This is only lightly tested; please make sure you have backups of your
database.

I'm not 100% sure about the indentation. No doubt Tomi will let me
know if it can be improved.

 contrib/nmbug/nmbug |   15 ++-
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/contrib/nmbug/nmbug b/contrib/nmbug/nmbug
index f003ef9..c66c526 100755
--- a/contrib/nmbug/nmbug
+++ b/contrib/nmbug/nmbug
@@ -270,17 +270,22 @@ sub do_sync {
 $D_action = '-';
   }

-  foreach my $pair (@{$status->{added}}) {
+  my $notmuch = spawn ({}, '|-', qw/notmuch tag --batch/)
+  or die 'notmuch tag --batch';

-notmuch ('tag', $A_action.$TAGPREFIX.$pair->{tag},
-'id:'.$pair->{id});
+  foreach my $pair (@{$status->{added}}) {
+print $notmuch $A_action.$TAGPREFIX.$pair->{tag}, " -- ",
+  'id:'.$pair->{id};
   }

   foreach my $pair (@{$status->{deleted}}) {
-notmuch ('tag', $D_action.$TAGPREFIX.$pair->{tag},
-'id:'.$pair->{id});
+print $notmuch $D_action.$TAGPREFIX.$pair->{tag},
+  'id:'.$pair->{id};
   }

+  unless (close $notmuch) {
+die "'notmuch tag --batch' exited with nonzero value\n";
+  }
 }


-- 
1.7.10.4

[PATCH 2/2] NEWS: News for 0.15.2

[da...@tethera.net]

From: David Bremner 

Another couple of build fixes.
---
 NEWS |   14 ++
 1 file changed, 14 insertions(+)

diff --git a/NEWS b/NEWS
index 97f2305..3ba5a50 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,17 @@
+Notmuch 0.15.2 (2013-01-29)
+===
+
+Build fixes
+---
+
+Update dependencies to avoid problems when building in parallel.
+
+Internal test framework changes
+---
+
+Adjust Emacs test watchdog mechanism to cope with `process-attributes`
+being unimplimented.
+
 Notmuch 0.15.1 (2013-01-24)
 =

-- 
1.7.10.4

[PATCH 1/2] test: delay watchdog checks in emacs.

[da...@tethera.net]

From: David Bremner 

Instead of checking immediately for the watched process, delay a
minute, or in the case that process-attributes returns nil, for two
minutes.  This is intended to cope with the case that
process-attributes is unimplimented, and returns always returns nil.
In this case, the watchdog check is the same as the two minute limit
imposed by timeout.
---
 test/test-lib.el |   13 ++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/test-lib.el b/test/test-lib.el
index dece811..d26b49f 100644
--- a/test/test-lib.el
+++ b/test/test-lib.el
@@ -77,12 +77,19 @@ invisible text."
(setq start next-pos)))
 str))

-(defun orphan-watchdog (pid)
+(defun orphan-watchdog-check (pid)
   "Periodically check that the process with id PID is still
 running, quit if it terminated."
   (if (not (process-attributes pid))
-  (kill-emacs)
-(run-at-time "1 min" nil 'orphan-watchdog pid)))
+  (kill-emacs)))
+
+(defun orphan-watchdog (pid)
+  "Initiate orphan watchdog check."
+  ; If process-attributes returns nil right away, that probably means
+  ; it is unimplimented. So we delay two minutes before killing emacs.
+  (if (process-attributes pid)
+  (run-at-time 60 60 'orphan-watchdog-check pid)
+(run-at-time 120 60 'orphan-watchdog-check pid)))

 (defun hook-counter (hook)
   "Count how many times a hook is called.  Increments
-- 
1.7.10.4

another bug fix release: 0.15.2, in progress.

[da...@tethera.net]

I plan to do another quick bug fix release, with aidecoe's parallel
build fix and a patch originally from Tomi to make the Emacs tests not
commit suicide right away on certain OSes.

I didn't include the parallel building patch here since it's already
in master.

[PATCH 2/2] CLI: add simple error handling for talloc logging

[da...@tethera.net]

From: David Bremner 

This really should have been there before. I think it's better to do
the actual operation and then possibly fail writing the memory log,
but it would not be too hard to change it to abort earlier.
---
 notmuch.c |   12 +++-
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/notmuch.c b/notmuch.c
index a674481..cfb009b 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -294,10 +294,6 @@ main (int argc, char *argv[])

ret = (command->function)(local, argc - opt_index, argv + 
opt_index);

-   /* in the future support for this environment variable may
-* be supplemented or replaced by command line arguments
-* --leak-report and/or --leak-report-full */
-
talloc_report = getenv ("NOTMUCH_TALLOC_REPORT");

/* this relies on the previous call to
@@ -305,7 +301,13 @@ main (int argc, char *argv[])

if (talloc_report && strcmp (talloc_report, "") != 0) {
FILE *report = fopen (talloc_report, "w");
-   talloc_report_full (NULL, report);
+   if (report) {
+   talloc_report_full (NULL, report);
+   } else {
+   ret = 1;
+   fprintf (stderr, "ERROR: unable to write talloc log. ");
+   perror (talloc_report);
+   }
}

return ret;
-- 
1.7.10.4

[PATCH 1/2] man: document NOTMUCH_TALLOC_REPORT environment variable

[da...@tethera.net]

From: David Bremner 

---
 man/man1/notmuch.1 |8 
 1 file changed, 8 insertions(+)

diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
index 0805be8..55e2d16 100644
--- a/man/man1/notmuch.1
+++ b/man/man1/notmuch.1
@@ -147,6 +147,14 @@ behavior of notmuch.
 .B NOTMUCH_CONFIG
 Specifies the location of the notmuch configuration file. Notmuch will
 use ${HOME}/.notmuch\-config if this variable is not set.
+
+.TP
+.B NOTMUCH_TALLOC_REPORT
+Location to write a talloc memory usage report. See
+.B talloc_enable_leak_report_full
+in \fBtalloc\fR(3)
+for more information.
+
 .SH SEE ALSO

 \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

minor fixes for talloc leak logging

[da...@tethera.net]

These are the remaining unapplied patches from 

  id:1358619958-21209-1-git-send-email-david at tethera.net

They are rebased to omit the actual command line argument, as suggested by 

 id:87d2wzwmd9.fsf at servo.finestructure.net

[PATCH 7/7] CLI: add simple error handling for talloc logging

[da...@tethera.net]

From: David Bremner 

This really should have been there before. I think it's better to do
the actual operation and then possibly fail writing the memory log,
but it would not be too hard to change it to abort earlier.
---
 notmuch.c |7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/notmuch.c b/notmuch.c
index f8d4b35..c3336e8 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -301,7 +301,12 @@ main (int argc, char *argv[])

if (leak_report && (strcmp (leak_report, "") != 0)) {
FILE *report = fopen (leak_report, "w");
-   talloc_report_full (NULL, report);
+   if (report) {
+   talloc_report_full (NULL, report);
+   } else {
+   ret = 1;
+   perror (leak_report);
+   }
}

return ret;
-- 
1.7.10.4

[PATCH 6/7] man: document NOTMUCH_TALLOC_REPORT environment variable

[da...@tethera.net]

From: David Bremner 

---
 man/man1/notmuch.1 |7 +++
 1 file changed, 7 insertions(+)

diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
index 5c58c41..391eb88 100644
--- a/man/man1/notmuch.1
+++ b/man/man1/notmuch.1
@@ -155,6 +155,13 @@ behavior of notmuch.
 .B NOTMUCH_CONFIG
 Specifies the location of the notmuch configuration file. Notmuch will
 use ${HOME}/.notmuch\-config if this variable is not set.
+
+.TP
+.B NOTMUCH_TALLOC_REPORT
+Location to write a talloc memory usage report. Overridden by the
+.B --leak-report
+option.
+
 .SH SEE ALSO

 \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[PATCH 5/7] CLI: add --leak-report top level option

[da...@tethera.net]

From: David Bremner 

This roughly mimics the samba4 argument. The presence of the command
line argument overrides any value of NOTMUCH_TALLOC_REPORT in the
environment.
---
 man/man1/notmuch.1 |8 
 notmuch.c  |   18 +++---
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
index 6bf9b2e..5c58c41 100644
--- a/man/man1/notmuch.1
+++ b/man/man1/notmuch.1
@@ -70,6 +70,14 @@ Print a synopsis of available commands and exit.
 Print the installed version of notmuch, and exit.
 .RE

+.RS 4
+.TP 4
+.BI \-\-leak-report= path
+
+Write a detailed report of all memory allocated via talloc to
+.I path
+.RE
+
 .SH COMMANDS


diff --git a/notmuch.c b/notmuch.c
index a674481..f8d4b35 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -250,11 +250,13 @@ main (int argc, char *argv[])
 command_t *command;
 unsigned int i;
 notmuch_bool_t print_help=FALSE, print_version=FALSE;
+const char* leak_report=NULL;
 int opt_index;

 notmuch_opt_desc_t options[] = {
{ NOTMUCH_OPT_BOOLEAN, _help, "help", 'h', 0 },
{ NOTMUCH_OPT_BOOLEAN, _version, "version", 'v', 0 },
+   { NOTMUCH_OPT_STRING, _report, "leak-report", 'l', 0 },
{ 0, 0, 0, 0, 0 }
 };

@@ -290,21 +292,15 @@ main (int argc, char *argv[])

if (strcmp (argv[opt_index], command->name) == 0) {
int ret;
-   char *talloc_report;

ret = (command->function)(local, argc - opt_index, argv + 
opt_index);

-   /* in the future support for this environment variable may
-* be supplemented or replaced by command line arguments
-* --leak-report and/or --leak-report-full */
-
-   talloc_report = getenv ("NOTMUCH_TALLOC_REPORT");
-
-   /* this relies on the previous call to
-* talloc_enable_null_tracking */
+   if (leak_report == NULL) {
+   leak_report = getenv ("NOTMUCH_TALLOC_REPORT");
+   }

-   if (talloc_report && strcmp (talloc_report, "") != 0) {
-   FILE *report = fopen (talloc_report, "w");
+   if (leak_report && (strcmp (leak_report, "") != 0)) {
+   FILE *report = fopen (leak_report, "w");
talloc_report_full (NULL, report);
}

-- 
1.7.10.4

[PATCH 4/7] man: document existing top level options

[da...@tethera.net]

From: David Bremner 

The options --help and --version were not documented before.  One
could quibble about how useful that documentation is, but we will soon
add more options.
---
 man/man1/notmuch.1 |   22 +-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
index 69805cb..6bf9b2e 100644
--- a/man/man1/notmuch.1
+++ b/man/man1/notmuch.1
@@ -21,7 +21,7 @@
 notmuch \- thread-based email index, search, and tagging
 .SH SYNOPSIS
 .B notmuch
-.IR command " [" args " ...]"
+.RI "[" option " ...] " command  " [" arg " ...]"
 .SH DESCRIPTION
 Notmuch is a command-line based program for indexing, searching,
 reading, and tagging large collections of email messages.
@@ -50,6 +50,26 @@ interfaces to notmuch. The emacs-based interface to notmuch 
(available under
 in the Notmuch source distribution) is probably the most widely used at
 this time.

+.SH OPTIONS
+
+Supported global options for
+.B notmuch
+include
+
+.RS 4
+.TP 4
+.B \-\-help
+
+Print a synopsis of available commands and exit.
+.RE
+
+.RS 4
+.TP 4
+.B \-\-version
+
+Print the installed version of notmuch, and exit.
+.RE
+
 .SH COMMANDS


-- 
1.7.10.4

[PATCH 3/7] CLI: convert top level argument parsing to use command-line-arguments

[da...@tethera.net]

From: David Bremner 

This isn't really a win for conciseness yet, but will make it easier
to add options.
---
 notmuch.c |   22 ++
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/notmuch.c b/notmuch.c
index f13fd27..a674481 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -249,6 +249,14 @@ main (int argc, char *argv[])
 void *local;
 command_t *command;
 unsigned int i;
+notmuch_bool_t print_help=FALSE, print_version=FALSE;
+int opt_index;
+
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_BOOLEAN, _help, "help", 'h', 0 },
+   { NOTMUCH_OPT_BOOLEAN, _version, "version", 'v', 0 },
+   { 0, 0, 0, 0, 0 }
+};

 talloc_enable_null_tracking ();

@@ -263,10 +271,16 @@ main (int argc, char *argv[])
 if (argc == 1)
return notmuch (local);

-if (strcmp (argv[1], "--help") == 0)
+opt_index = parse_arguments (argc, argv, options, 1);
+if (opt_index < 0) {
+   /* diagnostics already printed */
+   return 1;
+}
+
+if (print_help)
return notmuch_help_command (NULL, argc - 1, [1]);

-if (strcmp (argv[1], "--version") == 0) {
+if (print_version) {
printf ("notmuch " STRINGIFY(NOTMUCH_VERSION) "\n");
return 0;
 }
@@ -274,11 +288,11 @@ main (int argc, char *argv[])
 for (i = 0; i < ARRAY_SIZE (commands); i++) {
command = [i];

-   if (strcmp (argv[1], command->name) == 0) {
+   if (strcmp (argv[opt_index], command->name) == 0) {
int ret;
char *talloc_report;

-   ret = (command->function)(local, argc - 1, [1]);
+   ret = (command->function)(local, argc - opt_index, argv + 
opt_index);

/* in the future support for this environment variable may
 * be supplemented or replaced by command line arguments
-- 
1.7.10.4

[PATCH 2/7] CLI: remove alias machinery, and "part", "search-tags" commands

[da...@tethera.net]

From: David Bremner 

The commands are long deprecated, so removal is probably overdue. The
real motivation is to simplify argument handling for notmuch so that
we can migrate to the common argument parsing framework.
---
 NEWS   |8 
 devel/TODO |7 ---
 notmuch.c  |   50 +-
 3 files changed, 9 insertions(+), 56 deletions(-)

diff --git a/NEWS b/NEWS
index 1cb52dd..2ed472e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,11 @@
+Notmuch 0.16 (2013-MM-DD)
+=
+
+Command-Line Interface
+--
+
+Deprecated commands "part" and "search-tags" are removed.
+
 Notmuch 0.15 (2013-01-18)
 =

diff --git a/devel/TODO b/devel/TODO
index eb757af..3741f0e 100644
--- a/devel/TODO
+++ b/devel/TODO
@@ -118,13 +118,6 @@ file.
 Allow configuration for filename patterns that should be ignored when
 indexing.

-Replace the "notmuch part --part=id" command with "notmuch show
---part=id", (David Edmondson wants to rewrite some of "notmuch show" to
-provide more MIME-structure information in its output first).
-
-Replace the "notmuch search-tags" command with "notmuch search
---output=tags".
-
 Fix to avoid this ugly message:

(process:17197): gmime-CRITICAL **: g_mime_message_get_mime_part: 
assertion `GMIME_IS_MESSAGE (message)' failed
diff --git a/notmuch.c b/notmuch.c
index 4fc0973..f13fd27 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -31,18 +31,6 @@ typedef struct command {
 const char *summary;
 } command_t;

-#define MAX_ALIAS_SUBSTITUTIONS 3
-
-typedef struct alias {
-const char *name;
-const char *substitutions[MAX_ALIAS_SUBSTITUTIONS];
-} alias_t;
-
-alias_t aliases[] = {
-{ "part", { "show", "--format=raw"}},
-{ "search-tags", {"search", "--output=tags", "*"}}
-};
-
 static int
 notmuch_help_command (void *ctx, int argc, char *argv[]);

@@ -260,9 +248,7 @@ main (int argc, char *argv[])
 {
 void *local;
 command_t *command;
-alias_t *alias;
-unsigned int i, j;
-const char **argv_local;
+unsigned int i;

 talloc_enable_null_tracking ();

@@ -285,40 +271,6 @@ main (int argc, char *argv[])
return 0;
 }

-for (i = 0; i < ARRAY_SIZE (aliases); i++) {
-   alias = [i];
-
-   if (strcmp (argv[1], alias->name) == 0)
-   {
-   int substitutions;
-
-   argv_local = talloc_size (local, sizeof (char *) *
- (argc + MAX_ALIAS_SUBSTITUTIONS - 1));
-   if (argv_local == NULL) {
-   fprintf (stderr, "Out of memory.\n");
-   return 1;
-   }
-
-   /* Copy all substution arguments from the alias. */
-   argv_local[0] = argv[0];
-   for (j = 0; j < MAX_ALIAS_SUBSTITUTIONS; j++) {
-   if (alias->substitutions[j] == NULL)
-   break;
-   argv_local[j+1] = alias->substitutions[j];
-   }
-   substitutions = j;
-
-   /* And copy all original arguments (skipping the argument
-* that matched the alias of course. */
-   for (j = 2; j < (unsigned) argc; j++) {
-   argv_local[substitutions+j-1] = argv[j];
-   }
-
-   argc += substitutions - 1;
-   argv = (char **) argv_local;
-   }
-}
-
 for (i = 0; i < ARRAY_SIZE (commands); i++) {
command = [i];

-- 
1.7.10.4

[PATCH 1/7] emacs: don't use deprecated "notmuch search-tags" command

[da...@tethera.net]

From: David Bremner 

A followup patch will finally remove this command, so we need to stop
using it.
---
 emacs/notmuch-hello.el |2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index 6db62a0..00b78e1 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -504,7 +504,7 @@ Complete list of currently available key bindings:
  (notmuch-remove-if-not
   (lambda (tag)
 (not (member tag hide-tags)))
-  (process-lines notmuch-command "search-tags"
+  (process-lines notmuch-command "search" "--output=tags" "*"

 (defun notmuch-hello-insert-header ()
   "Insert the default notmuch-hello header."
-- 
1.7.10.4

update top level argument handling

[da...@tethera.net]

The only new feature here is an option --leak-report
to notmuch new, as requested in id:m2hangivfu.fsf at guru.guru-group.fi

There is also a bunch of cleanup of the argument handling. One
casualty of this is that the use of aliases (in particular "notmuch
part" and "notmuch search-tags" is no longer supported).

[PATCH] notmuch-tag: initialize with enum instead of 0.

[da...@tethera.net]

From: David Bremner 

This is just a cosmetic fix to make the "type" of ret more clear.
---
 notmuch-tag.c |2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index b54c55d..d9daf8f 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -97,7 +97,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const char 
*query_string,
 notmuch_query_t *query;
 notmuch_messages_t *messages;
 notmuch_message_t *message;
-int ret = 0;
+int ret = NOTMUCH_STATUS_SUCCESS;

 /* Optimize the query so it excludes messages that already have
  * the specified set of tags. */
-- 
1.7.10.4

[PATCH] NEWS: describe 'batch-tag' dump/restore/tag

[da...@tethera.net]

From: David Bremner 

Hopefully it is clear that more details are available in the man
pages.
---
 NEWS |   10 ++
 1 file changed, 10 insertions(+)

diff --git a/NEWS b/NEWS
index 1d29770..42bf276 100644
--- a/NEWS
+++ b/NEWS
@@ -41,6 +41,16 @@ Fixed `notmuch new` to skip ignored broken symlinks
   ignored files list.  Previously, it would abort when encountering
   broken symlink, even if it was ignored.

+New dump/restore format and tagging interface.
+
+  There is a new 'batch-tag' format for dump and restore that is more
+  robust, particularly with respect to tags and message-ids containing
+  whitespace.
+
+  `notmuch tag` now supports the ability to read tag operations and
+  queries from an input stream, in a format compatible with the new
+  dump/restore format.
+
 Bcc and Reply-To headers are now available in notmuch show json output

   The `notmuch show --format=json` now includes "Bcc" and "Reply-To" headers.
-- 
1.7.10.4

[PATCH 5/5] debian: note that ical bug is fixed

[da...@tethera.net]

From: David Bremner 

This was fixed a while ago in git, but not released yet.
---
 debian/changelog |6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 40b7695..6c6915f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,10 +4,10 @@ notmuch (0.15~rc1-1) experimental; urgency=low
   * Change priority to optional (Closes: #687217).
   * Remove Dm-Upload-Allowed field, as this is no longer used by
 Debian.
-  * Add python3 bindings, thanks to Jakub Wilk (Closes:
-#683515).
+  * Add python3 bindings, thanks to Jakub Wilk (Closes: #683515).
+  * Bug fix: ".ical attachment problem", (Closes: #688747).

- -- David Bremner   Wed, 16 Jan 2013 08:25:02 -0400
+ -- David Bremner   Wed, 16 Jan 2013 08:28:27 -0400

 notmuch (0.14-1) experimental; urgency=low

-- 
1.7.10.4

[PATCH 4/5] debian: add python 3 bindings

[da...@tethera.net]

From: David Bremner 

This patch is due to Jakub Wilk .

It does add a build depend on python3 for people using the "make
debian-snapshot" target.
---
 debian/changelog  |4 +++-
 debian/control|   14 ++
 debian/python-notmuch.install |2 +-
 debian/rules  |5 +
 4 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index f4925a1..40b7695 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,8 +4,10 @@ notmuch (0.15~rc1-1) experimental; urgency=low
   * Change priority to optional (Closes: #687217).
   * Remove Dm-Upload-Allowed field, as this is no longer used by
 Debian.
+  * Add python3 bindings, thanks to Jakub Wilk (Closes:
+#683515).

- -- David Bremner   Mon, 07 Jan 2013 21:40:52 -0400
+ -- David Bremner   Wed, 16 Jan 2013 08:25:02 -0400

 notmuch (0.14-1) experimental; urgency=low

diff --git a/debian/control b/debian/control
index 75b55a5..5bb0d05 100644
--- a/debian/control
+++ b/debian/control
@@ -14,6 +14,7 @@ Build-Depends:
  libtalloc-dev,
  libz-dev,
  python-all (>= 2.6.6-3~),
+ python3-all (>= 3.1.2-7~),
  emacs23-nox | emacs23 (>=23~) | emacs23-lucid (>=23~) |
  emacs24-nox | emacs24 (>=24~) | emacs24-lucid (>=24~),
  gdb,
@@ -75,6 +76,19 @@ Description: python interface to the notmuch mail search and 
index library
  This package provides a Python interface to the notmuch
  functionality, directly interfacing with a shared notmuch library.

+Package: python3-notmuch
+Architecture: all
+Section: python
+Depends: ${misc:Depends}, ${python3:Depends}, libnotmuch3
+Description: Python 3 interface to the notmuch mail search and index library
+ Notmuch is a system for indexing, searching, reading, and tagging
+ large collections of email messages in maildir or mh format. It uses
+ the Xapian library to provide fast, full-text search with a very
+ convenient search syntax.
+ .
+ This package provides a Python 3 interface to the notmuch
+ functionality, directly interfacing with a shared notmuch library.
+
 Package: notmuch-emacs
 Architecture: all
 Section: mail
diff --git a/debian/python-notmuch.install b/debian/python-notmuch.install
index 607c065..b2cc136 100644
--- a/debian/python-notmuch.install
+++ b/debian/python-notmuch.install
@@ -1 +1 @@
-usr/lib/python*
+usr/lib/python2*
diff --git a/debian/rules b/debian/rules
index 8e30353..45e970a 100755
--- a/debian/rules
+++ b/debian/rules
@@ -1,5 +1,7 @@
 #!/usr/bin/make -f

+python3_all = py3versions -s | xargs -n1 | xargs -t -I {} env {}
+
 %:
dh $@ --with python2

@@ -9,13 +11,16 @@ override_dh_auto_configure:
 override_dh_auto_build:
dh_auto_build
dh_auto_build --sourcedirectory bindings/python
+   cd bindings/python && $(python3_all) setup.py build
$(MAKE) -C contrib/notmuch-mutt

 override_dh_auto_clean:
dh_auto_clean
dh_auto_clean --sourcedirectory bindings/python
+   cd bindings/python && $(python3_all) setup.py clean -a
$(MAKE) -C contrib/notmuch-mutt clean

 override_dh_auto_install:
dh_auto_install
dh_auto_install --sourcedirectory bindings/python
+   cd bindings/python && $(python3_all) setup.py install 
--install-layout=deb --root=$(CURDIR)/debian/tmp
-- 
1.7.10.4

[PATCH 3/5] debian/compat: upgrade to compat level 9

[da...@tethera.net]

From: David Bremner 

- enable hardening

- fix dh syntax. Now that we have compat level 9, the old, wrong
  syntax is no longer accepted.

- update debian/libnotmuch{3,-dev}.install for multiarch.

- update versioned dependency on debhelper.
---
 debian/compat |2 +-
 debian/control|3 ++-
 debian/libnotmuch-dev.install |2 +-
 debian/libnotmuch3.install|2 +-
 debian/rules  |2 +-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/debian/compat b/debian/compat
index 7f8f011..ec63514 100644
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-7
+9
diff --git a/debian/control b/debian/control
index d434c53..75b55a5 100644
--- a/debian/control
+++ b/debian/control
@@ -7,7 +7,7 @@ Uploaders:
  martin f. krafft ,
  David Bremner 
 Build-Depends:
- debhelper (>= 7.0.50~),
+ debhelper (>= 9),
  pkg-config,
  libxapian-dev,
  libgmime-2.6-dev (>= 2.6.7~) | libgmime-2.4-dev,
@@ -39,6 +39,7 @@ Package: libnotmuch3
 Section: libs
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}
+Pre-Depends: ${misc:Pre-Depends}
 Description: thread-based email index, search and tagging (runtime)
  Notmuch is a system for indexing, searching, reading, and tagging
  large collections of email messages in maildir or mh format. It uses
diff --git a/debian/libnotmuch-dev.install b/debian/libnotmuch-dev.install
index 185dba4..96bbd63 100644
--- a/debian/libnotmuch-dev.install
+++ b/debian/libnotmuch-dev.install
@@ -1,2 +1,2 @@
 usr/include
-usr/lib/libnotmuch.so
+usr/lib/*/libnotmuch.so
diff --git a/debian/libnotmuch3.install b/debian/libnotmuch3.install
index da4fc25..a513b47 100644
--- a/debian/libnotmuch3.install
+++ b/debian/libnotmuch3.install
@@ -1 +1 @@
-usr/lib/libnotmuch.so.*
+usr/lib/*/libnotmuch.so.*
diff --git a/debian/rules b/debian/rules
index 603b3ab..8e30353 100755
--- a/debian/rules
+++ b/debian/rules
@@ -1,7 +1,7 @@
 #!/usr/bin/make -f

 %:
-   dh --with python2 $@
+   dh $@ --with python2

 override_dh_auto_configure:
dh_auto_configure -- --emacslispdir=/usr/share/emacs/site-lisp/notmuch
-- 
1.7.10.4

[PATCH 1/5] debian: change priority to optional.

[da...@tethera.net]

From: David Bremner 

The distinction between extra and optional is generally not very
important, except that being extra forces anything that depends on you
to be extra.
---
 debian/changelog |1 +
 debian/control   |2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index dd07aa1..2fa2cf6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 notmuch (0.15~rc1-1) experimental; urgency=low

   * New upstream release candidate.
+  * Change priority to optional (Closes: #687217).

  -- David Bremner   Mon, 07 Jan 2013 21:40:52 -0400

diff --git a/debian/control b/debian/control
index f725276..ed026b0 100644
--- a/debian/control
+++ b/debian/control
@@ -1,6 +1,6 @@
 Source: notmuch
 Section: mail
-Priority: extra
+Priority: optional
 Maintainer: Carl Worth 
 Uploaders:
  Jameson Graef Rollins ,
-- 
1.7.10.4

No subject

[da...@tethera.net]

Hi Gang;

Here are some proposed changes to the debian packaging for 0.15.

Most will probably be boring to people not familiar with debian
packaging, with the excepotion of 4/5, which has a shell pipeline with
two xargs in it, and almost can certainly be improved by several
readers of this list.

[PATCH 1/5] debian: change priority to optional.
[PATCH 2/5] debian: remove Dm-Upload-Allowed field.
[PATCH 3/5] debian/compat: upgrade to compat level 9
[PATCH 4/5] debian: add python 3 bindings
[PATCH 5/5] debian: note that ical bug is fixed

[Patch v2 9/9] test/tagging: add test for naked punctuation in tags; compare with quoting spaces.

[da...@tethera.net]

From: David Bremner 

This test also serves as documentation of the quoting
requirements. The comment lines are so that it exactly matches the man
page. Nothing more embarrassing than having an example in the man page
fail.
---
 test/tagging |   25 +
 1 file changed, 25 insertions(+)

diff --git a/test/tagging b/test/tagging
index 1717e72..1f5632c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -198,6 +198,31 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest "--batch: only space and % needs to be encoded."
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%23possible%5c +%26are +%28tags%29 +crazy%7b +inbox +match%2acrazy 
+space%20in%20tags +tag4 +tag5 +unread +winner -- id:msg-002 at 
notmuch-test-suite
++foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001 at 
notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: unicode message-ids'

 ${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
-- 
1.7.10.4

[Patch v2 8/9] man: document notmuch tag --batch, --input options

[da...@tethera.net]

From: Jani Nikula 

---
 man/man1/notmuch-tag.1 |   92 
 1 file changed, 92 insertions(+)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 9444aa4..3aa2fa5 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -6,6 +6,11 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms
 .B notmuch tag
 .RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"

+.B notmuch tag
+.RI "--batch"
+.RI "[ --input=<" filename "> ]"
+
+
 .SH DESCRIPTION

 Add/remove tags for all messages matching the search terms.
@@ -30,6 +35,93 @@ updates the maildir flags according to tag changes if the
 configuration option is enabled. See \fBnotmuch-config\fR(1) for
 details.

+Supported options for
+.B tag
+include
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read batch tagging operations from a file (stdin by default). This is more
+efficient than repeated
+.B notmuch tag
+invocations. See
+.B TAG FILE FORMAT
+below for the input format. This option is not compatible with
+specifying tagging on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR "\-\-input=" 
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+
+.SH TAG FILE FORMAT
+
+The input must consist of lines of the format:
+
+.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" query ">"
+
+Each line is interpreted similarly to
+.B notmuch tag
+command line arguments. The delimiter is one or more spaces ' '. Any
+characters in
+.RI < tag >
+.B may
+be hex-encoded with %NN where NN is the hexadecimal value of the
+character. To hex-encode a character with a multi-byte UTF-8 encoding,
+hex-encode each byte.
+Any spaces in 
+.B must
+be hex-encoded as %20. Any characters that are not
+part of
+.RI  < tag >
+.B must not
+be hex-encoded.
+
+In the future tag:"tag with spaces" style quoting may be supported for
+.RI < tag >
+as well;
+for this reason all double quote characters in
+.RI < tag >
+.B should
+be hex-encoded.
+
+The
+.RI < query >
+should be quoted using Xapian boolean term quoting rules: if a term
+contains whitespace or a close paren or starts with a double quote, it
+must be enclosed in double quotes (not including any prefix) and
+double quotes inside the term must be doubled (see below for
+examples).
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
+.SS EXAMPLE
+
+The following shows a valid input to batch tagging. Note that only the
+isolated '*' acts as a wildcard. Also note the two different quotings
+of the tag
+.B space in tags
+.
+.RS
+.nf
++winner *
++foo::bar%25 -- (One and Two) or (One and tag:winner)
++found::it -- tag:foo::bar%
+# ignore this line and the next
+
++space%20in%20tags -- Two
+# add tag '(tags)', among other stunts.
++crazy{ +(tags) + +#possible\ -- tag:"space in tags"
++match*crazy -- tag:crazy{
++some_tag -- id:"this is ""nauty)"""
+.fi
+.RE
+
 .SH SEE ALSO

 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[Patch v2 7/9] test/tagging: add test for exotic message-ids and batch tagging

[da...@tethera.net]

From: David Bremner 

The (now fixed) bug that this test revealed is that unquoted
message-ids with whitespace or other control characters in them are
split into several tokens by the Xapian query parser.
---
 test/tagging |   18 ++
 1 file changed, 18 insertions(+)

diff --git a/test/tagging b/test/tagging
index 417112b..1717e72 100755
--- a/test/tagging
+++ b/test/tagging
@@ -198,6 +198,24 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest '--batch: unicode message-ids'
+
+${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
+ --num-messages=100
+
+notmuch dump --format=batch-tag | sed 's/^.* -- /+common_tag -- /' | \
+sort > EXPECTED
+
+notmuch dump --format=batch-tag | sed 's/^.* -- /  -- /' | \
+notmuch restore --format=batch-tag
+
+notmuch tag --batch < EXPECTED
+
+notmuch dump --format=batch-tag| \
+sort > OUTPUT
+
+test_expect_equal_file EXPECTED OUTPUT
+
 test_expect_code 1 "Empty tag names" 'notmuch tag + One'

 test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
-- 
1.7.10.4

[Patch v2 6/9] test/tagging: add tests for exotic tags

[da...@tethera.net]

From: David Bremner 

We test quotes seperately because they matter to the query escaper.
---
 test/tagging |   66 ++
 1 file changed, 66 insertions(+)

diff --git a/test/tagging b/test/tagging
index 405ad7c..417112b 100755
--- a/test/tagging
+++ b/test/tagging
@@ -132,6 +132,72 @@ EOF
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest '--batch: tags with quotes'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%22%27%22%22%22%27 +inbox +tag5 +unread -- id:msg-001 at notmuch-test-suite
++%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002 at 
notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest '--batch: tags with punctuation and space'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e 
+inbox +tag4 +tag5 +unread -- id:msg-002 at notmuch-test-suite
++%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e 
+inbox +tag5 +unread -- id:msg-001 at notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest '--batch: unicode tags'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 
+=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d
 
+A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27
 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 
+P%c4%98%2f +R +inbox +tag4 +tag5 +unread 
+%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d 
+%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b
 +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 
+%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-002 at notmuch-test-suite
++%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 
+=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d
 
+A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27
 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 
+P%c4%98%2f +R +inbox +tag5 +unread 
+%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d 
+%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b
 +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 
+%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001 at notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_expect_code 1 "Empty tag names" 'notmuch tag + One'

 test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
-- 
1.7.10.4

[Patch v2 5/9] test/tagging: add basic tests for batch tagging functionality

[da...@tethera.net]

From: David Bremner 

This tests argument parsing, blank lines and comments, and basic hex
decoding functionality.
---
 test/tagging |   51 +++
 1 file changed, 51 insertions(+)

diff --git a/test/tagging b/test/tagging
index cd16585..405ad7c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,57 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\"  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)"

+test_begin_subtest "--batch"
+notmuch tag --batch < batch.in  < batch.expected < backup.tags
+notmuch tag --input=batch.in
+notmuch search \* | notmuch_search_sanitize > OUTPUT
+notmuch restore --format=batch-tag < backup.tags
+test_expect_equal_file batch.expected OUTPUT
+
+test_begin_subtest "--batch --input"
+notmuch dump --format=batch-tag > backup.tags
+notmuch tag --batch --input=batch.in
+notmuch search \* | notmuch_search_sanitize > OUTPUT
+notmuch restore --format=batch-tag < backup.tags
+test_expect_equal_file batch.expected OUTPUT
+
+test_begin_subtest "--batch, blank lines and comments"
+notmuch dump | sort > EXPECTED
+notmuch tag --batch < OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: checking error messages'
 notmuch dump --format=batch-tag > BACKUP
 notmuch tag --batch 

[Patch v2 4/9] test/tagging: add test for error messages of tag --batch

[da...@tethera.net]

From: David Bremner 

This is based on the similar test for notmuch restore, but the parser
in batch tagging mode is less tolerant of a few cases, in particular
those tested by illegal_tag.
---
 test/tagging |   35 +++
 1 file changed, 35 insertions(+)

diff --git a/test/tagging b/test/tagging
index 980ff92..cd16585 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,41 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\"  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)"

+test_begin_subtest '--batch: checking error messages'
+notmuch dump --format=batch-tag > BACKUP
+notmuch tag --batch 

[Patch v2 3/9] cli: add support for batch tagging operations to "notmuch tag"

[da...@tethera.net]

From: Jani Nikula 

Add support for batch tagging operations through stdin to "notmuch
tag". This can be enabled with the new --batch command line option to
"notmuch tag". The input must consist of lines of the format:

+|- [...] [--]  [...]

Each line is interpreted similarly to "notmuch tag" command line
arguments. The delimiter is one or more spaces ' '. Any characters in
 MAY be hex encoded with %NN where NN is the hexadecimal value of
the character. Any ' ' and '%' characters in  and MUST be hex
encoded (using %20 and %25, respectively). For future-proofing, any
'"' characters in  SHOULD be hex-encoded.

Any characters that are not part of  or
MUST NOT be hex encoded.

 is passed verbatim to Xapian

Leading and trailing space ' ' is ignored. Empty lines and lines
beginning with '#' are ignored.

Signed-off-by: Jani Nikula 

Hacked-like-crazy-by: David Bremner 
---
 notmuch-tag.c |   94 -
 1 file changed, 86 insertions(+), 8 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 8129912..7fc614d 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -128,6 +128,46 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 return interrupted;
 }

+static int
+tag_file (void *ctx, notmuch_database_t *notmuch, tag_op_flag_t flags,
+ FILE *input)
+{
+char *line = NULL;
+char *query_string = NULL;
+size_t line_size = 0;
+ssize_t line_len;
+int ret = 0;
+tag_op_list_t *tag_ops;
+
+tag_ops = tag_op_list_create (ctx);
+if (tag_ops == NULL) {
+   fprintf (stderr, "Out of memory.\n");
+   return 1;
+}
+
+while ((line_len = getline (, _size, input)) != -1 &&
+  ! interrupted) {
+
+   ret = parse_tag_line (ctx, line, TAG_FLAG_NONE,
+ _string, tag_ops);
+
+   if (ret > 0)
+   continue;
+
+   if (ret < 0)
+   break;
+
+   ret = tag_query (ctx, notmuch, query_string, tag_ops, flags);
+   if (ret)
+   break;
+}
+
+if (line)
+   free (line);
+
+return ret;
+}
+
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
@@ -137,6 +177,10 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 notmuch_database_t *notmuch;
 struct sigaction action;
 tag_op_flag_t tag_flags = TAG_FLAG_NONE;
+notmuch_bool_t batch = FALSE;
+FILE *input = stdin;
+char *input_file_name = NULL;
+int opt_index;
 int ret = 0;

 /* Setup our handler for SIGINT */
@@ -146,15 +190,43 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 action.sa_flags = SA_RESTART;
 sigaction (SIGINT, , NULL);

-tag_ops = tag_op_list_create (ctx);
-if (tag_ops == NULL) {
-   fprintf (stderr, "Out of memory.\n");
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_BOOLEAN, , "batch", 0, 0 },
+   { NOTMUCH_OPT_STRING, _file_name, "input", 'i', 0 },
+   { 0, 0, 0, 0, 0 }
+};
+
+opt_index = parse_arguments (argc, argv, options, 1);
+if (opt_index < 0)
return 1;
+
+if (input_file_name) {
+   batch = TRUE;
+   input = fopen (input_file_name, "r");
+   if (input == NULL) {
+   fprintf (stderr, "Error opening %s for reading: %s\n",
+input_file_name, strerror (errno));
+   return 1;
+   }
 }

-if (parse_tag_command_line (ctx, argc - 1, argv + 1,
-   _string, tag_ops))
-   return 1;
+if (batch) {
+   if (opt_index != argc) {
+   fprintf (stderr, "Can't specify both cmdline and stdin!\n");
+   return 1;
+   }
+} else {
+
+   tag_ops = tag_op_list_create (ctx);
+   if (tag_ops == NULL) {
+   fprintf (stderr, "Out of memory.\n");
+   return 1;
+   }
+
+   if (parse_tag_command_line (ctx, argc - opt_index, argv + opt_index,
+   _string, tag_ops))
+   return 1;
+}

 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
@@ -167,9 +239,15 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 if (notmuch_config_get_maildir_synchronize_flags (config))
tag_flags |= TAG_FLAG_MAILDIR_SYNC;

-ret = tag_query (ctx, notmuch, query_string, tag_ops, tag_flags);
+if (batch)
+   ret = tag_file (ctx, notmuch, tag_flags, input);
+else
+   ret = tag_query (ctx, notmuch, query_string, tag_ops, tag_flags);

 notmuch_database_destroy (notmuch);

-return ret;
+if (input != stdin)
+   fclose (input);
+
+return ret || interrupted;
 }
-- 
1.7.10.4

[Patch v2 2/9] notmuch-tag.c: convert to use tag-utils

[da...@tethera.net]

From: David Bremner 

Command line parsing is factored out into a function
parse_tag_command_line in tag-utils.c.

There is some duplicated code eliminated in tag_query, and a bunch of
translation from using the bare tag_op structs to using that tag-utils
API.
---
 notmuch-tag.c |   99 -
 tag-util.c|   51 +++--
 tag-util.h|   15 +
 3 files changed, 84 insertions(+), 81 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index fc9d43a..8129912 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -19,6 +19,7 @@
  */

 #include "notmuch-client.h"
+#include "tag-util.h"
 #include "string-util.h"

 static volatile sig_atomic_t interrupted;
@@ -36,14 +37,10 @@ handle_sigint (unused (int sig))
 interrupted = 1;
 }

-typedef struct {
-const char *tag;
-notmuch_bool_t remove;
-} tag_operation_t;

 static char *
 _optimize_tag_query (void *ctx, const char *orig_query_string,
-const tag_operation_t *tag_ops)
+const tag_op_list_t *list)
 {
 /* This is subtler than it looks.  Xapian ignores the '-' operator
  * at the beginning both queries and parenthesized groups and,
@@ -60,7 +57,7 @@ _optimize_tag_query (void *ctx, const char *orig_query_string,
 size_t i;

 /* Don't optimize if there are no tag changes. */
-if (tag_ops[0].tag == NULL)
+if (tag_op_list_size (list) == 0)
return talloc_strdup (ctx, orig_query_string);

 /* Build the new query string */
@@ -69,17 +66,17 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 else
query_string = talloc_asprintf (ctx, "( %s ) and (", orig_query_string);

-for (i = 0; tag_ops[i].tag && query_string; i++) {
+for (i = 0; i < tag_op_list_size (list) && query_string; i++) {
/* XXX in case of OOM, query_string will be deallocated when
 * ctx is, which might be at shutdown */
if (make_boolean_term (ctx,
-  "tag", tag_ops[i].tag,
+  "tag", tag_op_list_tag (list, i),
   , _len))
return NULL;

query_string = talloc_asprintf_append_buffer (
query_string, "%s%s%s", join,
-   tag_ops[i].remove ? "" : "not ",
+   tag_op_list_isremove (list, i) ? "" : "not ",
escaped);
join = " or ";
 }
@@ -91,17 +88,15 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 return query_string;
 }

-/* Tag messages matching 'query_string' according to 'tag_ops', which
- * must be an array of tagging operations terminated with an empty
- * element. */
+/* Tag messages matching 'query_string' according to 'tag_ops'
+ */
 static int
 tag_query (void *ctx, notmuch_database_t *notmuch, const char *query_string,
-  tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
+  tag_op_list_t *tag_ops, tag_op_flag_t flags)
 {
 notmuch_query_t *query;
 notmuch_messages_t *messages;
 notmuch_message_t *message;
-int i;

 /* Optimize the query so it excludes messages that already have
  * the specified set of tags. */
@@ -124,21 +119,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages) && ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-
-   notmuch_message_freeze (message);
-
-   for (i = 0; tag_ops[i].tag; i++) {
-   if (tag_ops[i].remove)
-   notmuch_message_remove_tag (message, tag_ops[i].tag);
-   else
-   notmuch_message_add_tag (message, tag_ops[i].tag);
-   }
-
-   notmuch_message_thaw (message);
-
-   if (synchronize_flags)
-   notmuch_message_tags_to_maildir_flags (message);
-
+   tag_op_list_apply (message, tag_ops, flags | TAG_FLAG_PRE_OPTIMIZED);
notmuch_message_destroy (message);
 }

@@ -150,15 +131,13 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
-tag_operation_t *tag_ops;
-int tag_ops_count = 0;
-char *query_string;
+tag_op_list_t *tag_ops = NULL;
+char *query_string = NULL;
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
 struct sigaction action;
-notmuch_bool_t synchronize_flags;
-int i;
-int ret;
+tag_op_flag_t tag_flags = TAG_FLAG_NONE;
+int ret = 0;

 /* Setup our handler for SIGINT */
 memset (, 0, sizeof (struct sigaction));
@@ -167,54 +146,15 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 action.sa_flags = SA_RESTART;
 sigaction (SIGINT, , NULL);

-argc--; argv++; /* skip subcommand argument */
-
-/* Array of tagging operations (add or remove), terminated with an
- * empty element. */
-tag_ops = 

[Patch v2 1/9] tag-util: factor out rules for illegal tags, use in parse_tag_line

[da...@tethera.net]

From: David Bremner 

This will allow us to be consistent between batch tagging and command
line tagging as far as what is an illegal tag.
---
 tag-util.c |   36 +++-
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/tag-util.c b/tag-util.c
index ca12b3b..0a4fe78 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -31,6 +31,30 @@ line_error (tag_parse_status_t status,
 return status;
 }

+/*
+ * Test tags for some forbidden cases.
+ *
+ * return: NULL if OK,
+ *explanatory message otherwise.
+ */
+
+static const char *
+illegal_tag (const char *tag, notmuch_bool_t remove)
+{
+
+if (*tag == '\0' && ! remove)
+   return "empty tag forbidden";
+
+/* This disallows adding the non-removable tag "-" and
+ * enables notmuch tag to take long options more easily.
+ */
+
+if (*tag == '-' && ! remove)
+   return "tag starting with '-' forbidden";
+
+return NULL;
+}
+
 tag_parse_status_t
 parse_tag_line (void *ctx, char *line,
tag_op_flag_t flags,
@@ -95,11 +119,13 @@ parse_tag_line (void *ctx, char *line,
remove = (*tok == '-');
tag = tok + 1;

-   /* Maybe refuse empty tags. */
-   if (! (flags & TAG_FLAG_BE_GENEROUS) && *tag == '\0') {
-   ret = line_error (TAG_PARSE_INVALID, line_for_error,
- "empty tag");
-   goto DONE;
+   /* Maybe refuse illegal tags. */
+   if (! (flags & TAG_FLAG_BE_GENEROUS)) {
+   const char *msg = illegal_tag (tag, remove);
+   if (msg) {
+   ret = line_error (TAG_PARSE_INVALID, line_for_error, msg);
+   goto DONE;
+   }
}

/* Decode tag. */
-- 
1.7.10.4

Xapian-quoting based batch tagging.

[da...@tethera.net]

This is essentially just a rebase of 

 id:1356464567-21779-1-git-send-email-david at tethera.net

with some commit-message fixups for 

[Patch v2 3/9] cli: add support for batch tagging operations to

as suggested by Jani

[Patch v2] notmuch-restore: handle empty input file, leading blank lines and comments.

[da...@tethera.net]

From: David Bremner 

This patch corrects several undesirable behaviours:

1) Empty files were not detected, leading to buffer read overrun.

2) An initial blank line cause restore to silently abort

3) Initial comment line caused format detection to fail
---
 notmuch-restore.c |   18 +-
 test/dump-restore |3 ---
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/notmuch-restore.c b/notmuch-restore.c
index d43546d..f436989 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -181,11 +181,6 @@ notmuch_restore_command (unused (void *ctx), int argc, 
char *argv[])
 argv[opt_index]);
return 1;
 }
-char *p;
-
-line_len = getline (, _size, input);
-if (line_len == 0)
-   return 0;

 tag_ops = tag_op_list_create (ctx);
 if (tag_ops == NULL) {
@@ -193,6 +188,19 @@ notmuch_restore_command (unused (void *ctx), int argc, 
char *argv[])
return 1;
 }

+do {
+   line_len = getline (, _size, input);
+
+   /* empty input file not considered an error */
+   if (line_len < 0)
+   return 0;
+
+} while ((line_len == 0) ||
+(line[0] == '#') ||
+/* the cast is safe because we checked about for line_len < 0 */
+(strspn (line, " \t\n") == (unsigned)line_len));
+
+char *p;
 for (p = line; (input_format == DUMP_FORMAT_AUTO) && *p; p++) {
if (*p == '(')
input_format = DUMP_FORMAT_SUP;
diff --git a/test/dump-restore b/test/dump-restore
index c2ddb92..ae30cd1 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -146,13 +146,11 @@ cat < comments-and-blanks
 EOF

 test_begin_subtest 'restoring empty file is not an error'
-test_subtest_known_broken
 notmuch restore < /dev/null 2>OUTPUT.$test_count
 cp /dev/null EXPECTED
 test_expect_equal_file EXPECTED OUTPUT.$test_count

 test_begin_subtest 'file of comments and blank lines is not an error'
-test_subtest_known_broken
 notmuch restore --input=comments-and-blanks
 ret_val=$?
 test_expect_equal "$ret_val" "0"
@@ -172,7 +170,6 @@ echo "yun1vjwegii.fsf at aiko.keithp.com (another_tag)" \
 >> leading-comments-blanks-sup

 test_begin_subtest 'detect format=sup with leading comments and blanks'
-test_subtest_known_broken
 notmuch restore --input=leading-comments-blanks-sup
 notmuch search --output=tags id:yun1vjwegii.fsf at aiko.keithp.com > 
OUTPUT.$test_count
 echo "another_tag" > EXPECTED
-- 
1.7.10.4

[Patch v2 4/4] perf-test: initial support for talloc leak report in memory tests

[da...@tethera.net]

From: David Bremner 

As with the valgrind logs, we print a (very) brief summary and leave
the log for inspection.
---
 performance-test/perf-test-lib.sh |5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/performance-test/perf-test-lib.sh 
b/performance-test/perf-test-lib.sh
index 10d05e0..9ee7661 100644
--- a/performance-test/perf-test-lib.sh
+++ b/performance-test/perf-test-lib.sh
@@ -126,13 +126,16 @@ memory_run ()
 test_count=$(($test_count+1))

 log_file=$log_dir/$test_count.log
+talloc_log=$log_dir/$test_count.talloc

 printf "[ %d ]\t%s\n" $test_count "$1"

-valgrind --leak-check=full --log-file="$log_file" $2
+NOTMUCH_TALLOC_REPORT="$talloc_log" valgrind --leak-check=full 
--log-file="$log_file" $2

 awk '/LEAK SUMMARY/,/suppressed/ { sub(/^==[0-9]*==/," "); print }' 
"$log_file"
 echo
+sed -n -e 's/.*[(]total *\([^)]*\)[)]/talloced at exit: \1/p' $talloc_log
+echo
 }

 memory_done ()
-- 
1.7.10.4

[Patch v2 3/4] notmuch-restore: use debug version of talloc_strndup

[da...@tethera.net]

From: David Bremner 

This gives line numbers for better debugging.
---
 notmuch-restore.c |9 +
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/notmuch-restore.c b/notmuch-restore.c
index c93f1ac..6111977 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -88,10 +88,11 @@ parse_sup_line (void *ctx, char *line,
return 1;
 }

-*query_str = talloc_strndup (ctx, line + match[1].rm_so,
-match[1].rm_eo - match[1].rm_so);
-file_tags = talloc_strndup (ctx, line + match[2].rm_so,
-   match[2].rm_eo - match[2].rm_so);
+*query_str = talloc_strndup_debug (ctx, line + match[1].rm_so,
+  match[1].rm_eo - match[1].rm_so);
+
+file_tags = talloc_strndup_debug (ctx, line + match[2].rm_so,
+ match[2].rm_eo - match[2].rm_so);

 char *tok = file_tags;
 size_t tok_len = 0;
-- 
1.7.10.4

[Patch v2 2/4] util: add talloc-extra.[ch]

[da...@tethera.net]

From: David Bremner 

These are intended to be simple wrappers to provide slightly better
debugging information than what talloc currently provides natively.
---
 notmuch-client.h|2 +-
 util/Makefile.local |2 +-
 util/talloc-extra.c |   14 ++
 util/talloc-extra.h |   18 ++
 4 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 util/talloc-extra.c
 create mode 100644 util/talloc-extra.h

diff --git a/notmuch-client.h b/notmuch-client.h
index d7b352e..5f28836 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -58,7 +58,7 @@ typedef GMimeCipherContext notmuch_crypto_context_t;
 #include 
 #include 

-#include 
+#include "talloc-extra.h"

 #define unused(x) x __attribute__ ((unused))

diff --git a/util/Makefile.local b/util/Makefile.local
index a11e35b..29c0ce6 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -4,7 +4,7 @@ dir := util
 extra_cflags += -I$(srcdir)/$(dir)

 libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
- $(dir)/string-util.c
+ $(dir)/string-util.c $(dir)/talloc-extra.c

 libutil_modules := $(libutil_c_srcs:.c=.o)

diff --git a/util/talloc-extra.c b/util/talloc-extra.c
new file mode 100644
index 000..4a5f9c0
--- /dev/null
+++ b/util/talloc-extra.c
@@ -0,0 +1,14 @@
+#include 
+#include "talloc-extra.h"
+
+char *
+talloc_strndup_named_const (void *ctx, const char *str,
+size_t len, const char *name)
+{
+char *ptr = talloc_strndup (ctx, str, len);
+
+if (ptr)
+   talloc_set_name_const(ptr, name);
+
+return ptr;
+}
diff --git a/util/talloc-extra.h b/util/talloc-extra.h
new file mode 100644
index 000..5b8ca28
--- /dev/null
+++ b/util/talloc-extra.h
@@ -0,0 +1,18 @@
+#ifndef _XTALLOC_H
+#define _XTALLOC_H
+
+#include 
+
+/* Like talloc_strndup, but take an extra parameter for the internal talloc
+ * name (for debugging) */
+
+char *
+talloc_strndup_named_const (void *ctx, const char *str,
+   size_t len, const char *name);
+
+/* use the __location__ macro from talloc.h to name a string according to its
+ * source location */
+
+#define talloc_strndup_debug(ctx, str, len) talloc_strndup_named_const (ctx, 
str, len, __location__)
+
+#endif
-- 
1.7.10.4

[Patch v2 1/4] CLI: add talloc leak report, controlled by an environment variable.

[da...@tethera.net]

From: David Bremner 

The argument handling in notmuch.c seems due for an overhaul, but
until then use an environment variable to specify a location to write
the talloc leak report to.  This is only enabled for the (interesting)
case where some notmuch subcommand is invoked.
---
 notmuch.c |   24 ++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/notmuch.c b/notmuch.c
index ee2892e..6b7aae8 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -321,8 +321,28 @@ main (int argc, char *argv[])
 for (i = 0; i < ARRAY_SIZE (commands); i++) {
command = [i];

-   if (strcmp (argv[1], command->name) == 0)
-   return (command->function)(local, argc - 1, [1]);
+   if (strcmp (argv[1], command->name) == 0) {
+   int ret;
+   char *talloc_report;
+
+   ret = (command->function)(local, argc - 1, [1]);
+
+   /* in the future support for this environment variable may
+* be supplemented or replaced by command line arguments
+* --leak-report and/or --leak-report-full */
+
+   talloc_report = getenv ("NOTMUCH_TALLOC_REPORT");
+
+   /* this relies on the previous call to
+* talloc_enable_null_tracking */
+
+   if (talloc_report && strcmp (talloc_report, "") != 0) {
+   FILE *report = fopen (talloc_report, "w");
+   talloc_report_full (NULL, report);
+   }
+
+   return ret;
+   }
 }

 fprintf (stderr, "Error: Unknown command '%s' (see \"notmuch help\")\n",
-- 
1.7.10.4

v2 talloc leak report

[da...@tethera.net]

Obsoletes 

  id:1355714648-23144-1-git-send-email-david at tethera.netZ

[Patch v2 1/4] CLI: add talloc leak report, controlled by an
- comments and formatting

[Patch v2 2/4] util: add talloc-extra.[ch]
- rename, fix dangerous memcpy

[Patch v2 3/4] notmuch-restore: use debug version of talloc_strndup
no change, except new names

[Patch v2 4/4] perf-test: initial support for talloc leak report in
New patch, use talloc reports in memory tests.

[PATCH 2/2] notmuch-restore: handle empty input file, leading blank lines and comments.

[da...@tethera.net]

From: David Bremner 

This patch corrects several undesirable behaviours:

1) Empty files were not detected, leading to buffer read overrun.

2) An initial blank line cause restore to silently abort

3) Initial comment line caused format detection to fail
---
 notmuch-restore.c |   17 -
 test/dump-restore |3 ---
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/notmuch-restore.c b/notmuch-restore.c
index 9ed9b51..c93f1ac 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -180,11 +180,6 @@ notmuch_restore_command (unused (void *ctx), int argc, 
char *argv[])
 argv[opt_index]);
return 1;
 }
-char *p;
-
-line_len = getline (, _size, input);
-if (line_len == 0)
-   return 0;

 tag_ops = tag_op_list_create (ctx);
 if (tag_ops == NULL) {
@@ -192,6 +187,18 @@ notmuch_restore_command (unused (void *ctx), int argc, 
char *argv[])
return 1;
 }

+do {
+   line_len = getline (, _size, input);
+
+   /* empty input file not considered an error */
+   if (line_len < 0)
+   return 0;
+
+} while ((line_len == 0) ||
+(line[0] == '#') ||
+(strspn (line, " \t\n") == strlen (line)));
+
+char *p;
 for (p = line; (input_format == DUMP_FORMAT_AUTO) && *p; p++) {
if (*p == '(')
input_format = DUMP_FORMAT_SUP;
diff --git a/test/dump-restore b/test/dump-restore
index c2ddb92..ae30cd1 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -146,13 +146,11 @@ cat < comments-and-blanks
 EOF

 test_begin_subtest 'restoring empty file is not an error'
-test_subtest_known_broken
 notmuch restore < /dev/null 2>OUTPUT.$test_count
 cp /dev/null EXPECTED
 test_expect_equal_file EXPECTED OUTPUT.$test_count

 test_begin_subtest 'file of comments and blank lines is not an error'
-test_subtest_known_broken
 notmuch restore --input=comments-and-blanks
 ret_val=$?
 test_expect_equal "$ret_val" "0"
@@ -172,7 +170,6 @@ echo "yun1vjwegii.fsf at aiko.keithp.com (another_tag)" \
 >> leading-comments-blanks-sup

 test_begin_subtest 'detect format=sup with leading comments and blanks'
-test_subtest_known_broken
 notmuch restore --input=leading-comments-blanks-sup
 notmuch search --output=tags id:yun1vjwegii.fsf at aiko.keithp.com > 
OUTPUT.$test_count
 echo "another_tag" > EXPECTED
-- 
1.7.10.4

[PATCH 1/2] test/dump-restore: new tests for empty files and leading comments/whitespace.

[da...@tethera.net]

From: David Bremner 

Three of these are marked broken; the third is a regression test,
since it passes by virtue of batch-tag being the default input format.
---
 test/dump-restore |   42 ++
 1 file changed, 42 insertions(+)

diff --git a/test/dump-restore b/test/dump-restore
index 6a989b6..c2ddb92 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -136,6 +136,48 @@ notmuch dump --format=batch-tag > BACKUP

 notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"

+# initial segment of file used for several tests below.
+cat < comments-and-blanks
+# this is a comment
+
+# next line has leading whitespace
+   
+
+EOF
+
+test_begin_subtest 'restoring empty file is not an error'
+test_subtest_known_broken
+notmuch restore < /dev/null 2>OUTPUT.$test_count
+cp /dev/null EXPECTED
+test_expect_equal_file EXPECTED OUTPUT.$test_count
+
+test_begin_subtest 'file of comments and blank lines is not an error'
+test_subtest_known_broken
+notmuch restore --input=comments-and-blanks
+ret_val=$?
+test_expect_equal "$ret_val" "0"
+
+cp comments-and-blanks leading-comments-blanks-batch-tag
+echo "+some_tag -- id:yun1vjwegii.fsf at aiko.keithp.com" \
+>> leading-comments-blanks-batch-tag
+
+test_begin_subtest 'detect format=batch-tag with leading comments and blanks'
+notmuch restore --input=leading-comments-blanks-batch-tag
+notmuch search --output=tags id:yun1vjwegii.fsf at aiko.keithp.com > 
OUTPUT.$test_count
+echo "some_tag" > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT.$test_count
+
+cp comments-and-blanks leading-comments-blanks-sup
+echo "yun1vjwegii.fsf at aiko.keithp.com (another_tag)" \
+>> leading-comments-blanks-sup
+
+test_begin_subtest 'detect format=sup with leading comments and blanks'
+test_subtest_known_broken
+notmuch restore --input=leading-comments-blanks-sup
+notmuch search --output=tags id:yun1vjwegii.fsf at aiko.keithp.com > 
OUTPUT.$test_count
+echo "another_tag" > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT.$test_count
+
 test_begin_subtest 'format=batch-tag, round trip with strange tags'
 notmuch dump --format=batch-tag > EXPECTED.$test_count
 notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
-- 
1.7.10.4

[PATCH] notmuch.c: run uncrustify

[da...@tethera.net]

From: David Bremner 

In anticipation of doing some updates to this code, it simplifies life
if the code is "uncrustify clean" to start with
---
 notmuch.c |   23 +++
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/notmuch.c b/notmuch.c
index 9516dfb..ee2892e 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -22,7 +22,7 @@

 #include "notmuch-client.h"

-typedef int (*command_function_t) (void *ctx, int argc, char *argv[]);
+typedef int (*command_function_t)(void *ctx, int argc, char *argv[]);

 typedef struct command {
 const char *name;
@@ -39,8 +39,8 @@ typedef struct alias {
 } alias_t;

 alias_t aliases[] = {
-{ "part", { "show", "--format=raw"}},
-{ "search-tags", {"search", "--output=tags", "*"}}
+{ "part", { "show", "--format=raw" } },
+{ "search-tags", { "search", "--output=tags", "*" } }
 };

 static int
@@ -66,7 +66,7 @@ static command_t commands[] = {
   "[options...]  [...]",
   "Construct a reply template for a set of messages." },
 { "tag", notmuch_tag_command,
-  "+|- [...] [--]  [...]" ,
+  "+|- [...] [--]  [...]",
   "Add/remove tags for all messages matching the search terms." },
 { "dump", notmuch_dump_command,
   "[] [--] []",
@@ -107,8 +107,8 @@ usage (FILE *out)

 fprintf (out, "\n");
 fprintf (out,
-"Use \"notmuch help \" for more details on each command\n"
-"and \"notmuch help search-terms\" for the common search-terms 
syntax.\n\n");
+"Use \"notmuch help \" for more details on each command\n"
+"and \"notmuch help search-terms\" for the common search-terms 
syntax.\n\n");
 }

 void
@@ -281,15 +281,14 @@ main (int argc, char *argv[])
return notmuch_help_command (NULL, argc - 1, [1]);

 if (strcmp (argv[1], "--version") == 0) {
-   printf ("notmuch " STRINGIFY(NOTMUCH_VERSION) "\n");
+   printf ("notmuch " STRINGIFY (NOTMUCH_VERSION) "\n");
return 0;
 }

 for (i = 0; i < ARRAY_SIZE (aliases); i++) {
alias = [i];

-   if (strcmp (argv[1], alias->name) == 0)
-   {
+   if (strcmp (argv[1], alias->name) == 0) {
int substitutions;

argv_local = talloc_size (local, sizeof (char *) *
@@ -304,14 +303,14 @@ main (int argc, char *argv[])
for (j = 0; j < MAX_ALIAS_SUBSTITUTIONS; j++) {
if (alias->substitutions[j] == NULL)
break;
-   argv_local[j+1] = alias->substitutions[j];
+   argv_local[j + 1] = alias->substitutions[j];
}
substitutions = j;

/* And copy all original arguments (skipping the argument
 * that matched the alias of course. */
for (j = 2; j < (unsigned) argc; j++) {
-   argv_local[substitutions+j-1] = argv[j];
+   argv_local[substitutions + j - 1] = argv[j];
}

argc += substitutions - 1;
@@ -323,7 +322,7 @@ main (int argc, char *argv[])
command = [i];

if (strcmp (argv[1], command->name) == 0)
-   return (command->function) (local, argc - 1, [1]);
+   return (command->function)(local, argc - 1, [1]);
 }

 fprintf (stderr, "Error: Unknown command '%s' (see \"notmuch help\")\n",
-- 
1.7.10.4

[PATCH 11/11] test/tagging: add test for naked punctuation in tags; compare with quoting spaces.

[da...@tethera.net]

From: David Bremner 

This test also serves as documentation of the quoting
requirements. The comment lines are so that it exactly matches the man
page. Nothing more embarrassing than having an example in the man page
fail.
---
 test/tagging |   25 +
 1 file changed, 25 insertions(+)

diff --git a/test/tagging b/test/tagging
index 1717e72..1f5632c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -198,6 +198,31 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest "--batch: only space and % needs to be encoded."
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%23possible%5c +%26are +%28tags%29 +crazy%7b +inbox +match%2acrazy 
+space%20in%20tags +tag4 +tag5 +unread +winner -- id:msg-002 at 
notmuch-test-suite
++foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001 at 
notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: unicode message-ids'

 ${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
-- 
1.7.10.4

[PATCH 10/11] man: document notmuch tag --batch, --input options

[da...@tethera.net]

From: Jani Nikula 

---
 man/man1/notmuch-tag.1 |   92 
 1 file changed, 92 insertions(+)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 9444aa4..3aa2fa5 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -6,6 +6,11 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms
 .B notmuch tag
 .RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"

+.B notmuch tag
+.RI "--batch"
+.RI "[ --input=<" filename "> ]"
+
+
 .SH DESCRIPTION

 Add/remove tags for all messages matching the search terms.
@@ -30,6 +35,93 @@ updates the maildir flags according to tag changes if the
 configuration option is enabled. See \fBnotmuch-config\fR(1) for
 details.

+Supported options for
+.B tag
+include
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read batch tagging operations from a file (stdin by default). This is more
+efficient than repeated
+.B notmuch tag
+invocations. See
+.B TAG FILE FORMAT
+below for the input format. This option is not compatible with
+specifying tagging on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR "\-\-input=" 
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+
+.SH TAG FILE FORMAT
+
+The input must consist of lines of the format:
+
+.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" query ">"
+
+Each line is interpreted similarly to
+.B notmuch tag
+command line arguments. The delimiter is one or more spaces ' '. Any
+characters in
+.RI < tag >
+.B may
+be hex-encoded with %NN where NN is the hexadecimal value of the
+character. To hex-encode a character with a multi-byte UTF-8 encoding,
+hex-encode each byte.
+Any spaces in 
+.B must
+be hex-encoded as %20. Any characters that are not
+part of
+.RI  < tag >
+.B must not
+be hex-encoded.
+
+In the future tag:"tag with spaces" style quoting may be supported for
+.RI < tag >
+as well;
+for this reason all double quote characters in
+.RI < tag >
+.B should
+be hex-encoded.
+
+The
+.RI < query >
+should be quoted using Xapian boolean term quoting rules: if a term
+contains whitespace or a close paren or starts with a double quote, it
+must be enclosed in double quotes (not including any prefix) and
+double quotes inside the term must be doubled (see below for
+examples).
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
+.SS EXAMPLE
+
+The following shows a valid input to batch tagging. Note that only the
+isolated '*' acts as a wildcard. Also note the two different quotings
+of the tag
+.B space in tags
+.
+.RS
+.nf
++winner *
++foo::bar%25 -- (One and Two) or (One and tag:winner)
++found::it -- tag:foo::bar%
+# ignore this line and the next
+
++space%20in%20tags -- Two
+# add tag '(tags)', among other stunts.
++crazy{ +(tags) + +#possible\ -- tag:"space in tags"
++match*crazy -- tag:crazy{
++some_tag -- id:"this is ""nauty)"""
+.fi
+.RE
+
 .SH SEE ALSO

 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[PATCH 09/11] notmuch-tag.1: tidy synopsis formatting, reference

[da...@tethera.net]

From: David Bremner 

Consistently use [...]; one less space. Use singular 
---
 man/man1/notmuch-tag.1 |7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 0f86582..9444aa4 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -4,20 +4,21 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms

 .SH SYNOPSIS
 .B notmuch tag
-.RI  "+<" tag> "|\-<" tag "> [...] [\-\-] <" search-term ">..."
+.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"

 .SH DESCRIPTION

 Add/remove tags for all messages matching the search terms.

 See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for .
+for details of the supported syntax for
+.RI < search-term >.

 Tags prefixed by '+' are added while those prefixed by '\-' are
 removed. For each message, tag removal is performed before tag
 addition.

-The beginning of  is recognized by the first
+The beginning of the search terms is recognized by the first
 argument that begins with neither '+' nor '\-'. Support for
 an initial search term beginning with '+' or '\-' is provided
 by allowing the user to specify a "\-\-" argument to separate
-- 
1.7.10.4

[PATCH 08/11] test/tagging: add test for exotic message-ids and batch tagging

[da...@tethera.net]

From: David Bremner 

The (now fixed) bug that this test revealed is that unquoted
message-ids with whitespace or other control characters in them are
split into several tokens by the Xapian query parser.
---
 test/tagging |   18 ++
 1 file changed, 18 insertions(+)

diff --git a/test/tagging b/test/tagging
index 417112b..1717e72 100755
--- a/test/tagging
+++ b/test/tagging
@@ -198,6 +198,24 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest '--batch: unicode message-ids'
+
+${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
+ --num-messages=100
+
+notmuch dump --format=batch-tag | sed 's/^.* -- /+common_tag -- /' | \
+sort > EXPECTED
+
+notmuch dump --format=batch-tag | sed 's/^.* -- /  -- /' | \
+notmuch restore --format=batch-tag
+
+notmuch tag --batch < EXPECTED
+
+notmuch dump --format=batch-tag| \
+sort > OUTPUT
+
+test_expect_equal_file EXPECTED OUTPUT
+
 test_expect_code 1 "Empty tag names" 'notmuch tag + One'

 test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
-- 
1.7.10.4

[PATCH 07/11] test/tagging: add tests for exotic tags

[da...@tethera.net]

From: David Bremner 

We test quotes seperately because they matter to the query escaper.
---
 test/tagging |   66 ++
 1 file changed, 66 insertions(+)

diff --git a/test/tagging b/test/tagging
index 405ad7c..417112b 100755
--- a/test/tagging
+++ b/test/tagging
@@ -132,6 +132,72 @@ EOF
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest '--batch: tags with quotes'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%22%27%22%22%22%27 +inbox +tag5 +unread -- id:msg-001 at notmuch-test-suite
++%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002 at 
notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest '--batch: tags with punctuation and space'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e 
+inbox +tag4 +tag5 +unread -- id:msg-002 at notmuch-test-suite
++%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e 
+inbox +tag5 +unread -- id:msg-001 at notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest '--batch: unicode tags'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 
+=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d
 
+A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27
 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 
+P%c4%98%2f +R +inbox +tag4 +tag5 +unread 
+%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d 
+%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b
 +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 
+%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-002 at notmuch-test-suite
++%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 
+=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d
 
+A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27
 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 
+P%c4%98%2f +R +inbox +tag5 +unread 
+%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d 
+%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b
 +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 
+%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001 at notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_expect_code 1 "Empty tag names" 'notmuch tag + One'

 test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
-- 
1.7.10.4

[PATCH 06/11] test/tagging: add basic tests for batch tagging functionality

[da...@tethera.net]

From: David Bremner 

This tests argument parsing, blank lines and comments, and basic hex
decoding functionality.
---
 test/tagging |   51 +++
 1 file changed, 51 insertions(+)

diff --git a/test/tagging b/test/tagging
index cd16585..405ad7c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,57 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\"  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)"

+test_begin_subtest "--batch"
+notmuch tag --batch < batch.in  < batch.expected < backup.tags
+notmuch tag --input=batch.in
+notmuch search \* | notmuch_search_sanitize > OUTPUT
+notmuch restore --format=batch-tag < backup.tags
+test_expect_equal_file batch.expected OUTPUT
+
+test_begin_subtest "--batch --input"
+notmuch dump --format=batch-tag > backup.tags
+notmuch tag --batch --input=batch.in
+notmuch search \* | notmuch_search_sanitize > OUTPUT
+notmuch restore --format=batch-tag < backup.tags
+test_expect_equal_file batch.expected OUTPUT
+
+test_begin_subtest "--batch, blank lines and comments"
+notmuch dump | sort > EXPECTED
+notmuch tag --batch < OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: checking error messages'
 notmuch dump --format=batch-tag > BACKUP
 notmuch tag --batch 

[PATCH 05/11] test/tagging: add test for error messages of tag --batch

[da...@tethera.net]

From: David Bremner 

This is based on the similar test for notmuch restore, but the parser
in batch tagging mode is less tolerant of a few cases, in particular
those tested by illegal_tag.
---
 test/tagging |   35 +++
 1 file changed, 35 insertions(+)

diff --git a/test/tagging b/test/tagging
index 980ff92..cd16585 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,41 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\"  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)"

+test_begin_subtest '--batch: checking error messages'
+notmuch dump --format=batch-tag > BACKUP
+notmuch tag --batch 

[PATCH 04/11] cli: add support for batch tagging operations to "notmuch tag"

[da...@tethera.net]

From: Jani Nikula 

Add support for batch tagging operations through stdin to "notmuch
tag". This can be enabled with the new --batch command line option to
"notmuch tag". The input must consist of lines of the format:

+|- [...] [--]  [...]

Each line is interpreted similarly to "notmuch tag" command line
arguments. The delimiter is one or more spaces ' '. Any characters in
 MAP be hex encoded with %NN where NN is the
hexadecimal value of the character. Any ' ' and '%' characters in
 and  MUST be hex encoded (using %20 and %25,
respectively). Any characters that are not part of  or
MUST NOT be hex encoded.

 is passed verbatim to Xapian

Leading and trailing space ' ' is ignored. Empty lines and lines
beginning with '#' are ignored.

Signed-off-by: Jani Nikula 

Hacked-like-crazy-by: David Bremner 
---
 notmuch-tag.c |   94 -
 1 file changed, 86 insertions(+), 8 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 8129912..7fc614d 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -128,6 +128,46 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 return interrupted;
 }

+static int
+tag_file (void *ctx, notmuch_database_t *notmuch, tag_op_flag_t flags,
+ FILE *input)
+{
+char *line = NULL;
+char *query_string = NULL;
+size_t line_size = 0;
+ssize_t line_len;
+int ret = 0;
+tag_op_list_t *tag_ops;
+
+tag_ops = tag_op_list_create (ctx);
+if (tag_ops == NULL) {
+   fprintf (stderr, "Out of memory.\n");
+   return 1;
+}
+
+while ((line_len = getline (, _size, input)) != -1 &&
+  ! interrupted) {
+
+   ret = parse_tag_line (ctx, line, TAG_FLAG_NONE,
+ _string, tag_ops);
+
+   if (ret > 0)
+   continue;
+
+   if (ret < 0)
+   break;
+
+   ret = tag_query (ctx, notmuch, query_string, tag_ops, flags);
+   if (ret)
+   break;
+}
+
+if (line)
+   free (line);
+
+return ret;
+}
+
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
@@ -137,6 +177,10 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 notmuch_database_t *notmuch;
 struct sigaction action;
 tag_op_flag_t tag_flags = TAG_FLAG_NONE;
+notmuch_bool_t batch = FALSE;
+FILE *input = stdin;
+char *input_file_name = NULL;
+int opt_index;
 int ret = 0;

 /* Setup our handler for SIGINT */
@@ -146,15 +190,43 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 action.sa_flags = SA_RESTART;
 sigaction (SIGINT, , NULL);

-tag_ops = tag_op_list_create (ctx);
-if (tag_ops == NULL) {
-   fprintf (stderr, "Out of memory.\n");
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_BOOLEAN, , "batch", 0, 0 },
+   { NOTMUCH_OPT_STRING, _file_name, "input", 'i', 0 },
+   { 0, 0, 0, 0, 0 }
+};
+
+opt_index = parse_arguments (argc, argv, options, 1);
+if (opt_index < 0)
return 1;
+
+if (input_file_name) {
+   batch = TRUE;
+   input = fopen (input_file_name, "r");
+   if (input == NULL) {
+   fprintf (stderr, "Error opening %s for reading: %s\n",
+input_file_name, strerror (errno));
+   return 1;
+   }
 }

-if (parse_tag_command_line (ctx, argc - 1, argv + 1,
-   _string, tag_ops))
-   return 1;
+if (batch) {
+   if (opt_index != argc) {
+   fprintf (stderr, "Can't specify both cmdline and stdin!\n");
+   return 1;
+   }
+} else {
+
+   tag_ops = tag_op_list_create (ctx);
+   if (tag_ops == NULL) {
+   fprintf (stderr, "Out of memory.\n");
+   return 1;
+   }
+
+   if (parse_tag_command_line (ctx, argc - opt_index, argv + opt_index,
+   _string, tag_ops))
+   return 1;
+}

 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
@@ -167,9 +239,15 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 if (notmuch_config_get_maildir_synchronize_flags (config))
tag_flags |= TAG_FLAG_MAILDIR_SYNC;

-ret = tag_query (ctx, notmuch, query_string, tag_ops, tag_flags);
+if (batch)
+   ret = tag_file (ctx, notmuch, tag_flags, input);
+else
+   ret = tag_query (ctx, notmuch, query_string, tag_ops, tag_flags);

 notmuch_database_destroy (notmuch);

-return ret;
+if (input != stdin)
+   fclose (input);
+
+return ret || interrupted;
 }
-- 
1.7.10.4

[PATCH 03/11] notmuch-tag.c: convert to use tag-utils

[da...@tethera.net]

From: David Bremner 

Command line parsing is factored out into a function
parse_tag_command_line in tag-utils.c.

There is some duplicated code eliminated in tag_query, and a bunch of
translation from using the bare tag_op structs to using that tag-utils
API.
---
 notmuch-tag.c |   99 -
 tag-util.c|   51 +++--
 tag-util.h|   15 +
 3 files changed, 84 insertions(+), 81 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index fc9d43a..8129912 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -19,6 +19,7 @@
  */

 #include "notmuch-client.h"
+#include "tag-util.h"
 #include "string-util.h"

 static volatile sig_atomic_t interrupted;
@@ -36,14 +37,10 @@ handle_sigint (unused (int sig))
 interrupted = 1;
 }

-typedef struct {
-const char *tag;
-notmuch_bool_t remove;
-} tag_operation_t;

 static char *
 _optimize_tag_query (void *ctx, const char *orig_query_string,
-const tag_operation_t *tag_ops)
+const tag_op_list_t *list)
 {
 /* This is subtler than it looks.  Xapian ignores the '-' operator
  * at the beginning both queries and parenthesized groups and,
@@ -60,7 +57,7 @@ _optimize_tag_query (void *ctx, const char *orig_query_string,
 size_t i;

 /* Don't optimize if there are no tag changes. */
-if (tag_ops[0].tag == NULL)
+if (tag_op_list_size (list) == 0)
return talloc_strdup (ctx, orig_query_string);

 /* Build the new query string */
@@ -69,17 +66,17 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 else
query_string = talloc_asprintf (ctx, "( %s ) and (", orig_query_string);

-for (i = 0; tag_ops[i].tag && query_string; i++) {
+for (i = 0; i < tag_op_list_size (list) && query_string; i++) {
/* XXX in case of OOM, query_string will be deallocated when
 * ctx is, which might be at shutdown */
if (make_boolean_term (ctx,
-  "tag", tag_ops[i].tag,
+  "tag", tag_op_list_tag (list, i),
   , _len))
return NULL;

query_string = talloc_asprintf_append_buffer (
query_string, "%s%s%s", join,
-   tag_ops[i].remove ? "" : "not ",
+   tag_op_list_isremove (list, i) ? "" : "not ",
escaped);
join = " or ";
 }
@@ -91,17 +88,15 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 return query_string;
 }

-/* Tag messages matching 'query_string' according to 'tag_ops', which
- * must be an array of tagging operations terminated with an empty
- * element. */
+/* Tag messages matching 'query_string' according to 'tag_ops'
+ */
 static int
 tag_query (void *ctx, notmuch_database_t *notmuch, const char *query_string,
-  tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
+  tag_op_list_t *tag_ops, tag_op_flag_t flags)
 {
 notmuch_query_t *query;
 notmuch_messages_t *messages;
 notmuch_message_t *message;
-int i;

 /* Optimize the query so it excludes messages that already have
  * the specified set of tags. */
@@ -124,21 +119,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages) && ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-
-   notmuch_message_freeze (message);
-
-   for (i = 0; tag_ops[i].tag; i++) {
-   if (tag_ops[i].remove)
-   notmuch_message_remove_tag (message, tag_ops[i].tag);
-   else
-   notmuch_message_add_tag (message, tag_ops[i].tag);
-   }
-
-   notmuch_message_thaw (message);
-
-   if (synchronize_flags)
-   notmuch_message_tags_to_maildir_flags (message);
-
+   tag_op_list_apply (message, tag_ops, flags | TAG_FLAG_PRE_OPTIMIZED);
notmuch_message_destroy (message);
 }

@@ -150,15 +131,13 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
-tag_operation_t *tag_ops;
-int tag_ops_count = 0;
-char *query_string;
+tag_op_list_t *tag_ops = NULL;
+char *query_string = NULL;
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
 struct sigaction action;
-notmuch_bool_t synchronize_flags;
-int i;
-int ret;
+tag_op_flag_t tag_flags = TAG_FLAG_NONE;
+int ret = 0;

 /* Setup our handler for SIGINT */
 memset (, 0, sizeof (struct sigaction));
@@ -167,54 +146,15 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 action.sa_flags = SA_RESTART;
 sigaction (SIGINT, , NULL);

-argc--; argv++; /* skip subcommand argument */
-
-/* Array of tagging operations (add or remove), terminated with an
- * empty element. */
-tag_ops = 

[PATCH 02/11] tag-util: factor out rules for illegal tags, use in parse_tag_line

[da...@tethera.net]

From: David Bremner 

This will allow us to be consistent between batch tagging and command
line tagging as far as what is an illegal tag.
---
 tag-util.c |   36 +++-
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/tag-util.c b/tag-util.c
index ca12b3b..0a4fe78 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -31,6 +31,30 @@ line_error (tag_parse_status_t status,
 return status;
 }

+/*
+ * Test tags for some forbidden cases.
+ *
+ * return: NULL if OK,
+ *explanatory message otherwise.
+ */
+
+static const char *
+illegal_tag (const char *tag, notmuch_bool_t remove)
+{
+
+if (*tag == '\0' && ! remove)
+   return "empty tag forbidden";
+
+/* This disallows adding the non-removable tag "-" and
+ * enables notmuch tag to take long options more easily.
+ */
+
+if (*tag == '-' && ! remove)
+   return "tag starting with '-' forbidden";
+
+return NULL;
+}
+
 tag_parse_status_t
 parse_tag_line (void *ctx, char *line,
tag_op_flag_t flags,
@@ -95,11 +119,13 @@ parse_tag_line (void *ctx, char *line,
remove = (*tok == '-');
tag = tok + 1;

-   /* Maybe refuse empty tags. */
-   if (! (flags & TAG_FLAG_BE_GENEROUS) && *tag == '\0') {
-   ret = line_error (TAG_PARSE_INVALID, line_for_error,
- "empty tag");
-   goto DONE;
+   /* Maybe refuse illegal tags. */
+   if (! (flags & TAG_FLAG_BE_GENEROUS)) {
+   const char *msg = illegal_tag (tag, remove);
+   if (msg) {
+   ret = line_error (TAG_PARSE_INVALID, line_for_error, msg);
+   goto DONE;
+   }
}

/* Decode tag. */
-- 
1.7.10.4

[PATCH 01/11] parse_tag_line: use enum for return value.

[da...@tethera.net]

From: David Bremner 

This is essentially cosmetic, since success=0 is promised by
the comments in tag-utils.h.
---
 tag-util.c |4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tag-util.c b/tag-util.c
index e4e5dda..ca12b3b 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -40,14 +40,14 @@ parse_tag_line (void *ctx, char *line,
 char *tok = line;
 size_t tok_len = 0;
 char *line_for_error;
-int ret = 0;
+tag_parse_status_t ret = TAG_PARSE_SUCCESS;

 chomp_newline (line);

 line_for_error = talloc_strdup (ctx, line);
 if (line_for_error == NULL) {
fprintf (stderr, "Error: out of memory\n");
-   return -1;
+   return TAG_PARSE_OUT_OF_MEMORY;
 }

 /* remove leading space */
-- 
1.7.10.4

Xapian-quoting based batch-tagging.

[da...@tethera.net]

This is an alternative version of 

 id:1356313183-9266-1-git-send-email-david at tethera.net

batch tagging patches rebased on top of 

 id:1356415076-5692-1-git-send-email-amdragon at mit.edu

This mainly consisted of removing 

 [Patch v9 04/17] notmuch-tag: factor out double quoting routine
 (superceded by one of Austin's patches)

 [Patch v9 05/17] util/string-util: add a new string tokenized function
 [Patch v9 06/17] unhex_and_quote: new function to quote hex-decoded queries
 [Patch v9 07/17] notmuch-restore: move query handling for batch
 (uneeded if query is passed verbatim to xapian)

 I also removed two tests, since they are about how we handle
 quoting:

 [Patch v9 13/17] test/tagging: add test for compound queries with batch 
tagging
 [Patch v9 17/17] test/tagging: add test for handling of parenthesized  
tag queries.

A few small fixes were needed to the tests, and a fair amount of
changes to the notmuch-tag man page.

Diffstat (against Austin's series) is as follows

 man/man1/notmuch-tag.1 |   99 -
 notmuch-tag.c  |  169 
 tag-util.c |   87 +--
 tag-util.h |   15 
 test/tagging   |  195 ++
 5 files changed, 480 insertions(+), 85 deletions(-)

[Patch v2 4/4] perf-test: add memory leak test for dump restore

[da...@tethera.net]

From: David Bremner 

In id:87vcc2q5n2.fsf at nikula.org, Jani points out a memory leak in the
current version of the sup restore code. Among other things, this test
is intended to verify a fix for that leak.
---
 performance-test/M01-dump-restore |   15 +++
 1 file changed, 15 insertions(+)
 create mode 100755 performance-test/M01-dump-restore

diff --git a/performance-test/M01-dump-restore 
b/performance-test/M01-dump-restore
new file mode 100755
index 000..be5894a
--- /dev/null
+++ b/performance-test/M01-dump-restore
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='dump and restore'
+
+. ./perf-test-lib.sh
+
+memory_start
+
+memory_run 'load nmbug tags' 'notmuch restore --accumulate 
--input=corpus.tags/nmbug.sup-dump'
+memory_run 'dump *' 'notmuch dump --output=tags.sup'
+memory_run 'restore *' 'notmuch restore --input=tags.sup'
+memory_run 'dump --format=batch-tag *' 'notmuch dump --format=batch-tag 
--output=tags.bt'
+memory_run 'restore --format=batch-tag *' 'notmuch restore --format=batch-tag 
--input=tags.bt'
+
+memory_done
-- 
1.7.10.4

[Patch v2 3/4] perf-test: initial version of memory test infrastructure.

[da...@tethera.net]

From: David Bremner 

The idea is run some code under valgrind --leak-check=full and report
a summary, leaving the user to peruse the log file if they want.

We go to some lengths to preserve the log files from accidental
overwriting; the full corpus takes about 3 hours to run under valgrind
on my machine.

The naming of the log directories may be slightly controversial; in
the unlikely event of two runs in less than a second, the log will be
overwritten. A previous version with mktemp+timestamp was dismissed as
overkill; just mktemp alone does not sort nicely.

One new test is included, to check notmuch new for memory leaks.
---
 performance-test/.gitignore   |1 +
 performance-test/M00-new  |   15 
 performance-test/Makefile.local   |   16 ++--
 performance-test/README   |   57 +++-
 performance-test/perf-test-lib.sh |   75 -
 5 files changed, 126 insertions(+), 38 deletions(-)
 create mode 100755 performance-test/M00-new

diff --git a/performance-test/.gitignore b/performance-test/.gitignore
index 6421a9a..f3f9be4 100644
--- a/performance-test/.gitignore
+++ b/performance-test/.gitignore
@@ -1,3 +1,4 @@
 tmp.*/
+log.*/
 corpus/
 notmuch.cache.*/
diff --git a/performance-test/M00-new b/performance-test/M00-new
new file mode 100755
index 000..99c3f52
--- /dev/null
+++ b/performance-test/M00-new
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. ./perf-test-lib.sh
+
+# ensure initial 'notmuch new' is run by memory_start
+uncache_database
+
+memory_start
+
+# run 'notmuch new' a second time, to test different code paths
+memory_run "notmuch new" "notmuch new"
+
+memory_done
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
index 57beb44..73aa963 100644
--- a/performance-test/Makefile.local
+++ b/performance-test/Makefile.local
@@ -4,14 +4,24 @@ dir := performance-test

 include $(dir)/version.sh

+TIME_TEST_SCRIPT := ${dir}/notmuch-time-test
+MEMORY_TEST_SCRIPT := ${dir}/notmuch-memory-test
+
 CORPUS_NAME := notmuch-email-corpus-$(PERFTEST_VERSION).tar.xz
 TXZFILE := ${dir}/download/${CORPUS_NAME}
 SIGFILE := ${TXZFILE}.asc
-TEST_SCRIPT := ${dir}/notmuch-perf-test
 DEFAULT_URL :=  http://notmuchmail.org/releases/${CORPUS_NAME}

+perf-test: time-test memory-test
+
 time-test: setup-perf-test all
-   $(TEST_SCRIPT) $(OPTIONS)
+   @echo
+   $(TIME_TEST_SCRIPT) $(OPTIONS)
+
+memory-test: setup-perf-test all
+   @echo
+   $(MEMORY_TEST_SCRIPT) $(OPTIONS)
+

 .PHONY: download-corpus setup-perf-test

@@ -29,4 +39,4 @@ $(TXZFILE):
 download-corpus:
wget -O ${TXZFILE} ${DEFAULT_URL}

-CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus $(dir)/notmuch.cache.*
+CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/log.* $(dir)/corpus 
$(dir)/notmuch.cache.*
diff --git a/performance-test/README b/performance-test/README
index d1fb6de..996724c 100644
--- a/performance-test/README
+++ b/performance-test/README
@@ -1,3 +1,10 @@
+Performance Tests
+-
+
+This directory contains two kinds of performance tests: time tests,
+and memory tests. The former use gnu time, and the latter use
+valgrind.
+
 Pre-requisites
 --

@@ -5,9 +12,10 @@ In addition to having notmuch, you need:

 - gpg
 - gnu tar
-- gnu time
+- gnu time (for the time tests)
 - xz. Some speedup can be gotten by installing "pixz", but this is
   probably only worthwhile if you are debugging the tests.
+- valgrind (for the memory tests)

 Getting set up to run tests:
 
@@ -36,34 +44,47 @@ for a list of mirrors.
 Running tests
 -

-The easiest way to run performance tests is to say "make time-test", (or
-simply run the notmuch-time-test script). Either command will run all
-available performance tests.
-
-Alternately, you can run a specific subset of tests by simply invoking
-one of the executable scripts in this directory, (such as ./basic).
-Each test script supports the following arguments
+The easiest way to run performance tests is to say "make perf-test".
+This will run all time and memory tests.  Be aware that the memory
+tests are quite time consuming when run on the full corpus, and that
+depending on your interests it may be more sensible to run "make
+time-test" or "make memory-test".  You can also invoke one of the
+scripts notmuch-time-test or notmuch-memory-test or run a more
+specific subset of tests by simply invoking one of the executable
+scripts in this directory, (such as ./T00-new).  Each test script
+supports the following arguments

 --small / --medium / --large   Choose corpus size.
 --debugEnable debugging. In particular don't 
delete
temporary directories.

+When using the make targets, you can pass arguments to all test
+scripts by defining the make variable OPTIONS.
+
 Writing tests
 -

-Have a look at "T01-dump-restore" for an 

[Patch v2 2/4] perf-test: rename current tests as "time tests"

[da...@tethera.net]

From: David Bremner 

This is almost entirely renaming files, except for updating a few
references to those file names, and changing the makefile target.

A new set of memory tests will be run separately because they take
much longer.
---
 performance-test/00-new|   15 ---
 performance-test/01-dump-restore   |   13 -
 performance-test/02-tag|   14 --
 performance-test/Makefile.local|2 +-
 performance-test/README|9 +
 performance-test/T00-new   |   15 +++
 performance-test/T01-dump-restore  |   13 +
 performance-test/T02-tag   |   14 ++
 performance-test/notmuch-perf-test |   27 ---
 performance-test/notmuch-time-test |   27 +++
 10 files changed, 75 insertions(+), 74 deletions(-)
 delete mode 100755 performance-test/00-new
 delete mode 100755 performance-test/01-dump-restore
 delete mode 100755 performance-test/02-tag
 create mode 100755 performance-test/T00-new
 create mode 100755 performance-test/T01-dump-restore
 create mode 100755 performance-test/T02-tag
 delete mode 100755 performance-test/notmuch-perf-test
 create mode 100755 performance-test/notmuch-time-test

diff --git a/performance-test/00-new b/performance-test/00-new
deleted file mode 100755
index 553bb8b..000
--- a/performance-test/00-new
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-test_description='notmuch new'
-
-. ./perf-test-lib.sh
-
-uncache_database
-
-time_start
-
-for i in $(seq 2 6); do
-time_run "notmuch new #$i" 'notmuch new'
-done
-
-time_done
diff --git a/performance-test/01-dump-restore b/performance-test/01-dump-restore
deleted file mode 100755
index b2ff940..000
--- a/performance-test/01-dump-restore
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-test_description='dump and restore'
-
-. ./perf-test-lib.sh
-
-time_start
-
-time_run 'load nmbug tags' 'notmuch restore --accumulate < 
corpus.tags/nmbug.sup-dump'
-time_run 'dump *' 'notmuch dump > tags.out'
-time_run 'restore *' 'notmuch restore < tags.out'
-
-time_done
diff --git a/performance-test/02-tag b/performance-test/02-tag
deleted file mode 100755
index 78ceccc..000
--- a/performance-test/02-tag
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-test_description='tagging'
-
-. ./perf-test-lib.sh
-
-time_start
-
-time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
-time_run 'tag * +existing_tag' "notmuch tag +new_tag '*'"
-time_run 'tag * -existing_tag' "notmuch tag -new_tag '*'"
-time_run 'tag * -missing_tag' "notmuch tag -new_tag '*'"
-
-time_done
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
index 3834e4d..57beb44 100644
--- a/performance-test/Makefile.local
+++ b/performance-test/Makefile.local
@@ -10,7 +10,7 @@ SIGFILE := ${TXZFILE}.asc
 TEST_SCRIPT := ${dir}/notmuch-perf-test
 DEFAULT_URL :=  http://notmuchmail.org/releases/${CORPUS_NAME}

-perf-test: setup-perf-test all
+time-test: setup-perf-test all
$(TEST_SCRIPT) $(OPTIONS)

 .PHONY: download-corpus setup-perf-test
diff --git a/performance-test/README b/performance-test/README
index 1481660..d1fb6de 100644
--- a/performance-test/README
+++ b/performance-test/README
@@ -36,8 +36,8 @@ for a list of mirrors.
 Running tests
 -

-The easiest way to run performance tests is to say "make perf-test", (or
-simply run the notmuch-perf-test script). Either command will run all
+The easiest way to run performance tests is to say "make time-test", (or
+simply run the notmuch-time-test script). Either command will run all
 available performance tests.

 Alternately, you can run a specific subset of tests by simply invoking
@@ -51,7 +51,7 @@ Each test script supports the following arguments
 Writing tests
 -

-Have a look at "01-dump-restore" for an example. Sourcing
+Have a look at "T01-dump-restore" for an example. Sourcing
 "perf-test-lib.sh" is mandatory.  Utility functions include

 - 'add_email_corpus' unpacks a set of messages and adds them to the database.
@@ -65,4 +65,5 @@ Have a look at "01-dump-restore" for an example. Sourcing

 Scripts are run in the order specified in notmuch-perf-test. In the
 future this order might be chosen automatically so please follow the
-convention of starting the name with two digits to specify the order.
+convention of starting the name with 'T' followed by two digits to
+specify the order.
diff --git a/performance-test/T00-new b/performance-test/T00-new
new file mode 100755
index 000..553bb8b
--- /dev/null
+++ b/performance-test/T00-new
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. ./perf-test-lib.sh
+
+uncache_database
+
+time_start
+
+for i in $(seq 2 6); do
+time_run "notmuch new #$i" 'notmuch new'
+done
+
+time_done
diff --git a/performance-test/T01-dump-restore 
b/performance-test/T01-dump-restore
new file mode 100755
index 000..b2ff940
--- /dev/null
+++ 

[Patch v2 1/4] perf-test: remove redundant "initial notmuch new"

[da...@tethera.net]

From: David Bremner 

The initial notmuch-new and caching are now done automatically by
time_start
---
 performance-test/00-new |4 
 1 file changed, 4 deletions(-)

diff --git a/performance-test/00-new b/performance-test/00-new
index 6f0b50c..553bb8b 100755
--- a/performance-test/00-new
+++ b/performance-test/00-new
@@ -8,10 +8,6 @@ uncache_database

 time_start

-time_run 'initial notmuch new' 'notmuch new'
-
-cache_database
-
 for i in $(seq 2 6); do
 time_run "notmuch new #$i" 'notmuch new'
 done
-- 
1.7.10.4

v2 of valgrind based memory tests

[da...@tethera.net]

These obsolete

  id:1355196820-29734-1-git-send-email-david at tethera.net

I tried to follow the suggestions of 

  id:20121216191121.GH6187 at mit.edu

pretty closely.

diff --git a/performance-test/M00-new b/performance-test/M00-new
index 733e9b0..99c3f52 100755
--- a/performance-test/M00-new
+++ b/performance-test/M00-new
@@ -9,6 +9,7 @@ uncache_database

 memory_start

+# run 'notmuch new' a second time, to test different code paths
 memory_run "notmuch new" "notmuch new"

 memory_done
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
index 357d800..73aa963 100644
--- a/performance-test/Makefile.local
+++ b/performance-test/Makefile.local
@@ -4,7 +4,6 @@ dir := performance-test

 include $(dir)/version.sh

-# these two are just make sure dir is expanded at the right time.
 TIME_TEST_SCRIPT := ${dir}/notmuch-time-test
 MEMORY_TEST_SCRIPT := ${dir}/notmuch-memory-test

@@ -17,11 +16,11 @@ perf-test: time-test memory-test

 time-test: setup-perf-test all
@echo
-   $(TIME_TEST_SCRIPT) $(TEST_OPTIONS)
+   $(TIME_TEST_SCRIPT) $(OPTIONS)

 memory-test: setup-perf-test all
@echo
-   $(MEMORY_TEST_SCRIPT) $(TEST_OPTIONS)
+   $(MEMORY_TEST_SCRIPT) $(OPTIONS)


 .PHONY: download-corpus setup-perf-test
diff --git a/performance-test/README b/performance-test/README
index 7eaf5f7..996724c 100644
--- a/performance-test/README
+++ b/performance-test/README
@@ -1,7 +1,7 @@
 Performance Tests
 -

-This directory contains two kinds of performance tests, time tests,
+This directory contains two kinds of performance tests: time tests,
 and memory tests. The former use gnu time, and the latter use
 valgrind.

@@ -12,7 +12,7 @@ In addition to having notmuch, you need:

 - gpg
 - gnu tar
-- gnu time (for the time tests).
+- gnu time (for the time tests)
 - xz. Some speedup can be gotten by installing "pixz", but this is
   probably only worthwhile if you are debugging the tests.
 - valgrind (for the memory tests)
@@ -59,13 +59,13 @@ supports the following arguments
temporary directories.

 When using the make targets, you can pass arguments to all test
-scripts by defining the make variable TEST_OPTIONS.
+scripts by defining the make variable OPTIONS.

 Writing tests
 -

-Have a look at "T01-dump-restore" for an example time test and and
-"M00-new" for an example memory tests. In both cases sourcing
+Have a look at "T01-dump-restore" for an example time test and
+"M00-new" for an example memory test. In both cases sourcing
 "perf-test-lib.sh" is mandatory.

 Basics:
diff --git a/performance-test/perf-test-lib.sh 
b/performance-test/perf-test-lib.sh
index 79eb2c5..10d05e0 100644
--- a/performance-test/perf-test-lib.sh
+++ b/performance-test/perf-test-lib.sh
@@ -89,11 +89,10 @@ add_email_corpus ()

 cp -lr $TAG_CORPUS $TMP_DIRECTORY/corpus.tags
 cp -lr $MAIL_CORPUS $MAIL_DIR
-
 }

-notmuch_new_with_cache () {
-
+notmuch_new_with_cache ()
+{
 if [ -d $DB_CACHE_DIR ]; then
cp -r $DB_CACHE_DIR ${MAIL_DIR}/.notmuch
 else
@@ -102,8 +101,8 @@ notmuch_new_with_cache () {
 fi
 }

-time_start () {
-
+time_start ()
+{
 add_email_corpus

 print_header
@@ -111,17 +110,19 @@ time_start () {
 notmuch_new_with_cache time_run
 }

-memory_start () {
-
+memory_start ()
+{
 add_email_corpus

-_timestamp=$(printf "%x" $(date +"%s"))
-log_dir=$(mktemp -d "${TEST_DIRECTORY}/log.$(basename 
$0)-$corpus_size-${_timestamp}-XX")
+local timestamp=$(date +%Y%m%dT%H%M%S)
+log_dir="${TEST_DIRECTORY}/log.$(basename $0)-$corpus_size-${timestamp}"
+mkdir -p ${log_dir}

 notmuch_new_with_cache memory_run
 }

-memory_run () {
+memory_run ()
+{
 test_count=$(($test_count+1))

 log_file=$log_dir/$test_count.log
@@ -134,11 +135,13 @@ memory_run () {
 echo
 }

-memory_done () {
+memory_done ()
+{
 time_done
 }

-cache_database () {
+cache_database ()
+{
 if [ -d $MAIL_DIR/.notmuch ]; then
cp -r $MAIL_DIR/.notmuch $DB_CACHE_DIR
 else
@@ -146,16 +149,18 @@ cache_database () {
 fi
 }

-uncache_database () {
+uncache_database ()
+{
 rm -rf $DB_CACHE_DIR
 }

-print_header () {
+print_header ()
+{
 printf "\t\t\tWall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n"
-
 }

-time_run () {
+time_run ()
+{
 printf "  %-22s" "$1"
 test_count=$(($test_count+1))
 if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
@@ -166,7 +171,8 @@ time_run () {
 return 0
 }

-time_done () {
+time_done ()
+{
 if [ "$test_failure" = "0" ]; then
rm -rf "$remove_tmp"
exit 0

[PATCH] devel: add script to run tests on a patch series.

[da...@tethera.net]

From: David Bremner 

This script is a thin wrapper around git rebase --interactive, that
allows the user to fine tune patches if they break the test suite, or
violate the coding style guidelines.

The user can always run "git rebase --continue" to ignore false positives.

I decided to use perl because inplace editing with sed is non-portable.
---

I have caught several formatting violations and at least one real bug 
using this as "last step before sending to the list".

No doubt the more talented shell-scripters / perl-hackers in the crowd
can can suggest some improvements.

 devel/check-commit |   10 ++
 devel/check-series |   33 +
 2 files changed, 43 insertions(+)
 create mode 100755 devel/check-commit
 create mode 100755 devel/check-series

diff --git a/devel/check-commit b/devel/check-commit
new file mode 100755
index 000..86648b8
--- /dev/null
+++ b/devel/check-commit
@@ -0,0 +1,10 @@
+# This script is mainly intended to be used by the check-series script
+# but you are welcome to use it as a standalone tool. It takes no
+# parameters and operates on the latest git commit (HEAD).
+
+set -e
+make test
+for file in $(git diff --name-only HEAD^ | grep '[.]\(c\|h\|cc\|hh\)'); do
+uncrustify --replace -c devel/uncrustify.cfg $file
+done
+git diff --quiet
diff --git a/devel/check-series b/devel/check-series
new file mode 100755
index 000..d48e70f
--- /dev/null
+++ b/devel/check-series
@@ -0,0 +1,33 @@
+#/bin/sh
+
+# Usage: check-series [upstream-commit]
+
+# Checks each commit in a patch series (topic branch) by running the
+# script devel/check-commit. If check-commit fails (exits with
+# non-zero status), the user is left in the middle of a git rebase, and
+# can fix the commit, e.g. using git commit --amend, followed by
+# "git rebase --continue". If all else fails, "git rebase --abort" should
+# get you back to where you started.
+#
+# NOTE: this runs "make test" many times, so it can take a while.
+#
+
+trap cleanup EXIT
+
+cleanup () {
+if [ -n "$tmpdir" ]; then
+   rm -rf $tmpdir
+fi
+}
+
+upstream=master
+if [ -n "$1" ]; then
+upstream="$1"
+fi
+
+# make sure we always run the most recent version of check-commit
+# in particular cope with it going away.
+tmpdir=$(mktemp -d)
+cp devel/check-commit $tmpdir
+
+GIT_SEQUENCE_EDITOR="perl -pi -e 's,^\s*([^#\s].*)$,\1\nexec 
$tmpdir/check-commit,'" git rebase -i $upstream
-- 
1.7.10.4

[Patch v9 17/17] test/tagging: add test for handling of parenthesized tag queries.

[da...@tethera.net]

From: David Bremner 

This test is intended to verify that '(' and ')' are passed through
unscathed to Xapian to parse.
---
 test/tagging |   20 
 1 file changed, 20 insertions(+)

diff --git a/test/tagging b/test/tagging
index 748d947..70be943 100755
--- a/test/tagging
+++ b/test/tagging
@@ -202,6 +202,26 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest '--batch: compound queries with parens'
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++compound%201 +compound3 +compound at 2 +compound at 3 +inbox +tag4 +tag5 
+unread -- id:msg-002 at notmuch-test-suite
++compound%201 +compound4 +compound at 2 +compound at 3 +inbox +tag5 +unread -- 
id:msg-001 at notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: unicode tags'
 notmuch dump --format=batch-tag > BACKUP

-- 
1.7.10.4

[Patch v9 16/17] test/tagging: add test for naked punctuation in tags; compare with quoting spaces.

[da...@tethera.net]

From: David Bremner 

This test also serves as documentation of the quoting
requirements. The comment lines are so that it exactly matches the man
page. Nothing more embarrassing than having an example in the man page
fail.
---
 test/tagging |   24 
 1 file changed, 24 insertions(+)

diff --git a/test/tagging b/test/tagging
index 5b48cb7..748d947 100755
--- a/test/tagging
+++ b/test/tagging
@@ -228,6 +228,30 @@ notmuch dump --format=batch-tag | sort > OUTPUT
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT

+test_begin_subtest "--batch: only space and parens need to be quoted"
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag --batch < EXPECTED
++%23possible%5c +%26are +%28tags%29 +crazy%7b +inbox +match%2acrazy 
+space%20in%20tags +tag4 +tag5 +unread +winner -- id:msg-002 at 
notmuch-test-suite
++foo%3a%3abar +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001 at 
notmuch-test-suite
+EOF
+
+notmuch dump --format=batch-tag | sort > OUTPUT
+notmuch restore --format=batch-tag < BACKUP
+test_expect_equal_file EXPECTED OUTPUT
+
 test_begin_subtest '--batch: unicode message-ids'

 ${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
-- 
1.7.10.4

[Patch v9 15/17] man: document notmuch tag --batch, --input options

[da...@tethera.net]

From: Jani Nikula 

---
 man/man1/notmuch-tag.1 |   79 
 1 file changed, 79 insertions(+)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 9444aa4..19ccf7e 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -6,6 +6,11 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms
 .B notmuch tag
 .RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"

+.B notmuch tag
+.RI "--batch"
+.RI "[ --input=<" filename "> ]"
+
+
 .SH DESCRIPTION

 Add/remove tags for all messages matching the search terms.
@@ -30,6 +35,80 @@ updates the maildir flags according to tag changes if the
 configuration option is enabled. See \fBnotmuch-config\fR(1) for
 details.

+Supported options for
+.B tag
+include
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read batch tagging operations from a file (stdin by default). This is more
+efficient than repeated
+.B notmuch tag
+invocations. See
+.B TAG FILE FORMAT
+below for the input format. This option is not compatible with
+specifying tagging on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR "\-\-input=" 
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+
+.SH TAG FILE FORMAT
+
+The input must consist of lines of the format:
+
+.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"
+
+Each line is interpreted similarly to
+.B notmuch tag
+command line arguments. The delimiter is one or more spaces ' '. Any
+characters in
+.RI < tag >
+and
+.RI < search-term >
+.B may
+be hex-encoded with %NN where NN is the hexadecimal value of the
+character. To hex-encode a character with a multi-byte UTF-8 encoding,
+hex-encode each byte.
+Any spaces in  and 
+.B must
+be hex-encoded as %20.  The ':' indicating a prefix like 'id:' or 'tag:',
+the '*' wildcard for all messages, and any characters that are not
+part of
+.RI  < tag >
+or
+.RI < search-term >
+.B must not
+be hex-encoded.  In particular parentheses used to bracket
+sub-expressions must not be encoded.
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
+.SS EXAMPLE
+
+The following shows a valid input to batch tagging. Note that only the
+isolated '*' acts as a wildcard.
+
+.RS
+.nf
++winner *
++foo::bar -- (One and Two) or (One and tag:winner)
++found::it -- tag:foo::bar
+# ignore this line and the next
+
++space%20in%20tags -- Two
+# hex encode tag '(tags)', among other stunts.
++crazy{ +%28tags%29 + +#possible\ -- tag:space%20in%20tags
++match*crazy -- tag:crazy{
+.fi
+.RE
+
 .SH SEE ALSO

 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4