[PATCH 6/6] ruby: Add bindings for notmuch_thread_get_messages

[Austin Clements]

---
 bindings/ruby/defs.h   |3 +++
 bindings/ruby/init.c   |1 +
 bindings/ruby/thread.c |   20 
 3 files changed, 24 insertions(+)

diff --git a/bindings/ruby/defs.h b/bindings/ruby/defs.h
index fe81b3f..5b44585 100644
--- a/bindings/ruby/defs.h
+++ b/bindings/ruby/defs.h
@@ -262,6 +262,9 @@ VALUE
 notmuch_rb_thread_get_toplevel_messages (VALUE self);

 VALUE
+notmuch_rb_thread_get_messages (VALUE self);
+
+VALUE
 notmuch_rb_thread_get_matched_messages (VALUE self);

 VALUE
diff --git a/bindings/ruby/init.c b/bindings/ruby/init.c
index f4931d3..663271d 100644
--- a/bindings/ruby/init.c
+++ b/bindings/ruby/init.c
@@ -306,6 +306,7 @@ Init_notmuch (void)
 rb_define_method (notmuch_rb_cThread, "thread_id", 
notmuch_rb_thread_get_thread_id, 0); /* in thread.c */
 rb_define_method (notmuch_rb_cThread, "total_messages", 
notmuch_rb_thread_get_total_messages, 0); /* in thread.c */
 rb_define_method (notmuch_rb_cThread, "toplevel_messages", 
notmuch_rb_thread_get_toplevel_messages, 0); /* in thread.c */
+rb_define_method (notmuch_rb_cThread, "messages", 
notmuch_rb_thread_get_messages, 0); /* in thread.c */
 rb_define_method (notmuch_rb_cThread, "matched_messages", 
notmuch_rb_thread_get_matched_messages, 0); /* in thread.c */
 rb_define_method (notmuch_rb_cThread, "authors", 
notmuch_rb_thread_get_authors, 0); /* in thread.c */
 rb_define_method (notmuch_rb_cThread, "subject", 
notmuch_rb_thread_get_subject, 0); /* in thread.c */
diff --git a/bindings/ruby/thread.c b/bindings/ruby/thread.c
index efe5aaf..56616d9 100644
--- a/bindings/ruby/thread.c
+++ b/bindings/ruby/thread.c
@@ -92,6 +92,26 @@ notmuch_rb_thread_get_toplevel_messages (VALUE self)
 }

 /*
+ * call-seq: THREAD.messages => MESSAGES
+ *
+ * Get a Notmuch::Messages iterator for the all messages in thread.
+ */
+VALUE
+notmuch_rb_thread_get_messages (VALUE self)
+{
+notmuch_messages_t *messages;
+notmuch_thread_t *thread;
+
+Data_Get_Notmuch_Thread (self, thread);
+
+messages = notmuch_thread_get_messages (thread);
+if (!messages)
+   rb_raise (notmuch_rb_eMemoryError, "Out of memory");
+
+return Data_Wrap_Struct (notmuch_rb_cMessages, NULL, NULL, messages);
+}
+
+/*
  * call-seq: THREAD.matched_messages => fixnum
  *
  * Get the number of messages in thread that matched the search
-- 
1.7.10.4

[PATCH 5/6] python: Add bindings for notmuch_thread_get_messages

[Austin Clements]

---
 bindings/python/notmuch/thread.py |   27 ++-
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/bindings/python/notmuch/thread.py 
b/bindings/python/notmuch/thread.py
index 009cb2b..0454dbd 100644
--- a/bindings/python/notmuch/thread.py
+++ b/bindings/python/notmuch/thread.py
@@ -128,11 +128,6 @@ class Thread(object):
in the thread. It will only iterate over the messages in the thread
which are not replies to other messages in the thread.

-   To iterate over all messages in the thread, the caller will need to
-   iterate over the result of :meth:`Message.get_replies` for each
-   top-level message (and do that recursively for the resulting
-   messages, etc.).
-
 :returns: :class:`Messages`
 :raises: :exc:`NotInitializedError` if query is not initialized
 :raises: :exc:`NullPointerError` if search_messages failed
@@ -147,6 +142,28 @@ class Thread(object):

 return Messages(msgs_p, self)

+"""notmuch_thread_get_messages"""
+_get_messages = nmlib.notmuch_thread_get_messages
+_get_messages.argtypes = [NotmuchThreadP]
+_get_messages.restype = NotmuchMessagesP
+
+def get_messages(self):
+"""Returns a :class:`Messages` iterator for all messages in 'thread'
+
+:returns: :class:`Messages`
+:raises: :exc:`NotInitializedError` if query is not initialized
+:raises: :exc:`NullPointerError` if get_messages failed
+"""
+if not self._thread:
+raise NotInitializedError()
+
+msgs_p = Thread._get_messages(self._thread)
+
+if not msgs_p:
+raise NullPointerError()
+
+return Messages(msgs_p, self)
+
 _get_matched_messages = nmlib.notmuch_thread_get_matched_messages
 _get_matched_messages.argtypes = [NotmuchThreadP]
 _get_matched_messages.restype = c_int
-- 
1.7.10.4

[PATCH 4/6] lib: Add an iterator over all messages in a thread

[Austin Clements]

Previously, getting the list of all messages in a thread required
recursively traversing the thread's message hierarchy, which was both
difficult and resulted in messages being out of order.  This adds a
public function to retrieve an iterator over all of the messages in a
thread in oldest-first order.
---
 lib/notmuch.h |   13 +++--
 lib/thread.cc |6 ++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index 3633bed..3739336 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -719,20 +719,21 @@ int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);

 /* Get a notmuch_messages_t iterator for the top-level messages in
- * 'thread'.
+ * 'thread' in oldest-first order.
  *
  * This iterator will not necessarily iterate over all of the messages
  * in the thread. It will only iterate over the messages in the thread
  * which are not replies to other messages in the thread.
- *
- * To iterate over all messages in the thread, the caller will need to
- * iterate over the result of notmuch_message_get_replies for each
- * top-level message (and do that recursively for the resulting
- * messages, etc.).
  */
 notmuch_messages_t *
 notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread);

+/* Get a notmuch_thread_t iterator for all messages in 'thread' in
+ * oldest-first order.
+ */
+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread);
+
 /* Get the number of messages in 'thread' that matched the search.
  *
  * This count includes only the messages in this thread that were
diff --git a/lib/thread.cc b/lib/thread.cc
index 45a7d1d..c126aac 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -508,6 +508,12 @@ notmuch_thread_get_toplevel_messages (notmuch_thread_t 
*thread)
 return _notmuch_messages_create (thread->toplevel_list);
 }

+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread)
+{
+return _notmuch_messages_create (thread->message_list);
+}
+
 const char *
 notmuch_thread_get_thread_id (notmuch_thread_t *thread)
 {
-- 
1.7.10.4

[PATCH 3/6] lib: Eliminate _notmuch_message_list_append

[Austin Clements]

This API invited micro-optimized and complicated list pointer
manipulation and is no longer used.
---
 lib/messages.c|   17 +++--
 lib/notmuch-private.h |4 
 2 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/lib/messages.c b/lib/messages.c
index 1121864..0eee569 100644
--- a/lib/messages.c
+++ b/lib/messages.c
@@ -42,19 +42,7 @@ _notmuch_message_list_create (const void *ctx)
 return list;
 }

-/* Append a single 'node' to the end of 'list'.
- */
-void
-_notmuch_message_list_append (notmuch_message_list_t *list,
- notmuch_message_node_t *node)
-{
-*(list->tail) = node;
-list->tail = >next;
-}
-
-/* Allocate a new node for 'message' and append it to the end of
- * 'list'.
- */
+/* Append 'message' to the end of 'list'. */
 void
 _notmuch_message_list_add_message (notmuch_message_list_t *list,
   notmuch_message_t *message)
@@ -64,7 +52,8 @@ _notmuch_message_list_add_message (notmuch_message_list_t 
*list,
 node->message = message;
 node->next = NULL;

-_notmuch_message_list_append (list, node);
+*(list->tail) = node;
+list->tail = >next;
 }

 notmuch_messages_t *
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index c054a0e..f38ccb3 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -429,10 +429,6 @@ notmuch_message_list_t *
 _notmuch_message_list_create (const void *ctx);

 void
-_notmuch_message_list_append (notmuch_message_list_t *list,
- notmuch_message_node_t *node);
-
-void
 _notmuch_message_list_add_message (notmuch_message_list_t *list,
   notmuch_message_t *message);

-- 
1.7.10.4

[PATCH 2/6] lib: Separate list of all messages from top-level messages

[Austin Clements]

Previously, thread.cc built up a list of all messages, then
proceeded to tear it apart to transform it into a list of
top-level messages.  Now we simply build a new list of top-level
messages.

This simplifies the interface to _notmuch_message_add_reply,
eliminates the pointer acrobatics from
_resolve_thread_relationships, and will enable us to do things
with the list of all messages in the following patches.
---
 lib/message.cc|4 ++--
 lib/notmuch-private.h |2 +-
 lib/thread.cc |   29 ++---
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index 978de06..171c580 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -462,9 +462,9 @@ notmuch_message_get_thread_id (notmuch_message_t *message)

 void
 _notmuch_message_add_reply (notmuch_message_t *message,
-   notmuch_message_node_t *reply)
+   notmuch_message_t *reply)
 {
-_notmuch_message_list_append (message->replies, reply);
+_notmuch_message_list_add_message (message->replies, reply);
 }

 notmuch_messages_t *
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 7a409f5..c054a0e 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -462,7 +462,7 @@ _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,

 void
 _notmuch_message_add_reply (notmuch_message_t *message,
-   notmuch_message_node_t *reply);
+   notmuch_message_t *reply);

 /* sha1.c */

diff --git a/lib/thread.cc b/lib/thread.cc
index aed87b1..45a7d1d 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -35,7 +35,11 @@ struct visible _notmuch_thread {
 char *authors;
 GHashTable *tags;

+/* All messages, oldest first. */
 notmuch_message_list_t *message_list;
+/* Top-level messages, oldest first. */
+notmuch_message_list_t *toplevel_list;
+
 GHashTable *message_hash;
 int total_messages;
 int matched_messages;
@@ -345,29 +349,22 @@ _thread_add_matched_message (notmuch_thread_t *thread,
 }

 static void
-_resolve_thread_relationships (unused (notmuch_thread_t *thread))
+_resolve_thread_relationships (notmuch_thread_t *thread)
 {
-notmuch_message_node_t **prev, *node;
+notmuch_message_node_t *node;
 notmuch_message_t *message, *parent;
 const char *in_reply_to;

-prev = >message_list->head;
-while ((node = *prev)) {
+for (node = thread->message_list->head; node; node = node->next) {
message = node->message;
in_reply_to = _notmuch_message_get_in_reply_to (message);
if (in_reply_to && strlen (in_reply_to) &&
g_hash_table_lookup_extended (thread->message_hash,
  in_reply_to, NULL,
  (void **) ))
-   {
-   *prev = node->next;
-   if (thread->message_list->tail == >next)
-   thread->message_list->tail = prev;
-   node->next = NULL;
-   _notmuch_message_add_reply (parent, node);
-   } else {
-   prev = &((*prev)->next);
-   }
+   _notmuch_message_add_reply (parent, message);
+   else
+   _notmuch_message_list_add_message (thread->toplevel_list, message);
 }

 /* XXX: After scanning through the entire list looking for parents
@@ -451,7 +448,9 @@ _notmuch_thread_create (void *ctx,
  free, NULL);

 thread->message_list = _notmuch_message_list_create (thread);
-if (unlikely (thread->message_list == NULL)) {
+thread->toplevel_list = _notmuch_message_list_create (thread);
+if (unlikely (thread->message_list == NULL ||
+ thread->toplevel_list == NULL)) {
thread = NULL;
goto DONE;
 }
@@ -506,7 +505,7 @@ _notmuch_thread_create (void *ctx,
 notmuch_messages_t *
 notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread)
 {
-return _notmuch_messages_create (thread->message_list);
+return _notmuch_messages_create (thread->toplevel_list);
 }

 const char *
-- 
1.7.10.4

[PATCH 1/6] lib: Clean up error handling in _notmuch_thread_create

[Austin Clements]

Previously, there were various opportunities for memory leaks in the
error-handling paths of this function.  Use a local talloc context and
some reparenting to make eliminate these leaks, while keeping the
control flow simple.
---
 lib/thread.cc |   33 +++--
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/lib/thread.cc b/lib/thread.cc
index e976d64..aed87b1 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -406,7 +406,8 @@ _notmuch_thread_create (void *ctx,
notmuch_string_list_t *exclude_terms,
notmuch_sort_t sort)
 {
-notmuch_thread_t *thread;
+void *local = talloc_new (ctx);
+notmuch_thread_t *thread = NULL;
 notmuch_message_t *seed_message;
 const char *thread_id;
 char *thread_id_query_string;
@@ -415,24 +416,23 @@ _notmuch_thread_create (void *ctx,
 notmuch_messages_t *messages;
 notmuch_message_t *message;

-seed_message = _notmuch_message_create (ctx, notmuch, seed_doc_id, NULL);
+seed_message = _notmuch_message_create (local, notmuch, seed_doc_id, NULL);
 if (! seed_message)
INTERNAL_ERROR ("Thread seed message %u does not exist", seed_doc_id);

 thread_id = notmuch_message_get_thread_id (seed_message);
-thread_id_query_string = talloc_asprintf (ctx, "thread:%s", thread_id);
+thread_id_query_string = talloc_asprintf (local, "thread:%s", thread_id);
 if (unlikely (thread_id_query_string == NULL))
-   return NULL;
+   goto DONE;

-thread_id_query = notmuch_query_create (notmuch, thread_id_query_string);
+thread_id_query = talloc_steal (
+   local, notmuch_query_create (notmuch, thread_id_query_string));
 if (unlikely (thread_id_query == NULL))
-   return NULL;
+   goto DONE;

-talloc_free (thread_id_query_string);
-
-thread = talloc (ctx, notmuch_thread_t);
+thread = talloc (local, notmuch_thread_t);
 if (unlikely (thread == NULL))
-   return NULL;
+   goto DONE;

 talloc_set_destructor (thread, _notmuch_thread_destructor);

@@ -451,8 +451,10 @@ _notmuch_thread_create (void *ctx,
  free, NULL);

 thread->message_list = _notmuch_message_list_create (thread);
-if (unlikely (thread->message_list == NULL))
-   return NULL;
+if (unlikely (thread->message_list == NULL)) {
+   thread = NULL;
+   goto DONE;
+}

 thread->message_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
  free, NULL);
@@ -489,12 +491,15 @@ _notmuch_thread_create (void *ctx,
_notmuch_message_close (message);
 }

-notmuch_query_destroy (thread_id_query);
-
 _resolve_thread_authors_string (thread);

 _resolve_thread_relationships (thread);

+/* Commit to returning thread. */
+talloc_steal (ctx, thread);
+
+  DONE:
+talloc_free (local);
 return thread;
 }

-- 
1.7.10.4

[PATCH 0/6] API for iterating over all messages in a thread

[Austin Clements]

This series adds a library API for iterating over all messages in a
thread in sorted order.  This is easy for the library to provide and
difficult to obtain from the current API.  Plus, if you don't count
the code added to the bindings, this series is actually a net
decrease of 4 lines of code because of simplifications it enables.

Do we want the API to do more?  Currently it's very minimal, but I can
imagine two ways it could be generalized.  It could take an argument
to indicate which message list to return, which could be all messages,
matched messages, top-level messages, or maybe even unmatched messages
(possibly all in terms of message flags).  It could also take an
argument indicating the desired sort order.  Currently, the caller can
use existing message flag APIs to distinguish matched and unmatched
messages and there's a separate function for the top-level messages.
However, if the API could do all of these things, it would subsume
various other API functions, such as notmuch_thread_get_*_date.

Also, is this the right name for the new API?  In particular, if we do
later want to add a function that returns, say, the list of matched
messages, we'll have a convention collision with
notmuch_thread_get_matched_messages, which returns only a count.

[PATCH 1/1] test: always source test-lib.sh as ./test-lib.sh

[David Bremner]

Tomi Ollila  writes:

>
> Changed the 9 files to execute '. ./test-lib.sh'. The test-lib.sh
> should never be loaded from directory in PATH.

Pushed.

d

[Patch v2 09/17] tag-util.[ch]: New files for common tagging routines

[David Bremner]

>
> Maybe something like the following formatted and consistency-tuned version:
>
> typedef enum { 
>TAG_FLAG_NONE = 0,
>
>/* Operations are synced to maildir, if possible.
> */
>TAG_FLAG_MAILDIR_SYNC = (1 << 0),
>
>/* Remove all tags from message before applying list.
> */
>TAG_FLAG_REMOVE_ALL = (1 << 1),
>
>/* Don't try to avoid database operations. Useful when we
> * know that message passed needs these operations.
> */
>TAG_FLAG_PRE_OPTIMIZED = (1 << 2),
>
>/* Accept strange tags that might be user error;
> * intended for use by notmuch-restore.
> */
>TAG_FLAG_BE_GENEROUS = (1 << 3)
>
> } tag_op_flag_t;
>

Applied.  We may have to fight uncrustify on this, but we both know who
likes to play with uncrustify config ;).

d

[PATCH] test: update dump-restore roundtripping test for batch-tag format

[da...@tethera.net]

From: David Bremner 

Now we can actually round trip these crazy tags and and message ids.
hex-xcode is no longer needed as it's built in.
---


I played with this a bit to try to make the diff nicer; I'm not sure
if I really improved much, but putting the sort into the original test
was a bug fix anyway. This is really for review; I don't think it will
apply without also resending 6/14.

 test/dump-restore |   14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 28474f5..e9ba79d 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -86,18 +86,16 @@ notmuch dump --output=dump-outfile-dash-inbox.actual -- 
from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual

 test_expect_success 'roundtripping random message-ids and tags' \
-'test_subtest_known_broken &&
- ${TEST_DIRECTORY}/random-corpus --num-messages=10 \
+ '${TEST_DIRECTORY}/random-corpus --num-messages=100 \
--config-path=${NOTMUCH_CONFIG} &&
- notmuch dump |
-   ${TEST_DIRECTORY}/hex-xcode --direction=encode |
+ notmuch dump --format=batch-tag |
sort > EXPECTED.$test_count &&
  notmuch tag -random-corpus tag:random-corpus &&
- ${TEST_DIRECTORY}/hex-xcode --direction=decode < EXPECTED.$test_count |
-   notmuch restore 2>/dev/null &&
- notmuch dump |
-   ${TEST_DIRECTORY}/hex-xcode --direction=encode |
+ notmuch restore --format=batch-tag < EXPECTED.$test_count &&
+ notmuch dump --format=batch-tag |
sort > OUTPUT.$test_count &&
  test_cmp EXPECTED.$test_count OUTPUT.$test_count'

 test_done
+
+# Note the database is "poisoned" for sup format at this point.
-- 
1.7.10.4

BUG: notmuch stop on death symlink even..

[calmar c.]

hi all, 

notmuch stops/exits on death symlink even when they are on the
ignore-list at the same time.

on: notmuch new ... arount 360maybe in notmuch-new.c


cheers
marco

-- 
   (o_  It rocks: LINUX + Command-Line-Interface
   //\   GPG: 0x59D90F4D
   V_/_ http://www.calmar.ws

[PATCH v2 7/7] emacs: make emacs use message-ids for tagging

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
> From: Mark Walters 
> 
> This makes emacs use the new --queries=true in search mode and use
> this for tagging.  This fixes the race condition in tagging from
> search mode so mark the tests fixed.
> ---
>  emacs/notmuch.el |   28 +---
>  test/emacs   |2 --
>  2 files changed, 25 insertions(+), 5 deletions(-)
> 
> diff --git a/emacs/notmuch.el b/emacs/notmuch.el
> index 64b9474..6e8ef83 100644
> --- a/emacs/notmuch.el
> +++ b/emacs/notmuch.el
> @@ -473,7 +473,8 @@ BEG."
>(let (output)
>  (notmuch-search-foreach-result beg end
>(lambda (pos)
> - (push (plist-get (notmuch-search-get-result pos) property) output)))
> + (let ((value (plist-get (notmuch-search-get-result pos) property)))
> +   (when value (push value output)

Why is this necessary?  (Assuming it is, it could use a comment, and
probably an update to the docstring.)

>  output))
>  
>  (defun notmuch-search-find-thread-id ( bare)
> @@ -483,6 +484,7 @@ If BARE is set then do not prefix with \"thread:\""
>(let ((thread (plist-get (notmuch-search-get-result) :thread)))
>  (when thread (concat (unless bare "thread:") thread
>  
> +

Unintentional?

>  (defun notmuch-search-find-thread-id-region (beg end)
>"Return a list of threads for the current region"
>(mapcar (lambda (thread) (concat "thread:" thread))
> @@ -492,6 +494,23 @@ If BARE is set then do not prefix with \"thread:\""
>"Return a search string for threads for the current region"
>(mapconcat 'identity (notmuch-search-find-thread-id-region beg end) " or 
> "))
>  
> +;; The following two functions are similar to the previous two but
> +;; they only match messages that were in the the thread when the
> +;; initial search was run. This means that they can be used where it
> +;; is important to avoid races: e.g. when tagging.
> +(defun notmuch-search-find-queries-region (beg end  only-matching)
> +  (interactive)
> +  "Return a list of queries for the current region"
> +  (append (notmuch-search-properties-in-region :matching_msg_query beg end)
> +   (unless only-matching
> + (notmuch-search-properties-in-region :nonmatching_msg_query beg 
> end

Two minor performance nits: Using nconc instead of append will avoid
copying the first list and swapping the two arguments will avoid
needlessly traversing the non-matching list when only-matching is nil.

> +
> +(defun notmuch-search-find-queries-region-search (beg end  
> only-matching)
> +  "Return a search string for messages in threads in the current region"
> +  (mapconcat 'identity

#'identity

> +  (notmuch-search-find-queries-region beg end only-matching)
> +  " or "))
> +
>  (defun notmuch-search-find-authors ()
>"Return the authors for the current thread"
>(plist-get (notmuch-search-get-result) :authors))
> @@ -575,7 +594,7 @@ and will also appear in a buffer named \"*Notmuch 
> errors*\"."
>  
>  (defun notmuch-search-tag-region (beg end  tag-changes)
>"Change tags for threads in the given region."
> -  (let ((search-string (notmuch-search-find-thread-id-region-search beg 
> end)))
> +  (let ((search-string (notmuch-search-find-queries-region-search beg end)))
>  (setq tag-changes (funcall 'notmuch-tag search-string tag-changes))
>  (notmuch-search-foreach-result beg end
>(lambda (pos)
> @@ -851,7 +870,9 @@ non-authors is found, assume that all of the authors 
> match."
>  
>  See `notmuch-tag' for information on the format of TAG-CHANGES."
>(interactive)
> -  (apply 'notmuch-tag notmuch-search-query-string tag-changes))
> +  (apply 'notmuch-tag (notmuch-search-find-queries-region-search
> +(point-min) (point-max) t)
> +  tag-changes))
>  
>  (defun notmuch-search-buffer-title (query)
>"Returns the title for a buffer with notmuch search results."
> @@ -948,6 +969,7 @@ Other optional parameters are used as follows:
>"notmuch-search" buffer
>notmuch-command "search"
>"--format=json"
> +  "--output=with-queries"

--output=with-queries or --queries=true?

>(if oldest-first
>"--sort=oldest-first"
>  "--sort=newest-first")
> diff --git a/test/emacs b/test/emacs
> index 3788439..132768f 100755
> --- a/test/emacs
> +++ b/test/emacs
> @@ -123,7 +123,6 @@ output=$(notmuch search $os_x_darwin_thread | 
> notmuch_search_sanitize)
>  test_expect_equal "$output" "thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
> Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
> unread)"
>  
>  test_begin_subtest "Tag all matching messages from search view"
> -test_subtest_known_broken
>  notmuch tag +test-tag-race from:cworth
>  test_emacs "(notmuch-search \"tag:test-tag-race\")
>   (notmuch-test-wait)"
> @@ -135,7 +134,6 @@ notmuch tag -test-tag-race '*'
>  notmuch 

[PATCH v2 6/7] cli: allow search mode to include msg-ids with JSON output

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
> From: Mark Walters 
> 
> This adds a --queries=true option which modifies the summary output of
> notmuch search by including two extra query strings with each result:
> one query string specifies all matching messages and one query string
> all non-matching messages. Currently these are just lists of message
> ids joined with " or " but that could change in future.
> 
> Currently this is not implemented for text format.
> ---
>  notmuch-search.c |   95 ++---
>  1 files changed, 89 insertions(+), 6 deletions(-)
> 
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 830c4e4..c8fc9a6 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -26,7 +26,8 @@ typedef enum {
>  OUTPUT_THREADS,
>  OUTPUT_MESSAGES,
>  OUTPUT_FILES,
> -OUTPUT_TAGS
> +OUTPUT_TAGS,
> +OUTPUT_SUMMARY_WITH_QUERIES
>  } output_t;
>  
>  static char *
> @@ -46,6 +47,57 @@ sanitize_string (const void *ctx, const char *str)
>  return out;
>  }
>  
> +/* This function takes a message id and returns an escaped string
> + * which can be used as a Xapian query. This involves prefixing with
> + * `id:', putting the id inside double quotes, and doubling any
> + * occurence of a double quote in the message id itself.*/
> +static char *
> +xapian_escape_id (const void *ctx,
> +const char *msg_id)
> +{
> +const char *c;
> +char *escaped_msg_id;
> +escaped_msg_id = talloc_strdup (ctx, "id:\"");

talloc_strdup can fail.

> +for (c=msg_id; *c; c++)

Missing spaces around =.

> + if (*c == '"')
> + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "\"\"");
> + else
> + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "%c", *c);

.. as can talloc_asprintf_append.

> +escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "\"");
> +return escaped_msg_id;
> +}

Unfortunately, this approach will cause reallocation and copying for
every character in msg_id (as well as requiring talloc_asprintf_append
to re-scan escaped_msg_id to find the end on every iteration).  How
about pre-allocating a large enough buffer and keeping your position
in it, like

/* This function takes a message id and returns an escaped string
 * which can be used as a Xapian query. This involves prefixing with
 * `id:', putting the id inside double quotes, and doubling any
 * occurence of a double quote in the message id itself. Returns NULL
 * if memory allocation fails. */
static char *
xapian_escape_id (const void *ctx,
  const char *msg_id)
{
const char *in;
char *out;
char *escaped_msg_id = talloc_array (ctx, char, 6 + strlen (msg_id) * 2);
if (!escaped_msg_id)
return NULL;
strcpy (escaped_msg_id, "id:\"");
out = escaped_msg_id + 4;
for (in = msg_id; *in; ++in) {
if (*in == '"')
*(out++) = '"';
*(out++) = *in;
}
strcpy(out, "\"");
return escaped_msg_id;
}

> +
> +static char *
> +output_msg_query (const void *ctx,
> + sprinter_t *format,
> + notmuch_bool_t matching,
> + notmuch_bool_t first,
> + notmuch_messages_t *messages)
> +{
> +notmuch_message_t *message;
> +char *query, *escaped_msg_id;
> +query = talloc_strdup (ctx, "");
> +for (;
> +  notmuch_messages_valid (messages);
> +  notmuch_messages_move_to_next (messages))
> +{
> + message = notmuch_messages_get (messages);
> + if (notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_MATCH) == 
> matching) {
> + escaped_msg_id = xapian_escape_id (ctx, 
> notmuch_message_get_message_id (message));

Two long lines.

> + if (first) {
> + query = talloc_asprintf_append (query, "%s", escaped_msg_id);
> + first = FALSE;
> + }
> + else

"} else".

> + query = talloc_asprintf_append (query, " or %s", 
> escaped_msg_id);

The "or" is unnecessary, since id is registered with the query parser
as an exclusive boolean term.

You could simplify this to

  query = talloc_asprintf_append (query, "%s%s", first ? "" : " ", 
  escaped_msg_id);

Technically this loop has the same O(n^2) problem as xapian_escape_id
and query_string_from_args, but given that threads rarely have more
than a few dozen messages in them, perhaps it doesn't matter.  OTOH,
this may deal poorly with pathological threads (autogenerated messages
and such).

I wonder if we should have some simple linear-time talloc string
accumulation abstraction in util/...

> + talloc_free (escaped_msg_id);
> + }
> + /* output_msg_query already starts with an ` or' */
> + query = talloc_asprintf_append (query, "%s", output_msg_query (ctx, 
> format, matching, first, notmuch_message_get_replies (message)));

Oof, how unfortunate.  I've got a patch that adds an iterator over all
of the messages in a 

[Patch v2 17/17] tag-util: optimization of tag application

[da...@tethera.net]

From: David Bremner 

The idea is not to bother with restore operations if they don't change
the set of tags. This is actually a relatively common case.

In order to avoid fancy datastructures, this method is quadratic in
the number of tags; at least on my mail database this doesn't seem to
be a big problem.
---
 notmuch-tag.c |2 +-
 tag-util.c|   59 +
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 8a8af0b..e4fca67 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -140,7 +140,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages) && ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-   tag_op_list_apply (message, tag_ops, flags);
+   tag_op_list_apply (message, tag_ops, flags | TAG_FLAG_PRE_OPTIMIZED);
notmuch_message_destroy (message);
 }

diff --git a/tag-util.c b/tag-util.c
index 287cc67..2bb8355 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -111,6 +111,62 @@ message_error (notmuch_message_t *message,
 fprintf (stderr, "Status: %s\n", notmuch_status_to_string (status));
 }

+static int
+makes_changes (notmuch_message_t *message,
+  tag_op_list_t *list,
+  tag_op_flag_t flags)
+{
+
+int i;
+
+notmuch_tags_t *tags;
+notmuch_bool_t changes = FALSE;
+
+/* First, do we delete an existing tag? */
+changes = FALSE;
+for (tags = notmuch_message_get_tags (message);
+! changes && notmuch_tags_valid (tags);
+notmuch_tags_move_to_next (tags)) {
+   const char *cur_tag = notmuch_tags_get (tags);
+   int last_op =  (flags & TAG_FLAG_REMOVE_ALL) ? -1 : 0;
+
+   for (i = 0; i < list->count; i++) {
+   if (strcmp (cur_tag, list->ops[i].tag) == 0) {
+   last_op = list->ops[i].remove ? -1 : 1;
+   }
+   }
+
+   changes = (last_op == -1);
+}
+notmuch_tags_destroy (tags);
+
+if (changes)
+   return TRUE;
+
+/* Now check for adding new tags */
+for (i = 0; i < list->count; i++) {
+   notmuch_bool_t exists = FALSE;
+
+   for (tags = notmuch_message_get_tags (message);
+notmuch_tags_valid (tags);
+notmuch_tags_move_to_next (tags)) {
+   const char *cur_tag = notmuch_tags_get (tags);
+   if (strcmp (cur_tag, list->ops[i].tag) == 0) {
+   exists = TRUE;
+   break;
+   }
+   }
+   notmuch_tags_destroy (tags);
+
+   /* the following test is conservative, it's ok to think we
+* make changes when we don't */
+   if ( ! exists && ! list->ops[i].remove )
+   return TRUE;
+}
+return FALSE;
+
+}
+
 notmuch_status_t
 tag_op_list_apply (notmuch_message_t *message,
   tag_op_list_t *list,
@@ -121,6 +177,9 @@ tag_op_list_apply (notmuch_message_t *message,
 notmuch_status_t status = 0;
 tag_operation_t *tag_ops = list->ops;

+if (! (flags & TAG_FLAG_PRE_OPTIMIZED) && ! makes_changes (message, list, 
flags))
+   return NOTMUCH_STATUS_SUCCESS;
+
 status = notmuch_message_freeze (message);
 if (status) {
message_error (message, status, "freezing message");
-- 
1.7.10.4

[Patch v2 16/17] notmuch-{dump, restore}.1: document new format options

[da...@tethera.net]

From: David Bremner 

More or less arbitrarily, notmuch-dump.1 gets the more detailed
description of the format.
---
 man/man1/notmuch-dump.1|   58 +++
 man/man1/notmuch-restore.1 |   59 +++-
 2 files changed, 111 insertions(+), 6 deletions(-)

diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
index 230deec..9f59905 100644
--- a/man/man1/notmuch-dump.1
+++ b/man/man1/notmuch-dump.1
@@ -5,6 +5,7 @@ notmuch-dump \- creates a plain-text dump of the tags of each 
message
 .SH SYNOPSIS

 .B "notmuch dump"
+.RB  [ "\-\-format=(sup|batch-tag)"  "] [--]"
 .RI "[ --output=<" filename "> ] [--]"
 .RI "[ <" search-term ">...]"

@@ -19,6 +20,63 @@ recreated from the messages themselves.  The output of 
notmuch dump is
 therefore the only critical thing to backup (and much more friendly to
 incremental backup than the native database files.)

+.TP 4
+.B \-\-format=(sup|batch-tag)
+
+Notmuch restore supports two plain text dump formats, both with one message-id
+per line, followed by a list of tags.
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
+compatible with the format of files produced by sup-dump.
+So if you've previously been using sup for mail, then the
+.B "notmuch restore"
+command provides you a way to import all of your tags (or labels as
+sup calls them).
+Each line has the following form
+
+.RS 4
+.RI < message-id >
+.B (
+.RI < tag "> ..."
+.B )
+
+with zero or more tags are separated by spaces. Note that (malformed)
+message-ids may contain arbitrary non-null characters. Note also
+that tags with spaces will not be correctly restored with this format.
+
+.RE
+
+.RE
+.RS 4
+.TP 4
+.B batch-tag
+
+The
+.B batch-tag
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.
+Each line has the form
+
+.RS 4
+.RI "+<" "encoded-tag" "> " "" "+<" "encoded-tag" "> ... -- " "" " <" 
encoded-message-id >
+
+where encoded means that every byte not matching the regex
+.B [A-Za-z0-9+-_@=.:,]
+is replace by
+.B %nn
+where nn is the two digit hex encoding.
+The astute reader will notice this is a special case of the batch input
+format for \fBnotmuch-tag\fR(1).
+
+.RE
+
+
 With no search terms, a dump of all messages in the database will be
 generated.  A "--" argument instructs notmuch that the
 remaining arguments are search terms.
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
index 2fa8733..3860829 100644
--- a/man/man1/notmuch-restore.1
+++ b/man/man1/notmuch-restore.1
@@ -6,6 +6,7 @@ notmuch-restore \- restores the tags from the given file (see 
notmuch dump)

 .B "notmuch restore"
 .RB [ "--accumulate" ]
+.RB [ "--format=(auto|batch-tag|sup)" ]
 .RI "[ --input=<" filename "> ]"

 .SH DESCRIPTION
@@ -15,19 +16,51 @@ Restores the tags from the given file (see

 The input is read from the given filename, if any, or from stdin.

-Note: The dump file format is specifically chosen to be
+
+Supported options for
+.B restore
+include
+.RS 4
+.TP 4
+.B \-\-accumulate
+
+The union of the existing and new tags is applied, instead of
+replacing each message's tags as they are read in from the dump file.
+
+.RE
+.RS 4
+.TP 4
+.B \-\-format=(sup|batch-tag|auto)
+
+Notmuch restore supports two plain text dump formats, with one message-id
+per line, and a list of tags.
+For details of the actual formats, see \fBnotmuch-dump\fR(1).
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
 compatible with the format of files produced by sup-dump.
 So if you've previously been using sup for mail, then the
 .B "notmuch restore"
 command provides you a way to import all of your tags (or labels as
 sup calls them).

-The --accumulate switch causes the union of the existing and new tags to be
-applied, instead of replacing each message's tags as they are read in from the
-dump file.
+.RE
+.RS 4
+.TP 4
+.B batch-tag

-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for .
+The
+.B batch-tag
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.  This
+format hex-escapes all characters those outside of a small character
+set, intended to be suitable for e.g. pathnames in most UNIX-like
+systems.

 .B "notmuch restore"
 updates the maildir flags according to tag changes if the
@@ -36,6 +69,20 @@ configuration option is enabled. See \fBnotmuch-config\fR(1) 
for
 details.

 .RE
+
+.RS 4
+.TP 4
+.B auto
+
+This option (the default) tries to guess the format from the
+input. For correctly formed input in either supported format, this
+heuristic, based the fact that batch-tag format contains no parentheses,
+should be accurate.
+
+.RE
+
+.RE
+
 .SH SEE ALSO

 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[Patch v2 15/17] test: second set of dump/restore --format=batch-tag tests

[da...@tethera.net]

From: David Bremner 

These one need the completed functionality in notmuch-restore. Fairly
exotic tags are tested, but no weird message id's.

We test each possible input to autodetection, both explicit (with
--format=auto) and implicit (without --format).
---
 test/dump-restore |   92 +
 1 file changed, 92 insertions(+)

diff --git a/test/dump-restore b/test/dump-restore
index e08b656..23af2b7 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,6 +85,98 @@ test_begin_subtest "dump --output=outfile -- from:cworth"
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual

+test_begin_subtest "Check for a safe set of message-ids"
+notmuch search --output=messages from:cworth > EXPECTED.$test_count
+notmuch search --output=messages from:cworth |\
+   $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
+test_begin_subtest "format=batch-tag, # round-trip"
+notmuch dump --format=sup | sort > EXPECTED.$test_count
+notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
+notmuch dump --format=sup | sort > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "format=batch-tag, # blank lines and comments"
+notmuch dump --format=batch-tag| sort > EXPECTED.$test_count
+notmuch restore < OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "format=batch-tag, # reverse-round-trip empty tag"
+cat  
OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+
+tag2=$(printf 'this\n tag\t has\n spaces')
+enc2=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag2")
+
+enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
+tag3=$($TEST_DIRECTORY/hex-xcode --direction=decode $enc3)
+
+notmuch dump --format=batch-tag > BACKUP
+
+notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"
+
+test_begin_subtest 'format=batch-tag, round trip with strange tags'
+notmuch dump --format=batch-tag > EXPECTED.$test_count
+notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
+notmuch dump --format=batch-tag > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, checking encoded output'
+cp /dev/null EXPECTED.$test_count
+notmuch dump --format=batch-tag -- from:cworth |\
+awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count
+
+notmuch dump --format=batch-tag -- from:cworth  > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'restoring sane tags'
+notmuch restore --format=batch-tag < BACKUP
+notmuch dump --format=batch-tag > OUTPUT.$test_count
+test_expect_equal_file BACKUP OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, restore=auto'
+notmuch dump --format=batch-tag > EXPECTED.$test_count
+notmuch tag -inbox -unread "*"
+notmuch restore --format=auto < EXPECTED.$test_count
+notmuch dump --format=batch-tag > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=auto'
+notmuch dump --format=sup > EXPECTED.$test_count
+notmuch tag -inbox -unread "*"
+notmuch restore --format=auto < EXPECTED.$test_count
+notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, restore=default'
+notmuch dump --format=batch-tag > EXPECTED.$test_count
+notmuch tag -inbox -unread "*"
+notmuch restore < EXPECTED.$test_count
+notmuch dump --format=batch-tag > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=default'
+notmuch dump --format=sup > EXPECTED.$test_count
+notmuch tag -inbox -unread "*"
+notmuch restore < EXPECTED.$test_count
+notmuch dump --format=sup > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+notmuch dump --format=batch-tag --output=save.tags

 test_expect_success 'roundtripping random message-ids and tags' \
 '${TEST_DIRECTORY}/random-corpus --num-messages=100 
--config-path=${NOTMUCH_CONFIG} &&
-- 
1.7.10.4

[Patch v2 14/17] test: update dump-restore roundtripping test for batch-tag format

[da...@tethera.net]

From: David Bremner 

Now we can actually round trip these crazy tags and and message ids.
hex-xcode is no longer needed as it's built in.
---
 test/dump-restore |   13 +++--
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index a2204fb..e08b656 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,13 +85,14 @@ test_begin_subtest "dump --output=outfile -- from:cworth"
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual

+
 test_expect_success 'roundtripping random message-ids and tags' \
-'test_subtest_known_broken &&
- ${TEST_DIRECTORY}/random-corpus --num-messages=10 
--config-path=${NOTMUCH_CONFIG} &&
- notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode > 
EXPECTED.$test_count &&
+'${TEST_DIRECTORY}/random-corpus --num-messages=100 
--config-path=${NOTMUCH_CONFIG} &&
+ notmuch dump --format=batch-tag | sort > EXPECTED.$test_count &&
  notmuch tag -random-corpus tag:random-corpus &&
- ${TEST_DIRECTORY}/hex-xcode --direction=decode < EXPECTED.$test_count | 
notmuch restore 2>/dev/null &&
- notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode > 
OUTPUT.$test_count &&
+ notmuch restore --format=batch-tag --input=EXPECTED.$test_count &&
+ notmuch dump --format=batch-tag | sort > OUTPUT.$test_count &&
  test_cmp EXPECTED.$test_count OUTPUT.$test_count'
-
 test_done
+
+# Note the database is "poisoned" for sup format at this point.
-- 
1.7.10.4

[Patch v2 13/17] notmuch-restore: add support for input format 'batch-tag'

[da...@tethera.net]

From: David Bremner 

This is the same as the batch input for notmuch tag, except by default
it removes all tags before modifying a given message id and only "id:"
is supported.
---
 notmuch-restore.c |  199 +
 1 file changed, 125 insertions(+), 74 deletions(-)

diff --git a/notmuch-restore.c b/notmuch-restore.c
index f03dcac..22fcd2d 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -19,18 +19,22 @@
  */

 #include "notmuch-client.h"
+#include "dump-restore-private.h"
+#include "tag-util.h"
+#include "string-util.h"
+
+static volatile sig_atomic_t interrupted;
+static regex_t regex;

 static int
-tag_message (notmuch_database_t *notmuch, const char *message_id,
-char *file_tags, notmuch_bool_t remove_all,
-notmuch_bool_t synchronize_flags)
+tag_message (unused (void *ctx),
+notmuch_database_t *notmuch,
+const char *message_id,
+tag_op_list_t *tag_ops,
+tag_op_flag_t flags)
 {
 notmuch_status_t status;
-notmuch_tags_t *db_tags;
-char *db_tags_str;
 notmuch_message_t *message = NULL;
-const char *tag;
-char *next;
 int ret = 0;

 status = notmuch_database_find_message (notmuch, message_id, );
@@ -44,55 +48,63 @@ tag_message (notmuch_database_t *notmuch, const char 
*message_id,

 /* In order to detect missing messages, this check/optimization is
  * intentionally done *after* first finding the message. */
-if (! remove_all && (file_tags == NULL || *file_tags == '\0'))
-   goto DONE;
-
-db_tags_str = NULL;
-for (db_tags = notmuch_message_get_tags (message);
-notmuch_tags_valid (db_tags);
-notmuch_tags_move_to_next (db_tags)) {
-   tag = notmuch_tags_get (db_tags);
-
-   if (db_tags_str)
-   db_tags_str = talloc_asprintf_append (db_tags_str, " %s", tag);
-   else
-   db_tags_str = talloc_strdup (message, tag);
-}
+if ( (flags & TAG_FLAG_REMOVE_ALL) || (tag_op_list_size (tag_ops)))
+   tag_op_list_apply (message, tag_ops, flags);

-if (((file_tags == NULL || *file_tags == '\0') &&
-(db_tags_str == NULL || *db_tags_str == '\0')) ||
-   (file_tags && db_tags_str && strcmp (file_tags, db_tags_str) == 0))
-   goto DONE;
+if (message)
+   notmuch_message_destroy (message);

-notmuch_message_freeze (message);
+return ret;
+}

-if (remove_all)
-   notmuch_message_remove_all_tags (message);
+static int
+parse_sup_line (void *ctx, char *line,
+   char **query_str, tag_op_list_t *tag_ops)
+{

-next = file_tags;
-while (next) {
-   tag = strsep (, " ");
-   if (*tag == '\0')
-   continue;
-   status = notmuch_message_add_tag (message, tag);
-   if (status) {
-   fprintf (stderr, "Error applying tag %s to message %s:\n",
-tag, message_id);
-   fprintf (stderr, "%s\n", notmuch_status_to_string (status));
-   ret = 1;
-   }
+regmatch_t match[3];
+char *file_tags;
+int rerr;
+
+tag_op_list_reset (tag_ops);
+
+chomp_newline (line);
+
+/* Silently ignore blank lines */
+if (line[0] == '\0') {
+   return 1;
+}
+
+rerr = xregexec (, line, 3, match, 0);
+if (rerr == REG_NOMATCH) {
+   fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+line);
+   return 1;
 }

-notmuch_message_thaw (message);
+*query_str = talloc_strndup (ctx, line + match[1].rm_so,
+match[1].rm_eo - match[1].rm_so);
+file_tags = talloc_strndup (ctx, line + match[2].rm_so,
+   match[2].rm_eo - match[2].rm_so);

-if (synchronize_flags)
-   notmuch_message_tags_to_maildir_flags (message);
+char *tok = file_tags;
+size_t tok_len = 0;

-  DONE:
-if (message)
-   notmuch_message_destroy (message);
+tag_op_list_reset (tag_ops);
+
+while ((tok = strtok_len (tok + tok_len, " ", _len)) != NULL) {
+
+   if (*(tok + tok_len) != '\0') {
+   *(tok + tok_len) = '\0';
+   tok_len++;
+   }
+
+   if (tag_op_list_append (ctx, tag_ops, tok, FALSE))
+   return -1;
+}
+
+return 0;

-return ret;
 }

 int
@@ -100,16 +112,19 @@ notmuch_restore_command (unused (void *ctx), int argc, 
char *argv[])
 {
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
-notmuch_bool_t synchronize_flags;
 notmuch_bool_t accumulate = FALSE;
+tag_op_flag_t flags = 0;
+tag_op_list_t *tag_ops;
+
 char *input_file_name = NULL;
 FILE *input = stdin;
 char *line = NULL;
 size_t line_size;
 ssize_t line_len;
-regex_t regex;
-int rerr;
+
+int ret = 0;
 int opt_index;
+int input_format = DUMP_FORMAT_AUTO;

 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
@@ -119,9 +134,15 @@ 

[Patch v2 12/17] man: document notmuch tag --batch, --input options

[da...@tethera.net]

From: Jani Nikula 

---
 man/man1/notmuch-tag.1 |   52 +++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 0f86582..751db7b 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -4,7 +4,12 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms

 .SH SYNOPSIS
 .B notmuch tag
-.RI  "+<" tag> "|\-<" tag "> [...] [\-\-] <" search-term ">..."
+.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" search-terms ">"
+
+.B notmuch tag
+.RI "--batch"
+.RI "[ --input=<" filename "> ]"
+

 .SH DESCRIPTION

@@ -29,6 +34,51 @@ updates the maildir flags according to tag changes if the
 configuration option is enabled. See \fBnotmuch-config\fR(1) for
 details.

+Supported options for
+.B tag
+include
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read batch tagging operations from standard input. This is more
+efficient than repeated
+.B notmuch tag
+invocations. See
+.B TAG FILE FORMAT
+below for the input format. This option is not compatible with
+specifying tagging on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR "\-\-input=" 
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+
+.SH TAG FILE FORMAT
+
+The input must consist of lines of the format:
+
+.RI "T +<" tag ">|\-<" tag "> [...] [\-\-] <" search-terms ">"
+
+Each line is interpreted similarly to
+.B notmuch tag
+command line arguments. The delimiter is one or more spaces ' '. Any
+characters in  and 
+.B may
+be hex encoded with %NN where NN is the hexadecimal value of the
+character. Any ' ' and '%' characters in  and 
+.B must
+be hex encoded (using %20 and %25, respectively). Any characters that
+are not part of  or 
+.B must not
+be hex encoded.
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
 .SH SEE ALSO

 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

[Patch v2 11/17] test: add test for notmuch tag --batch option

[da...@tethera.net]

From: Jani Nikula 

Basic test of functionality, along with all combinations of options.
---
 test/tagging |   46 ++
 1 file changed, 46 insertions(+)

diff --git a/test/tagging b/test/tagging
index 980ff92..e5b8315 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,52 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\"  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)"

+test_begin_subtest "--batch"
+notmuch tag --batch < batch.in  < EXPECTED.$test_count
+notmuch tag --batch < OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
 test_expect_code 1 "Empty tag names" 'notmuch tag + One'

 test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
-- 
1.7.10.4

[Patch v2 10/17] cli: add support for batch tagging operations to "notmuch tag"

[da...@tethera.net]

From: Jani Nikula 

Add support for batch tagging operations through stdin to "notmuch
tag". This can be enabled with the new --stdin command line option to
"notmuch tag". The input must consist of lines of the format:

+|- [...] [--] 

Each line is interpreted similarly to "notmuch tag" command line
arguments. The delimiter is one or more spaces ' '. Any characters in
 and  MAY be hex encoded with %NN where NN is the
hexadecimal value of the character. Any ' ' and '%' characters in
 and  MUST be hex encoded (using %20 and %25,
respectively). Any characters that are not part of  or
 MUST NOT be hex encoded.

Leading and trailing space ' ' is ignored. Empty lines and lines
beginning with '#' are ignored.

Signed-off-by: Jani Nikula 

Hacked-like-crazy-by: David Bremner 
---
 notmuch-tag.c |  194 +++--
 1 file changed, 118 insertions(+), 76 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 88d559b..8a8af0b 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -19,6 +19,7 @@
  */

 #include "notmuch-client.h"
+#include "tag-util.h"

 static volatile sig_atomic_t interrupted;

@@ -54,14 +55,9 @@ _escape_tag (char *buf, const char *tag)
 return buf;
 }

-typedef struct {
-const char *tag;
-notmuch_bool_t remove;
-} tag_operation_t;
-
 static char *
 _optimize_tag_query (void *ctx, const char *orig_query_string,
-const tag_operation_t *tag_ops)
+const tag_op_list_t *list)
 {
 /* This is subtler than it looks.  Xapian ignores the '-' operator
  * at the beginning both queries and parenthesized groups and,
@@ -73,19 +69,20 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,

 char *escaped, *query_string;
 const char *join = "";
-int i;
+size_t i;
 unsigned int max_tag_len = 0;

 /* Don't optimize if there are no tag changes. */
-if (tag_ops[0].tag == NULL)
+if (tag_op_list_size (list) == 0)
return talloc_strdup (ctx, orig_query_string);

 /* Allocate a buffer for escaping tags.  This is large enough to
  * hold a fully escaped tag with every character doubled plus
  * enclosing quotes and a NUL. */
-for (i = 0; tag_ops[i].tag; i++)
-   if (strlen (tag_ops[i].tag) > max_tag_len)
-   max_tag_len = strlen (tag_ops[i].tag);
+for (i = 0; i < tag_op_list_size (list); i++)
+   if (strlen (tag_op_list_tag (list, i)) > max_tag_len)
+   max_tag_len = strlen (tag_op_list_tag (list, i));
+
 escaped = talloc_array (ctx, char, max_tag_len * 2 + 3);
 if (! escaped)
return NULL;
@@ -96,11 +93,11 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 else
query_string = talloc_asprintf (ctx, "( %s ) and (", orig_query_string);

-for (i = 0; tag_ops[i].tag && query_string; i++) {
+for (i = 0; i < tag_op_list_size (list) && query_string; i++) {
query_string = talloc_asprintf_append_buffer (
query_string, "%s%stag:%s", join,
-   tag_ops[i].remove ? "" : "not ",
-   _escape_tag (escaped, tag_ops[i].tag));
+   tag_op_list_isremove (list, i) ? "" : "not ",
+   _escape_tag (escaped, tag_op_list_tag (list, i)));
join = " or ";
 }

@@ -116,12 +113,11 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
  * element. */
 static int
 tag_query (void *ctx, notmuch_database_t *notmuch, const char *query_string,
-  tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
+  tag_op_list_t *tag_ops, tag_op_flag_t flags)
 {
 notmuch_query_t *query;
 notmuch_messages_t *messages;
 notmuch_message_t *message;
-int i;

 /* Optimize the query so it excludes messages that already have
  * the specified set of tags. */
@@ -144,21 +140,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages) && ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-
-   notmuch_message_freeze (message);
-
-   for (i = 0; tag_ops[i].tag; i++) {
-   if (tag_ops[i].remove)
-   notmuch_message_remove_tag (message, tag_ops[i].tag);
-   else
-   notmuch_message_add_tag (message, tag_ops[i].tag);
-   }
-
-   notmuch_message_thaw (message);
-
-   if (synchronize_flags)
-   notmuch_message_tags_to_maildir_flags (message);
-
+   tag_op_list_apply (message, tag_ops, flags);
notmuch_message_destroy (message);
 }

@@ -170,15 +152,17 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
-tag_operation_t *tag_ops;
-int tag_ops_count = 0;
-char *query_string;
+tag_op_list_t *tag_ops = NULL;
+char *query_string = NULL;
 notmuch_config_t *config;
 

[Patch v2 09/17] tag-util.[ch]: New files for common tagging routines

[da...@tethera.net]

From: David Bremner 

These are meant to be shared between notmuch-tag and notmuch-restore.

The bulk of the routines implement a "tag operation list" abstract
data type act as a structured representation of a set of tag
operations (typically coming from a single tag command or line of
input).
---
 Makefile.local |1 +
 tag-util.c |  264 
 tag-util.h |  120 ++
 3 files changed, 385 insertions(+)
 create mode 100644 tag-util.c
 create mode 100644 tag-util.h

diff --git a/Makefile.local b/Makefile.local
index 2b91946..854867d 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -274,6 +274,7 @@ notmuch_client_srcs =   \
query-string.c  \
mime-node.c \
crypto.c\
+   tag-util.c

 notmuch_client_modules = $(notmuch_client_srcs:.c=.o)

diff --git a/tag-util.c b/tag-util.c
new file mode 100644
index 000..287cc67
--- /dev/null
+++ b/tag-util.c
@@ -0,0 +1,264 @@
+#include "string-util.h"
+#include "tag-util.h"
+#include "hex-escape.h"
+
+struct _tag_operation_t {
+const char *tag;
+notmuch_bool_t remove;
+};
+
+struct _tag_op_list_t {
+tag_operation_t *ops;
+int count;
+int size;
+};
+
+int
+parse_tag_line (void *ctx, char *line,
+   tag_op_flag_t flags,
+   char **query_string,
+   tag_op_list_t *tag_ops)
+{
+char *tok = line;
+size_t tok_len = 0;
+
+chomp_newline (line);
+
+/* remove leading space */
+while (*tok == ' ' || *tok == '\t')
+   tok++;
+
+/* Skip empty and comment lines. */
+if (*tok == '\0' || *tok == '#')
+   return 1;
+
+tag_op_list_reset (tag_ops);
+
+/* Parse tags. */
+while ((tok = strtok_len (tok + tok_len, " ", _len)) != NULL) {
+   notmuch_bool_t remove;
+   char *tag;
+
+   /* Optional explicit end of tags marker. */
+   if (strncmp (tok, "--", tok_len) == 0) {
+   tok = strtok_len (tok + tok_len, " ", _len);
+   break;
+   }
+
+   /* Implicit end of tags. */
+   if (*tok != '-' && *tok != '+')
+   break;
+
+   /* If tag is terminated by NUL, there's no query string. */
+   if (*(tok + tok_len) == '\0') {
+   tok = NULL;
+   break;
+   }
+
+   /* Terminate, and start next token after terminator. */
+   *(tok + tok_len++) = '\0';
+
+   remove = (*tok == '-');
+   tag = tok + 1;
+
+   /* Maybe refuse empty tags. */
+   if (!(flags & TAG_FLAG_BE_GENEROUS) && *tag == '\0') {
+   tok = NULL;
+   break;
+   }
+
+   /* Decode tag. */
+   if (hex_decode_inplace (tag) != HEX_SUCCESS) {
+   tok = NULL;
+   break;
+   }
+
+   if (tag_op_list_append (ctx, tag_ops, tag, remove))
+   return -1;
+}
+
+if (tok == NULL || tag_ops->count == 0) {
+   /* FIXME: line has been modified! */
+   fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+line);
+   return 1;
+}
+
+/* tok now points to the query string */
+if (hex_decode_inplace (tok) != HEX_SUCCESS) {
+   /* FIXME: line has been modified! */
+   fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
+line);
+   return 1;
+}
+
+*query_string = tok;
+
+return 0;
+}
+
+static inline void
+message_error (notmuch_message_t *message,
+  notmuch_status_t status,
+  const char *format, ...)
+{
+va_list va_args;
+
+va_start (va_args, format);
+
+vfprintf (stderr, format, va_args);
+fprintf (stderr, "Message-ID: %s\n", notmuch_message_get_message_id 
(message));
+fprintf (stderr, "Status: %s\n", notmuch_status_to_string (status));
+}
+
+notmuch_status_t
+tag_op_list_apply (notmuch_message_t *message,
+  tag_op_list_t *list,
+  tag_op_flag_t flags)
+{
+int i;
+
+notmuch_status_t status = 0;
+tag_operation_t *tag_ops = list->ops;
+
+status = notmuch_message_freeze (message);
+if (status) {
+   message_error (message, status, "freezing message");
+   return status;
+}
+
+if (flags & TAG_FLAG_REMOVE_ALL) {
+   status = notmuch_message_remove_all_tags (message);
+   if (status) {
+   message_error (message, status, "removing all tags" );
+   return status;
+   }
+}
+
+for (i = 0; i < list->count; i++) {
+   if (tag_ops[i].remove) {
+   status = notmuch_message_remove_tag (message, tag_ops[i].tag);
+   if (status) {
+   message_error (message, status, "removing tag %s", 
tag_ops[i].tag);
+   return status;
+   }
+   } else {
+   status = notmuch_message_add_tag (message, tag_ops[i].tag);
+   if (status) {
+   message_error (message, status, "adding tag %s", 
tag_ops[i].tag);
+ 

[Patch v2 08/17] util: add string-util.[ch]

[da...@tethera.net]

From: David Bremner 

This is to give a home to strtok_len. It's a bit silly to add a header
for one routine, but it needs to be shared between several compilation
units (or at least that's the most natural design).
---
 util/Makefile.local |3 ++-
 util/string-util.c  |   34 ++
 util/string-util.h  |   19 +++
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 util/string-util.c
 create mode 100644 util/string-util.h

diff --git a/util/Makefile.local b/util/Makefile.local
index 3ca623e..a11e35b 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,8 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)

-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
+ $(dir)/string-util.c

 libutil_modules := $(libutil_c_srcs:.c=.o)

diff --git a/util/string-util.c b/util/string-util.c
new file mode 100644
index 000..44f8cd3
--- /dev/null
+++ b/util/string-util.c
@@ -0,0 +1,34 @@
+/* string-util.c -  Extra or enhanced routines for null terminated strings.
+ *
+ * Copyright (c) 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula 
+ */
+
+
+#include "string-util.h"
+
+char *
+strtok_len (char *s, const char *delim, size_t *len)
+{
+/* skip initial delims */
+s += strspn (s, delim);
+
+/* length of token */
+*len = strcspn (s, delim);
+
+return *len ? s : NULL;
+}
diff --git a/util/string-util.h b/util/string-util.h
new file mode 100644
index 000..696da40
--- /dev/null
+++ b/util/string-util.h
@@ -0,0 +1,19 @@
+#ifndef _STRING_UTIL_H
+#define _STRING_UTIL_H
+
+#include 
+
+/* like strtok(3), but without state, and doesn't modify s. usage pattern:
+ *
+ * const char *tok = input;
+ * const char *delim = " \t";
+ * size_t tok_len = 0;
+ *
+ * while ((tok = strtok_len (tok + tok_len, delim, _len)) != NULL) {
+ * // do stuff with string tok of length tok_len
+ * }
+ */
+
+char *strtok_len (char *s, const char *delim, size_t *len);
+
+#endif
-- 
1.7.10.4

[Patch v2 07/17] notmuch-dump: add --format=(batch-tag|sup)

[da...@tethera.net]

From: David Bremner 

sup is the old format, and remains the default, at least until
restore is converted to parse this format.

Each line of the batch- format is valid for notmuch tag, (with the
"notmuch tag" omitted from the front of the line). The dump format
only uses query strings of a single message-id.

Each space seperated tag/message-id is 'hex-encoded' to remove
troubling characters.  In particular this format won't have the same
problem with e.g. spaces in message-ids or tags; they will be
round-trip-able.
---
 dump-restore-private.h |   13 +
 notmuch-dump.c |   42 --
 2 files changed, 49 insertions(+), 6 deletions(-)
 create mode 100644 dump-restore-private.h

diff --git a/dump-restore-private.h b/dump-restore-private.h
new file mode 100644
index 000..896a004
--- /dev/null
+++ b/dump-restore-private.h
@@ -0,0 +1,13 @@
+#ifndef DUMP_RESTORE_PRIVATE_H
+#define DUMP_RESTORE_PRIVATE_H
+
+#include "hex-escape.h"
+#include "command-line-arguments.h"
+
+typedef enum dump_formats {
+DUMP_FORMAT_AUTO,
+DUMP_FORMAT_BATCH_TAG,
+DUMP_FORMAT_SUP
+} dump_format_t;
+
+#endif
diff --git a/notmuch-dump.c b/notmuch-dump.c
index 88f598a..045ca9e 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,6 +19,7 @@
  */

 #include "notmuch-client.h"
+#include "dump-restore-private.h"

 int
 notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
@@ -43,7 +44,13 @@ notmuch_dump_command (unused (void *ctx), int argc, char 
*argv[])
 char *output_file_name = NULL;
 int opt_index;

+int output_format = DUMP_FORMAT_SUP;
+
 notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_KEYWORD, _format, "format", 'f',
+ (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
+ { "batch-tag", DUMP_FORMAT_BATCH_TAG },
+ { 0, 0 } } },
{ NOTMUCH_OPT_STRING, _file_name, "output", 'o', 0  },
{ 0, 0, 0, 0, 0 }
 };
@@ -83,27 +90,50 @@ notmuch_dump_command (unused (void *ctx), int argc, char 
*argv[])
  */
 notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);

+char *buffer = NULL;
+size_t buffer_size = 0;
+
 for (messages = notmuch_query_search_messages (query);
 notmuch_messages_valid (messages);
 notmuch_messages_move_to_next (messages)) {
int first = 1;
+   const char *message_id;
+
message = notmuch_messages_get (messages);
+   message_id = notmuch_message_get_message_id (message);

-   fprintf (output,
-"%s (", notmuch_message_get_message_id (message));
+   if (output_format == DUMP_FORMAT_SUP) {
+   fprintf (output, "%s (", message_id);
+   }

for (tags = notmuch_message_get_tags (message);
 notmuch_tags_valid (tags);
 notmuch_tags_move_to_next (tags)) {
-   if (! first)
-   fprintf (output, " ");
+   const char *tag_str = notmuch_tags_get (tags);

-   fprintf (output, "%s", notmuch_tags_get (tags));
+   if (! first)
+   fputs (" ", output);

first = 0;
+
+   if (output_format == DUMP_FORMAT_SUP) {
+   fputs (tag_str, output);
+   } else {
+   if (hex_encode (notmuch, tag_str,
+   , _size) != HEX_SUCCESS)
+   return 1;
+   fprintf (output, "+%s", buffer);
+   }
}

-   fprintf (output, ")\n");
+   if (output_format == DUMP_FORMAT_SUP) {
+   fputs (")\n", output);
+   } else {
+   if (hex_encode (notmuch, message_id,
+   , _size) != HEX_SUCCESS)
+   return 1;
+   fprintf (output, " -- id:%s\n", buffer);
+   }

notmuch_message_destroy (message);
 }
-- 
1.7.10.4

[Patch v2 06/17] test: add broken roundtrip test

[da...@tethera.net]

From: David Bremner 

We demonstrate the current notmuch restore parser being confused by
message-id's and tags containing non alpha numeric characters
(particularly space and parentheses are problematic because they are
not escaped by notmuch dump).

We save the files as hex escaped on disk so that the output from the
failing test will not confuse the terminal emulator of people running
the test.
---
 test/dump-restore |9 +
 1 file changed, 9 insertions(+)

diff --git a/test/dump-restore b/test/dump-restore
index b05399c..a2204fb 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,4 +85,13 @@ test_begin_subtest "dump --output=outfile -- from:cworth"
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual

+test_expect_success 'roundtripping random message-ids and tags' \
+'test_subtest_known_broken &&
+ ${TEST_DIRECTORY}/random-corpus --num-messages=10 
--config-path=${NOTMUCH_CONFIG} &&
+ notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode > 
EXPECTED.$test_count &&
+ notmuch tag -random-corpus tag:random-corpus &&
+ ${TEST_DIRECTORY}/hex-xcode --direction=decode < EXPECTED.$test_count | 
notmuch restore 2>/dev/null &&
+ notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode > 
OUTPUT.$test_count &&
+ test_cmp EXPECTED.$test_count OUTPUT.$test_count'
+
 test_done
-- 
1.7.10.4

[Patch v2 05/17] test: add generator for random "stub" messages

[da...@tethera.net]

From: David Bremner 

Initial use case is testing dump and restore, so we only have
message-ids and tags.

The message ID's are nothing like RFC compliant, but it doesn't seem
any harder to roundtrip random UTF-8 strings than RFC-compliant ones.

Tags are UTF-8, even though notmuch is in principle more generous than
that.

updated for id:m2wr04ocro.fsf at guru.guru-group.fi

- talk about Unicode value rather some specific encoding
- call talloc_realloc less times
---
 test/.gitignore  |1 +
 test/Makefile.local  |   10 +++
 test/basic   |1 +
 test/random-corpus.c |  204 ++
 4 files changed, 216 insertions(+)
 create mode 100644 test/random-corpus.c

diff --git a/test/.gitignore b/test/.gitignore
index be7ab5e..1eff7ce 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -4,4 +4,5 @@ smtp-dummy
 symbol-test
 arg-test
 hex-xcode
+random-corpus
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index 8479f91..6a9f15e 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -16,6 +16,14 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o 
util/libutil.a
 $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@ -ltalloc

+random_corpus_deps =  $(dir)/random-corpus.o  $(dir)/database-test.o \
+   notmuch-config.o command-line-arguments.o \
+   lib/libnotmuch.a util/libutil.a \
+   parse-time-string/libparse-time-string.a
+
+$(dir)/random-corpus: $(random_corpus_deps)
+   $(call quiet,CC) $(CFLAGS_FINAL) $^ -o $@ $(CONFIGURE_LDFLAGS)
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@

@@ -29,6 +37,7 @@ $(dir)/parse-time: $(dir)/parse-time.o 
parse-time-string/parse-time-string.o

 TEST_BINARIES=$(dir)/arg-test \
  $(dir)/hex-xcode \
+ $(dir)/random-corpus \
  $(dir)/parse-time \
  $(dir)/smtp-dummy \
  $(dir)/symbol-test
@@ -46,5 +55,6 @@ CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/arg-test $(dir)/arg-test.o \
 $(dir)/hex-xcode $(dir)/hex-xcode.o \
 $(dir)/database-test.o \
+$(dir)/random-corpus $(dir)/random-corpus.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/basic b/test/basic
index 2a571ac..f93469f 100755
--- a/test/basic
+++ b/test/basic
@@ -59,6 +59,7 @@ available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -perm 
+111  \
 ! -name hex-xcode  \
 ! -name notmuch-test   \
 ! -name parse-time \
+! -name random-corpus  \
 ! -name smtp-dummy \
 ! -name symbol-test\
 ! -name test-verbose   \
diff --git a/test/random-corpus.c b/test/random-corpus.c
new file mode 100644
index 000..085bda0
--- /dev/null
+++ b/test/random-corpus.c
@@ -0,0 +1,204 @@
+/*
+ * Generate a random corpus of stub messages.
+ *
+ * Initial use case is testing dump and restore, so we only have
+ * message-ids and tags.
+ *
+ * Generated message-id's and tags are intentionally nasty.
+ *
+ * Copyright (c) 2012 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "notmuch-client.h"
+#include "command-line-arguments.h"
+#include "database-test.h"
+
+/* Current largest Unicode value defined. Note that most of these will
+ * be printed as boxes in most fonts.
+ */
+
+#define GLYPH_MAX 0x10FFFE
+
+static gunichar
+random_unichar ()
+{
+int start = 1, stop = GLYPH_MAX;
+int class = random() % 2;
+
+/*
+ *  Choose about half ascii as test characters, as ascii
+ *  punctation and whitespace is the main cause of problems for
+ *  the (old) restore parser
+*/
+switch (class) {
+case 0:
+   /* ascii */
+   start = 0x01;
+   stop = 0x7f;
+   break;
+case 1:
+   /* the rest of unicode */
+   start = 0x80;
+   stop = GLYPH_MAX;
+}
+
+if (start == stop)
+   return start;
+else
+   return start + (random() % (stop - start + 1));
+}
+
+static char *

[Patch v2 04/17] test: add database routines for testing

[da...@tethera.net]

From: David Bremner 

Initially, provide a way to create "stub" messages in the notmuch
database without corresponding files.  This is essentially cut and
paste from lib/database.cc. This is a seperate file since we don't
want to export these symbols from libnotmuch or bloat the library with
non-exported code.
---
 test/Makefile.local  |1 +
 test/database-test.c |   71 ++
 test/database-test.h |   21 +++
 3 files changed, 93 insertions(+)
 create mode 100644 test/database-test.c
 create mode 100644 test/database-test.h

diff --git a/test/Makefile.local b/test/Makefile.local
index 8da4c56..8479f91 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -45,5 +45,6 @@ CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/symbol-test $(dir)/symbol-test.o \
 $(dir)/arg-test $(dir)/arg-test.o \
 $(dir)/hex-xcode $(dir)/hex-xcode.o \
+$(dir)/database-test.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/database-test.c b/test/database-test.c
new file mode 100644
index 000..739e03b
--- /dev/null
+++ b/test/database-test.c
@@ -0,0 +1,71 @@
+/*
+ * Database routines intended only for testing, not exported from
+ * library.
+ *
+ * Copyright (c) 2012 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner 
+ */
+
+#include "notmuch-private.h"
+#include "database-test.h"
+
+notmuch_status_t
+notmuch_database_add_stub_message (notmuch_database_t *notmuch,
+  const char *message_id,
+  const char **tags)
+{
+const char **tag;
+notmuch_status_t ret;
+notmuch_private_status_t private_status;
+notmuch_message_t *message;
+
+ret = _notmuch_database_ensure_writable (notmuch);
+if (ret)
+   return ret;
+
+message = _notmuch_message_create_for_message_id (notmuch,
+ message_id,
+ _status);
+if (message == NULL) {
+   return COERCE_STATUS (private_status,
+ "Unexpected status value from 
_notmuch_message_create_for_message_id");
+
+}
+
+if (private_status != NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+   return NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+_notmuch_message_add_term (message, "type", "mail");
+
+if (tags) {
+   ret = notmuch_message_freeze (message);
+   if (ret)
+   return ret;
+
+   for (tag = tags; *tag; tag++) {
+   ret = notmuch_message_add_tag (message, *tag);
+   if (ret)
+   return ret;
+   }
+}
+
+ret = notmuch_message_thaw (message);
+if (ret)
+   return ret;
+
+return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/test/database-test.h b/test/database-test.h
new file mode 100644
index 000..84f7988
--- /dev/null
+++ b/test/database-test.h
@@ -0,0 +1,21 @@
+#ifndef _DATABASE_TEST_H
+#define _DATABASE_TEST_H
+/* Add a new stub message to the given notmuch database.
+ *
+ * At least the following return values are possible:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
+ * ID as another message already in the database.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ * mode so no message can be added.
+ */
+
+notmuch_status_t
+notmuch_database_add_stub_message (notmuch_database_t *database,
+  const char *message_id,
+  const char **tag_list);
+
+#endif
-- 
1.7.10.4

[Patch v2 03/17] test/hex-escaping: new test for hex escaping routines

[da...@tethera.net]

From: David Bremner 

These are more like unit tests, to (try to) make sure the library
functionality is working before building more complicated things on
top of it.
---
 test/hex-escaping |   26 ++
 test/notmuch-test |1 +
 2 files changed, 27 insertions(+)
 create mode 100755 test/hex-escaping

diff --git a/test/hex-escaping b/test/hex-escaping
new file mode 100755
index 000..f34cc8c
--- /dev/null
+++ b/test/hex-escaping
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+test_description="hex encoding and decoding"
+. ./test-lib.sh
+
+test_begin_subtest "round trip"
+find $TEST_DIRECTORY/corpus -type f -print | sort | xargs cat > EXPECTED
+$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | 
$TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "punctuation"
+tag1='comic_swear=$&^%$^%\\//-+$^%$'
+tag_enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
+test_expect_equal "$tag_enc1" 
"comic_swear=%24%26%5e%25%24%5e%25%5c%5c%2f%2f-+%24%5e%25%24"
+
+test_begin_subtest "round trip newlines"
+printf 'this\n tag\t has\n spaces\n' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=encode  < EXPECTED.$test_count |\
+   $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest "round trip 8bit chars"
+echo '%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a' > EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=decode  < EXPECTED.$test_count |\
+   $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 9a1b375..d2e90e2 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -60,6 +60,7 @@ TESTS="
   emacs-hello
   emacs-show
   missing-headers
+  hex-escaping
   parse-time-string
   search-date
 "
-- 
1.7.10.4

[Patch v2 02/17] test/hex-xcode: new test binary

[da...@tethera.net]

From: David Bremner 

This program is used both as a test-bed/unit-tester for
../util/hex-escape.c, and also as a utility in future tests of dump
and restore.
---
 test/.gitignore |1 +
 test/Makefile.local |   13 ++-
 test/basic  |1 +
 test/hex-xcode.c|  103 +++
 4 files changed, 116 insertions(+), 2 deletions(-)
 create mode 100644 test/hex-xcode.c

diff --git a/test/.gitignore b/test/.gitignore
index e63c689..be7ab5e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,4 +3,5 @@ corpus.mail
 smtp-dummy
 symbol-test
 arg-test
+hex-xcode
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index 9ae130a..8da4c56 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -13,6 +13,9 @@ smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
 $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@

+$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
+   $(call quiet,CC) -I. $^ -o $@ -ltalloc
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@

@@ -24,8 +27,13 @@ $(dir)/parse-time: $(dir)/parse-time.o 
parse-time-string/parse-time-string.o

 .PHONY: test check

-test-binaries: $(dir)/arg-test $(dir)/smtp-dummy $(dir)/symbol-test \
-   $(dir)/parse-time
+TEST_BINARIES=$(dir)/arg-test \
+ $(dir)/hex-xcode \
+ $(dir)/parse-time \
+ $(dir)/smtp-dummy \
+ $(dir)/symbol-test
+
+test-binaries: $(TEST_BINARIES)

 test:  all test-binaries
@${dir}/notmuch-test $(OPTIONS)
@@ -36,5 +44,6 @@ SRCS := $(SRCS) $(smtp_dummy_srcs)
 CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/symbol-test $(dir)/symbol-test.o \
 $(dir)/arg-test $(dir)/arg-test.o \
+$(dir)/hex-xcode $(dir)/hex-xcode.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/basic b/test/basic
index 1b842d2..2a571ac 100755
--- a/test/basic
+++ b/test/basic
@@ -56,6 +56,7 @@ tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
 available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -perm +111  \
 ! -name aggregate-results.sh   \
 ! -name arg-test   \
+! -name hex-xcode  \
 ! -name notmuch-test   \
 ! -name parse-time \
 ! -name smtp-dummy \
diff --git a/test/hex-xcode.c b/test/hex-xcode.c
new file mode 100644
index 000..eec6541
--- /dev/null
+++ b/test/hex-xcode.c
@@ -0,0 +1,103 @@
+/* No, nothing to to with IDE from Apple Inc.
+   testbed for ../util/hex-escape.c.
+
+   usage:
+   hex-xcode [--direction=(encode|decode)] [--omit-newline] < file
+   hex-xcode [--direction=(encode|decode)] [--omit-newline] arg1 arg2 arg3 ...
+
+ */
+
+#include "notmuch-client.h"
+#include "hex-escape.h"
+#include 
+
+
+enum direction {
+ENCODE,
+DECODE
+};
+
+static int
+xcode (void *ctx, enum direction dir, char *in, char **buf_p, size_t *size_p)
+{
+hex_status_t status;
+
+if (dir == ENCODE)
+   status = hex_encode (ctx, in, buf_p, size_p);
+else
+   status = hex_decode (ctx, in, buf_p, size_p);
+
+if (status == HEX_SUCCESS)
+   fputs (*buf_p, stdout);
+
+return status;
+}
+
+
+int
+main (int argc, char **argv)
+{
+
+
+enum direction dir = DECODE;
+int omit_newline = FALSE;
+
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_KEYWORD, , "direction", 'd',
+ (notmuch_keyword_t []){ { "encode", ENCODE },
+ { "decode", DECODE },
+ { 0, 0 } } },
+   { NOTMUCH_OPT_BOOLEAN, _newline, "omit-newline", 'n', 0 },
+   { 0, 0, 0, 0, 0 }
+};
+
+int opt_index = parse_arguments (argc, argv, options, 1);
+
+if (opt_index < 0)
+   exit (1);
+
+void *ctx = talloc_new (NULL);
+
+char *line = NULL;
+size_t line_size;
+ssize_t line_len;
+
+char *buffer = NULL;
+size_t buf_size = 0;
+
+notmuch_bool_t read_stdin = TRUE;
+
+for (; opt_index < argc; opt_index++) {
+
+   if (xcode (ctx, dir, argv[opt_index],
+  , _size) != HEX_SUCCESS)
+   return 1;
+
+   if (!omit_newline)
+   putchar ('\n');
+
+   read_stdin = FALSE;
+}
+
+if (!read_stdin)
+   return 0;
+
+while ((line_len = getline (, _size, stdin)) != -1) {
+
+   chomp_newline (line);
+
+   if (xcode (ctx, dir, line, , _size) != HEX_SUCCESS)
+   return 1;
+
+   if (!omit_newline)
+   putchar ('\n');
+
+}
+
+if (line)
+   free (line);
+
+talloc_free (ctx);
+
+return 0;
+}
-- 
1.7.10.4

[Patch v2 01/17] hex-escape: (en|de)code strings to/from restricted character set

[da...@tethera.net]

From: David Bremner 

The character set is chosen to be suitable for pathnames, and the same
as that used by contrib/nmbug

[With additions by Jani Nikula]
---
 util/Makefile.local |2 +-
 util/hex-escape.c   |  168 +++
 util/hex-escape.h   |   41 +
 3 files changed, 210 insertions(+), 1 deletion(-)
 create mode 100644 util/hex-escape.c
 create mode 100644 util/hex-escape.h

diff --git a/util/Makefile.local b/util/Makefile.local
index c7cae61..3ca623e 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,7 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)

-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c

 libutil_modules := $(libutil_c_srcs:.c=.o)

diff --git a/util/hex-escape.c b/util/hex-escape.c
new file mode 100644
index 000..d8905d0
--- /dev/null
+++ b/util/hex-escape.c
@@ -0,0 +1,168 @@
+/* hex-escape.c -  Manage encoding and decoding of byte strings into path names
+ *
+ * Copyright (c) 2011 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner 
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include "error_util.h"
+#include "hex-escape.h"
+
+static const size_t default_buf_size = 1024;
+
+static const char *output_charset =
+"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.:,";
+
+static const char escape_char = '%';
+
+static int
+is_output (char c)
+{
+return (strchr (output_charset, c) != NULL);
+}
+
+static int
+maybe_realloc (void *ctx, size_t needed, char **out, size_t *out_size)
+{
+if (*out_size < needed) {
+
+   if (*out == NULL)
+   *out = talloc_size (ctx, needed);
+   else
+   *out = talloc_realloc (ctx, *out, char, needed);
+
+   if (*out == NULL)
+   return 0;
+
+   *out_size = needed;
+}
+return 1;
+}
+
+hex_status_t
+hex_encode (void *ctx, const char *in, char **out, size_t *out_size)
+{
+
+const unsigned char *p;
+char *q;
+
+size_t escape_count = 0;
+size_t len = 0;
+size_t needed;
+
+assert (ctx); assert (in); assert (out); assert (out_size);
+
+for (p = (unsigned char *) in; *p; p++) {
+   escape_count += (!is_output (*p));
+   len++;
+}
+
+needed = len + escape_count * 2 + 1;
+
+if (*out == NULL)
+   *out_size = 0;
+
+if (!maybe_realloc (ctx, needed, out, out_size))
+   return HEX_OUT_OF_MEMORY;
+
+q = *out;
+p = (unsigned char *) in;
+
+while (*p) {
+   if (is_output (*p)) {
+   *q++ = *p++;
+   } else {
+   sprintf (q, "%%%02x", *p++);
+   q += 3;
+   }
+}
+
+*q = '\0';
+return HEX_SUCCESS;
+}
+
+/* Hex decode 'in' to 'out'.
+ *
+ * This must succeed for in == out to support hex_decode_inplace().
+ */
+static hex_status_t
+hex_decode_internal (const char *in, unsigned char *out)
+{
+char buf[3];
+
+while (*in) {
+   if (*in == escape_char) {
+   char *endp;
+
+   /* This also handles unexpected end-of-string. */
+   if (!isxdigit ((unsigned char) in[1]) ||
+   !isxdigit ((unsigned char) in[2]))
+   return HEX_SYNTAX_ERROR;
+
+   buf[0] = in[1];
+   buf[1] = in[2];
+   buf[2] = '\0';
+
+   *out = strtoul (buf, , 16);
+
+   if (endp != buf + 2)
+   return HEX_SYNTAX_ERROR;
+
+   in += 3;
+   out++;
+   } else {
+   *out++ = *in++;
+   }
+}
+
+*out = '\0';
+
+return HEX_SUCCESS;
+}
+
+hex_status_t
+hex_decode_inplace (char *s)
+{
+/* A decoded string is never longer than the encoded one, so it is
+ * safe to decode a string onto itself. */
+return hex_decode_internal (s, (unsigned char *) s);
+}
+
+hex_status_t
+hex_decode (void *ctx, const char *in, char **out, size_t * out_size)
+{
+const char *p;
+size_t escape_count = 0;
+size_t needed = 0;
+
+assert (ctx); assert (in); assert (out); assert (out_size);
+
+size_t len = strlen (in);
+
+for (p = in; *p; p++)
+   escape_count += (*p == escape_char);
+
+needed = len - escape_count * 2 + 1;
+
+if (!maybe_realloc (ctx, needed, out, out_size))
+   return HEX_OUT_OF_MEMORY;
+
+return hex_decode_internal 

V2 of batch-tagging plus new dump/restore

[da...@tethera.net]

this obsoletes: id:1353265498-3839-1-git-send-email-david at tethera.net

Changes based on Mark's review

- add commments for each function  in tag-util.h

- inline tag_op_list_from_string in the one place it was called 

- remove NULL terminator from tag_op_list (for mark); along with
  tag_op_list_t being opaque, I thought is was ok to leave off the
  comment on ops. But I could be convinced.

- make tag_op_list_t typedef opaque

- add BE_GENEROUS flag to parser. Currently this enables empty tags.

- handle blank lines and comments in restore and batch tagging

- fixed the commit message about "notmuch-new"

Changes based on Ethan's review:

- remove the dreaded . from first line of commit messages

- convert if on l49 of database-test.c into explicit return on error

- add comments and parens for num_tags and this_mid_len in random_corpus.c

- add check for message-id's less than length 1. I wasn't sure why we
  would disallow all message ids of length 1.

- remove declaration of parse_tag_stream

- explain "query_string += 3"

- fix typo in notmuch-dump.1

[PATCH v2 4/7] emacs: make emacs tagging use the stdin query functionality

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
> From: Mark Walters 
> 
> In preparation for the use of large queries in some cases make tagging
> from emacs use the new query on stdin functionality. Currently uses
> this for all tagging (as I could not see a reason not to).
> ---
>  emacs/notmuch-tag.el |   14 +-
>  1 files changed, 9 insertions(+), 5 deletions(-)
> 
> diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
> index 4fce3a9..4634b0d 100644
> --- a/emacs/notmuch-tag.el
> +++ b/emacs/notmuch-tag.el
> @@ -59,9 +59,10 @@ the messages that were tagged"
>(setq search-terms (list "*")))
>(split-string
> (with-output-to-string
> - (with-current-buffer standard-output
> -   (apply 'call-process notmuch-command nil t
> -   nil "search" "--output=tags" "--exclude=false" search-terms)))
> + (with-temp-buffer
> +   (insert (mapconcat 'identity search-terms " "))

#'identity ?

> +   (apply 'call-process-region (point-min) (point-max) notmuch-command 
> nil

#'call-process-region ?

> +   standard-output nil "search" "--output=tags" "--exclude=false" 
> (list "-"

If you use funcall instead of apply here, you won't need to put "-" in
a list.

Also, the lines seem a little long (but maybe that's just diff and
quoting?)

> "\n+" t))
>  
>  (defun notmuch-select-tag-with-completion (prompt  search-terms)
> @@ -134,8 +135,11 @@ notmuch-after-tag-hook will be run."
>   tag-changes)
>(unless (null tag-changes)
>  (run-hooks 'notmuch-before-tag-hook)
> -(apply 'notmuch-call-notmuch-process "tag"
> -(append tag-changes (list "--" query)))
> +(with-temp-buffer
> +  (insert query)
> +  (apply 'notmuch-call-notmuch-process-region

#'notmuch-call-notmuch-process-region ?

> +  (point-min) (point-max)
> +  "tag" (append tag-changes (list "--" "-"
>  (run-hooks 'notmuch-after-tag-hook))
>;; in all cases we return tag-changes as a list
>tag-changes)

[PATCH v2 1/7] cli: allow query to come from stdin

[Mark Walters]

I should have said the code for reading from stdin was 
stolen from Peter's patch for notmuch-insert
id:1343223767-9812-4-git-send-email-novalazy at gmail.com
(errors are of course mine)

My apologies for not mentioning this in the commit message.

Mark



On Sat, 24 Nov 2012, markwalters1009  wrote:
> From: Mark Walters 
>
> After this series there will be times when a caller will want to pass
> a very large query string to notmuch (eg a list of 10,000 message-ids)
> and this can exceed the size of ARG_MAX. Hence allow notmuch to take
> the query from stdin (if the query is -).
> ---
>  query-string.c |   41 +
>  1 files changed, 41 insertions(+), 0 deletions(-)
>
> diff --git a/query-string.c b/query-string.c
> index 6536512..b1fbdeb 100644
> --- a/query-string.c
> +++ b/query-string.c
> @@ -20,6 +20,44 @@
>  
>  #include "notmuch-client.h"
>  
> +/* Read a single query string from STDIN, using
> + * 'ctx' as the talloc owner for all allocations.
> + *
> + * This function returns NULL in case of insufficient memory or read
> + * errors.
> + */
> +static char *
> +query_string_from_stdin (void *ctx)
> +{
> +char *query_string;
> +char buf[4096];
> +ssize_t remain;
> +
> +query_string = talloc_strdup (ctx, "");
> +if (query_string == NULL)
> + return NULL;
> +
> +for (;;) {
> + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
> + if (remain == 0)
> + break;
> + if (remain < 0) {
> + if (errno == EINTR)
> + continue;
> + fprintf (stderr, "Error: reading from standard input: %s\n",
> +  strerror (errno));
> + return NULL;
> + }
> +
> + buf[remain] = '\0';
> + query_string = talloc_strdup_append (query_string, buf);
> + if (query_string == NULL)
> + return NULL;
> +}
> +
> +return query_string;
> +}
> +
>  /* Construct a single query string from the passed arguments, using
>   * 'ctx' as the talloc owner for all allocations.
>   *
> @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
>  char *query_string;
>  int i;
>  
> +if ((argc == 1) && (strcmp ("-", argv[0]) == 0))
> + return query_string_from_stdin (ctx);
> +
>  query_string = talloc_strdup (ctx, "");
>  if (query_string == NULL)
>   return NULL;
> -- 
> 1.7.9.1

[PATCH v2 7/7] emacs: make emacs use message-ids for tagging

[markwalters1009]

From: Mark Walters 

This makes emacs use the new --queries=true in search mode and use
this for tagging.  This fixes the race condition in tagging from
search mode so mark the tests fixed.
---
 emacs/notmuch.el |   28 +---
 test/emacs   |2 --
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 64b9474..6e8ef83 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -473,7 +473,8 @@ BEG."
   (let (output)
 (notmuch-search-foreach-result beg end
   (lambda (pos)
-   (push (plist-get (notmuch-search-get-result pos) property) output)))
+   (let ((value (plist-get (notmuch-search-get-result pos) property)))
+ (when value (push value output)
 output))

 (defun notmuch-search-find-thread-id ( bare)
@@ -483,6 +484,7 @@ If BARE is set then do not prefix with \"thread:\""
   (let ((thread (plist-get (notmuch-search-get-result) :thread)))
 (when thread (concat (unless bare "thread:") thread

+
 (defun notmuch-search-find-thread-id-region (beg end)
   "Return a list of threads for the current region"
   (mapcar (lambda (thread) (concat "thread:" thread))
@@ -492,6 +494,23 @@ If BARE is set then do not prefix with \"thread:\""
   "Return a search string for threads for the current region"
   (mapconcat 'identity (notmuch-search-find-thread-id-region beg end) " or "))

+;; The following two functions are similar to the previous two but
+;; they only match messages that were in the the thread when the
+;; initial search was run. This means that they can be used where it
+;; is important to avoid races: e.g. when tagging.
+(defun notmuch-search-find-queries-region (beg end  only-matching)
+  (interactive)
+  "Return a list of queries for the current region"
+  (append (notmuch-search-properties-in-region :matching_msg_query beg end)
+ (unless only-matching
+   (notmuch-search-properties-in-region :nonmatching_msg_query beg 
end
+
+(defun notmuch-search-find-queries-region-search (beg end  
only-matching)
+  "Return a search string for messages in threads in the current region"
+  (mapconcat 'identity
+(notmuch-search-find-queries-region beg end only-matching)
+" or "))
+
 (defun notmuch-search-find-authors ()
   "Return the authors for the current thread"
   (plist-get (notmuch-search-get-result) :authors))
@@ -575,7 +594,7 @@ and will also appear in a buffer named \"*Notmuch 
errors*\"."

 (defun notmuch-search-tag-region (beg end  tag-changes)
   "Change tags for threads in the given region."
-  (let ((search-string (notmuch-search-find-thread-id-region-search beg end)))
+  (let ((search-string (notmuch-search-find-queries-region-search beg end)))
 (setq tag-changes (funcall 'notmuch-tag search-string tag-changes))
 (notmuch-search-foreach-result beg end
   (lambda (pos)
@@ -851,7 +870,9 @@ non-authors is found, assume that all of the authors match."

 See `notmuch-tag' for information on the format of TAG-CHANGES."
   (interactive)
-  (apply 'notmuch-tag notmuch-search-query-string tag-changes))
+  (apply 'notmuch-tag (notmuch-search-find-queries-region-search
+  (point-min) (point-max) t)
+tag-changes))

 (defun notmuch-search-buffer-title (query)
   "Returns the title for a buffer with notmuch search results."
@@ -948,6 +969,7 @@ Other optional parameters are used as follows:
 "notmuch-search" buffer
 notmuch-command "search"
 "--format=json"
+"--output=with-queries"
 (if oldest-first
 "--sort=oldest-first"
   "--sort=newest-first")
diff --git a/test/emacs b/test/emacs
index 3788439..132768f 100755
--- a/test/emacs
+++ b/test/emacs
@@ -123,7 +123,6 @@ output=$(notmuch search $os_x_darwin_thread | 
notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
unread)"

 test_begin_subtest "Tag all matching messages from search view"
-test_subtest_known_broken
 notmuch tag +test-tag-race from:cworth
 test_emacs "(notmuch-search \"tag:test-tag-race\")
(notmuch-test-wait)"
@@ -135,7 +134,6 @@ notmuch tag -test-tag-race '*'
 notmuch tag -test-tag-race-2 '*'

 test_begin_subtest "Change tags from search view: another message arriving 
after thread lookup"
-test_subtest_known_broken
 typsos_id="878we4qdqf.fsf at yoom.home.cworth.org"
 typsos_thread=$(notmuch search --output=threads id:$typsos_id)
 test_emacs "(notmuch-search \"$typsos_thread\")
-- 
1.7.9.1

[PATCH v2 6/7] cli: allow search mode to include msg-ids with JSON output

[markwalters1009]

From: Mark Walters 

This adds a --queries=true option which modifies the summary output of
notmuch search by including two extra query strings with each result:
one query string specifies all matching messages and one query string
all non-matching messages. Currently these are just lists of message
ids joined with " or " but that could change in future.

Currently this is not implemented for text format.
---
 notmuch-search.c |   95 ++---
 1 files changed, 89 insertions(+), 6 deletions(-)

diff --git a/notmuch-search.c b/notmuch-search.c
index 830c4e4..c8fc9a6 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -26,7 +26,8 @@ typedef enum {
 OUTPUT_THREADS,
 OUTPUT_MESSAGES,
 OUTPUT_FILES,
-OUTPUT_TAGS
+OUTPUT_TAGS,
+OUTPUT_SUMMARY_WITH_QUERIES
 } output_t;

 static char *
@@ -46,6 +47,57 @@ sanitize_string (const void *ctx, const char *str)
 return out;
 }

+/* This function takes a message id and returns an escaped string
+ * which can be used as a Xapian query. This involves prefixing with
+ * `id:', putting the id inside double quotes, and doubling any
+ * occurence of a double quote in the message id itself.*/
+static char *
+xapian_escape_id (const void *ctx,
+  const char *msg_id)
+{
+const char *c;
+char *escaped_msg_id;
+escaped_msg_id = talloc_strdup (ctx, "id:\"");
+for (c=msg_id; *c; c++)
+   if (*c == '"')
+   escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "\"\"");
+   else
+   escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "%c", *c);
+escaped_msg_id = talloc_asprintf_append (escaped_msg_id, "\"");
+return escaped_msg_id;
+}
+
+static char *
+output_msg_query (const void *ctx,
+   sprinter_t *format,
+   notmuch_bool_t matching,
+   notmuch_bool_t first,
+   notmuch_messages_t *messages)
+{
+notmuch_message_t *message;
+char *query, *escaped_msg_id;
+query = talloc_strdup (ctx, "");
+for (;
+notmuch_messages_valid (messages);
+notmuch_messages_move_to_next (messages))
+{
+   message = notmuch_messages_get (messages);
+   if (notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_MATCH) == 
matching) {
+   escaped_msg_id = xapian_escape_id (ctx, 
notmuch_message_get_message_id (message));
+   if (first) {
+   query = talloc_asprintf_append (query, "%s", escaped_msg_id);
+   first = FALSE;
+   }
+   else
+   query = talloc_asprintf_append (query, " or %s", 
escaped_msg_id);
+   talloc_free (escaped_msg_id);
+   }
+   /* output_msg_query already starts with an ` or' */
+   query = talloc_asprintf_append (query, "%s", output_msg_query (ctx, 
format, matching, first, notmuch_message_get_replies (message)));
+}
+return query;
+}
+
 static int
 do_search_threads (sprinter_t *format,
   notmuch_query_t *query,
@@ -88,7 +140,7 @@ do_search_threads (sprinter_t *format,
format->string (format,
notmuch_thread_get_thread_id (thread));
format->separator (format);
-   } else { /* output == OUTPUT_SUMMARY */
+   } else { /* output == OUTPUT_SUMMARY or OUTPUT_SUMMARY_WITH_QUERIES */
void *ctx_quote = talloc_new (thread);
const char *authors = notmuch_thread_get_authors (thread);
const char *subject = notmuch_thread_get_subject (thread);
@@ -108,7 +160,7 @@ do_search_threads (sprinter_t *format,
relative_date = notmuch_time_relative_date (ctx_quote, date);

if (format->is_text_printer) {
-/* Special case for the text formatter */
+   /* Special case for the text formatter */
printf ("thread:%s %12s [%d/%d] %s; %s (",
thread_id,
relative_date,
@@ -133,8 +185,6 @@ do_search_threads (sprinter_t *format,
format->string (format, subject);
}

-   talloc_free (ctx_quote);
-
format->map_key (format, "tags");
format->begin_list (format);

@@ -145,7 +195,7 @@ do_search_threads (sprinter_t *format,
const char *tag = notmuch_tags_get (tags);

if (format->is_text_printer) {
-  /* Special case for the text formatter */
+   /* Special case for the text formatter */
if (first_tag)
first_tag = FALSE;
else
@@ -160,8 +210,25 @@ do_search_threads (sprinter_t *format,
printf (")");

format->end (format);
+
+   if (output == OUTPUT_SUMMARY_WITH_QUERIES) {
+   char *query;
+   query = output_msg_query (ctx_quote, format, TRUE, TRUE, 
notmuch_thread_get_toplevel_messages (thread));
+   if 

[PATCH v2 5/7] test: test for race when tagging from emacs search

[markwalters1009]

From: Mark Walters 

When tagging from search view in emacs there is a race condition: it
tags all messages in the thread even ones which arrived after the
search was made. This can cause dataloss (if, for example, a thread is
archived it could archive messages the user has never seen).

Mark this test known broken.
---
 test/emacs |   23 +++
 1 files changed, 23 insertions(+), 0 deletions(-)

diff --git a/test/emacs b/test/emacs
index 77265b0..3788439 100755
--- a/test/emacs
+++ b/test/emacs
@@ -122,6 +122,29 @@ test_emacs "(notmuch-search \"$os_x_darwin_thread\")
 output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
unread)"

+test_begin_subtest "Tag all matching messages from search view"
+test_subtest_known_broken
+notmuch tag +test-tag-race from:cworth
+test_emacs "(notmuch-search \"tag:test-tag-race\")
+   (notmuch-test-wait)"
+notmuch tag +test-tag-race "id:1258471718-6781-2-git-send-email-dottedmag at 
dottedmag.net"
+test_emacs "(execute-kbd-macro \"*+test-tag-race-2\")"
+output=$(notmuch count tag:test-tag-race-2)
+test_expect_equal "$output" "12"
+notmuch tag -test-tag-race '*'
+notmuch tag -test-tag-race-2 '*'
+
+test_begin_subtest "Change tags from search view: another message arriving 
after thread lookup"
+test_subtest_known_broken
+typsos_id="878we4qdqf.fsf at yoom.home.cworth.org"
+typsos_thread=$(notmuch search --output=threads id:$typsos_id)
+test_emacs "(notmuch-search \"$typsos_thread\")
+   (notmuch-test-wait)"
+add_message "[subject]=\"new-thread-message\"" "[date]=\"Sat, 01 Jan 2000 
12:00:00 -\"" "[body]=\"new-thread-message\"" 
"[in-reply-to]=\"<$typsos_id>\""
+test_emacs "(execute-kbd-macro \"+tag-from-search-view -unread\")"
+output=$(notmuch search tag:tag-from-search-view | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2009-11-18 [2/3] Ingmar Vanhassel, 
Carl Worth| Notmuch Test Suite; [notmuch] [PATCH] Typsos (inbox 
tag-from-search-view unread)"
+
 test_begin_subtest "Add tag from notmuch-show view"
 test_emacs "(notmuch-show \"$os_x_darwin_thread\")
(execute-kbd-macro \"+tag-from-show-view\")"
-- 
1.7.9.1

[PATCH v2 4/7] emacs: make emacs tagging use the stdin query functionality

[markwalters1009]

From: Mark Walters 

In preparation for the use of large queries in some cases make tagging
from emacs use the new query on stdin functionality. Currently uses
this for all tagging (as I could not see a reason not to).
---
 emacs/notmuch-tag.el |   14 +-
 1 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
index 4fce3a9..4634b0d 100644
--- a/emacs/notmuch-tag.el
+++ b/emacs/notmuch-tag.el
@@ -59,9 +59,10 @@ the messages that were tagged"
   (setq search-terms (list "*")))
   (split-string
(with-output-to-string
- (with-current-buffer standard-output
-   (apply 'call-process notmuch-command nil t
- nil "search" "--output=tags" "--exclude=false" search-terms)))
+ (with-temp-buffer
+   (insert (mapconcat 'identity search-terms " "))
+   (apply 'call-process-region (point-min) (point-max) notmuch-command nil
+ standard-output nil "search" "--output=tags" "--exclude=false" 
(list "-"
"\n+" t))

 (defun notmuch-select-tag-with-completion (prompt  search-terms)
@@ -134,8 +135,11 @@ notmuch-after-tag-hook will be run."
tag-changes)
   (unless (null tag-changes)
 (run-hooks 'notmuch-before-tag-hook)
-(apply 'notmuch-call-notmuch-process "tag"
-  (append tag-changes (list "--" query)))
+(with-temp-buffer
+  (insert query)
+  (apply 'notmuch-call-notmuch-process-region
+(point-min) (point-max)
+"tag" (append tag-changes (list "--" "-"
 (run-hooks 'notmuch-after-tag-hook))
   ;; in all cases we return tag-changes as a list
   tag-changes)
-- 
1.7.9.1

[PATCH v2 2/7] test: for the new query from stdin functionality

[markwalters1009]

From: Mark Walters 

---
 test/tagging |9 +
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/test/tagging b/test/tagging
index 980ff92..eb7d61c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -19,6 +19,15 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag3 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)"

+test_begin_subtest "Adding tags. Query from stdin"
+echo -n "subject:One" | notmuch tag +intag1 +intag2 -- -
+echo DONE
+output=$(notmuch search \* | notmuch_search_sanitize)
+test_expect_equal "$output" "\
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox intag1 intag2 
tag3 unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)"
+notmuch tag -intag1 -intag2 \*
+
 test_expect_code 1 "No tag operations" 'notmuch tag One'
 test_expect_code 1 "No query" 'notmuch tag +tag2'

-- 
1.7.9.1

[PATCH v2 1/7] cli: allow query to come from stdin

[markwalters1009]

From: Mark Walters 

After this series there will be times when a caller will want to pass
a very large query string to notmuch (eg a list of 10,000 message-ids)
and this can exceed the size of ARG_MAX. Hence allow notmuch to take
the query from stdin (if the query is -).
---
 query-string.c |   41 +
 1 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/query-string.c b/query-string.c
index 6536512..b1fbdeb 100644
--- a/query-string.c
+++ b/query-string.c
@@ -20,6 +20,44 @@

 #include "notmuch-client.h"

+/* Read a single query string from STDIN, using
+ * 'ctx' as the talloc owner for all allocations.
+ *
+ * This function returns NULL in case of insufficient memory or read
+ * errors.
+ */
+static char *
+query_string_from_stdin (void *ctx)
+{
+char *query_string;
+char buf[4096];
+ssize_t remain;
+
+query_string = talloc_strdup (ctx, "");
+if (query_string == NULL)
+   return NULL;
+
+for (;;) {
+   remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
+   if (remain == 0)
+   break;
+   if (remain < 0) {
+   if (errno == EINTR)
+   continue;
+   fprintf (stderr, "Error: reading from standard input: %s\n",
+strerror (errno));
+   return NULL;
+   }
+
+   buf[remain] = '\0';
+   query_string = talloc_strdup_append (query_string, buf);
+   if (query_string == NULL)
+   return NULL;
+}
+
+return query_string;
+}
+
 /* Construct a single query string from the passed arguments, using
  * 'ctx' as the talloc owner for all allocations.
  *
@@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
 char *query_string;
 int i;

+if ((argc == 1) && (strcmp ("-", argv[0]) == 0))
+   return query_string_from_stdin (ctx);
+
 query_string = talloc_strdup (ctx, "");
 if (query_string == NULL)
return NULL;
-- 
1.7.9.1

[PATCH v2 0/7] Fix emacs tagging race

[markwalters1009]

This is version 2 of this series: version 1 is at
id:1352487491-31512-1-git-send-email-markwalters1009 at gmail.com but
this is a much more complete version. Version 1 roughly corresponds to
patches 5-7.

The first two patches allows queries to come from stdin (if the query
string is "-"). This is necessary to avoid ARGMAX limits in some
cases. They are independent of the rest of the series. The main thing
needed for these two (apart from review!) is a manpage but I wasn't
sure whether that should go in notmuch-search-terms or somewhere else.

Patches 3 and 4 make the emacs interface use this new functionality to
pass the tagging query. These two patches depend on the previous two
but are independent of the later patches. Note that it is possible (if
unlikely) to trigger the ARGMAX problem in current notmuch: highlight
most or all of a large search buffer and then try to tag the region.

Patches 5-7 actually fix the race. They do this by appending two query
strings to each search: one query string for the matching messages and
one for the non-matching messages. The front-end can then combine
these query strings to make sure it only tags messages that were
present/matched when the search buffer was created.

The main changes from v1 are to append query-string rather than all
the message-ids (so if we had a better way of constructing the queries
we could switch to that later) and to use Austin's suggestion of
--queries=true to add the queries. I think we do want the choice as
appending the string could easily double the size of the output.

This version (since rebasing and tidying) is not heavily tested (all
tests pass) but I have been running a similar version for some time
without problems.

Best wishes

Mark




Mark Walters (7):
  cli: allow query to come from stdin
  test: for the new query from stdin functionality
  emacs: notmuch.el split call-process into call-process-region
  emacs: make emacs tagging use the stdin query functionality
  test: test for race when tagging from emacs search
  cli: allow search mode to include msg-ids with JSON output
  emacs: make emacs use message-ids for tagging

 emacs/notmuch-tag.el |   14 +---
 emacs/notmuch.el |   47 
 notmuch-search.c |   95 ++---
 query-string.c   |   41 +
 test/emacs   |   21 +++
 test/tagging |9 +
 6 files changed, 208 insertions(+), 19 deletions(-)

-- 
1.7.9.1

[PATCH v2 1/7] cli: allow query to come from stdin

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
> From: Mark Walters 
> 
> After this series there will be times when a caller will want to pass
> a very large query string to notmuch (eg a list of 10,000 message-ids)
> and this can exceed the size of ARG_MAX. Hence allow notmuch to take
> the query from stdin (if the query is -).
> ---
>  query-string.c |   41 +
>  1 files changed, 41 insertions(+), 0 deletions(-)
> 
> diff --git a/query-string.c b/query-string.c
> index 6536512..b1fbdeb 100644
> --- a/query-string.c
> +++ b/query-string.c
> @@ -20,6 +20,44 @@
>  
>  #include "notmuch-client.h"
>  
> +/* Read a single query string from STDIN, using
> + * 'ctx' as the talloc owner for all allocations.
> + *
> + * This function returns NULL in case of insufficient memory or read
> + * errors.
> + */
> +static char *
> +query_string_from_stdin (void *ctx)
> +{
> +char *query_string;
> +char buf[4096];
> +ssize_t remain;
> +
> +query_string = talloc_strdup (ctx, "");
> +if (query_string == NULL)
> + return NULL;
> +
> +for (;;) {
> + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
> + if (remain == 0)
> + break;
> + if (remain < 0) {
> + if (errno == EINTR)
> + continue;
> + fprintf (stderr, "Error: reading from standard input: %s\n",
> +  strerror (errno));

talloc_free (query_string) ?

> + return NULL;
> + }
> +
> + buf[remain] = '\0';
> + query_string = talloc_strdup_append (query_string, buf);

Eliminate the NUL in buf and instead
 talloc_strndup_append (query_string, buf, remain) ?

Should there be some (large) bound on the size of the query string to
prevent runaway?

> + if (query_string == NULL)

Technically it would be good to talloc_free the old pointer here, too.

> + return NULL;
> +}
> +
> +return query_string;
> +}
> +

This whole approach is O(n^2), which might actually matter for large
query strings.  How about (tested, but only a little):

#define MAX_QUERY_STRING_LENGTH (16 * 1024 * 1024)

/* Read a single query string from STDIN, using 'ctx' as the talloc
 * owner for all allocations.
 *
 * This function returns NULL in case of insufficient memory or read
 * errors.
 */
static char *
query_string_from_stdin (void *ctx)
{
char *query_string = NULL, *new_qs;
size_t pos = 0, end = 0;
ssize_t got;

for (;;) {
if (end - pos < 512) {
end = MAX(end * 2, 1024);
if (end >= MAX_QUERY_STRING_LENGTH) {
fprintf (stderr, "Error: query too long\n");
goto FAIL;
}
new_qs = talloc_realloc (ctx, query_string, char, end);
if (new_qs == NULL)
goto FAIL;
query_string = new_qs;
}

got = read (STDIN_FILENO, query_string + pos, end - pos - 1);
if (got == 0)
break;
if (got < 0) {
   if (errno == EINTR)
   continue;
   fprintf (stderr, "Error: reading from standard input: %s\n",
strerror (errno));
   goto FAIL;
}
pos += got;
}

query_string[pos] = '\0';
return query_string;

 FAIL:
talloc_free (query_string);
return NULL;
}

>  /* Construct a single query string from the passed arguments, using
>   * 'ctx' as the talloc owner for all allocations.
>   *
> @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
>  char *query_string;
>  int i;
>  
> +if ((argc == 1) && (strcmp ("-", argv[0]) == 0))
> + return query_string_from_stdin (ctx);
> +
>  query_string = talloc_strdup (ctx, "");
>  if (query_string == NULL)
>   return NULL;

[PATCH v2 0/7] Fix emacs tagging race

[markwalters1009]

This is version 2 of this series: version 1 is at
id:1352487491-31512-1-git-send-email-markwalters1...@gmail.com but
this is a much more complete version. Version 1 roughly corresponds to
patches 5-7.

The first two patches allows queries to come from stdin (if the query
string is -). This is necessary to avoid ARGMAX limits in some
cases. They are independent of the rest of the series. The main thing
needed for these two (apart from review!) is a manpage but I wasn't
sure whether that should go in notmuch-search-terms or somewhere else.

Patches 3 and 4 make the emacs interface use this new functionality to
pass the tagging query. These two patches depend on the previous two
but are independent of the later patches. Note that it is possible (if
unlikely) to trigger the ARGMAX problem in current notmuch: highlight
most or all of a large search buffer and then try to tag the region.

Patches 5-7 actually fix the race. They do this by appending two query
strings to each search: one query string for the matching messages and
one for the non-matching messages. The front-end can then combine
these query strings to make sure it only tags messages that were
present/matched when the search buffer was created.

The main changes from v1 are to append query-string rather than all
the message-ids (so if we had a better way of constructing the queries
we could switch to that later) and to use Austin's suggestion of
--queries=true to add the queries. I think we do want the choice as
appending the string could easily double the size of the output.

This version (since rebasing and tidying) is not heavily tested (all
tests pass) but I have been running a similar version for some time
without problems.

Best wishes

Mark




Mark Walters (7):
  cli: allow query to come from stdin
  test: for the new query from stdin functionality
  emacs: notmuch.el split call-process into call-process-region
  emacs: make emacs tagging use the stdin query functionality
  test: test for race when tagging from emacs search
  cli: allow search mode to include msg-ids with JSON output
  emacs: make emacs use message-ids for tagging

 emacs/notmuch-tag.el |   14 +---
 emacs/notmuch.el |   47 
 notmuch-search.c |   95 ++---
 query-string.c   |   41 +
 test/emacs   |   21 +++
 test/tagging |9 +
 6 files changed, 208 insertions(+), 19 deletions(-)

-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 2/7] test: for the new query from stdin functionality

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

---
 test/tagging |9 +
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/test/tagging b/test/tagging
index 980ff92..eb7d61c 100755
--- a/test/tagging
+++ b/test/tagging
@@ -19,6 +19,15 @@ test_expect_equal $output \
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag3 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)
 
+test_begin_subtest Adding tags. Query from stdin
+echo -n subject:One | notmuch tag +intag1 +intag2 -- -
+echo DONE
+output=$(notmuch search \* | notmuch_search_sanitize)
+test_expect_equal $output \
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox intag1 intag2 
tag3 unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)
+notmuch tag -intag1 -intag2 \*
+
 test_expect_code 1 No tag operations 'notmuch tag One'
 test_expect_code 1 No query 'notmuch tag +tag2'
 
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 1/7] cli: allow query to come from stdin

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

After this series there will be times when a caller will want to pass
a very large query string to notmuch (eg a list of 10,000 message-ids)
and this can exceed the size of ARG_MAX. Hence allow notmuch to take
the query from stdin (if the query is -).
---
 query-string.c |   41 +
 1 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/query-string.c b/query-string.c
index 6536512..b1fbdeb 100644
--- a/query-string.c
+++ b/query-string.c
@@ -20,6 +20,44 @@
 
 #include notmuch-client.h
 
+/* Read a single query string from STDIN, using
+ * 'ctx' as the talloc owner for all allocations.
+ *
+ * This function returns NULL in case of insufficient memory or read
+ * errors.
+ */
+static char *
+query_string_from_stdin (void *ctx)
+{
+char *query_string;
+char buf[4096];
+ssize_t remain;
+
+query_string = talloc_strdup (ctx, );
+if (query_string == NULL)
+   return NULL;
+
+for (;;) {
+   remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
+   if (remain == 0)
+   break;
+   if (remain  0) {
+   if (errno == EINTR)
+   continue;
+   fprintf (stderr, Error: reading from standard input: %s\n,
+strerror (errno));
+   return NULL;
+   }
+
+   buf[remain] = '\0';
+   query_string = talloc_strdup_append (query_string, buf);
+   if (query_string == NULL)
+   return NULL;
+}
+
+return query_string;
+}
+
 /* Construct a single query string from the passed arguments, using
  * 'ctx' as the talloc owner for all allocations.
  *
@@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
 char *query_string;
 int i;
 
+if ((argc == 1)  (strcmp (-, argv[0]) == 0))
+   return query_string_from_stdin (ctx);
+
 query_string = talloc_strdup (ctx, );
 if (query_string == NULL)
return NULL;
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 3/7] emacs: notmuch.el split call-process into call-process-region

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

We add a new function notmuch-call-process-region so that functions
can call notmuch with some region sent to stdin. This is preparation
for using the new query from stdin functionality.
---
 emacs/notmuch.el |   19 ++-
 1 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index f9454d8..64b9474 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -533,15 +533,17 @@ If BARE is set then do not prefix with \thread:\
   (let ((message-id (notmuch-search-find-thread-id)))
 (notmuch-mua-new-reply message-id prompt-for-sender nil)))
 
-(defun notmuch-call-notmuch-process (rest args)
-  Synchronously invoke \notmuch\ with the given list of arguments.
+(defun notmuch-call-notmuch-process-region (beg end rest args)
+  Synchronously invoke \notmuch\ with the given list of arguments and pipe 
region.
 
-Output from the process will be presented to the user as an error
-and will also appear in a buffer named \*Notmuch errors*\.
+The region from beg to end in the current buffer will be piped to
+stdin for the notmuch process.  Output from the process will be
+presented to the user as an error and will also appear in a
+buffer named \*Notmuch errors*\.
   (let ((error-buffer (get-buffer-create *Notmuch errors*)))
 (with-current-buffer error-buffer
(erase-buffer))
-(if (eq (apply 'call-process notmuch-command nil error-buffer nil args) 0)
+(if (eq (apply 'call-process-region beg end notmuch-command nil 
error-buffer nil args) 0)
(point)
   (progn
(with-current-buffer error-buffer
@@ -550,6 +552,13 @@ and will also appear in a buffer named \*Notmuch 
errors*\.
(error (buffer-substring beg end))
))
 
+(defun notmuch-call-notmuch-process (rest args)
+  Synchronously invoke \notmuch\ with the given list of arguments.
+
+Output from the process will be presented to the user as an error
+and will also appear in a buffer named \*Notmuch errors*\.
+  (apply 'notmuch-call-notmuch-process-region (point) (point) args))
+
 (defun notmuch-search-set-tags (tags optional pos)
   (let ((new-result (plist-put (notmuch-search-get-result pos) :tags tags)))
 (notmuch-search-update-result new-result pos)))
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 4/7] emacs: make emacs tagging use the stdin query functionality

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

In preparation for the use of large queries in some cases make tagging
from emacs use the new query on stdin functionality. Currently uses
this for all tagging (as I could not see a reason not to).
---
 emacs/notmuch-tag.el |   14 +-
 1 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
index 4fce3a9..4634b0d 100644
--- a/emacs/notmuch-tag.el
+++ b/emacs/notmuch-tag.el
@@ -59,9 +59,10 @@ the messages that were tagged
   (setq search-terms (list *)))
   (split-string
(with-output-to-string
- (with-current-buffer standard-output
-   (apply 'call-process notmuch-command nil t
- nil search --output=tags --exclude=false search-terms)))
+ (with-temp-buffer
+   (insert (mapconcat 'identity search-terms  ))
+   (apply 'call-process-region (point-min) (point-max) notmuch-command nil
+ standard-output nil search --output=tags --exclude=false 
(list -
\n+ t))
 
 (defun notmuch-select-tag-with-completion (prompt rest search-terms)
@@ -134,8 +135,11 @@ notmuch-after-tag-hook will be run.
tag-changes)
   (unless (null tag-changes)
 (run-hooks 'notmuch-before-tag-hook)
-(apply 'notmuch-call-notmuch-process tag
-  (append tag-changes (list -- query)))
+(with-temp-buffer
+  (insert query)
+  (apply 'notmuch-call-notmuch-process-region
+(point-min) (point-max)
+tag (append tag-changes (list -- -
 (run-hooks 'notmuch-after-tag-hook))
   ;; in all cases we return tag-changes as a list
   tag-changes)
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 5/7] test: test for race when tagging from emacs search

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

When tagging from search view in emacs there is a race condition: it
tags all messages in the thread even ones which arrived after the
search was made. This can cause dataloss (if, for example, a thread is
archived it could archive messages the user has never seen).

Mark this test known broken.
---
 test/emacs |   23 +++
 1 files changed, 23 insertions(+), 0 deletions(-)

diff --git a/test/emacs b/test/emacs
index 77265b0..3788439 100755
--- a/test/emacs
+++ b/test/emacs
@@ -122,6 +122,29 @@ test_emacs (notmuch-search \$os_x_darwin_thread\)
 output=$(notmuch search $os_x_darwin_thread | notmuch_search_sanitize)
 test_expect_equal $output thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
unread)
 
+test_begin_subtest Tag all matching messages from search view
+test_subtest_known_broken
+notmuch tag +test-tag-race from:cworth
+test_emacs (notmuch-search \tag:test-tag-race\)
+   (notmuch-test-wait)
+notmuch tag +test-tag-race 
id:1258471718-6781-2-git-send-email-dotted...@dottedmag.net
+test_emacs (execute-kbd-macro \*+test-tag-race-2\)
+output=$(notmuch count tag:test-tag-race-2)
+test_expect_equal $output 12
+notmuch tag -test-tag-race '*'
+notmuch tag -test-tag-race-2 '*'
+
+test_begin_subtest Change tags from search view: another message arriving 
after thread lookup
+test_subtest_known_broken
+typsos_id=878we4qdqf@yoom.home.cworth.org
+typsos_thread=$(notmuch search --output=threads id:$typsos_id)
+test_emacs (notmuch-search \$typsos_thread\)
+   (notmuch-test-wait)
+add_message [subject]=\new-thread-message\ [date]=\Sat, 01 Jan 2000 
12:00:00 -\ [body]=\new-thread-message\ 
[in-reply-to]=\$typsos_id\
+test_emacs (execute-kbd-macro \+tag-from-search-view -unread\)
+output=$(notmuch search tag:tag-from-search-view | notmuch_search_sanitize)
+test_expect_equal $output thread:XXX   2009-11-18 [2/3] Ingmar Vanhassel, 
Carl Worth| Notmuch Test Suite; [notmuch] [PATCH] Typsos (inbox 
tag-from-search-view unread)
+
 test_begin_subtest Add tag from notmuch-show view
 test_emacs (notmuch-show \$os_x_darwin_thread\)
(execute-kbd-macro \+tag-from-show-view\)
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 6/7] cli: allow search mode to include msg-ids with JSON output

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

This adds a --queries=true option which modifies the summary output of
notmuch search by including two extra query strings with each result:
one query string specifies all matching messages and one query string
all non-matching messages. Currently these are just lists of message
ids joined with  or  but that could change in future.

Currently this is not implemented for text format.
---
 notmuch-search.c |   95 ++---
 1 files changed, 89 insertions(+), 6 deletions(-)

diff --git a/notmuch-search.c b/notmuch-search.c
index 830c4e4..c8fc9a6 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -26,7 +26,8 @@ typedef enum {
 OUTPUT_THREADS,
 OUTPUT_MESSAGES,
 OUTPUT_FILES,
-OUTPUT_TAGS
+OUTPUT_TAGS,
+OUTPUT_SUMMARY_WITH_QUERIES
 } output_t;
 
 static char *
@@ -46,6 +47,57 @@ sanitize_string (const void *ctx, const char *str)
 return out;
 }
 
+/* This function takes a message id and returns an escaped string
+ * which can be used as a Xapian query. This involves prefixing with
+ * `id:', putting the id inside double quotes, and doubling any
+ * occurence of a double quote in the message id itself.*/
+static char *
+xapian_escape_id (const void *ctx,
+  const char *msg_id)
+{
+const char *c;
+char *escaped_msg_id;
+escaped_msg_id = talloc_strdup (ctx, id:\);
+for (c=msg_id; *c; c++)
+   if (*c == '')
+   escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \\);
+   else
+   escaped_msg_id = talloc_asprintf_append (escaped_msg_id, %c, *c);
+escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \);
+return escaped_msg_id;
+}
+
+static char *
+output_msg_query (const void *ctx,
+   sprinter_t *format,
+   notmuch_bool_t matching,
+   notmuch_bool_t first,
+   notmuch_messages_t *messages)
+{
+notmuch_message_t *message;
+char *query, *escaped_msg_id;
+query = talloc_strdup (ctx, );
+for (;
+notmuch_messages_valid (messages);
+notmuch_messages_move_to_next (messages))
+{
+   message = notmuch_messages_get (messages);
+   if (notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_MATCH) == 
matching) {
+   escaped_msg_id = xapian_escape_id (ctx, 
notmuch_message_get_message_id (message));
+   if (first) {
+   query = talloc_asprintf_append (query, %s, escaped_msg_id);
+   first = FALSE;
+   }
+   else
+   query = talloc_asprintf_append (query,  or %s, 
escaped_msg_id);
+   talloc_free (escaped_msg_id);
+   }
+   /* output_msg_query already starts with an ` or' */
+   query = talloc_asprintf_append (query, %s, output_msg_query (ctx, 
format, matching, first, notmuch_message_get_replies (message)));
+}
+return query;
+}
+
 static int
 do_search_threads (sprinter_t *format,
   notmuch_query_t *query,
@@ -88,7 +140,7 @@ do_search_threads (sprinter_t *format,
format-string (format,
notmuch_thread_get_thread_id (thread));
format-separator (format);
-   } else { /* output == OUTPUT_SUMMARY */
+   } else { /* output == OUTPUT_SUMMARY or OUTPUT_SUMMARY_WITH_QUERIES */
void *ctx_quote = talloc_new (thread);
const char *authors = notmuch_thread_get_authors (thread);
const char *subject = notmuch_thread_get_subject (thread);
@@ -108,7 +160,7 @@ do_search_threads (sprinter_t *format,
relative_date = notmuch_time_relative_date (ctx_quote, date);
 
if (format-is_text_printer) {
-/* Special case for the text formatter */
+   /* Special case for the text formatter */
printf (thread:%s %12s [%d/%d] %s; %s (,
thread_id,
relative_date,
@@ -133,8 +185,6 @@ do_search_threads (sprinter_t *format,
format-string (format, subject);
}
 
-   talloc_free (ctx_quote);
-
format-map_key (format, tags);
format-begin_list (format);
 
@@ -145,7 +195,7 @@ do_search_threads (sprinter_t *format,
const char *tag = notmuch_tags_get (tags);
 
if (format-is_text_printer) {
-  /* Special case for the text formatter */
+   /* Special case for the text formatter */
if (first_tag)
first_tag = FALSE;
else
@@ -160,8 +210,25 @@ do_search_threads (sprinter_t *format,
printf ());
 
format-end (format);
+
+   if (output == OUTPUT_SUMMARY_WITH_QUERIES) {
+   char *query;
+   query = output_msg_query (ctx_quote, format, TRUE, TRUE, 
notmuch_thread_get_toplevel_messages (thread));
+   if (strlen (query)) {
+ 

[PATCH v2 7/7] emacs: make emacs use message-ids for tagging

[markwalters1009]

From: Mark Walters markwalters1...@gmail.com

This makes emacs use the new --queries=true in search mode and use
this for tagging.  This fixes the race condition in tagging from
search mode so mark the tests fixed.
---
 emacs/notmuch.el |   28 +---
 test/emacs   |2 --
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 64b9474..6e8ef83 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -473,7 +473,8 @@ BEG.
   (let (output)
 (notmuch-search-foreach-result beg end
   (lambda (pos)
-   (push (plist-get (notmuch-search-get-result pos) property) output)))
+   (let ((value (plist-get (notmuch-search-get-result pos) property)))
+ (when value (push value output)
 output))
 
 (defun notmuch-search-find-thread-id (optional bare)
@@ -483,6 +484,7 @@ If BARE is set then do not prefix with \thread:\
   (let ((thread (plist-get (notmuch-search-get-result) :thread)))
 (when thread (concat (unless bare thread:) thread
 
+
 (defun notmuch-search-find-thread-id-region (beg end)
   Return a list of threads for the current region
   (mapcar (lambda (thread) (concat thread: thread))
@@ -492,6 +494,23 @@ If BARE is set then do not prefix with \thread:\
   Return a search string for threads for the current region
   (mapconcat 'identity (notmuch-search-find-thread-id-region beg end)  or ))
 
+;; The following two functions are similar to the previous two but
+;; they only match messages that were in the the thread when the
+;; initial search was run. This means that they can be used where it
+;; is important to avoid races: e.g. when tagging.
+(defun notmuch-search-find-queries-region (beg end optional only-matching)
+  (interactive)
+  Return a list of queries for the current region
+  (append (notmuch-search-properties-in-region :matching_msg_query beg end)
+ (unless only-matching
+   (notmuch-search-properties-in-region :nonmatching_msg_query beg 
end
+
+(defun notmuch-search-find-queries-region-search (beg end optional 
only-matching)
+  Return a search string for messages in threads in the current region
+  (mapconcat 'identity
+(notmuch-search-find-queries-region beg end only-matching)
+ or ))
+
 (defun notmuch-search-find-authors ()
   Return the authors for the current thread
   (plist-get (notmuch-search-get-result) :authors))
@@ -575,7 +594,7 @@ and will also appear in a buffer named \*Notmuch 
errors*\.
 
 (defun notmuch-search-tag-region (beg end optional tag-changes)
   Change tags for threads in the given region.
-  (let ((search-string (notmuch-search-find-thread-id-region-search beg end)))
+  (let ((search-string (notmuch-search-find-queries-region-search beg end)))
 (setq tag-changes (funcall 'notmuch-tag search-string tag-changes))
 (notmuch-search-foreach-result beg end
   (lambda (pos)
@@ -851,7 +870,9 @@ non-authors is found, assume that all of the authors match.
 
 See `notmuch-tag' for information on the format of TAG-CHANGES.
   (interactive)
-  (apply 'notmuch-tag notmuch-search-query-string tag-changes))
+  (apply 'notmuch-tag (notmuch-search-find-queries-region-search
+  (point-min) (point-max) t)
+tag-changes))
 
 (defun notmuch-search-buffer-title (query)
   Returns the title for a buffer with notmuch search results.
@@ -948,6 +969,7 @@ Other optional parameters are used as follows:
 notmuch-search buffer
 notmuch-command search
 --format=json
+--output=with-queries
 (if oldest-first
 --sort=oldest-first
   --sort=newest-first)
diff --git a/test/emacs b/test/emacs
index 3788439..132768f 100755
--- a/test/emacs
+++ b/test/emacs
@@ -123,7 +123,6 @@ output=$(notmuch search $os_x_darwin_thread | 
notmuch_search_sanitize)
 test_expect_equal $output thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
unread)
 
 test_begin_subtest Tag all matching messages from search view
-test_subtest_known_broken
 notmuch tag +test-tag-race from:cworth
 test_emacs (notmuch-search \tag:test-tag-race\)
(notmuch-test-wait)
@@ -135,7 +134,6 @@ notmuch tag -test-tag-race '*'
 notmuch tag -test-tag-race-2 '*'
 
 test_begin_subtest Change tags from search view: another message arriving 
after thread lookup
-test_subtest_known_broken
 typsos_id=878we4qdqf@yoom.home.cworth.org
 typsos_thread=$(notmuch search --output=threads id:$typsos_id)
 test_emacs (notmuch-search \$typsos_thread\)
-- 
1.7.9.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 1/7] cli: allow query to come from stdin

[Mark Walters]

I should have said the code for reading from stdin was 
stolen from Peter's patch for notmuch-insert
id:1343223767-9812-4-git-send-email-noval...@gmail.com
(errors are of course mine)

My apologies for not mentioning this in the commit message.

Mark



On Sat, 24 Nov 2012, markwalters1009 markwalters1...@gmail.com wrote:
 From: Mark Walters markwalters1...@gmail.com

 After this series there will be times when a caller will want to pass
 a very large query string to notmuch (eg a list of 10,000 message-ids)
 and this can exceed the size of ARG_MAX. Hence allow notmuch to take
 the query from stdin (if the query is -).
 ---
  query-string.c |   41 +
  1 files changed, 41 insertions(+), 0 deletions(-)

 diff --git a/query-string.c b/query-string.c
 index 6536512..b1fbdeb 100644
 --- a/query-string.c
 +++ b/query-string.c
 @@ -20,6 +20,44 @@
  
  #include notmuch-client.h
  
 +/* Read a single query string from STDIN, using
 + * 'ctx' as the talloc owner for all allocations.
 + *
 + * This function returns NULL in case of insufficient memory or read
 + * errors.
 + */
 +static char *
 +query_string_from_stdin (void *ctx)
 +{
 +char *query_string;
 +char buf[4096];
 +ssize_t remain;
 +
 +query_string = talloc_strdup (ctx, );
 +if (query_string == NULL)
 + return NULL;
 +
 +for (;;) {
 + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
 + if (remain == 0)
 + break;
 + if (remain  0) {
 + if (errno == EINTR)
 + continue;
 + fprintf (stderr, Error: reading from standard input: %s\n,
 +  strerror (errno));
 + return NULL;
 + }
 +
 + buf[remain] = '\0';
 + query_string = talloc_strdup_append (query_string, buf);
 + if (query_string == NULL)
 + return NULL;
 +}
 +
 +return query_string;
 +}
 +
  /* Construct a single query string from the passed arguments, using
   * 'ctx' as the talloc owner for all allocations.
   *
 @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
  char *query_string;
  int i;
  
 +if ((argc == 1)  (strcmp (-, argv[0]) == 0))
 + return query_string_from_stdin (ctx);
 +
  query_string = talloc_strdup (ctx, );
  if (query_string == NULL)
   return NULL;
 -- 
 1.7.9.1
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 1/7] cli: allow query to come from stdin

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
 From: Mark Walters markwalters1...@gmail.com
 
 After this series there will be times when a caller will want to pass
 a very large query string to notmuch (eg a list of 10,000 message-ids)
 and this can exceed the size of ARG_MAX. Hence allow notmuch to take
 the query from stdin (if the query is -).
 ---
  query-string.c |   41 +
  1 files changed, 41 insertions(+), 0 deletions(-)
 
 diff --git a/query-string.c b/query-string.c
 index 6536512..b1fbdeb 100644
 --- a/query-string.c
 +++ b/query-string.c
 @@ -20,6 +20,44 @@
  
  #include notmuch-client.h
  
 +/* Read a single query string from STDIN, using
 + * 'ctx' as the talloc owner for all allocations.
 + *
 + * This function returns NULL in case of insufficient memory or read
 + * errors.
 + */
 +static char *
 +query_string_from_stdin (void *ctx)
 +{
 +char *query_string;
 +char buf[4096];
 +ssize_t remain;
 +
 +query_string = talloc_strdup (ctx, );
 +if (query_string == NULL)
 + return NULL;
 +
 +for (;;) {
 + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
 + if (remain == 0)
 + break;
 + if (remain  0) {
 + if (errno == EINTR)
 + continue;
 + fprintf (stderr, Error: reading from standard input: %s\n,
 +  strerror (errno));

talloc_free (query_string) ?

 + return NULL;
 + }
 +
 + buf[remain] = '\0';
 + query_string = talloc_strdup_append (query_string, buf);

Eliminate the NUL in buf and instead
 talloc_strndup_append (query_string, buf, remain) ?

Should there be some (large) bound on the size of the query string to
prevent runaway?

 + if (query_string == NULL)

Technically it would be good to talloc_free the old pointer here, too.

 + return NULL;
 +}
 +
 +return query_string;
 +}
 +

This whole approach is O(n^2), which might actually matter for large
query strings.  How about (tested, but only a little):

#define MAX_QUERY_STRING_LENGTH (16 * 1024 * 1024)

/* Read a single query string from STDIN, using 'ctx' as the talloc
 * owner for all allocations.
 *
 * This function returns NULL in case of insufficient memory or read
 * errors.
 */
static char *
query_string_from_stdin (void *ctx)
{
char *query_string = NULL, *new_qs;
size_t pos = 0, end = 0;
ssize_t got;

for (;;) {
if (end - pos  512) {
end = MAX(end * 2, 1024);
if (end = MAX_QUERY_STRING_LENGTH) {
fprintf (stderr, Error: query too long\n);
goto FAIL;
}
new_qs = talloc_realloc (ctx, query_string, char, end);
if (new_qs == NULL)
goto FAIL;
query_string = new_qs;
}

got = read (STDIN_FILENO, query_string + pos, end - pos - 1);
if (got == 0)
break;
if (got  0) {
   if (errno == EINTR)
   continue;
   fprintf (stderr, Error: reading from standard input: %s\n,
strerror (errno));
   goto FAIL;
}
pos += got;
}

query_string[pos] = '\0';
return query_string;

 FAIL:
talloc_free (query_string);
return NULL;
}

  /* Construct a single query string from the passed arguments, using
   * 'ctx' as the talloc owner for all allocations.
   *
 @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
  char *query_string;
  int i;
  
 +if ((argc == 1)  (strcmp (-, argv[0]) == 0))
 + return query_string_from_stdin (ctx);
 +
  query_string = talloc_strdup (ctx, );
  if (query_string == NULL)
   return NULL;
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

V2 of batch-tagging plus new dump/restore

[david]

this obsoletes: id:1353265498-3839-1-git-send-email-da...@tethera.net

Changes based on Mark's review

- add commments for each function  in tag-util.h

- inline tag_op_list_from_string in the one place it was called 

- remove NULL terminator from tag_op_list (for mark); along with
  tag_op_list_t being opaque, I thought is was ok to leave off the
  comment on ops. But I could be convinced.

- make tag_op_list_t typedef opaque

- add BE_GENEROUS flag to parser. Currently this enables empty tags.

- handle blank lines and comments in restore and batch tagging

- fixed the commit message about notmuch-new

Changes based on Ethan's review:

- remove the dreaded . from first line of commit messages

- convert if on l49 of database-test.c into explicit return on error

- add comments and parens for num_tags and this_mid_len in random_corpus.c

- add check for message-id's less than length 1. I wasn't sure why we
  would disallow all message ids of length 1.

- remove declaration of parse_tag_stream

- explain query_string += 3

- fix typo in notmuch-dump.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 02/17] test/hex-xcode: new test binary

[david]

From: David Bremner brem...@debian.org

This program is used both as a test-bed/unit-tester for
../util/hex-escape.c, and also as a utility in future tests of dump
and restore.
---
 test/.gitignore |1 +
 test/Makefile.local |   13 ++-
 test/basic  |1 +
 test/hex-xcode.c|  103 +++
 4 files changed, 116 insertions(+), 2 deletions(-)
 create mode 100644 test/hex-xcode.c

diff --git a/test/.gitignore b/test/.gitignore
index e63c689..be7ab5e 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,4 +3,5 @@ corpus.mail
 smtp-dummy
 symbol-test
 arg-test
+hex-xcode
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index 9ae130a..8da4c56 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -13,6 +13,9 @@ smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
 $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@
 
+$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
+   $(call quiet,CC) -I. $^ -o $@ -ltalloc
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@
 
@@ -24,8 +27,13 @@ $(dir)/parse-time: $(dir)/parse-time.o 
parse-time-string/parse-time-string.o
 
 .PHONY: test check
 
-test-binaries: $(dir)/arg-test $(dir)/smtp-dummy $(dir)/symbol-test \
-   $(dir)/parse-time
+TEST_BINARIES=$(dir)/arg-test \
+ $(dir)/hex-xcode \
+ $(dir)/parse-time \
+ $(dir)/smtp-dummy \
+ $(dir)/symbol-test
+
+test-binaries: $(TEST_BINARIES)
 
 test:  all test-binaries
@${dir}/notmuch-test $(OPTIONS)
@@ -36,5 +44,6 @@ SRCS := $(SRCS) $(smtp_dummy_srcs)
 CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/symbol-test $(dir)/symbol-test.o \
 $(dir)/arg-test $(dir)/arg-test.o \
+$(dir)/hex-xcode $(dir)/hex-xcode.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/basic b/test/basic
index 1b842d2..2a571ac 100755
--- a/test/basic
+++ b/test/basic
@@ -56,6 +56,7 @@ tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
 available=$(find $TEST_DIRECTORY -maxdepth 1 -type f -perm +111  \
 ! -name aggregate-results.sh   \
 ! -name arg-test   \
+! -name hex-xcode  \
 ! -name notmuch-test   \
 ! -name parse-time \
 ! -name smtp-dummy \
diff --git a/test/hex-xcode.c b/test/hex-xcode.c
new file mode 100644
index 000..eec6541
--- /dev/null
+++ b/test/hex-xcode.c
@@ -0,0 +1,103 @@
+/* No, nothing to to with IDE from Apple Inc.
+   testbed for ../util/hex-escape.c.
+
+   usage:
+   hex-xcode [--direction=(encode|decode)] [--omit-newline]  file
+   hex-xcode [--direction=(encode|decode)] [--omit-newline] arg1 arg2 arg3 ...
+
+ */
+
+#include notmuch-client.h
+#include hex-escape.h
+#include assert.h
+
+
+enum direction {
+ENCODE,
+DECODE
+};
+
+static int
+xcode (void *ctx, enum direction dir, char *in, char **buf_p, size_t *size_p)
+{
+hex_status_t status;
+
+if (dir == ENCODE)
+   status = hex_encode (ctx, in, buf_p, size_p);
+else
+   status = hex_decode (ctx, in, buf_p, size_p);
+
+if (status == HEX_SUCCESS)
+   fputs (*buf_p, stdout);
+
+return status;
+}
+
+
+int
+main (int argc, char **argv)
+{
+
+
+enum direction dir = DECODE;
+int omit_newline = FALSE;
+
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_KEYWORD, dir, direction, 'd',
+ (notmuch_keyword_t []){ { encode, ENCODE },
+ { decode, DECODE },
+ { 0, 0 } } },
+   { NOTMUCH_OPT_BOOLEAN, omit_newline, omit-newline, 'n', 0 },
+   { 0, 0, 0, 0, 0 }
+};
+
+int opt_index = parse_arguments (argc, argv, options, 1);
+
+if (opt_index  0)
+   exit (1);
+
+void *ctx = talloc_new (NULL);
+
+char *line = NULL;
+size_t line_size;
+ssize_t line_len;
+
+char *buffer = NULL;
+size_t buf_size = 0;
+
+notmuch_bool_t read_stdin = TRUE;
+
+for (; opt_index  argc; opt_index++) {
+
+   if (xcode (ctx, dir, argv[opt_index],
+  buffer, buf_size) != HEX_SUCCESS)
+   return 1;
+
+   if (!omit_newline)
+   putchar ('\n');
+
+   read_stdin = FALSE;
+}
+
+if (!read_stdin)
+   return 0;
+
+while ((line_len = getline (line, line_size, stdin)) != -1) {
+
+   chomp_newline (line);
+
+   if (xcode (ctx, dir, line, buffer, buf_size) != HEX_SUCCESS)
+   return 1;
+
+   if (!omit_newline)
+   putchar ('\n');
+
+}
+
+if (line)
+   free (line);
+
+talloc_free (ctx);
+
+return 0;
+}
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 01/17] hex-escape: (en|de)code strings to/from restricted character set

[david]

From: David Bremner brem...@debian.org

The character set is chosen to be suitable for pathnames, and the same
as that used by contrib/nmbug

[With additions by Jani Nikula]
---
 util/Makefile.local |2 +-
 util/hex-escape.c   |  168 +++
 util/hex-escape.h   |   41 +
 3 files changed, 210 insertions(+), 1 deletion(-)
 create mode 100644 util/hex-escape.c
 create mode 100644 util/hex-escape.h

diff --git a/util/Makefile.local b/util/Makefile.local
index c7cae61..3ca623e 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,7 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)
 
-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
 
 libutil_modules := $(libutil_c_srcs:.c=.o)
 
diff --git a/util/hex-escape.c b/util/hex-escape.c
new file mode 100644
index 000..d8905d0
--- /dev/null
+++ b/util/hex-escape.c
@@ -0,0 +1,168 @@
+/* hex-escape.c -  Manage encoding and decoding of byte strings into path names
+ *
+ * Copyright (c) 2011 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner da...@tethera.net
+ */
+
+#include assert.h
+#include string.h
+#include talloc.h
+#include ctype.h
+#include error_util.h
+#include hex-escape.h
+
+static const size_t default_buf_size = 1024;
+
+static const char *output_charset =
+ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-_@=.:,;
+
+static const char escape_char = '%';
+
+static int
+is_output (char c)
+{
+return (strchr (output_charset, c) != NULL);
+}
+
+static int
+maybe_realloc (void *ctx, size_t needed, char **out, size_t *out_size)
+{
+if (*out_size  needed) {
+
+   if (*out == NULL)
+   *out = talloc_size (ctx, needed);
+   else
+   *out = talloc_realloc (ctx, *out, char, needed);
+
+   if (*out == NULL)
+   return 0;
+
+   *out_size = needed;
+}
+return 1;
+}
+
+hex_status_t
+hex_encode (void *ctx, const char *in, char **out, size_t *out_size)
+{
+
+const unsigned char *p;
+char *q;
+
+size_t escape_count = 0;
+size_t len = 0;
+size_t needed;
+
+assert (ctx); assert (in); assert (out); assert (out_size);
+
+for (p = (unsigned char *) in; *p; p++) {
+   escape_count += (!is_output (*p));
+   len++;
+}
+
+needed = len + escape_count * 2 + 1;
+
+if (*out == NULL)
+   *out_size = 0;
+
+if (!maybe_realloc (ctx, needed, out, out_size))
+   return HEX_OUT_OF_MEMORY;
+
+q = *out;
+p = (unsigned char *) in;
+
+while (*p) {
+   if (is_output (*p)) {
+   *q++ = *p++;
+   } else {
+   sprintf (q, %%%02x, *p++);
+   q += 3;
+   }
+}
+
+*q = '\0';
+return HEX_SUCCESS;
+}
+
+/* Hex decode 'in' to 'out'.
+ *
+ * This must succeed for in == out to support hex_decode_inplace().
+ */
+static hex_status_t
+hex_decode_internal (const char *in, unsigned char *out)
+{
+char buf[3];
+
+while (*in) {
+   if (*in == escape_char) {
+   char *endp;
+
+   /* This also handles unexpected end-of-string. */
+   if (!isxdigit ((unsigned char) in[1]) ||
+   !isxdigit ((unsigned char) in[2]))
+   return HEX_SYNTAX_ERROR;
+
+   buf[0] = in[1];
+   buf[1] = in[2];
+   buf[2] = '\0';
+
+   *out = strtoul (buf, endp, 16);
+
+   if (endp != buf + 2)
+   return HEX_SYNTAX_ERROR;
+
+   in += 3;
+   out++;
+   } else {
+   *out++ = *in++;
+   }
+}
+
+*out = '\0';
+
+return HEX_SUCCESS;
+}
+
+hex_status_t
+hex_decode_inplace (char *s)
+{
+/* A decoded string is never longer than the encoded one, so it is
+ * safe to decode a string onto itself. */
+return hex_decode_internal (s, (unsigned char *) s);
+}
+
+hex_status_t
+hex_decode (void *ctx, const char *in, char **out, size_t * out_size)
+{
+const char *p;
+size_t escape_count = 0;
+size_t needed = 0;
+
+assert (ctx); assert (in); assert (out); assert (out_size);
+
+size_t len = strlen (in);
+
+for (p = in; *p; p++)
+   escape_count += (*p == escape_char);
+
+needed = len - escape_count * 2 + 1;
+
+if (!maybe_realloc (ctx, needed, out, out_size))
+   return 

[Patch v2 06/17] test: add broken roundtrip test

[david]

From: David Bremner brem...@debian.org

We demonstrate the current notmuch restore parser being confused by
message-id's and tags containing non alpha numeric characters
(particularly space and parentheses are problematic because they are
not escaped by notmuch dump).

We save the files as hex escaped on disk so that the output from the
failing test will not confuse the terminal emulator of people running
the test.
---
 test/dump-restore |9 +
 1 file changed, 9 insertions(+)

diff --git a/test/dump-restore b/test/dump-restore
index b05399c..a2204fb 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,4 +85,13 @@ test_begin_subtest dump --output=outfile -- from:cworth
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
+test_expect_success 'roundtripping random message-ids and tags' \
+'test_subtest_known_broken 
+ ${TEST_DIRECTORY}/random-corpus --num-messages=10 
--config-path=${NOTMUCH_CONFIG} 
+ notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
EXPECTED.$test_count 
+ notmuch tag -random-corpus tag:random-corpus 
+ ${TEST_DIRECTORY}/hex-xcode --direction=decode  EXPECTED.$test_count | 
notmuch restore 2/dev/null 
+ notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
OUTPUT.$test_count 
+ test_cmp EXPECTED.$test_count OUTPUT.$test_count'
+
 test_done
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 03/17] test/hex-escaping: new test for hex escaping routines

[david]

From: David Bremner brem...@debian.org

These are more like unit tests, to (try to) make sure the library
functionality is working before building more complicated things on
top of it.
---
 test/hex-escaping |   26 ++
 test/notmuch-test |1 +
 2 files changed, 27 insertions(+)
 create mode 100755 test/hex-escaping

diff --git a/test/hex-escaping b/test/hex-escaping
new file mode 100755
index 000..f34cc8c
--- /dev/null
+++ b/test/hex-escaping
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+test_description=hex encoding and decoding
+. ./test-lib.sh
+
+test_begin_subtest round trip
+find $TEST_DIRECTORY/corpus -type f -print | sort | xargs cat  EXPECTED
+$TEST_DIRECTORY/hex-xcode --direction=encode  EXPECTED | 
$TEST_DIRECTORY/hex-xcode --direction=decode  OUTPUT
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest punctuation
+tag1='comic_swear=$^%$^%\\//-+$^%$'
+tag_enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode $tag1)
+test_expect_equal $tag_enc1 
comic_swear=%24%26%5e%25%24%5e%25%5c%5c%2f%2f-+%24%5e%25%24
+
+test_begin_subtest round trip newlines
+printf 'this\n tag\t has\n spaces\n'  EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=encode   EXPECTED.$test_count |\
+   $TEST_DIRECTORY/hex-xcode --direction=decode  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest round trip 8bit chars
+echo '%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'  EXPECTED.$test_count
+$TEST_DIRECTORY/hex-xcode --direction=decode   EXPECTED.$test_count |\
+   $TEST_DIRECTORY/hex-xcode --direction=encode  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 9a1b375..d2e90e2 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -60,6 +60,7 @@ TESTS=
   emacs-hello
   emacs-show
   missing-headers
+  hex-escaping
   parse-time-string
   search-date
 
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 07/17] notmuch-dump: add --format=(batch-tag|sup)

[david]

From: David Bremner brem...@debian.org

sup is the old format, and remains the default, at least until
restore is converted to parse this format.

Each line of the batch- format is valid for notmuch tag, (with the
notmuch tag omitted from the front of the line). The dump format
only uses query strings of a single message-id.

Each space seperated tag/message-id is 'hex-encoded' to remove
troubling characters.  In particular this format won't have the same
problem with e.g. spaces in message-ids or tags; they will be
round-trip-able.
---
 dump-restore-private.h |   13 +
 notmuch-dump.c |   42 --
 2 files changed, 49 insertions(+), 6 deletions(-)
 create mode 100644 dump-restore-private.h

diff --git a/dump-restore-private.h b/dump-restore-private.h
new file mode 100644
index 000..896a004
--- /dev/null
+++ b/dump-restore-private.h
@@ -0,0 +1,13 @@
+#ifndef DUMP_RESTORE_PRIVATE_H
+#define DUMP_RESTORE_PRIVATE_H
+
+#include hex-escape.h
+#include command-line-arguments.h
+
+typedef enum dump_formats {
+DUMP_FORMAT_AUTO,
+DUMP_FORMAT_BATCH_TAG,
+DUMP_FORMAT_SUP
+} dump_format_t;
+
+#endif
diff --git a/notmuch-dump.c b/notmuch-dump.c
index 88f598a..045ca9e 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,6 +19,7 @@
  */
 
 #include notmuch-client.h
+#include dump-restore-private.h
 
 int
 notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
@@ -43,7 +44,13 @@ notmuch_dump_command (unused (void *ctx), int argc, char 
*argv[])
 char *output_file_name = NULL;
 int opt_index;
 
+int output_format = DUMP_FORMAT_SUP;
+
 notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_KEYWORD, output_format, format, 'f',
+ (notmuch_keyword_t []){ { sup, DUMP_FORMAT_SUP },
+ { batch-tag, DUMP_FORMAT_BATCH_TAG },
+ { 0, 0 } } },
{ NOTMUCH_OPT_STRING, output_file_name, output, 'o', 0  },
{ 0, 0, 0, 0, 0 }
 };
@@ -83,27 +90,50 @@ notmuch_dump_command (unused (void *ctx), int argc, char 
*argv[])
  */
 notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
 
+char *buffer = NULL;
+size_t buffer_size = 0;
+
 for (messages = notmuch_query_search_messages (query);
 notmuch_messages_valid (messages);
 notmuch_messages_move_to_next (messages)) {
int first = 1;
+   const char *message_id;
+
message = notmuch_messages_get (messages);
+   message_id = notmuch_message_get_message_id (message);
 
-   fprintf (output,
-%s (, notmuch_message_get_message_id (message));
+   if (output_format == DUMP_FORMAT_SUP) {
+   fprintf (output, %s (, message_id);
+   }
 
for (tags = notmuch_message_get_tags (message);
 notmuch_tags_valid (tags);
 notmuch_tags_move_to_next (tags)) {
-   if (! first)
-   fprintf (output,  );
+   const char *tag_str = notmuch_tags_get (tags);
 
-   fprintf (output, %s, notmuch_tags_get (tags));
+   if (! first)
+   fputs ( , output);
 
first = 0;
+
+   if (output_format == DUMP_FORMAT_SUP) {
+   fputs (tag_str, output);
+   } else {
+   if (hex_encode (notmuch, tag_str,
+   buffer, buffer_size) != HEX_SUCCESS)
+   return 1;
+   fprintf (output, +%s, buffer);
+   }
}
 
-   fprintf (output, )\n);
+   if (output_format == DUMP_FORMAT_SUP) {
+   fputs ()\n, output);
+   } else {
+   if (hex_encode (notmuch, message_id,
+   buffer, buffer_size) != HEX_SUCCESS)
+   return 1;
+   fprintf (output,  -- id:%s\n, buffer);
+   }
 
notmuch_message_destroy (message);
 }
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 04/17] test: add database routines for testing

[david]

From: David Bremner brem...@debian.org

Initially, provide a way to create stub messages in the notmuch
database without corresponding files.  This is essentially cut and
paste from lib/database.cc. This is a seperate file since we don't
want to export these symbols from libnotmuch or bloat the library with
non-exported code.
---
 test/Makefile.local  |1 +
 test/database-test.c |   71 ++
 test/database-test.h |   21 +++
 3 files changed, 93 insertions(+)
 create mode 100644 test/database-test.c
 create mode 100644 test/database-test.h

diff --git a/test/Makefile.local b/test/Makefile.local
index 8da4c56..8479f91 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -45,5 +45,6 @@ CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/symbol-test $(dir)/symbol-test.o \
 $(dir)/arg-test $(dir)/arg-test.o \
 $(dir)/hex-xcode $(dir)/hex-xcode.o \
+$(dir)/database-test.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/database-test.c b/test/database-test.c
new file mode 100644
index 000..739e03b
--- /dev/null
+++ b/test/database-test.c
@@ -0,0 +1,71 @@
+/*
+ * Database routines intended only for testing, not exported from
+ * library.
+ *
+ * Copyright (c) 2012 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner da...@tethera.net
+ */
+
+#include notmuch-private.h
+#include database-test.h
+
+notmuch_status_t
+notmuch_database_add_stub_message (notmuch_database_t *notmuch,
+  const char *message_id,
+  const char **tags)
+{
+const char **tag;
+notmuch_status_t ret;
+notmuch_private_status_t private_status;
+notmuch_message_t *message;
+
+ret = _notmuch_database_ensure_writable (notmuch);
+if (ret)
+   return ret;
+
+message = _notmuch_message_create_for_message_id (notmuch,
+ message_id,
+ private_status);
+if (message == NULL) {
+   return COERCE_STATUS (private_status,
+ Unexpected status value from 
_notmuch_message_create_for_message_id);
+
+}
+
+if (private_status != NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+   return NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+_notmuch_message_add_term (message, type, mail);
+
+if (tags) {
+   ret = notmuch_message_freeze (message);
+   if (ret)
+   return ret;
+
+   for (tag = tags; *tag; tag++) {
+   ret = notmuch_message_add_tag (message, *tag);
+   if (ret)
+   return ret;
+   }
+}
+
+ret = notmuch_message_thaw (message);
+if (ret)
+   return ret;
+
+return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/test/database-test.h b/test/database-test.h
new file mode 100644
index 000..84f7988
--- /dev/null
+++ b/test/database-test.h
@@ -0,0 +1,21 @@
+#ifndef _DATABASE_TEST_H
+#define _DATABASE_TEST_H
+/* Add a new stub message to the given notmuch database.
+ *
+ * At least the following return values are possible:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
+ * ID as another message already in the database.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ * mode so no message can be added.
+ */
+
+notmuch_status_t
+notmuch_database_add_stub_message (notmuch_database_t *database,
+  const char *message_id,
+  const char **tag_list);
+
+#endif
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 05/17] test: add generator for random stub messages

[david]

From: David Bremner brem...@debian.org

Initial use case is testing dump and restore, so we only have
message-ids and tags.

The message ID's are nothing like RFC compliant, but it doesn't seem
any harder to roundtrip random UTF-8 strings than RFC-compliant ones.

Tags are UTF-8, even though notmuch is in principle more generous than
that.

updated for id:m2wr04ocro@guru.guru-group.fi

- talk about Unicode value rather some specific encoding
- call talloc_realloc less times
---
 test/.gitignore  |1 +
 test/Makefile.local  |   10 +++
 test/basic   |1 +
 test/random-corpus.c |  204 ++
 4 files changed, 216 insertions(+)
 create mode 100644 test/random-corpus.c

diff --git a/test/.gitignore b/test/.gitignore
index be7ab5e..1eff7ce 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -4,4 +4,5 @@ smtp-dummy
 symbol-test
 arg-test
 hex-xcode
+random-corpus
 tmp.*
diff --git a/test/Makefile.local b/test/Makefile.local
index 8479f91..6a9f15e 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -16,6 +16,14 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o 
util/libutil.a
 $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
$(call quiet,CC) -I. $^ -o $@ -ltalloc
 
+random_corpus_deps =  $(dir)/random-corpus.o  $(dir)/database-test.o \
+   notmuch-config.o command-line-arguments.o \
+   lib/libnotmuch.a util/libutil.a \
+   parse-time-string/libparse-time-string.a
+
+$(dir)/random-corpus: $(random_corpus_deps)
+   $(call quiet,CC) $(CFLAGS_FINAL) $^ -o $@ $(CONFIGURE_LDFLAGS)
+
 $(dir)/smtp-dummy: $(smtp_dummy_modules)
$(call quiet,CC) $^ -o $@
 
@@ -29,6 +37,7 @@ $(dir)/parse-time: $(dir)/parse-time.o 
parse-time-string/parse-time-string.o
 
 TEST_BINARIES=$(dir)/arg-test \
  $(dir)/hex-xcode \
+ $(dir)/random-corpus \
  $(dir)/parse-time \
  $(dir)/smtp-dummy \
  $(dir)/symbol-test
@@ -46,5 +55,6 @@ CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
 $(dir)/arg-test $(dir)/arg-test.o \
 $(dir)/hex-xcode $(dir)/hex-xcode.o \
 $(dir)/database-test.o \
+$(dir)/random-corpus $(dir)/random-corpus.o \
 $(dir)/parse-time $(dir)/parse-time.o \
 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/basic b/test/basic
index 2a571ac..f93469f 100755
--- a/test/basic
+++ b/test/basic
@@ -59,6 +59,7 @@ available=$(find $TEST_DIRECTORY -maxdepth 1 -type f -perm 
+111  \
 ! -name hex-xcode  \
 ! -name notmuch-test   \
 ! -name parse-time \
+! -name random-corpus  \
 ! -name smtp-dummy \
 ! -name symbol-test\
 ! -name test-verbose   \
diff --git a/test/random-corpus.c b/test/random-corpus.c
new file mode 100644
index 000..085bda0
--- /dev/null
+++ b/test/random-corpus.c
@@ -0,0 +1,204 @@
+/*
+ * Generate a random corpus of stub messages.
+ *
+ * Initial use case is testing dump and restore, so we only have
+ * message-ids and tags.
+ *
+ * Generated message-id's and tags are intentionally nasty.
+ *
+ * Copyright (c) 2012 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner da...@tethera.net
+ */
+
+#include stdlib.h
+#include assert.h
+#include talloc.h
+#include string.h
+#include glib.h
+#include math.h
+
+#include notmuch-client.h
+#include command-line-arguments.h
+#include database-test.h
+
+/* Current largest Unicode value defined. Note that most of these will
+ * be printed as boxes in most fonts.
+ */
+
+#define GLYPH_MAX 0x10FFFE
+
+static gunichar
+random_unichar ()
+{
+int start = 1, stop = GLYPH_MAX;
+int class = random() % 2;
+
+/*
+ *  Choose about half ascii as test characters, as ascii
+ *  punctation and whitespace is the main cause of problems for
+ *  the (old) restore parser
+*/
+switch (class) {
+case 0:
+   /* ascii */
+   start = 0x01;
+   stop = 0x7f;
+   break;
+case 1:
+   /* the rest of unicode */
+   start = 0x80;
+   stop = GLYPH_MAX;
+}
+
+if (start == stop)
+   return start;
+else
+   return start + (random() 

[Patch v2 08/17] util: add string-util.[ch]

[david]

From: David Bremner brem...@debian.org

This is to give a home to strtok_len. It's a bit silly to add a header
for one routine, but it needs to be shared between several compilation
units (or at least that's the most natural design).
---
 util/Makefile.local |3 ++-
 util/string-util.c  |   34 ++
 util/string-util.h  |   19 +++
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 util/string-util.c
 create mode 100644 util/string-util.h

diff --git a/util/Makefile.local b/util/Makefile.local
index 3ca623e..a11e35b 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -3,7 +3,8 @@
 dir := util
 extra_cflags += -I$(srcdir)/$(dir)
 
-libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c
+libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
+ $(dir)/string-util.c
 
 libutil_modules := $(libutil_c_srcs:.c=.o)
 
diff --git a/util/string-util.c b/util/string-util.c
new file mode 100644
index 000..44f8cd3
--- /dev/null
+++ b/util/string-util.c
@@ -0,0 +1,34 @@
+/* string-util.c -  Extra or enhanced routines for null terminated strings.
+ *
+ * Copyright (c) 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula j...@nikula.org
+ */
+
+
+#include string-util.h
+
+char *
+strtok_len (char *s, const char *delim, size_t *len)
+{
+/* skip initial delims */
+s += strspn (s, delim);
+
+/* length of token */
+*len = strcspn (s, delim);
+
+return *len ? s : NULL;
+}
diff --git a/util/string-util.h b/util/string-util.h
new file mode 100644
index 000..696da40
--- /dev/null
+++ b/util/string-util.h
@@ -0,0 +1,19 @@
+#ifndef _STRING_UTIL_H
+#define _STRING_UTIL_H
+
+#include string.h
+
+/* like strtok(3), but without state, and doesn't modify s. usage pattern:
+ *
+ * const char *tok = input;
+ * const char *delim =  \t;
+ * size_t tok_len = 0;
+ *
+ * while ((tok = strtok_len (tok + tok_len, delim, tok_len)) != NULL) {
+ * // do stuff with string tok of length tok_len
+ * }
+ */
+
+char *strtok_len (char *s, const char *delim, size_t *len);
+
+#endif
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 14/17] test: update dump-restore roundtripping test for batch-tag format

[david]

From: David Bremner brem...@debian.org

Now we can actually round trip these crazy tags and and message ids.
hex-xcode is no longer needed as it's built in.
---
 test/dump-restore |   13 +++--
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index a2204fb..e08b656 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,13 +85,14 @@ test_begin_subtest dump --output=outfile -- from:cworth
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
+
 test_expect_success 'roundtripping random message-ids and tags' \
-'test_subtest_known_broken 
- ${TEST_DIRECTORY}/random-corpus --num-messages=10 
--config-path=${NOTMUCH_CONFIG} 
- notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
EXPECTED.$test_count 
+'${TEST_DIRECTORY}/random-corpus --num-messages=100 
--config-path=${NOTMUCH_CONFIG} 
+ notmuch dump --format=batch-tag | sort  EXPECTED.$test_count 
  notmuch tag -random-corpus tag:random-corpus 
- ${TEST_DIRECTORY}/hex-xcode --direction=decode  EXPECTED.$test_count | 
notmuch restore 2/dev/null 
- notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
OUTPUT.$test_count 
+ notmuch restore --format=batch-tag --input=EXPECTED.$test_count 
+ notmuch dump --format=batch-tag | sort  OUTPUT.$test_count 
  test_cmp EXPECTED.$test_count OUTPUT.$test_count'
-
 test_done
+
+# Note the database is poisoned for sup format at this point.
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 09/17] tag-util.[ch]: New files for common tagging routines

[david]

From: David Bremner brem...@debian.org

These are meant to be shared between notmuch-tag and notmuch-restore.

The bulk of the routines implement a tag operation list abstract
data type act as a structured representation of a set of tag
operations (typically coming from a single tag command or line of
input).
---
 Makefile.local |1 +
 tag-util.c |  264 
 tag-util.h |  120 ++
 3 files changed, 385 insertions(+)
 create mode 100644 tag-util.c
 create mode 100644 tag-util.h

diff --git a/Makefile.local b/Makefile.local
index 2b91946..854867d 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -274,6 +274,7 @@ notmuch_client_srcs =   \
query-string.c  \
mime-node.c \
crypto.c\
+   tag-util.c
 
 notmuch_client_modules = $(notmuch_client_srcs:.c=.o)
 
diff --git a/tag-util.c b/tag-util.c
new file mode 100644
index 000..287cc67
--- /dev/null
+++ b/tag-util.c
@@ -0,0 +1,264 @@
+#include string-util.h
+#include tag-util.h
+#include hex-escape.h
+
+struct _tag_operation_t {
+const char *tag;
+notmuch_bool_t remove;
+};
+
+struct _tag_op_list_t {
+tag_operation_t *ops;
+int count;
+int size;
+};
+
+int
+parse_tag_line (void *ctx, char *line,
+   tag_op_flag_t flags,
+   char **query_string,
+   tag_op_list_t *tag_ops)
+{
+char *tok = line;
+size_t tok_len = 0;
+
+chomp_newline (line);
+
+/* remove leading space */
+while (*tok == ' ' || *tok == '\t')
+   tok++;
+
+/* Skip empty and comment lines. */
+if (*tok == '\0' || *tok == '#')
+   return 1;
+
+tag_op_list_reset (tag_ops);
+
+/* Parse tags. */
+while ((tok = strtok_len (tok + tok_len,  , tok_len)) != NULL) {
+   notmuch_bool_t remove;
+   char *tag;
+
+   /* Optional explicit end of tags marker. */
+   if (strncmp (tok, --, tok_len) == 0) {
+   tok = strtok_len (tok + tok_len,  , tok_len);
+   break;
+   }
+
+   /* Implicit end of tags. */
+   if (*tok != '-'  *tok != '+')
+   break;
+
+   /* If tag is terminated by NUL, there's no query string. */
+   if (*(tok + tok_len) == '\0') {
+   tok = NULL;
+   break;
+   }
+
+   /* Terminate, and start next token after terminator. */
+   *(tok + tok_len++) = '\0';
+
+   remove = (*tok == '-');
+   tag = tok + 1;
+
+   /* Maybe refuse empty tags. */
+   if (!(flags  TAG_FLAG_BE_GENEROUS)  *tag == '\0') {
+   tok = NULL;
+   break;
+   }
+
+   /* Decode tag. */
+   if (hex_decode_inplace (tag) != HEX_SUCCESS) {
+   tok = NULL;
+   break;
+   }
+
+   if (tag_op_list_append (ctx, tag_ops, tag, remove))
+   return -1;
+}
+
+if (tok == NULL || tag_ops-count == 0) {
+   /* FIXME: line has been modified! */
+   fprintf (stderr, Warning: Ignoring invalid input line: %s\n,
+line);
+   return 1;
+}
+
+/* tok now points to the query string */
+if (hex_decode_inplace (tok) != HEX_SUCCESS) {
+   /* FIXME: line has been modified! */
+   fprintf (stderr, Warning: Ignoring invalid input line: %s\n,
+line);
+   return 1;
+}
+
+*query_string = tok;
+
+return 0;
+}
+
+static inline void
+message_error (notmuch_message_t *message,
+  notmuch_status_t status,
+  const char *format, ...)
+{
+va_list va_args;
+
+va_start (va_args, format);
+
+vfprintf (stderr, format, va_args);
+fprintf (stderr, Message-ID: %s\n, notmuch_message_get_message_id 
(message));
+fprintf (stderr, Status: %s\n, notmuch_status_to_string (status));
+}
+
+notmuch_status_t
+tag_op_list_apply (notmuch_message_t *message,
+  tag_op_list_t *list,
+  tag_op_flag_t flags)
+{
+int i;
+
+notmuch_status_t status = 0;
+tag_operation_t *tag_ops = list-ops;
+
+status = notmuch_message_freeze (message);
+if (status) {
+   message_error (message, status, freezing message);
+   return status;
+}
+
+if (flags  TAG_FLAG_REMOVE_ALL) {
+   status = notmuch_message_remove_all_tags (message);
+   if (status) {
+   message_error (message, status, removing all tags );
+   return status;
+   }
+}
+
+for (i = 0; i  list-count; i++) {
+   if (tag_ops[i].remove) {
+   status = notmuch_message_remove_tag (message, tag_ops[i].tag);
+   if (status) {
+   message_error (message, status, removing tag %s, 
tag_ops[i].tag);
+   return status;
+   }
+   } else {
+   status = notmuch_message_add_tag (message, tag_ops[i].tag);
+   if (status) {
+   message_error (message, status, adding tag %s, 
tag_ops[i].tag);
+   return status;
+   

[Patch v2 11/17] test: add test for notmuch tag --batch option

[david]

From: Jani Nikula j...@nikula.org

Basic test of functionality, along with all combinations of options.
---
 test/tagging |   46 ++
 1 file changed, 46 insertions(+)

diff --git a/test/tagging b/test/tagging
index 980ff92..e5b8315 100755
--- a/test/tagging
+++ b/test/tagging
@@ -46,6 +46,52 @@ test_expect_equal $output \
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (:\  inbox tag1 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag1 tag4 unread)
 
+test_begin_subtest --batch
+notmuch tag --batch EOF
+# %20 is a space in tag
+-:%20 -tag1 +tag5 +tag6 -- One
++tag1 -tag1 -tag4 +tag4 -- Two
+-tag6 One
++tag5 Two
+EOF
+output=$(notmuch search \* | notmuch_search_sanitize)
+test_expect_equal $output \
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag5 unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag4 tag5 unread)
+
+ batch.in  EOF
+# %20 is a space in tag
+-:%20 -tag1 +tag5 +tag6 -- One
++tag1 -tag1 -tag4 +tag4 -- Two
+-tag6 One
++tag5 Two
+EOF
+
+test_begin_subtest --input
+notmuch tag --input=batch.in
+output=$(notmuch search \* | notmuch_search_sanitize)
+test_expect_equal $output \
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag5 unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag4 tag5 unread)
+
+test_begin_subtest --batch --input
+notmuch tag --batch --input=batch.in
+output=$(notmuch search \* | notmuch_search_sanitize)
+test_expect_equal $output \
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag5 unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag4 tag5 unread)
+
+test_begin_subtest --batch, blank lines and comments
+notmuch dump | sort  EXPECTED.$test_count
+notmuch tag --batch EOF
+# this line is a comment; the next has only white space
+
+
+# the previous line is empty
+EOF
+notmuch dump | sort  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
 test_expect_code 1 Empty tag names 'notmuch tag + One'
 
 test_expect_code 1 Tag name beginning with - 'notmuch tag +- One'
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 12/17] man: document notmuch tag --batch, --input options

[david]

From: Jani Nikula j...@nikula.org

---
 man/man1/notmuch-tag.1 |   52 +++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
index 0f86582..751db7b 100644
--- a/man/man1/notmuch-tag.1
+++ b/man/man1/notmuch-tag.1
@@ -4,7 +4,12 @@ notmuch-tag \- add/remove tags for all messages matching the 
search terms
 
 .SH SYNOPSIS
 .B notmuch tag
-.RI  + tag |\- tag  [...] [\-\-]  search-term ...
+.RI + tag |\- tag  [...] [\-\-]  search-terms 
+
+.B notmuch tag
+.RI --batch
+.RI [ --input= filename  ]
+
 
 .SH DESCRIPTION
 
@@ -29,6 +34,51 @@ updates the maildir flags according to tag changes if the
 configuration option is enabled. See \fBnotmuch-config\fR(1) for
 details.
 
+Supported options for
+.B tag
+include
+.RS 4
+.TP 4
+.BR \-\-batch
+
+Read batch tagging operations from standard input. This is more
+efficient than repeated
+.B notmuch tag
+invocations. See
+.B TAG FILE FORMAT
+below for the input format. This option is not compatible with
+specifying tagging on the command line.
+.RE
+
+.RS 4
+.TP 4
+.BR \-\-input= filename
+
+Read input from given file, instead of from stdin. Implies
+.BR --batch .
+
+.SH TAG FILE FORMAT
+
+The input must consist of lines of the format:
+
+.RI T + tag |\- tag  [...] [\-\-]  search-terms 
+
+Each line is interpreted similarly to
+.B notmuch tag
+command line arguments. The delimiter is one or more spaces ' '. Any
+characters in tag and search-terms
+.B may
+be hex encoded with %NN where NN is the hexadecimal value of the
+character. Any ' ' and '%' characters in tag and search-terms
+.B must
+be hex encoded (using %20 and %25, respectively). Any characters that
+are not part of tag or search-terms
+.B must not
+be hex encoded.
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 17/17] tag-util: optimization of tag application

[david]

From: David Bremner brem...@debian.org

The idea is not to bother with restore operations if they don't change
the set of tags. This is actually a relatively common case.

In order to avoid fancy datastructures, this method is quadratic in
the number of tags; at least on my mail database this doesn't seem to
be a big problem.
---
 notmuch-tag.c |2 +-
 tag-util.c|   59 +
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 8a8af0b..e4fca67 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -140,7 +140,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages)  ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-   tag_op_list_apply (message, tag_ops, flags);
+   tag_op_list_apply (message, tag_ops, flags | TAG_FLAG_PRE_OPTIMIZED);
notmuch_message_destroy (message);
 }
 
diff --git a/tag-util.c b/tag-util.c
index 287cc67..2bb8355 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -111,6 +111,62 @@ message_error (notmuch_message_t *message,
 fprintf (stderr, Status: %s\n, notmuch_status_to_string (status));
 }
 
+static int
+makes_changes (notmuch_message_t *message,
+  tag_op_list_t *list,
+  tag_op_flag_t flags)
+{
+
+int i;
+
+notmuch_tags_t *tags;
+notmuch_bool_t changes = FALSE;
+
+/* First, do we delete an existing tag? */
+changes = FALSE;
+for (tags = notmuch_message_get_tags (message);
+! changes  notmuch_tags_valid (tags);
+notmuch_tags_move_to_next (tags)) {
+   const char *cur_tag = notmuch_tags_get (tags);
+   int last_op =  (flags  TAG_FLAG_REMOVE_ALL) ? -1 : 0;
+
+   for (i = 0; i  list-count; i++) {
+   if (strcmp (cur_tag, list-ops[i].tag) == 0) {
+   last_op = list-ops[i].remove ? -1 : 1;
+   }
+   }
+
+   changes = (last_op == -1);
+}
+notmuch_tags_destroy (tags);
+
+if (changes)
+   return TRUE;
+
+/* Now check for adding new tags */
+for (i = 0; i  list-count; i++) {
+   notmuch_bool_t exists = FALSE;
+
+   for (tags = notmuch_message_get_tags (message);
+notmuch_tags_valid (tags);
+notmuch_tags_move_to_next (tags)) {
+   const char *cur_tag = notmuch_tags_get (tags);
+   if (strcmp (cur_tag, list-ops[i].tag) == 0) {
+   exists = TRUE;
+   break;
+   }
+   }
+   notmuch_tags_destroy (tags);
+
+   /* the following test is conservative, it's ok to think we
+* make changes when we don't */
+   if ( ! exists  ! list-ops[i].remove )
+   return TRUE;
+}
+return FALSE;
+
+}
+
 notmuch_status_t
 tag_op_list_apply (notmuch_message_t *message,
   tag_op_list_t *list,
@@ -121,6 +177,9 @@ tag_op_list_apply (notmuch_message_t *message,
 notmuch_status_t status = 0;
 tag_operation_t *tag_ops = list-ops;
 
+if (! (flags  TAG_FLAG_PRE_OPTIMIZED)  ! makes_changes (message, list, 
flags))
+   return NOTMUCH_STATUS_SUCCESS;
+
 status = notmuch_message_freeze (message);
 if (status) {
message_error (message, status, freezing message);
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[Patch v2 15/17] test: second set of dump/restore --format=batch-tag tests

[david]

From: David Bremner brem...@debian.org

These one need the completed functionality in notmuch-restore. Fairly
exotic tags are tested, but no weird message id's.

We test each possible input to autodetection, both explicit (with
--format=auto) and implicit (without --format).
---
 test/dump-restore |   92 +
 1 file changed, 92 insertions(+)

diff --git a/test/dump-restore b/test/dump-restore
index e08b656..23af2b7 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -85,6 +85,98 @@ test_begin_subtest dump --output=outfile -- from:cworth
 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
+test_begin_subtest Check for a safe set of message-ids
+notmuch search --output=messages from:cworth  EXPECTED.$test_count
+notmuch search --output=messages from:cworth |\
+   $TEST_DIRECTORY/hex-xcode --direction=encode  OUTPUT.$test_count
+test_expect_equal_file OUTPUT.$test_count EXPECTED.$test_count
+
+test_begin_subtest format=batch-tag, # round-trip
+notmuch dump --format=sup | sort  EXPECTED.$test_count
+notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
+notmuch dump --format=sup | sort  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest format=batch-tag, # blank lines and comments
+notmuch dump --format=batch-tag| sort  EXPECTED.$test_count
+notmuch restore EOF
+# this line is a comment; the next has only white space
+
+
+# the previous line is empty
+EOF
+notmuch dump --format=batch-tag | sort  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest format=batch-tag, # reverse-round-trip empty tag
+cat EOF EXPECTED.$test_count
++ -- id:20091117232137.ga7...@griffis1.net
+EOF
+notmuch restore --format=batch-tag  EXPECTED.$test_count
+notmuch dump --format=batch-tag id:20091117232137.ga7...@griffis1.net  
OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+tag1='comic_swear=$^%$^%\\//-+$^%$'
+enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode $tag1)
+
+tag2=$(printf 'this\n tag\t has\n spaces')
+enc2=$($TEST_DIRECTORY/hex-xcode --direction=encode $tag2)
+
+enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
+tag3=$($TEST_DIRECTORY/hex-xcode --direction=decode $enc3)
+
+notmuch dump --format=batch-tag  BACKUP
+
+notmuch tag +$tag1 +$tag2 +$tag3 -inbox -unread *
+
+test_begin_subtest 'format=batch-tag, round trip with strange tags'
+notmuch dump --format=batch-tag  EXPECTED.$test_count
+notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
+notmuch dump --format=batch-tag  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, checking encoded output'
+cp /dev/null EXPECTED.$test_count
+notmuch dump --format=batch-tag -- from:cworth |\
+awk { print \+$enc1 +$enc2 +$enc3 -- \ \$5 }  EXPECTED.$test_count
+
+notmuch dump --format=batch-tag -- from:cworth   OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'restoring sane tags'
+notmuch restore --format=batch-tag  BACKUP
+notmuch dump --format=batch-tag  OUTPUT.$test_count
+test_expect_equal_file BACKUP OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, restore=auto'
+notmuch dump --format=batch-tag  EXPECTED.$test_count
+notmuch tag -inbox -unread *
+notmuch restore --format=auto  EXPECTED.$test_count
+notmuch dump --format=batch-tag  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=auto'
+notmuch dump --format=sup  EXPECTED.$test_count
+notmuch tag -inbox -unread *
+notmuch restore --format=auto  EXPECTED.$test_count
+notmuch dump --format=sup  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=batch-tag, restore=default'
+notmuch dump --format=batch-tag  EXPECTED.$test_count
+notmuch tag -inbox -unread *
+notmuch restore  EXPECTED.$test_count
+notmuch dump --format=batch-tag  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+test_begin_subtest 'format=sup, restore=default'
+notmuch dump --format=sup  EXPECTED.$test_count
+notmuch tag -inbox -unread *
+notmuch restore  EXPECTED.$test_count
+notmuch dump --format=sup  OUTPUT.$test_count
+test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
+
+notmuch dump --format=batch-tag --output=save.tags
 
 test_expect_success 'roundtripping random message-ids and tags' \
 '${TEST_DIRECTORY}/random-corpus --num-messages=100 
--config-path=${NOTMUCH_CONFIG} 
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org

[Patch v2 16/17] notmuch-{dump, restore}.1: document new format options

[david]

From: David Bremner brem...@debian.org

More or less arbitrarily, notmuch-dump.1 gets the more detailed
description of the format.
---
 man/man1/notmuch-dump.1|   58 +++
 man/man1/notmuch-restore.1 |   59 +++-
 2 files changed, 111 insertions(+), 6 deletions(-)

diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
index 230deec..9f59905 100644
--- a/man/man1/notmuch-dump.1
+++ b/man/man1/notmuch-dump.1
@@ -5,6 +5,7 @@ notmuch-dump \- creates a plain-text dump of the tags of each 
message
 .SH SYNOPSIS
 
 .B notmuch dump
+.RB  [ \-\-format=(sup|batch-tag)  ] [--]
 .RI [ --output= filename  ] [--]
 .RI [  search-term ...]
 
@@ -19,6 +20,63 @@ recreated from the messages themselves.  The output of 
notmuch dump is
 therefore the only critical thing to backup (and much more friendly to
 incremental backup than the native database files.)
 
+.TP 4
+.B \-\-format=(sup|batch-tag)
+
+Notmuch restore supports two plain text dump formats, both with one message-id
+per line, followed by a list of tags.
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
+compatible with the format of files produced by sup-dump.
+So if you've previously been using sup for mail, then the
+.B notmuch restore
+command provides you a way to import all of your tags (or labels as
+sup calls them).
+Each line has the following form
+
+.RS 4
+.RI  message-id 
+.B (
+.RI  tag  ...
+.B )
+
+with zero or more tags are separated by spaces. Note that (malformed)
+message-ids may contain arbitrary non-null characters. Note also
+that tags with spaces will not be correctly restored with this format.
+
+.RE
+
+.RE
+.RS 4
+.TP 4
+.B batch-tag
+
+The
+.B batch-tag
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.
+Each line has the form
+
+.RS 4
+.RI + encoded-tag+ encoded-tag  ... -- 
encoded-message-id 
+
+where encoded means that every byte not matching the regex
+.B [A-Za-z0-9+-_@=.:,]
+is replace by
+.B %nn
+where nn is the two digit hex encoding.
+The astute reader will notice this is a special case of the batch input
+format for \fBnotmuch-tag\fR(1).
+
+.RE
+
+
 With no search terms, a dump of all messages in the database will be
 generated.  A -- argument instructs notmuch that the
 remaining arguments are search terms.
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
index 2fa8733..3860829 100644
--- a/man/man1/notmuch-restore.1
+++ b/man/man1/notmuch-restore.1
@@ -6,6 +6,7 @@ notmuch-restore \- restores the tags from the given file (see 
notmuch dump)
 
 .B notmuch restore
 .RB [ --accumulate ]
+.RB [ --format=(auto|batch-tag|sup) ]
 .RI [ --input= filename  ]
 
 .SH DESCRIPTION
@@ -15,19 +16,51 @@ Restores the tags from the given file (see
 
 The input is read from the given filename, if any, or from stdin.
 
-Note: The dump file format is specifically chosen to be
+
+Supported options for
+.B restore
+include
+.RS 4
+.TP 4
+.B \-\-accumulate
+
+The union of the existing and new tags is applied, instead of
+replacing each message's tags as they are read in from the dump file.
+
+.RE
+.RS 4
+.TP 4
+.B \-\-format=(sup|batch-tag|auto)
+
+Notmuch restore supports two plain text dump formats, with one message-id
+per line, and a list of tags.
+For details of the actual formats, see \fBnotmuch-dump\fR(1).
+
+.RS 4
+.TP 4
+.B sup
+
+The
+.B sup
+dump file format is specifically chosen to be
 compatible with the format of files produced by sup-dump.
 So if you've previously been using sup for mail, then the
 .B notmuch restore
 command provides you a way to import all of your tags (or labels as
 sup calls them).
 
-The --accumulate switch causes the union of the existing and new tags to be
-applied, instead of replacing each message's tags as they are read in from the
-dump file.
+.RE
+.RS 4
+.TP 4
+.B batch-tag
 
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for search-terms.
+The
+.B batch-tag
+dump format is intended to more robust against malformed message-ids
+and tags containing whitespace or non-\fBascii\fR(7) characters.  This
+format hex-escapes all characters those outside of a small character
+set, intended to be suitable for e.g. pathnames in most UNIX-like
+systems.
 
 .B notmuch restore
 updates the maildir flags according to tag changes if the
@@ -36,6 +69,20 @@ configuration option is enabled. See \fBnotmuch-config\fR(1) 
for
 details.
 
 .RE
+
+.RS 4
+.TP 4
+.B auto
+
+This option (the default) tries to guess the format from the
+input. For correctly formed input in either supported format, this
+heuristic, based the fact that batch-tag format contains no parentheses,
+should be accurate.
+
+.RE
+
+.RE
+
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-- 
1.7.10.4

___
notmuch mailing list

[Patch v2 10/17] cli: add support for batch tagging operations to notmuch tag

[david]

From: Jani Nikula j...@nikula.org

Add support for batch tagging operations through stdin to notmuch
tag. This can be enabled with the new --stdin command line option to
notmuch tag. The input must consist of lines of the format:

+tag|-tag [...] [--] search-terms

Each line is interpreted similarly to notmuch tag command line
arguments. The delimiter is one or more spaces ' '. Any characters in
tag and search-terms MAY be hex encoded with %NN where NN is the
hexadecimal value of the character. Any ' ' and '%' characters in
tag and search-terms MUST be hex encoded (using %20 and %25,
respectively). Any characters that are not part of tag or
search-terms MUST NOT be hex encoded.

Leading and trailing space ' ' is ignored. Empty lines and lines
beginning with '#' are ignored.

Signed-off-by: Jani Nikula j...@nikula.org

Hacked-like-crazy-by: David Bremner da...@tethera.net
---
 notmuch-tag.c |  194 +++--
 1 file changed, 118 insertions(+), 76 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 88d559b..8a8af0b 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -19,6 +19,7 @@
  */
 
 #include notmuch-client.h
+#include tag-util.h
 
 static volatile sig_atomic_t interrupted;
 
@@ -54,14 +55,9 @@ _escape_tag (char *buf, const char *tag)
 return buf;
 }
 
-typedef struct {
-const char *tag;
-notmuch_bool_t remove;
-} tag_operation_t;
-
 static char *
 _optimize_tag_query (void *ctx, const char *orig_query_string,
-const tag_operation_t *tag_ops)
+const tag_op_list_t *list)
 {
 /* This is subtler than it looks.  Xapian ignores the '-' operator
  * at the beginning both queries and parenthesized groups and,
@@ -73,19 +69,20 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 
 char *escaped, *query_string;
 const char *join = ;
-int i;
+size_t i;
 unsigned int max_tag_len = 0;
 
 /* Don't optimize if there are no tag changes. */
-if (tag_ops[0].tag == NULL)
+if (tag_op_list_size (list) == 0)
return talloc_strdup (ctx, orig_query_string);
 
 /* Allocate a buffer for escaping tags.  This is large enough to
  * hold a fully escaped tag with every character doubled plus
  * enclosing quotes and a NUL. */
-for (i = 0; tag_ops[i].tag; i++)
-   if (strlen (tag_ops[i].tag)  max_tag_len)
-   max_tag_len = strlen (tag_ops[i].tag);
+for (i = 0; i  tag_op_list_size (list); i++)
+   if (strlen (tag_op_list_tag (list, i))  max_tag_len)
+   max_tag_len = strlen (tag_op_list_tag (list, i));
+
 escaped = talloc_array (ctx, char, max_tag_len * 2 + 3);
 if (! escaped)
return NULL;
@@ -96,11 +93,11 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
 else
query_string = talloc_asprintf (ctx, ( %s ) and (, orig_query_string);
 
-for (i = 0; tag_ops[i].tag  query_string; i++) {
+for (i = 0; i  tag_op_list_size (list)  query_string; i++) {
query_string = talloc_asprintf_append_buffer (
query_string, %s%stag:%s, join,
-   tag_ops[i].remove ?  : not ,
-   _escape_tag (escaped, tag_ops[i].tag));
+   tag_op_list_isremove (list, i) ?  : not ,
+   _escape_tag (escaped, tag_op_list_tag (list, i)));
join =  or ;
 }
 
@@ -116,12 +113,11 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
  * element. */
 static int
 tag_query (void *ctx, notmuch_database_t *notmuch, const char *query_string,
-  tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
+  tag_op_list_t *tag_ops, tag_op_flag_t flags)
 {
 notmuch_query_t *query;
 notmuch_messages_t *messages;
 notmuch_message_t *message;
-int i;
 
 /* Optimize the query so it excludes messages that already have
  * the specified set of tags. */
@@ -144,21 +140,7 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 notmuch_messages_valid (messages)  ! interrupted;
 notmuch_messages_move_to_next (messages)) {
message = notmuch_messages_get (messages);
-
-   notmuch_message_freeze (message);
-
-   for (i = 0; tag_ops[i].tag; i++) {
-   if (tag_ops[i].remove)
-   notmuch_message_remove_tag (message, tag_ops[i].tag);
-   else
-   notmuch_message_add_tag (message, tag_ops[i].tag);
-   }
-
-   notmuch_message_thaw (message);
-
-   if (synchronize_flags)
-   notmuch_message_tags_to_maildir_flags (message);
-
+   tag_op_list_apply (message, tag_ops, flags);
notmuch_message_destroy (message);
 }
 
@@ -170,15 +152,17 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
-tag_operation_t *tag_ops;
-int tag_ops_count = 0;
-char *query_string;
+tag_op_list_t *tag_ops = NULL;
+char 

Re: [PATCH v2 4/7] emacs: make emacs tagging use the stdin query functionality

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
 From: Mark Walters markwalters1...@gmail.com
 
 In preparation for the use of large queries in some cases make tagging
 from emacs use the new query on stdin functionality. Currently uses
 this for all tagging (as I could not see a reason not to).
 ---
  emacs/notmuch-tag.el |   14 +-
  1 files changed, 9 insertions(+), 5 deletions(-)
 
 diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
 index 4fce3a9..4634b0d 100644
 --- a/emacs/notmuch-tag.el
 +++ b/emacs/notmuch-tag.el
 @@ -59,9 +59,10 @@ the messages that were tagged
(setq search-terms (list *)))
(split-string
 (with-output-to-string
 - (with-current-buffer standard-output
 -   (apply 'call-process notmuch-command nil t
 -   nil search --output=tags --exclude=false search-terms)))
 + (with-temp-buffer
 +   (insert (mapconcat 'identity search-terms  ))

#'identity ?

 +   (apply 'call-process-region (point-min) (point-max) notmuch-command 
 nil

#'call-process-region ?

 +   standard-output nil search --output=tags --exclude=false 
 (list -

If you use funcall instead of apply here, you won't need to put - in
a list.

Also, the lines seem a little long (but maybe that's just diff and
quoting?)

 \n+ t))
  
  (defun notmuch-select-tag-with-completion (prompt rest search-terms)
 @@ -134,8 +135,11 @@ notmuch-after-tag-hook will be run.
   tag-changes)
(unless (null tag-changes)
  (run-hooks 'notmuch-before-tag-hook)
 -(apply 'notmuch-call-notmuch-process tag
 -(append tag-changes (list -- query)))
 +(with-temp-buffer
 +  (insert query)
 +  (apply 'notmuch-call-notmuch-process-region

#'notmuch-call-notmuch-process-region ?

 +  (point-min) (point-max)
 +  tag (append tag-changes (list -- -
  (run-hooks 'notmuch-after-tag-hook))
;; in all cases we return tag-changes as a list
tag-changes)
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 6/7] cli: allow search mode to include msg-ids with JSON output

[Tomi Ollila]

On Sat, Nov 24 2012, markwalters1009 wrote:

 From: Mark Walters markwalters1...@gmail.com

 This adds a --queries=true option which modifies the summary output of
 notmuch search by including two extra query strings with each result:
 one query string specifies all matching messages and one query string
 all non-matching messages. Currently these are just lists of message
 ids joined with  or  but that could change in future.


Please see (mostly formatting) comments inline below.


 Currently this is not implemented for text format.
 ---
  notmuch-search.c |   95 ++---
  1 files changed, 89 insertions(+), 6 deletions(-)

 diff --git a/notmuch-search.c b/notmuch-search.c
 index 830c4e4..c8fc9a6 100644
 --- a/notmuch-search.c
 +++ b/notmuch-search.c
 @@ -26,7 +26,8 @@ typedef enum {
  OUTPUT_THREADS,
  OUTPUT_MESSAGES,
  OUTPUT_FILES,
 -OUTPUT_TAGS
 +OUTPUT_TAGS,
 +OUTPUT_SUMMARY_WITH_QUERIES
  } output_t;
  
  static char *
 @@ -46,6 +47,57 @@ sanitize_string (const void *ctx, const char *str)
  return out;
  }
  
 +/* This function takes a message id and returns an escaped string
 + * which can be used as a Xapian query. This involves prefixing with
 + * `id:', putting the id inside double quotes, and doubling any
 + * occurence of a double quote in the message id itself.*/
 +static char *
 +xapian_escape_id (const void *ctx,
 +const char *msg_id)

second line indentation, not at '(' level as elsewhere

 +{
 +const char *c;
 +char *escaped_msg_id;
 +escaped_msg_id = talloc_strdup (ctx, id:\);
 +for (c=msg_id; *c; c++)

spacing above

 + if (*c == '')
 + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \\);
 + else
 + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, %c, *c);
 +escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \);
 +return escaped_msg_id;
 +}

If Austin sees fit he can comment the O(...) of the addition of msgids
to query strings -- it would take quite an overhaul to the functionality
if the escaped msgid's were directly written to query string...

 +
 +static char *
 +output_msg_query (const void *ctx,
 + sprinter_t *format,
 + notmuch_bool_t matching,
 + notmuch_bool_t first,
 + notmuch_messages_t *messages)

indentation level above

 +{
 +notmuch_message_t *message;
 +char *query, *escaped_msg_id;
 +query = talloc_strdup (ctx, );
 +for (;
 +  notmuch_messages_valid (messages);
 +  notmuch_messages_move_to_next (messages))
 +{
 + message = notmuch_messages_get (messages);
 + if (notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_MATCH) == 
 matching) {
 + escaped_msg_id = xapian_escape_id (ctx, 
 notmuch_message_get_message_id (message));
 + if (first) {
 + query = talloc_asprintf_append (query, %s, escaped_msg_id);
 + first = FALSE;
 + }
 + else
 + query = talloc_asprintf_append (query,  or %s, 
 escaped_msg_id);
 + talloc_free (escaped_msg_id);
 + }
 + /* output_msg_query already starts with an ` or' */
 + query = talloc_asprintf_append (query, %s, output_msg_query (ctx, 
 format, matching, first, notmuch_message_get_replies (message)));
 +}
 +return query;
 +}
 +
  static int
  do_search_threads (sprinter_t *format,
  notmuch_query_t *query,
 @@ -88,7 +140,7 @@ do_search_threads (sprinter_t *format,
   format-string (format,
   notmuch_thread_get_thread_id (thread));
   format-separator (format);
 - } else { /* output == OUTPUT_SUMMARY */
 + } else { /* output == OUTPUT_SUMMARY or OUTPUT_SUMMARY_WITH_QUERIES */
   void *ctx_quote = talloc_new (thread);
   const char *authors = notmuch_thread_get_authors (thread);
   const char *subject = notmuch_thread_get_subject (thread);
 @@ -108,7 +160,7 @@ do_search_threads (sprinter_t *format,
   relative_date = notmuch_time_relative_date (ctx_quote, date);
  
   if (format-is_text_printer) {
 -/* Special case for the text formatter */
 +   /* Special case for the text formatter */

irrelevant spacing change

   printf (thread:%s %12s [%d/%d] %s; %s (,
   thread_id,
   relative_date,
 @@ -133,8 +185,6 @@ do_search_threads (sprinter_t *format,
   format-string (format, subject);
   }
  
 - talloc_free (ctx_quote);
 -
   format-map_key (format, tags);
   format-begin_list (format);
  
 @@ -145,7 +195,7 @@ do_search_threads (sprinter_t *format,
   const char *tag = notmuch_tags_get (tags);
  
   if (format-is_text_printer) {
 -  /* Special case for the text formatter */
 + /* Special case for the text formatter */

irrelevant spacing 

Re: [PATCH v2 1/7] cli: allow query to come from stdin

[Tomi Ollila]

On Sat, Nov 24 2012, markwalters1009 wrote:

 From: Mark Walters markwalters1...@gmail.com

 After this series there will be times when a caller will want to pass
 a very large query string to notmuch (eg a list of 10,000 message-ids)
 and this can exceed the size of ARG_MAX. Hence allow notmuch to take
 the query from stdin (if the query is -).


 ---
  query-string.c |   41 +
  1 files changed, 41 insertions(+), 0 deletions(-)

 diff --git a/query-string.c b/query-string.c
 index 6536512..b1fbdeb 100644
 --- a/query-string.c
 +++ b/query-string.c
 @@ -20,6 +20,44 @@
  
  #include notmuch-client.h
  
 +/* Read a single query string from STDIN, using
 + * 'ctx' as the talloc owner for all allocations.
 + *
 + * This function returns NULL in case of insufficient memory or read
 + * errors.
 + */
 +static char *
 +query_string_from_stdin (void *ctx)

Austin provided pretty nice alternative implementation of 
query_string_from_stdin() in his reply so I decline to
comment minor formatting issue below :D

 +{
 +char *query_string;
 +char buf[4096];
 +ssize_t remain;
 +
 +query_string = talloc_strdup (ctx, );
 +if (query_string == NULL)
 + return NULL;
 +
 +for (;;) {
 + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1);
 + if (remain == 0)
 + break;
 + if (remain  0) {
 + if (errno == EINTR)
 + continue;
 + fprintf (stderr, Error: reading from standard input: %s\n,
 +  strerror (errno));
 + return NULL;
 + }
 +
 + buf[remain] = '\0';
 + query_string = talloc_strdup_append (query_string, buf);
 + if (query_string == NULL)
 + return NULL;
 +}
 +
 +return query_string;
 +}
 +
  /* Construct a single query string from the passed arguments, using
   * 'ctx' as the talloc owner for all allocations.
   *
 @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[])
  char *query_string;
  int i;
  
 +if ((argc == 1)  (strcmp (-, argv[0]) == 0))

the argument order in strcmp() is not consistent with all the 
other uses of strcmp () in the codebase.

 + return query_string_from_stdin (ctx);
 +
  query_string = talloc_strdup (ctx, );
  if (query_string == NULL)
   return NULL;
 -- 
 1.7.9.1

Tomi
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [Patch v2 14/17] test: update dump-restore roundtripping test for batch-tag format

[Tomi Ollila]

On Sat, Nov 24 2012, da...@tethera.net wrote:

 From: David Bremner brem...@debian.org

 Now we can actually round trip these crazy tags and and message ids.
 hex-xcode is no longer needed as it's built in.
 ---
  test/dump-restore |   13 +++--
  1 file changed, 7 insertions(+), 6 deletions(-)

 diff --git a/test/dump-restore b/test/dump-restore
 index a2204fb..e08b656 100755
 --- a/test/dump-restore
 +++ b/test/dump-restore
 @@ -85,13 +85,14 @@ test_begin_subtest dump --output=outfile -- from:cworth
  notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
  test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
  
 +
  test_expect_success 'roundtripping random message-ids and tags' \
 -'test_subtest_known_broken 
 - ${TEST_DIRECTORY}/random-corpus --num-messages=10 
 --config-path=${NOTMUCH_CONFIG} 
 - notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
 EXPECTED.$test_count 
 +'${TEST_DIRECTORY}/random-corpus --num-messages=100 
 --config-path=${NOTMUCH_CONFIG} 
 + notmuch dump --format=batch-tag | sort  EXPECTED.$test_count 
   notmuch tag -random-corpus tag:random-corpus 
 - ${TEST_DIRECTORY}/hex-xcode --direction=decode  EXPECTED.$test_count | 
 notmuch restore 2/dev/null 
 - notmuch dump | ${TEST_DIRECTORY}/hex-xcode --direction=encode  
 OUTPUT.$test_count 
 + notmuch restore --format=batch-tag --input=EXPECTED.$test_count 
 + notmuch dump --format=batch-tag | sort  OUTPUT.$test_count 
   test_cmp EXPECTED.$test_count OUTPUT.$test_count'
 -
  test_done

In most cases there is empty line before `test_done` -- also other 
unrelated whitespace changes disturb review (especially when one is
tired ;)

 +
 +# Note the database is poisoned for sup format at this point.
 -- 
 1.7.10.4

Tomi
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [Patch v2 09/17] tag-util.[ch]: New files for common tagging routines

[Tomi Ollila]

On Sat, Nov 24 2012, da...@tethera.net wrote:

 From: David Bremner brem...@debian.org

 These are meant to be shared between notmuch-tag and notmuch-restore.

 The bulk of the routines implement a tag operation list abstract
 data type act as a structured representation of a set of tag
 operations (typically coming from a single tag command or line of
 input).
 ---

dumping all except beginning of tag-util.h...

  tag-util.h |  120 ++
 diff --git a/tag-util.h b/tag-util.h
 new file mode 100644
 index 000..508806f
 --- /dev/null
 +++ b/tag-util.h
 @@ -0,0 +1,120 @@
 +#ifndef _TAG_UTIL_H
 +#define _TAG_UTIL_H
 +
 +#include notmuch-client.h
 +
 +typedef struct _tag_operation_t tag_operation_t;
 +typedef struct _tag_op_list_t tag_op_list_t;
 +
 +#define TAG_OP_LIST_INITIAL_SIZE 10
 +
 +/* Use powers of 2 */
 +typedef enum  { TAG_FLAG_NONE = 0,
 + /* Operations are synced to maildir, if possible */
 +
 + TAG_FLAG_MAILDIR_SYNC = 1,
 +
 + /* Remove all tags from message before applying
 +  * list */
 +
 + TAG_FLAG_REMOVE_ALL = 2,
 +
 + /* Don't try to avoid database operations.  Useful
 +  * when we know that message passed needs these
 +  *  operations. */
 +
 + TAG_FLAG_PRE_OPTIMIZED = 4,
 +
 + /* Accept strange tags that might be user error;
 +intended for use by notmuch-restore.
 + */
 +
 + TAG_FLAG_BE_GENEROUS = 8} tag_op_flag_t;
 +

Maybe something like the following formatted and consistency-tuned version:

typedef enum { 
   TAG_FLAG_NONE = 0,

   /* Operations are synced to maildir, if possible.
*/
   TAG_FLAG_MAILDIR_SYNC = (1  0),

   /* Remove all tags from message before applying list.
*/
   TAG_FLAG_REMOVE_ALL = (1  1),

   /* Don't try to avoid database operations. Useful when we
* know that message passed needs these operations.
*/
   TAG_FLAG_PRE_OPTIMIZED = (1  2),

   /* Accept strange tags that might be user error;
* intended for use by notmuch-restore.
*/
   TAG_FLAG_BE_GENEROUS = (1  3)

} tag_op_flag_t;

Tomi
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH v2 6/7] cli: allow search mode to include msg-ids with JSON output

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
 From: Mark Walters markwalters1...@gmail.com
 
 This adds a --queries=true option which modifies the summary output of
 notmuch search by including two extra query strings with each result:
 one query string specifies all matching messages and one query string
 all non-matching messages. Currently these are just lists of message
 ids joined with  or  but that could change in future.
 
 Currently this is not implemented for text format.
 ---
  notmuch-search.c |   95 ++---
  1 files changed, 89 insertions(+), 6 deletions(-)
 
 diff --git a/notmuch-search.c b/notmuch-search.c
 index 830c4e4..c8fc9a6 100644
 --- a/notmuch-search.c
 +++ b/notmuch-search.c
 @@ -26,7 +26,8 @@ typedef enum {
  OUTPUT_THREADS,
  OUTPUT_MESSAGES,
  OUTPUT_FILES,
 -OUTPUT_TAGS
 +OUTPUT_TAGS,
 +OUTPUT_SUMMARY_WITH_QUERIES
  } output_t;
  
  static char *
 @@ -46,6 +47,57 @@ sanitize_string (const void *ctx, const char *str)
  return out;
  }
  
 +/* This function takes a message id and returns an escaped string
 + * which can be used as a Xapian query. This involves prefixing with
 + * `id:', putting the id inside double quotes, and doubling any
 + * occurence of a double quote in the message id itself.*/
 +static char *
 +xapian_escape_id (const void *ctx,
 +const char *msg_id)
 +{
 +const char *c;
 +char *escaped_msg_id;
 +escaped_msg_id = talloc_strdup (ctx, id:\);

talloc_strdup can fail.

 +for (c=msg_id; *c; c++)

Missing spaces around =.

 + if (*c == '')
 + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \\);
 + else
 + escaped_msg_id = talloc_asprintf_append (escaped_msg_id, %c, *c);

.. as can talloc_asprintf_append.

 +escaped_msg_id = talloc_asprintf_append (escaped_msg_id, \);
 +return escaped_msg_id;
 +}

Unfortunately, this approach will cause reallocation and copying for
every character in msg_id (as well as requiring talloc_asprintf_append
to re-scan escaped_msg_id to find the end on every iteration).  How
about pre-allocating a large enough buffer and keeping your position
in it, like

/* This function takes a message id and returns an escaped string
 * which can be used as a Xapian query. This involves prefixing with
 * `id:', putting the id inside double quotes, and doubling any
 * occurence of a double quote in the message id itself. Returns NULL
 * if memory allocation fails. */
static char *
xapian_escape_id (const void *ctx,
  const char *msg_id)
{
const char *in;
char *out;
char *escaped_msg_id = talloc_array (ctx, char, 6 + strlen (msg_id) * 2);
if (!escaped_msg_id)
return NULL;
strcpy (escaped_msg_id, id:\);
out = escaped_msg_id + 4;
for (in = msg_id; *in; ++in) {
if (*in == '')
*(out++) = '';
*(out++) = *in;
}
strcpy(out, \);
return escaped_msg_id;
}

 +
 +static char *
 +output_msg_query (const void *ctx,
 + sprinter_t *format,
 + notmuch_bool_t matching,
 + notmuch_bool_t first,
 + notmuch_messages_t *messages)
 +{
 +notmuch_message_t *message;
 +char *query, *escaped_msg_id;
 +query = talloc_strdup (ctx, );
 +for (;
 +  notmuch_messages_valid (messages);
 +  notmuch_messages_move_to_next (messages))
 +{
 + message = notmuch_messages_get (messages);
 + if (notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_MATCH) == 
 matching) {
 + escaped_msg_id = xapian_escape_id (ctx, 
 notmuch_message_get_message_id (message));

Two long lines.

 + if (first) {
 + query = talloc_asprintf_append (query, %s, escaped_msg_id);
 + first = FALSE;
 + }
 + else

} else.

 + query = talloc_asprintf_append (query,  or %s, 
 escaped_msg_id);

The or is unnecessary, since id is registered with the query parser
as an exclusive boolean term.

You could simplify this to

  query = talloc_asprintf_append (query, %s%s, first ?  :  , 
  escaped_msg_id);

Technically this loop has the same O(n^2) problem as xapian_escape_id
and query_string_from_args, but given that threads rarely have more
than a few dozen messages in them, perhaps it doesn't matter.  OTOH,
this may deal poorly with pathological threads (autogenerated messages
and such).

I wonder if we should have some simple linear-time talloc string
accumulation abstraction in util/...

 + talloc_free (escaped_msg_id);
 + }
 + /* output_msg_query already starts with an ` or' */
 + query = talloc_asprintf_append (query, %s, output_msg_query (ctx, 
 format, matching, first, notmuch_message_get_replies (message)));

Oof, how unfortunate.  I've got a patch that adds an iterator over all
of the messages in a thread, which would make this much simpler (I
don't know how we've gotten this far without such an 

Re: [PATCH v2 7/7] emacs: make emacs use message-ids for tagging

[Austin Clements]

Quoth markwalters1009 on Nov 24 at  1:20 pm:
 From: Mark Walters markwalters1...@gmail.com
 
 This makes emacs use the new --queries=true in search mode and use
 this for tagging.  This fixes the race condition in tagging from
 search mode so mark the tests fixed.
 ---
  emacs/notmuch.el |   28 +---
  test/emacs   |2 --
  2 files changed, 25 insertions(+), 5 deletions(-)
 
 diff --git a/emacs/notmuch.el b/emacs/notmuch.el
 index 64b9474..6e8ef83 100644
 --- a/emacs/notmuch.el
 +++ b/emacs/notmuch.el
 @@ -473,7 +473,8 @@ BEG.
(let (output)
  (notmuch-search-foreach-result beg end
(lambda (pos)
 - (push (plist-get (notmuch-search-get-result pos) property) output)))
 + (let ((value (plist-get (notmuch-search-get-result pos) property)))
 +   (when value (push value output)

Why is this necessary?  (Assuming it is, it could use a comment, and
probably an update to the docstring.)

  output))
  
  (defun notmuch-search-find-thread-id (optional bare)
 @@ -483,6 +484,7 @@ If BARE is set then do not prefix with \thread:\
(let ((thread (plist-get (notmuch-search-get-result) :thread)))
  (when thread (concat (unless bare thread:) thread
  
 +

Unintentional?

  (defun notmuch-search-find-thread-id-region (beg end)
Return a list of threads for the current region
(mapcar (lambda (thread) (concat thread: thread))
 @@ -492,6 +494,23 @@ If BARE is set then do not prefix with \thread:\
Return a search string for threads for the current region
(mapconcat 'identity (notmuch-search-find-thread-id-region beg end)  or 
 ))
  
 +;; The following two functions are similar to the previous two but
 +;; they only match messages that were in the the thread when the
 +;; initial search was run. This means that they can be used where it
 +;; is important to avoid races: e.g. when tagging.
 +(defun notmuch-search-find-queries-region (beg end optional only-matching)
 +  (interactive)
 +  Return a list of queries for the current region
 +  (append (notmuch-search-properties-in-region :matching_msg_query beg end)
 +   (unless only-matching
 + (notmuch-search-properties-in-region :nonmatching_msg_query beg 
 end

Two minor performance nits: Using nconc instead of append will avoid
copying the first list and swapping the two arguments will avoid
needlessly traversing the non-matching list when only-matching is nil.

 +
 +(defun notmuch-search-find-queries-region-search (beg end optional 
 only-matching)
 +  Return a search string for messages in threads in the current region
 +  (mapconcat 'identity

#'identity

 +  (notmuch-search-find-queries-region beg end only-matching)
 +   or ))
 +
  (defun notmuch-search-find-authors ()
Return the authors for the current thread
(plist-get (notmuch-search-get-result) :authors))
 @@ -575,7 +594,7 @@ and will also appear in a buffer named \*Notmuch 
 errors*\.
  
  (defun notmuch-search-tag-region (beg end optional tag-changes)
Change tags for threads in the given region.
 -  (let ((search-string (notmuch-search-find-thread-id-region-search beg 
 end)))
 +  (let ((search-string (notmuch-search-find-queries-region-search beg end)))
  (setq tag-changes (funcall 'notmuch-tag search-string tag-changes))
  (notmuch-search-foreach-result beg end
(lambda (pos)
 @@ -851,7 +870,9 @@ non-authors is found, assume that all of the authors 
 match.
  
  See `notmuch-tag' for information on the format of TAG-CHANGES.
(interactive)
 -  (apply 'notmuch-tag notmuch-search-query-string tag-changes))
 +  (apply 'notmuch-tag (notmuch-search-find-queries-region-search
 +(point-min) (point-max) t)
 +  tag-changes))
  
  (defun notmuch-search-buffer-title (query)
Returns the title for a buffer with notmuch search results.
 @@ -948,6 +969,7 @@ Other optional parameters are used as follows:
notmuch-search buffer
notmuch-command search
--format=json
 +  --output=with-queries

--output=with-queries or --queries=true?

(if oldest-first
--sort=oldest-first
  --sort=newest-first)
 diff --git a/test/emacs b/test/emacs
 index 3788439..132768f 100755
 --- a/test/emacs
 +++ b/test/emacs
 @@ -123,7 +123,6 @@ output=$(notmuch search $os_x_darwin_thread | 
 notmuch_search_sanitize)
  test_expect_equal $output thread:XXX   2009-11-18 [4/4] Jjgod Jiang, 
 Alexander Botero-Lowry; [notmuch] Mac OS X/Darwin compatibility issues (inbox 
 unread)
  
  test_begin_subtest Tag all matching messages from search view
 -test_subtest_known_broken
  notmuch tag +test-tag-race from:cworth
  test_emacs (notmuch-search \tag:test-tag-race\)
   (notmuch-test-wait)
 @@ -135,7 +134,6 @@ notmuch tag -test-tag-race '*'
  notmuch tag -test-tag-race-2 '*'
  
  test_begin_subtest Change tags from search view: another message arriving 
 

[PATCH] test: update dump-restore roundtripping test for batch-tag format

[david]

From: David Bremner brem...@debian.org

Now we can actually round trip these crazy tags and and message ids.
hex-xcode is no longer needed as it's built in.
---


I played with this a bit to try to make the diff nicer; I'm not sure
if I really improved much, but putting the sort into the original test
was a bug fix anyway. This is really for review; I don't think it will
apply without also resending 6/14.

 test/dump-restore |   14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/dump-restore b/test/dump-restore
index 28474f5..e9ba79d 100755
--- a/test/dump-restore
+++ b/test/dump-restore
@@ -86,18 +86,16 @@ notmuch dump --output=dump-outfile-dash-inbox.actual -- 
from:cworth
 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 
 test_expect_success 'roundtripping random message-ids and tags' \
-'test_subtest_known_broken 
- ${TEST_DIRECTORY}/random-corpus --num-messages=10 \
+ '${TEST_DIRECTORY}/random-corpus --num-messages=100 \
--config-path=${NOTMUCH_CONFIG} 
- notmuch dump |
-   ${TEST_DIRECTORY}/hex-xcode --direction=encode |
+ notmuch dump --format=batch-tag |
sort  EXPECTED.$test_count 
  notmuch tag -random-corpus tag:random-corpus 
- ${TEST_DIRECTORY}/hex-xcode --direction=decode  EXPECTED.$test_count |
-   notmuch restore 2/dev/null 
- notmuch dump |
-   ${TEST_DIRECTORY}/hex-xcode --direction=encode |
+ notmuch restore --format=batch-tag  EXPECTED.$test_count 
+ notmuch dump --format=batch-tag |
sort  OUTPUT.$test_count 
  test_cmp EXPECTED.$test_count OUTPUT.$test_count'
 
 test_done
+
+# Note the database is poisoned for sup format at this point.
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 00/20] insert command

[Peter Wang]

This series mainly addresses the issues raised by Mark:

- check talloc failures
- deadlock in maildir_open_tmp
- stricter file modes (0600 and 0700)
- shared tag operation parser with notmuch-tag.c
- simplified mkdir_parents
- trap SIGINT
- fsync after writing and rename
- added a couple of tests
- man page wording
- comments

Due to new restriction on tags beginning with '-', an argument beginning with
-- is no longer ambiguous so I have removed the optional -- separator
between options and tag operations.

Peter Wang (20):
  tag: factor out tag operation parsing
  tag: make tag operation parser public
  cli: add stub for insert command
  insert: open Maildir tmp file
  insert: copy stdin to Maildir tmp file
  insert: move file from Maildir tmp to new
  insert: add new message to database
  insert: support --folder option
  insert: prevent writes outside Maildir hierarchy
  insert: apply default tags to new message
  insert: parse command-line tag operations
  insert: apply command-line tag operations
  insert: add --create-folder option
  insert: fsync after writing tmp file
  insert: fsync new directory after rename
  insert: trap SIGINT and clean up
  insert: add copyright line from notmuch-deliver
  test: add tests for insert
  man: document 'insert' command
  man: reference notmuch-insert.1

 Makefile.local  |   1 +
 man/Makefile.local  |   1 +
 man/man1/notmuch-config.1   |   4 +-
 man/man1/notmuch-count.1|   4 +-
 man/man1/notmuch-dump.1 |   4 +-
 man/man1/notmuch-insert.1   |  60 +
 man/man1/notmuch-new.1  |   4 +-
 man/man1/notmuch-reply.1|   3 +-
 man/man1/notmuch-restore.1  |   3 +-
 man/man1/notmuch-search.1   |   3 +-
 man/man1/notmuch-show.1 |   3 +-
 man/man1/notmuch-tag.1  |   3 +-
 man/man1/notmuch.1  |   3 +-
 man/man5/notmuch-hooks.5|   4 +-
 man/man7/notmuch-search-terms.7 |   3 +-
 notmuch-client.h|  12 +
 notmuch-insert.c| 493 
 notmuch-tag.c   |  79 ---
 notmuch.c   |   3 +
 test/insert |  93 
 test/notmuch-test   |   1 +
 21 files changed, 733 insertions(+), 51 deletions(-)
 create mode 100644 man/man1/notmuch-insert.1
 create mode 100644 notmuch-insert.c
 create mode 100755 test/insert

-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 01/20] tag: factor out tag operation parsing

[Peter Wang]

Factor out parsing of +tag, -tag operations from argv
into a separate function.
---
 notmuch-tag.c | 66 +--
 1 file changed, 41 insertions(+), 25 deletions(-)

diff --git a/notmuch-tag.c b/notmuch-tag.c
index 88d559b..35a76db 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -167,11 +167,48 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 return interrupted;
 }
 
+/* Parse +tag and -tag operations between argv[i] and argv[argc-1].
+ * The array tag_ops must be at least argc - i elements long.
+ * Returns the index into argv where parsing stopped, or -1 on error. */
+static int
+parse_tag_operations (int i, int argc, char *argv[],
+ tag_operation_t *tag_ops, int *tag_ops_count)
+{
+*tag_ops_count = 0;
+
+for (; i  argc; i++) {
+   if (strcmp (argv[i], --) == 0) {
+   i++;
+   break;
+   }
+   if (argv[i][0] == '+' || argv[i][0] == '-') {
+   if (argv[i][0] == '+'  argv[i][1] == '\0') {
+   fprintf (stderr, Error: tag names cannot be empty.\n);
+   return -1;
+   }
+   if (argv[i][0] == '+'  argv[i][1] == '-') {
+   /* This disallows adding the non-removable tag - and
+* enables notmuch tag to take long options in the
+* future. */
+   fprintf (stderr, Error: tag names must not start with '-'.\n);
+   return -1;
+   }
+   tag_ops[*tag_ops_count].tag = argv[i] + 1;
+   tag_ops[*tag_ops_count].remove = (argv[i][0] == '-');
+   (*tag_ops_count)++;
+   } else {
+   break;
+   }
+}
+
+return i;
+}
+
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
 tag_operation_t *tag_ops;
-int tag_ops_count = 0;
+int tag_ops_count;
 char *query_string;
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
@@ -197,30 +234,9 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
return 1;
 }
 
-for (i = 0; i  argc; i++) {
-   if (strcmp (argv[i], --) == 0) {
-   i++;
-   break;
-   }
-   if (argv[i][0] == '+' || argv[i][0] == '-') {
-   if (argv[i][0] == '+'  argv[i][1] == '\0') {
-   fprintf (stderr, Error: tag names cannot be empty.\n);
-   return 1;
-   }
-   if (argv[i][0] == '+'  argv[i][1] == '-') {
-   /* This disallows adding the non-removable tag - and
-* enables notmuch tag to take long options in the
-* future. */
-   fprintf (stderr, Error: tag names must not start with '-'.\n);
-   return 1;
-   }
-   tag_ops[tag_ops_count].tag = argv[i] + 1;
-   tag_ops[tag_ops_count].remove = (argv[i][0] == '-');
-   tag_ops_count++;
-   } else {
-   break;
-   }
-}
+i = parse_tag_operations (0, argc, argv, tag_ops, tag_ops_count);
+if (i  0)
+   return 1;
 
 tag_ops[tag_ops_count].tag = NULL;
 
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 02/20] tag: make tag operation parser public

[Peter Wang]

Make the tag operation parser accessible outside notmuch-tag.c
so it can be reused by the upcoming insert command.
---
 notmuch-client.h |  9 +
 notmuch-tag.c| 17 ++---
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/notmuch-client.h b/notmuch-client.h
index ae9344b..a7c3df2 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -65,6 +65,11 @@ typedef GMimeCipherContext notmuch_crypto_context_t;
 #define STRINGIFY(s) STRINGIFY_(s)
 #define STRINGIFY_(s) #s
 
+typedef struct {
+const char *tag;
+notmuch_bool_t remove;
+} notmuch_tag_operation_t;
+
 typedef struct mime_node mime_node_t;
 struct sprinter;
 struct notmuch_show_params;
@@ -159,6 +164,10 @@ notmuch_cat_command (void *ctx, int argc, char *argv[]);
 int
 notmuch_config_command (void *ctx, int argc, char *argv[]);
 
+int
+parse_tag_operations (int i, int argc, char *argv[],
+ notmuch_tag_operation_t *tag_ops, int *tag_ops_count);
+
 const char *
 notmuch_time_relative_date (const void *ctx, time_t then);
 
diff --git a/notmuch-tag.c b/notmuch-tag.c
index 35a76db..831a0e4 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -54,14 +54,9 @@ _escape_tag (char *buf, const char *tag)
 return buf;
 }
 
-typedef struct {
-const char *tag;
-notmuch_bool_t remove;
-} tag_operation_t;
-
 static char *
 _optimize_tag_query (void *ctx, const char *orig_query_string,
-const tag_operation_t *tag_ops)
+const notmuch_tag_operation_t *tag_ops)
 {
 /* This is subtler than it looks.  Xapian ignores the '-' operator
  * at the beginning both queries and parenthesized groups and,
@@ -116,7 +111,7 @@ _optimize_tag_query (void *ctx, const char 
*orig_query_string,
  * element. */
 static int
 tag_query (void *ctx, notmuch_database_t *notmuch, const char *query_string,
-  tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
+  notmuch_tag_operation_t *tag_ops, notmuch_bool_t synchronize_flags)
 {
 notmuch_query_t *query;
 notmuch_messages_t *messages;
@@ -170,9 +165,9 @@ tag_query (void *ctx, notmuch_database_t *notmuch, const 
char *query_string,
 /* Parse +tag and -tag operations between argv[i] and argv[argc-1].
  * The array tag_ops must be at least argc - i elements long.
  * Returns the index into argv where parsing stopped, or -1 on error. */
-static int
+int
 parse_tag_operations (int i, int argc, char *argv[],
- tag_operation_t *tag_ops, int *tag_ops_count)
+ notmuch_tag_operation_t *tag_ops, int *tag_ops_count)
 {
 *tag_ops_count = 0;
 
@@ -207,7 +202,7 @@ parse_tag_operations (int i, int argc, char *argv[],
 int
 notmuch_tag_command (void *ctx, int argc, char *argv[])
 {
-tag_operation_t *tag_ops;
+notmuch_tag_operation_t *tag_ops;
 int tag_ops_count;
 char *query_string;
 notmuch_config_t *config;
@@ -228,7 +223,7 @@ notmuch_tag_command (void *ctx, int argc, char *argv[])
 
 /* Array of tagging operations (add or remove), terminated with an
  * empty element. */
-tag_ops = talloc_array (ctx, tag_operation_t, argc + 1);
+tag_ops = talloc_array (ctx, notmuch_tag_operation_t, argc + 1);
 if (tag_ops == NULL) {
fprintf (stderr, Out of memory.\n);
return 1;
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 03/20] cli: add stub for insert command

[Peter Wang]

The notmuch insert command should read a message from standard input
and deliver it to a Maildir folder, and then incorporate the message
into the notmuch database.  Essentially it moves the functionality of
notmuch-deliver into notmuch.

Though it could be used as an alternative to notmuch new, the reason
I want this is to allow my notmuch frontend to add postponed or sent
messages to the mail store and notmuch database, without resorting to
another tool (e.g. notmuch-deliver) nor directly modifying the maildir.
---
 Makefile.local   |  1 +
 notmuch-client.h |  3 +++
 notmuch-insert.c | 43 +++
 notmuch.c|  3 +++
 4 files changed, 50 insertions(+)
 create mode 100644 notmuch-insert.c

diff --git a/Makefile.local b/Makefile.local
index 2b91946..6bbd588 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -261,6 +261,7 @@ notmuch_client_srcs =   \
notmuch-config.c\
notmuch-count.c \
notmuch-dump.c  \
+   notmuch-insert.c\
notmuch-new.c   \
notmuch-reply.c \
notmuch-restore.c   \
diff --git a/notmuch-client.h b/notmuch-client.h
index a7c3df2..c0d0e93 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -135,6 +135,9 @@ int
 notmuch_dump_command (void *ctx, int argc, char *argv[]);
 
 int
+notmuch_insert_command (void *ctx, int argc, char *argv[]);
+
+int
 notmuch_new_command (void *ctx, int argc, char *argv[]);
 
 int
diff --git a/notmuch-insert.c b/notmuch-insert.c
new file mode 100644
index 000..21424cf
--- /dev/null
+++ b/notmuch-insert.c
@@ -0,0 +1,43 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2012 Peter Wang
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Peter Wang noval...@gmail.com
+ */
+
+#include notmuch-client.h
+
+int
+notmuch_insert_command (void *ctx, int argc, char *argv[])
+{
+notmuch_config_t *config;
+notmuch_database_t *notmuch;
+const char *db_path;
+
+config = notmuch_config_open (ctx, NULL, NULL);
+if (config == NULL)
+   return 1;
+
+db_path = notmuch_config_get_database_path (config);
+
+if (notmuch_database_open (notmuch_config_get_database_path (config),
+  NOTMUCH_DATABASE_MODE_READ_WRITE, notmuch))
+   return 1;
+
+notmuch_database_destroy (notmuch);
+
+return 1;
+}
diff --git a/notmuch.c b/notmuch.c
index 477a09c..86239fd 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -53,6 +53,9 @@ static command_t commands[] = {
 { new, notmuch_new_command,
   [options...],
   Find and import new messages to the notmuch database. },
+{ insert, notmuch_insert_command,
+  [options...] [--] [+tag|-tag ...]  message,
+  Add a new message into the maildir and notmuch database. },
 { search, notmuch_search_command,
   [options...] search-terms [...],
   Search for messages matching the given search terms. },
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 04/20] insert: open Maildir tmp file

[Peter Wang]

Open a unique file in the Maildir tmp directory.
The new message is not yet copied into the file.
---
 notmuch-insert.c | 105 ++-
 1 file changed, 104 insertions(+), 1 deletion(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 21424cf..371fb47 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -20,12 +20,107 @@
 
 #include notmuch-client.h
 
+#include sys/types.h
+#include sys/stat.h
+#include fcntl.h
+
+/* Like gethostname but guarantees that a null-terminated hostname is
+ * returned, even if it has to make one up.
+ * Returns true unless hostname contains a slash. */
+static notmuch_bool_t
+safe_gethostname (char *hostname, size_t len)
+{
+if (gethostname (hostname, len) == -1) {
+   strncpy (hostname, unknown, len);
+}
+hostname[len - 1] = '\0';
+
+return (strchr (hostname, '/') == NULL);
+}
+
+/* Open a unique file in the Maildir 'tmp' directory.
+ * Returns the file descriptor on success, or -1 on failure.
+ * On success, file paths into the 'tmp' and 'new' directories are returned
+ * via tmppath and newpath. */
+static int
+maildir_open_tmp_file (void *ctx, const char *dir,
+  char **tmppath, char **newpath)
+{
+pid_t pid;
+char hostname[256];
+struct timeval tv;
+char *filename;
+int fd = -1;
+
+/* We follow the Dovecot file name generation algorithm. */
+pid = getpid ();
+if (! safe_gethostname (hostname, sizeof (hostname))) {
+   fprintf (stderr, Error: invalid host name.\n);
+   return -1;
+}
+do {
+   gettimeofday (tv, NULL);
+   filename = talloc_asprintf (ctx, %ld.M%ldP%d.%s,
+   tv.tv_sec, tv.tv_usec, pid, hostname);
+   if (! filename) {
+   fprintf (stderr, Out of memory\n);
+   return -1;
+   }
+
+   *tmppath = talloc_asprintf (ctx, %s/tmp/%s, dir, filename);
+   if (! *tmppath) {
+   fprintf (stderr, Out of memory\n);
+   return -1;
+   }
+
+   fd = open (*tmppath, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600);
+} while (fd == -1  errno == EEXIST);
+
+if (fd == -1) {
+   fprintf (stderr, Error: opening %s: %s\n, *tmppath, strerror (errno));
+   return -1;
+}
+
+*newpath = talloc_asprintf (ctx, %s/new/%s, dir, filename);
+if (! *newpath) {
+   fprintf (stderr, Out of memory\n);
+   close (fd);
+   unlink (*tmppath);
+   return -1;
+}
+
+talloc_free (filename);
+
+return fd;
+}
+
+static notmuch_bool_t
+insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
+   const char *dir)
+{
+char *tmppath;
+char *newpath;
+int fdout;
+
+fdout = maildir_open_tmp_file (ctx, dir, tmppath, newpath);
+if (fdout  0) {
+   return FALSE;
+}
+
+/* For now we just delete the tmp file immediately. */
+close (fdout);
+unlink (tmppath);
+return FALSE;
+}
+
 int
 notmuch_insert_command (void *ctx, int argc, char *argv[])
 {
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
 const char *db_path;
+char *maildir;
+notmuch_bool_t ret;
 
 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
@@ -33,11 +128,19 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 
 db_path = notmuch_config_get_database_path (config);
 
+maildir = talloc_asprintf (ctx, %s, db_path);
+if (! maildir) {
+   fprintf (stderr, Out of memory\n);
+   return 1;
+}
+
 if (notmuch_database_open (notmuch_config_get_database_path (config),
   NOTMUCH_DATABASE_MODE_READ_WRITE, notmuch))
return 1;
 
+ret = insert_message (ctx, notmuch, STDIN_FILENO, maildir);
+
 notmuch_database_destroy (notmuch);
 
-return 1;
+return (ret) ? 0 : 1;
 }
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 05/20] insert: copy stdin to Maildir tmp file

[Peter Wang]

Read the new message from standard input into the Maildir tmp file.
---
 notmuch-insert.c | 51 +++
 1 file changed, 47 insertions(+), 4 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 371fb47..88e8533 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -94,6 +94,47 @@ maildir_open_tmp_file (void *ctx, const char *dir,
 return fd;
 }
 
+/* Copy the contents of fdin into fdout. */
+static notmuch_bool_t
+copy_fd_data (int fdin, int fdout)
+{
+char buf[4096];
+char *p;
+ssize_t remain;
+ssize_t written;
+
+for (;;) {
+   remain = read (fdin, buf, sizeof(buf));
+   if (remain == 0)
+   break;
+   if (remain  0) {
+   if (errno == EINTR)
+   continue;
+   fprintf (stderr, Error: reading from standard input: %s\n,
+strerror (errno));
+   return FALSE;
+   }
+
+   p = buf;
+   do {
+   written = write (fdout, p, remain);
+   if (written == 0)
+   return FALSE;
+   if (written  0) {
+   if (errno == EINTR)
+   continue;
+   fprintf (stderr, Error: writing to temporary file: %s,
+strerror (errno));
+   return FALSE;
+   }
+   p += written;
+   remain -= written;
+   } while (remain  0);
+}
+
+return TRUE;
+}
+
 static notmuch_bool_t
 insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
const char *dir)
@@ -101,16 +142,18 @@ insert_message (void *ctx, notmuch_database_t *notmuch, 
int fdin,
 char *tmppath;
 char *newpath;
 int fdout;
+notmuch_bool_t ret;
 
 fdout = maildir_open_tmp_file (ctx, dir, tmppath, newpath);
 if (fdout  0) {
return FALSE;
 }
-
-/* For now we just delete the tmp file immediately. */
+ret = copy_fd_data (fdin, fdout);
 close (fdout);
-unlink (tmppath);
-return FALSE;
+if (!ret) {
+   unlink (tmppath);
+}
+return ret;
 }
 
 int
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 06/20] insert: move file from Maildir tmp to new

[Peter Wang]

Atomically move the new message file from the Maildir 'tmp' directory
to 'new'.
---
 notmuch-insert.c | 21 +
 1 file changed, 21 insertions(+)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 88e8533..63a04dc 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -94,6 +94,24 @@ maildir_open_tmp_file (void *ctx, const char *dir,
 return fd;
 }
 
+/* Atomically move the new message file from the Maildir 'tmp' directory
+ * to the 'new' directory.
+ *
+ * We follow the Dovecot recommendation to simply use rename()
+ * instead of link() and unlink().  See also:
+ * http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_delivery
+ */
+static notmuch_bool_t
+maildir_move_tmp_to_new (const char *tmppath, const char *newpath)
+{
+if (rename (tmppath, newpath) != 0) {
+   fprintf (stderr, Error: rename() failed: %s\n, strerror (errno));
+   return FALSE;
+}
+
+return TRUE;
+}
+
 /* Copy the contents of fdin into fdout. */
 static notmuch_bool_t
 copy_fd_data (int fdin, int fdout)
@@ -150,6 +168,9 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int 
fdin,
 }
 ret = copy_fd_data (fdin, fdout);
 close (fdout);
+if (ret) {
+   ret = maildir_move_tmp_to_new (tmppath, newpath);
+}
 if (!ret) {
unlink (tmppath);
 }
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 07/20] insert: add new message to database

[Peter Wang]

Add the new message to the notmuch database, renaming the file to encode
notmuch tags as maildir flags.
---
 notmuch-insert.c | 49 -
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 63a04dc..020dc29 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -153,6 +153,44 @@ copy_fd_data (int fdin, int fdout)
 return TRUE;
 }
 
+/* Add the specified message file to the notmuch database.
+ * The file is renamed to encode notmuch tags as maildir flags. */
+static notmuch_bool_t
+add_file_to_database (notmuch_database_t *notmuch, const char *path)
+{
+notmuch_message_t *message;
+notmuch_status_t status;
+
+status = notmuch_database_add_message (notmuch, path, message);
+switch (status) {
+case NOTMUCH_STATUS_SUCCESS:
+   break;
+case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
+   fprintf (stderr, Warning: duplicate message.\n);
+   break;
+default:
+case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+case NOTMUCH_STATUS_OUT_OF_MEMORY:
+case NOTMUCH_STATUS_FILE_ERROR:
+case NOTMUCH_STATUS_NULL_POINTER:
+case NOTMUCH_STATUS_TAG_TOO_LONG:
+case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
+case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
+case NOTMUCH_STATUS_LAST_STATUS:
+   fprintf (stderr, Error: failed to add `%s' to notmuch database: %s\n,
+path, notmuch_status_to_string (status));
+   return FALSE;
+}
+
+notmuch_message_tags_to_maildir_flags (message);
+
+notmuch_message_destroy (message);
+
+return TRUE;
+}
+
 static notmuch_bool_t
 insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
const char *dir)
@@ -173,8 +211,17 @@ insert_message (void *ctx, notmuch_database_t *notmuch, 
int fdin,
 }
 if (!ret) {
unlink (tmppath);
+   return FALSE;
 }
-return ret;
+
+ret = add_file_to_database (notmuch, newpath);
+if (!ret) {
+   /* XXX maybe there should be an option to keep the file in maildir? */
+   unlink (newpath);
+   return FALSE;
+}
+
+return TRUE;
 }
 
 int
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 08/20] insert: support --folder option

[Peter Wang]

Allow the new message to be inserted into a folder within the Maildir
hierarchy instead of the top-level folder.
---
 notmuch-insert.c | 21 -
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 020dc29..a50eacc 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -230,16 +230,35 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
 const char *db_path;
+const char *folder = NULL;
 char *maildir;
+int opt_index;
 notmuch_bool_t ret;
 
+notmuch_opt_desc_t options[] = {
+   { NOTMUCH_OPT_STRING, folder, folder, 0, 0 },
+   { NOTMUCH_OPT_END, 0, 0, 0, 0 }
+};
+
+opt_index = parse_arguments (argc, argv, options, 1);
+
+if (opt_index  0) {
+   fprintf (stderr, Error: bad argument to notmuch insert: %s\n,
+argv[-opt_index]);
+   return 1;
+}
+
 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
return 1;
 
 db_path = notmuch_config_get_database_path (config);
 
-maildir = talloc_asprintf (ctx, %s, db_path);
+if (folder != NULL) {
+   maildir = talloc_asprintf (ctx, %s/%s, db_path, folder);
+} else {
+   maildir = talloc_asprintf (ctx, %s, db_path);
+}
 if (! maildir) {
fprintf (stderr, Out of memory\n);
return 1;
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 10/20] insert: apply default tags to new message

[Peter Wang]

Apply the new.tags to messages added by 'insert'.  This mirrors the
behaviour if the message were delivered by a separate tool followed by
'notmuch new'.
---
 notmuch-insert.c | 25 -
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 022f7cd..362da66 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -170,13 +170,15 @@ copy_fd_data (int fdin, int fdout)
 return TRUE;
 }
 
-/* Add the specified message file to the notmuch database.
+/* Add the specified message file to the notmuch database, applying tags.
  * The file is renamed to encode notmuch tags as maildir flags. */
 static notmuch_bool_t
-add_file_to_database (notmuch_database_t *notmuch, const char *path)
+add_file_to_database (notmuch_database_t *notmuch, const char *path,
+ const char **new_tags)
 {
 notmuch_message_t *message;
 notmuch_status_t status;
+int i;
 
 status = notmuch_database_add_message (notmuch, path, message);
 switch (status) {
@@ -201,6 +203,16 @@ add_file_to_database (notmuch_database_t *notmuch, const 
char *path)
return FALSE;
 }
 
+notmuch_message_freeze (message);
+
+/* Apply the new.tags, as would happen were the message added by
+ * 'notmuch new'. */
+for (i = 0; new_tags[i]; i++) {
+   notmuch_message_add_tag (message, new_tags[i]);
+}
+
+notmuch_message_thaw (message);
+
 notmuch_message_tags_to_maildir_flags (message);
 
 notmuch_message_destroy (message);
@@ -210,7 +222,7 @@ add_file_to_database (notmuch_database_t *notmuch, const 
char *path)
 
 static notmuch_bool_t
 insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
-   const char *dir)
+   const char *dir, const char **new_tags)
 {
 char *tmppath;
 char *newpath;
@@ -231,7 +243,7 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int 
fdin,
return FALSE;
 }
 
-ret = add_file_to_database (notmuch, newpath);
+ret = add_file_to_database (notmuch, newpath, new_tags);
 if (!ret) {
/* XXX maybe there should be an option to keep the file in maildir? */
unlink (newpath);
@@ -247,6 +259,8 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
 const char *db_path;
+const char **new_tags;
+size_t new_tags_length;
 const char *folder = NULL;
 char *maildir;
 int opt_index;
@@ -270,6 +284,7 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
return 1;
 
 db_path = notmuch_config_get_database_path (config);
+new_tags = notmuch_config_get_new_tags (config, new_tags_length);
 
 if (folder != NULL) {
if (! check_folder_name (folder)) {
@@ -289,7 +304,7 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
   NOTMUCH_DATABASE_MODE_READ_WRITE, notmuch))
return 1;
 
-ret = insert_message (ctx, notmuch, STDIN_FILENO, maildir);
+ret = insert_message (ctx, notmuch, STDIN_FILENO, maildir, new_tags);
 
 notmuch_database_destroy (notmuch);
 
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 11/20] insert: parse command-line tag operations

[Peter Wang]

Reuse the parser in notmuch-tag.c to parse +tag and -tag
operations on the 'insert' command-line.
---
 notmuch-insert.c | 23 +++
 1 file changed, 23 insertions(+)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 362da66..e4631a2 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -262,6 +262,8 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 const char **new_tags;
 size_t new_tags_length;
 const char *folder = NULL;
+notmuch_tag_operation_t *tag_ops;
+int tag_ops_count;
 char *maildir;
 int opt_index;
 notmuch_bool_t ret;
@@ -279,6 +281,27 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
return 1;
 }
 
+/* Array of tagging operations (add or remove), terminated with an
+ * empty element. */
+tag_ops = talloc_array (ctx, notmuch_tag_operation_t, argc - opt_index + 
1);
+if (tag_ops == NULL) {
+   fprintf (stderr, Out of memory.\n);
+   return 1;
+}
+
+opt_index = parse_tag_operations (opt_index, argc, argv,
+ tag_ops, tag_ops_count);
+if (opt_index  0)
+   return 1;
+
+tag_ops[tag_ops_count].tag = NULL;
+
+if (opt_index != argc) {
+   fprintf (stderr, Error: bad argument to notmuch insert: %s\n,
+argv[opt_index]);
+   return 1;
+}
+
 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
return 1;
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 12/20] insert: apply command-line tag operations

[Peter Wang]

Apply the +tag and -tag operations specified on the command-line.
---
 notmuch-insert.c | 19 +++
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index e4631a2..81a528c 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -174,7 +174,8 @@ copy_fd_data (int fdin, int fdout)
  * The file is renamed to encode notmuch tags as maildir flags. */
 static notmuch_bool_t
 add_file_to_database (notmuch_database_t *notmuch, const char *path,
- const char **new_tags)
+ const char **new_tags,
+ const notmuch_tag_operation_t *tag_ops)
 {
 notmuch_message_t *message;
 notmuch_status_t status;
@@ -211,6 +212,14 @@ add_file_to_database (notmuch_database_t *notmuch, const 
char *path,
notmuch_message_add_tag (message, new_tags[i]);
 }
 
+/* Apply the tags specified on the command-line. */
+for (i = 0; tag_ops[i].tag; i++) {
+   if (tag_ops[i].remove)
+   notmuch_message_remove_tag (message, tag_ops[i].tag);
+   else
+   notmuch_message_add_tag (message, tag_ops[i].tag);
+}
+
 notmuch_message_thaw (message);
 
 notmuch_message_tags_to_maildir_flags (message);
@@ -222,7 +231,8 @@ add_file_to_database (notmuch_database_t *notmuch, const 
char *path,
 
 static notmuch_bool_t
 insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
-   const char *dir, const char **new_tags)
+   const char *dir, const char **new_tags,
+   const notmuch_tag_operation_t *tag_ops)
 {
 char *tmppath;
 char *newpath;
@@ -243,7 +253,7 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int 
fdin,
return FALSE;
 }
 
-ret = add_file_to_database (notmuch, newpath, new_tags);
+ret = add_file_to_database (notmuch, newpath, new_tags, tag_ops);
 if (!ret) {
/* XXX maybe there should be an option to keep the file in maildir? */
unlink (newpath);
@@ -327,7 +337,8 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
   NOTMUCH_DATABASE_MODE_READ_WRITE, notmuch))
return 1;
 
-ret = insert_message (ctx, notmuch, STDIN_FILENO, maildir, new_tags);
+ret = insert_message (ctx, notmuch, STDIN_FILENO, maildir, new_tags,
+ tag_ops);
 
 notmuch_database_destroy (notmuch);
 
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 13/20] insert: add --create-folder option

[Peter Wang]

Support an option to create a new folder in the maildir.
---
 notmuch-insert.c | 92 
 1 file changed, 92 insertions(+)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 81a528c..b7aef95 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -55,6 +55,91 @@ check_folder_name (const char *folder)
 }
 }
 
+/* Make the given directory but succeed if it already exists. */
+static int
+mkdir_exists_ok (const char *path, int mode)
+{
+int ret;
+
+ret = mkdir (path, mode);
+if (ret == 0 || errno == EEXIST)
+   return 0;
+else
+   return ret;
+}
+
+/* Make the given directory including its parent directory as necessary.
+ * Return 0 on success, -1 on error. */
+static int
+mkdir_parents (char *path, int mode)
+{
+struct stat st;
+char *start;
+char *end;
+int ret;
+
+/* First check the common case: directory already exists. */
+if (stat (path, st) == 0)
+   return (S_ISDIR (st.st_mode)) ? 0 : -1;
+
+for (start = path; *start != '\0'; start = end + 1) {
+   /* start points to the first unprocessed character.
+* Find the next slash from start onwards. */
+   end = strchr (start, '/');
+
+   /* If there are no more slashes then all the parent directories
+* have been made.  Now attempt to make the whole path. */
+   if (end == NULL)
+   return mkdir_exists_ok (path, mode);
+
+   /* Make the path up to the next slash, unless the current
+* directory component is actually empty. */
+   if (end  start) {
+   *end = '\0';
+   ret = mkdir_exists_ok (path, mode);
+   *end = '/';
+   if (ret != 0)
+   return ret;
+   }
+}
+
+return 0;
+}
+
+/* Create the given maildir folder, i.e. dir and its subdirectories
+ * 'cur', 'new', 'tmp'. */
+static notmuch_bool_t
+maildir_create_folder (void *ctx, const char *dir)
+{
+const int mode = 0700;
+char *subdir;
+char *tail;
+
+/* Create 'cur' directory, including parent directories. */
+subdir = talloc_asprintf (ctx, %s/cur, dir);
+if (! subdir) {
+   fprintf (stderr, Out of memory.\n);
+   return FALSE;
+}
+if (mkdir_parents (subdir, mode) != 0)
+   return FALSE;
+
+tail = subdir + strlen (subdir) - 3;
+
+/* Create 'new' directory. */
+strcpy (tail, new);
+if (mkdir_exists_ok (subdir, mode) != 0)
+   return FALSE;
+
+/* Create 'tmp' directory. */
+strcpy (tail, tmp);
+if (mkdir_exists_ok (subdir, mode) != 0)
+   return FALSE;
+
+talloc_free (subdir);
+return TRUE;
+}
+
 /* Open a unique file in the Maildir 'tmp' directory.
  * Returns the file descriptor on success, or -1 on failure.
  * On success, file paths into the 'tmp' and 'new' directories are returned
@@ -272,6 +357,7 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 const char **new_tags;
 size_t new_tags_length;
 const char *folder = NULL;
+notmuch_bool_t create_folder = FALSE;
 notmuch_tag_operation_t *tag_ops;
 int tag_ops_count;
 char *maildir;
@@ -280,6 +366,7 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 
 notmuch_opt_desc_t options[] = {
{ NOTMUCH_OPT_STRING, folder, folder, 0, 0 },
+   { NOTMUCH_OPT_BOOLEAN, create_folder, create-folder, 0, 0 },
{ NOTMUCH_OPT_END, 0, 0, 0, 0 }
 };
 
@@ -332,6 +419,11 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
fprintf (stderr, Out of memory\n);
return 1;
 }
+if (create_folder  ! maildir_create_folder (ctx, maildir)) {
+   fprintf (stderr, Error: creating maildir %s: %s\n,
+maildir, strerror (errno));
+   return 1;
+}
 
 if (notmuch_database_open (notmuch_config_get_database_path (config),
   NOTMUCH_DATABASE_MODE_READ_WRITE, notmuch))
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 14/20] insert: fsync after writing tmp file

[Peter Wang]

Flush the tmp file to disk after writing for durability.
---
 notmuch-insert.c | 4 
 1 file changed, 4 insertions(+)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index b7aef95..f09c579 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -329,6 +329,10 @@ insert_message (void *ctx, notmuch_database_t *notmuch, 
int fdin,
return FALSE;
 }
 ret = copy_fd_data (fdin, fdout);
+if (ret  fsync (fdout) != 0) {
+   fprintf (stderr, Error: fsync failed: %s\n, strerror (errno));
+   ret = FALSE;
+}
 close (fdout);
 if (ret) {
ret = maildir_move_tmp_to_new (tmppath, newpath);
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

Re: [Patch v2 09/17] tag-util.[ch]: New files for common tagging routines

[David Bremner]

 Maybe something like the following formatted and consistency-tuned version:

 typedef enum { 
TAG_FLAG_NONE = 0,

/* Operations are synced to maildir, if possible.
 */
TAG_FLAG_MAILDIR_SYNC = (1  0),

/* Remove all tags from message before applying list.
 */
TAG_FLAG_REMOVE_ALL = (1  1),

/* Don't try to avoid database operations. Useful when we
 * know that message passed needs these operations.
 */
TAG_FLAG_PRE_OPTIMIZED = (1  2),

/* Accept strange tags that might be user error;
 * intended for use by notmuch-restore.
 */
TAG_FLAG_BE_GENEROUS = (1  3)

 } tag_op_flag_t;


Applied.  We may have to fight uncrustify on this, but we both know who
likes to play with uncrustify config ;).

d

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 15/20] insert: fsync new directory after rename

[Peter Wang]

After moving the file from the 'tmp' to the 'new' directory,
fsync on the 'new' directory for durability.
---
 notmuch-insert.c | 39 ---
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index f09c579..831b322 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -143,10 +143,10 @@ maildir_create_folder (void *ctx, const char *dir)
 /* Open a unique file in the Maildir 'tmp' directory.
  * Returns the file descriptor on success, or -1 on failure.
  * On success, file paths into the 'tmp' and 'new' directories are returned
- * via tmppath and newpath. */
+ * via tmppath and newpath, and the path of the 'new' directory in newdir. */
 static int
 maildir_open_tmp_file (void *ctx, const char *dir,
-  char **tmppath, char **newpath)
+  char **tmppath, char **newpath, char **newdir)
 {
 pid_t pid;
 char hostname[256];
@@ -183,8 +183,9 @@ maildir_open_tmp_file (void *ctx, const char *dir,
return -1;
 }
 
+*newdir = talloc_asprintf (ctx, %s/new, dir);
 *newpath = talloc_asprintf (ctx, %s/new/%s, dir, filename);
-if (! *newpath) {
+if (! *newdir || ! *newpath) {
fprintf (stderr, Out of memory\n);
close (fd);
unlink (*tmppath);
@@ -204,14 +205,31 @@ maildir_open_tmp_file (void *ctx, const char *dir,
  * http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_delivery
  */
 static notmuch_bool_t
-maildir_move_tmp_to_new (const char *tmppath, const char *newpath)
+maildir_move_tmp_to_new (const char *tmppath, const char *newpath,
+const char *newdir)
 {
+notmuch_bool_t ret;
+int fd;
+
 if (rename (tmppath, newpath) != 0) {
fprintf (stderr, Error: rename() failed: %s\n, strerror (errno));
return FALSE;
 }
 
-return TRUE;
+/* Sync the 'new' directory after rename for durability. */
+ret = TRUE;
+fd = open (newdir, O_RDONLY);
+if (fd == -1) {
+   fprintf (stderr, Error: open() dir failed: %s\n, strerror (errno));
+   ret = FALSE;
+}
+if (ret  fsync (fd) != 0) {
+   fprintf (stderr, Error: fsync() dir failed: %s\n, strerror (errno));
+   ret = FALSE;
+}
+if (fd != -1)
+   close (fd);
+return ret;
 }
 
 /* Copy the contents of fdin into fdout. */
@@ -307,6 +325,12 @@ add_file_to_database (notmuch_database_t *notmuch, const 
char *path,
 
 notmuch_message_thaw (message);
 
+/* notmuch_message_tags_to_maildir_flags may rename the message file
+ * once more, and does so without fsyncing the directory afterwards.
+ * rename() is atomic so after a crash the file should appear under
+ * the old or new name. notmuch new should be able to rename the file
+ * again if required, so another fsync is not required, I think.
+ */
 notmuch_message_tags_to_maildir_flags (message);
 
 notmuch_message_destroy (message);
@@ -321,10 +345,11 @@ insert_message (void *ctx, notmuch_database_t *notmuch, 
int fdin,
 {
 char *tmppath;
 char *newpath;
+char *newdir;
 int fdout;
 notmuch_bool_t ret;
 
-fdout = maildir_open_tmp_file (ctx, dir, tmppath, newpath);
+fdout = maildir_open_tmp_file (ctx, dir, tmppath, newpath, newdir);
 if (fdout  0) {
return FALSE;
 }
@@ -335,7 +360,7 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int 
fdin,
 }
 close (fdout);
 if (ret) {
-   ret = maildir_move_tmp_to_new (tmppath, newpath);
+   ret = maildir_move_tmp_to_new (tmppath, newpath, newdir);
 }
 if (!ret) {
unlink (tmppath);
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 16/20] insert: trap SIGINT and clean up

[Peter Wang]

The only potentially long-running part of the 'insert' command should be
copying stdin to the 'tmp' file.  If SIGINT is received during the
copying process, abort and clean up the file in 'tmp'.  At all other
points, just ignore the signal and continue.
---
 notmuch-insert.c | 27 +--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 831b322..28653ee 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -24,6 +24,21 @@
 #include sys/stat.h
 #include fcntl.h
 
+static volatile sig_atomic_t interrupted;
+
+static void
+handle_sigint (unused (int sig))
+{
+static char msg[] = Stopping... \n;
+
+/* This write is opportunistic, so it's okay to ignore the
+ * result.  It is not required for correctness, and if it does
+ * fail or produce a short write, we want to get out of the signal
+ * handler as quickly as possible, not retry it. */
+IGNORE_RESULT (write (2, msg, sizeof (msg) - 1));
+interrupted = 1;
+}
+
 /* Like gethostname but guarantees that a null-terminated hostname is
  * returned, even if it has to make one up.
  * Returns true unless hostname contains a slash. */
@@ -241,7 +256,7 @@ copy_fd_data (int fdin, int fdout)
 ssize_t remain;
 ssize_t written;
 
-for (;;) {
+while (! interrupted) {
remain = read (fdin, buf, sizeof(buf));
if (remain == 0)
break;
@@ -270,7 +285,7 @@ copy_fd_data (int fdin, int fdout)
} while (remain  0);
 }
 
-return TRUE;
+return ! interrupted;
 }
 
 /* Add the specified message file to the notmuch database, applying tags.
@@ -382,6 +397,7 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
 {
 notmuch_config_t *config;
 notmuch_database_t *notmuch;
+struct sigaction action;
 const char *db_path;
 const char **new_tags;
 size_t new_tags_length;
@@ -428,6 +444,13 @@ notmuch_insert_command (void *ctx, int argc, char *argv[])
return 1;
 }
 
+/* Setup our handler for SIGINT */
+memset (action, 0, sizeof (struct sigaction));
+action.sa_handler = handle_sigint;
+sigemptyset (action.sa_mask);
+action.sa_flags = SA_RESTART;
+sigaction (SIGINT, action, NULL);
+
 config = notmuch_config_open (ctx, NULL, NULL);
 if (config == NULL)
return 1;
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 17/20] insert: add copyright line from notmuch-deliver

[Peter Wang]

The 'insert' implementation was based partly on notmuch-deliver.
---
 notmuch-insert.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/notmuch-insert.c b/notmuch-insert.c
index 28653ee..5c65582 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -2,6 +2,9 @@
  *
  * Copyright © 2012 Peter Wang
  *
+ * Based in part on notmuch-deliver
+ * Copyright © 2010 Ali Polatel
+ *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 18/20] test: add tests for insert

[Peter Wang]

Add tests for new 'insert' command.
---
 test/insert   | 93 +++
 test/notmuch-test |  1 +
 2 files changed, 94 insertions(+)
 create mode 100755 test/insert

diff --git a/test/insert b/test/insert
new file mode 100755
index 000..d821a41
--- /dev/null
+++ b/test/insert
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+test_description='notmuch insert'
+. ./test-lib.sh
+
+# Create directories and database before inserting.
+mkdir -p $MAIL_DIR/{cur,new,tmp}
+mkdir -p $MAIL_DIR/Drafts/{cur,new,tmp}
+notmuch new  /dev/null
+
+# We use generate_message to create the temporary message file.
+# It happens to be in the mail directory already but that is okay.
+
+test_begin_subtest Insert message, default
+generate_message \
+[subject]=\insert-subject\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message\
+notmuch insert  $gen_msg_filename
+test_expect_equal `notmuch count subject:insert-subject tag:unread` 1
+
+test_begin_subtest Insert message, add tag
+generate_message \
+[subject]=\insert-subject-addtag\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-addtag\
+notmuch insert +custom  $gen_msg_filename
+test_expect_equal `notmuch count tag:custom` 1
+
+test_begin_subtest Insert message, add/remove tag
+generate_message \
+[subject]=\insert-subject-addrmtag\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-addrmtag\
+notmuch insert +custom -unread  $gen_msg_filename
+test_expect_equal `notmuch count tag:custom NOT tag:unread` 1
+
+test_begin_subtest Insert message, folder
+generate_message \
+[subject]=\insert-subject-draft\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-draft\
+notmuch insert --folder=Drafts  $gen_msg_filename
+test_expect_equal `notmuch count folder:Drafts` 1
+
+test_begin_subtest Insert message, folder and tags
+generate_message \
+[subject]=\insert-subject-draft\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-draft\
+notmuch insert --folder=Drafts +draft -unread  $gen_msg_filename
+test_expect_equal `notmuch count folder:Drafts tag:draft NOT tag:unread` 1
+
+test_begin_subtest Insert message, non-existent folder
+generate_message \
+[subject]=\insert-subject-nonexistfolder\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-nonexistfolder\
+notmuch insert --folder=nonesuch  $gen_msg_filename
+test_expect_equal $? 1
+
+test_begin_subtest Insert message, create folder
+generate_message \
+[subject]=\insert-subject-createfolder\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-createfolder\
+notmuch insert --folder=F --create-folder +folder  $gen_msg_filename
+test_expect_equal `notmuch count folder:F tag:folder` 1
+
+test_begin_subtest Insert message, create subfolder
+generate_message \
+[subject]=\insert-subject-createfolder\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-createfolder\
+notmuch insert --folder=F/G/H/I/J --create-folder +folder  $gen_msg_filename
+test_expect_equal `notmuch count folder:F/G/H/I/J tag:folder` 1
+
+test_begin_subtest Insert message, create existing subfolder
+generate_message \
+[subject]=\insert-subject-createfolder\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-createfolder\
+notmuch insert --folder=F/G/H/I/J --create-folder +folder  $gen_msg_filename
+test_expect_equal `notmuch count folder:F/G/H/I/J tag:folder` 2
+
+test_begin_subtest Insert message, create invalid subfolder
+generate_message \
+[subject]=\insert-subject-createinvalidfolder\ \
+[date]=\Sat, 01 Jan 2000 12:00:00 -\ \
+[body]=\insert-message-createinvalidfolder\
+notmuch insert --folder=../G --create-folder  $gen_msg_filename
+test_expect_equal $? 1
+
+test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 9a1b375..09be44c 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -22,6 +22,7 @@ TESTS=
   config
   new
   count
+  insert
   search
   search-output
   search-by-folder
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 19/20] man: document 'insert' command

[Peter Wang]

Add initial documentation for notmuch insert command.
---
 man/Makefile.local|  1 +
 man/man1/notmuch-insert.1 | 60 +++
 2 files changed, 61 insertions(+)
 create mode 100644 man/man1/notmuch-insert.1

diff --git a/man/Makefile.local b/man/Makefile.local
index 72e2a18..216aaa0 100644
--- a/man/Makefile.local
+++ b/man/Makefile.local
@@ -12,6 +12,7 @@ MAN1 := \
$(dir)/man1/notmuch-count.1 \
$(dir)/man1/notmuch-dump.1 \
$(dir)/man1/notmuch-restore.1 \
+   $(dir)/man1/notmuch-insert.1 \
$(dir)/man1/notmuch-new.1 \
$(dir)/man1/notmuch-reply.1 \
$(dir)/man1/notmuch-search.1 \
diff --git a/man/man1/notmuch-insert.1 b/man/man1/notmuch-insert.1
new file mode 100644
index 000..8b1c4e9
--- /dev/null
+++ b/man/man1/notmuch-insert.1
@@ -0,0 +1,60 @@
+.TH NOTMUCH-INSERT 1 2012-xx-xx Notmuch 0.xx
+.SH NAME
+notmuch-insert \- add a message to the maildir and notmuch database
+.SH SYNOPSIS
+
+.B notmuch insert
+.RI [ options ]
+.RI [ + tag |\- tag  ... ]
+
+.SH DESCRIPTION
+
+.B notmuch insert
+reads a message from standard input
+and delivers it to the specified maildir folder,
+then incorporates the message into the notmuch database.
+It is an alternative to using a separate tool to deliver
+the message then running
+.B notmuch new
+afterwards.
+
+The new message will be tagged with the tags specified by the
+.B new.tags
+configuration option.
+Additional tagging operations may be specified on the command-line:
+tags prefixed by '+' are added while
+those prefixed by '\-' are removed.
+
+Option arguments must appear before any tag operation arguments.
+Supported options for
+.B insert
+include
+.RS 4
+.TP 4
+.BI --folder= folder 
+
+Deliver the message to the specified folder,
+relative to the top-level directory given by the value of
+\fBdatabase.path\fR.
+The default is to deliver to the top-level directory.
+
+.RE
+
+.RS 4
+.TP 4
+.B --create-folder
+
+Try to create the folder named by the
+.B --folder
+option, if it does not exist.
+Otherwise the folder must already exist for mail
+delivery to succeed.
+
+.RE
+.SH SEE ALSO
+
+\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
+\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-reply\fR(1),
+\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
+\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
+\fBnotmuch-tag\fR(1)
-- 
1.7.12.1

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 20/20] man: reference notmuch-insert.1

[Peter Wang]

Add references to notmuch-insert.1 from other man pages.
---
 man/man1/notmuch-config.1   | 4 ++--
 man/man1/notmuch-count.1| 4 ++--
 man/man1/notmuch-dump.1 | 4 ++--
 man/man1/notmuch-new.1  | 4 ++--
 man/man1/notmuch-reply.1| 3 ++-
 man/man1/notmuch-restore.1  | 3 ++-
 man/man1/notmuch-search.1   | 3 ++-
 man/man1/notmuch-show.1 | 3 ++-
 man/man1/notmuch-tag.1  | 3 ++-
 man/man1/notmuch.1  | 3 ++-
 man/man5/notmuch-hooks.5| 4 ++--
 man/man7/notmuch-search-terms.7 | 3 ++-
 12 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/man/man1/notmuch-config.1 b/man/man1/notmuch-config.1
index 10c9fd6..d526627 100644
--- a/man/man1/notmuch-config.1
+++ b/man/man1/notmuch-config.1
@@ -152,7 +152,7 @@ use ${HOME}/.notmuch\-config if this variable is not set.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-count\fR(1), \fBnotmuch-dump\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1), \fBnotmuch-reply\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
+\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
+\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-count.1 b/man/man1/notmuch-count.1
index 9b55e7c..4161324 100644
--- a/man/man1/notmuch-count.1
+++ b/man/man1/notmuch-count.1
@@ -52,7 +52,7 @@ count (the default) or not.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-dump\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1), \fBnotmuch-reply\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
+\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
+\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
index 230deec..109fd26 100644
--- a/man/man1/notmuch-dump.1
+++ b/man/man1/notmuch-dump.1
@@ -30,7 +30,7 @@ for details of the supported syntax for search-terms.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1), \fBnotmuch-reply\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
+\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
+\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-new.1 b/man/man1/notmuch-new.1
index 0acfeac..af2daff 100644
--- a/man/man1/notmuch-new.1
+++ b/man/man1/notmuch-new.1
@@ -64,7 +64,7 @@ Prevents hooks from being run.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-reply\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
+\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1),
+\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-reply.1 b/man/man1/notmuch-reply.1
index d264060..020e252 100644
--- a/man/man1/notmuch-reply.1
+++ b/man/man1/notmuch-reply.1
@@ -96,7 +96,8 @@ replying to multiple messages at once, but the JSON format 
does not.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1),
+\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
+\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
 \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
index 2fa8733..c23ea2c 100644
--- a/man/man1/notmuch-restore.1
+++ b/man/man1/notmuch-restore.1
@@ -39,7 +39,8 @@ details.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1),
+\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
+\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
 \fBnotmuch-reply\fR(1), \fBnotmuch-search\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 \fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-search.1 b/man/man1/notmuch-search.1
index 6ccd3b8..8af2a71 100644
--- a/man/man1/notmuch-search.1
+++ b/man/man1/notmuch-search.1
@@ -128,7 +128,8 @@ is the number of matching non-excluded messages in the 
thread.
 .SH SEE ALSO
 
 \fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-new\fR(1),
+\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
+\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
 \fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
 \fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
 

Re: [PATCH 1/1] test: always source test-lib.sh as ./test-lib.sh

[David Bremner]

Tomi Ollila tomi.oll...@iki.fi writes:


 Changed the 9 files to execute '. ./test-lib.sh'. The test-lib.sh
 should never be loaded from directory in PATH.

Pushed.

d
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 0/6] API for iterating over all messages in a thread

[Austin Clements]

This series adds a library API for iterating over all messages in a
thread in sorted order.  This is easy for the library to provide and
difficult to obtain from the current API.  Plus, if you don't count
the code added to the bindings, this series is actually a net
decrease of 4 lines of code because of simplifications it enables.

Do we want the API to do more?  Currently it's very minimal, but I can
imagine two ways it could be generalized.  It could take an argument
to indicate which message list to return, which could be all messages,
matched messages, top-level messages, or maybe even unmatched messages
(possibly all in terms of message flags).  It could also take an
argument indicating the desired sort order.  Currently, the caller can
use existing message flag APIs to distinguish matched and unmatched
messages and there's a separate function for the top-level messages.
However, if the API could do all of these things, it would subsume
various other API functions, such as notmuch_thread_get_*_date.

Also, is this the right name for the new API?  In particular, if we do
later want to add a function that returns, say, the list of matched
messages, we'll have a convention collision with
notmuch_thread_get_matched_messages, which returns only a count.

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/6] lib: Clean up error handling in _notmuch_thread_create

[Austin Clements]

Previously, there were various opportunities for memory leaks in the
error-handling paths of this function.  Use a local talloc context and
some reparenting to make eliminate these leaks, while keeping the
control flow simple.
---
 lib/thread.cc |   33 +++--
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/lib/thread.cc b/lib/thread.cc
index e976d64..aed87b1 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -406,7 +406,8 @@ _notmuch_thread_create (void *ctx,
notmuch_string_list_t *exclude_terms,
notmuch_sort_t sort)
 {
-notmuch_thread_t *thread;
+void *local = talloc_new (ctx);
+notmuch_thread_t *thread = NULL;
 notmuch_message_t *seed_message;
 const char *thread_id;
 char *thread_id_query_string;
@@ -415,24 +416,23 @@ _notmuch_thread_create (void *ctx,
 notmuch_messages_t *messages;
 notmuch_message_t *message;
 
-seed_message = _notmuch_message_create (ctx, notmuch, seed_doc_id, NULL);
+seed_message = _notmuch_message_create (local, notmuch, seed_doc_id, NULL);
 if (! seed_message)
INTERNAL_ERROR (Thread seed message %u does not exist, seed_doc_id);
 
 thread_id = notmuch_message_get_thread_id (seed_message);
-thread_id_query_string = talloc_asprintf (ctx, thread:%s, thread_id);
+thread_id_query_string = talloc_asprintf (local, thread:%s, thread_id);
 if (unlikely (thread_id_query_string == NULL))
-   return NULL;
+   goto DONE;
 
-thread_id_query = notmuch_query_create (notmuch, thread_id_query_string);
+thread_id_query = talloc_steal (
+   local, notmuch_query_create (notmuch, thread_id_query_string));
 if (unlikely (thread_id_query == NULL))
-   return NULL;
+   goto DONE;
 
-talloc_free (thread_id_query_string);
-
-thread = talloc (ctx, notmuch_thread_t);
+thread = talloc (local, notmuch_thread_t);
 if (unlikely (thread == NULL))
-   return NULL;
+   goto DONE;
 
 talloc_set_destructor (thread, _notmuch_thread_destructor);
 
@@ -451,8 +451,10 @@ _notmuch_thread_create (void *ctx,
  free, NULL);
 
 thread-message_list = _notmuch_message_list_create (thread);
-if (unlikely (thread-message_list == NULL))
-   return NULL;
+if (unlikely (thread-message_list == NULL)) {
+   thread = NULL;
+   goto DONE;
+}
 
 thread-message_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
  free, NULL);
@@ -489,12 +491,15 @@ _notmuch_thread_create (void *ctx,
_notmuch_message_close (message);
 }
 
-notmuch_query_destroy (thread_id_query);
-
 _resolve_thread_authors_string (thread);
 
 _resolve_thread_relationships (thread);
 
+/* Commit to returning thread. */
+talloc_steal (ctx, thread);
+
+  DONE:
+talloc_free (local);
 return thread;
 }
 
-- 
1.7.10.4

___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch