[PATCH v3 3/3] Use the structured format printer in notmuch search.
This patch uses the previously introduced structured format printer in
notmuch-search.
Most of this patch is necessary to keep the text output unchanged. All tests
pass, both text and JSON are printed exactly the same as they were before.
---
notmuch-search.c | 275 +-
1 file changed, 148 insertions(+), 127 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..321b32a 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -19,6 +19,7 @@
*/
#include "notmuch-client.h"
+#include "structured-output.h"
typedef enum {
OUTPUT_SUMMARY,
@@ -28,6 +29,7 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* legacy, only needed for non-structured text output */
typedef struct search_format {
const char *results_start;
const char *item_start;
@@ -64,6 +66,7 @@ format_thread_text (const void *ctx,
const int total,
const char *authors,
const char *subject);
+
static const search_format_t format_text = {
"",
"",
@@ -78,35 +81,6 @@ static const search_format_t format_text = {
};
static void
-format_item_id_json (const void *ctx,
-const char *item_type,
-const char *item_id);
-
-static void
-format_thread_json (const void *ctx,
- const char *thread_id,
- const time_t date,
- const int matched,
- const int total,
- const char *authors,
- const char *subject);
-
-/* Any changes to the JSON format should be reflected in the file
- * devel/schemata. */
-static const search_format_t format_json = {
-"[",
- "{",
- format_item_id_json,
- format_thread_json,
- "\"tags\": [",
- "\"%s\"", ", ",
- "]", ",\n",
- "}",
-"]\n",
-"]\n",
-};
-
-static void
format_item_id_text (unused (const void *ctx),
const char *item_type,
const char *item_id)
@@ -153,50 +127,9 @@ format_thread_text (const void *ctx,
talloc_free (ctx_quote);
}
-static void
-format_item_id_json (const void *ctx,
-unused (const char *item_type),
-const char *item_id)
-{
-void *ctx_quote = talloc_new (ctx);
-
-printf ("%s", json_quote_str (ctx_quote, item_id));
-
-talloc_free (ctx_quote);
-
-}
-
-static void
-format_thread_json (const void *ctx,
- const char *thread_id,
- const time_t date,
- const int matched,
- const int total,
- const char *authors,
- const char *subject)
-{
-void *ctx_quote = talloc_new (ctx);
-
-printf ("\"thread\": %s,\n"
- "\"timestamp\": %ld,\n"
- "\"date_relative\": \"%s\",\n"
- "\"matched\": %d,\n"
- "\"total\": %d,\n"
- "\"authors\": %s,\n"
- "\"subject\": %s,\n",
- json_quote_str (ctx_quote, thread_id),
- date,
- notmuch_time_relative_date (ctx, date),
- matched,
- total,
- json_quote_str (ctx_quote, authors),
- json_quote_str (ctx_quote, subject));
-
-talloc_free (ctx_quote);
-}
-
static int
-do_search_threads (const search_format_t *format,
+do_search_threads (const structure_printer_t *format,
+ void *state,
notmuch_query_t *query,
notmuch_sort_t sort,
output_t output,
@@ -209,6 +142,8 @@ do_search_threads (const search_format_t *format,
time_t date;
int first_thread = 1;
int i;
+int outermost_level = 0;
+int items_level = 0;
if (offset < 0) {
offset += notmuch_query_count_threads (query);
@@ -220,7 +155,11 @@ do_search_threads (const search_format_t *format,
if (threads == NULL)
return 1;
-fputs (format->results_start, stdout);
+if (format == unstructured_text_printer) {
+ fputs(format_text.results_start, stdout);
+} else { /* structured output */
+ outermost_level = format->list(state);
+}
for (i = 0;
notmuch_threads_valid (threads) && (limit < 0 || i < offset + limit);
@@ -235,43 +174,85 @@ do_search_threads (const search_format_t *format,
continue;
}
- if (! first_thread)
- fputs (format->item_sep, stdout);
+ if (format == unstructured_text_printer && ! first_thread)
+ fputs (format_text.item_sep, stdout);
if (output == OUTPUT_THREADS) {
- format->item_id (thread, "thread:",
-notmuch_thread_get_thread_id (thread));
+ if (format == unstructured_text_printer) {
+ format_text.item_id (thread, "thread:",
+notmuch_thread_get_thread_id (thread));
+ } else { /*
[PATCH v3 2/3] Adding structured output formatter for JSON.
This formatter prints exactly the same structure as the existing JSON
formatter, but uses the newly introduced structured formatting primitives.
---
structured-output.c | 159 +++
structured-output.h | 48
2 files changed, 207 insertions(+)
create mode 100644 structured-output.c
diff --git a/structured-output.c b/structured-output.c
new file mode 100644
index 000..e10fba4
--- /dev/null
+++ b/structured-output.c
@@ -0,0 +1,159 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright ? 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth
+ */
+
+#include "structured-output.h"
+
+structure_printer_t *
+unstructured_text_printer = NULL;
+
+structure_printer_t
+json_structure_printer = {
+_map,
+_list,
+_pop,
+_map_key,
+_number,
+_string,
+_bool,
+_initial_state
+};
+static int
+enter_level(void *st, const char *marker, int type) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if (state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(",", output);
+ if (state->level == 1)
+ fputs("\n", output);
+ else
+ fputs(" ", output);
+}
+if (state->stack != NULL) {
+ state->stack->first_already_seen = TRUE;
+}
+fputs(marker, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el->type = type;
+el->first_already_seen = FALSE;
+el->rest = state->stack;
+state->stack = el;
+return state->level++;
+}
+
+int
+json_map(void *st)
+{
+return enter_level(st, "{", TYPE_JSON_MAP);
+}
+
+int
+json_list(void *st)
+{
+return enter_level(st, "[", TYPE_JSON_ARRAY);
+}
+
+void
+json_pop(void *st, int level)
+{
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+for (i = state->level; i > level; i--) {
+ json_list_t *tos = state->stack;
+ if (tos->type == TYPE_JSON_MAP) {
+ fputs("}", output);
+ }
+ if (tos->type == TYPE_JSON_ARRAY) {
+ fputs("]", output);
+ }
+ state->stack = tos->rest;
+ state->level--;
+ talloc_free(tos);
+}
+if (state->level == 0)
+ fputs("\n", output);
+}
+
+void
+json_map_key(void *st, const char *key)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if (state->stack != NULL && state->stack->first_already_seen) {
+ fputs(",\n", output);
+}
+fputs("\"", output);
+fputs(key, output);
+fputs("\": ", output);
+}
+
+void
+json_number(void *st, int val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if (state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(", ", output);
+}
+state->stack->first_already_seen = TRUE;
+fprintf(output, "%i", val);
+}
+
+void
+json_string(void *st, const char *val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+void *ctx = talloc_new(0);
+if (state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(",", output);
+ if (state->level == 1)
+ fputs("\n", output);
+ else
+ fputs(" ", output);
+}
+
+state->stack->first_already_seen = TRUE;
+fprintf(output, "%s", json_quote_str(ctx, val));
+talloc_free(ctx);
+}
+
+void
+json_bool(void *st, notmuch_bool_t val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if (val)
+ fputs("true", output);
+else
+ fputs("false", output);
+}
+
+void *
+json_initial_state(const struct structure_printer *sp, FILE *output)
+{
+(void)sp;
+json_state_t *st = talloc(0, json_state_t);
+st->level = 0;
+st->stack = NULL;
+st->output = output;
+return st;
+}
diff --git a/structured-output.h b/structured-output.h
index b43afe0..cba3882 100644
--- a/structured-output.h
+++ b/structured-output.h
@@ -62,3 +62,51 @@ typedef struct structure_printer {
} structure_printer_t;
+/* JSON structure printer */
+
+/* single linked list
[PATCH v3 1/3] Add support for structured output formatters.
This patch adds a new type structure_printer, which is used for structured
formatting, e.g. JSON or S-Expressions.
The structure contains the following function pointers:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use state to insert delimiters etc. automatically when
appropriate.
Example:
int top, one;
top = map(state);
map_key(state, "foo");
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, one);
map_key(state, "bar");
map(state);
map_key(state, "baaz");
string(state, "hello world");
pop(state, top);
would output JSON as follows:
{"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
---
Makefile.local |1 +
structured-output.h | 64 +++
2 files changed, 65 insertions(+)
create mode 100644 structured-output.h
diff --git a/Makefile.local b/Makefile.local
index a890df2..9b3db5e 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -286,6 +286,7 @@ notmuch_client_srcs = \
notmuch-reply.c \
notmuch-restore.c \
notmuch-search.c\
+ structured-output.c \
notmuch-setup.c \
notmuch-show.c \
notmuch-tag.c \
diff --git a/structured-output.h b/structured-output.h
new file mode 100644
index 000..b43afe0
--- /dev/null
+++ b/structured-output.h
@@ -0,0 +1,64 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright ? 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth
+ */
+
+#include "notmuch-client.h"
+
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, "foo");
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
+pop(state, one);
+map_key(state, "bar");
+map(state);
+map_key(state, "baaz");
+string(state, "hello world");
+pop(state, top);
+
+would output JSON as follows:
+
+{"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
+
+ */
+typedef struct structure_printer {
+
[PATCH] v2: Added better support for multiple structured output formats.
On Tue, 10 Jul 2012, craven at gmx.net wrote:
> As discussed in , this patch adds
> support for new structured output formats (like s-expressions) by using
> stateful structure_printers. An implementation of the JSON structure
> printer that passes all tests is included. The output for JSON (and
> text) is identical to the current output. S-Expressions will be added in
> a later patch.
>
Hi I have some more comments inline below. I agree with Jamie that it
would be nice to split it into smaller chunks but I am not sure that is
practical to do.
> A large part of this patch just implements the differentiation between
> structured and non-structured output (all the code within
> "if(format == unstructured_text_printer)").
I also agree with Jamie that the text output is annoying. However, at
least until Austin's async json parser goes in even the emacs interface
is using it so I think we have to live with it for a while (at the very
least we would need to give clients a reasonable time to migrate).
Anyway, I think this patch has to support it.
> In a second patch, the structured output code should be isolated in a
> separate file, and also used in all other parts of notmuch.
>
> The interface is a structure structure_printer, which contains the following
> methods:
>
> - initial_state: is called to create a state object, that is passed to all
> invocations. This should be used to keep track of the output file and
> everything else necessary to correctly format output.
> - map: is called when a new map (associative array, dictionary) is started.
> map_key and the primitives (string, number, bool) are used alternatingly to
> add key/value pairs. pop is used to close the map (see there). This function
> must return a nesting level identifier that can be used to close all nested
> structures (maps and lists), backing out to the returned nesting level.
> - list: is called when a new list (array, vector) is started. the primitives
> (string, number, bool) are used consecutively to add values to the list. pop
> is used to close the list. This function must return a nesting level
> identifier that can be used to close all nested structures (maps and lists),
> backing out to the returned nesting level.
> - pop: is called to return to a given nesting level. All lists and maps with
> a deeper nesting level must be closed.
> - number, string, bool: output one element of the specific type.
>
> All functions should use the state object to insert delimiters etc.
> automatically when appropriate.
>
> Example:
> int top, one;
> top = map(state);
> map_key(state, "foo");
> one = list(state);
> number(state, 1);
> number(state, 2);
> number(state, 3);
> pop(state, i);
> map_key(state, "bar");
> map(state);
> map_key(state, "baaz");
> string(state, "hello world");
> pop(state, top);
>
> would output JSON as follows:
>
> {"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
> ---
> notmuch-search.c | 491
> ---
> 1 file changed, 361 insertions(+), 130 deletions(-)
>
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 3be296d..412 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -28,6 +28,210 @@ typedef enum {
> OUTPUT_TAGS
> } output_t;
>
> +/* structured formatting, useful for JSON, S-Expressions, ...
> +
> +- initial_state: is called to create a state object, that is passed to all
> invocations. This should be used to keep track of the output file and
> everything else necessary to correctly format output.
> +- map: is called when a new map (associative array, dictionary) is started.
> map_key and the primitives (string, number, bool) are used alternatingly to
> add key/value pairs. pop is used to close the map (see there). This function
> must return a nesting level identifier that can be used to close all nested
> structures (maps and lists), backing out to the returned nesting level.
> +- list: is called when a new list (array, vector) is started. the primitives
> (string, number, bool) are used consecutively to add values to the list. pop
> is used to close the list. This function must return a nesting level
> identifier that can be used to close all nested structures (maps and lists),
> backing out to the returned nesting level.
> +- pop: is called to return to a given nesting level. All lists and maps with
> a deeper nesting level must be closed.
> +- number, string, bool: output one element of the specific type.
> +
> +All functions should use state to insert delimiters etc. automatically when
> appropriate.
> +
> +Example:
> +int top, one;
> +top = map(state);
> +map_key(state, "foo");
> +one = list(state);
> +number(state, 1);
> +number(state, 2);
> +number(state, 3);
> +pop(state, i);
I think `i` should be `one` or vice versa (and the same in the commit message)?
> +map_key(state, "bar");
> +map(state);
> +map_key(state, "baaz");
> +string(state, "hello world");
> +pop(state, top);
> +
> +would output
Tests
I have been thinking a little bit about the current situation with regards to tests. There are quite a lot of tests in the review queue that have been there for quite some time without much interest, but I do think we are rather short of tests. (*) I wonder if we could have a sort of staging area for tests where they could roughly go in without review and would only be run by something like make stage-test or make all-tests or something. The hope is that this would encourage more tests. The idea would be not that a patch author would have to make sure they all pass, but if they previously passed and no longer do then the author would know *something* about the output had changed. One example is the emacs elide test by Pieter id:"1329684990-12504-3-git-send-email-pieter at praet.org" which does still pass after my substantial change to the way elide is done (and is not currently covered in the test suite) Of course if someone does review one of these staging tests then they can be moved into the real tests. Best wishes Mark (*) for example the structured output patch accidentally changed the output of notmuch search --output=threads --format=json but that was not caught by the tests.
deprecating legacy text output
On Tue, 10 Jul 2012 10:04:10 -0700, Jameson Graef Rollins wrote: > On Tue, Jul 10 2012, Mark Walters wrote: > > I think I would also add something saying that the text format is just > > different (and that a significant chunk of the patch is just that). > > Can we not just dump this output format once and for all? Does anything > use it? And if so can we just modify it to use a more sensible format? I think we need to at least deprecate the format for a release or so, as there are definitely people using it in the wild. As far as I know the at least the Vim interface uses it (what about notmuch-mutt?), in addition to whatever homebrewed scripts might use it. Since scriptability is one of the "notmuch core values", I guess we should also provide something more script friendly than json. As you say, it doesn't have to be exactly like what we have now. d
post-new [was: Re: query on a subset of messages ?]
On Jul 10, 2012 12:59 PM, "Sebastien Binet" wrote: > > Jani Nikula writes: > > > On Jul 9, 2012 8:12 PM, "Jameson Graef Rollins" < jrollins at finestructure.net> > > wrote: > >> > >> On Mon, Jul 09 2012, Sebastien Binet wrote: > >> > hum... is post-new supposed to be run even if there is no new message ? > >> > >> Hi, Sebastian. Yes, I think it runs regardless if there are any new > >> messages or not. > > > > That's correct; only errors in notmuch new cause post-new hook to be > > skipped. > > ok. I thought using the post-new hook would have saved some i/o > resources over my current setup: > offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh > > where notmuch-tag.sh is (in pseudo-code): > ## > /usr/bin/notmuch new > > for tag,query in tag-queries: > tag_new $tag $query > > ## > > is there any advantage of using post-new compared to this setup ? There's no functional advantage. It does keep your initial tagging script connected with notmuch new rather than offlinemap, if you ever need to run notmuch new on its own. If your tagging setup is really complicated, you could have some-tag in new.tags config, and bail out early if notmuch count tag:some-tag outputs 0 (and obviously notmuch tag -some-tag tag:some-tag later in the script). Just a thought. J. > > -s -- next part -- An HTML attachment was scrubbed... URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/c069b7cf/attachment.html>
notmuch-emacs and bbdb
As far as I can tell, notmuch doesn't integrate as smoothly with bbdb as older emacs mailclients. I'm especially looking for a snarf function that distinguishes sender from recipient. How do other people use bbdb with notmuch? Does anyone have lisp code like that which ships with bbdb for other clients? If I were to find time to write such code, what would you like it to do? cheers, bergey
[PATCH] v2: Added better support for multiple structured output formats.
As discussed in , this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included. The output for JSON (and
text) is identical to the current output. S-Expressions will be added in
a later patch.
A large part of this patch just implements the differentiation between
structured and non-structured output (all the code within
"if(format == unstructured_text_printer)").
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
The interface is a structure structure_printer, which contains the following
methods:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use the state object to insert delimiters etc.
automatically when appropriate.
Example:
int top, one;
top = map(state);
map_key(state, "foo");
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, i);
map_key(state, "bar");
map(state);
map_key(state, "baaz");
string(state, "hello world");
pop(state, top);
would output JSON as follows:
{"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
---
notmuch-search.c | 491 ---
1 file changed, 361 insertions(+), 130 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..412 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,210 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, "foo");
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
+pop(state, i);
+map_key(state, "bar");
+map(state);
+map_key(state, "baaz");
+string(state, "hello world");
+pop(state, top);
+
+would output JSON as follows:
+
+{"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
+
+ */
+typedef struct structure_printer {
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
+} structure_printer_t;
+
+/* JSON structure printer */
+
+/* single linked list implementation for keeping track of the array/map
nesting state */
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
[PATCH] v2: Added better support for multiple structured output formats.
Since it would be great to use the structure printer for show as well,
it would make sense to put it in its own source file, where it can
easily be shared between commands.
There are a few systematic code formatting problems in your patch. To
be consistent with other notmuch code, there should be a space between
function names and parameter lists (in both declarations and calls)
and there should be a space between a keyword and a paren (e.g., "if
(" and "for ("). In a function definition, the function name should
start on a new line (this makes it easy to grep for a function's
definition). Also, in general, try to keep lines under 80 characters
wide (we're not very consistent about this one, but it would be nice
to not become even less consistent about it).
More detailed comments below.
Quoth craven at gmx.net on Jul 10 at 3:30 pm:
> As discussed in , this patch adds
> support for new structured output formats (like s-expressions) by using
> stateful structure_printers. An implementation of the JSON structure
> printer that passes all tests is included. The output for JSON (and
> text) is identical to the current output. S-Expressions will be added in
> a later patch.
>
> A large part of this patch just implements the differentiation between
> structured and non-structured output (all the code within
> "if(format == unstructured_text_printer)").
>
> In a second patch, the structured output code should be isolated in a
> separate file, and also used in all other parts of notmuch.
>
> The interface is a structure structure_printer, which contains the following
> methods:
>
> - initial_state: is called to create a state object, that is passed to all
> invocations. This should be used to keep track of the output file and
> everything else necessary to correctly format output.
> - map: is called when a new map (associative array, dictionary) is started.
> map_key and the primitives (string, number, bool) are used alternatingly to
> add key/value pairs. pop is used to close the map (see there). This function
> must return a nesting level identifier that can be used to close all nested
> structures (maps and lists), backing out to the returned nesting level.
> - list: is called when a new list (array, vector) is started. the primitives
> (string, number, bool) are used consecutively to add values to the list. pop
> is used to close the list. This function must return a nesting level
> identifier that can be used to close all nested structures (maps and lists),
> backing out to the returned nesting level.
> - pop: is called to return to a given nesting level. All lists and maps with
> a deeper nesting level must be closed.
> - number, string, bool: output one element of the specific type.
>
> All functions should use the state object to insert delimiters etc.
> automatically when appropriate.
>
> Example:
> int top, one;
> top = map(state);
> map_key(state, "foo");
> one = list(state);
> number(state, 1);
> number(state, 2);
> number(state, 3);
> pop(state, i);
> map_key(state, "bar");
> map(state);
> map_key(state, "baaz");
> string(state, "hello world");
> pop(state, top);
>
> would output JSON as follows:
>
> {"foo": [1, 2, 3], "bar": { "baaz": "hello world"}}
> ---
> notmuch-search.c | 491
> ---
> 1 file changed, 361 insertions(+), 130 deletions(-)
>
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 3be296d..412 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -28,6 +28,210 @@ typedef enum {
> OUTPUT_TAGS
> } output_t;
>
> +/* structured formatting, useful for JSON, S-Expressions, ...
> +
> +- initial_state: is called to create a state object, that is passed to all
> invocations. This should be used to keep track of the output file and
> everything else necessary to correctly format output.
> +- map: is called when a new map (associative array, dictionary) is started.
> map_key and the primitives (string, number, bool) are used alternatingly to
> add key/value pairs. pop is used to close the map (see there). This function
> must return a nesting level identifier that can be used to close all nested
> structures (maps and lists), backing out to the returned nesting level.
> +- list: is called when a new list (array, vector) is started. the primitives
> (string, number, bool) are used consecutively to add values to the list. pop
> is used to close the list. This function must return a nesting level
> identifier that can be used to close all nested structures (maps and lists),
> backing out to the returned nesting level.
> +- pop: is called to return to a given nesting level. All lists and maps with
> a deeper nesting level must be closed.
> +- number, string, bool: output one element of the specific type.
> +
> +All functions should use state to insert delimiters etc. automatically when
> appropriate.
> +
> +Example:
> +int top, one;
> +top = map(state);
> +map_key(state, "foo");
>
[PATCH] FIXED: Added better support for multiple structured output formats.
> notmuch-search.c | 453
> +++
> 1 file changed, 326 insertions(+), 127 deletions(-)
Hi for such a large patch this was surprisingly easy to follow (though I
am not saying I have worked through all the details). However, there are
some preliminary comments below which I think are worth fixing before a
full review as they will make it easier for other reviewers. One thing
that would help a lot is some comments.
Best wishes
Mark
> As discussed in , this patch adds
> support for new structured output formats (like s-expressions) by using
> stateful structure_printers. An implementation of the JSON structure
> printer that passes all tests is included.
If I understand it correctly, the output should be identical to before?
If that is the case I think it's worth saying.
> Structure printers have functions for starting a map, a list, closing
> any number of these two, printing a map key, a number value, a string
> value, and boolean values. By threading a state through the
> invocations, arbitrary structured formatting can be achieved.
I think I would also add something saying that the text format is just
different (and that a significant chunk of the patch is just that).
> In a second patch, the structured output code should be isolated in a
> separate file, and also used in all other parts of notmuch.
> ---
>
> diff --git a/notmuch-search.c b/notmuch-search.c
> index 3be296d..3413b79 100644
> --- a/notmuch-search.c
> +++ b/notmuch-search.c
> @@ -28,6 +28,181 @@ typedef enum {
> OUTPUT_TAGS
> } output_t;
>
> +typedef void * structure_printer_state_t;
> +
> +typedef struct structure_printer {
> +int (*map)(void *state);
> +int (*list)(void *state);
> +void (*pop)(void *state, int level);
> +void (*map_key)(void *state, const char *key);
> +void (*number)(void *state, int val);
> +void (*string)(void *state, const char *val);
> +void (*bool)(void *state, notmuch_bool_t val);
> +void *(*initial_state)(const struct structure_printer *sp, FILE *output);
> +} structure_printer_t;
I think this needs some comments on what these functions do. number,
string and boolean are relatively clear (but saying "output a number"
etc is worthwhile). But what map and list do is much less clear and
definitely deserves a comment. And it is definitely unclear what they
are meant to return.
I would also say something about the variable state: eg "the variable
`state` can contain any state the structure_printer wishes to maintain".
> +
> +/* JSON structure printer */
> +
> +typedef struct json_list {
> +int type;
> +int first_already_seen;
> +struct json_list *rest;
> +} json_list_t;
> +
> +#define TYPE_JSON_MAP 1
> +#define TYPE_JSON_ARRAY 2
> +
> +typedef struct json_state {
> +FILE *output;
> +json_list_t *stack;
> +int level;
> +} json_state_t;
> +
> +int json_map(void *state);
> +int json_list(void *state);
> +void json_pop(void *state, int level);
> +void json_map_key(void *state, const char *key);
> +void json_number(void *state, int val);
> +void json_string(void *state, const char *val);
> +void json_bool(void *state, notmuch_bool_t val);
> +void *json_initial_state(const struct structure_printer *sp, FILE *output);
> +
> +int json_map(void *st) {
> +json_state_t *state = (json_state_t*)st;
> +FILE *output = state->output;
> +if(state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
> state->stack->first_already_seen) {
> + fputs(",", output);
> + if(state->level == 1)
> + fputs("\n", output);
> + else
> + fputs(" ", output);
> +}
> +if(state->stack != NULL) {
> + state->stack->first_already_seen = TRUE;
> +}
> +fputs("{", output);
> +void *ctx_json_map = talloc_new (0);
> +json_list_t *el = talloc(ctx_json_map, json_list_t);
> +el->type = TYPE_JSON_MAP;
> +el->first_already_seen = FALSE;
> +el->rest = state->stack;
> +state->stack = el;
> +return state->level++;
> +}
> +
> +int json_list(void *st) {
> +json_state_t *state = (json_state_t*)st;
> +FILE *output = state->output;
> +if(state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
> state->stack->first_already_seen) {
> + fputs(",", output);
> + if(state->level == 1)
> + fputs("\n", output);
> + else
> + fputs(" ", output);
> +}
> +if(state->stack != NULL) {
> + state->stack->first_already_seen = TRUE;
> +}
> +fputs("[", output);
> +void *ctx_json_map = talloc_new (0);
> +json_list_t *el = talloc(ctx_json_map, json_list_t);
> +el->type = TYPE_JSON_ARRAY;
> +el->first_already_seen = FALSE;
> +el->rest = state->stack;
> +state->stack = el;
> +return state->level++;
> +}
> +
> +void json_pop(void *st, int level) {
> +int i;
> +json_state_t *state = (json_state_t*)st;
> +FILE *output = state->output;
> +for(i = state->level; i > level;
[PATCH] FIXED: Added better support for multiple structured output formats.
Quoth Jameson Graef Rollins on Jul 10 at 10:04 am: > On Tue, Jul 10 2012, Mark Walters wrote: > > I think I would also add something saying that the text format is just > > different (and that a significant chunk of the patch is just that). > > Can we not just dump this output format once and for all? Does anything > use it? And if so can we just modify it to use a more sensible format? Currently Emacs uses this output format (though id:"1341870162-17782-1-git-send-email-amdragon at mit.edu" aims to fix this). Also, I assume most scripted uses of notmuch use the text format (probably not the --output=summary text format, but certainly files, messages, and tags).
[PATCH] FIXED: Added better support for multiple structured output formats.
Sorry, the original patch I sent was missing a small part, here the full
patch:
As discussed in , this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included.
Structure printers have functions for starting a map, a list, closing
any number of these two, printing a map key, a number value, a string
value, and boolean values. By threading a state through the
invocations, arbitrary structured formatting can be achieved.
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
---
notmuch-search.c | 453 +++
1 file changed, 326 insertions(+), 127 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..3413b79 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,181 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+typedef void * structure_printer_state_t;
+
+typedef struct structure_printer {
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
+} structure_printer_t;
+
+/* JSON structure printer */
+
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
+typedef struct json_state {
+FILE *output;
+json_list_t *stack;
+int level;
+} json_state_t;
+
+int json_map(void *state);
+int json_list(void *state);
+void json_pop(void *state, int level);
+void json_map_key(void *state, const char *key);
+void json_number(void *state, int val);
+void json_string(void *state, const char *val);
+void json_bool(void *state, notmuch_bool_t val);
+void *json_initial_state(const struct structure_printer *sp, FILE *output);
+
+int json_map(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if(state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(",", output);
+ if(state->level == 1)
+ fputs("\n", output);
+ else
+ fputs(" ", output);
+}
+if(state->stack != NULL) {
+ state->stack->first_already_seen = TRUE;
+}
+fputs("{", output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el->type = TYPE_JSON_MAP;
+el->first_already_seen = FALSE;
+el->rest = state->stack;
+state->stack = el;
+return state->level++;
+}
+
+int json_list(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if(state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(",", output);
+ if(state->level == 1)
+ fputs("\n", output);
+ else
+ fputs(" ", output);
+}
+if(state->stack != NULL) {
+ state->stack->first_already_seen = TRUE;
+}
+fputs("[", output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el->type = TYPE_JSON_ARRAY;
+el->first_already_seen = FALSE;
+el->rest = state->stack;
+state->stack = el;
+return state->level++;
+}
+
+void json_pop(void *st, int level) {
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+for(i = state->level; i > level; i--) {
+ json_list_t *tos = state->stack;
+ if(tos->type == TYPE_JSON_MAP) {
+ fputs("}", output);
+ }
+ if(tos->type == TYPE_JSON_ARRAY) {
+ fputs("]", output);
+ }
+ state->stack = tos->rest;
+ state->level--;
+ talloc_free(tos);
+}
+if(state->level == 0)
+ fputs("\n", output);
+}
+
+void json_map_key(void *st, const char *key) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if(state->stack != NULL && state->stack->first_already_seen) {
+ fputs(",\n", output);
+}
+fputs("\"", output);
+fputs(key, output);
+fputs("\": ", output);
+}
+
+void json_number(void *st, int val) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+if(state->stack != NULL && state->stack->type == TYPE_JSON_ARRAY &&
state->stack->first_already_seen) {
+ fputs(", ", output);
+}
+state->stack->first_already_seen = TRUE;
+fprintf(output, "%i", val);
+}
+
+void json_string(void *st, const char *val) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state->output;
+void *ctx =
ANN: Speedee, a web-based notmuch client
Hi, Sounds great! https://speedee.bugsplat.info/#search/ruby throws at me: NoMethodError at / undefined method `close' for nil:NilClass file: app.rb location: block in line: 16 Looking forward to play with you demo :) Thanks! ~ elf Pavlik ~ Excerpts from Peter Keen's message of 2012-07-10 05:14:45 +: > Hi guys, > > Over the weekend I threw together the prototype for a web-based notmuch > client named Speedee. It's a little Ruby webapp with a Backbone.js front > end, using Twitter Bootstrap and a free theme from Bootswatch. It doesn't > do much yet. Basically, it can list threads matching a search and show you > those threads. It reads the notmuch config file to find the database. It > also reads the file for a new section named "speedee searches", where you > can define named searches to be included in the sidebar. The list of All > Tags is also included in the sidebar, assuming the version of notmuch > you're running has my ruby bindings patch from yesterday. > > At this point, my plan is to add a super basic mail composer and sending > facility and then live with it for a few months. I've setup a demo using > the notmuch mbox file as sample data. Feel free to poke around [1] and let > me know what you think! > > Source: https://github.com/peterkeen/speedee > Demo: https://speedee.bugsplat.info/#search/ruby > > > [1]: The demo is using a self-signed wildcard cert. SHA1 fingerprint: 01 B6 > 5D 47 EC 4A BB 27 E6 49 A1 0F 5A 48 8F BD CC 33 F2 F8
post-new [was: Re: query on a subset of messages ?]
Jani Nikula writes: > On Jul 9, 2012 8:12 PM, "Jameson Graef Rollins" finestructure.net> > wrote: >> >> On Mon, Jul 09 2012, Sebastien Binet wrote: >> > hum... is post-new supposed to be run even if there is no new message ? >> >> Hi, Sebastian. Yes, I think it runs regardless if there are any new >> messages or not. > > That's correct; only errors in notmuch new cause post-new hook to be > skipped. ok. I thought using the post-new hook would have saved some i/o resources over my current setup: offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh where notmuch-tag.sh is (in pseudo-code): ## /usr/bin/notmuch new for tag,query in tag-queries: tag_new $tag $query ## is there any advantage of using post-new compared to this setup ? -s -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 197 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/8e3e0334/attachment.pgp>
[PATCH] FIXED: Added better support for multiple structured output formats.
On Tue, Jul 10 2012, Austin Clements wrote: > Quoth Jameson Graef Rollins on Jul 10 at 10:04 am: >> On Tue, Jul 10 2012, Mark Walters wrote: >> > I think I would also add something saying that the text format is just >> > different (and that a significant chunk of the patch is just that). >> >> Can we not just dump this output format once and for all? Does anything >> use it? And if so can we just modify it to use a more sensible format? > > Currently Emacs uses this output format (though > id:"1341870162-17782-1-git-send-email-amdragon at mit.edu" aims to fix > this). Also, I assume most scripted uses of notmuch use the text > format (probably not the --output=summary text format, but certainly > files, messages, and tags). Sorry, in a momentary brain freeze I confused the search and show text output. You're right that the text search output is more widely used. jamie. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 835 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/1889c9ec/attachment.pgp>
[PATCH] FIXED: Added better support for multiple structured output formats.
On Tue, Jul 10 2012, Mark Walters wrote: > I think I would also add something saying that the text format is just > different (and that a significant chunk of the patch is just that). Can we not just dump this output format once and for all? Does anything use it? And if so can we just modify it to use a more sensible format? jamie. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 835 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/05948522/attachment.pgp>
[PATCH] FIXED: Added better support for multiple structured output formats.
On Tue, Jul 10 2012, craven at gmx.net wrote: > Sorry, the original patch I sent was missing a small part, here the full > patch: > > As discussed in , this patch adds > support for new structured output formats (like s-expressions) by using > stateful structure_printers. An implementation of the JSON structure > printer that passes all tests is included. > > Structure printers have functions for starting a map, a list, closing > any number of these two, printing a map key, a number value, a string > value, and boolean values. By threading a state through the > invocations, arbitrary structured formatting can be achieved. > > In a second patch, the structured output code should be isolated in a > separate file, and also used in all other parts of notmuch. Hi, Peter. Thanks for submitting this. I would really like to see this patch broken up into multiple smaller and more atomic patches. Smaller patches are much easier to read and review. For instance, all the new formatting functions could be added in their own patch (and already in their own file if that's appropriate). Then the existing json output could then be modified to use the new formatters. There is also json output in notmuch show and reply as well, so they should probably also be modified to use the new formatter. Thanks! jamie. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 835 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/74c64ff7/attachment.pgp>
post-new [was: Re: query on a subset of messages ?]
On Tue, Jul 10 2012, Sebastien Binet wrote: > ok. I thought using the post-new hook would have saved some i/o > resources over my current setup: > offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh > > where notmuch-tag.sh is (in pseudo-code): > ## > /usr/bin/notmuch new > > for tag,query in tag-queries: > tag_new $tag $query > > ## > > is there any advantage of using post-new compared to this setup ? The hooks are just there for convenience, and as Jani says, they're executed any time notmuch new is run, in any context. In your setup it looks like it would also save you having to maintain another script (notmuch-tag.sh) since that script in it's entirety could just be the post-new script, and the offlineimap.postsynchook could just execute notmuch new directly. jamie. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 835 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/f68c2784/attachment.pgp>
[PATCH 1/3] emacs: add no-display arg to notmuch-hello-refresh-hook
On Jul 10, 2012 4:49 AM, "Austin Clements" wrote: > > Quoth Jani Nikula on Jun 15 at 6:53 pm: > > Add no-display arg to notmuch-hello-refresh-hook to allow each hook to > > decide what is appropriate when no-display is t, which is typically > > the case when called non-interactively. This is used by the following > > patch. > > > > This breaks existing hooks people might have, which will now need to > > accept the argument. > > > > Signed-off-by: Jani Nikula > > This seems like an overloaded use of no-display. If I'm reading the > code right, no-display indicates whether or not the notmuch-hello > buffer should be switched to and seems like a workaround for some > particular corner-case (I'm not even sure what). This seems like a > strange condition to predicate a hook on (but maybe I just don't > understand). What condition, abstractly speaking, is > notmuch-hello-refresh-status-message trying to run under? IIUC, no-display is useful for calling refresh from outside of emacs, e.g. from post-new hook in an automated fashion, so you can have an up-to-date buffer when you switch to it. There's no point in displaying the refresh message when you don't also switch to the buffer, is there? And this way you'll get the diff between the manual (through user interaction) refreshes of the buffer, not between two cron jobs. J. -- next part -- An HTML attachment was scrubbed... URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/cf9914bb/attachment.html>
ANN: Speedee, a web-based notmuch client
On Tue, Jul 10, 2012 at 5:08 AM, elf Pavlik wrote: > Hi, > > Sounds great! > > https://speedee.bugsplat.info/#search/ruby throws at me: > > NoMethodError at / > undefined method `close' for nil:NilClass > > file: app.rb > location: block in > line: 16 > > Looking forward to play with you demo :) > > Thanks! > ~ elf Pavlik ~ > Interesting. Looks like something was leaking file handles. It appears to be all better after a simple patch and a restart. Thanks! -- next part -- An HTML attachment was scrubbed... URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/f0282786/attachment.html>
ANN: Speedee, a web-based notmuch client
On Mon, Jul 09 2012, Peter Keen wrote: > Over the weekend I threw together the prototype for a web-based notmuch > client named Speedee. It's a little Ruby webapp with a Backbone.js front > end, using Twitter Bootstrap and a free theme from Bootswatch. It doesn't > do much yet. Basically, it can list threads matching a search and show you > those threads. It reads the notmuch config file to find the database. It > also reads the file for a new section named "speedee searches", where you > can define named searches to be included in the sidebar. The list of All > Tags is also included in the sidebar, assuming the version of notmuch > you're running has my ruby bindings patch from yesterday. > > At this point, my plan is to add a super basic mail composer and sending > facility and then live with it for a few months. I've setup a demo using > the notmuch mbox file as sample data. Feel free to poke around [1] and let > me know what you think! > > Source: https://github.com/peterkeen/speedee > Demo: https://speedee.bugsplat.info/#search/ruby Cool! I like it. I can imagine adding fancier formatting and coloring in the show pages, but it's a really nice simple platform to start with. Very nice. Looking forward to watching it develop! jamie. -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 835 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20120710/e3e1e654/attachment.pgp>
S/MIME support
On Mon, Jul 09 2012, "Bryant, Daniel B." wrote:
> I was able to get signature verification working with your patchset
> (with a caveat) but not decryption.
Hi, Daniel. I guess I'm only partially happy to hear that! I
definitely do appreciate the feedback, though.
> The caveat is that GMime is still borked with handling signatures with
> content type application/x-pkcs7-signature
> (vs. application/pkcs7-signature, which works fine). This is upstream
> GNOME bug #674032 that was supposed to have been fixed in GMime 2.6.9,
> but that original fix is also broken.
Ah, I didn't notice that:
https://bugzilla.gnome.org/show_bug.cgi?id=674032
Encouragingly, it sounds like Jeffery is working on it.
> One possible workaround is to twiddle the content-type of the
> signature part (and the corresponding protocol in the multipart/signed
> part). I implemented this by looping over each message part in
> mime_node_open() and modifying as necessary using the following logic:
>
>
> GMimeContentType *content_type = g_mime_object_get_content_type (part);
>
> const char *subtype = g_mime_content_type_get_media_subtype
> (content_type);
> const char *protocol = g_mime_content_type_get_parameter (content_type,
> "protocol");
>
> if (!strcmp(subtype, "x-pkcs7-signature")) {
> g_mime_content_type_set_media_subtype (content_type,
> "pkcs7-signature");
> }
>
> if (protocol && !strcmp(protocol, "application/x-pkcs7-signature")) {
> g_mime_content_type_set_parameter (content_type,
> "protocol","application/pkcs7-signature");
> }
We could do this, but I would certainly prefer that we fix gmime to
handle both types properly.
> All of my S/MIME encrypted mail consists of single part messages with
> content-type "application/x-pkcs7-mime". These conform to RFC3851,
> section 3.3/3.4. (sample messages are included in the RFC as
> well). This fails to be decrypted by notmuch because the mime node
> traversal code assumes that every encrypted message is
> multipart/encrypted, which appears to only be true for PGP/MIME.
Thanks for the great example of why we need tests!
Would you (or anyone) be willing to start putting together some tests
that include messages encrypted according to this RFC? I think adding
some tests to the test/crypto script would be a great place to start.
jamie.
-- next part --
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 835 bytes
Desc: not available
URL:
<http://notmuchmail.org/pipermail/notmuch/attachments/20120710/7606e81b/attachment-0001.pgp>
RE: S/MIME support
On Mon, Jul 09 2012, Bryant, Daniel B. dan.bry...@jhuapl.edu wrote:
I was able to get signature verification working with your patchset
(with a caveat) but not decryption.
Hi, Daniel. I guess I'm only partially happy to hear that! I
definitely do appreciate the feedback, though.
The caveat is that GMime is still borked with handling signatures with
content type application/x-pkcs7-signature
(vs. application/pkcs7-signature, which works fine). This is upstream
GNOME bug #674032 that was supposed to have been fixed in GMime 2.6.9,
but that original fix is also broken.
Ah, I didn't notice that:
https://bugzilla.gnome.org/show_bug.cgi?id=674032
Encouragingly, it sounds like Jeffery is working on it.
One possible workaround is to twiddle the content-type of the
signature part (and the corresponding protocol in the multipart/signed
part). I implemented this by looping over each message part in
mime_node_open() and modifying as necessary using the following logic:
GMimeContentType *content_type = g_mime_object_get_content_type (part);
const char *subtype = g_mime_content_type_get_media_subtype
(content_type);
const char *protocol = g_mime_content_type_get_parameter (content_type,
protocol);
if (!strcmp(subtype, x-pkcs7-signature)) {
g_mime_content_type_set_media_subtype (content_type,
pkcs7-signature);
}
if (protocol !strcmp(protocol, application/x-pkcs7-signature)) {
g_mime_content_type_set_parameter (content_type,
protocol,application/pkcs7-signature);
}
We could do this, but I would certainly prefer that we fix gmime to
handle both types properly.
All of my S/MIME encrypted mail consists of single part messages with
content-type application/x-pkcs7-mime. These conform to RFC3851,
section 3.3/3.4. (sample messages are included in the RFC as
well). This fails to be decrypted by notmuch because the mime node
traversal code assumes that every encrypted message is
multipart/encrypted, which appears to only be true for PGP/MIME.
Thanks for the great example of why we need tests!
Would you (or anyone) be willing to start putting together some tests
that include messages encrypted according to this RFC? I think adding
some tests to the test/crypto script would be a great place to start.
jamie.
pgpG8NzNvo4cI.pgp
Description: PGP signature
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
Re: ANN: Speedee, a web-based notmuch client
On Mon, Jul 09 2012, Peter Keen peter.k...@bugsplat.info wrote: Over the weekend I threw together the prototype for a web-based notmuch client named Speedee. It's a little Ruby webapp with a Backbone.js front end, using Twitter Bootstrap and a free theme from Bootswatch. It doesn't do much yet. Basically, it can list threads matching a search and show you those threads. It reads the notmuch config file to find the database. It also reads the file for a new section named speedee searches, where you can define named searches to be included in the sidebar. The list of All Tags is also included in the sidebar, assuming the version of notmuch you're running has my ruby bindings patch from yesterday. At this point, my plan is to add a super basic mail composer and sending facility and then live with it for a few months. I've setup a demo using the notmuch mbox file as sample data. Feel free to poke around [1] and let me know what you think! Source: https://github.com/peterkeen/speedee Demo: https://speedee.bugsplat.info/#search/ruby Cool! I like it. I can imagine adding fancier formatting and coloring in the show pages, but it's a really nice simple platform to start with. Very nice. Looking forward to watching it develop! jamie. pgpRlUPku6VAg.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: ANN: Speedee, a web-based notmuch client
On Tue, Jul 10, 2012 at 7:14 AM, Peter Keen peter.k...@bugsplat.info wrote: let me know what you think! This is awesome, I love it! ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: post-new [was: Re: query on a subset of messages ?]
Jani Nikula j...@nikula.org writes: On Jul 9, 2012 8:12 PM, Jameson Graef Rollins jroll...@finestructure.net wrote: On Mon, Jul 09 2012, Sebastien Binet bi...@cern.ch wrote: hum... is post-new supposed to be run even if there is no new message ? Hi, Sebastian. Yes, I think it runs regardless if there are any new messages or not. That's correct; only errors in notmuch new cause post-new hook to be skipped. ok. I thought using the post-new hook would have saved some i/o resources over my current setup: offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh where notmuch-tag.sh is (in pseudo-code): ## /usr/bin/notmuch new for tag,query in tag-queries: tag_new $tag $query ## is there any advantage of using post-new compared to this setup ? -s pgpUHrMkBjuFt.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[PATCH] Added better support for multiple structured output formats.
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included.
Structure printers have functions for starting a map, a list, closing
any number of these two, printing a map key, a number value, a string
value, and boolean values. By threading a state through the
invocations, arbitrary structured formatting can be achieved.
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
---
notmuch-search.c | 458 ++-
1 file changed, 322 insertions(+), 136 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index fa5086e..3413b79 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,19 +28,181 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+typedef void * structure_printer_state_t;
+
typedef struct structure_printer {
-int (*map)(struct structure_printer_t *sp);
-int (*list)(struct structure_printer_t *sp);
-void (*pop)(struct structure_printer_t *sp, int level);
-void (*map_key)(struct structure_printer_t *sp, const char *key);
-void (*number)(struct structure_printer_t *sp, int val);
-void (*string)(struct structure_printer_t *sp, const char *val);
-void (*bool)(struct structure_printer_t *sp, notmuch_bool_t val);
-void *(*initial_context)(struct structure_printer_t *sp);
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
} structure_printer_t;
-struct structure_printer_t *new_search_format_structure_printer(FILE *stream,
struct search_format_t *search_format);
+/* JSON structure printer */
+
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
+typedef struct json_state {
+FILE *output;
+json_list_t *stack;
+int level;
+} json_state_t;
+
+int json_map(void *state);
+int json_list(void *state);
+void json_pop(void *state, int level);
+void json_map_key(void *state, const char *key);
+void json_number(void *state, int val);
+void json_string(void *state, const char *val);
+void json_bool(void *state, notmuch_bool_t val);
+void *json_initial_state(const struct structure_printer *sp, FILE *output);
+
+int json_map(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs({, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_MAP;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+int json_list(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs([, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_ARRAY;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+void json_pop(void *st, int level) {
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+for(i = state-level; i level; i--) {
+ json_list_t *tos = state-stack;
+ if(tos-type == TYPE_JSON_MAP) {
+ fputs(}, output);
+ }
+ if(tos-type == TYPE_JSON_ARRAY) {
+ fputs(], output);
+ }
+ state-stack = tos-rest;
+ state-level--;
+ talloc_free(tos);
+}
+if(state-level == 0)
+ fputs(\n, output);
+}
+
+void json_map_key(void *st, const char *key) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-first_already_seen) {
+ fputs(,\n, output);
+}
+fputs(\, output);
+fputs(key, output);
+fputs(\: , output);
+}
+
+void
[PATCH] FIXED: Added better support for multiple structured output formats.
Sorry, the original patch I sent was missing a small part, here the full
patch:
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included.
Structure printers have functions for starting a map, a list, closing
any number of these two, printing a map key, a number value, a string
value, and boolean values. By threading a state through the
invocations, arbitrary structured formatting can be achieved.
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
---
notmuch-search.c | 453 +++
1 file changed, 326 insertions(+), 127 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..3413b79 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,181 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+typedef void * structure_printer_state_t;
+
+typedef struct structure_printer {
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
+} structure_printer_t;
+
+/* JSON structure printer */
+
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
+typedef struct json_state {
+FILE *output;
+json_list_t *stack;
+int level;
+} json_state_t;
+
+int json_map(void *state);
+int json_list(void *state);
+void json_pop(void *state, int level);
+void json_map_key(void *state, const char *key);
+void json_number(void *state, int val);
+void json_string(void *state, const char *val);
+void json_bool(void *state, notmuch_bool_t val);
+void *json_initial_state(const struct structure_printer *sp, FILE *output);
+
+int json_map(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs({, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_MAP;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+int json_list(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs([, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_ARRAY;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+void json_pop(void *st, int level) {
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+for(i = state-level; i level; i--) {
+ json_list_t *tos = state-stack;
+ if(tos-type == TYPE_JSON_MAP) {
+ fputs(}, output);
+ }
+ if(tos-type == TYPE_JSON_ARRAY) {
+ fputs(], output);
+ }
+ state-stack = tos-rest;
+ state-level--;
+ talloc_free(tos);
+}
+if(state-level == 0)
+ fputs(\n, output);
+}
+
+void json_map_key(void *st, const char *key) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-first_already_seen) {
+ fputs(,\n, output);
+}
+fputs(\, output);
+fputs(key, output);
+fputs(\: , output);
+}
+
+void json_number(void *st, int val) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(, , output);
+}
+state-stack-first_already_seen = TRUE;
+fprintf(output, %i, val);
+}
+
+void json_string(void *st, const char *val) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+void *ctx = talloc_new(0);
+if(state-stack != NULL state-stack-type ==
Re: ANN: Speedee, a web-based notmuch client
Hi, Sounds great! https://speedee.bugsplat.info/#search/ruby throws at me: NoMethodError at / undefined method `close' for nil:NilClass file: app.rb location: block in class:App line: 16 Looking forward to play with you demo :) Thanks! ~ elf Pavlik ~ Excerpts from Peter Keen's message of 2012-07-10 05:14:45 +: Hi guys, Over the weekend I threw together the prototype for a web-based notmuch client named Speedee. It's a little Ruby webapp with a Backbone.js front end, using Twitter Bootstrap and a free theme from Bootswatch. It doesn't do much yet. Basically, it can list threads matching a search and show you those threads. It reads the notmuch config file to find the database. It also reads the file for a new section named speedee searches, where you can define named searches to be included in the sidebar. The list of All Tags is also included in the sidebar, assuming the version of notmuch you're running has my ruby bindings patch from yesterday. At this point, my plan is to add a super basic mail composer and sending facility and then live with it for a few months. I've setup a demo using the notmuch mbox file as sample data. Feel free to poke around [1] and let me know what you think! Source: https://github.com/peterkeen/speedee Demo: https://speedee.bugsplat.info/#search/ruby [1]: The demo is using a self-signed wildcard cert. SHA1 fingerprint: 01 B6 5D 47 EC 4A BB 27 E6 49 A1 0F 5A 48 8F BD CC 33 F2 F8 ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [PATCH] FIXED: Added better support for multiple structured output formats.
notmuch-search.c | 453
+++
1 file changed, 326 insertions(+), 127 deletions(-)
Hi for such a large patch this was surprisingly easy to follow (though I
am not saying I have worked through all the details). However, there are
some preliminary comments below which I think are worth fixing before a
full review as they will make it easier for other reviewers. One thing
that would help a lot is some comments.
Best wishes
Mark
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included.
If I understand it correctly, the output should be identical to before?
If that is the case I think it's worth saying.
Structure printers have functions for starting a map, a list, closing
any number of these two, printing a map key, a number value, a string
value, and boolean values. By threading a state through the
invocations, arbitrary structured formatting can be achieved.
I think I would also add something saying that the text format is just
different (and that a significant chunk of the patch is just that).
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
---
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..3413b79 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,181 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+typedef void * structure_printer_state_t;
+
+typedef struct structure_printer {
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
+} structure_printer_t;
I think this needs some comments on what these functions do. number,
string and boolean are relatively clear (but saying output a number
etc is worthwhile). But what map and list do is much less clear and
definitely deserves a comment. And it is definitely unclear what they
are meant to return.
I would also say something about the variable state: eg the variable
`state` can contain any state the structure_printer wishes to maintain.
+
+/* JSON structure printer */
+
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
+typedef struct json_state {
+FILE *output;
+json_list_t *stack;
+int level;
+} json_state_t;
+
+int json_map(void *state);
+int json_list(void *state);
+void json_pop(void *state, int level);
+void json_map_key(void *state, const char *key);
+void json_number(void *state, int val);
+void json_string(void *state, const char *val);
+void json_bool(void *state, notmuch_bool_t val);
+void *json_initial_state(const struct structure_printer *sp, FILE *output);
+
+int json_map(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs({, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_MAP;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+int json_list(void *st) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if(state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if(state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if(state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs([, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = TYPE_JSON_ARRAY;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+void json_pop(void *st, int level) {
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+for(i = state-level; i level; i--) {
+ json_list_t *tos = state-stack;
+ if(tos-type == TYPE_JSON_MAP) {
+ fputs(}, output);
+ }
+ if(tos-type ==
Re: post-new [was: Re: query on a subset of messages ?]
On Jul 10, 2012 12:59 PM, Sebastien Binet bi...@cern.ch wrote: Jani Nikula j...@nikula.org writes: On Jul 9, 2012 8:12 PM, Jameson Graef Rollins jroll...@finestructure.net wrote: On Mon, Jul 09 2012, Sebastien Binet bi...@cern.ch wrote: hum... is post-new supposed to be run even if there is no new message ? Hi, Sebastian. Yes, I think it runs regardless if there are any new messages or not. That's correct; only errors in notmuch new cause post-new hook to be skipped. ok. I thought using the post-new hook would have saved some i/o resources over my current setup: offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh where notmuch-tag.sh is (in pseudo-code): ## /usr/bin/notmuch new for tag,query in tag-queries: tag_new $tag $query ## is there any advantage of using post-new compared to this setup ? There's no functional advantage. It does keep your initial tagging script connected with notmuch new rather than offlinemap, if you ever need to run notmuch new on its own. If your tagging setup is really complicated, you could have some-tag in new.tags config, and bail out early if notmuch count tag:some-tag outputs 0 (and obviously notmuch tag -some-tag tag:some-tag later in the script). Just a thought. J. -s ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[PATCH] v2: Added better support for multiple structured output formats.
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included. The output for JSON (and
text) is identical to the current output. S-Expressions will be added in
a later patch.
A large part of this patch just implements the differentiation between
structured and non-structured output (all the code within
if(format == unstructured_text_printer)).
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
The interface is a structure structure_printer, which contains the following
methods:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use the state object to insert delimiters etc.
automatically when appropriate.
Example:
int top, one;
top = map(state);
map_key(state, foo);
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, i);
map_key(state, bar);
map(state);
map_key(state, baaz);
string(state, hello world);
pop(state, top);
would output JSON as follows:
{foo: [1, 2, 3], bar: { baaz: hello world}}
---
notmuch-search.c | 491 ---
1 file changed, 361 insertions(+), 130 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..412 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,210 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, foo);
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
+pop(state, i);
+map_key(state, bar);
+map(state);
+map_key(state, baaz);
+string(state, hello world);
+pop(state, top);
+
+would output JSON as follows:
+
+{foo: [1, 2, 3], bar: { baaz: hello world}}
+
+ */
+typedef struct structure_printer {
+int (*map)(void *state);
+int (*list)(void *state);
+void (*pop)(void *state, int level);
+void (*map_key)(void *state, const char *key);
+void (*number)(void *state, int val);
+void (*string)(void *state, const char *val);
+void (*bool)(void *state, notmuch_bool_t val);
+void *(*initial_state)(const struct structure_printer *sp, FILE *output);
+} structure_printer_t;
+
+/* JSON structure printer */
+
+/* single linked list implementation for keeping track of the array/map
nesting state */
+typedef struct json_list {
+int type;
+int first_already_seen;
+struct json_list *rest;
+} json_list_t;
+
+#define TYPE_JSON_MAP 1
+#define TYPE_JSON_ARRAY 2
+
Re: ANN: Speedee, a web-based notmuch client
On Tue, Jul 10, 2012 at 5:08 AM, elf Pavlik perpetual-trip...@wwelves.orgwrote: Hi, Sounds great! https://speedee.bugsplat.info/#search/ruby throws at me: NoMethodError at / undefined method `close' for nil:NilClass file: app.rb location: block in class:App line: 16 Looking forward to play with you demo :) Thanks! ~ elf Pavlik ~ Interesting. Looks like something was leaking file handles. It appears to be all better after a simple patch and a restart. Thanks! ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: post-new [was: Re: query on a subset of messages ?]
On Tue, Jul 10 2012, Sebastien Binet bi...@cern.ch wrote: ok. I thought using the post-new hook would have saved some i/o resources over my current setup: offlineimap.postsynchook = ~/emacs/notmuch-lib/notmuch-tag.sh where notmuch-tag.sh is (in pseudo-code): ## /usr/bin/notmuch new for tag,query in tag-queries: tag_new $tag $query ## is there any advantage of using post-new compared to this setup ? The hooks are just there for convenience, and as Jani says, they're executed any time notmuch new is run, in any context. In your setup it looks like it would also save you having to maintain another script (notmuch-tag.sh) since that script in it's entirety could just be the post-new script, and the offlineimap.postsynchook could just execute notmuch new directly. jamie. pgpkOUAXRnsHY.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Tests
I have been thinking a little bit about the current situation with regards to tests. There are quite a lot of tests in the review queue that have been there for quite some time without much interest, but I do think we are rather short of tests. (*) I wonder if we could have a sort of staging area for tests where they could roughly go in without review and would only be run by something like make stage-test or make all-tests or something. The hope is that this would encourage more tests. The idea would be not that a patch author would have to make sure they all pass, but if they previously passed and no longer do then the author would know *something* about the output had changed. One example is the emacs elide test by Pieter id:1329684990-12504-3-git-send-email-pie...@praet.org which does still pass after my substantial change to the way elide is done (and is not currently covered in the test suite) Of course if someone does review one of these staging tests then they can be moved into the real tests. Best wishes Mark (*) for example the structured output patch accidentally changed the output of notmuch search --output=threads --format=json search but that was not caught by the tests. ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [PATCH] FIXED: Added better support for multiple structured output formats.
On Tue, Jul 10 2012, Mark Walters markwalters1...@gmail.com wrote: I think I would also add something saying that the text format is just different (and that a significant chunk of the patch is just that). Can we not just dump this output format once and for all? Does anything use it? And if so can we just modify it to use a more sensible format? jamie. pgpciY52neVXL.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [PATCH] FIXED: Added better support for multiple structured output formats.
Quoth Jameson Graef Rollins on Jul 10 at 10:04 am: On Tue, Jul 10 2012, Mark Walters markwalters1...@gmail.com wrote: I think I would also add something saying that the text format is just different (and that a significant chunk of the patch is just that). Can we not just dump this output format once and for all? Does anything use it? And if so can we just modify it to use a more sensible format? Currently Emacs uses this output format (though id:1341870162-17782-1-git-send-email-amdra...@mit.edu aims to fix this). Also, I assume most scripted uses of notmuch use the text format (probably not the --output=summary text format, but certainly files, messages, and tags). ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [PATCH] v2: Added better support for multiple structured output formats.
On Tue, 10 Jul 2012, cra...@gmx.net wrote:
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included. The output for JSON (and
text) is identical to the current output. S-Expressions will be added in
a later patch.
Hi I have some more comments inline below. I agree with Jamie that it
would be nice to split it into smaller chunks but I am not sure that is
practical to do.
A large part of this patch just implements the differentiation between
structured and non-structured output (all the code within
if(format == unstructured_text_printer)).
I also agree with Jamie that the text output is annoying. However, at
least until Austin's async json parser goes in even the emacs interface
is using it so I think we have to live with it for a while (at the very
least we would need to give clients a reasonable time to migrate).
Anyway, I think this patch has to support it.
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
The interface is a structure structure_printer, which contains the following
methods:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to
add key/value pairs. pop is used to close the map (see there). This function
must return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop
is used to close the list. This function must return a nesting level
identifier that can be used to close all nested structures (maps and lists),
backing out to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with
a deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use the state object to insert delimiters etc.
automatically when appropriate.
Example:
int top, one;
top = map(state);
map_key(state, foo);
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, i);
map_key(state, bar);
map(state);
map_key(state, baaz);
string(state, hello world);
pop(state, top);
would output JSON as follows:
{foo: [1, 2, 3], bar: { baaz: hello world}}
---
notmuch-search.c | 491
---
1 file changed, 361 insertions(+), 130 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..412 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,210 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to
add key/value pairs. pop is used to close the map (see there). This function
must return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop
is used to close the list. This function must return a nesting level
identifier that can be used to close all nested structures (maps and lists),
backing out to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with
a deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, foo);
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
+pop(state, i);
I think `i` should be `one` or vice versa (and the same in the commit message)?
+map_key(state, bar);
+map(state);
+map_key(state, baaz);
+string(state, hello world);
+pop(state, top);
+
+would output JSON as follows:
+
+{foo: [1, 2, 3], bar: { baaz: hello world}}
+
+ */
+typedef struct
Re: [PATCH] FIXED: Added better support for multiple structured output formats.
On Tue, Jul 10 2012, Austin Clements amdra...@mit.edu wrote: Quoth Jameson Graef Rollins on Jul 10 at 10:04 am: On Tue, Jul 10 2012, Mark Walters markwalters1...@gmail.com wrote: I think I would also add something saying that the text format is just different (and that a significant chunk of the patch is just that). Can we not just dump this output format once and for all? Does anything use it? And if so can we just modify it to use a more sensible format? Currently Emacs uses this output format (though id:1341870162-17782-1-git-send-email-amdra...@mit.edu aims to fix this). Also, I assume most scripted uses of notmuch use the text format (probably not the --output=summary text format, but certainly files, messages, and tags). Sorry, in a momentary brain freeze I confused the search and show text output. You're right that the text search output is more widely used. jamie. pgpXoqvwtYeFE.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[PATCH v3 1/3] Add support for structured output formatters.
This patch adds a new type structure_printer, which is used for structured
formatting, e.g. JSON or S-Expressions.
The structure contains the following function pointers:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use state to insert delimiters etc. automatically when
appropriate.
Example:
int top, one;
top = map(state);
map_key(state, foo);
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, one);
map_key(state, bar);
map(state);
map_key(state, baaz);
string(state, hello world);
pop(state, top);
would output JSON as follows:
{foo: [1, 2, 3], bar: { baaz: hello world}}
---
Makefile.local |1 +
structured-output.h | 64 +++
2 files changed, 65 insertions(+)
create mode 100644 structured-output.h
diff --git a/Makefile.local b/Makefile.local
index a890df2..9b3db5e 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -286,6 +286,7 @@ notmuch_client_srcs = \
notmuch-reply.c \
notmuch-restore.c \
notmuch-search.c\
+ structured-output.c \
notmuch-setup.c \
notmuch-show.c \
notmuch-tag.c \
diff --git a/structured-output.h b/structured-output.h
new file mode 100644
index 000..b43afe0
--- /dev/null
+++ b/structured-output.h
@@ -0,0 +1,64 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth cwo...@cworth.org
+ */
+
+#include notmuch-client.h
+
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to add
key/value pairs. pop is used to close the map (see there). This function must
return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop is
used to close the list. This function must return a nesting level identifier
that can be used to close all nested structures (maps and lists), backing out
to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with a
deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, foo);
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
+pop(state, one);
+map_key(state, bar);
+map(state);
+map_key(state, baaz);
+string(state, hello world);
+pop(state, top);
+
+would output JSON as follows:
+
+{foo: [1, 2, 3], bar: { baaz: hello world}}
+
+ */
+typedef struct structure_printer {
+int (*map)(void
[PATCH v3 2/3] Adding structured output formatter for JSON.
This formatter prints exactly the same structure as the existing JSON
formatter, but uses the newly introduced structured formatting primitives.
---
structured-output.c | 159 +++
structured-output.h | 48
2 files changed, 207 insertions(+)
create mode 100644 structured-output.c
diff --git a/structured-output.c b/structured-output.c
new file mode 100644
index 000..e10fba4
--- /dev/null
+++ b/structured-output.c
@@ -0,0 +1,159 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth cwo...@cworth.org
+ */
+
+#include structured-output.h
+
+structure_printer_t *
+unstructured_text_printer = NULL;
+
+structure_printer_t
+json_structure_printer = {
+json_map,
+json_list,
+json_pop,
+json_map_key,
+json_number,
+json_string,
+json_bool,
+json_initial_state
+};
+static int
+enter_level(void *st, const char *marker, int type) {
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if (state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if (state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+if (state-stack != NULL) {
+ state-stack-first_already_seen = TRUE;
+}
+fputs(marker, output);
+void *ctx_json_map = talloc_new (0);
+json_list_t *el = talloc(ctx_json_map, json_list_t);
+el-type = type;
+el-first_already_seen = FALSE;
+el-rest = state-stack;
+state-stack = el;
+return state-level++;
+}
+
+int
+json_map(void *st)
+{
+return enter_level(st, {, TYPE_JSON_MAP);
+}
+
+int
+json_list(void *st)
+{
+return enter_level(st, [, TYPE_JSON_ARRAY);
+}
+
+void
+json_pop(void *st, int level)
+{
+int i;
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+for (i = state-level; i level; i--) {
+ json_list_t *tos = state-stack;
+ if (tos-type == TYPE_JSON_MAP) {
+ fputs(}, output);
+ }
+ if (tos-type == TYPE_JSON_ARRAY) {
+ fputs(], output);
+ }
+ state-stack = tos-rest;
+ state-level--;
+ talloc_free(tos);
+}
+if (state-level == 0)
+ fputs(\n, output);
+}
+
+void
+json_map_key(void *st, const char *key)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if (state-stack != NULL state-stack-first_already_seen) {
+ fputs(,\n, output);
+}
+fputs(\, output);
+fputs(key, output);
+fputs(\: , output);
+}
+
+void
+json_number(void *st, int val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if (state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(, , output);
+}
+state-stack-first_already_seen = TRUE;
+fprintf(output, %i, val);
+}
+
+void
+json_string(void *st, const char *val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+void *ctx = talloc_new(0);
+if (state-stack != NULL state-stack-type == TYPE_JSON_ARRAY
state-stack-first_already_seen) {
+ fputs(,, output);
+ if (state-level == 1)
+ fputs(\n, output);
+ else
+ fputs( , output);
+}
+
+state-stack-first_already_seen = TRUE;
+fprintf(output, %s, json_quote_str(ctx, val));
+talloc_free(ctx);
+}
+
+void
+json_bool(void *st, notmuch_bool_t val)
+{
+json_state_t *state = (json_state_t*)st;
+FILE *output = state-output;
+if (val)
+ fputs(true, output);
+else
+ fputs(false, output);
+}
+
+void *
+json_initial_state(const struct structure_printer *sp, FILE *output)
+{
+(void)sp;
+json_state_t *st = talloc(0, json_state_t);
+st-level = 0;
+st-stack = NULL;
+st-output = output;
+return st;
+}
diff --git a/structured-output.h b/structured-output.h
index b43afe0..cba3882 100644
--- a/structured-output.h
+++ b/structured-output.h
@@ -62,3 +62,51 @@ typedef struct structure_printer {
} structure_printer_t;
+/* JSON structure printer */
+
+/* single linked list implementation for keeping track of the array/map
nesting
[PATCH v3 3/3] Use the structured format printer in notmuch search.
This patch uses the previously introduced structured format printer in
notmuch-search.
Most of this patch is necessary to keep the text output unchanged. All tests
pass, both text and JSON are printed exactly the same as they were before.
---
notmuch-search.c | 275 +-
1 file changed, 148 insertions(+), 127 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..321b32a 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -19,6 +19,7 @@
*/
#include notmuch-client.h
+#include structured-output.h
typedef enum {
OUTPUT_SUMMARY,
@@ -28,6 +29,7 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* legacy, only needed for non-structured text output */
typedef struct search_format {
const char *results_start;
const char *item_start;
@@ -64,6 +66,7 @@ format_thread_text (const void *ctx,
const int total,
const char *authors,
const char *subject);
+
static const search_format_t format_text = {
,
,
@@ -78,35 +81,6 @@ static const search_format_t format_text = {
};
static void
-format_item_id_json (const void *ctx,
-const char *item_type,
-const char *item_id);
-
-static void
-format_thread_json (const void *ctx,
- const char *thread_id,
- const time_t date,
- const int matched,
- const int total,
- const char *authors,
- const char *subject);
-
-/* Any changes to the JSON format should be reflected in the file
- * devel/schemata. */
-static const search_format_t format_json = {
-[,
- {,
- format_item_id_json,
- format_thread_json,
- \tags\: [,
- \%s\, , ,
- ], ,\n,
- },
-]\n,
-]\n,
-};
-
-static void
format_item_id_text (unused (const void *ctx),
const char *item_type,
const char *item_id)
@@ -153,50 +127,9 @@ format_thread_text (const void *ctx,
talloc_free (ctx_quote);
}
-static void
-format_item_id_json (const void *ctx,
-unused (const char *item_type),
-const char *item_id)
-{
-void *ctx_quote = talloc_new (ctx);
-
-printf (%s, json_quote_str (ctx_quote, item_id));
-
-talloc_free (ctx_quote);
-
-}
-
-static void
-format_thread_json (const void *ctx,
- const char *thread_id,
- const time_t date,
- const int matched,
- const int total,
- const char *authors,
- const char *subject)
-{
-void *ctx_quote = talloc_new (ctx);
-
-printf (\thread\: %s,\n
- \timestamp\: %ld,\n
- \date_relative\: \%s\,\n
- \matched\: %d,\n
- \total\: %d,\n
- \authors\: %s,\n
- \subject\: %s,\n,
- json_quote_str (ctx_quote, thread_id),
- date,
- notmuch_time_relative_date (ctx, date),
- matched,
- total,
- json_quote_str (ctx_quote, authors),
- json_quote_str (ctx_quote, subject));
-
-talloc_free (ctx_quote);
-}
-
static int
-do_search_threads (const search_format_t *format,
+do_search_threads (const structure_printer_t *format,
+ void *state,
notmuch_query_t *query,
notmuch_sort_t sort,
output_t output,
@@ -209,6 +142,8 @@ do_search_threads (const search_format_t *format,
time_t date;
int first_thread = 1;
int i;
+int outermost_level = 0;
+int items_level = 0;
if (offset 0) {
offset += notmuch_query_count_threads (query);
@@ -220,7 +155,11 @@ do_search_threads (const search_format_t *format,
if (threads == NULL)
return 1;
-fputs (format-results_start, stdout);
+if (format == unstructured_text_printer) {
+ fputs(format_text.results_start, stdout);
+} else { /* structured output */
+ outermost_level = format-list(state);
+}
for (i = 0;
notmuch_threads_valid (threads) (limit 0 || i offset + limit);
@@ -235,43 +174,85 @@ do_search_threads (const search_format_t *format,
continue;
}
- if (! first_thread)
- fputs (format-item_sep, stdout);
+ if (format == unstructured_text_printer ! first_thread)
+ fputs (format_text.item_sep, stdout);
if (output == OUTPUT_THREADS) {
- format-item_id (thread, thread:,
-notmuch_thread_get_thread_id (thread));
+ if (format == unstructured_text_printer) {
+ format_text.item_id (thread, thread:,
+notmuch_thread_get_thread_id (thread));
+ } else { /* structured output */
+ format-string(state,
Re: [PATCH] v2: Added better support for multiple structured output formats.
Since it would be great to use the structure printer for show as well,
it would make sense to put it in its own source file, where it can
easily be shared between commands.
There are a few systematic code formatting problems in your patch. To
be consistent with other notmuch code, there should be a space between
function names and parameter lists (in both declarations and calls)
and there should be a space between a keyword and a paren (e.g., if
( and for (). In a function definition, the function name should
start on a new line (this makes it easy to grep for a function's
definition). Also, in general, try to keep lines under 80 characters
wide (we're not very consistent about this one, but it would be nice
to not become even less consistent about it).
More detailed comments below.
Quoth cra...@gmx.net on Jul 10 at 3:30 pm:
As discussed in id:20120121220407.gk16...@mit.edu, this patch adds
support for new structured output formats (like s-expressions) by using
stateful structure_printers. An implementation of the JSON structure
printer that passes all tests is included. The output for JSON (and
text) is identical to the current output. S-Expressions will be added in
a later patch.
A large part of this patch just implements the differentiation between
structured and non-structured output (all the code within
if(format == unstructured_text_printer)).
In a second patch, the structured output code should be isolated in a
separate file, and also used in all other parts of notmuch.
The interface is a structure structure_printer, which contains the following
methods:
- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to
add key/value pairs. pop is used to close the map (see there). This function
must return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop
is used to close the list. This function must return a nesting level
identifier that can be used to close all nested structures (maps and lists),
backing out to the returned nesting level.
- pop: is called to return to a given nesting level. All lists and maps with
a deeper nesting level must be closed.
- number, string, bool: output one element of the specific type.
All functions should use the state object to insert delimiters etc.
automatically when appropriate.
Example:
int top, one;
top = map(state);
map_key(state, foo);
one = list(state);
number(state, 1);
number(state, 2);
number(state, 3);
pop(state, i);
map_key(state, bar);
map(state);
map_key(state, baaz);
string(state, hello world);
pop(state, top);
would output JSON as follows:
{foo: [1, 2, 3], bar: { baaz: hello world}}
---
notmuch-search.c | 491
---
1 file changed, 361 insertions(+), 130 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index 3be296d..412 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -28,6 +28,210 @@ typedef enum {
OUTPUT_TAGS
} output_t;
+/* structured formatting, useful for JSON, S-Expressions, ...
+
+- initial_state: is called to create a state object, that is passed to all
invocations. This should be used to keep track of the output file and
everything else necessary to correctly format output.
+- map: is called when a new map (associative array, dictionary) is started.
map_key and the primitives (string, number, bool) are used alternatingly to
add key/value pairs. pop is used to close the map (see there). This function
must return a nesting level identifier that can be used to close all nested
structures (maps and lists), backing out to the returned nesting level.
+- list: is called when a new list (array, vector) is started. the primitives
(string, number, bool) are used consecutively to add values to the list. pop
is used to close the list. This function must return a nesting level
identifier that can be used to close all nested structures (maps and lists),
backing out to the returned nesting level.
+- pop: is called to return to a given nesting level. All lists and maps with
a deeper nesting level must be closed.
+- number, string, bool: output one element of the specific type.
+
+All functions should use state to insert delimiters etc. automatically when
appropriate.
+
+Example:
+int top, one;
+top = map(state);
+map_key(state, foo);
+one = list(state);
+number(state, 1);
+number(state, 2);
+number(state, 3);
[PATCH] contib/nmbug/nmbug-status: leftover whitespaces, indentation quoting
The initial nmbug-status was pretty consistent in it's whitespacing
but a few lines had some leftover slips. Those are now corrected.
Also, most of the code used ' as quoting char. As in Python one can
use ' and interchangeably some code used instead of '. However
the usage of those were inconsistent. Now all quotes that python
parses are ':s (only quoted content uses :s).
No functional changes.
---
I tested that this produces identical output compared to the unpatched version.
contrib/nmbug/nmbug-status | 41 -
1 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/contrib/nmbug/nmbug-status b/contrib/nmbug/nmbug-status
index 6aa86a0..2abb3e4 100755
--- a/contrib/nmbug/nmbug-status
+++ b/contrib/nmbug/nmbug-status
@@ -18,10 +18,10 @@ import subprocess
# parse command line arguments
parser = argparse.ArgumentParser()
-parser.add_argument(--text, help=output plain text format,
-action=store_true)
+parser.add_argument('--text', help='output plain text format',
+action='store_true')
-parser.add_argument(--config, help=load config from given file)
+parser.add_argument('--config', help='load config from given file')
args = parser.parse_args()
@@ -31,18 +31,18 @@ args = parser.parse_args()
if args.config != None:
fp = open(args.config)
else:
-nmbhome = os.getenv('NMBGIT', os.path.expanduser(~/.nmbug))
+nmbhome = os.getenv('NMBGIT', os.path.expanduser('~/.nmbug'))
# read only the first line from the pipe
-sha1 = subprocess.Popen(['git', '--git-dir', nmbhome,
- 'show-ref', '-s', 'config'],
- stdout=subprocess.PIPE).stdout.readline()
+sha1 = subprocess.Popen(['git', '--git-dir', nmbhome,
+ 'show-ref', '-s', 'config'],
+stdout=subprocess.PIPE).stdout.readline()
sha1 = sha1.rstrip()
-fp = subprocess.Popen(['git', '--git-dir', nmbhome,
-'cat-file', 'blob', sha1+':status-config.json'],
- stdout=subprocess.PIPE).stdout
+fp = subprocess.Popen(['git', '--git-dir', nmbhome,
+ 'cat-file', 'blob', sha1+':status-config.json'],
+ stdout=subprocess.PIPE).stdout
config = json.load(fp)
@@ -60,11 +60,10 @@ def clear_last():
def print_view(title, query, comment):
-query_string = and .join(query)
+query_string = ' and '.join(query)
q_new = notmuch.Query(db, query_string)
q_new.set_sort(notmuch.Query.SORT.OLDEST_FIRST)
-
last['thread_id'] = ''
if output_format == 'html':
@@ -94,7 +93,7 @@ def print_view(title, query, comment):
val = rfc822.parseaddr(val)[0]
if last[header] == val:
-out[header] =
+out[header] = ''
else:
out[header] = val.encode('utf-8')
last[header] = val
@@ -109,14 +108,14 @@ def print_view(title, query, comment):
else:
br = ''
out['subject'] = 'a href=http://mid.gmane.org/%s;%s/a' \
-% (urllib.quote(mid), out['subject'])
-
-print trtd%s %s % (br, out['date'])
-print /tdtd%s %s % (br, out['id'])
-print /td/tr
-print trtd%s % out['from']
-print /tdtd%s % out['subject']
-print /td/tr\n
+% (urllib.quote(mid), out['subject'])
+
+print ' trtd%s %s' % (br, out['date'])
+print '/tdtd%s %s' % (br, out['id'])
+print '/td/tr'
+print ' trtd%s' % out['from']
+print '/tdtd%s' % out['subject']
+print '/td/tr\n'
else:
print '%(date)-10.10s %(from)-20.20s
%(subject)-40.40s\n%(id)72s\n' % out
--
1.7.1
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
deprecating legacy text output
On Tue, 10 Jul 2012 10:04:10 -0700, Jameson Graef Rollins jroll...@finestructure.net wrote: On Tue, Jul 10 2012, Mark Walters markwalters1...@gmail.com wrote: I think I would also add something saying that the text format is just different (and that a significant chunk of the patch is just that). Can we not just dump this output format once and for all? Does anything use it? And if so can we just modify it to use a more sensible format? I think we need to at least deprecate the format for a release or so, as there are definitely people using it in the wild. As far as I know the at least the Vim interface uses it (what about notmuch-mutt?), in addition to whatever homebrewed scripts might use it. Since scriptability is one of the notmuch core values, I guess we should also provide something more script friendly than json. As you say, it doesn't have to be exactly like what we have now. d ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
notmuch-emacs and bbdb
As far as I can tell, notmuch doesn't integrate as smoothly with bbdb as older emacs mailclients. I'm especially looking for a snarf function that distinguishes sender from recipient. How do other people use bbdb with notmuch? Does anyone have lisp code like that which ships with bbdb for other clients? If I were to find time to write such code, what would you like it to do? cheers, bergey ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
