[notmuch] [PATCH] libtoolize notmuch.
Here is a patch based on quad's libtool branch rebased on current master. It has been tested and verified to work on my machine --- Makefile | 27 --- Makefile.local | 12 lib/Makefile.local |8 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 021fdb8..9022a39 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ WARN_CFLAGS=$(WARN_CXXFLAGS) -Wmissing-declarations # Additional programs that are used during the compilation process. EMACS ?= emacs +libtool = libtool +# Needed so libtool replaces parameters in a modern fashion. +CC = gcc +CXX = g++ # Lowercase to avoid clash with GZIP environment variable for passing # arguments to gzip. gzip = gzip @@ -43,34 +47,35 @@ include Makefile.local # user how to enable verbose compiles. ifeq ($(V),) quiet_DOC := "Use \"$(MAKE) V=1\" to see the verbose compile lines.\n" -quiet = @printf $(quiet_DOC)$(eval quiet_DOC:=)" $1 $2$@\n"; $($1) +V = 0 endif # The user has explicitly enabled quiet compilation. ifeq ($(V),0) -quiet = @printf " $1 $@\n"; $($1) +quiet = @printf $(quiet_DOC)$(eval quiet_DOC:=)" $1 $2$@\n"; $($1) +libtool += --silent endif # Otherwise, print the full command line. quiet ?= $($1) -%.o: %.cc $(all_deps) - $(call quiet,CXX,$(CXXFLAGS)) -c $(FINAL_CXXFLAGS) $< -o $@ +%.lo: %.cc $(all_deps) + $(call quiet,libtool,$(CXXFLAGS)) --mode=compile $(CXX) -c $(FINAL_CXXFLAGS) $< -o $@ -%.o: %.c $(all_deps) - $(call quiet,CC,$(CFLAGS)) -c $(FINAL_CFLAGS) $< -o $@ +%.lo: %.c $(all_deps) + $(call quiet,libtool,$(CFLAGS)) --mode=compile $(CC) -c $(FINAL_CFLAGS) $< -o $@ %.elc: %.el $(call quiet,EMACS) -batch -f batch-byte-compile $< .deps/%.d: %.c $(all_deps) @set -e; rm -f $@; mkdir -p $$(dirname $@) ; \ - $(CC) -M $(CPPFLAGS) $(FINAL_CFLAGS) $< > $@. 2>/dev/null ; \ - sed 's,'$$(basename $*)'\.o[ :]*,$*.o $@ : ,g' < $@. > $@; \ + $(CC) -M $(CPPFLAGS) $(FINAL_CFLAGS) $< > $@.; \ + sed 's,'$$(basename $*)'\.lo[ :]*,$*.lo $@ : ,g' < $@. > $@; \ rm -f $@. .deps/%.d: %.cc $(all_deps) @set -e; rm -f $@; mkdir -p $$(dirname $@) ; \ - $(CXX) -M $(CPPFLAGS) $(FINAL_CXXFLAGS) $< > $@. 2>/dev/null ; \ - sed 's,'$$(basename $*)'\.o[ :]*,$*.o $@ : ,g' < $@. > $@; \ + $(CXX) -M $(CPPFLAGS) $(FINAL_CXXFLAGS) $< > $@.; \ + sed 's,'$$(basename $*)'\.lo[ :]*,$*.lo $@ : ,g' < $@. > $@; \ rm -f $@. DEPS := $(SRCS:%.c=.deps/%.d) @@ -79,4 +84,4 @@ DEPS := $(DEPS:%.cc=.deps/%.d) .PHONY : clean clean: - rm -f $(CLEAN); rm -rf .deps + $(libtool) --mode=clean rm -f $(CLEAN); rm -rf .deps diff --git a/Makefile.local b/Makefile.local index 933ff4c..ccff76c 100644 --- a/Makefile.local +++ b/Makefile.local @@ -20,9 +20,9 @@ notmuch_client_srcs = \ query-string.c \ show-message.c -notmuch_client_modules = $(notmuch_client_srcs:.c=.o) -notmuch: $(notmuch_client_modules) lib/notmuch.a - $(call quiet,CXX,$(LDFLAGS)) $^ $(FINAL_LDFLAGS) -o $@ +notmuch_client_modules = $(notmuch_client_srcs:.c=.lo) +notmuch: $(notmuch_client_modules) lib/libnotmuch.la + $(call quiet,libtool,$(LDFLAGS)) --mode=link $(CXX) $^ $(FINAL_LDFLAGS) -o $@ notmuch.1.gz: notmuch.1 $(call quiet,gzip) --stdout $^ > $@ @@ -32,8 +32,12 @@ install: all notmuch.1.gz do \ install -d $$d ; \ done ; - install notmuch $(DESTDIR)$(prefix)/bin/ + $(libtool) --mode=install install -c lib/libnotmuch.la $(DESTDIR)$(prefix)/lib/ + $(libtool) --mode=install install -c notmuch $(DESTDIR)$(prefix)/bin/ install -m0644 notmuch.1.gz $(DESTDIR)$(prefix)/share/man/man1/ + install contrib/notmuch-completion.bash \ + $(DESTDIR)$(bash_completion_dir)/notmuch + $(libtool) --mode=finish $(DESTDIR)$(prefix)/lib/ install-emacs: install emacs for d in $(DESTDIR)/$(emacs_lispdir) ; \ diff --git a/lib/Makefile.local b/lib/Makefile.local index 70489e1..b03dff6 100644 --- a/lib/Makefile.local +++ b/lib/Makefile.local @@ -17,9 +17,9 @@ libnotmuch_cxx_srcs = \ $(dir)/query.cc \ $(dir)/thread.cc -libnotmuch_modules = $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o) -$(dir)/notmuch.a: $(libnotmuch_modules) - $(call quiet,AR) rcs $@ $^ +libnotmuch_modules = $(libnotmuch_c_srcs:.c=.lo) $(libnotmuch_cxx_srcs:.cc=.lo) +$(dir)/libnotmuch.la: $(libnotmuch_modules) + $(call quiet,libtool) --mode=link $(CXX) -rpath $(DESTDIR)$(prefix)/lib -o $@ $^ SRCS := $(SRCS) $(libnotmuch_c_srcs) $(libnotmuch_cxx_srcs) -CLEAN := $(CLEAN) $(libnotmuch_modules) $(dir)/notmuch.a +CLEAN := $(CLEAN) $(libnotmuch_modules) $(dir)/notmuch.la -- 1.6.3.3
[notmuch] Thoughts on notmuch and Lua
Carl Worth yazm??: > On Fri, 15 Jan 2010 13:16:00 +1300, martin f krafft > wrote: > > Lua for hooks has the advantage that the hooks can be executed in > > the context of manipulateable objects. On the other hand, hooks in > > the style of run-parts directories are more flexible and accessible, > > and could always be invoked as filters for the manipulateable data. > > Good point. > > One thing that notmuch has in its favor already is that there's a > command-line interface that provides all of the interesting > functionality. So it's already a fairly trivial matter to "hook" things > like "notmuch new" by writing a script in whatever language you prefer > to invoke things like "notmuch search" and "notmuch tag". > > And that's what I'm doing already, (with a shell script), to tag > newly-arrived messages. > > The only part that's really missing here is a directory where scripts > could be dropped in and get automatically invoked. > > What do you think, Ali? Would an approach like that satisfy the things > you had in mind for hooks? > It might, here are some thoughts and questions to help you elaborate: - How will these scripts manipulate data? e.g.: A "tagger" script may get the new mail from stdin and print out the new tags which notmuch will read and apply to the message. A "search-filter" script may get search results from stdin and filter them. Just my initial thoughts. This script approach has both upsides and downsides. I have some experience about this because I wrote a tool called mpdcron? which executes hooks based on mpd events. First I thought it would be cool to execute scripts based on events by passing data using environment variables. However this approach didn't give hooks enough power to save state information etc. so I decided to add module support through GModule. I may be going a bit off topic here but what I'm trying to explain is scripts may not give power-users enough options and is practically slower than having Lua, or another scripting language, support. In my humble opinion the former is just a hack while the latter is the proper way to do it. P.S.: As I stated in the beginning you could have elaborated what you meant by script support and I'm just commenting on what I've understood. ?: http://alip.github.com/mpdcron/ > -Carl -- Regards, Ali Polatel -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 198 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/43640272/attachment.pgp>
[notmuch] SWIG (and particularly Python) bindings
Excerpts from Scott Robinson's message of Wed Dec 30 06:34:15 -0500 2009: > Excerpts from Adrian Perez de Castro's message of Wed Dec 30 02:52:23 -0800 > 2009: > > BTW, I think that if more bindings start to appear, Notmuch might be built > > as a shared library, to avoid duplicating it everywhere. One option may be > > using *just* libtool but not the rest of auto-foo tools (for the record: > > I agree with Carl that they are slow and wicked). > > http://github.com/quad/notmuch/tree/libtool Is this branch being considered for merge? The build output arguably isn't as pretty as it was before, but it seems to work. Moreover, my SWIG patches currently depend upon having a shared library involved. - Ben
[notmuch] indexing mail?
Hi Olly, > "Olly" == Olly Betts writes: Olly> On 2010-01-15, Dirk-Jan C Binnema wrote: >>> "Olly" == Olly Betts writes: Olly> Not a full patch, but I already posted what this code should look like Olly> to handle both systems without d_type, and those which return DT_UNKNOWN: >> Olly> http://article.gmane.org/gmane.mail.notmuch.general/1044 >> static gboolean >> _set_dtype (const char* path, struct dirent *entry) Olly> Underscore prefixed identifiers are reserved by ISO C at file-scope; Olly> using them yourself is undefined behaviour... Ah, thanks for reminding, I thought it was __ and _C (capital), but you are right: , (7.1.3 Reserved identifiers) | All identifiers that begin with an underscore and either an uppercase letter | or another underscore are always reserved for any use. | | ? All identifiers that begin with an underscore are always reserved for use as | identifiers with file scope in both the ordinary and tag name spaces. ` >> /* we only care about dirs, regular files and links */ >> if (S_ISREG (statbuf.st_mode)) entry-> d_type = DT_REG; >> else if (S_ISDIR (statbuf.st_mode)) entry-> d_type = DT_DIR; >> else if (S_ISLNK (statbuf.st_mode)) entry-> d_type = DT_LNK; Olly> This addresses the case where the FS returns DT_UNKNOWN for d_type, Olly> but doesn't deal with the case of platforms where struct dirent has Olly> no d_type member - from the Linux readdir man page: Olly> The only fields in the dirent structure that are mandated by Olly> POSIX.1 are: d_name[], of unspecified size, with at most NAME_MAX Olly> characters preceding the terminating null byte; and (as an XSI Olly> extension) d_ino. The other fields are unstandardized, and not Olly> present on all systems; see NOTES below for some further details. Olly> And in NOTES: Olly> Other than Linux, the d_type field is available mainly only on BSD Olly> systems. Yes, my patch could me generalized a bit more, just like your patch could not hardcode the '/'-separator :) In practice though, what Unices in use today do not support d_type? Best wishes, Dirk, -- Dirk-Jan C. Binnema Helsinki, Finland e:djcb at djcbsoftware.nl w:www.djcbsoftware.nl pgp: D09C E664 897D 7D39 5047 A178 E96A C7A1 017D DA3C
[notmuch] indexing mail?
On 2010-01-15, Dirk-Jan C Binnema wrote: >Olly> Other than Linux, the d_type field is available mainly only on BSD >Olly> systems. > > Yes, my patch could me generalized a bit more, just like your patch could not > hardcode the '/'-separator :) Well, '/' works as a directory separator for all Unix systems and also for Microsoft Windows at this level. Is there a system which doesn't accept '/' in this place which is still relevant? Personally I don't see the point in aiming for portability to systems like Mac OS 9, RISC OS, and VMS in 2010... > In practice though, what Unices in use today do not support d_type? Solaris 10 doesn't for starters. I don't have ready access to the other non-Linux, non-BSD Unix flavours to check those right now. Cheers, Olly
[notmuch] [PATCH] libtoolize notmuch.
Here is a patch based on quad's libtool branch rebased on current master. It has been tested and verified to work on my machine --- Makefile | 27 --- Makefile.local | 12 lib/Makefile.local |8 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 021fdb8..9022a39 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ WARN_CFLAGS=$(WARN_CXXFLAGS) -Wmissing-declarations # Additional programs that are used during the compilation process. EMACS ?= emacs +libtool = libtool +# Needed so libtool replaces parameters in a modern fashion. +CC = gcc +CXX = g++ # Lowercase to avoid clash with GZIP environment variable for passing # arguments to gzip. gzip = gzip @@ -43,34 +47,35 @@ include Makefile.local # user how to enable verbose compiles. ifeq ($(V),) quiet_DOC := "Use \"$(MAKE) V=1\" to see the verbose compile lines.\n" -quiet = @printf $(quiet_DOC)$(eval quiet_DOC:=)" $1 $2$...@\n"; $($1) +V = 0 endif # The user has explicitly enabled quiet compilation. ifeq ($(V),0) -quiet = @printf " $1 $...@\n"; $($1) +quiet = @printf $(quiet_DOC)$(eval quiet_DOC:=)" $1 $2$...@\n"; $($1) +libtool += --silent endif # Otherwise, print the full command line. quiet ?= $($1) -%.o: %.cc $(all_deps) - $(call quiet,CXX,$(CXXFLAGS)) -c $(FINAL_CXXFLAGS) $< -o $@ +%.lo: %.cc $(all_deps) + $(call quiet,libtool,$(CXXFLAGS)) --mode=compile $(CXX) -c $(FINAL_CXXFLAGS) $< -o $@ -%.o: %.c $(all_deps) - $(call quiet,CC,$(CFLAGS)) -c $(FINAL_CFLAGS) $< -o $@ +%.lo: %.c $(all_deps) + $(call quiet,libtool,$(CFLAGS)) --mode=compile $(CC) -c $(FINAL_CFLAGS) $< -o $@ %.elc: %.el $(call quiet,EMACS) -batch -f batch-byte-compile $< .deps/%.d: %.c $(all_deps) @set -e; rm -f $@; mkdir -p $$(dirname $@) ; \ - $(CC) -M $(CPPFLAGS) $(FINAL_CFLAGS) $< > $...@. 2>/dev/null ; \ - sed 's,'$$(basename $*)'\.o[ :]*,$*.o $@ : ,g' < $...@. > $@; \ + $(CC) -M $(CPPFLAGS) $(FINAL_CFLAGS) $< > $...@.; \ + sed 's,'$$(basename $*)'\.lo[ :]*,$*.lo $@ : ,g' < $...@. > $@; \ rm -f $...@. .deps/%.d: %.cc $(all_deps) @set -e; rm -f $@; mkdir -p $$(dirname $@) ; \ - $(CXX) -M $(CPPFLAGS) $(FINAL_CXXFLAGS) $< > $...@. 2>/dev/null ; \ - sed 's,'$$(basename $*)'\.o[ :]*,$*.o $@ : ,g' < $...@. > $@; \ + $(CXX) -M $(CPPFLAGS) $(FINAL_CXXFLAGS) $< > $...@.; \ + sed 's,'$$(basename $*)'\.lo[ :]*,$*.lo $@ : ,g' < $...@. > $@; \ rm -f $...@. DEPS := $(SRCS:%.c=.deps/%.d) @@ -79,4 +84,4 @@ DEPS := $(DEPS:%.cc=.deps/%.d) .PHONY : clean clean: - rm -f $(CLEAN); rm -rf .deps + $(libtool) --mode=clean rm -f $(CLEAN); rm -rf .deps diff --git a/Makefile.local b/Makefile.local index 933ff4c..ccff76c 100644 --- a/Makefile.local +++ b/Makefile.local @@ -20,9 +20,9 @@ notmuch_client_srcs = \ query-string.c \ show-message.c -notmuch_client_modules = $(notmuch_client_srcs:.c=.o) -notmuch: $(notmuch_client_modules) lib/notmuch.a - $(call quiet,CXX,$(LDFLAGS)) $^ $(FINAL_LDFLAGS) -o $@ +notmuch_client_modules = $(notmuch_client_srcs:.c=.lo) +notmuch: $(notmuch_client_modules) lib/libnotmuch.la + $(call quiet,libtool,$(LDFLAGS)) --mode=link $(CXX) $^ $(FINAL_LDFLAGS) -o $@ notmuch.1.gz: notmuch.1 $(call quiet,gzip) --stdout $^ > $@ @@ -32,8 +32,12 @@ install: all notmuch.1.gz do \ install -d $$d ; \ done ; - install notmuch $(DESTDIR)$(prefix)/bin/ + $(libtool) --mode=install install -c lib/libnotmuch.la $(DESTDIR)$(prefix)/lib/ + $(libtool) --mode=install install -c notmuch $(DESTDIR)$(prefix)/bin/ install -m0644 notmuch.1.gz $(DESTDIR)$(prefix)/share/man/man1/ + install contrib/notmuch-completion.bash \ + $(DESTDIR)$(bash_completion_dir)/notmuch + $(libtool) --mode=finish $(DESTDIR)$(prefix)/lib/ install-emacs: install emacs for d in $(DESTDIR)/$(emacs_lispdir) ; \ diff --git a/lib/Makefile.local b/lib/Makefile.local index 70489e1..b03dff6 100644 --- a/lib/Makefile.local +++ b/lib/Makefile.local @@ -17,9 +17,9 @@ libnotmuch_cxx_srcs = \ $(dir)/query.cc \ $(dir)/thread.cc -libnotmuch_modules = $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o) -$(dir)/notmuch.a: $(libnotmuch_modules) - $(call quiet,AR) rcs $@ $^ +libnotmuch_modules = $(libnotmuch_c_srcs:.c=.lo) $(libnotmuch_cxx_srcs:.cc=.lo) +$(dir)/libnotmuch.la: $(libnotmuch_modules) + $(call quiet,libtool) --mode=link $(CXX) -rpath $(DESTDIR)$(prefix)/lib -o $@ $^ SRCS := $(SRCS) $(libnotmuch_c_srcs) $(libnotmuch_cxx_srcs) -CLEAN := $(CLEAN) $(libnotmuch_modules) $(dir)/notmuch.a +CLEAN := $(CLEAN) $(libnotmuch_modules) $(dir)/notmuch.la -- 1.6.3.3 ___ notmuch mailin
Re: [notmuch] SWIG (and particularly Python) bindings
Excerpts from Scott Robinson's message of Wed Dec 30 06:34:15 -0500 2009: > Excerpts from Adrian Perez de Castro's message of Wed Dec 30 02:52:23 -0800 > 2009: > > BTW, I think that if more bindings start to appear, Notmuch might be built > > as a shared library, to avoid duplicating it everywhere. One option may be > > using *just* libtool but not the rest of auto-foo tools (for the record: > > I agree with Carl that they are slow and wicked). > > http://github.com/quad/notmuch/tree/libtool Is this branch being considered for merge? The build output arguably isn't as pretty as it was before, but it seems to work. Moreover, my SWIG patches currently depend upon having a shared library involved. - Ben ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [notmuch] Bug with commit 2e96464f9705be4ec772280cad71a6c9d5831e6f
I still confirm the bug.
The problem is due to relying on non-standardized fields of directory entries
(i.e. d_type), which don't behave the
same on reiserfs than on ext2 (I use reiserfs).
The following ugly patch "solves" my problem.
diff --git a/notmuch-new.c b/notmuch-new.c
index b740ee2..42d9f89 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -359,7 +359,7 @@ add_files_recursive (notmuch_database_t *notmuch,
if (! S_ISREG (st.st_mode))
continue;
- } else if (entry->d_type != DT_REG) {
+ } else if (entry->d_type != DT_REG && entry->d_type != 0) {
continue;
}
With this patch, mails are added by the database, but some directories also
have d_type set to 0
and can slip through.
It seems that the only portable way to test for directory types is stat, which
is expensive... I don't known
what's the best way to go.
Matthieu
- Mail Original -
De: "Ali Polatel"
À: notmuch@notmuchmail.org
Envoyé: Mercredi 13 Janvier 2010 11h21:52 GMT +01:00 Amsterdam / Berlin / Berne
/ Rome / Stockholm / Vienne
Objet: Re: [notmuch] Bug with commit 2e96464f9705be4ec772280cad71a6c9d5831e6f
Ali Polatel yazmış:
> ra...@free.fr yazmış:
> > Hello,
> >
> > I just updated notmuch and now notmuch new cannot update my mail anymore...
> > It tells me that there are
> > 700 files found, but tells that there's no new mail.
> >
> > I did a git bisect, which tells me the first bad commit is commit
> > 2e96464f9705be4ec772280cad71a6c9d5831e6f.
> >
> > I did not try to use the new xapian database or to update xapian; maybe
> > this is the problem.
> >
> > I tested with several tools to get mail in the maildir format, including
> > mb2md and getmail, and I always get the problem.
> Same problem here, I tried upgrading xapian to xapian-core-1.1.3_svn13824,
> the problem persists.
> Here's what happens here:
> 3074 a...@harikalardiyari> rm -fr .maildir/.notmuch
> 3075 a...@harikalardiyari> notmuch new
> Found 210302 total files (that's not much mail).
> No new mail.
> 3076 a...@harikalardiyari> notmuch search from:a...@exherbo.org
> 3077 a...@harikalardiyari>
>
After doing git bisect which gave random results and testing notmuch
using cworth's notmuch-test script on another file system I figured out
my problem was due to file system corruption.
Sorry for the noise...
--
Regards,
Ali Polatel
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
patch
Description: Binary data
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] asynch operations protocol
On Fri, 15 Jan 2010 09:59:54 -0400, David Bremner wrote:
> Is this over/under engineered? I spent roughly as long on the design as
> it took me to type :). Maybe the whole session id thing is redundant and
> could be done at the socket level. Or, getting more serious about the
> whole thing, maybe each queue operation should return an identifier.
The asynchronous interface I work with most is MPI. There you get a
Request object when the operation is initiated and you can
{test,block}{one,some,any,all}, where the latter takes a list of
requests. These variants are all useful, but of course they could be
implemented as needed. I don't think that being able to support these
variants places any particular burden on the design.
I believe in performing operations with appropriate granularity, so I
wouldn't expect cases where you need to manage thousands of active
requests, thus I'm not sure the "session" grouping offers any real
benefit. In any case, I'm not in favor of a single global flush.
Jed
Re: [notmuch] Thoughts on notmuch and Lua
On Fri, 15 Jan 2010 23:09:34 +0200, Ali Polatel wrote: > Carl Worth yazmış: > > What do you think, Ali? Would an approach like that satisfy the things > > you had in mind for hooks? > > It might, here are some thoughts and questions to help you elaborate: > > - How will these scripts manipulate data? > e.g.: A "tagger" script may get the new mail from stdin and print out > the new tags which notmuch will read and apply to the message. This can happen with current notmuch entirely with no real "hook" support. We've talked about switching from default tags of "inbox" and "unread" to simply having new mail tagged with a "new" tag. So a "tagger" script could operate simply by doing a "notmuch search" for messages with the "new" tag and could iterate over the filenames to process actual messages. (We don't have support now for emitting just filenames from a "notmuch search", but we have patches for that, and I'll be applying one soon.) So that's a taste for the "scriptability" I see in the current notmuch system that makes it really much more flexible than any "hooks" system. Additional flexibility comes from: * User can run a script like this at any point---not merely when messages are added. * User script isn't restricted to dealing only with "new" messages, but can act on any set of messages based on any search constraint, (or even all messages in the database). The results can then be applied by simply calling "notmuch tag" as needed. And if there are any performance problems there we can fix them, (such as, perhaps we'll end up wanting this script to be able to invoke a single process for all of its tagging rather than calling "notmuch tag" over and over). > A "search-filter" script may get search results from stdin and filter > them. Just my initial thoughts. And how would this search functionality and filtering be different than the search functionality provided by notmuch itself? I can think of at least a couple of ways it might be different: 1. It would be nice to be able to filter based on tags that are present in a thread, though perhaps not present in any message matching the original search. An obvious application of this is the "thread muting" feature, where once a message is tagged as "muted", no messages delivered to that thread in the future will appear in the inbox. This is a feature I'd like to put into the core of notmuch such that one passes a query to match messages and then also a second query to filter based on the collected tags in threads. Something like: notmuch search tag:inbox --filter="not tag:muted" 2. There are other details available at the thread level that are not available at the level at which message-based searching happens. A simple example of this would be the ability to search for threads with a single message, (perhaps checking to ensure that all requests had gotten at least one reply). But one can imagine more complex things as well, "Show me all threads where ImportantPerson sent a message and where I never replied in the thread." For this kind of thing, I think we simply want to build on the output of "notmuch search". The current output isn't very usable for this, but with things like the structured json output, etc. (which, again, I hope to be merging soon), it would be quite easy to write new tools that accept that output and provide additional searching and filtering, etc. And that tool could provide lua-based scripting or whatever else is desired. So my feeling is that if anything can live outside of notmuch, then it should, and should simply build on top of notmuch output. (And we should fix notmuch output to support that well.) And anything that must live within notmuch (or is best supported there), we should see if we can't just make that a core part of notmuch itself, (such as the --filter option I showed above). I'm *still* not wanting to squelch any experimentation with embedding scripting languages inside notmuch, or anything else. I'm just still not seeing anything that requires this. Look at the amount of emacs-lisp code we've written, for example, and the various things it does, (hiding away citations, etc.). That's all "scripting code", but that sits easily on top of the existing notmuch command-line tool. I think I'd prefer to keep that nice clean boundary until we find something that really requires changing that. But, show me something really cool that requires it, and you might convince me. :-) -Carl pgpNLUVWuTVpx.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] Thoughts on notmuch and Lua
On Fri, 15 Jan 2010 23:09:34 +0200, Ali Polatel wrote: > Carl Worth yazm??: > > What do you think, Ali? Would an approach like that satisfy the things > > you had in mind for hooks? > > It might, here are some thoughts and questions to help you elaborate: > > - How will these scripts manipulate data? > e.g.: A "tagger" script may get the new mail from stdin and print out > the new tags which notmuch will read and apply to the message. This can happen with current notmuch entirely with no real "hook" support. We've talked about switching from default tags of "inbox" and "unread" to simply having new mail tagged with a "new" tag. So a "tagger" script could operate simply by doing a "notmuch search" for messages with the "new" tag and could iterate over the filenames to process actual messages. (We don't have support now for emitting just filenames from a "notmuch search", but we have patches for that, and I'll be applying one soon.) So that's a taste for the "scriptability" I see in the current notmuch system that makes it really much more flexible than any "hooks" system. Additional flexibility comes from: * User can run a script like this at any point---not merely when messages are added. * User script isn't restricted to dealing only with "new" messages, but can act on any set of messages based on any search constraint, (or even all messages in the database). The results can then be applied by simply calling "notmuch tag" as needed. And if there are any performance problems there we can fix them, (such as, perhaps we'll end up wanting this script to be able to invoke a single process for all of its tagging rather than calling "notmuch tag" over and over). > A "search-filter" script may get search results from stdin and filter > them. Just my initial thoughts. And how would this search functionality and filtering be different than the search functionality provided by notmuch itself? I can think of at least a couple of ways it might be different: 1. It would be nice to be able to filter based on tags that are present in a thread, though perhaps not present in any message matching the original search. An obvious application of this is the "thread muting" feature, where once a message is tagged as "muted", no messages delivered to that thread in the future will appear in the inbox. This is a feature I'd like to put into the core of notmuch such that one passes a query to match messages and then also a second query to filter based on the collected tags in threads. Something like: notmuch search tag:inbox --filter="not tag:muted" 2. There are other details available at the thread level that are not available at the level at which message-based searching happens. A simple example of this would be the ability to search for threads with a single message, (perhaps checking to ensure that all requests had gotten at least one reply). But one can imagine more complex things as well, "Show me all threads where ImportantPerson sent a message and where I never replied in the thread." For this kind of thing, I think we simply want to build on the output of "notmuch search". The current output isn't very usable for this, but with things like the structured json output, etc. (which, again, I hope to be merging soon), it would be quite easy to write new tools that accept that output and provide additional searching and filtering, etc. And that tool could provide lua-based scripting or whatever else is desired. So my feeling is that if anything can live outside of notmuch, then it should, and should simply build on top of notmuch output. (And we should fix notmuch output to support that well.) And anything that must live within notmuch (or is best supported there), we should see if we can't just make that a core part of notmuch itself, (such as the --filter option I showed above). I'm *still* not wanting to squelch any experimentation with embedding scripting languages inside notmuch, or anything else. I'm just still not seeing anything that requires this. Look at the amount of emacs-lisp code we've written, for example, and the various things it does, (hiding away citations, etc.). That's all "scripting code", but that sits easily on top of the existing notmuch command-line tool. I think I'd prefer to keep that nice clean boundary until we find something that really requires changing that. But, show me something really cool that requires it, and you might convince me. :-) -Carl -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/aaacd133/attachment.pgp>
[notmuch] Xapian::DatabaseError on notmuch new upgrade
Looks like this was just a bug in Xapian HEAD, today's build works just fine and I'm enjoying the faster CHERT backend. Bulk tagging is really loads faster now, and the interactivity is better than before, but SPC still has a noticeable pause (at least 200 ms) since (apparently) it's doing an fsync, and this is worse when the disk is getting hit by some other job (like when I'm reading email while something big compiles). It wouldn't bother me at all if I lost my last few seconds of interactive tagging due to something catastrophic like losing power. I think there is still (post #250) a case for supporting some asynchronous operations. Jed
Re: [notmuch] indexing mail?
On 2010-01-15, Dirk-Jan C Binnema wrote: >Olly> Other than Linux, the d_type field is available mainly only on BSD >Olly> systems. > > Yes, my patch could me generalized a bit more, just like your patch could not > hardcode the '/'-separator :) Well, '/' works as a directory separator for all Unix systems and also for Microsoft Windows at this level. Is there a system which doesn't accept '/' in this place which is still relevant? Personally I don't see the point in aiming for portability to systems like Mac OS 9, RISC OS, and VMS in 2010... > In practice though, what Unices in use today do not support d_type? Solaris 10 doesn't for starters. I don't have ready access to the other non-Linux, non-BSD Unix flavours to check those right now. Cheers, Olly ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] Thoughts on notmuch and Lua
also sprach Carl Worth [2010.01.15.1200 +1300]: > > Lua has many advantages over other scripting languages when it > > comes to integration with a C program. It has a very clean and > > easy C API, the overhead of running Lua scripts is not noticable > > among other things. > > I've definitely heard lots of good things about "lua > embedability". So if we do decide to provide hooks, then lua would > seem like a logical option to look at first. I've never looked at > it closely myself though. Lua for hooks has the advantage that the hooks can be executed in the context of manipulateable objects. On the other hand, hooks in the style of run-parts directories are more flexible and accessible, and could always be invoked as filters for the manipulateable data. -- martin | http://madduck.net/ | http://two.sentenc.es/ "imagine if every thursday your shoes exploded if you tied them the usual way. this happens to us all the time with computers, and nobody thinks of complaining." -- jeff raskin spamtraps: madduck.bogus at madduck.net -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 198 bytes Desc: Digital signature (see http://martin-krafft.net/gpg/) URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/66e1dde1/attachment.pgp>
Re: [notmuch] Thoughts on notmuch and Lua
Carl Worth yazmış: > On Fri, 15 Jan 2010 13:16:00 +1300, martin f krafft > wrote: > > Lua for hooks has the advantage that the hooks can be executed in > > the context of manipulateable objects. On the other hand, hooks in > > the style of run-parts directories are more flexible and accessible, > > and could always be invoked as filters for the manipulateable data. > > Good point. > > One thing that notmuch has in its favor already is that there's a > command-line interface that provides all of the interesting > functionality. So it's already a fairly trivial matter to "hook" things > like "notmuch new" by writing a script in whatever language you prefer > to invoke things like "notmuch search" and "notmuch tag". > > And that's what I'm doing already, (with a shell script), to tag > newly-arrived messages. > > The only part that's really missing here is a directory where scripts > could be dropped in and get automatically invoked. > > What do you think, Ali? Would an approach like that satisfy the things > you had in mind for hooks? > It might, here are some thoughts and questions to help you elaborate: - How will these scripts manipulate data? e.g.: A "tagger" script may get the new mail from stdin and print out the new tags which notmuch will read and apply to the message. A "search-filter" script may get search results from stdin and filter them. Just my initial thoughts. This script approach has both upsides and downsides. I have some experience about this because I wrote a tool called mpdcron¹ which executes hooks based on mpd events. First I thought it would be cool to execute scripts based on events by passing data using environment variables. However this approach didn't give hooks enough power to save state information etc. so I decided to add module support through GModule. I may be going a bit off topic here but what I'm trying to explain is scripts may not give power-users enough options and is practically slower than having Lua, or another scripting language, support. In my humble opinion the former is just a hack while the latter is the proper way to do it. P.S.: As I stated in the beginning you could have elaborated what you meant by script support and I'm just commenting on what I've understood. ¹: http://alip.github.com/mpdcron/ > -Carl -- Regards, Ali Polatel pgp5dmSI14ERl.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [notmuch] Thoughts on notmuch and Lua
On Fri, 15 Jan 2010 13:16:00 +1300, martin f krafft wrote: > Lua for hooks has the advantage that the hooks can be executed in > the context of manipulateable objects. On the other hand, hooks in > the style of run-parts directories are more flexible and accessible, > and could always be invoked as filters for the manipulateable data. Good point. One thing that notmuch has in its favor already is that there's a command-line interface that provides all of the interesting functionality. So it's already a fairly trivial matter to "hook" things like "notmuch new" by writing a script in whatever language you prefer to invoke things like "notmuch search" and "notmuch tag". And that's what I'm doing already, (with a shell script), to tag newly-arrived messages. The only part that's really missing here is a directory where scripts could be dropped in and get automatically invoked. What do you think, Ali? Would an approach like that satisfy the things you had in mind for hooks? -Carl pgpTUijV9mm08.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] Thoughts on notmuch and Lua
On Fri, 15 Jan 2010 13:16:00 +1300, martin f krafft wrote: > Lua for hooks has the advantage that the hooks can be executed in > the context of manipulateable objects. On the other hand, hooks in > the style of run-parts directories are more flexible and accessible, > and could always be invoked as filters for the manipulateable data. Good point. One thing that notmuch has in its favor already is that there's a command-line interface that provides all of the interesting functionality. So it's already a fairly trivial matter to "hook" things like "notmuch new" by writing a script in whatever language you prefer to invoke things like "notmuch search" and "notmuch tag". And that's what I'm doing already, (with a shell script), to tag newly-arrived messages. The only part that's really missing here is a directory where scripts could be dropped in and get automatically invoked. What do you think, Ali? Would an approach like that satisfy the things you had in mind for hooks? -Carl -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/d26d0c3b/attachment.pgp>
Re: [notmuch] indexing mail?
On Fri, 15 Jan 2010 21:57:32 +0200, Dirk-Jan C. Binnema wrote: > Olly> Underscore prefixed identifiers are reserved by ISO C at file-scope; > Olly> using them yourself is undefined behaviour... > > Ah, thanks for reminding, I thought it was __ and _C (capital), but you are > right: > > , (7.1.3 Reserved identifiers) > | All identifiers that begin with an underscore and either an uppercase > letter > | or another underscore are always reserved for any use. > | > | — All identifiers that begin with an underscore are always reserved for use > as > | identifiers with file scope in both the ordinary and tag name spaces. > ` But please don't be too strict about this. Please feel very free to use any identifier with a _notmuch prefix. And really, feel free to use just about any underscore-prefixed identifier that you want that doesn't clash with anything on your system. Then if we do identify an actual clash somewhere then we can fix it. I think it was stupid of Posix to steal _ and a reserved prefix, and I really don't have a problem ignoring that. This is like I described in a recent mail---trying to prevent all portability problems just isn't worth the effort. It's much easier to fix problems that actually occur in practice. -Carl pgpKA3Dr7krKE.pgp Description: PGP signature ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] indexing mail?
On Fri, 15 Jan 2010 21:57:32 +0200, Dirk-Jan C. Binnema wrote: > Olly> Underscore prefixed identifiers are reserved by ISO C at file-scope; > Olly> using them yourself is undefined behaviour... > > Ah, thanks for reminding, I thought it was __ and _C (capital), but you are > right: > > , (7.1.3 Reserved identifiers) > | All identifiers that begin with an underscore and either an uppercase > letter > | or another underscore are always reserved for any use. > | > | ? All identifiers that begin with an underscore are always reserved for use > as > | identifiers with file scope in both the ordinary and tag name spaces. > ` But please don't be too strict about this. Please feel very free to use any identifier with a _notmuch prefix. And really, feel free to use just about any underscore-prefixed identifier that you want that doesn't clash with anything on your system. Then if we do identify an actual clash somewhere then we can fix it. I think it was stupid of Posix to steal _ and a reserved prefix, and I really don't have a problem ignoring that. This is like I described in a recent mail---trying to prevent all portability problems just isn't worth the effort. It's much easier to fix problems that actually occur in practice. -Carl -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/95f8a69e/attachment.pgp>
Re: [notmuch] indexing mail?
Hi Olly, > "Olly" == Olly Betts writes: Olly> On 2010-01-15, Dirk-Jan C Binnema wrote: >>> "Olly" == Olly Betts writes: Olly> Not a full patch, but I already posted what this code should look like Olly> to handle both systems without d_type, and those which return DT_UNKNOWN: >> Olly> http://article.gmane.org/gmane.mail.notmuch.general/1044 >> static gboolean >> _set_dtype (const char* path, struct dirent *entry) Olly> Underscore prefixed identifiers are reserved by ISO C at file-scope; Olly> using them yourself is undefined behaviour... Ah, thanks for reminding, I thought it was __ and _C (capital), but you are right: , (7.1.3 Reserved identifiers) | All identifiers that begin with an underscore and either an uppercase letter | or another underscore are always reserved for any use. | | — All identifiers that begin with an underscore are always reserved for use as | identifiers with file scope in both the ordinary and tag name spaces. ` >> /* we only care about dirs, regular files and links */ >> if (S_ISREG (statbuf.st_mode)) entry-> d_type = DT_REG; >> else if (S_ISDIR (statbuf.st_mode)) entry-> d_type = DT_DIR; >> else if (S_ISLNK (statbuf.st_mode)) entry-> d_type = DT_LNK; Olly> This addresses the case where the FS returns DT_UNKNOWN for d_type, Olly> but doesn't deal with the case of platforms where struct dirent has Olly> no d_type member - from the Linux readdir man page: Olly> The only fields in the dirent structure that are mandated by Olly> POSIX.1 are: d_name[], of unspecified size, with at most NAME_MAX Olly> characters preceding the terminating null byte; and (as an XSI Olly> extension) d_ino. The other fields are unstandardized, and not Olly> present on all systems; see NOTES below for some further details. Olly> And in NOTES: Olly> Other than Linux, the d_type field is available mainly only on BSD Olly> systems. Yes, my patch could me generalized a bit more, just like your patch could not hardcode the '/'-separator :) In practice though, what Unices in use today do not support d_type? Best wishes, Dirk, -- Dirk-Jan C. Binnema Helsinki, Finland e:d...@djcbsoftware.nl w:www.djcbsoftware.nl pgp: D09C E664 897D 7D39 5047 A178 E96A C7A1 017D DA3C ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] Threading
also sprach Carl Worth [2010.01.15.1108 +1300]: > > Reading is one thing. Information storage and organisation is > > another. After a message is delivered (and read) to my mailbox, > > it's really mine and I can (and should be able) to affix it and > > integrate it into my organisational scheme any way I want, don't > > you think? > > A fair point. > > I don't see this being something I'm going to spend any time > implementing. I just wouldn't use the functionality myself. But > I would be happy to integrate patches if someone came up with > some. Maybe I should try to persuade you in person. Just today I referenced a discussion I had with a client's ISP, which was done via a web-based support system (custhelp.com). They send you e-mail for every post you or they make to the thread, but those e-mails do not reference each other. Fortunately, I stitched them together and when I searched for the correspondence in my mailstore, I had the entire thread available to me, which was handy (thanks to mutt's useful thread handling abilities). -- martin | http://madduck.net/ | http://two.sentenc.es/ "this week dragged past me so slowly; the days fell on their knees..." -- david bowie spamtraps: madduck.bogus at madduck.net -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 198 bytes Desc: Digital signature (see http://martin-krafft.net/gpg/) URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/9dc8de9c/attachment-0001.pgp>
[notmuch] Idea for storing tags
also sprach Carl Worth [2010.01.15.1124 +1300]: > > You might have marked a message 'read' on one machine and if the two > > get out of sync on another machine, you might have the same message > > unread there. > > That's a different issue though. With two databases there's clearly the > opportunity for the two databases to be out of synch. > > But you talked about the database being out of synch with respect to the > mailstore. And that's something I just don't understand, (given the > assumption that all tags are stored in the database---which was the > explicit description of the case of interest). Yes, we are talking about the situation where the tagstore is seperate from the mailstore, and that they are both synchronised with a server, or between machines, separately. If for some reason you only synchronise the mailstore ? say because the connection drops before the sync of the tagstore completes ? then you end up with an out-of-sync situation, because the mailstore-sync will have pulled in a new message, but not the associated tags. So if you had already read this message on another machine and tagged it 'done', then it would show up on this machine as 'new' without the 'done' tag, because the tags were not synchronised. The only way to really solve this is by transferring a message and its tags in a transactional way. > > Shouldn't this just be solved? I've had formail+procmail delete my > > duplicates for 10+ years, and while I don't like the fact that > > I usually get the CC before the list mail, and thus cannot filter on > > Delivered-To, I have never looked back. > > Notmuch has access to all the information it needs to allow you to > delete the CC version once the list mail arrives. So you could do > notmuch-based deletion now and avoid losing the Delivered-To header if > you want. Of course. I hadn't thought that far. However, there are still benefits to formail, namely avoiding having to run duplicates through potentially expensive spamfilters. > I think that synchronizing the mail store and synchronizing the > tags information are tasks that have different requirements, and > for which we may well want different tools. Fair enough. Maybe I am just paranoid about the stores getting out of sync (see above). -- martin | http://madduck.net/ | http://two.sentenc.es/ "we all know linux is great... it does infinite loops in 5 seconds." -- linus torvalds spamtraps: madduck.bogus at madduck.net -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 198 bytes Desc: Digital signature (see http://martin-krafft.net/gpg/) URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/ca89e75e/attachment.pgp>
[notmuch] [PATCH] [RFC] notmuch search: Return a non-zero exitcode if the search returns no hits.
If the user is explicitly searching for a message, then if notmuch
fails to find it, it is useful to set a failure exit code. This makes it
easier for scripting. However, we will want to then distinguish between
fatal errors (such as out-of-memory, invalid arguments, corrupt db, etc)
from the simple no message. To that end, we introduce a set of
enumerated exit-codes which may prove useful to use consistently across
notmuch. (I'm not that convinced highly differentiated errors is
particularly useful, though separate CONFIG_ERROR and DATABASE_ERROR
would seem to have value for scripts.)
---
notmuch-search.c | 34 --
1 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index dc44eb6..6ca903b 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -20,7 +20,17 @@
#include "notmuch-client.h"
-static void
+/* XXX After some review, make these universal? */
+enum {
+NOTMUCH_EXIT_SUCCESS = 0,
+NOTMUCH_EXIT_NOT_FOUND,
+NOTMUCH_EXIT_INVALID_ARGUMENT,
+NOTMUCH_EXIT_SYSTEM_ERROR, /* XXX just out-of-memory? */
+NOTMUCH_EXIT_CONFIG_ERROR,
+NOTMUCH_EXIT_DATABASE_ERROR,
+};
+
+static int
do_search_threads (const void *ctx,
notmuch_query_t *query,
notmuch_sort_t sort)
@@ -30,6 +40,7 @@ do_search_threads (const void *ctx,
notmuch_tags_t *tags;
time_t date;
const char *relative_date;
+int count = 0;
for (threads = notmuch_query_search_threads (query);
notmuch_threads_has_more (threads);
@@ -67,7 +78,10 @@ do_search_threads (const void *ctx,
printf (")\n");
notmuch_thread_destroy (thread);
+ count++;
}
+
+return count;
}
int
@@ -94,11 +108,11 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
sort = NOTMUCH_SORT_NEWEST_FIRST;
} else {
fprintf (stderr, "Invalid value for --sort: %s\n", opt);
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
} else {
fprintf (stderr, "Unrecognized option: %s\n", argv[i]);
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
}
@@ -107,35 +121,35 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
config = notmuch_config_open (ctx, NULL, NULL);
if (config == NULL)
- return 1;
+ return NOTMUCH_EXIT_CONFIG_ERROR;
notmuch = notmuch_database_open (notmuch_config_get_database_path (config),
NOTMUCH_DATABASE_MODE_READ_ONLY);
if (notmuch == NULL)
- return 1;
+ return NOTMUCH_EXIT_DATABASE_ERROR;
query_str = query_string_from_args (ctx, argc, argv);
if (query_str == NULL) {
fprintf (stderr, "Out of memory.\n");
- return 1;
+ return NOTMUCH_EXIT_SYSTEM_ERROR;
}
if (*query_str == '\0') {
fprintf (stderr, "Error: notmuch search requires at least one search
term.\n");
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
query = notmuch_query_create (notmuch, query_str);
if (query == NULL) {
fprintf (stderr, "Out of memory\n");
- return 1;
+ return NOTMUCH_EXIT_SYSTEM_ERROR;
}
notmuch_query_set_sort (query, sort);
-do_search_threads (ctx, query, sort);
+i = do_search_threads (ctx, query, sort);
notmuch_query_destroy (query);
notmuch_database_close (notmuch);
-return 0;
+return i ? NOTMUCH_EXIT_SUCCESS : NOTMUCH_EXIT_NOT_FOUND;
}
--
1.6.6
[notmuch] asynch operations protocol
On Fri, 15 Jan 2010 14:49:14 +0100, Jed Brown wrote: > It wouldn't bother me at all if I lost my last few seconds of > interactive tagging due to something catastrophic like losing power. I > think there is still (post #250) a case for supporting some asynchronous > operations. I was wondering what the protocol would be like for non-blocking commands to a notmuch daemon. I have no experience with these things, but I was thinking something along the lines of (not worrying about wire encoding) open_session - basically just generates a unique id to allow grabbing results of commands to be collected. queue command session arguments I guess the command dispatcher could just be a thin replacement for command-line argument parsing. gather session return all output from session flush session close session Is this over/under engineered? I spent roughly as long on the design as it took me to type :). Maybe the whole session id thing is redundant and could be done at the socket level. Or, getting more serious about the whole thing, maybe each queue operation should return an identifier. OK, discuss amongst yourselves ;) d
[notmuch] Potential problem using Git for mail (was: Idea for storing tags)
also sprach Asheesh Laroia [2010.01.14.2112 +1300]: > Sure. But the MDA doesn't need to do the commit immediately. Since > (presumably) we're using Maildir, the MDA on the mail receiving > server is going to generate filenames that won't cause conflicts. > So it's okay to leave the files uncommitted. So when does the commit happen? > When I did the "git merge", git would create the Maildir files in > ~/Maildir/cur/... non-atomically. This might be something that the Git people could address if it was brought up on the mailing list. Then again, it might not be possible without going via a temporary file, which I doubt will fly. I suppose that I never actually considered merges on the IMAP server side, but obviously the IMAP server has to work off a clone, and that means it needs to merge. > Dovecot would notice the file in ~/Maildir/cur/ and think, "This > file must be ready!" So it would parse it even though git hadn't > finished writing it. This caused me to only see partial headers in > Alpine since Dovecot parsed it before it was a complete message. I wonder if a custom merge driver could address this to properly use ?/tmp/ to assemble the message and only then move it. -- martin | http://madduck.net/ | http://two.sentenc.es/ "this week dragged past me so slowly; the days fell on their knees..." -- david bowie spamtraps: madduck.bogus at madduck.net -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 198 bytes Desc: Digital signature (see http://martin-krafft.net/gpg/) URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/7814b895/attachment.pgp>
[notmuch] indexing mail?
> "Olly" == Olly Betts writes:
Olly> On 2010-01-14, Carl Worth wrote:
>> On Thu, 14 Jan 2010 18:38:54 +0100, Adrian Perez de Castro wrote:
>>> I am using XFS, which always returns DT_UNKNOWN. Taking into account
that
>>> there is a good deal of people using filesystems other than the ones you
>>> mention, and that other non-linux filesystems may also return
DT_UNKNOWN,
>>> in my opinion there should be a fall-back. I will try to post a patch
>>> Anytime Soon=E2=84=A2.
>>
>> We definitely want the fallback. I can attempt to code it, but I don't
>> have ready access to an afflicted filesystem, so I'd need help testing
>> anyway.
>>
>> I'd love to see a patch for this bug soon. Be sure to CC me when the
>> patch is sent and that will help me commit it sooner.
Olly> Not a full patch, but I already posted what this code should look like
Olly> to handle both systems without d_type, and those which return
DT_UNKNOWN:
Olly> http://article.gmane.org/gmane.mail.notmuch.general/1044
I take a slighly different approach in mu:
/* if the file system does not support entry->d_type, we add it ourselves
* this is slower (extra stat) but at least it works
*/
static gboolean
_set_dtype (const char* path, struct dirent *entry)
{
struct stat statbuf;
char fullpath[4096];
snprintf (fullpath, sizeof(fullpath), "%s%c%s",
path, G_DIR_SEPARATOR, entry->d_name);
if (stat (fullpath, &statbuf) != 0) {
g_warning ("stat failed on %s: %s", fullpath,
strerror(errno));
return FALSE;
}
/* we only care about dirs, regular files and links */
if (S_ISREG (statbuf.st_mode))
entry->d_type = DT_REG;
else if (S_ISDIR (statbuf.st_mode))
entry->d_type = DT_DIR;
else if (S_ISLNK (statbuf.st_mode))
entry->d_type = DT_LNK;
return TRUE;
}
and then in some other places:
/* handle FSs that don't support entry->d_type */
if (entry->d_type == DT_UNKNOWN)
_set_dtype (dirname, entry);
Note, that is untested as of yet.
Best wishes,
Dirk.
--
Dirk-Jan C. Binnema Helsinki, Finland
e:djcb at djcbsoftware.nl w:www.djcbsoftware.nl
pgp: D09C E664 897D 7D39 5047 A178 E96A C7A1 017D DA3C
[notmuch] indexing mail?
On Fri, 15 Jan 2010 08:47:10 +0200, Dirk-Jan C. Binnema wrote: > I take a slighly different approach in mu: works for me, thanks Dirk. patch against HEAD is attached ( i hope. i'm new to emacs mail :D ) It takes around half an hour for my 60K mail on reiserfs, but it did take 10 minutes already on ext4. I suggest having a different approach to feed new mail in, such as: for i in (fetchmail) do notmuch new < $i done -- next part -- A non-text attachment was scrubbed... Name: not available Type: application/octet-stream Size: 1439 bytes Desc: not available URL: <http://notmuchmail.org/pipermail/notmuch/attachments/20100115/2253434b/attachment.obj>
[notmuch] indexing mail?
On 2010-01-15, Dirk-Jan C Binnema wrote: >> "Olly" == Olly Betts writes: >Olly> Not a full patch, but I already posted what this code should look > like >Olly> to handle both systems without d_type, and those which return > DT_UNKNOWN: > >Olly> http://article.gmane.org/gmane.mail.notmuch.general/1044 > static gboolean > _set_dtype (const char* path, struct dirent *entry) Underscore prefixed identifiers are reserved by ISO C at file-scope; using them yourself is undefined behaviour... > /* we only care about dirs, regular files and links */ > if (S_ISREG (statbuf.st_mode)) > entry->d_type = DT_REG; > else if (S_ISDIR (statbuf.st_mode)) > entry->d_type = DT_DIR; > else if (S_ISLNK (statbuf.st_mode)) > entry->d_type = DT_LNK; This addresses the case where the FS returns DT_UNKNOWN for d_type, but doesn't deal with the case of platforms where struct dirent has no d_type member - from the Linux readdir man page: The only fields in the dirent structure that are mandated by POSIX.1 are: d_name[], of unspecified size, with at most NAME_MAX characters preceding the terminating null byte; and (as an XSI extension) d_ino. The other fields are unstandardized, and not present on all systems; see NOTES below for some further details. And in NOTES: Other than Linux, the d_type field is available mainly only on BSD systems. Cheers, Olly
Re: [notmuch] asynch operations protocol
On Fri, 15 Jan 2010 09:59:54 -0400, David Bremner wrote:
> Is this over/under engineered? I spent roughly as long on the design as
> it took me to type :). Maybe the whole session id thing is redundant and
> could be done at the socket level. Or, getting more serious about the
> whole thing, maybe each queue operation should return an identifier.
The asynchronous interface I work with most is MPI. There you get a
Request object when the operation is initiated and you can
{test,block}{one,some,any,all}, where the latter takes a list of
requests. These variants are all useful, but of course they could be
implemented as needed. I don't think that being able to support these
variants places any particular burden on the design.
I believe in performing operations with appropriate granularity, so I
wouldn't expect cases where you need to manage thousands of active
requests, thus I'm not sure the "session" grouping offers any real
benefit. In any case, I'm not in favor of a single global flush.
Jed
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] asynch operations protocol
On Fri, 15 Jan 2010 14:49:14 +0100, Jed Brown wrote: > It wouldn't bother me at all if I lost my last few seconds of > interactive tagging due to something catastrophic like losing power. I > think there is still (post #250) a case for supporting some asynchronous > operations. I was wondering what the protocol would be like for non-blocking commands to a notmuch daemon. I have no experience with these things, but I was thinking something along the lines of (not worrying about wire encoding) open_session - basically just generates a unique id to allow grabbing results of commands to be collected. queue command session arguments I guess the command dispatcher could just be a thin replacement for command-line argument parsing. gather session return all output from session flush session close session Is this over/under engineered? I spent roughly as long on the design as it took me to type :). Maybe the whole session id thing is redundant and could be done at the socket level. Or, getting more serious about the whole thing, maybe each queue operation should return an identifier. OK, discuss amongst yourselves ;) d ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
Re: [notmuch] Xapian::DatabaseError on notmuch new upgrade
Looks like this was just a bug in Xapian HEAD, today's build works just fine and I'm enjoying the faster CHERT backend. Bulk tagging is really loads faster now, and the interactivity is better than before, but SPC still has a noticeable pause (at least 200 ms) since (apparently) it's doing an fsync, and this is worse when the disk is getting hit by some other job (like when I'm reading email while something big compiles). It wouldn't bother me at all if I lost my last few seconds of interactive tagging due to something catastrophic like losing power. I think there is still (post #250) a case for supporting some asynchronous operations. Jed ___ notmuch mailing list notmuch@notmuchmail.org http://notmuchmail.org/mailman/listinfo/notmuch
[notmuch] Notmuch performance problems on OSX
On 2010-01-14, Oliver Charles wrote: > I've installed the latest notmuch from Git at this time of writing, > along with Xapian from SVN head. However, just tagging a single thread > with only one message seems to take too long: One difference between OS X and other systems is that OS X supports the F_FULLSYNC ioctl, and other systems don't (currently, at least AFAIK) and Xapian uses that if it is available to ensure that changes have actually made it to disk: http://trac.xapian.org/ticket/288 On other systems, it uses fdatasync() or fsync(), which typically just ensure that the data has left the OS - it can sit in disk controller or drive caches for potentially seconds longer. This call happens once per table for every (explicit or implicit) flush on a database. I can see an issue here which is that currently Xapian writes the base file for the table, then syncs it, then does the next table. I bet it would be more efficient to write them all and then sync them all, especially with F_FULLSYNC. I'll take a look at doing that, and have created a ticket for it: http://trac.xapian.org/ticket/426 If after that this is still causing problems, it should probably be made configurable what (if any) flushing is done. If you're on a UPS-backed server, you probably don't need such paranoia. Cheers, Olly
[notmuch] indexing mail?
On 2010-01-14, Carl Worth wrote: > On Thu, 14 Jan 2010 18:38:54 +0100, Adrian Perez de Castro igalia.com> wrote: >> I am using XFS, which always returns DT_UNKNOWN. Taking into account that >> there is a good deal of people using filesystems other than the ones you >> mention, and that other non-linux filesystems may also return DT_UNKNOWN, >> in my opinion there should be a fall-back. I will try to post a patch >> Anytime Soon=E2=84=A2. > > We definitely want the fallback. I can attempt to code it, but I don't > have ready access to an afflicted filesystem, so I'd need help testing > anyway. > > I'd love to see a patch for this bug soon. Be sure to CC me when the > patch is sent and that will help me commit it sooner. Not a full patch, but I already posted what this code should look like to handle both systems without d_type, and those which return DT_UNKNOWN: http://article.gmane.org/gmane.mail.notmuch.general/1044 Cheers, Olly
[notmuch] [PATCH] [RFC] notmuch search: Return a non-zero exitcode if the search returns no hits.
If the user is explicitly searching for a message, then if notmuch
fails to find it, it is useful to set a failure exit code. This makes it
easier for scripting. However, we will want to then distinguish between
fatal errors (such as out-of-memory, invalid arguments, corrupt db, etc)
from the simple no message. To that end, we introduce a set of
enumerated exit-codes which may prove useful to use consistently across
notmuch. (I'm not that convinced highly differentiated errors is
particularly useful, though separate CONFIG_ERROR and DATABASE_ERROR
would seem to have value for scripts.)
---
notmuch-search.c | 34 --
1 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/notmuch-search.c b/notmuch-search.c
index dc44eb6..6ca903b 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -20,7 +20,17 @@
#include "notmuch-client.h"
-static void
+/* XXX After some review, make these universal? */
+enum {
+NOTMUCH_EXIT_SUCCESS = 0,
+NOTMUCH_EXIT_NOT_FOUND,
+NOTMUCH_EXIT_INVALID_ARGUMENT,
+NOTMUCH_EXIT_SYSTEM_ERROR, /* XXX just out-of-memory? */
+NOTMUCH_EXIT_CONFIG_ERROR,
+NOTMUCH_EXIT_DATABASE_ERROR,
+};
+
+static int
do_search_threads (const void *ctx,
notmuch_query_t *query,
notmuch_sort_t sort)
@@ -30,6 +40,7 @@ do_search_threads (const void *ctx,
notmuch_tags_t *tags;
time_t date;
const char *relative_date;
+int count = 0;
for (threads = notmuch_query_search_threads (query);
notmuch_threads_has_more (threads);
@@ -67,7 +78,10 @@ do_search_threads (const void *ctx,
printf (")\n");
notmuch_thread_destroy (thread);
+ count++;
}
+
+return count;
}
int
@@ -94,11 +108,11 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
sort = NOTMUCH_SORT_NEWEST_FIRST;
} else {
fprintf (stderr, "Invalid value for --sort: %s\n", opt);
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
} else {
fprintf (stderr, "Unrecognized option: %s\n", argv[i]);
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
}
@@ -107,35 +121,35 @@ notmuch_search_command (void *ctx, int argc, char *argv[])
config = notmuch_config_open (ctx, NULL, NULL);
if (config == NULL)
- return 1;
+ return NOTMUCH_EXIT_CONFIG_ERROR;
notmuch = notmuch_database_open (notmuch_config_get_database_path (config),
NOTMUCH_DATABASE_MODE_READ_ONLY);
if (notmuch == NULL)
- return 1;
+ return NOTMUCH_EXIT_DATABASE_ERROR;
query_str = query_string_from_args (ctx, argc, argv);
if (query_str == NULL) {
fprintf (stderr, "Out of memory.\n");
- return 1;
+ return NOTMUCH_EXIT_SYSTEM_ERROR;
}
if (*query_str == '\0') {
fprintf (stderr, "Error: notmuch search requires at least one search
term.\n");
- return 1;
+ return NOTMUCH_EXIT_INVALID_ARGUMENT;
}
query = notmuch_query_create (notmuch, query_str);
if (query == NULL) {
fprintf (stderr, "Out of memory\n");
- return 1;
+ return NOTMUCH_EXIT_SYSTEM_ERROR;
}
notmuch_query_set_sort (query, sort);
-do_search_threads (ctx, query, sort);
+i = do_search_threads (ctx, query, sort);
notmuch_query_destroy (query);
notmuch_database_close (notmuch);
-return 0;
+return i ? NOTMUCH_EXIT_SUCCESS : NOTMUCH_EXIT_NOT_FOUND;
}
--
1.6.6
___
notmuch mailing list
notmuch@notmuchmail.org
http://notmuchmail.org/mailman/listinfo/notmuch
