[PATCH v2 0/2] cli: support regex in new.ignore

[Jani Nikula]

The rest of the patches from id:cover.1504280923.git.j...@nikula.org
with review addressed. Be more strict about parsing new.ignore, and give
sensible error messages on failures. Also improve the text in man page a
bit.

BR,
Jani.

Jani Nikula (2):
  cli/new: support // in new.ignore
  test: test regexp based new.ignore

 doc/man1/notmuch-config.rst |  21 +--
 notmuch-new.c   | 134 
 test/T050-new.sh|  22 
 3 files changed, 162 insertions(+), 15 deletions(-)

-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2 1/2] cli/new: support // in new.ignore

[Jani Nikula]

Add support for using // style regular expressions in
new.ignore, mixed with the old style verbatim file and directory
basenames. The regex is matched against the relative path from the
database path.
---
 doc/man1/notmuch-config.rst |  21 +--
 notmuch-new.c   | 134 
 2 files changed, 140 insertions(+), 15 deletions(-)

diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 6a51e64f1517..0af86f9beee4 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -75,11 +75,22 @@ The available configuration items are described below.
 Default: ``unread;inbox``.
 
 **new.ignore**
-A list of file and directory names, without path, that will not
-be searched for messages by **notmuch new**. All the files and
-directories matching any of the names specified here will be
-ignored, regardless of the location in the mail store directory
-hierarchy.
+A list to specify files and directories that will not be
+searched for messages by **notmuch new**. Each entry in the
+list is either:
+
+  A file or a directory name, without path, that will be
+  ignored, regardless of the location in the mail store
+  directory hierarchy.
+
+Or:
+
+  A regular expression delimited with // that will be matched
+  against the path of the file or directory relative to the
+  database path. Matching files and directories will be
+  ignored. The beginning and end of string must be explictly
+  anchored. For example, /.*/foo$/ would match "bar/foo" and
+  "bar/baz/foo", but not "foo" or "bar/foobar".
 
 Default: empty list.
 
diff --git a/notmuch-new.c b/notmuch-new.c
index 50597b75c07e..7e12b73a7883 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -42,13 +42,17 @@ enum verbosity {
 };
 
 typedef struct {
+const char *db_path;
+
 int output_is_a_tty;
 enum verbosity verbosity;
 notmuch_bool_t debug;
 const char **new_tags;
 size_t new_tags_length;
-const char **new_ignore;
-size_t new_ignore_length;
+const char **ignore_verbatim;
+size_t ignore_verbatim_length;
+regex_t *ignore_regex;
+size_t ignore_regex_length;
 
 int total_files;
 int processed_files;
@@ -240,18 +244,125 @@ _special_directory (const char *entry)
 return strcmp (entry, ".") == 0 || strcmp (entry, "..") == 0;
 }
 
+static notmuch_bool_t
+_setup_ignore (notmuch_config_t *config, add_files_state_t *state)
+{
+const char **ignore_list, **ignore;
+int nregex = 0, nverbatim = 0;
+const char **verbatim = NULL;
+regex_t *regex = NULL;
+
+ignore_list = notmuch_config_get_new_ignore (config, NULL);
+if (! ignore_list)
+   return TRUE;
+
+for (ignore = ignore_list; *ignore; ignore++) {
+   const char *s = *ignore;
+   size_t len = strlen (s);
+
+   if (len == 0) {
+   fprintf (stderr, "Error: Empty string in new.ignore list\n");
+   return FALSE;
+   }
+
+   if (s[0] == '/') {
+   regex_t *preg;
+   char *r;
+   int rerr;
+
+   if (len < 3 || s[len - 1] != '/') {
+   fprintf (stderr, "Error: Malformed pattern '%s' in 
new.ignore\n",
+s);
+   return FALSE;
+   }
+
+   r = talloc_strndup (config, s + 1, len - 2);
+   regex = talloc_realloc (config, regex, regex_t, nregex + 1);
+   preg = [nregex];
+
+   rerr = regcomp (preg, r, REG_EXTENDED | REG_NOSUB);
+   if (rerr) {
+   size_t error_size = regerror (rerr, preg, NULL, 0);
+   char *error = talloc_size (r, error_size);
+
+   regerror (rerr, preg, error, error_size);
+
+   fprintf (stderr, "Error: Invalid regex '%s' in new.ignore: 
%s\n",
+r, error);
+   return FALSE;
+   }
+   nregex++;
+
+   talloc_free (r);
+   } else {
+   verbatim = talloc_realloc (config, verbatim, const char *,
+  nverbatim + 1);
+   verbatim[nverbatim++] = s;
+   }
+}
+
+state->ignore_regex = regex;
+state->ignore_regex_length = nregex;
+state->ignore_verbatim = verbatim;
+state->ignore_verbatim_length = nverbatim;
+
+return TRUE;
+}
+
+static char *
+_get_relative_path (const char *db_path, const char *dirpath, const char 
*entry)
+{
+size_t db_path_len = strlen (db_path);
+
+/* paranoia? */
+if (strncmp (dirpath, db_path, db_path_len) != 0) {
+   fprintf (stderr, "Warning: '%s' is not a subdirectory of '%s'\n",
+dirpath, db_path);
+   return NULL;
+}
+
+dirpath += db_path_len;
+while (*dirpath == '/')
+   dirpath++;
+
+if (*dirpath)
+   return talloc_asprintf (NULL, "%s/%s", dirpath, entry);
+

[PATCH v2 2/2] test: test regexp based new.ignore

[Jani Nikula]

Just some basics.
---
 test/T050-new.sh | 22 ++
 1 file changed, 22 insertions(+)

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 272ed417aa2e..ee9ce08c8e86 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -259,6 +259,28 @@ ln -s i_do_not_exist "${MAIL_DIR}"/broken_link
 output=$(NOTMUCH_NEW 2>&1)
 test_expect_equal "$output" "No new mail."
 
+test_begin_subtest "Ignore files and directories specified in new.ignore 
(regexp)"
+notmuch config set new.ignore ".git" "/^bro.*ink\$/" "/ignored.*file/"
+output=$(NOTMUCH_NEW --debug 2>&1 | sort)
+test_expect_equal "$output" \
+"(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/broken_link
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/ignored_file
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/one/ignored_file
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/one/two/ignored_file
+(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/one/two/three/.git
+(D) add_files, pass 1: explicitly ignoring 
${MAIL_DIR}/one/two/three/ignored_file
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/.git
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/broken_link
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/ignored_file
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/one/ignored_file
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/one/two/ignored_file
+(D) add_files, pass 2: explicitly ignoring ${MAIL_DIR}/one/two/three/.git
+(D) add_files, pass 2: explicitly ignoring 
${MAIL_DIR}/one/two/three/ignored_file
+No new mail."
+
 test_begin_subtest "Quiet: No new mail."
 output=$(NOTMUCH_NEW --quiet)
 test_expect_equal "$output" ""
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: No documents in index...

[David Bremner]

"Bruce C. Dillahunty"  writes:

> Hello all... first time poster :-)
>
> Used notmuch for several years with mutt, but been away a while and coming 
> back. Different machine, etc. In the past it always "just worked", but not so 
> much now.
>
> I'm on a Linux box, notmuch built from git (tried 0.25 and latest git pull), 
> xapian 1.4.4.
>
> I configure it, run "notmuch new" and it chews through an appropriate number 
> (285000+) emails. It generates some large index files in 
> ~/Maildir/.notmuch/xapian
>
> But not results to anything.
>
> notmuch count returns "0"
>
> xapian-delve ~/Maildir/.notmuch/xapian 

Does the notmuch test suite pass? (make test)

Can you share the output of "notmuch config list"? I'm particularly
interested in the value of new.ignore

It's a long shot, but you could try moving $MAILDIR/.notmuch and
.notmuch-config out of the way and starting over with "notmuch setup"

d


___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch