[PATCH 4/4] doc: document notmuch new --full-scan

[David Bremner]

---
 doc/man1/notmuch-new.rst | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst
index 16af5492..b0016ccc 100644
--- a/doc/man1/notmuch-new.rst
+++ b/doc/man1/notmuch-new.rst
@@ -59,6 +59,12 @@ Supported options for **new** include
 
 See also ``index.decrypt`` in **notmuch-config(1)**.
 
+``--full-scan``
+By default notmuch-new uses directory modification times (mtimes)
+to optimize the scanning of directories for new mail. This option
+allows turning that optimization off e.g. for testing or
+debugging.
+
 EXIT STATUS
 ===
 
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 3/4] test: use --full-scan in T050-new.sh

[David Bremner]

Wherever the test relies on directories being scanned, this option
should be used to avoid skipping them due to mtimes on directories
matching the database.
---
 test/T050-new.sh | 16 +++-
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 1f42..40c5e6fa 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -99,7 +99,7 @@ notmuch new > /dev/null
 
 mv "${MAIL_DIR}"/dir "${MAIL_DIR}"/dir-renamed
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --full-scan)
 test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
${MAIL_DIR}/dir for deletion from database
 No new mail. Detected 3 file renames."
 
@@ -107,7 +107,7 @@ No new mail. Detected 3 file renames."
 test_begin_subtest "Deleted directory"
 rm -rf "${MAIL_DIR}"/dir-renamed
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --full-scan)
 test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
${MAIL_DIR}/dir-renamed for deletion from database
 No new mail. Removed 3 messages."
 
@@ -126,7 +126,7 @@ test_begin_subtest "Deleted directory (end of list)"
 
 rm -rf "${MAIL_DIR}"/zzz
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --full-scan)
 test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover directory 
${MAIL_DIR}/zzz for deletion from database
 No new mail. Removed 3 messages."
 
@@ -177,7 +177,7 @@ test_begin_subtest "Deleted two-level directory"
 
 rm -rf "${MAIL_DIR}"/two
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --full-scan)
 test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover directory 
${MAIL_DIR}/two for deletion from database
 No new mail. Removed 3 messages."
 
@@ -223,7 +223,7 @@ Subject: Test mbox message 2
 
 Body 2.
 EOF
-output=$(NOTMUCH_NEW --debug 2>&1)
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1)
 test_expect_equal "$output" \
 "Note: Ignoring non-mail file: ${MAIL_DIR}/.git/config
 Note: Ignoring non-mail file: ${MAIL_DIR}/.ignored_hidden_file
@@ -258,11 +258,9 @@ test_expect_equal_file EXPECTED OUTPUT
 test_begin_subtest "Ignore files and directories specified in new.ignore 
(multiple occurrences)"
 notmuch config set new.ignore .git ignored_file .ignored_hidden_file
 notmuch new > /dev/null # ensure that files/folders will be printed in ASCII 
order.
-touch "${MAIL_DIR}"/.git # change .git's mtime for notmuch new to rescan.
-touch "${MAIL_DIR}"  # likewise for MAIL_DIR
 mkdir -p "${MAIL_DIR}"/one/two/three/.git
 touch "${MAIL_DIR}"/{one,one/two,one/two/three}/ignored_file
-output=$(NOTMUCH_NEW --debug 2>&1 | sort)
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1 | sort)
 test_expect_equal "$output" \
 "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
 (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
@@ -289,7 +287,7 @@ test_expect_equal "$output" "No new mail."
 
 test_begin_subtest "Ignore files and directories specified in new.ignore 
(regexp)"
 notmuch config set new.ignore ".git" "/^bro.*ink\$/" "/ignored.*file/"
-output=$(NOTMUCH_NEW --debug 2>&1 | sort)
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1 | sort)
 test_expect_equal "$output" \
 "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
 (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/4] test: add tests for notmuch new --full-scan

[David Bremner]

Most of these just check that adding the flag does not break existing
functionality. The one test that does check the full-scan
functionality had to be rewritten to output debugging info.
---
 test/T050-new.sh | 32 ++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 9025fa7a..1f42 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -12,6 +12,10 @@ generate_message
 output=$(NOTMUCH_NEW --debug)
 test_expect_equal "$output" "Added 1 new message to the database."
 
+test_begin_subtest "Single message (full-scan)"
+generate_message
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1)
+test_expect_equal "$output" "Added 1 new message to the database."
 
 test_begin_subtest "Multiple new messages"
 generate_message
@@ -19,11 +23,19 @@ generate_message
 output=$(NOTMUCH_NEW --debug)
 test_expect_equal "$output" "Added 2 new messages to the database."
 
+test_begin_subtest "Multiple new messages (full-scan)"
+generate_message
+generate_message
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1)
+test_expect_equal "$output" "Added 2 new messages to the database."
 
 test_begin_subtest "No new messages (non-empty DB)"
 output=$(NOTMUCH_NEW --debug)
 test_expect_equal "$output" "No new mail."
 
+test_begin_subtest "No new messages (full-scan)"
+output=$(NOTMUCH_NEW --debug --full-scan 2>&1)
+test_expect_equal "$output" "No new mail."
 
 test_begin_subtest "New directories"
 rm -rf "${MAIL_DIR}"/* "${MAIL_DIR}"/.notmuch
@@ -224,8 +236,24 @@ test_begin_subtest "Ignore files and directories specified 
in new.ignore"
 generate_message
 notmuch config set new.ignore .git ignored_file .ignored_hidden_file
 touch "${MAIL_DIR}"/.git # change .git's mtime for notmuch new to rescan.
-output=$(NOTMUCH_NEW 2>&1)
-test_expect_equal "$output" "Added 1 new message to the database."
+NOTMUCH_NEW --debug 2>&1 | sort > OUTPUT
+cat < EXPECTED
+(D) add_files, pass 1: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/.git
+(D) add_files, pass 1: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/.ignored_hidden_file
+(D) add_files, pass 1: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/ignored_file
+(D) add_files, pass 2: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/.git
+(D) add_files, pass 2: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/.ignored_hidden_file
+(D) add_files, pass 2: explicitly ignoring 
/home/bremner/software/upstream/notmuch/test/tmp.T050-new/mail/ignored_file
+Added 1 new message to the database.
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "Ignore files and directories specified in new.ignore 
(full-scan)"
+generate_message
+notmuch config set new.ignore .git ignored_file .ignored_hidden_file
+NOTMUCH_NEW --debug --full-scan 2>&1 | sort > OUTPUT
+# reuse EXPECTED from previous test
+test_expect_equal_file EXPECTED OUTPUT
 
 test_begin_subtest "Ignore files and directories specified in new.ignore 
(multiple occurrences)"
 notmuch config set new.ignore .git ignored_file .ignored_hidden_file
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/4] CLI/new: add full-scan option

[David Bremner]

By default notmuch-new uses directory mtimes to optimize the scanning of
directories for new mail. This option allows turning that optimization
off e.g. for testing or debugging.
---
 notmuch-new.c | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/notmuch-new.c b/notmuch-new.c
index c4345705..6a54a1a1 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -47,6 +47,7 @@ typedef struct {
 int output_is_a_tty;
 enum verbosity verbosity;
 bool debug;
+bool full_scan;
 const char **new_tags;
 size_t new_tags_length;
 const char **ignore_verbatim;
@@ -527,7 +528,7 @@ add_files (notmuch_database_t *notmuch,
  * mistakenly return the total number of directory entries, since
  * that only inflates the count beyond 2.
  */
-if (directory && fs_mtime == db_mtime && st.st_nlink == 2) {
+if (directory && (! state->full_scan) && fs_mtime == db_mtime && 
st.st_nlink == 2) {
/* There's one catch: pass 1 below considers symlinks to
 * directories to be directories, but these don't increase the
 * file system link count.  So, only bail early if the
@@ -618,7 +619,7 @@ add_files (notmuch_database_t *notmuch,
  * being discovered until the clock catches up and the directory
  * is modified again).
  */
-if (directory && fs_mtime == db_mtime)
+if (directory && (! state->full_scan) && fs_mtime == db_mtime)
goto DONE;
 
 /* If the database has never seen this directory before, we can
@@ -1053,6 +1054,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, 
char *argv[])
 add_files_state_t add_files_state = {
.verbosity = VERBOSITY_NORMAL,
.debug = false,
+   .full_scan = false,
.output_is_a_tty = isatty (fileno (stdout)),
 };
 struct timeval tv_start;
@@ -1073,6 +1075,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, 
char *argv[])
{ .opt_bool = , .name = "quiet" },
{ .opt_bool = , .name = "verbose" },
{ .opt_bool = _files_state.debug, .name = "debug" },
+   { .opt_bool = _files_state.full_scan, .name = "full-scan" },
{ .opt_bool = , .name = "hooks" },
{ .opt_inherit = notmuch_shared_indexing_options },
{ .opt_inherit = notmuch_shared_options },
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: notmuch show --decrypt=stash

[Daniel Kahn Gillmor]

On Tue 2018-01-09 18:12:23 -0500, Daniel Kahn Gillmor wrote:
> This is a revision of the series initially introduced in
> id:20171212025225.11854-1-...@fifthhorseman.net, with minor updates to
> accomodate the recent release of notmuch 0.26 (yay!)
>
> This series allows "notmuch show" to index the cleartext and stash the
> session keys of an encrypted message while displaying it.

I'd really appreciate review for this series!  I think it adds useful
functionality to notmuch, enabling a particular workflow for people to
use the cleartext index who don't want mail being decrypted in the
background.

Please take a look and let me know what you think!

   --dkg
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: Database corruption after clean rebuild

[Gregor Zattler]

Hi David,
* David Bremner  [2018-04-29; 11:27]:
> Gregor Zattler  writes:
>> I also had this database corruption, I waited for the fix to land
>> in notmuch 0.26.2, build it, moved the xapian directory away, did
>> a notmuch new and restored the tags from a dump.  But the problem
>> remains:
>>
>> ~$ xapian-check ~/Mail/.notmuch/xapian
>> docdata:
>> blocksize=8K items=10841 firstunused=75 revision=82 levels=1 root=2
>> B-tree checked okay
>> docdata table structure checked OK
>>
>> termlist:
>> blocksize=8K items=1893162 firstunused=368983 revision=82 levels=3 
>> root=177608
>> xapian-check: DatabaseError: 1 unused block(s) missing from the free list, 
>> first is 0
>
> This is a known bug in Xapian, fixed in xapian master. The message will
> go away if you run notmuch-compact, or you can just ignore it.

OK, thanks.  Gregor

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH 2/2] test: use --no-mtime-opt in T050-new.sh

[Jani Nikula]

On Sun, 29 Apr 2018, David Bremner  wrote:
> Wherever the test relies on directories being scanned, this option
> should be used to avoid skipping them due to mtimes on directories
> matching the database.

I think you could additionally remove a few touch calls in the
test. Some of them do actually create empty files, but some of them just
touch directories to force rescans.

Otherwise, LGTM.

BR,
Jani.

> ---
>  test/T050-new.sh | 14 +++---
>  1 file changed, 7 insertions(+), 7 deletions(-)
>
> diff --git a/test/T050-new.sh b/test/T050-new.sh
> index 9025fa7a..12dba471 100755
> --- a/test/T050-new.sh
> +++ b/test/T050-new.sh
> @@ -87,7 +87,7 @@ notmuch new > /dev/null
>  
>  mv "${MAIL_DIR}"/dir "${MAIL_DIR}"/dir-renamed
>  
> -output=$(NOTMUCH_NEW --debug)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt)
>  test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
> ${MAIL_DIR}/dir for deletion from database
>  No new mail. Detected 3 file renames."
>  
> @@ -95,7 +95,7 @@ No new mail. Detected 3 file renames."
>  test_begin_subtest "Deleted directory"
>  rm -rf "${MAIL_DIR}"/dir-renamed
>  
> -output=$(NOTMUCH_NEW --debug)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt)
>  test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
> ${MAIL_DIR}/dir-renamed for deletion from database
>  No new mail. Removed 3 messages."
>  
> @@ -114,7 +114,7 @@ test_begin_subtest "Deleted directory (end of list)"
>  
>  rm -rf "${MAIL_DIR}"/zzz
>  
> -output=$(NOTMUCH_NEW --debug)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt)
>  test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover 
> directory ${MAIL_DIR}/zzz for deletion from database
>  No new mail. Removed 3 messages."
>  
> @@ -165,7 +165,7 @@ test_begin_subtest "Deleted two-level directory"
>  
>  rm -rf "${MAIL_DIR}"/two
>  
> -output=$(NOTMUCH_NEW --debug)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt)
>  test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover 
> directory ${MAIL_DIR}/two for deletion from database
>  No new mail. Removed 3 messages."
>  
> @@ -211,7 +211,7 @@ Subject: Test mbox message 2
>  
>  Body 2.
>  EOF
> -output=$(NOTMUCH_NEW --debug 2>&1)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1)
>  test_expect_equal "$output" \
>  "Note: Ignoring non-mail file: ${MAIL_DIR}/.git/config
>  Note: Ignoring non-mail file: ${MAIL_DIR}/.ignored_hidden_file
> @@ -234,7 +234,7 @@ touch "${MAIL_DIR}"/.git # change .git's mtime for 
> notmuch new to rescan.
>  touch "${MAIL_DIR}"  # likewise for MAIL_DIR
>  mkdir -p "${MAIL_DIR}"/one/two/three/.git
>  touch "${MAIL_DIR}"/{one,one/two,one/two/three}/ignored_file
> -output=$(NOTMUCH_NEW --debug 2>&1 | sort)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1 | sort)
>  test_expect_equal "$output" \
>  "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
>  (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
> @@ -261,7 +261,7 @@ test_expect_equal "$output" "No new mail."
>  
>  test_begin_subtest "Ignore files and directories specified in new.ignore 
> (regexp)"
>  notmuch config set new.ignore ".git" "/^bro.*ink\$/" "/ignored.*file/"
> -output=$(NOTMUCH_NEW --debug 2>&1 | sort)
> +output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1 | sort)
>  test_expect_equal "$output" \
>  "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
>  (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
> -- 
> 2.17.0
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH 1/2] CLI/new: add mtime-opt option

[Jani Nikula]

On Sun, 29 Apr 2018, David Bremner  wrote:
> This option, on by default, controls whether mtimes are used to
> optimize the scanning of directories. The intent is to turn this
> optimization off for certain tests.
> ---
>  notmuch-new.c | 9 ++---
>  1 file changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/notmuch-new.c b/notmuch-new.c
> index c4345705..099bbbae 100644
> --- a/notmuch-new.c
> +++ b/notmuch-new.c
> @@ -47,6 +47,7 @@ typedef struct {
>  int output_is_a_tty;
>  enum verbosity verbosity;
>  bool debug;
> +bool mtime_opt;
>  const char **new_tags;
>  size_t new_tags_length;
>  const char **ignore_verbatim;
> @@ -527,7 +528,7 @@ add_files (notmuch_database_t *notmuch,
>   * mistakenly return the total number of directory entries, since
>   * that only inflates the count beyond 2.
>   */
> -if (directory && fs_mtime == db_mtime && st.st_nlink == 2) {
> +if (directory && state->mtime_opt && fs_mtime == db_mtime && st.st_nlink 
> == 2) {
>   /* There's one catch: pass 1 below considers symlinks to
>* directories to be directories, but these don't increase the
>* file system link count.  So, only bail early if the
> @@ -618,7 +619,7 @@ add_files (notmuch_database_t *notmuch,
>   * being discovered until the clock catches up and the directory
>   * is modified again).
>   */
> -if (directory && fs_mtime == db_mtime)
> +if (directory && state->mtime_opt && fs_mtime == db_mtime)
>   goto DONE;
>  
>  /* If the database has never seen this directory before, we can
> @@ -771,7 +772,7 @@ add_files (notmuch_database_t *notmuch,
>   * the database because a message could be delivered later in this
>   * same second.  This may lead to unnecessary re-scans, but it
>   * avoids overlooking messages. */
> -if (fs_mtime != stat_time)
> +if (state->mtime_opt && fs_mtime != stat_time)
>   _filename_list_add (state->directory_mtimes, path)->mtime = fs_mtime;

I don't think we should skip this part. We've done a full scan now, so
we should record that in the database so a subsequent scan doesn't have
to.

>  
>DONE:
> @@ -1053,6 +1054,7 @@ notmuch_new_command (notmuch_config_t *config, int 
> argc, char *argv[])
>  add_files_state_t add_files_state = {
>   .verbosity = VERBOSITY_NORMAL,
>   .debug = false,
> + .mtime_opt = true,
>   .output_is_a_tty = isatty (fileno (stdout)),
>  };
>  struct timeval tv_start;
> @@ -1073,6 +1075,7 @@ notmuch_new_command (notmuch_config_t *config, int 
> argc, char *argv[])
>   { .opt_bool = , .name = "quiet" },
>   { .opt_bool = , .name = "verbose" },
>   { .opt_bool = _files_state.debug, .name = "debug" },
> + { .opt_bool = _files_state.mtime_opt, .name = "mtime-opt" },

--full-scan? --force? --force-scan?

I think we've had people ask how they can have notmuch scan some
directories again, for some reason or another. Maybe to test their
ignores, and they can't have notmuch rescan the directory without
touching it. IMHO --no-mtime-opt doesn't sound very intuitive for that
purpose.

Otherwise, seems like a thing we should add.

BR,
Jani.

>   { .opt_bool = , .name = "hooks" },
>   { .opt_inherit = notmuch_shared_indexing_options },
>   { .opt_inherit = notmuch_shared_options },
> -- 
> 2.17.0
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: Database corruption after clean rebuild

[David Bremner]

Gregor Zattler  writes:

> Hi notmuch developers,
>
> I also had this database corruption, I waited for the fix to land
> in notmuch 0.26.2, build it, moved the xapian directory away, did
> a notmuch new and restored the tags from a dump.  But the problem
> remains:
>
> ~$ xapian-check ~/Mail/.notmuch/xapian
> docdata:
> blocksize=8K items=10841 firstunused=75 revision=82 levels=1 root=2
> B-tree checked okay
> docdata table structure checked OK
>
> termlist:
> blocksize=8K items=1893162 firstunused=368983 revision=82 levels=3 root=177608
> xapian-check: DatabaseError: 1 unused block(s) missing from the free list, 
> first is 0

This is a known bug in Xapian, fixed in xapian master. The message will
go away if you run notmuch-compact, or you can just ignore it.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/2] test: use --no-mtime-opt in T050-new.sh

[David Bremner]

Wherever the test relies on directories being scanned, this option
should be used to avoid skipping them due to mtimes on directories
matching the database.
---
 test/T050-new.sh | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 9025fa7a..12dba471 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -87,7 +87,7 @@ notmuch new > /dev/null
 
 mv "${MAIL_DIR}"/dir "${MAIL_DIR}"/dir-renamed
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt)
 test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
${MAIL_DIR}/dir for deletion from database
 No new mail. Detected 3 file renames."
 
@@ -95,7 +95,7 @@ No new mail. Detected 3 file renames."
 test_begin_subtest "Deleted directory"
 rm -rf "${MAIL_DIR}"/dir-renamed
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt)
 test_expect_equal "$output" "(D) add_files, pass 2: queuing passed directory 
${MAIL_DIR}/dir-renamed for deletion from database
 No new mail. Removed 3 messages."
 
@@ -114,7 +114,7 @@ test_begin_subtest "Deleted directory (end of list)"
 
 rm -rf "${MAIL_DIR}"/zzz
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt)
 test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover directory 
${MAIL_DIR}/zzz for deletion from database
 No new mail. Removed 3 messages."
 
@@ -165,7 +165,7 @@ test_begin_subtest "Deleted two-level directory"
 
 rm -rf "${MAIL_DIR}"/two
 
-output=$(NOTMUCH_NEW --debug)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt)
 test_expect_equal "$output" "(D) add_files, pass 3: queuing leftover directory 
${MAIL_DIR}/two for deletion from database
 No new mail. Removed 3 messages."
 
@@ -211,7 +211,7 @@ Subject: Test mbox message 2
 
 Body 2.
 EOF
-output=$(NOTMUCH_NEW --debug 2>&1)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1)
 test_expect_equal "$output" \
 "Note: Ignoring non-mail file: ${MAIL_DIR}/.git/config
 Note: Ignoring non-mail file: ${MAIL_DIR}/.ignored_hidden_file
@@ -234,7 +234,7 @@ touch "${MAIL_DIR}"/.git # change .git's mtime for notmuch 
new to rescan.
 touch "${MAIL_DIR}"  # likewise for MAIL_DIR
 mkdir -p "${MAIL_DIR}"/one/two/three/.git
 touch "${MAIL_DIR}"/{one,one/two,one/two/three}/ignored_file
-output=$(NOTMUCH_NEW --debug 2>&1 | sort)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1 | sort)
 test_expect_equal "$output" \
 "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
 (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
@@ -261,7 +261,7 @@ test_expect_equal "$output" "No new mail."
 
 test_begin_subtest "Ignore files and directories specified in new.ignore 
(regexp)"
 notmuch config set new.ignore ".git" "/^bro.*ink\$/" "/ignored.*file/"
-output=$(NOTMUCH_NEW --debug 2>&1 | sort)
+output=$(NOTMUCH_NEW --debug --no-mtime-opt 2>&1 | sort)
 test_expect_equal "$output" \
 "(D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.git
 (D) add_files, pass 1: explicitly ignoring ${MAIL_DIR}/.ignored_hidden_file
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/2] CLI/new: add mtime-opt option

[David Bremner]

This option, on by default, controls whether mtimes are used to
optimize the scanning of directories. The intent is to turn this
optimization off for certain tests.
---
 notmuch-new.c | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/notmuch-new.c b/notmuch-new.c
index c4345705..099bbbae 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -47,6 +47,7 @@ typedef struct {
 int output_is_a_tty;
 enum verbosity verbosity;
 bool debug;
+bool mtime_opt;
 const char **new_tags;
 size_t new_tags_length;
 const char **ignore_verbatim;
@@ -527,7 +528,7 @@ add_files (notmuch_database_t *notmuch,
  * mistakenly return the total number of directory entries, since
  * that only inflates the count beyond 2.
  */
-if (directory && fs_mtime == db_mtime && st.st_nlink == 2) {
+if (directory && state->mtime_opt && fs_mtime == db_mtime && st.st_nlink 
== 2) {
/* There's one catch: pass 1 below considers symlinks to
 * directories to be directories, but these don't increase the
 * file system link count.  So, only bail early if the
@@ -618,7 +619,7 @@ add_files (notmuch_database_t *notmuch,
  * being discovered until the clock catches up and the directory
  * is modified again).
  */
-if (directory && fs_mtime == db_mtime)
+if (directory && state->mtime_opt && fs_mtime == db_mtime)
goto DONE;
 
 /* If the database has never seen this directory before, we can
@@ -771,7 +772,7 @@ add_files (notmuch_database_t *notmuch,
  * the database because a message could be delivered later in this
  * same second.  This may lead to unnecessary re-scans, but it
  * avoids overlooking messages. */
-if (fs_mtime != stat_time)
+if (state->mtime_opt && fs_mtime != stat_time)
_filename_list_add (state->directory_mtimes, path)->mtime = fs_mtime;
 
   DONE:
@@ -1053,6 +1054,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, 
char *argv[])
 add_files_state_t add_files_state = {
.verbosity = VERBOSITY_NORMAL,
.debug = false,
+   .mtime_opt = true,
.output_is_a_tty = isatty (fileno (stdout)),
 };
 struct timeval tv_start;
@@ -1073,6 +1075,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, 
char *argv[])
{ .opt_bool = , .name = "quiet" },
{ .opt_bool = , .name = "verbose" },
{ .opt_bool = _files_state.debug, .name = "debug" },
+   { .opt_bool = _files_state.mtime_opt, .name = "mtime-opt" },
{ .opt_bool = , .name = "hooks" },
{ .opt_inherit = notmuch_shared_indexing_options },
{ .opt_inherit = notmuch_shared_options },
-- 
2.17.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: Database corruption after clean rebuild

[Gregor Zattler]

Hi notmuch developers,

I also had this database corruption, I waited for the fix to land
in notmuch 0.26.2, build it, moved the xapian directory away, did
a notmuch new and restored the tags from a dump.  But the problem
remains:

~$ xapian-check ~/Mail/.notmuch/xapian
docdata:
blocksize=8K items=10841 firstunused=75 revision=82 levels=1 root=2
B-tree checked okay
docdata table structure checked OK

termlist:
blocksize=8K items=1893162 firstunused=368983 revision=82 levels=3 root=177608
xapian-check: DatabaseError: 1 unused block(s) missing from the free list, 
first is 0


this is very similar to the old database which I had moved away:
~$ xapian-check ~/Mail/.notmuch/xapian-2018-04-29-00-22/
docdata:
blocksize=8K items=10863 firstunused=78 revision=59623 levels=1 root=2
B-tree checked okay
docdata table structure checked OK

termlist:
blocksize=8K items=1894648 firstunused=360821 revision=59623 levels=3 
root=360580
xapian-check: DatabaseError: 1 unused block(s) missing from the free list, 
first is 0


Now I did notmuch compact and the database check says there are
no errors.


This seems to me as if the fix had not helped or there is another problem.


$ notmuch --version
notmuch 0.26.2+26~g9e158fb
~$ xapian-compact --version
xapian-compact - xapian-core 1.4.3

Thanks for developing notmuch, Gregor


* Javier Garcia  [2018-04-07; 17:09]:
> Unfortunately I can't share my emails without the approval of other
> parties. The minimum subsets that trigger the error are in the range of
> 1000-5000 mails, so asking each and everyone of them is out of my reach.
> I tried to replicate the problem using just spam folders without success.
>
> The following is a solid workaround I've stumbled upon. Afew no longer
> complains and database corruption is gone.
>
> $ notmuch compact
> $ xapian-check ~/.mail/.notmuch/xapian
>    
>    No errors found
>
> I built xapian-core 1.50 but I can't compile notmuch 0.26.1 against it.
> I will wait and test again in a few weeks.
>
> If you are interested in my setup, the error happens with this minimal
> configuration.
>
> #~/.config/afew/config
> [Filter.1]
> query = 'folder:"//(INBOX|Inbox|inbox)$/" AND (NOT tag:inbox)'
> tags = +inbox;-new
> message = Messages in INBOX folder are tagged as inbox
>
> [Filter.2]
> query = '(NOT folder:"//(INBOX|Inbox|inbox)$/") AND (tag:inbox)'
> tags = -inbox
> message = Messages not in INBOX folder cannot be inbox
>
> #~/.notmuch-config
> [database]
> path=/home-path/.mail
> [new]
> tags=new
>
> On 07/04/18 12:51, David Bremner wrote:
>> Javier Garcia  writes:
>>
>>> I've applied the path to notmuch 0.26.1 without success.
>>>
>>> $ rm -rf ~/.mail/.notmuch
>>> $ LD_LIBRARY_PATH=/hidden-path/notmuch-0.26.1/lib/:$LD_LIBRARY_PATH
>>> ./notmuch new
>>>    Found 20065 total files (that's not much mail).
>>>    Processed 20065 total files in 58s (341 files/sec.).
>>>    Added 19605 new messages to the database.
>>>
>>> $ xapian-check .mail/.notmuch/xapian/
>>>    docdata:
>>>    blocksize=8K items=63 firstunused=1 revision=2 levels=0 root=0
>>>    B-tree checked okay
>>>    docdata table structure checked OK
>>>    termlist:
>>>    blocksize=8K items=43520 firstunused=8293 revision=2 levels=2 root=748
>>>    xapian-check: DatabaseError: 1 unused block(s) missing from the free
>>> list, first is 0
>> OK, so probably not related to reference loops (although that patch is
>> not very well tested).  It's not clear how notmuch is triggering it, but
>> this looks like the same bug in Xapian that olly fixed recently [1].
>>
>> A possible next step is to try building xapian master, and linking
>> notmuch against that.
>>
>> Maybe Patrick or Justus (in copy) has some idea why you're only seeing
>> problems in afew.
>>
>> Another debugging direction is to try to duplicate your problem with
>> some subset of mail that you're willing to share (bisection is the usual
>> strategy).
>>
>> [1] https://notmuchmail.org/pipermail/notmuch/2018/026369.html
>
> ___
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch
Ciao; Gregor
-- 
 -... --- .-. . -.. ..--.. ...-.-

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: non-deterministic behaviour of new.ignore (regexp) test

[Jani Nikula]

On Sat, 28 Apr 2018, David Bremner  wrote:
> For me the following seems to consistently fail after between 30 and 500
> attempts
>
> export NOTMUCH_TEST_QUIET=yes; count=0; while ./T050-new.sh; do (( 
> count++ )); echo $count; done

I believe this happens because the directory mtime is unchanged from the
previous scan in the test, and we skip the directory before we could
ignore the files. Quoting add_files():

/* If the directory's modification time in the filesystem is the
 * same as what we recorded in the database the last time we
 * scanned it, then we can skip the second pass entirely.
 *
 * We test for strict equality here to avoid a bug that can happen
 * if the system clock jumps backward, (preventing new mail from
 * being discovered until the clock catches up and the directory
 * is modified again).
 */

I can't reproduce if I add this to the test:

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 9025fa7aa63e..0db76f47130b 100755
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -260,6 +260,7 @@ output=$(NOTMUCH_NEW 2>&1)
 test_expect_equal "$output" "No new mail."
 
 test_begin_subtest "Ignore files and directories specified in new.ignore 
(regexp)"
+touch "${MAIL_DIR}" # force rescan of the top level directory
 notmuch config set new.ignore ".git" "/^bro.*ink\$/" "/ignored.*file/"
 output=$(NOTMUCH_NEW --debug 2>&1 | sort)
 test_expect_equal "$output" \

---

However, I'm not sure even that is enough if all this happens in the
same second. I think the way notmuch new is written, it may skip as long
as it ensures a subsequent scan will catch the modified files:

/* If the directory's mtime is the same as the wall-clock time
 * when we stat'ed the directory, we skip updating the mtime in
 * the database because a message could be delivered later in this
 * same second.  This may lead to unnecessary re-scans, but it
 * avoids overlooking messages. */

I think we can make the problem less likely with the touch, but as
everything gets faster, we might hit this more and more. One approach
might be a notmuch new --force option that would rescan all directories
regardless of mtimes. We could use this for testing (except when we're
testing the optimization).

BR,
Jani.
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH v2] move more http -> https

[Daniel Kahn Gillmor]

Correct URLs that have crept into the notmuch codebase with http://
when https:// is possible.

As part of this conversion, this changeset also indicates the current
preferred upstream URLs for both gmime and sup.  the new URLs are
https-enabled, the old ones are not.

This also fixes T310-emacs.sh, thanks to Bremner for catching it.
---
 AUTHORS| 2 +-
 INSTALL| 4 ++--
 NEWS   | 2 +-
 bindings/python/docs/COPYING   | 2 +-
 bindings/python/docs/source/conf.py| 2 +-
 configure  | 6 +++---
 devel/nmbug/doc/man5/notmuch-report.json.5.rst | 6 +++---
 devel/nmbug/notmuch-report | 2 +-
 emacs/notmuch-show.el  | 2 +-
 lib/message-property.cc| 2 +-
 lib/string-map.c   | 2 +-
 notmuch-reindex.c  | 2 +-
 test/T310-emacs.sh | 2 +-
 13 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 6d0f2de8..5fe5006f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -3,7 +3,7 @@ But there's really not much that he's done. There's been a lot 
of
 standing on shoulders here:
 
 William Morgan deserves credit for providing the primary inspiration
-for Notmuch with his program Sup (http://sup.rubyforge.org/).
+for Notmuch with his program Sup (https://sup-heliotrope.github.io/).
 
 Some people have contributed code that has made it into Notmuch
 without their specific knowledge (but with their full permission
diff --git a/INSTALL b/INSTALL
index 69f9df7e..056e9fac 100644
--- a/INSTALL
+++ b/INSTALL
@@ -46,7 +46,7 @@ Talloc, and zlib which are each described below:
Without GMime, Notmuch would not be able to extract and index
the actual text from email message encoded as BASE64, etc.
 
-   GMime is available from http://spruce.sourceforge.net/gmime/
+   GMime is available from https://github.com/jstedfast/gmime
 
Talloc
--
@@ -70,7 +70,7 @@ Talloc, and zlib which are each described below:
Notmuch needs the transparent write feature of zlib introduced
in version 1.2.5.2 (Dec. 2011).
 
-   zlib is available from http://zlib.net
+   zlib is available from https://zlib.net
 
 Building Documentation
 --
diff --git a/NEWS b/NEWS
index fff8a08c..2b8d7bc8 100644
--- a/NEWS
+++ b/NEWS
@@ -1588,7 +1588,7 @@ uses `@{upstream}` instead of `FETCH_HEAD` to track 
upstream changes.
   2. Remove your `NMBGIT` repository (e.g. `mv .nmbug .nmbug.bak`).
   3. Use the new `clone` command to create a fresh clone:
 
-nmbug clone http://nmbug.tethera.net/git/nmbug-tags.git
+nmbug clone https://nmbug.notmuchmail.org/git/nmbug-tags.git
 
   4. If you had local commits in step 1, add a remote for that
  repository and fetch them into the new repository.
diff --git a/bindings/python/docs/COPYING b/bindings/python/docs/COPYING
index 2a000655..e6000869 100644
--- a/bindings/python/docs/COPYING
+++ b/bindings/python/docs/COPYING
@@ -1,7 +1,7 @@
 GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. 
+ Copyright (C) 2007 Free Software Foundation, Inc. 
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
diff --git a/bindings/python/docs/source/conf.py 
b/bindings/python/docs/source/conf.py
index 5107a96e..5b901c4e 100644
--- a/bindings/python/docs/source/conf.py
+++ b/bindings/python/docs/source/conf.py
@@ -215,4 +215,4 @@ latex_documents = [
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/configure b/configure
index b177b141..1a6bf295 100755
--- a/configure
+++ b/configure
@@ -734,18 +734,18 @@ EOF
 fi
 if [ $have_zlib -eq 0 ]; then
echo "  zlib library (>= version 1.2.5.2, including development files 
such as headers)"
-   echo "  http://zlib.net/;
+   echo "  https://zlib.net/;
echo
 fi
 if [ $have_gmime -eq 0 ]; then
echo "  GMime 2.6 library >= $GMIME_MINVER"
echo "  (including development files such as headers)"
-   echo "  http://spruce.sourceforge.net/gmime/;
+   echo "  https://github.com/jstedfast/gmime/;
echo
 fi
 if [ $have_glib -eq 0 ]; then
echo "  Glib library >= 2.22 (including development files such as 
headers)"
-   echo "  http://ftp.gnome.org/pub/gnome/sources/glib/;
+   echo "  https://ftp.gnome.org/pub/gnome/sources/glib/;
echo
 fi
 if [ $have_talloc -eq 0 ]; then
diff --git