#3691: Add LMDB support
---------------------------+----------------------
  Reporter:  rsc           |      Owner:  mutt-dev
      Type:  enhancement   |     Status:  new
  Priority:  minor         |  Milestone:
 Component:  header cache  |    Version:
Resolution:                |   Keywords:
---------------------------+----------------------

Comment (by neverpanic):

 LMDB performs much better than Tokyo Cabinet while reading, so this is
 definitely nice to have. However, I've noticed that while writing the
 cache, your code starts a transaction for every single write operation,
 which causes this to be much slower than I think it needs to be.

 Here's a quick comparison when opening a 130k mail mailbox with empty
 cache. In all cases I asked mutt to exit by pressing 'q' while it was
 generating the cache.
 {{{
 rm ./headers.lmdb*; time ./mutt -e 'set header_cache = "./headers.lmdb"'
 -f ~/Mail/MacPorts/tickets

 real    0m29.271s
 user    0m8.320s
 sys     0m11.525s
 }}}

 The same command with changes that explicitly start a single transaction
 before updating the cache:
 {{{
 real    0m10.819s
 user    0m6.886s
 sys     0m2.762s
 }}}

 I'm including the changes I used to generate these numbers below, but note
 that this is nowhere near ready for a merge as-is. I think `hcache.c`
 shouldn't abstract the need for a transaction away from the users of
 header cache, because those users alone know best whether they are trying
 to run multiple updates in sequence.
 {{{
 #!patch
 diff --git a/hcache.c b/hcache.c
 index f3ca50f..8f2d009 100644
 --- a/hcache.c
 +++ b/hcache.c
 @@ -100,6 +100,7 @@ struct header_cache
  {
    MDB_env *env;
    MDB_txn *txn;
 +  MDB_txn *wtxn;
    MDB_dbi db;
    char *folder;
    unsigned int crc;
 @@ -898,6 +899,7 @@ mutt_hcache_store_raw (header_cache_t* h, const char*
 filename, void* data,
  #elif HAVE_LMDB
    MDB_val key;
    MDB_val databuf;
 +  MDB_txn *local_txn;
    MDB_txn *txn;
    size_t folderlen;
    int rc;
 @@ -929,20 +931,32 @@ mutt_hcache_store_raw (header_cache_t* h, const
 char* filename, void* data,
    key.mv_size = ksize;
    databuf.mv_data = data;
    databuf.mv_size = dlen;
 -  rc = mdb_txn_begin(h->env, NULL, 0, &txn);
 -  if (rc != MDB_SUCCESS)
 -  {
 -    fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
 -    return rc;
 +
 +  txn = NULL;
 +  local_txn = NULL;
 +  if (h->wtxn) {
 +    txn = h->wtxn;
 +  } else {
 +    rc = mdb_txn_begin(h->env, NULL, 0, &local_txn);
 +    if (rc != MDB_SUCCESS)
 +    {
 +      fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
 +      return rc;
 +    }
 +    txn = local_txn;
    }
    rc = mdb_put(txn, h->db, &key, &databuf, 0);
    if (rc != MDB_SUCCESS)
    {
      fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc));
 -    mdb_txn_abort(txn);
 +    if (local_txn) {
 +      mdb_txn_abort(txn);
 +    }
      return rc;
    }
 -  rc = mdb_txn_commit(txn);
 +  if (local_txn) {
 +    rc = mdb_txn_commit(txn);
 +  }
    return rc;
  #else
    strncpy(path, h->folder, sizeof (path));
 @@ -1324,6 +1338,7 @@ hcache_open_lmdb (struct header_cache* h, const
 char* path)
    int rc;

    h->txn = NULL;
 +  h->wtxn = NULL;

    rc = mdb_env_create(&h->env);
    if (rc != MDB_SUCCESS)
 @@ -1414,6 +1429,38 @@ mutt_hcache_delete(header_cache_t *h, const char
 *filename,
    mdb_txn_commit(txn);
    return rc;
  }
 +
 +int mutt_hcache_txn_begin(header_cache_t *h) {
 +  if (!h) {
 +    return -1;
 +  }
 +
 +  return mdb_txn_begin(h->env, NULL, 0, &h->wtxn);
 +}
 +int mutt_hcache_txn_commit(header_cache_t *h) {
 +  int rc;
 +
 +  if (!h || !h->wtxn) {
 +    return -1;
 +  }
 +
 +  rc = mdb_txn_commit(h->wtxn);
 +  if (rc != MDB_SUCCESS) {
 +    h->wtxn = NULL;
 +  }
 +
 +  return rc;
 +}
 +int mutt_hcache_txn_abort(header_cache_t *h) {
 +  if (!h || !h->wtxn) {
 +    return -1;
 +  }
 +
 +  mdb_txn_abort(h->wtxn);
 +  h->wtxn = NULL;
 +
 +  return 0;
 +}
  #endif

  header_cache_t *
 diff --git a/hcache.h b/hcache.h
 index ca0b425..10eead1 100644
 --- a/hcache.h
 +++ b/hcache.h
 @@ -45,6 +45,10 @@ int mutt_hcache_store_raw (header_cache_t *h, const
 char* filename, void* data,
                             size_t dlen, size_t(*keylen) (const char*
 fn));
  int mutt_hcache_delete(header_cache_t *h, const char *filename, size_t
 (*keylen)(const char *fn));

 +int mutt_hcache_txn_begin(header_cache_t *h);
 +int mutt_hcache_txn_commit(header_cache_t *h);
 +int mutt_hcache_txn_abort(header_cache_t *h);
 +
  const char *mutt_hcache_backend (void);

  #endif /* _HCACHE_H_ */
 diff --git a/mh.c b/mh.c
 index c90679e..62325be 100644
 --- a/mh.c
 +++ b/mh.c
 @@ -1154,6 +1154,7 @@ static void maildir_delayed_parsing (CONTEXT * ctx,
 struct maildir **md,

  #if USE_HCACHE
    hc = mutt_hcache_open (HeaderCache, ctx->path, NULL);
 +  mutt_hcache_txn_begin (hc);
  #endif

    for (p = *md, count = 0; p; p = p->next, count++)
 @@ -1216,6 +1217,7 @@ static void maildir_delayed_parsing (CONTEXT * ctx,
 struct maildir **md,
      last = p;
     }
  #if USE_HCACHE
 +  mutt_hcache_txn_commit (hc);
    mutt_hcache_close (hc);
  #endif

 @@ -1898,6 +1900,13 @@ int mh_sync_mailbox_message (CONTEXT * ctx, int
 msgno)
      char path[_POSIX_PATH_MAX], tmp[_POSIX_PATH_MAX];
      HEADER *h = ctx->hdrs[msgno];

 +#if USE_HCACHE
 +    if (hc)
 +    {
 +      mutt_hcache_txn_begin (hc);
 +    }
 +#endif
 +
      if (h->deleted && (ctx->magic != MUTT_MAILDIR || !option
 (OPTMAILDIRTRASH)))
      {
        snprintf (path, sizeof (path), "%s/%s", ctx->path, h->path);
 @@ -1935,26 +1944,41 @@ int mh_sync_mailbox_message (CONTEXT * ctx, int
 msgno)
        if (ctx->magic == MUTT_MAILDIR)
        {
         if (maildir_sync_message (ctx, msgno) == -1)
 -         return -1;
 +          goto errout;
        }
        else
        {
         if (mh_sync_message (ctx, msgno) == -1)
 -         return -1;
 +          goto errout;
        }
      }

  #if USE_HCACHE
 -    if (hc && h->changed)
 +    if (hc)
      {
 -      if (ctx->magic == MUTT_MAILDIR)
 -       mutt_hcache_store (hc, h->path + 3, h, 0, &maildir_hcache_keylen,
 MUTT_GENERATE_UIDVALIDITY);
 -      else if (ctx->magic == MUTT_MH)
 -       mutt_hcache_store (hc, h->path, h, 0, strlen,
 MUTT_GENERATE_UIDVALIDITY);
 +      if (h->changed)
 +      {
 +        if (ctx->magic == MUTT_MAILDIR)
 +          mutt_hcache_store (hc, h->path + 3, h, 0,
 &maildir_hcache_keylen, MUTT_GENERATE_UIDVALIDITY);
 +        else if (ctx->magic == MUTT_MH)
 +          mutt_hcache_store (hc, h->path, h, 0, strlen,
 MUTT_GENERATE_UIDVALIDITY);
 +      }
 +
 +      mutt_hcache_txn_commit(hc);
      }
 +
  #endif

      return 0;
 +
 +errout:
 +#if USE_HCACHE
 +    if (hc)
 +    {
 +      mutt_hcache_txn_abort(hc);
 +    }
 +#endif
 +    return -1;
  }

  int mh_sync_mailbox (CONTEXT * ctx, int *index_hint)
 }}}

--
Ticket URL: <https://dev.mutt.org/trac/ticket/3691#comment:4>
Mutt <http://www.mutt.org/>
The Mutt mail user agent

Reply via email to