#3691: Add LMDB support
---------------------------+----------------------
Reporter: rsc | Owner: mutt-dev
Type: enhancement | Status: new
Priority: minor | Milestone:
Component: header cache | Version:
Resolution: | Keywords:
---------------------------+----------------------
Comment (by neverpanic):
LMDB performs much better than Tokyo Cabinet while reading, so this is
definitely nice to have. However, I've noticed that while writing the
cache, your code starts a transaction for every single write operation,
which causes this to be much slower than I think it needs to be.
Here's a quick comparison when opening a 130k mail mailbox with empty
cache. In all cases I asked mutt to exit by pressing 'q' while it was
generating the cache.
{{{
rm ./headers.lmdb*; time ./mutt -e 'set header_cache = "./headers.lmdb"'
-f ~/Mail/MacPorts/tickets
real 0m29.271s
user 0m8.320s
sys 0m11.525s
}}}
The same command with changes that explicitly start a single transaction
before updating the cache:
{{{
real 0m10.819s
user 0m6.886s
sys 0m2.762s
}}}
I'm including the changes I used to generate these numbers below, but note
that this is nowhere near ready for a merge as-is. I think `hcache.c`
shouldn't abstract the need for a transaction away from the users of
header cache, because those users alone know best whether they are trying
to run multiple updates in sequence.
{{{
#!patch
diff --git a/hcache.c b/hcache.c
index f3ca50f..8f2d009 100644
--- a/hcache.c
+++ b/hcache.c
@@ -100,6 +100,7 @@ struct header_cache
{
MDB_env *env;
MDB_txn *txn;
+ MDB_txn *wtxn;
MDB_dbi db;
char *folder;
unsigned int crc;
@@ -898,6 +899,7 @@ mutt_hcache_store_raw (header_cache_t* h, const char*
filename, void* data,
#elif HAVE_LMDB
MDB_val key;
MDB_val databuf;
+ MDB_txn *local_txn;
MDB_txn *txn;
size_t folderlen;
int rc;
@@ -929,20 +931,32 @@ mutt_hcache_store_raw (header_cache_t* h, const
char* filename, void* data,
key.mv_size = ksize;
databuf.mv_data = data;
databuf.mv_size = dlen;
- rc = mdb_txn_begin(h->env, NULL, 0, &txn);
- if (rc != MDB_SUCCESS)
- {
- fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
- return rc;
+
+ txn = NULL;
+ local_txn = NULL;
+ if (h->wtxn) {
+ txn = h->wtxn;
+ } else {
+ rc = mdb_txn_begin(h->env, NULL, 0, &local_txn);
+ if (rc != MDB_SUCCESS)
+ {
+ fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
+ return rc;
+ }
+ txn = local_txn;
}
rc = mdb_put(txn, h->db, &key, &databuf, 0);
if (rc != MDB_SUCCESS)
{
fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc));
- mdb_txn_abort(txn);
+ if (local_txn) {
+ mdb_txn_abort(txn);
+ }
return rc;
}
- rc = mdb_txn_commit(txn);
+ if (local_txn) {
+ rc = mdb_txn_commit(txn);
+ }
return rc;
#else
strncpy(path, h->folder, sizeof (path));
@@ -1324,6 +1338,7 @@ hcache_open_lmdb (struct header_cache* h, const
char* path)
int rc;
h->txn = NULL;
+ h->wtxn = NULL;
rc = mdb_env_create(&h->env);
if (rc != MDB_SUCCESS)
@@ -1414,6 +1429,38 @@ mutt_hcache_delete(header_cache_t *h, const char
*filename,
mdb_txn_commit(txn);
return rc;
}
+
+int mutt_hcache_txn_begin(header_cache_t *h) {
+ if (!h) {
+ return -1;
+ }
+
+ return mdb_txn_begin(h->env, NULL, 0, &h->wtxn);
+}
+int mutt_hcache_txn_commit(header_cache_t *h) {
+ int rc;
+
+ if (!h || !h->wtxn) {
+ return -1;
+ }
+
+ rc = mdb_txn_commit(h->wtxn);
+ if (rc != MDB_SUCCESS) {
+ h->wtxn = NULL;
+ }
+
+ return rc;
+}
+int mutt_hcache_txn_abort(header_cache_t *h) {
+ if (!h || !h->wtxn) {
+ return -1;
+ }
+
+ mdb_txn_abort(h->wtxn);
+ h->wtxn = NULL;
+
+ return 0;
+}
#endif
header_cache_t *
diff --git a/hcache.h b/hcache.h
index ca0b425..10eead1 100644
--- a/hcache.h
+++ b/hcache.h
@@ -45,6 +45,10 @@ int mutt_hcache_store_raw (header_cache_t *h, const
char* filename, void* data,
size_t dlen, size_t(*keylen) (const char*
fn));
int mutt_hcache_delete(header_cache_t *h, const char *filename, size_t
(*keylen)(const char *fn));
+int mutt_hcache_txn_begin(header_cache_t *h);
+int mutt_hcache_txn_commit(header_cache_t *h);
+int mutt_hcache_txn_abort(header_cache_t *h);
+
const char *mutt_hcache_backend (void);
#endif /* _HCACHE_H_ */
diff --git a/mh.c b/mh.c
index c90679e..62325be 100644
--- a/mh.c
+++ b/mh.c
@@ -1154,6 +1154,7 @@ static void maildir_delayed_parsing (CONTEXT * ctx,
struct maildir **md,
#if USE_HCACHE
hc = mutt_hcache_open (HeaderCache, ctx->path, NULL);
+ mutt_hcache_txn_begin (hc);
#endif
for (p = *md, count = 0; p; p = p->next, count++)
@@ -1216,6 +1217,7 @@ static void maildir_delayed_parsing (CONTEXT * ctx,
struct maildir **md,
last = p;
}
#if USE_HCACHE
+ mutt_hcache_txn_commit (hc);
mutt_hcache_close (hc);
#endif
@@ -1898,6 +1900,13 @@ int mh_sync_mailbox_message (CONTEXT * ctx, int
msgno)
char path[_POSIX_PATH_MAX], tmp[_POSIX_PATH_MAX];
HEADER *h = ctx->hdrs[msgno];
+#if USE_HCACHE
+ if (hc)
+ {
+ mutt_hcache_txn_begin (hc);
+ }
+#endif
+
if (h->deleted && (ctx->magic != MUTT_MAILDIR || !option
(OPTMAILDIRTRASH)))
{
snprintf (path, sizeof (path), "%s/%s", ctx->path, h->path);
@@ -1935,26 +1944,41 @@ int mh_sync_mailbox_message (CONTEXT * ctx, int
msgno)
if (ctx->magic == MUTT_MAILDIR)
{
if (maildir_sync_message (ctx, msgno) == -1)
- return -1;
+ goto errout;
}
else
{
if (mh_sync_message (ctx, msgno) == -1)
- return -1;
+ goto errout;
}
}
#if USE_HCACHE
- if (hc && h->changed)
+ if (hc)
{
- if (ctx->magic == MUTT_MAILDIR)
- mutt_hcache_store (hc, h->path + 3, h, 0, &maildir_hcache_keylen,
MUTT_GENERATE_UIDVALIDITY);
- else if (ctx->magic == MUTT_MH)
- mutt_hcache_store (hc, h->path, h, 0, strlen,
MUTT_GENERATE_UIDVALIDITY);
+ if (h->changed)
+ {
+ if (ctx->magic == MUTT_MAILDIR)
+ mutt_hcache_store (hc, h->path + 3, h, 0,
&maildir_hcache_keylen, MUTT_GENERATE_UIDVALIDITY);
+ else if (ctx->magic == MUTT_MH)
+ mutt_hcache_store (hc, h->path, h, 0, strlen,
MUTT_GENERATE_UIDVALIDITY);
+ }
+
+ mutt_hcache_txn_commit(hc);
}
+
#endif
return 0;
+
+errout:
+#if USE_HCACHE
+ if (hc)
+ {
+ mutt_hcache_txn_abort(hc);
+ }
+#endif
+ return -1;
}
int mh_sync_mailbox (CONTEXT * ctx, int *index_hint)
}}}
--
Ticket URL: <https://dev.mutt.org/trac/ticket/3691#comment:4>
Mutt <http://www.mutt.org/>
The Mutt mail user agent
