Re: [Mutt] #3691: Add LMDB support

[Mutt]

#3691: Add LMDB support
---+--
  Reporter:  rsc   |  Owner:  mutt-dev
  Type:  enhancement   | Status:  new
  Priority:  minor |  Milestone:
 Component:  header cache  |Version:
Resolution:|   Keywords:
---+--
Changes (by neverpanic):

 * Attachment "2016-09-19-diff-re-use-transactions.patch" added.

 Diff to the version in comment 8 that re-uses transactions

--
Ticket URL: 
Mutt 
The Mutt mail user agent

Re: [Mutt] #3691: Add LMDB support

[Mutt]

#3691: Add LMDB support
---+--
  Reporter:  rsc   |  Owner:  mutt-dev
  Type:  enhancement   | Status:  new
  Priority:  minor |  Milestone:
 Component:  header cache  |Version:
Resolution:|   Keywords:
---+--

Comment (by gahr2):

 I have an alternative approach, which is, let's keep the last (read /
 write) transaction open and reuse it until a transaction for a different
 mode (write / read) is required. At that point, reset it if it was a read
 transaction and commit it if it was a write transaction. I noticed a
 relevant speed up with this patch.


 {{{#!diff
 diff -r f1f1af650910 configure.ac
 --- a/configure.ac  Tue May 24 12:08:46 2016 -0700
 +++ b/configure.ac  Mon Sep 19 14:18:01 2016 +
 @@ -855,6 +855,7 @@
  AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if
 it is available]))
  AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if
 it is available]))
  AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4
 if gdbm is not available]))
 +AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if
 gdbm is not available]))

  db_found=no
  if test x$enable_hcache = xyes
 @@ -899,6 +900,15 @@
  db_requested=bdb
fi
  fi
 +if test -n "$with_lmdb" && test "$with_lmdb" != "no"
 +then
 +  if test "$db_requested" != "auto"
 +  then
 +AC_MSG_ERROR([more than one header cache engine requested.])
 +  else
 +db_requested=lmdb
 +  fi
 +fi

  dnl -- Tokyo Cabinet --
  if test "$with_tokyocabinet" != "no" \
 @@ -1042,6 +1052,34 @@
  fi
  fi

 +dnl -- LMDB --
 +if test x$with_lmdb != xno && test $db_found = no \
 +   && test "$db_requested" = auto -o "$db_requested" = lmdb
 +then
 +if test "$with_lmdb" != "yes"
 +then
 +  CPPFLAGS="$CPPFLAGS -I$with_lmdb/include"
 +  LDFLAGS="$LDFLAGS -L$with_lmdb/lib"
 +fi
 +saved_LIBS="$LIBS"
 +LIBS="$LIBS -llmdb"
 +AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[
 +ac_cv_mdbenvcreate=no
 +AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],
 [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[])
 +])
 +LIBS="$saved_LIBS"
 +if test "$ac_cv_mdbenvcreate" = yes
 +then
 +  AC_DEFINE(HAVE_LMDB, 1, [LMDB Support])
 +  MUTTLIBS="$MUTTLIBS -llmdb"
 +  db_found=lmdb
 +fi
 +if test "$db_requested" != auto && test "$db_found" !=
 "$db_requested"
 +then
 +  AC_MSG_ERROR([LMDB could not be used. Check config.log for
 details.])
 +fi
 +fi
 +
  if test $db_found = no
  then
  AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4
 for hcache])
 diff -r f1f1af650910 hcache.c
 --- a/hcache.c  Tue May 24 12:08:46 2016 -0700
 +++ b/hcache.c  Mon Sep 19 14:18:01 2016 +
 @@ -32,6 +32,9 @@
  #include 
  #elif HAVE_DB4
  #include 
 +#elif HAVE_LMDB
 +#define LMDB_DB_SIZE(1024 * 1024 * 1024)
 +#include 
  #endif

  #include 
 @@ -83,6 +86,52 @@

  static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
  static void mutt_hcache_dbt_empty_init(DBT * dbt);
 +#elif HAVE_LMDB
 +struct header_cache
 +{
 +  MDB_env *env;
 +  MDB_txn *txn;
 +  MDB_dbi db;
 +  char *folder;
 +  unsigned int crc;
 +  int last_mode; // 0: read, 1: write
 +};
 +
 +static int mdb_get_r_txn(header_cache_t *h)
 +{
 +  if (h->txn && h->last_mode == 1)
 +  {
 +mdb_txn_commit(h->txn);
 +h->txn = NULL;
 +  }
 +  h->last_mode = 0;
 +  if (!h->txn)
 +  {
 +return mdb_txn_begin(h->env, NULL, MDB_RDONLY, >txn);
 +  }
 +  else
 +  {
 +return mdb_txn_renew(h->txn);
 +  }
 +}
 +
 +static int mdb_get_w_txn(header_cache_t *h)
 +{
 +  if (h->txn && h->last_mode == 0)
 +  {
 +mdb_txn_reset(h->txn);
 +h->txn = NULL;
 +  }
 +  h->last_mode = 1;
 +  if (h->txn)
 +  {
 +return MDB_SUCCESS;
 +  }
 +  else
 +  {
 +return mdb_txn_begin(h->env, NULL, 0, >txn);
 +  }
 +}
  #endif

  typedef union
 @@ -732,6 +793,11 @@
  #elif HAVE_DB4
DBT key;
DBT data;
 +#elif HAVE_LMDB
 +  MDB_val key;
 +  MDB_val data;
 +  size_t folderlen;
 +  int rc;
  #endif

if (!h)
 @@ -748,6 +814,43 @@
h->db->get(h->db, NULL, , , 0);

return data.data;
 +#elif HAVE_LMDB
 +  strncpy(path, h->folder, sizeof (path));
 +  safe_strcat(path, sizeof (path), filename);
 +
 +  folderlen = strlen(h->folder);
 +  ksize = folderlen + keylen(path + folderlen);
 +  key.mv_data = (char *)path;
 +  key.mv_size = ksize;
 +  data.mv_data = NULL;
 +  data.mv_size = 0;
 +  rc = mdb_get_r_txn(h);
 +  if (rc != MDB_SUCCESS)
 +  {
 +h->txn = NULL;
 +fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc));
 +return NULL;
 +  }
 +  rc = mdb_get(h->txn, h->db, , );
 +  if (rc ==