Wietse Venema wrote:
Howard Chu:
Wietse Venema wrote:
Howard Chu:
My apologies for sending an inadequately tested patch. This is also needed for
read-only access/enumeration to work:
I suggest letting the code cool down for 24 hours. I used to have
code freezes of a week in TCP Wrapper days. A week is unfortunately
no longer an option as things evolve at internet speeds. But 24
hours is still doable.
Looks like I still missed a few places in the docs. Need to add a description
of mdb_map_size to proto/postconf.proto and add MDB_README to the
proto/Makefile.in.
I'm also wondering if it's necessary to provide a config keyword for MDB's
max_readers parameter. The default is 126. If there can be more than 126
threads (and/or processes) reading concurrently from a single table, this
needs to be settable.
If each Postfix process counts as one reader, then a limit of 128
won't be sufficient. The default_process_limit value is 100. That
means there can be 100 smtpd processes and 100 cleanup processes
OK, reposting the entire patch set with this taken into account.
--
-- Howard Chu
CTO, Symas Corp. http://www.symas.com
Director, Highland Sun http://highlandsun.com/hyc/
Chief Architect, OpenLDAP http://www.openldap.org/project/
>From 4df72d4728d97803ca1d5cc67cf311d82703ea46 Mon Sep 17 00:00:00 2001
From: Howard Chu <h...@symas.com>
Date: Thu, 20 Sep 2012 09:27:14 -0700
Subject: [PATCH] Add support for OpenLDAP MDB
---
proto/MDB_README.html | 109 +++++++++
proto/Makefile.in | 8 +
proto/postconf.proto | 11 +
src/global/Makefile.in | 4 +-
src/global/data_redirect.c | 2 +
src/global/mail_params.c | 9 +
src/global/mail_params.h | 7 +
src/global/mkmap.h | 1 +
src/global/mkmap_mdb.c | 109 +++++++++
src/global/mkmap_open.c | 4 +
src/util/Makefile.in | 9 +-
src/util/dict_mdb.c | 535 ++++++++++++++++++++++++++++++++++++++++++++
src/util/dict_mdb.h | 41 ++++
src/util/dict_open.c | 4 +
14 files changed, 848 insertions(+), 5 deletions(-)
create mode 100644 proto/MDB_README.html
create mode 100644 src/global/mkmap_mdb.c
create mode 100644 src/util/dict_mdb.c
create mode 100644 src/util/dict_mdb.h
diff --git a/proto/MDB_README.html b/proto/MDB_README.html
new file mode 100644
index 0000000..042ecef
--- /dev/null
+++ b/proto/MDB_README.html
@@ -0,0 +1,109 @@
+<!doctype html public "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd";>
+
+<html>
+
+<head>
+
+<title>Postfix OpenLDAP MDB Howto</title>
+
+<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
+
+</head>
+
+<body>
+
+<h1><img src="postfix-logo.jpg" width="203" height="98" ALT="">Postfix OpenLDAP MDB Howto</h1>
+
+<hr>
+
+<h2>Introduction</h2>
+
+<p> Postfix uses databases of various kinds to store and look up
+information. Postfix databases are specified as "type:name".
+OpenLDAP MDB implements the Postfix database type "mdb".
+The name of a Postfix OpenLDAP MDB database is the name
+of the database file without the ".mdb" suffix. OpenLDAP MDB databases
+are maintained with the postmap(1) command. </p>
+
+<p> This document describes: </p>
+
+<ol>
+
+<li> <p> How to build Postfix <a href="#with_mdb">with OpenLDAP
+MDB support</a>. </p>
+
+<li> <p> How to <a href="#configure">configure</a> MDB settings. </p>
+
+<li> <p> Missing <a href="#pthread">pthread</a> library trouble. </p>
+
+</ol>
+
+<h2><a name="with_mdb">Building Postfix with OpenLDAP MDB support</a></h2>
+
+<p> Postfix normally does not enable OpenLDAP MDB support.
+To build Postfix after you installed OpenLDAP MDB from
+source code, use something like: </p>
+
+<blockquote>
+<pre>
+% make makefiles CCARGS="-DHAS_MDB -I/usr/local/include" \
+ AUXLIBS="-L/usr/local/lib -lmdb"
+% make
+</pre>
+</blockquote>
+
+<p> Solaris needs this: </p>
+
+<blockquote>
+<pre>
+% make makefiles CCARGS="-DHAS_MDB -I/usr/local/include" \
+ AUXLIBS="-R/usr/local/lib -L/usr/local/lib -lmdb"
+% make
+</pre>
+</blockquote>
+
+<p> The exact pathnames depend on how OpenLDAP MDB was installed. </p>
+
+<h2><a name="configure">Configure MDB settings</a></h2>
+
+<p> Postfix provides a configuration parameter that controls how
+large an OpenLDAP MDB database may grow. </p>
+
+<ul>
+
+<li> <p> mdb_map_size (default: 10 MBytes per
+table). This setting controls how large any OpenLDAP MDB database
+may grow. It must be set large enough to accommodate the largest
+table that Postfix will use. </p>
+
+</ul>
+
+<h2><a name="pthread">Missing pthread library trouble</a></h2>
+
+<p> When building Postfix fails with: </p>
+
+<blockquote>
+<pre>
+undefined reference to `pthread_mutexattr_destroy'
+undefined reference to `pthread_mutexattr_init'
+undefined reference to `pthread_mutex_lock'
+</pre>
+</blockquote>
+
+<p> Add the "-lpthread" library to the "make makefiles" command. </p>
+
+<blockquote>
+<pre>
+% make makefiles .... AUXLIBS="... -lpthread"
+</pre>
+</blockquote>
+
+<p> Source code for OpenLDAP MDB is available at
+http://www.openldap.org.
+More information is available at
+http://highlandsun.com/hyc/mdb/. </p>
+
+</body>
+
+</html>
diff --git a/proto/Makefile.in b/proto/Makefile.in
index d71454f..6da7317 100644
--- a/proto/Makefile.in
+++ b/proto/Makefile.in
@@ -23,6 +23,7 @@ HTML = ../html/ADDRESS_CLASS_README.html \
../html/LDAP_README.html \
../html/LINUX_README.html \
../html/LOCAL_RECIPIENT_README.html ../html/MAILDROP_README.html \
+ ../html/MDB_README.html \
../html/MEMCACHE_README.html \
../html/MILTER_README.html \
../html/MULTI_INSTANCE_README.html \
@@ -64,6 +65,7 @@ README = ../README_FILES/ADDRESS_CLASS_README \
../README_FILES/LDAP_README \
../README_FILES/LINUX_README \
../README_FILES/LOCAL_RECIPIENT_README ../README_FILES/MAILDROP_README \
+ ../README_FILES/MDB_README \
../README_FILES/MEMCACHE_README \
../README_FILES/MILTER_README \
../README_FILES/MULTI_INSTANCE_README \
@@ -201,6 +203,9 @@ clobber:
../html/MAILDROP_README.html: MAILDROP_README.html
$(POSTLINK) $? >$@
+../html/MDB_README.html: MDB_README.html
+ $(POSTLINK) $? >$@
+
../html/MEMCACHE_README.html: MEMCACHE_README.html
$(POSTLINK) $? >$@
@@ -360,6 +365,9 @@ clobber:
../README_FILES/MAILDROP_README: MAILDROP_README.html
$(HT2READ) $? >$@
+../README_FILES/MDB_README: MDB_README.html
+ $(HT2READ) $? >$@
+
../README_FILES/MEMCACHE_README: MEMCACHE_README.html
$(HT2READ) $? >$@
diff --git a/proto/postconf.proto b/proto/postconf.proto
index f03af49..6222f14 100644
--- a/proto/postconf.proto
+++ b/proto/postconf.proto
@@ -2824,6 +2824,17 @@ The default time unit is d (days).
Specify 0 when mail delivery should be tried only once.
</p>
+%PARAM mdb_map_size 10485760
+
+<p>
+The per-table size limit for programs that create OpenLDAP MDB
+tables. Specify a byte count.
+</p>
+
+<p>
+This feature is available in Postfix 2.10 and later.
+</p>
+
%PARAM message_size_limit 10240000
<p>
diff --git a/src/global/Makefile.in b/src/global/Makefile.in
index 664d772..92d93b8 100644
--- a/src/global/Makefile.in
+++ b/src/global/Makefile.in
@@ -16,7 +16,7 @@ SRCS = abounce.c anvil_clnt.c been_here.c bounce.c bounce_log.c \
mail_params.c mail_pathname.c mail_queue.c mail_run.c \
mail_scan_dir.c mail_stream.c mail_task.c mail_trigger.c maps.c \
mark_corrupt.c match_parent_style.c mbox_conf.c mbox_open.c \
- mime_state.c mkmap_cdb.c mkmap_db.c mkmap_dbm.c mkmap_open.c \
+ mime_state.c mkmap_cdb.c mkmap_db.c mkmap_dbm.c mkmap_mdb.c mkmap_open.c \
mkmap_sdbm.c msg_stats_print.c msg_stats_scan.c mynetworks.c \
mypwd.c namadr_list.c off_cvt.c opened.c own_inet_addr.c \
pipe_command.c post_mail.c quote_821_local.c quote_822_local.c \
@@ -50,7 +50,7 @@ OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \
mail_params.o mail_pathname.o mail_queue.o mail_run.o \
mail_scan_dir.o mail_stream.o mail_task.o mail_trigger.o maps.o \
mark_corrupt.o match_parent_style.o mbox_conf.o mbox_open.o \
- mime_state.o mkmap_cdb.o mkmap_db.o mkmap_dbm.o mkmap_open.o \
+ mime_state.o mkmap_cdb.o mkmap_db.o mkmap_dbm.o mkmap_mdb.o mkmap_open.o \
mkmap_sdbm.o msg_stats_print.o msg_stats_scan.o mynetworks.o \
mypwd.o namadr_list.o off_cvt.o opened.o own_inet_addr.o \
pipe_command.o post_mail.o quote_821_local.o quote_822_local.o \
diff --git a/src/global/data_redirect.c b/src/global/data_redirect.c
index 096e58c..f914155 100644
--- a/src/global/data_redirect.c
+++ b/src/global/data_redirect.c
@@ -72,6 +72,7 @@
#include <dict_db.h>
#include <dict_dbm.h>
#include <dict_cdb.h>
+#include <dict_mdb.h>
#include <warn_stat.h>
/* Global directory. */
@@ -99,6 +100,7 @@ static const NAME_CODE data_redirect_map_types[] = {
DICT_TYPE_HASH, 1,
DICT_TYPE_BTREE, 1,
DICT_TYPE_DBM, 1,
+ DICT_TYPE_MDB, 1,
DICT_TYPE_CDB, 1, /* not a read-write map type */
"sdbm", 1, /* legacy 3rd-party TLS */
"dbz", 1, /* just in case */
diff --git a/src/global/mail_params.c b/src/global/mail_params.c
index 0e098b3..a14afe6 100644
--- a/src/global/mail_params.c
+++ b/src/global/mail_params.c
@@ -96,6 +96,7 @@
/* char *var_proxywrite_service;
/* int var_db_create_buf;
/* int var_db_read_buf;
+/* int var_mdb_map_size;
/* int var_mime_maxdepth;
/* int var_mime_bound_len;
/* int var_header_limit;
@@ -177,6 +178,9 @@
#ifdef HAS_DB
#include <dict_db.h>
#endif
+#ifdef HAS_MDB
+#include <dict_mdb.h>
+#endif
#include <inet_proto.h>
#include <vstring_vstream.h>
#include <iostuff.h>
@@ -284,6 +288,7 @@ char *var_proxymap_service;
char *var_proxywrite_service;
int var_db_create_buf;
int var_db_read_buf;
+int var_mdb_map_size;
int var_mime_maxdepth;
int var_mime_bound_len;
int var_header_limit;
@@ -596,6 +601,7 @@ void mail_params_init()
VAR_FAULT_INJ_CODE, DEF_FAULT_INJ_CODE, &var_fault_inj_code, 0, 0,
VAR_DB_CREATE_BUF, DEF_DB_CREATE_BUF, &var_db_create_buf, 1, 0,
VAR_DB_READ_BUF, DEF_DB_READ_BUF, &var_db_read_buf, 1, 0,
+ VAR_MDB_MAP_SIZE, DEF_MDB_MAP_SIZE, &var_mdb_map_size, 1, 0,
VAR_HEADER_LIMIT, DEF_HEADER_LIMIT, &var_header_limit, 1, 0,
VAR_TOKEN_LIMIT, DEF_TOKEN_LIMIT, &var_token_limit, 1, 0,
VAR_MIME_MAXDEPTH, DEF_MIME_MAXDEPTH, &var_mime_maxdepth, 1, 0,
@@ -713,6 +719,9 @@ void mail_params_init()
#ifdef HAS_DB
dict_db_cache_size = var_db_read_buf;
#endif
+#ifdef HAS_MDB
+ dict_mdb_map_size = var_mdb_map_size;
+#endif
inet_windowsize = var_inet_windowsize;
/*
diff --git a/src/global/mail_params.h b/src/global/mail_params.h
index 57135a0..935ba22 100644
--- a/src/global/mail_params.h
+++ b/src/global/mail_params.h
@@ -2728,6 +2728,13 @@ extern int var_db_create_buf;
extern int var_db_read_buf;
/*
+ * OpenLDAP MDB memory map size.
+ */
+#define VAR_MDB_MAP_SIZE "mdb_map_size"
+#define DEF_MDB_MAP_SIZE (16 * 1024 *1024)
+extern int var_mdb_map_size;
+
+ /*
* Named queue file attributes.
*/
#define VAR_QATTR_COUNT_LIMIT "queue_file_attribute_count_limit"
diff --git a/src/global/mkmap.h b/src/global/mkmap.h
index d74a912..e9f3a57 100644
--- a/src/global/mkmap.h
+++ b/src/global/mkmap.h
@@ -39,6 +39,7 @@ extern MKMAP *mkmap_dbm_open(const char *);
extern MKMAP *mkmap_cdb_open(const char *);
extern MKMAP *mkmap_hash_open(const char *);
extern MKMAP *mkmap_btree_open(const char *);
+extern MKMAP *mkmap_mdb_open(const char *);
extern MKMAP *mkmap_sdbm_open(const char *);
extern MKMAP *mkmap_proxy_open(const char *);
extern MKMAP *mkmap_fail_open(const char *);
diff --git a/src/global/mkmap_mdb.c b/src/global/mkmap_mdb.c
new file mode 100644
index 0000000..06df11a
--- /dev/null
+++ b/src/global/mkmap_mdb.c
@@ -0,0 +1,109 @@
+/*++
+/* NAME
+/* mkmap_mdb 3
+/* SUMMARY
+/* create or open database, MDB style
+/* SYNOPSIS
+/* #include <mkmap.h>
+/*
+/* MKMAP *mkmap_mdb_open(path)
+/* const char *path;
+/*
+/* DESCRIPTION
+/* This module implements support for creating MDB databases.
+/*
+/* mkmap_mdb_open() takes a file name, appends the ".mdb"
+/* suffix, and does whatever initialization is required
+/* before the OpenLDAP MDB open routine is called.
+/*
+/* All errors are fatal.
+/* SEE ALSO
+/* dict_mdb(3), MDB dictionary interface.
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+/* System library. */
+
+#include <sys_defs.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Utility library. */
+
+#include <msg.h>
+#include <mymalloc.h>
+#include <stringops.h>
+#include <dict.h>
+#include <dict_mdb.h>
+#include <myflock.h>
+#include <warn_stat.h>
+
+/* Global library. */
+
+#include <mail_conf.h>
+#include <mail_params.h>
+
+/* Application-specific. */
+
+#include "mkmap.h"
+
+#ifdef HAS_MDB
+#ifdef PATH_MDB_H
+#include PATH_MDB_H
+#else
+#include <mdb.h>
+#endif
+
+int var_proc_limit;
+
+/* mkmap_mdb_open */
+
+MKMAP *mkmap_mdb_open(const char *path)
+{
+ MKMAP *mkmap = (MKMAP *) mymalloc(sizeof(*mkmap));
+ static const CONFIG_INT_TABLE int_table[] = {
+ VAR_PROC_LIMIT, DEF_PROC_LIMIT, &var_proc_limit, 1, 0,
+ 0,
+ };
+
+ get_mail_conf_int_table(int_table);
+
+ /*
+ * Override the default per-table map size for map (re)builds.
+ *
+ * mdb_map_size is defined in util/dict_mdb.c and defaults to 10MB.
+ * It needs to be large enough to contain the largest tables in use.
+ *
+ * XXX This should be specified via the DICT interface so that the buffer
+ * size becomes an object property, instead of being specified by poking
+ * a global variable so that it becomes a class property.
+ */
+ dict_mdb_map_size = var_mdb_map_size;
+
+ /*
+ * Set the max number of concurrent readers per table. This is the
+ * maximum number of postfix processes, plus some extra for CLI users.
+ */
+ dict_mdb_max_readers = var_proc_limit * 2 + 16;
+
+ /*
+ * Fill in the generic members.
+ */
+ mkmap->open = dict_mdb_open;
+ mkmap->after_open = 0;
+ mkmap->after_close = 0;
+
+ /*
+ * MDB uses MVCC so it needs no special lock management here.
+ */
+
+ return (mkmap);
+}
+#endif
diff --git a/src/global/mkmap_open.c b/src/global/mkmap_open.c
index d939d44..918c855 100644
--- a/src/global/mkmap_open.c
+++ b/src/global/mkmap_open.c
@@ -65,6 +65,7 @@
#include <dict_db.h>
#include <dict_cdb.h>
#include <dict_dbm.h>
+#include <dict_mdb.h>
#include <dict_sdbm.h>
#include <dict_proxy.h>
#include <dict_fail.h>
@@ -101,6 +102,9 @@ static const MKMAP_OPEN_INFO mkmap_types[] = {
DICT_TYPE_HASH, mkmap_hash_open,
DICT_TYPE_BTREE, mkmap_btree_open,
#endif
+#ifdef HAS_MDB
+ DICT_TYPE_MDB, mkmap_mdb_open,
+#endif
DICT_TYPE_FAIL, mkmap_fail_open,
0,
};
diff --git a/src/util/Makefile.in b/src/util/Makefile.in
index 4bef131..56e44e2 100644
--- a/src/util/Makefile.in
+++ b/src/util/Makefile.in
@@ -4,7 +4,7 @@ SRCS = alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \
attr_scan_plain.c auto_clnt.c base64_code.c basename.c binhash.c \
chroot_uid.c cidr_match.c clean_env.c close_on_exec.c concatenate.c \
ctable.c dict.c dict_alloc.c dict_cdb.c dict_cidr.c dict_db.c \
- dict_dbm.c dict_debug.c dict_env.c dict_ht.c dict_ni.c dict_nis.c \
+ dict_dbm.c dict_debug.c dict_env.c dict_ht.c dict_mdb.c dict_ni.c dict_nis.c \
dict_nisplus.c dict_open.c dict_pcre.c dict_regexp.c dict_sdbm.c \
dict_static.c dict_tcp.c dict_unix.c dir_forest.c doze.c dummy_read.c \
dummy_write.c duplex_pipe.c environ.c events.c exec_command.c \
@@ -41,7 +41,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \
chroot_uid.o cidr_match.o clean_env.o close_on_exec.o concatenate.o \
ctable.o dict.o dict_alloc.o dict_cdb.o dict_cidr.o dict_db.o \
- dict_dbm.o dict_debug.o dict_env.o dict_ht.o dict_ni.o dict_nis.o \
+ dict_dbm.o dict_debug.o dict_env.o dict_ht.o dict_mdb.o dict_ni.o dict_nis.o \
dict_nisplus.o dict_open.o dict_pcre.o dict_regexp.o dict_sdbm.o \
dict_static.o dict_tcp.o dict_unix.o dir_forest.o doze.o dummy_read.o \
dummy_write.o duplex_pipe.o environ.o events.o exec_command.o \
@@ -76,7 +76,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \
chroot_uid.h cidr_match.h clean_env.h connect.h ctable.h dict.h \
dict_cdb.h dict_cidr.h dict_db.h dict_dbm.h dict_env.h dict_ht.h \
- dict_ni.h dict_nis.h dict_nisplus.h dict_pcre.h dict_regexp.h \
+ dict_mdb.h dict_ni.h dict_nis.h dict_nisplus.h dict_pcre.h dict_regexp.h \
dict_sdbm.h dict_static.h dict_tcp.h dict_unix.h dir_forest.h \
events.h exec_command.h find_inet.h fsspace.h fullname.h \
get_domainname.h get_hostname.h hex_code.h hex_quote.h host_port.h \
@@ -983,6 +983,9 @@ dict_ht.o: sys_defs.h
dict_ht.o: vbuf.h
dict_ht.o: vstream.h
dict_ht.o: vstring.h
+dict_mdb.o: dict.h
+dict_mdb.o: dict_mdb.h
+dict_mdb.o: dict_mdb.c
dict_ni.o: dict_ni.c
dict_ni.o: sys_defs.h
dict_nis.o: argv.h
diff --git a/src/util/dict_mdb.c b/src/util/dict_mdb.c
new file mode 100644
index 0000000..387f76d
--- /dev/null
+++ b/src/util/dict_mdb.c
@@ -0,0 +1,535 @@
+/*++
+/* NAME
+/* dict_mdb 3
+/* SUMMARY
+/* dictionary manager interface to OpenLDAP MDB files
+/* SYNOPSIS
+/* #include <dict_mdb.h>
+/*
+/* DICT *dict_mdb_open(path, open_flags, dict_flags)
+/* const char *name;
+/* const char *path;
+/* int open_flags;
+/* int dict_flags;
+/* DESCRIPTION
+/* dict_mdb_open() opens the named MDB database and makes it available
+/* via the generic interface described in dict_open(3).
+/*
+/* The dict_mdb_map_size variable specifies a non-default per-table
+/* memory map size. The map size is 10MB. The map size is also the
+/* maximum size the table can grow to, so it must be set large enough
+/* to accomodate the largest tables in use.
+/* DIAGNOSTICS
+/* Fatal errors: cannot open file, file write error, out of memory.
+/* SEE ALSO
+/* dict(3) generic dictionary manager
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+#include "sys_defs.h"
+
+#ifdef HAS_MDB
+
+/* System library. */
+
+#include <sys/stat.h>
+#ifdef PATH_MDB_H
+#include PATH_MDB_H
+#else
+#include <mdb.h>
+#endif
+#include <string.h>
+#include <unistd.h>
+
+/* Utility library. */
+
+#include "msg.h"
+#include "mymalloc.h"
+#include "htable.h"
+#include "iostuff.h"
+#include "vstring.h"
+#include "myflock.h"
+#include "stringops.h"
+#include "dict.h"
+#include "dict_mdb.h"
+#include "warn_stat.h"
+
+/* Application-specific. */
+
+typedef struct {
+ DICT dict; /* generic members */
+ MDB_env *env; /* MDB environment */
+ MDB_dbi dbi; /* database handle */
+ MDB_txn *txn; /* write transaction for O_TRUNC */
+ MDB_cursor *cursor; /* for sequence ops */
+ VSTRING *key_buf; /* key buffer */
+ VSTRING *val_buf; /* result buffer */
+} DICT_MDB;
+
+#define SCOPY(buf, data, size) \
+ vstring_str(vstring_strncpy(buf ? buf : (buf = vstring_alloc(10)), data, size))
+
+size_t dict_mdb_map_size = (10 * 1024 * 1024); /* 10MB default mmap size */
+unsigned int dict_mdb_max_readers = 216; /* 200 postfix processes, plus some extra */
+
+/* dict_mdb_lookup - find database entry */
+
+static const char *dict_mdb_lookup(DICT *dict, const char *name)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ const char *result = 0;
+ int status, klen;
+
+ dict->error = 0;
+ klen = strlen(name);
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_lookup: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+
+ /*
+ * Start a read transaction if there's no global txn.
+ */
+ if (dict_mdb->txn)
+ txn = dict_mdb->txn;
+ else if ((status = mdb_txn_begin(dict_mdb->env, NULL, MDB_RDONLY, &txn)))
+ msg_fatal("%s: txn_begin(read) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * See if this MDB file was written with one null byte appended to key
+ * and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen + 1;
+ status = mdb_get(txn, dict_mdb->dbi, &mdb_key, &mdb_value);
+ if (!status) {
+ dict->flags &= ~DICT_FLAG_TRY0NULL;
+ result = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ }
+ }
+
+ /*
+ * See if this MDB file was written with no null byte appended to key and
+ * value.
+ */
+ if (result == 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen;
+ status = mdb_get(txn, dict_mdb->dbi, &mdb_key, &mdb_value);
+ if (!status) {
+ dict->flags &= ~DICT_FLAG_TRY1NULL;
+ result = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ }
+ }
+
+ /*
+ * Close the read txn if it's not the global txn.
+ */
+ if (!dict_mdb->txn)
+ mdb_txn_abort(txn);
+
+ return (result);
+}
+
+/* dict_mdb_update - add or update database entry */
+
+static int dict_mdb_update(DICT *dict, const char *name, const char *value)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ int status;
+
+ dict->error = 0;
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_update: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+ mdb_key.mv_data = (void *) name;
+ mdb_value.mv_data = (void *) value;
+ mdb_key.mv_size = strlen(name);
+ mdb_value.mv_size = strlen(value);
+
+ /*
+ * If undecided about appending a null byte to key and value, choose a
+ * default depending on the platform.
+ */
+ if ((dict->flags & DICT_FLAG_TRY1NULL)
+ && (dict->flags & DICT_FLAG_TRY0NULL)) {
+#ifdef MDB_NO_TRAILING_NULL
+ dict->flags &= ~DICT_FLAG_TRY1NULL;
+#else
+ dict->flags &= ~DICT_FLAG_TRY0NULL;
+#endif
+ }
+
+ /*
+ * Optionally append a null byte to key and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_size++;
+ mdb_value.mv_size++;
+ }
+
+ /*
+ * Start a write transaction if there's no global txn.
+ */
+ if (dict_mdb->txn)
+ txn = dict_mdb->txn;
+ else if ((status = mdb_txn_begin(dict_mdb->env, NULL, 0, &txn)))
+ msg_fatal("%s: txn_begin(write) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * Do the update.
+ */
+ status = mdb_put(txn, dict_mdb->dbi, &mdb_key, &mdb_value,
+ (dict->flags & DICT_FLAG_DUP_REPLACE) ? 0 : MDB_NOOVERWRITE);
+ if (status) {
+ if (status == MDB_KEYEXIST) {
+ if (dict->flags & DICT_FLAG_DUP_IGNORE)
+ /* void */ ;
+ else if (dict->flags & DICT_FLAG_DUP_WARN)
+ msg_warn("%s: duplicate entry: \"%s\"", dict_mdb->dict.name, name);
+ else
+ msg_fatal("%s: duplicate entry: \"%s\"", dict_mdb->dict.name, name);
+ } else {
+ msg_fatal("error writing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ }
+ }
+
+ /*
+ * Commit the transaction if it's not the global txn.
+ */
+ if (!dict_mdb->txn && ((status = mdb_txn_commit(txn))))
+ msg_fatal("error committing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ return (status);
+}
+
+/* dict_mdb_delete - delete one entry from the dictionary */
+
+static int dict_mdb_delete(DICT *dict, const char *name)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_txn *txn;
+ int status = 1, klen, rc;
+
+ dict->error = 0;
+ klen = strlen(name);
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_delete: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+
+ /*
+ * Start a write transaction if there's no global txn.
+ */
+ if (dict_mdb->txn)
+ txn = dict_mdb->txn;
+ else if ((status = mdb_txn_begin(dict_mdb->env, NULL, 0, &txn)))
+ msg_fatal("%s: txn_begin(write) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * See if this MDB file was written with one null byte appended to key
+ * and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen + 1;
+ status = mdb_del(txn, dict_mdb->dbi, &mdb_key, NULL);
+ if (status) {
+ if (status == MDB_NOTFOUND)
+ status = 1;
+ else
+ msg_fatal("error deleting from %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ } else {
+ dict->flags &= ~DICT_FLAG_TRY0NULL; /* found */
+ }
+ }
+
+ /*
+ * See if this MDB file was written with no null byte appended to key and
+ * value.
+ */
+ if (status > 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen;
+ status = mdb_del(txn, dict_mdb->dbi, &mdb_key, NULL);
+ if (status) {
+ if (status == MDB_NOTFOUND)
+ status = 1;
+ else
+ msg_fatal("error deleting from %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ } else {
+ dict->flags &= ~DICT_FLAG_TRY1NULL; /* found */
+ }
+ }
+
+ /*
+ * Commit the transaction if it's not the global txn.
+ */
+ if (!dict_mdb->txn && ((rc = mdb_txn_commit(txn))))
+ msg_fatal("error committing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(rc));
+
+ return (status);
+}
+
+/* traverse the dictionary */
+
+static int dict_mdb_sequence(DICT *dict, int function,
+ const char **key, const char **value)
+{
+ const char *myname = "dict_mdb_sequence";
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ MDB_cursor_op op;
+ int status;
+
+ dict->error = 0;
+
+ /*
+ * Determine the seek function.
+ */
+ switch (function) {
+ case DICT_SEQ_FUN_FIRST:
+ op = MDB_FIRST;
+ break;
+ case DICT_SEQ_FUN_NEXT:
+ op = MDB_NEXT;
+ break;
+ default:
+ msg_panic("%s: invalid function: %d", myname, function);
+ }
+
+ /*
+ * Open a read transaction and cursor if needed.
+ */
+ if (dict_mdb->cursor == 0) {
+ if ((status = mdb_txn_begin(dict_mdb->env, NULL, MDB_RDONLY, &txn)))
+ msg_fatal("%s: txn_begin(read) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+ if ((status = mdb_cursor_open(txn, dict_mdb->dbi, &dict_mdb->cursor)))
+ msg_fatal("%s: cursor_open dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+ }
+
+ /*
+ * Database lookup.
+ */
+ status = mdb_cursor_get(dict_mdb->cursor, &mdb_key, &mdb_value, op);
+ if (status && status != MDB_NOTFOUND)
+ msg_fatal("%s: seeking dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ if (status == MDB_NOTFOUND) {
+ /*
+ * Caller must read to end, to ensure cursor gets closed.
+ */
+ status = 1;
+ txn = mdb_cursor_txn(dict_mdb->cursor);
+ mdb_cursor_close(dict_mdb->cursor);
+ mdb_txn_abort(txn);
+ dict_mdb->cursor = 0;
+ } else {
+
+ /*
+ * Copy the key so that it is guaranteed null terminated.
+ */
+ *key = SCOPY(dict_mdb->key_buf, mdb_key.mv_data, mdb_key.mv_size);
+
+ if (mdb_value.mv_data != 0 && mdb_value.mv_size > 0) {
+
+ /*
+ * Copy the value so that it is guaranteed null terminated.
+ */
+ *value = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ status = 0;
+ }
+ }
+
+ return (status);
+}
+
+/* dict_mdb_lock - noop lock handler */
+
+static int dict_mdb_lock(DICT *dict, int unused_op)
+{
+ /* MDB does its own concurrency control */
+ return 0;
+}
+
+/* dict_mdb_close - disassociate from data base */
+
+static void dict_mdb_close(DICT *dict)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+
+ if (dict_mdb->txn) {
+ int status = mdb_txn_commit(dict_mdb->txn);
+ if (status)
+ msg_fatal("%s: closing dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+ dict_mdb->cursor = NULL;
+ }
+ if (dict_mdb->cursor) {
+ MDB_txn *txn = mdb_cursor_txn(dict_mdb->cursor);
+ mdb_cursor_close(dict_mdb->cursor);
+ mdb_txn_abort(txn);
+ }
+ if (dict_mdb->dict.stat_fd >= 0)
+ close(dict_mdb->dict.stat_fd);
+ mdb_env_close(dict_mdb->env);
+ if (dict_mdb->key_buf)
+ vstring_free(dict_mdb->key_buf);
+ if (dict_mdb->val_buf)
+ vstring_free(dict_mdb->val_buf);
+ if (dict->fold_buf)
+ vstring_free(dict->fold_buf);
+ dict_free(dict);
+}
+
+/* dict_mdb_open - open MDB data base */
+
+DICT *dict_mdb_open(const char *path, int open_flags, int dict_flags)
+{
+ DICT_MDB *dict_mdb;
+ struct stat st;
+ MDB_env *env;
+ MDB_txn *txn;
+ MDB_dbi dbi;
+ char *mdb_path;
+ int env_flags, status;
+
+ mdb_path = concatenate(path, ".mdb", (char *) 0);
+
+ env_flags = MDB_NOSUBDIR;
+ if (open_flags == O_RDONLY)
+ env_flags |= MDB_RDONLY;
+
+ if ((status = mdb_env_create(&env)))
+ msg_fatal("env_create %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_env_set_mapsize(env, dict_mdb_map_size)))
+ msg_fatal("env_set_mapsize %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_env_set_maxreaders(env, dict_mdb_max_readers)))
+ msg_fatal("env_set_maxreaders %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_env_open(env, mdb_path, env_flags, 0644)))
+ msg_fatal("env_open %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_txn_begin(env, NULL, env_flags & MDB_RDONLY, &txn)))
+ msg_fatal("txn_begin %s: %s", mdb_path, mdb_strerror(status));
+
+ /* mdb_open requires a txn, but since the default DB always exists
+ * in an MDB environment, we don't need to do anything else with
+ * the txn.
+ */
+ if ((status = mdb_open(txn, NULL, 0, &dbi)))
+ msg_fatal("mdb_open %s: %s", mdb_path, mdb_strerror(status));
+
+ /* However, if O_TRUNC was specified, we need to do it now.
+ * Also with O_TRUNC we keep this write txn for as long as the
+ * database is open, since we'll probably be doing a bulk import
+ * immediately after.
+ */
+ if (open_flags & O_TRUNC) {
+ if ((status = mdb_drop(txn, dbi, 0)))
+ msg_fatal("truncate %s: %s", mdb_path, mdb_strerror(status));
+ } else {
+ mdb_txn_abort(txn);
+ txn = NULL;
+ }
+
+ dict_mdb = (DICT_MDB *) dict_alloc(DICT_TYPE_MDB, path, sizeof(*dict_mdb));
+ dict_mdb->dict.lookup = dict_mdb_lookup;
+ dict_mdb->dict.update = dict_mdb_update;
+ dict_mdb->dict.delete = dict_mdb_delete;
+ dict_mdb->dict.sequence = dict_mdb_sequence;
+ dict_mdb->dict.close = dict_mdb_close;
+ dict_mdb->dict.lock = dict_mdb_lock;
+ dict_mdb->dict.stat_fd = open(mdb_path, O_RDONLY);
+ if (fstat(dict_mdb->dict.stat_fd, &st) < 0)
+ msg_fatal("dict_mdb_open: fstat: %m");
+ dict_mdb->dict.mtime = st.st_mtime;
+ dict_mdb->dict.owner.uid = st.st_uid;
+ dict_mdb->dict.owner.status = (st.st_uid != 0);
+
+ /*
+ * Warn if the source file is newer than the indexed file, except when
+ * the source file changed only seconds ago.
+ */
+ if ((dict_flags & DICT_FLAG_LOCK) != 0
+ && stat(path, &st) == 0
+ && st.st_mtime > dict_mdb->dict.mtime
+ && st.st_mtime < time((time_t *) 0) - 100)
+ msg_warn("database %s is older than source file %s", mdb_path, path);
+
+ close_on_exec(dict_mdb->dict.stat_fd, CLOSE_ON_EXEC);
+ dict_mdb->dict.flags = dict_flags | DICT_FLAG_FIXED;
+ if ((dict_flags & (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL)) == 0)
+ dict_mdb->dict.flags |= (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL);
+ if (dict_flags & DICT_FLAG_FOLD_FIX)
+ dict_mdb->dict.fold_buf = vstring_alloc(10);
+ dict_mdb->env = env;
+ dict_mdb->dbi = dbi;
+
+ /* Save the write txn if we opened with O_TRUNC */
+ dict_mdb->txn = txn;
+
+ dict_mdb->cursor = 0;
+ dict_mdb->key_buf = 0;
+ dict_mdb->val_buf = 0;
+
+ myfree(mdb_path);
+
+ return (DICT_DEBUG (&dict_mdb->dict));
+}
+
+#endif
diff --git a/src/util/dict_mdb.h b/src/util/dict_mdb.h
new file mode 100644
index 0000000..e114c73
--- /dev/null
+++ b/src/util/dict_mdb.h
@@ -0,0 +1,41 @@
+#ifndef _DICT_MDB_H_INCLUDED_
+#define _DICT_MDB_H_INCLUDED_
+
+/*++
+/* NAME
+/* dict_mdb 3h
+/* SUMMARY
+/* dictionary manager interface to OpenLDAP MDB files
+/* SYNOPSIS
+/* #include <dict_mdb.h>
+/* DESCRIPTION
+/* .nf
+
+ /*
+ * Utility library.
+ */
+#include <dict.h>
+
+ /*
+ * External interface.
+ */
+#define DICT_TYPE_MDB "mdb"
+
+extern DICT *dict_mdb_open(const char *, int, int);
+
+ /*
+ * XXX Should be part of the DICT interface.
+ */
+extern size_t dict_mdb_map_size;
+extern unsigned int dict_mdb_max_readers;
+
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+#endif
diff --git a/src/util/dict_open.c b/src/util/dict_open.c
index 0f17785..53404cb 100644
--- a/src/util/dict_open.c
+++ b/src/util/dict_open.c
@@ -229,6 +229,7 @@
#include <dict_sdbm.h>
#include <dict_dbm.h>
#include <dict_db.h>
+#include <dict_mdb.h>
#include <dict_nis.h>
#include <dict_nisplus.h>
#include <dict_ni.h>
@@ -271,6 +272,9 @@ static const DICT_OPEN_INFO dict_open_info[] = {
DICT_TYPE_HASH, dict_hash_open,
DICT_TYPE_BTREE, dict_btree_open,
#endif
+#ifdef HAS_MDB
+ DICT_TYPE_MDB, dict_mdb_open,
+#endif
#ifdef HAS_NIS
DICT_TYPE_NIS, dict_nis_open,
#endif
--
1.7.9.5
