The attached patches add support for OpenLDAP's MDB (Memory-Mapped Database)
library to the postfix-2.10-20120908 source.
I don't consider this a final patch; pretty sure it works but also would like
feedback on how some things were done. Also, I didn't see any instructions
anywhere on how to write the corresponding _README file, it kind of looks like
nroff output. manpage updates also need to be written.
The first patch adds a lock handler to the dict interface. I needed this
first, because MDB is fully transactional and does its own locking/concurrency
management. In particular, it does MVCC so readers always run lockless; since
they're fully isolated there's no actual need for "shared locks."
I think this was a necessary feature for the dict interface in general;
arbitrary apps shouldn't need to know anything about how a particular table
implementation handles locking. In particular, dict implementations ought to
be free to use semaphores, process-shared mutexes, or anything else that comes
in handy.
I'm happy to write up the corresponding doc updates, when someone tells me
where to begin. In the meantime, you can read up on our MDB library here:
http://highlandsun.com/hyc/mdb/
--
-- Howard Chu
CTO, Symas Corp. http://www.symas.com
Director, Highland Sun http://highlandsun.com/hyc/
Chief Architect, OpenLDAP http://www.openldap.org/project/
>From 7ad5443c4f7586618e0f222c74812b80e1194a72 Mon Sep 17 00:00:00 2001
From: Howard Chu <h...@symas.com>
Date: Thu, 20 Sep 2012 07:30:12 -0700
Subject: [PATCH 1/2] Add locking to dict interface
---
src/tls/tls_scache.c | 4 +---
src/util/dict.c | 22 ++++++++++++++++++++++
src/util/dict.h | 4 ++++
src/util/dict_alloc.c | 9 +++++++++
src/util/dict_db.c | 1 +
src/util/dict_dbm.c | 1 +
src/util/dict_debug.c | 16 ++++++++++++++++
src/util/dict_open.c | 2 +-
src/util/dict_sdbm.c | 1 +
9 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/src/tls/tls_scache.c b/src/tls/tls_scache.c
index 7eefd90..52f6ebb 100644
--- a/src/tls/tls_scache.c
+++ b/src/tls/tls_scache.c
@@ -458,10 +458,8 @@ TLS_SCACHE *tls_scache_open(const char *dbname, const char *cache_label,
/*
* Sanity checks.
*/
- if (dict->lock_fd < 0)
- msg_fatal("dictionary %s is not a regular file", dbname);
#ifdef SINGLE_UPDATER
- if (myflock(dict->lock_fd, INTERNAL_LOCK,
+ if (dict->lock(dict, INTERNAL_LOCK,
MYFLOCK_OP_EXCLUSIVE | MYFLOCK_OP_NOWAIT) < 0)
msg_fatal("cannot lock dictionary %s for exclusive use: %m", dbname);
#endif
diff --git a/src/util/dict.c b/src/util/dict.c
index 173544c..f1e53ed 100644
--- a/src/util/dict.c
+++ b/src/util/dict.c
@@ -147,6 +147,8 @@
/* Each entry is stored in the dictionary named by \fIdict_name\fR.
/* The result is zero if the file could not be opened.
/*
+/* dict_lock_file() acquires a lock on the named dictionary.
+/*
/* dict_load_fp() reads name-value entries from an open stream.
/* It has the same semantics as the dict_load_file_xt() function.
/*
@@ -362,6 +364,26 @@ int dict_delete(const char *dict_name, const char *member)
return (dict ? dict->delete(dict, member) : DICT_STAT_FAIL);
}
+/* dict_lock_file - lock dictionary */
+
+int dict_lock_file(const char *dict_name, int style, int op)
+{
+ const char *myname = "dict_lock_file";
+ DICT *dict;
+
+ DICT_FIND_FOR_LOOKUP(dict, dict_name);
+ if (msg_verbose > 1)
+ msg_info("%s: lock_file %s", myname, dict_name);
+ return (dict ? dict->lock(dict, style, op) : DICT_STAT_FAIL);
+}
+
+/* dict_lock_default - default lock handler */
+
+int dict_lock_default(DICT *dict, int style, int op)
+{
+ return myflock(dict->lock_fd, style, op);
+}
+
/* dict_sequence - traverse dictionary */
int dict_sequence(const char *dict_name, const int func,
diff --git a/src/util/dict.h b/src/util/dict.h
index f8e91a4..d714a21 100644
--- a/src/util/dict.h
+++ b/src/util/dict.h
@@ -48,6 +48,7 @@ typedef struct DICT {
int (*delete) (struct DICT *, const char *);
int (*sequence) (struct DICT *, int, const char **, const char **);
void (*close) (struct DICT *);
+ int (*lock) (struct DICT *, int, int);
int lock_fd; /* for dict_update() lock */
int stat_fd; /* change detection */
time_t mtime; /* mod time at open */
@@ -155,6 +156,8 @@ extern int dict_update(const char *, const char *, const char *);
extern const char *dict_lookup(const char *, const char *);
extern int dict_delete(const char *, const char *);
extern int dict_sequence(const char *, const int, const char **, const char **);
+extern int dict_lock_file(const char *, const int, const int);
+extern int dict_lock_default(DICT *, const int, const int);
extern int dict_load_file_xt(const char *, const char *);
extern void dict_load_fp(const char *, VSTREAM *);
extern const char *dict_eval(const char *, const char *, int);
@@ -171,6 +174,7 @@ extern void dict_open_register(const char *, DICT *(*) (const char *, int, int))
#define dict_put(dp, key, val) (dp)->update((dp), (key), (val))
#define dict_del(dp, key) (dp)->delete((dp), (key))
#define dict_seq(dp, f, key, val) (dp)->sequence((dp), (f), (key), (val))
+#define dict_lock(dp, style, op) (dp)->lock((dp), (style), (op))
#define dict_close(dp) (dp)->close(dp)
typedef void (*DICT_WALK_ACTION) (const char *, DICT *, char *);
extern void dict_walk(DICT_WALK_ACTION, char *);
diff --git a/src/util/dict_alloc.c b/src/util/dict_alloc.c
index 03321f1..30fe450 100644
--- a/src/util/dict_alloc.c
+++ b/src/util/dict_alloc.c
@@ -103,6 +103,14 @@ static void dict_default_close(DICT *dict)
dict->type, dict->name);
}
+/* dict_default_lock - trap unimplemented operation */
+
+static void dict_default_lock(DICT *dict, int unused_style, int unused_op)
+{
+ msg_fatal("table %s:%s: lock operation is not supported",
+ dict->type, dict->name);
+}
+
/* dict_alloc - allocate dictionary object, initialize super-class */
DICT *dict_alloc(const char *dict_type, const char *dict_name, ssize_t size)
@@ -117,6 +125,7 @@ DICT *dict_alloc(const char *dict_type, const char *dict_name, ssize_t size)
dict->delete = dict_default_delete;
dict->sequence = dict_default_sequence;
dict->close = dict_default_close;
+ dict->lock = dict_default_lock;
dict->lock_fd = -1;
dict->stat_fd = -1;
dict->mtime = 0;
diff --git a/src/util/dict_db.c b/src/util/dict_db.c
index 93ee480..b0a1805 100644
--- a/src/util/dict_db.c
+++ b/src/util/dict_db.c
@@ -719,6 +719,7 @@ static DICT *dict_db_open(const char *class, const char *path, int open_flags,
dict_db->dict.delete = dict_db_delete;
dict_db->dict.sequence = dict_db_sequence;
dict_db->dict.close = dict_db_close;
+ dict_db->dict.lock = dict_lock_default;
dict_db->dict.lock_fd = dbfd;
dict_db->dict.stat_fd = dbfd;
if (fstat(dict_db->dict.stat_fd, &st) < 0)
diff --git a/src/util/dict_dbm.c b/src/util/dict_dbm.c
index 37e1463..ea24dfd 100644
--- a/src/util/dict_dbm.c
+++ b/src/util/dict_dbm.c
@@ -452,6 +452,7 @@ DICT *dict_dbm_open(const char *path, int open_flags, int dict_flags)
dict_dbm->dict.delete = dict_dbm_delete;
dict_dbm->dict.sequence = dict_dbm_sequence;
dict_dbm->dict.close = dict_dbm_close;
+ dict_dbm->dict.lock = dict_lock_default;
dict_dbm->dict.lock_fd = dbm_dirfno(dbm);
dict_dbm->dict.stat_fd = dbm_pagfno(dbm);
if (dict_dbm->dict.lock_fd == dict_dbm->dict.stat_fd)
diff --git a/src/util/dict_debug.c b/src/util/dict_debug.c
index 3d9a443..cff1129 100644
--- a/src/util/dict_debug.c
+++ b/src/util/dict_debug.c
@@ -123,6 +123,21 @@ static void dict_debug_close(DICT *dict)
dict_free(dict);
}
+/* dict_debug_lock - log operation */
+
+static int dict_debug_lock(DICT *dict, int style, int op)
+{
+ DICT_DEBUG *dict_debug = (DICT_DEBUG *) dict;
+ DICT *real_dict = dict_debug->real_dict;
+ int result;
+
+ result = dict_lock(real_dict, style, op);
+ msg_info("%s:%s lock: %s", dict->type, dict->name,
+ result == 0 ? "success" : real_dict->error ?
+ "error" : "failed");
+ DICT_ERR_VAL_RETURN(dict, real_dict->error, result);
+}
+
/* dict_debug - encapsulate dictionary object and install proxies */
DICT *dict_debug(DICT *real_dict)
@@ -137,6 +152,7 @@ DICT *dict_debug(DICT *real_dict)
dict_debug->dict.delete = dict_debug_delete;
dict_debug->dict.sequence = dict_debug_sequence;
dict_debug->dict.close = dict_debug_close;
+ dict_debug->dict.lock = dict_debug_lock;
dict_debug->real_dict = real_dict;
return (&dict_debug->dict);
}
diff --git a/src/util/dict_open.c b/src/util/dict_open.c
index cd90e08..cba0f45 100644
--- a/src/util/dict_open.c
+++ b/src/util/dict_open.c
@@ -353,7 +353,7 @@ DICT *dict_open3(const char *dict_type, const char *dict_name,
if (dict_flags & DICT_FLAG_LOCK)
msg_panic("%s: attempt to open %s:%s with both \"open\" lock and \"access\" lock",
myname, dict_type, dict_name);
- if (myflock(dict->lock_fd, INTERNAL_LOCK,
+ if (dict->lock(dict, INTERNAL_LOCK,
MYFLOCK_OP_EXCLUSIVE | MYFLOCK_OP_NOWAIT) < 0)
msg_fatal("%s:%s: unable to get exclusive lock: %m",
dict_type, dict_name);
diff --git a/src/util/dict_sdbm.c b/src/util/dict_sdbm.c
index 23371dc..612ba11 100644
--- a/src/util/dict_sdbm.c
+++ b/src/util/dict_sdbm.c
@@ -445,6 +445,7 @@ DICT *dict_sdbm_open(const char *path, int open_flags, int dict_flags)
dict_sdbm->dict.delete = dict_sdbm_delete;
dict_sdbm->dict.sequence = dict_sdbm_sequence;
dict_sdbm->dict.close = dict_sdbm_close;
+ dict_sdbm->dict.lock = dict_lock_default;
dict_sdbm->dict.lock_fd = sdbm_dirfno(dbm);
dict_sdbm->dict.stat_fd = sdbm_pagfno(dbm);
if (fstat(dict_sdbm->dict.stat_fd, &st) < 0)
--
1.7.9.5
>From f0314fea36ae850020f602912319099953a960c1 Mon Sep 17 00:00:00 2001
From: Howard Chu <h...@symas.com>
Date: Thu, 20 Sep 2012 09:27:14 -0700
Subject: [PATCH] Add support for OpenLDAP MDB
---
makedefs | 7 +
src/global/Makefile.in | 4 +-
src/global/data_redirect.c | 2 +
src/global/mail_params.c | 9 +
src/global/mail_params.h | 7 +
src/global/mkmap.h | 1 +
src/global/mkmap_mdb.c | 141 +++++++++++++
src/global/mkmap_open.c | 4 +
src/util/Makefile.in | 9 +-
src/util/dict_mdb.c | 484 ++++++++++++++++++++++++++++++++++++++++++++
src/util/dict_mdb.h | 40 ++++
src/util/dict_open.c | 4 +
12 files changed, 707 insertions(+), 5 deletions(-)
create mode 100644 src/global/mkmap_mdb.c
create mode 100644 src/util/dict_mdb.c
create mode 100644 src/util/dict_mdb.h
diff --git a/makedefs b/makedefs
index d9149b4..e6beb9f 100644
--- a/makedefs
+++ b/makedefs
@@ -620,6 +620,13 @@ esac
# ;;
#esac
+case "$CCARGS" in
+*-DNO_MDB*) ;;
+*) CCARGS="$CCARGS -DHAS_MDB"
+ AUXLIBS="$AUXLIBS -lmdb"
+ ;;
+esac
+
#
# PCRE 3.x has a pcre-config utility so we don't have to guess.
#
diff --git a/src/global/Makefile.in b/src/global/Makefile.in
index 8e11da5..2a7f062 100644
--- a/src/global/Makefile.in
+++ b/src/global/Makefile.in
@@ -16,7 +16,7 @@ SRCS = abounce.c anvil_clnt.c been_here.c bounce.c bounce_log.c \
mail_params.c mail_pathname.c mail_queue.c mail_run.c \
mail_scan_dir.c mail_stream.c mail_task.c mail_trigger.c maps.c \
mark_corrupt.c match_parent_style.c mbox_conf.c mbox_open.c \
- mime_state.c mkmap_cdb.c mkmap_db.c mkmap_dbm.c mkmap_open.c \
+ mime_state.c mkmap_cdb.c mkmap_db.c mkmap_dbm.c mkmap_mdb.c mkmap_open.c \
mkmap_sdbm.c msg_stats_print.c msg_stats_scan.c mynetworks.c \
mypwd.c namadr_list.c off_cvt.c opened.c own_inet_addr.c \
pipe_command.c post_mail.c quote_821_local.c quote_822_local.c \
@@ -50,7 +50,7 @@ OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \
mail_params.o mail_pathname.o mail_queue.o mail_run.o \
mail_scan_dir.o mail_stream.o mail_task.o mail_trigger.o maps.o \
mark_corrupt.o match_parent_style.o mbox_conf.o mbox_open.o \
- mime_state.o mkmap_cdb.o mkmap_db.o mkmap_dbm.o mkmap_open.o \
+ mime_state.o mkmap_cdb.o mkmap_db.o mkmap_dbm.o mkmap_mdb.o mkmap_open.o \
mkmap_sdbm.o msg_stats_print.o msg_stats_scan.o mynetworks.o \
mypwd.o namadr_list.o off_cvt.o opened.o own_inet_addr.o \
pipe_command.o post_mail.o quote_821_local.o quote_822_local.o \
diff --git a/src/global/data_redirect.c b/src/global/data_redirect.c
index 096e58c..f914155 100644
--- a/src/global/data_redirect.c
+++ b/src/global/data_redirect.c
@@ -72,6 +72,7 @@
#include <dict_db.h>
#include <dict_dbm.h>
#include <dict_cdb.h>
+#include <dict_mdb.h>
#include <warn_stat.h>
/* Global directory. */
@@ -99,6 +100,7 @@ static const NAME_CODE data_redirect_map_types[] = {
DICT_TYPE_HASH, 1,
DICT_TYPE_BTREE, 1,
DICT_TYPE_DBM, 1,
+ DICT_TYPE_MDB, 1,
DICT_TYPE_CDB, 1, /* not a read-write map type */
"sdbm", 1, /* legacy 3rd-party TLS */
"dbz", 1, /* just in case */
diff --git a/src/global/mail_params.c b/src/global/mail_params.c
index 0e098b3..a14afe6 100644
--- a/src/global/mail_params.c
+++ b/src/global/mail_params.c
@@ -96,6 +96,7 @@
/* char *var_proxywrite_service;
/* int var_db_create_buf;
/* int var_db_read_buf;
+/* int var_mdb_map_size;
/* int var_mime_maxdepth;
/* int var_mime_bound_len;
/* int var_header_limit;
@@ -177,6 +178,9 @@
#ifdef HAS_DB
#include <dict_db.h>
#endif
+#ifdef HAS_MDB
+#include <dict_mdb.h>
+#endif
#include <inet_proto.h>
#include <vstring_vstream.h>
#include <iostuff.h>
@@ -284,6 +288,7 @@ char *var_proxymap_service;
char *var_proxywrite_service;
int var_db_create_buf;
int var_db_read_buf;
+int var_mdb_map_size;
int var_mime_maxdepth;
int var_mime_bound_len;
int var_header_limit;
@@ -596,6 +601,7 @@ void mail_params_init()
VAR_FAULT_INJ_CODE, DEF_FAULT_INJ_CODE, &var_fault_inj_code, 0, 0,
VAR_DB_CREATE_BUF, DEF_DB_CREATE_BUF, &var_db_create_buf, 1, 0,
VAR_DB_READ_BUF, DEF_DB_READ_BUF, &var_db_read_buf, 1, 0,
+ VAR_MDB_MAP_SIZE, DEF_MDB_MAP_SIZE, &var_mdb_map_size, 1, 0,
VAR_HEADER_LIMIT, DEF_HEADER_LIMIT, &var_header_limit, 1, 0,
VAR_TOKEN_LIMIT, DEF_TOKEN_LIMIT, &var_token_limit, 1, 0,
VAR_MIME_MAXDEPTH, DEF_MIME_MAXDEPTH, &var_mime_maxdepth, 1, 0,
@@ -713,6 +719,9 @@ void mail_params_init()
#ifdef HAS_DB
dict_db_cache_size = var_db_read_buf;
#endif
+#ifdef HAS_MDB
+ dict_mdb_map_size = var_mdb_map_size;
+#endif
inet_windowsize = var_inet_windowsize;
/*
diff --git a/src/global/mail_params.h b/src/global/mail_params.h
index 8913eed..f26b4aa 100644
--- a/src/global/mail_params.h
+++ b/src/global/mail_params.h
@@ -2722,6 +2722,13 @@ extern int var_db_create_buf;
extern int var_db_read_buf;
/*
+ * OpenLDAP MDB memory map size.
+ */
+#define VAR_MDB_MAP_SIZE "mdb_map_size"
+#define DEF_MDB_MAP_SIZE (16 * 1024 *1024)
+extern int var_mdb_map_size;
+
+ /*
* Named queue file attributes.
*/
#define VAR_QATTR_COUNT_LIMIT "queue_file_attribute_count_limit"
diff --git a/src/global/mkmap.h b/src/global/mkmap.h
index d74a912..e9f3a57 100644
--- a/src/global/mkmap.h
+++ b/src/global/mkmap.h
@@ -39,6 +39,7 @@ extern MKMAP *mkmap_dbm_open(const char *);
extern MKMAP *mkmap_cdb_open(const char *);
extern MKMAP *mkmap_hash_open(const char *);
extern MKMAP *mkmap_btree_open(const char *);
+extern MKMAP *mkmap_mdb_open(const char *);
extern MKMAP *mkmap_sdbm_open(const char *);
extern MKMAP *mkmap_proxy_open(const char *);
extern MKMAP *mkmap_fail_open(const char *);
diff --git a/src/global/mkmap_mdb.c b/src/global/mkmap_mdb.c
new file mode 100644
index 0000000..e79b8c4
--- /dev/null
+++ b/src/global/mkmap_mdb.c
@@ -0,0 +1,141 @@
+/*++
+/* NAME
+/* mkmap_mdb 3
+/* SUMMARY
+/* create or open database, MDB style
+/* SYNOPSIS
+/* #include <mkmap.h>
+/*
+/* MKMAP *mkmap_mdb_open(path)
+/* const char *path;
+/*
+/* DESCRIPTION
+/* This module implements support for creating MDB databases.
+/*
+/* mkmap_mdb_open() takes a file name, appends the ".mdb"
+/* suffix, and does whatever initialization is required
+/* before the OpenLDAP MDB open routine is called.
+/*
+/* All errors are fatal.
+/* SEE ALSO
+/* dict_mdb(3), MDB dictionary interface.
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+/* System library. */
+
+#include <sys_defs.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Utility library. */
+
+#include <msg.h>
+#include <mymalloc.h>
+#include <stringops.h>
+#include <dict.h>
+#include <dict_mdb.h>
+#include <myflock.h>
+#include <warn_stat.h>
+
+/* Global library. */
+
+#include <mail_params.h>
+
+/* Application-specific. */
+
+#include "mkmap.h"
+
+#ifdef HAS_MDB
+#ifdef PATH_MDB_H
+#include PATH_MDB_H
+#else
+#include <mdb.h>
+#endif
+
+typedef struct MKMAP_MDB {
+ MKMAP mkmap; /* parent class */
+ char *lock_file; /* path name */
+ int lock_fd; /* -1 or open locked file */
+} MKMAP_MDB;
+
+/* mkmap_mdb_after_close - clean up after closing database */
+
+static void mkmap_mdb_after_close(MKMAP *mp)
+{
+ MKMAP_MDB *mkmap = (MKMAP_MDB *) mp;
+
+ if (mkmap->lock_fd >= 0 && close(mkmap->lock_fd) < 0)
+ msg_warn("close %s: %m", mkmap->lock_file);
+ myfree(mkmap->lock_file);
+}
+
+/* mkmap_mdb_after_open - lock newly created database */
+
+static void mkmap_mdb_after_open(MKMAP *mp)
+{
+ MKMAP_MDB *mkmap = (MKMAP_MDB *) mp;
+
+ if (mkmap->lock_fd < 0) {
+ if ((mkmap->lock_fd = open(mkmap->lock_file, O_RDWR, 0644)) < 0)
+ msg_fatal("open lockfile %s: %m", mkmap->lock_file);
+ if (myflock(mkmap->lock_fd, INTERNAL_LOCK, MYFLOCK_OP_EXCLUSIVE) < 0)
+ msg_fatal("lock %s: %m", mkmap->lock_file);
+ }
+}
+
+/* mkmap_mdb_open - lock existing database */
+
+MKMAP *mkmap_mdb_open(const char *path)
+{
+ MKMAP_MDB *mkmap = (MKMAP_MDB *) mymalloc(sizeof(*mkmap));
+ struct stat st;
+
+ /*
+ * Override the default per-table map size for map (re)builds.
+ *
+ * mdb_map_size is defined in util/dict_mdb.c and defaults to 10MB.
+ * It needs to be large enough to contain the largest tables in use.
+ *
+ * XXX This should be specified via the DICT interface so that the buffer
+ * size becomes an object property, instead of being specified by poking
+ * a global variable so that it becomes a class property.
+ */
+ dict_mdb_map_size = var_mdb_map_size;
+
+ /*
+ * Fill in the generic members.
+ */
+ mkmap->lock_file = concatenate(path, ".mdb", (char *) 0);
+ mkmap->mkmap.open = dict_mdb_open;
+ mkmap->mkmap.after_open = mkmap_mdb_after_open;
+ mkmap->mkmap.after_close = mkmap_mdb_after_close;
+
+ /*
+ * Unfortunately, not all systems that might support mdb databases
+ * support locking on open(), so we open the file before updating it.
+ */
+ if ((mkmap->lock_fd = open(mkmap->lock_file, O_RDWR, 0644)) < 0) {
+ if (errno != ENOENT)
+ msg_fatal("open %s: %m", mkmap->lock_file);
+ }
+
+ /*
+ * Get an exclusive lock - we're going to change the database so we can't
+ * have any spectators.
+ */
+ else {
+ if (myflock(mkmap->lock_fd, INTERNAL_LOCK, MYFLOCK_OP_EXCLUSIVE) < 0)
+ msg_fatal("lock %s: %m", mkmap->lock_file);
+ }
+
+ return (&mkmap->mkmap);
+}
+#endif
diff --git a/src/global/mkmap_open.c b/src/global/mkmap_open.c
index d939d44..918c855 100644
--- a/src/global/mkmap_open.c
+++ b/src/global/mkmap_open.c
@@ -65,6 +65,7 @@
#include <dict_db.h>
#include <dict_cdb.h>
#include <dict_dbm.h>
+#include <dict_mdb.h>
#include <dict_sdbm.h>
#include <dict_proxy.h>
#include <dict_fail.h>
@@ -101,6 +102,9 @@ static const MKMAP_OPEN_INFO mkmap_types[] = {
DICT_TYPE_HASH, mkmap_hash_open,
DICT_TYPE_BTREE, mkmap_btree_open,
#endif
+#ifdef HAS_MDB
+ DICT_TYPE_MDB, mkmap_mdb_open,
+#endif
DICT_TYPE_FAIL, mkmap_fail_open,
0,
};
diff --git a/src/util/Makefile.in b/src/util/Makefile.in
index 48990a6..7c73679 100644
--- a/src/util/Makefile.in
+++ b/src/util/Makefile.in
@@ -4,7 +4,7 @@ SRCS = alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \
attr_scan_plain.c auto_clnt.c base64_code.c basename.c binhash.c \
chroot_uid.c cidr_match.c clean_env.c close_on_exec.c concatenate.c \
ctable.c dict.c dict_alloc.c dict_cdb.c dict_cidr.c dict_db.c \
- dict_dbm.c dict_debug.c dict_env.c dict_ht.c dict_ni.c dict_nis.c \
+ dict_dbm.c dict_debug.c dict_env.c dict_ht.c dict_mdb.c dict_ni.c dict_nis.c \
dict_nisplus.c dict_open.c dict_pcre.c dict_regexp.c dict_sdbm.c \
dict_static.c dict_tcp.c dict_unix.c dir_forest.c doze.c dummy_read.c \
dummy_write.c duplex_pipe.c environ.c events.c exec_command.c \
@@ -41,7 +41,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \
chroot_uid.o cidr_match.o clean_env.o close_on_exec.o concatenate.o \
ctable.o dict.o dict_alloc.o dict_cdb.o dict_cidr.o dict_db.o \
- dict_dbm.o dict_debug.o dict_env.o dict_ht.o dict_ni.o dict_nis.o \
+ dict_dbm.o dict_debug.o dict_env.o dict_ht.o dict_mdb.o dict_ni.o dict_nis.o \
dict_nisplus.o dict_open.o dict_pcre.o dict_regexp.o dict_sdbm.o \
dict_static.o dict_tcp.o dict_unix.o dir_forest.o doze.o dummy_read.o \
dummy_write.o duplex_pipe.o environ.o events.o exec_command.o \
@@ -76,7 +76,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \
chroot_uid.h cidr_match.h clean_env.h connect.h ctable.h dict.h \
dict_cdb.h dict_cidr.h dict_db.h dict_dbm.h dict_env.h dict_ht.h \
- dict_ni.h dict_nis.h dict_nisplus.h dict_pcre.h dict_regexp.h \
+ dict_mdb.h dict_ni.h dict_nis.h dict_nisplus.h dict_pcre.h dict_regexp.h \
dict_sdbm.h dict_static.h dict_tcp.h dict_unix.h dir_forest.h \
events.h exec_command.h find_inet.h fsspace.h fullname.h \
get_domainname.h get_hostname.h hex_code.h hex_quote.h host_port.h \
@@ -975,6 +975,9 @@ dict_ht.o: sys_defs.h
dict_ht.o: vbuf.h
dict_ht.o: vstream.h
dict_ht.o: vstring.h
+dict_mdb.o: dict.h
+dict_mdb.o: dict_mdb.h
+dict_mdb.o: dict_mdb.c
dict_ni.o: dict_ni.c
dict_ni.o: sys_defs.h
dict_nis.o: argv.h
diff --git a/src/util/dict_mdb.c b/src/util/dict_mdb.c
new file mode 100644
index 0000000..2197469
--- /dev/null
+++ b/src/util/dict_mdb.c
@@ -0,0 +1,484 @@
+/*++
+/* NAME
+/* dict_mdb 3
+/* SUMMARY
+/* dictionary manager interface to OpenLDAP MDB files
+/* SYNOPSIS
+/* #include <dict_mdb.h>
+/*
+/* DICT *dict_mdb_open(path, open_flags, dict_flags)
+/* const char *name;
+/* const char *path;
+/* int open_flags;
+/* int dict_flags;
+/* DESCRIPTION
+/* dict_mdb_open() opens the named MDB database and makes it available
+/* via the generic interface described in dict_open(3).
+/*
+/* The dict_mdb_map_size variable specifies a non-default per-table
+/* memory map size. The map size is 10MB. The map size is also the
+/* maximum size the table can grow to, so it must be set large enough
+/* to accomodate the largest tables in use.
+/* DIAGNOSTICS
+/* Fatal errors: cannot open file, file write error, out of memory.
+/* SEE ALSO
+/* dict(3) generic dictionary manager
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+#include "sys_defs.h"
+
+#ifdef HAS_MDB
+
+/* System library. */
+
+#include <sys/stat.h>
+#ifdef PATH_MDB_H
+#include PATH_MDB_H
+#else
+#include <mdb.h>
+#endif
+#include <string.h>
+#include <unistd.h>
+
+/* Utility library. */
+
+#include "msg.h"
+#include "mymalloc.h"
+#include "htable.h"
+#include "iostuff.h"
+#include "vstring.h"
+#include "myflock.h"
+#include "stringops.h"
+#include "dict.h"
+#include "dict_mdb.h"
+#include "warn_stat.h"
+
+/* Application-specific. */
+
+typedef struct {
+ DICT dict; /* generic members */
+ MDB_env *env; /* MDB environment */
+ MDB_dbi dbi; /* database handle */
+ MDB_cursor *cursor; /* open database */
+ VSTRING *key_buf; /* key buffer */
+ VSTRING *val_buf; /* result buffer */
+} DICT_MDB;
+
+#define SCOPY(buf, data, size) \
+ vstring_str(vstring_strncpy(buf ? buf : (buf = vstring_alloc(10)), data, size))
+
+size_t dict_mdb_map_size = (10 * 1024 * 1024); /* 10MB default mmap size */
+
+/* dict_mdb_lookup - find database entry */
+
+static const char *dict_mdb_lookup(DICT *dict, const char *name)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ const char *result = 0;
+ int status, klen;
+
+ dict->error = 0;
+ klen = strlen(name);
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_lookup: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+
+ /*
+ * Start a read transaction.
+ */
+ if ((status = mdb_txn_begin(dict_mdb->env, NULL, MDB_RDONLY, &txn)))
+ msg_fatal("%s: txn_begin(read) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * See if this MDB file was written with one null byte appended to key
+ * and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen + 1;
+ status = mdb_get(txn, dict_mdb->dbi, &mdb_key, &mdb_value);
+ if (!status) {
+ dict->flags &= ~DICT_FLAG_TRY0NULL;
+ result = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ }
+ }
+
+ /*
+ * See if this MDB file was written with no null byte appended to key and
+ * value.
+ */
+ if (result == 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen;
+ status = mdb_get(txn, dict_mdb->dbi, &mdb_key, &mdb_value);
+ if (!status) {
+ dict->flags &= ~DICT_FLAG_TRY1NULL;
+ result = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ }
+ }
+
+ /*
+ * Close the read txn.
+ */
+ mdb_txn_abort(txn);
+
+ return (result);
+}
+
+/* dict_mdb_update - add or update database entry */
+
+static int dict_mdb_update(DICT *dict, const char *name, const char *value)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ int status;
+
+ dict->error = 0;
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_update: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+ mdb_key.mv_data = (void *) name;
+ mdb_value.mv_data = (void *) value;
+ mdb_key.mv_size = strlen(name);
+ mdb_value.mv_size = strlen(value);
+
+ /*
+ * If undecided about appending a null byte to key and value, choose a
+ * default depending on the platform.
+ */
+ if ((dict->flags & DICT_FLAG_TRY1NULL)
+ && (dict->flags & DICT_FLAG_TRY0NULL)) {
+#ifdef MDB_NO_TRAILING_NULL
+ dict->flags &= ~DICT_FLAG_TRY1NULL;
+#else
+ dict->flags &= ~DICT_FLAG_TRY0NULL;
+#endif
+ }
+
+ /*
+ * Optionally append a null byte to key and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_size++;
+ mdb_value.mv_size++;
+ }
+
+ /*
+ * Start a write transaction.
+ */
+ if ((status = mdb_txn_begin(dict_mdb->env, NULL, 0, &txn)))
+ msg_fatal("%s: txn_begin(write) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * Do the update.
+ */
+ status = mdb_put(txn, dict_mdb->dbi, &mdb_key, &mdb_value,
+ (dict->flags & DICT_FLAG_DUP_REPLACE) ? 0 : MDB_NOOVERWRITE);
+ if (status) {
+ if (status == MDB_KEYEXIST) {
+ if (dict->flags & DICT_FLAG_DUP_IGNORE)
+ /* void */ ;
+ else if (dict->flags & DICT_FLAG_DUP_WARN)
+ msg_warn("%s: duplicate entry: \"%s\"", dict_mdb->dict.name, name);
+ else
+ msg_fatal("%s: duplicate entry: \"%s\"", dict_mdb->dict.name, name);
+ } else {
+ msg_fatal("error writing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ }
+ }
+
+ /*
+ * Commit the transaction.
+ */
+ if ((status = mdb_txn_commit(txn)))
+ msg_fatal("error committing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ return (status);
+}
+
+/* dict_mdb_delete - delete one entry from the dictionary */
+
+static int dict_mdb_delete(DICT *dict, const char *name)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_txn *txn;
+ int status = 1, klen, rc;
+
+ dict->error = 0;
+ klen = strlen(name);
+
+ /*
+ * Sanity check.
+ */
+ if ((dict->flags & (DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL)) == 0)
+ msg_panic("dict_mdb_delete: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
+
+ /*
+ * Optionally fold the key.
+ */
+ if (dict->flags & DICT_FLAG_FOLD_FIX) {
+ if (dict->fold_buf == 0)
+ dict->fold_buf = vstring_alloc(10);
+ vstring_strcpy(dict->fold_buf, name);
+ name = lowercase(vstring_str(dict->fold_buf));
+ }
+
+ /*
+ * Start a write transaction.
+ */
+ if ((status = mdb_txn_begin(dict_mdb->env, NULL, 0, &txn)))
+ msg_fatal("%s: txn_begin(write) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ /*
+ * See if this MDB file was written with one null byte appended to key
+ * and value.
+ */
+ if (dict->flags & DICT_FLAG_TRY1NULL) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen + 1;
+ status = mdb_del(txn, dict_mdb->dbi, &mdb_key, NULL);
+ if (status) {
+ if (status == MDB_NOTFOUND)
+ status = 1;
+ else
+ msg_fatal("error deleting from %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ } else {
+ dict->flags &= ~DICT_FLAG_TRY0NULL; /* found */
+ }
+ }
+
+ /*
+ * See if this MDB file was written with no null byte appended to key and
+ * value.
+ */
+ if (status > 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
+ mdb_key.mv_data = (void *) name;
+ mdb_key.mv_size = klen;
+ status = mdb_del(txn, dict_mdb->dbi, &mdb_key, NULL);
+ if (status) {
+ if (status == MDB_NOTFOUND)
+ status = 1;
+ else
+ msg_fatal("error deleting from %s: %s", dict_mdb->dict.name, mdb_strerror(status));
+ } else {
+ dict->flags &= ~DICT_FLAG_TRY1NULL; /* found */
+ }
+ }
+
+ /*
+ * Commit the transaction.
+ */
+ if ((rc = mdb_txn_commit(txn)))
+ msg_fatal("error committing MDB database %s: %s", dict_mdb->dict.name, mdb_strerror(rc));
+
+ return (status);
+}
+
+/* traverse the dictionary */
+
+static int dict_mdb_sequence(DICT *dict, int function,
+ const char **key, const char **value)
+{
+ const char *myname = "dict_mdb_sequence";
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+ MDB_val mdb_key;
+ MDB_val mdb_value;
+ MDB_txn *txn;
+ MDB_cursor_op op;
+ int status;
+
+ dict->error = 0;
+
+ /*
+ * Determine the seek function.
+ */
+ switch (function) {
+ case DICT_SEQ_FUN_FIRST:
+ op = MDB_FIRST;
+ break;
+ case DICT_SEQ_FUN_NEXT:
+ op = MDB_NEXT;
+ break;
+ default:
+ msg_panic("%s: invalid function: %d", myname, function);
+ }
+
+ /*
+ * Open a read transaction and cursor if needed.
+ */
+ if (dict_mdb->cursor == 0) {
+ if ((status = mdb_txn_begin(dict_mdb->env, NULL, MDB_RDONLY, &txn)))
+ msg_fatal("%s: txn_begin(read) dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+ if ((status = mdb_cursor_open(txn, dict_mdb->dbi, &dict_mdb->cursor)))
+ msg_fatal("%s: cursor_open dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+ }
+
+ /*
+ * Database lookup.
+ */
+ status = mdb_cursor_get(dict_mdb->cursor, &mdb_key, &mdb_value, op);
+ if (status && status != MDB_NOTFOUND)
+ msg_fatal("%s: seeking dictionary: %s", dict_mdb->dict.name, mdb_strerror(status));
+
+ if (status == MDB_NOTFOUND) {
+ status = 1;
+ } else {
+
+ /*
+ * Copy the key so that it is guaranteed null terminated.
+ */
+ *key = SCOPY(dict_mdb->key_buf, mdb_key.mv_data, mdb_key.mv_size);
+
+ if (mdb_value.mv_data != 0 && mdb_value.mv_size > 0) {
+
+ /*
+ * Copy the value so that it is guaranteed null terminated.
+ */
+ *value = SCOPY(dict_mdb->val_buf, mdb_value.mv_data, mdb_value.mv_size);
+ status = 0;
+ }
+ }
+
+ /*
+ * FIXME: someone needs to close the cursor and read txn.
+ */
+
+ return (status);
+}
+
+/* dict_mdb_lock - noop lock handler */
+
+static int dict_mdb_lock(DICT *dict, int unused_style, int unused_op)
+{
+ /* MDB does its own concurrency control */
+ return 0;
+}
+
+/* dict_mdb_close - disassociate from data base */
+
+static void dict_mdb_close(DICT *dict)
+{
+ DICT_MDB *dict_mdb = (DICT_MDB *) dict;
+
+ if (dict_mdb->cursor) {
+ MDB_txn *txn = mdb_cursor_txn(dict_mdb->cursor);
+ mdb_cursor_close(dict_mdb->cursor);
+ mdb_txn_abort(txn);
+ }
+ if (dict_mdb->dict.stat_fd >= 0)
+ close(dict_mdb->dict.stat_fd);
+ mdb_env_close(dict_mdb->env);
+ if (dict_mdb->key_buf)
+ vstring_free(dict_mdb->key_buf);
+ if (dict_mdb->val_buf)
+ vstring_free(dict_mdb->val_buf);
+ if (dict->fold_buf)
+ vstring_free(dict->fold_buf);
+ dict_free(dict);
+}
+
+/* dict_mdb_open - open MDB data base */
+
+DICT *dict_mdb_open(const char *path, int open_flags, int dict_flags)
+{
+ DICT_MDB *dict_mdb;
+ struct stat st;
+ MDB_env *env;
+ char *mdb_path;
+ int env_flags, status;
+
+ mdb_path = concatenate(path, ".mdb", (char *) 0);
+
+ env_flags = MDB_NOSUBDIR;
+ if (open_flags == O_RDONLY)
+ env_flags |= MDB_RDONLY;
+
+ if ((status = mdb_env_create(&env)))
+ msg_fatal("env_create %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_env_set_mapsize(env, dict_mdb_map_size)))
+ msg_fatal("env_set_mapsize %s: %s", mdb_path, mdb_strerror(status));
+
+ if ((status = mdb_env_open(env, mdb_path, env_flags, 0644)))
+ msg_fatal("env_open %s: %s", mdb_path, mdb_strerror(status));
+
+ dict_mdb = (DICT_MDB *) dict_alloc(DICT_TYPE_MDB, path, sizeof(*dict_mdb));
+ dict_mdb->dict.lookup = dict_mdb_lookup;
+ dict_mdb->dict.update = dict_mdb_update;
+ dict_mdb->dict.delete = dict_mdb_delete;
+ dict_mdb->dict.sequence = dict_mdb_sequence;
+ dict_mdb->dict.close = dict_mdb_close;
+ dict_mdb->dict.lock = dict_mdb_lock;
+ dict_mdb->dict.lock_fd = 0;
+ dict_mdb->dict.stat_fd = open(mdb_path, O_RDONLY);
+ if (fstat(dict_mdb->dict.stat_fd, &st) < 0)
+ msg_fatal("dict_mdb_open: fstat: %m");
+ dict_mdb->dict.mtime = st.st_mtime;
+ dict_mdb->dict.owner.uid = st.st_uid;
+ dict_mdb->dict.owner.status = (st.st_uid != 0);
+
+ /*
+ * Warn if the source file is newer than the indexed file, except when
+ * the source file changed only seconds ago.
+ */
+ if ((dict_flags & DICT_FLAG_LOCK) != 0
+ && stat(path, &st) == 0
+ && st.st_mtime > dict_mdb->dict.mtime
+ && st.st_mtime < time((time_t *) 0) - 100)
+ msg_warn("database %s is older than source file %s", mdb_path, path);
+
+ close_on_exec(dict_mdb->dict.stat_fd, CLOSE_ON_EXEC);
+ dict_mdb->dict.flags = dict_flags | DICT_FLAG_FIXED;
+ if ((dict_flags & (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL)) == 0)
+ dict_mdb->dict.flags |= (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL);
+ if (dict_flags & DICT_FLAG_FOLD_FIX)
+ dict_mdb->dict.fold_buf = vstring_alloc(10);
+ dict_mdb->env = env;
+ dict_mdb->key_buf = 0;
+ dict_mdb->val_buf = 0;
+
+ myfree(mdb_path);
+
+ return (DICT_DEBUG (&dict_mdb->dict));
+}
+
+#endif
diff --git a/src/util/dict_mdb.h b/src/util/dict_mdb.h
new file mode 100644
index 0000000..551d836
--- /dev/null
+++ b/src/util/dict_mdb.h
@@ -0,0 +1,40 @@
+#ifndef _DICT_MDB_H_INCLUDED_
+#define _DICT_MDB_H_INCLUDED_
+
+/*++
+/* NAME
+/* dict_mdb 3h
+/* SUMMARY
+/* dictionary manager interface to OpenLDAP MDB files
+/* SYNOPSIS
+/* #include <dict_mdb.h>
+/* DESCRIPTION
+/* .nf
+
+ /*
+ * Utility library.
+ */
+#include <dict.h>
+
+ /*
+ * External interface.
+ */
+#define DICT_TYPE_MDB "mdb"
+
+extern DICT *dict_mdb_open(const char *, int, int);
+
+ /*
+ * XXX Should be part of the DICT interface.
+ */
+extern size_t dict_mdb_map_size;
+
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+#endif
diff --git a/src/util/dict_open.c b/src/util/dict_open.c
index cba0f45..43bcde9 100644
--- a/src/util/dict_open.c
+++ b/src/util/dict_open.c
@@ -226,6 +226,7 @@
#include <dict_sdbm.h>
#include <dict_dbm.h>
#include <dict_db.h>
+#include <dict_mdb.h>
#include <dict_nis.h>
#include <dict_nisplus.h>
#include <dict_ni.h>
@@ -268,6 +269,9 @@ static const DICT_OPEN_INFO dict_open_info[] = {
DICT_TYPE_HASH, dict_hash_open,
DICT_TYPE_BTREE, dict_btree_open,
#endif
+#ifdef HAS_MDB
+ DICT_TYPE_MDB, dict_mdb_open,
+#endif
#ifdef HAS_NIS
DICT_TYPE_NIS, dict_nis_open,
#endif
--
1.7.9.5
