Sorry, but I could not understand the meaning of back-bdb residing in HEAD.
I have attached a diff over openldap-2.3.35 for changes regarding BDB
in-memory.
This mainly has changes for keeping logs,environment and databases in
memory.
The bdb in-memory code is under #ifdef BDB_INMEMORY
I am trying out different environment cache sizes to check any performance
difference.
Thanks and Regards,
Suhel
On 8/31/07, Howard Chu <[EMAIL PROTECTED]> wrote:
> Suhel Momin wrote:
> > Hi,
> >
> > I have made changes such that backend BDB ( ver 4.5 ) resides in
> > memory instead of on disk . This is with default slapd configuration.
> > I was hoping to have a better performance with this.
>
> back-bdb as currently residing in HEAD already yields efficiencies of 95%
> of
> available system bandwidth. It is extremely unlikely that you are going to
> be
> able to improve the performance significantly, most changes you could make
> to
> the code at this point will only yield performance losses.
>
> > But what I now see
> > is that do_add takes much more time then what it use to take when BDB
> > was on-Disk. Indexing is done on objectclass in both the cases.
> >
> > Any pointers on why this could be an issue?
>
> --
> -- Howard Chu
> Chief Architect, Symas Corp. http://www.symas.com
> Director, Highland Sun http://highlandsun.com/hyc/
> Chief Architect, OpenLDAP http://www.openldap.org/project/
>
diff -Narup openldap-2.3.35/servers/slapd/back-bdb/dbcache.c inmemory_openldap-2.3.35/servers/slapd/back-bdb/dbcache.c
--- openldap-2.3.35/servers/slapd/back-bdb/dbcache.c 2007-01-03 03:14:00.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-bdb/dbcache.c 2007-09-04 19:05:57.000000000 +0530
@@ -116,6 +116,12 @@ bdb_db_cache(
file = ch_malloc( strlen( name ) + sizeof(BDB_SUFFIX) );
sprintf( file, "%s" BDB_SUFFIX, name );
+#ifdef BDB_INMEMORY
+ /* DB_MPOOL_NOFILE is set to specify database will remain in memory*/
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
+
+
#ifdef HAVE_EBCDIC
__atoe( file );
#endif
@@ -129,10 +135,16 @@ bdb_db_cache(
(SLAP_TOOL_QUICK|SLAP_TRUNCATE_MODE))
flags |= DB_TRUNCATE;
+#ifdef BDB_INMEMORY
+ rc = DB_OPEN( db->bdi_db,
+ NULL, /*No need to specify the name as the database will reside in-memory*/
+ NULL /* name */,
+ BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
+#else
rc = DB_OPEN( db->bdi_db,
file, NULL /* name */,
BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
-
+#endif
ch_free( file );
if( rc != 0 ) {
diff -Narup openldap-2.3.35/servers/slapd/back-bdb/init.c inmemory_openldap-2.3.35/servers/slapd/back-bdb/init.c
--- openldap-2.3.35/servers/slapd/back-bdb/init.c 2007-02-25 00:58:11.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-bdb/init.c 2007-09-04 19:01:25.000000000 +0530
@@ -58,10 +58,17 @@ bdb_db_init( BackendDB *be )
bdb = (struct bdb_info *) ch_calloc( 1, sizeof(struct bdb_info) );
/* DBEnv parameters */
+
+#ifdef BDB_INMEMORY
+ /* keep logs in memory*/
+ bdb->bi_dbenv_home = NULL;
+ bdb->bi_dbenv_xflags = DB_LOG_INMEMORY;
+ bdb->bi_dbenv_mode = 0;
+#else
bdb->bi_dbenv_home = ch_strdup( SLAPD_DEFAULT_DB_DIR );
bdb->bi_dbenv_xflags = 0;
bdb->bi_dbenv_mode = SLAPD_DEFAULT_DB_MODE;
-
+#endif
bdb->bi_cache.c_maxsize = DEFAULT_CACHE_SIZE;
bdb->bi_cache.c_minfree = 1;
@@ -278,8 +285,12 @@ shm_retry:
LDAP_XSTRING(bdb_db_open) ": dbenv_open(%s)\n",
bdb->bi_dbenv_home, 0, 0);
+#ifdef BDB_INMEMORY
+ /* DB_PRIVATE flag is added to keep environment in memory */
+ flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_PRIVATE;
+#else
flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
-
+#endif
if ( !quick )
flags |= BDB_TXN_FLAGS;
@@ -359,6 +370,9 @@ shm_retry:
goto fail;
}
+#ifdef BDB_INMEMORY
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
if( i == BDB_ID2ENTRY ) {
if ( slapMode & SLAP_TOOL_MODE )
db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
@@ -391,6 +405,29 @@ shm_retry:
BDB_PAGESIZE );
}
+
+#ifdef BDB_INMEMORY
+
+#ifdef HAVE_EBCDIC
+ strcpy( path, bdbi_databases[i].file );
+ __atoe( path );
+ rc = DB_OPEN( db->bdi_db,
+ /* path*/NULL, /* don't need path for in-memory db*/
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#else
+ rc = DB_OPEN( db->bdi_db,
+ /* bdbi_databases[i].file*/NULL, /* don't need file for in-memory db */
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#endif
+
+#else
+
#ifdef HAVE_EBCDIC
strcpy( path, bdbi_databases[i].file );
__atoe( path );
@@ -409,6 +446,7 @@ shm_retry:
bdb->bi_dbenv_mode );
#endif
+#endif /*BDB_INMEMORY */
if ( rc != 0 ) {
char buf[SLAP_TEXT_BUFLEN];
diff -Narup openldap-2.3.35/servers/slapd/back-hdb/dbcache.c inmemory_openldap-2.3.35/servers/slapd/back-hdb/dbcache.c
--- openldap-2.3.35/servers/slapd/back-hdb/dbcache.c 2007-01-03 03:14:00.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-hdb/dbcache.c 2007-09-04 19:05:57.000000000 +0530
@@ -116,6 +116,12 @@ bdb_db_cache(
file = ch_malloc( strlen( name ) + sizeof(BDB_SUFFIX) );
sprintf( file, "%s" BDB_SUFFIX, name );
+#ifdef BDB_INMEMORY
+ /* DB_MPOOL_NOFILE is set to specify database will remain in memory*/
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
+
+
#ifdef HAVE_EBCDIC
__atoe( file );
#endif
@@ -129,10 +135,16 @@ bdb_db_cache(
(SLAP_TOOL_QUICK|SLAP_TRUNCATE_MODE))
flags |= DB_TRUNCATE;
+#ifdef BDB_INMEMORY
+ rc = DB_OPEN( db->bdi_db,
+ NULL, /*No need to specify the name as the database will reside in-memory*/
+ NULL /* name */,
+ BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
+#else
rc = DB_OPEN( db->bdi_db,
file, NULL /* name */,
BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
-
+#endif
ch_free( file );
if( rc != 0 ) {
diff -Narup openldap-2.3.35/servers/slapd/back-hdb/init.c inmemory_openldap-2.3.35/servers/slapd/back-hdb/init.c
--- openldap-2.3.35/servers/slapd/back-hdb/init.c 2007-02-25 00:58:11.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-hdb/init.c 2007-09-04 19:01:25.000000000 +0530
@@ -58,10 +58,17 @@ bdb_db_init( BackendDB *be )
bdb = (struct bdb_info *) ch_calloc( 1, sizeof(struct bdb_info) );
/* DBEnv parameters */
+
+#ifdef BDB_INMEMORY
+ /* keep logs in memory*/
+ bdb->bi_dbenv_home = NULL;
+ bdb->bi_dbenv_xflags = DB_LOG_INMEMORY;
+ bdb->bi_dbenv_mode = 0;
+#else
bdb->bi_dbenv_home = ch_strdup( SLAPD_DEFAULT_DB_DIR );
bdb->bi_dbenv_xflags = 0;
bdb->bi_dbenv_mode = SLAPD_DEFAULT_DB_MODE;
-
+#endif
bdb->bi_cache.c_maxsize = DEFAULT_CACHE_SIZE;
bdb->bi_cache.c_minfree = 1;
@@ -278,8 +285,12 @@ shm_retry:
LDAP_XSTRING(bdb_db_open) ": dbenv_open(%s)\n",
bdb->bi_dbenv_home, 0, 0);
+#ifdef BDB_INMEMORY
+ /* DB_PRIVATE flag is added to keep environment in memory */
+ flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_PRIVATE;
+#else
flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
-
+#endif
if ( !quick )
flags |= BDB_TXN_FLAGS;
@@ -359,6 +370,9 @@ shm_retry:
goto fail;
}
+#ifdef BDB_INMEMORY
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
if( i == BDB_ID2ENTRY ) {
if ( slapMode & SLAP_TOOL_MODE )
db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
@@ -391,6 +405,29 @@ shm_retry:
BDB_PAGESIZE );
}
+
+#ifdef BDB_INMEMORY
+
+#ifdef HAVE_EBCDIC
+ strcpy( path, bdbi_databases[i].file );
+ __atoe( path );
+ rc = DB_OPEN( db->bdi_db,
+ /* path*/NULL, /* don't need path for in-memory db*/
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#else
+ rc = DB_OPEN( db->bdi_db,
+ /* bdbi_databases[i].file*/NULL, /* don't need file for in-memory db */
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#endif
+
+#else
+
#ifdef HAVE_EBCDIC
strcpy( path, bdbi_databases[i].file );
__atoe( path );
@@ -409,6 +446,7 @@ shm_retry:
bdb->bi_dbenv_mode );
#endif
+#endif /*BDB_INMEMORY */
if ( rc != 0 ) {
char buf[SLAP_TEXT_BUFLEN];
