This is a multi-part message in MIME format. --------------060806060106060808080102 Content-Type: text/plain; charset=windows-1251; format=flowed Content-Transfer-Encoding: 7bit
The attached files is derived from OpenLDAP Software. All of the modifications to OpenLDAP Software represented in the following patch(es) were developed by Peter-Service LLC, Moscow, Russia. Peter-Service LLC has not assigned rights and/or interest in this work to any party. I, Leonid Yuriev am authorized by Peter-Service LLC, my employer, to release this work under the following terms. Peter-Service LLC hereby places the following modifications to OpenLDAP Software (and only these modifications) into the public domain. Hence, these modifications may be freely used and/or redistributed for any purpose with or without attribution and/or other notice. --------------060806060106060808080102 Content-Type: text/x-patch; name="0001-lmdb-ITS-7974-a-reading-lag-for-dreamcatcher.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="0001-lmdb-ITS-7974-a-reading-lag-for-dreamcatcher.patch" >From a30ece9b236b7217481c086fd27b133bd3404317 Mon Sep 17 00:00:00 2001 From: Leo Yuriev <[email protected]> Date: Tue, 21 Oct 2014 15:34:22 +0400 Subject: [PATCH 1/2] lmdb: ITS#7974 a reading lag for dreamcatcher. --- libraries/liblmdb/lmdb.h | 11 +++++++++++ libraries/liblmdb/mdb.c | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index a3ca62e..82eff14 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -1571,6 +1571,17 @@ void mdb_env_set_oomkiller(MDB_env *env, MDB_oomkiller_func *oomkiller); * @return A #MDB_oomkiller_func function or NULL if disabled. */ MDB_oomkiller_func* mdb_env_get_oomkiller(MDB_env *env); + + /** @brief Returns a reading lag. + * + * Returns an information for estimate how much given read-only + * transaction is lagging relative the to actual head. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[out] percent Percentage of page allocation in the database. + * @return Number of transactions committed after the given was started for read, or -1 on failure. + */ +int mdb_txn_straggler(MDB_txn *txnm, int *percent); /** @} */ #ifdef __cplusplus diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index e60d83d..a417c9b 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -2823,6 +2823,26 @@ mdb_dbis_update(MDB_txn *txn, int keep) env->me_numdbs = n; } +int +mdb_txn_straggler(MDB_txn *txn, int *percent) +{ + MDB_env *env; + MDB_meta *meta; + txnid_t lag; + + if (! txn || ! txn->mt_u.reader) + return -1; + + env = txn->mt_env; + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + if (percent) { + long cent = env->me_maxpg / 100; + *percent = (meta->mm_last_pg + cent / 2 + 1) / (cent ? cent : 1); + } + lag = meta->mm_txnid - txn->mt_u.reader->mr_txnid; + return (0 > (int) lag) ? ~0u >> 1: lag; +} + /** Common code for #mdb_txn_reset() and #mdb_txn_abort(). * May be called twice for readonly txns: First reset it, then abort. * @param[in] txn the transaction handle to reset -- 2.1.0 --------------060806060106060808080102 Content-Type: text/x-patch; name="0002-slapd-ITS-7974-dreamcatcher-feature.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="0002-slapd-ITS-7974-dreamcatcher-feature.patch" >From 133cec8eadc93fe3083d110e58259ba2a067908c Mon Sep 17 00:00:00 2001 From: Leo Yuriev <[email protected]> Date: Tue, 21 Oct 2014 15:38:28 +0400 Subject: [PATCH 2/2] slapd: ITS#7974 dreamcatcher feature. --- servers/slapd/back-mdb/config.c | 46 +++++++++++++++++++++++++++++- servers/slapd/back-mdb/search.c | 62 ++++++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 29 deletions(-) diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c index b54da49..65034b1 100644 --- a/servers/slapd/back-mdb/config.c +++ b/servers/slapd/back-mdb/config.c @@ -39,7 +39,8 @@ enum { MDB_MAXREADERS, MDB_MAXSIZE, MDB_MODE, - MDB_SSTACK + MDB_SSTACK, + MDB_DREAMCATCHER }; static ConfigTable mdbcfg[] = { @@ -74,6 +75,10 @@ static ConfigTable mdbcfg[] = { mdb_cf_gen, "( OLcfgDbAt:12.2 NAME 'olcDbMaxSize' " "DESC 'Maximum size of DB in bytes' " "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, + { "dreamcatcher", "lag> <percentage", 3, 3, 0, ARG_MAGIC|MDB_DREAMCATCHER, + mdb_cf_gen, "( OLcfgDbAt:12.4 NAME 'olcDbDreamcatcher' " + "DESC 'Dreamcatcher to avoids withhold of reclaiming' " + "SYNTAX OMsDirectoryString SINGLE-VALUE )",NULL, NULL }, { "mode", "mode", 2, 2, 0, ARG_MAGIC|MDB_MODE, mdb_cf_gen, "( OLcfgDbAt:0.3 NAME 'olcDbMode' " "DESC 'Unix permissions of database files' " @@ -319,6 +324,23 @@ mdb_cf_gen( ConfigArgs *c ) } break; + case MDB_DREAMCATCHER: + if ( mdb->mi_renew_lag ) { + char buf[64]; + struct berval bv; + bv.bv_len = snprintf( buf, sizeof(buf), "%ld %ld", + (long) mdb->mi_renew_lag, (long) mdb->mi_renew_percent ); + if ( bv.bv_len > 0 && bv.bv_len < sizeof(buf) ) { + bv.bv_val = buf; + value_add_one( &c->rvalue_vals, &bv ); + } else { + rc = 1; + } + } else { + rc = 1; + } + break; + case MDB_DIRECTORY: if ( mdb->mi_dbenv_home ) { c->value_string = ch_strdup( mdb->mi_dbenv_home ); @@ -389,6 +411,10 @@ mdb_cf_gen( ConfigArgs *c ) } mdb->mi_txn_cp = 0; break; + case MDB_DREAMCATCHER: + mdb->mi_renew_lag = 0; + mdb->mi_renew_percent = 0; + break; case MDB_DIRECTORY: mdb->mi_flags |= MDB_RE_OPEN; ch_free( mdb->mi_dbenv_home ); @@ -574,6 +600,24 @@ mdb_cf_gen( ConfigArgs *c ) } } break; + case MDB_DREAMCATCHER: { + long l; + if ( lutil_atolx( &l, c->argv[1], 0 ) != 0 || l < 1 ) { + fprintf( stderr, "%s: " + "invalid lag \"%s\" in \"dreamcatcher\".\n", + c->log, c->argv[1] ); + return 1; + } + mdb->mi_renew_lag = l; + if ( lutil_atolx( &l, c->argv[2], 0 ) != 0 || l < 0 || l > 100 ) { + fprintf( stderr, "%s: " + "invalid percentage \"%s\" in \"dreamcatcher\".\n", + c->log, c->argv[2] ); + return 1; + } + mdb->mi_renew_percent = l; + } break; + case MDB_DIRECTORY: { FILE *f; char *ptr, *testpath; diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c index 98b5f25..381cb80 100644 --- a/servers/slapd/back-mdb/search.c +++ b/servers/slapd/back-mdb/search.c @@ -340,22 +340,29 @@ typedef struct ww_ctx { * case return an LDAP_BUSY error - let the client know this search * couldn't succeed, but might succeed on a retry. */ + +static void +mdb_befree( Operation *op, ww_ctx *ww ) +{ + assert(! ww->flag); + if ( ww->mcd ) { + MDB_val key, data; + mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT ); + memcpy( &ww->key, key.mv_data, sizeof(ID) ); + ww->data.mv_size = data.mv_size; + ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); + memcpy(ww->data.mv_data, data.mv_data, data.mv_size); + } + mdb_txn_reset( ww->txn ); + ww->flag = 1; +} + static void mdb_writewait( Operation *op, slap_callback *sc ) { ww_ctx *ww = sc->sc_private; - if ( !ww->flag ) { - if ( ww->mcd ) { - MDB_val key, data; - mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT ); - memcpy( &ww->key, key.mv_data, sizeof(ID) ); - ww->data.mv_size = data.mv_size; - ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx ); - memcpy(ww->data.mv_data, data.mv_data, data.mv_size); - } - mdb_txn_reset( ww->txn ); - ww->flag = 1; - } + if ( !ww->flag ) + mdb_befree( op, ww ); } static int @@ -796,7 +803,6 @@ loop_begin: goto done; } - if ( nsubs < ncand ) { unsigned i; /* Is this entry in the candidate list? */ @@ -1034,14 +1040,6 @@ notfound: ber_bvarray_free( erefs ); rs->sr_ref = NULL; - if ( wwctx.flag ) { - rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd ); - if ( rs->sr_err ) { - send_ldap_result( op, rs ); - goto done; - } - } - goto loop_continue; } @@ -1096,13 +1094,6 @@ notfound: } goto done; } - if ( wwctx.flag ) { - rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd ); - if ( rs->sr_err ) { - send_ldap_result( op, rs ); - goto done; - } - } } } else { @@ -1113,6 +1104,21 @@ notfound: } loop_continue: + if ( ! wwctx.flag && mdb->mi_renew_lag ) { + int percentage, lag = mdb_txn_straggler( ltid, &percentage ); + if ( lag >= mdb->mi_renew_lag && percentage >= mdb->mi_renew_percent ) { + Debug( LDAP_DEBUG_TRACE, "dreamcather: lag %d, percentage %u%%\n", lag, percentage, 0 ); + mdb_befree( op, &wwctx ); + } + } + if ( wwctx.flag ) { + rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd ); + if ( rs->sr_err ) { + send_ldap_result( op, rs ); + goto done; + } + } + if( e != NULL ) { if ( e != base ) mdb_entry_return( op, e ); -- 2.1.0 --------------060806060106060808080102--
