[email protected] wrote: > Robins George wrote: >> Hello Howard, > > Hello again. Revisiting this trace and walkthrough: > >> we are not able to reproduce the issue consistently so test code is not >> available at this time, and I do believe this issue will be present in the >> latest code as well since Myk also reported the same.. Attaching analysis >> from our side, hope it helps. >> >> The call stack what we have seen is, >> >> #0 0x0011b62e in mdb_cursor_put (mc=0xffe52d18, key=0xffe52e74, >> data=0xffe52e6c, flags=0) at mdb.c:6688 >> #1 0x0011c8ec in mdb_put (txn=0xdc13f008, dbi=2, key=0xffe52e74, >> data=0xffe52e6c, flags=0) at mdb.c:8771 >> #2 0x00b792b8 in LMDB::LMDBContext::createSession (this=0x80ca418, >> sid=0x8297be4 "sidebf3708f0fd9fcb8e062529da47adf51402dfc4600000000+") at >> lmdbint.cc:460 >> #3 0x08053564 in updateLMDB (request=..., forward=@0xffe5315f) at >> sessionserver.cc:1075 >> #4 processRequestWithoutResponse (request=..., forward=@0xffe5315f) at >> sessionserver.cc:1160 >> #5 0x08056a80 in processRequest (this=0x80b2ac0, fd=33) at >> sessionserver.cc:1189 >> #6 readFromSock (this=0x80b2ac0, fd=33) at sessionserver.cc:1342 >> #7 SockHandler::ioReady (this=0x80b2ac0, fd=33) at sessionserver.cc:1455 >> #8 0x00ba2592 in runCoreDispatcher (default_t=<value optimized out>, >> flags=-1) at fds.cc:889 >> #9 0x00ba2ff7 in DSEvntFds::runDispatcher () at fds.cc:945 >> #10 0x080559c6 in main () at sessionserver.cc:1739 >> >> Here it is helpful to examine frames #0 and #1 in detail in relation to the >> relevant source code /libraries/liblmdb/mdb.c. >> >> Frame #1 >> >> #1 0x0011c8ec in mdb_put (txn=0xdc13f008, dbi=2, key=0xffe52e74, >> data=0xffe52e6c, flags=0) at mdb.c:8771 >> mc = {mc_next = 0x0, mc_backup = 0x0, mc_xcursor = 0x0, mc_txn = >> 0xdc13f008, mc_dbi = 2, mc_db = 0xdc13f088, mc_dbx = 0xe1215038, mc_dbflag = >> 0xdc1cf09a "\033", '\032' <repeats 51 times>, mc_snum = 0, mc_top = 0, >> mc_flags = 0, mc_pg = {0x0, 0x80b2ac0, 0xffe52d68, 0x28afce, 0xdc13f008, >> 0x80ce9b0, 0xffe52dc8, 0x4c5ff4, 0x2b, 0x80b2ac0, 0xffe52d98, 0x49209a, >> 0x2b, 0x4c5ff4, 0x2121cb, 0x21576c, 0x8297d34, 0x3497af, 0x21576c, 0x21316a, >> 0x8297d34, 0x80ac7b0, 0x1e, 0x46eed6, 0x2b, 0x0, 0x80ce9b0, 0x4c5ff4, >> 0x80ac7b0, 0x8297d34, 0xffe52df8, 0x46fb96}, mc_ki = {0, 2089, 51120, 2058, >> 30, 0, 16796, 17, 11760, 65509, 3, 0, 32040, 2089, 30, 0, 0, 0, 0, 0, 3, 0, >> 24564, 76, 51120, 2058, 10944, 2059, 11800, 65509, 64758, 70}} <-------- >> >> mx = {mx_cursor = {mc_next = 0x38, mc_backup = 0x3, mc_xcursor = >> 0x0, mc_txn = 0x0, mc_dbi = 0, mc_db = 0x28, mc_dbx = 0x0, mc_dbflag = 0x0, >> mc_snum = 3, mc_top = 0, mc_flags = 17, mc_pg = {0x3a93d8, 0x10, 0x0, 0x18, >> 0x3a93a0, 0x0, 0x3a93d0, 0x289abe, 0x3a93a0, 0xffe52d3f, 0xffe52c68, >> 0x28afce, 0xffe52dbc, 0xffe52db4, 0x3, 0x4c5ff4, 0x11, 0x36c99b, 0x6e, 0x77, >> 0x7c, 0x5b, 0x38, 0x289abe, 0x0, 0x0, 0x0, 0x28, 0x0, 0x0, 0x3, 0x10}, mc_ki >> = {37848, 58, 51611, 54, 110, 0, 119, 0, 124, 0, 91, 0, 58, 0, 39614, 40, 0, >> 0, 0, 0, 0, 0, 160, 0, 0, 0, 2, 0, 18, 0, 136, 0}}, mx_db = {md_pad = >> 3838936, md_flags = 51611, md_depth = 54, md_branch_pages = 110, >> md_leaf_pages = 119, md_overflow_pages = 124, md_entries = 91, md_root = >> 56}, mx_dbx = {md_name = {mv_size = 6, mv_data = 0x0}, md_cmp = 0, md_dcmp = >> 0, md_rel = 0x40, md_relctx = 0x0}, mx_dbflag = 0 '\000'} >> rc = 0 >> >> And source code for mdb_put() is from mdb.c >> >> int >> mdb_put(MDB_txn *txn, MDB_dbi dbi, >> MDB_val *key, MDB_val *data, unsigned int flags) >> { >> MDB_cursor mc; >> MDB_xcursor mx; >> int rc; >> >> if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) >> return EINVAL; >> >> if (flags & >> ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) >> return EINVAL; >> >> if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) >> return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; >> >> mdb_cursor_init(&mc, txn, dbi, &mx); <-------- see source code below >> mc.mc_next = txn->mt_cursors[dbi]; >> txn->mt_cursors[dbi] = &mc; >> rc = mdb_cursor_put(&mc, key, data, flags); <-------- see Frame #0 >> txn->mt_cursors[dbi] = mc.mc_next; >> return rc; >> } >> >> >> Source Code For mdb_cursor_init() From mdb.c >> /** Initialize a cursor for a given transaction and database. */ >> static void >> mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) >> { >> mc->mc_next = NULL; >> mc->mc_backup = NULL; >> mc->mc_dbi = dbi; >> mc->mc_txn = txn; >> mc->mc_db = &txn->mt_dbs[dbi]; >> mc->mc_dbx = &txn->mt_dbxs[dbi]; >> mc->mc_dbflag = &txn->mt_dbflags[dbi]; >> mc->mc_snum = 0; <-------- >> mc->mc_top = 0; <-------- >> mc->mc_pg[0] = 0; <-------- >> mc->mc_ki[0] = 0; >> mc->mc_flags = 0; >> if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { >> mdb_tassert(txn, mx != NULL); >> mc->mc_xcursor = mx; >> mdb_xcursor_init0(mc); >> } else { >> mc->mc_xcursor = NULL; >> } >> if (*mc->mc_dbflag & DB_STALE) { <-------- false *mc->mc_dbflag = 27, >> DB_STALE = 0x02 /**< Named-DB record is older than txnID */ > > 27 = 0x1B, this DB is stale and this test is true, not false. > >> mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); >> } >> } >> >> From this we know that when mdb_cursor_put() is called, the following values >> remain assigned: >> >> mc->mc_snum = 0; <-------- >> mc->mc_top = 0; <-------- >> mc->mc_pg[0] = 0; <-------- >> >> and its noted these two values still have their initial values at the time >> of the segmentation fault. >> >> (gdb) fr 0 >> #0 0x0011b62e in mdb_cursor_put (mc=0xffe52d18, key=0xffe52e74, >> data=0xffe52e6c, flags=0) at mdb.c:6688 >> 6688 in mdb.c >> (gdb) p mc->mc_top >> $1 = 0 >> (gdb) p mc->mc_pg[mc->mc_top] >> $19 = (MDB_page *) 0x0 >> >> Let us consider how the path taken through mdb_cursor_put() could account >> for these null variable values which result in a segmentation fault. >> >> Frame #0 >> >> #0 0x0011b62e in mdb_cursor_put (mc=0xffe52d18, key=0xffe52e74, >> data=0xffe52e6c, flags=0) at mdb.c:6688 <-------- flags = 0 >> env = 0x80ce9b0 >> leaf = <value optimized out> >> fp = <value optimized out> >> mp = 0x38 >> sub_root = 0x0 >> fp_flags = <value optimized out> >> xdata = {mv_size = 3838880, mv_data = 0xdc1cf09a} >> rdata = 0xffe52e6c >> dkey = {mv_size = 0, mv_data = 0x3a93cc} >> olddata = {mv_size = 22, mv_data = 0x36c820} >> dummy = {md_pad = 22, md_flags = 11128, md_depth = 65509, >> md_branch_pages = 16, md_leaf_pages = 1507329, md_overflow_pages = 0, >> md_entries = 3838928, md_root = 3833844} >> do_sub = 0 >> insert_key = -30798 <-------- #define MDB_NOTFOUND (-30798) >> insert_data = -30798 <-------- #define MDB_NOTFOUND (-30798) >> mcount = 0 >> dcount = 0 >> nospill = 0 <-------- >> nsize = <value optimized out> >> rc = 0 >> rc2 = <value optimized out> >> nflags = 0 <-------- >> __func__ = "mdb_cursor_put" >> >> We know that mc->mc_top and mc->mc_pg[0] were initialized to 0 in >> mdb_cursor_init(). As my annotations of the source code below suggest, >> careful analysis reveals that the paths actually taken through >> mdb_cursor_put() do not modify these assignments. >> >> Excerpted Source Code For mdb_cursor_put() From mdb.c >> >> int >> mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, >> unsigned int flags) >> { >> MDB_env *env; >> MDB_node *leaf = NULL; >> MDB_page *fp, *mp, *sub_root = NULL; >> uint16_t fp_flags; >> MDB_val xdata, *rdata, dkey, olddata; >> MDB_db dummy; >> int do_sub = 0, insert_key, insert_data; >> unsigned int mcount = 0, dcount = 0, nospill; >> size_t nsize; >> int rc, rc2; >> unsigned int nflags; >> DKBUF; >> >> if (mc == NULL || key == NULL) >> return EINVAL; >> >> env = mc->mc_txn->mt_env; >> >> /* Check this first so counter will always be zero on any >> * early failures. >> */ >> if (flags & MDB_MULTIPLE) { >> dcount = data[1].mv_size; >> data[1].mv_size = 0; >> if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) >> return MDB_INCOMPATIBLE; >> } >> >> nospill = flags & MDB_NOSPILL; >> flags &= ~MDB_NOSPILL; >> >> if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) >> return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : >> MDB_BAD_TXN; >> >> if (key->mv_size-1 >= ENV_MAXKEY(env)) >> return MDB_BAD_VALSIZE; >> >> #if SIZE_MAX > MAXDATASIZE >> if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? >> ENV_MAXKEY(env) : MAXDATASIZE)) >> return MDB_BAD_VALSIZE; >> #else >> if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > >> ENV_MAXKEY(env)) >> return MDB_BAD_VALSIZE; >> #endif >> >> DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", >> DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); >> >> dkey.mv_size = 0; >> >> if (flags == MDB_CURRENT) { <-------- false flags = 0 >> if (!(mc->mc_flags & C_INITIALIZED)) >> return EINVAL; >> rc = MDB_SUCCESS; >> } else if (mc->mc_db->md_root == P_INVALID) { <-------- false, >> mc->mc_db->md_root = 68 > > I might have missed it, did you show the entire contents of *mc->mc_db > somewhere? > >> /* new database, cursor has nothing to point to */ >> mc->mc_snum = 0; >> mc->mc_top = 0; >> mc->mc_flags &= ~C_INITIALIZED; >> rc = MDB_NO_ROOT; >> } else { <-------- executed >> int exact = 0; >> MDB_val d2; >> if (flags & MDB_APPEND) { <-------- false flags = 0 >> MDB_val k2; >> rc = mdb_cursor_last(mc, &k2, &d2); >> if (rc == 0) { >> rc = mc->mc_dbx->md_cmp(key, &k2); >> if (rc > 0) { >> rc = MDB_NOTFOUND; >> mc->mc_ki[mc->mc_top]++; >> } else { >> /* new key is <= last key */ >> rc = MDB_KEYEXIST; >> } >> } >> } else {<-------- executed >> rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); >> <-------- returns -30798 which is MDB_NOTFOUND > > This is an important step. mdb_cursor_set only returns MDB_NOTFOUND when > mc->mc_pg[mc->mc_top] != NULL. > There are no code paths that can return this error code without setting the > rootpage pointer mc->mc_pg[0] non-NULL. > The only path that can allow this is if mc->mc_db->md_root == P_INVALID (the > tree is empty) and we already know > that's not true in this case. Nothing in LMDB will change these values > between the time mdb_cursor_set > returned and the time that mdb_cursor_put tries to reference mc->mc_pg[]. As > such this sounds like a > memory overwrite corruption from some other thread, unrelated to LMDB. > You can verify this by adding an assert after the mdb_cursor_set() call: mdb_cassert(mc, rc != MDB_NOTFOUND || mc->mc_pg != NULL);
At this point I would check to see if your threads' stack sizes are large enough. Since this cursor came from mdb_put, it lives on the stack, and a stack overrun is the most likely culprit. -- -- Howard Chu CTO, Symas Corp. http://www.symas.com Director, Highland Sun http://highlandsun.com/hyc/ Chief Architect, OpenLDAP http://www.openldap.org/project/
