Update of /cvsroot/monetdb/pathfinder/runtime
In directory sc8-pr-cvs16:/tmp/cvs-serv24390
Modified Files:
pathfinder.mx pf_support.mx
Log Message:
- use msync for incremental shredding, rather than save
in a mixed update load with updates and shreds, saving the entire bat
also saves the dirty pages caused by the updates rather than just he
shred. performance-wise it is better to batch those until the next checkpoint.
- fix perf issue that caused the nsloc index to be ignored on small
context sequences (surfaced in xmark Q6/Q7)
Index: pathfinder.mx
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/runtime/pathfinder.mx,v
retrieving revision 1.353
retrieving revision 1.354
diff -u -d -r1.353 -r1.354
--- pathfinder.mx 1 May 2007 13:21:49 -0000 1.353
+++ pathfinder.mx 2 May 2007 12:35:27 -0000 1.354
@@ -301,7 +301,6 @@
@:[EMAIL PROTECTED](ATTR_OWN, 19, void, bat, void, oid, PRE_BASE)@
@:[EMAIL PROTECTED](ATTR_QN, 20, void, bat, void, oid, PRE_BASE)@
@:[EMAIL PROTECTED](ATTR_PROP, 21, void, bat, void, oid, PRE_BASE)@
-
@:[EMAIL PROTECTED](QN_NID, 22, void, bat, void, oid, PRE_BASE)@
@:[EMAIL PROTECTED](QN_NID_DEL, 23, void, bat, void, oid, PRE_BASE)@
@:[EMAIL PROTECTED](QN_NID_INS, 24, void, bat, void, oid, PRE_BASE)@
@@ -2461,8 +2460,10 @@
# it may be that the other thread has done _ws_open_coll (adding itself to
# ws_overlaps_ws, so we noticed it) but not yet __ws_open_coll(), which
- # requires the coll_shortlock (possed by ws_isolate())
- if (not(isnil(ws.seqbase())) and ws.fetch(CONT_NAME).texist(colname)) {
+ # requires the coll_shortlock (possessed by ws_isolate())
+ if (ws.fetch(CONT_NAME).texist(colname) and
+ not(isnil(ws.seqbase()))) # transactions set ws.seqbase nil when
isolation is no longer required
+ {
if (ws_log_active) {
[EMAIL PROTECTED]()
ws_log(wsid, "ws_coll_isolate: ancestor_nid-" + str($t));
@@ -2493,18 +2494,18 @@
var modified_pid := modified_page.leftjoin(map_pid).tmark([EMAIL
PROTECTED]);
# isolate the affected positions
+ nid_rid.isolate(modified_nid, false);
+ attr_own.isolate(modified_attr, false);
+ attr_qn.isolate(modified_attr, false);
+ attr_prop.isolate(modified_attr, false);
pre_size.isolate(ancestor_pre, false);
+
+ # isolate full pages
pre_size.isolate(modified_pid, true);
pre_level.isolate(modified_pid,true);
pre_prop.isolate(modified_pid, true);
pre_kind.isolate(modified_pid, true);
- if (map_pid.ttype() = oid) {
- pre_nid.isolate(modified_pid, true);
- nid_rid.isolate(modified_nid, false);
- }
- attr_own.isolate(modified_attr, false);
- attr_qn.isolate(modified_attr, false);
- attr_prop.isolate(modified_attr, false);
+ pre_nid.isolate(modified_pid, true);
}
if (ws_log_active)
ws_log(wsid, "ws_coll_isolate: exec-" + str(ws_logtime - usec()));
@@ -2516,17 +2517,16 @@
BAT[void,oid] modified_attr) : void
{
var wsid := ws_id(ws);
- lock_set(pf_short);
- var overlap;
- var err := CATCH(overlap :=
reverse(ws_overlaps_ws).uselect(wsid).kunique().mirror());
- lock_unset(pf_short);
- if (not(isnil(err)))
- ERROR(err);
var runtime := ws.fetch(CONT_RUNTIME).fetch(cont);
var page_ws := runtime.fetch(RT_PAGE_WS);
var attr_ws := runtime.fetch(RT_ATTR_WS);
var ws_logtime;
+ lock_set(pf_short);
+ var overlap, err := CATCH(overlap :=
reverse(ws_overlaps_ws).uselect(wsid).kunique().mirror());
+ lock_unset(pf_short);
+ if (not(isnil(err))) ERROR(err);
+
if (ws_log_active) {
ws_logtime := usec();
ws_log(wsid, "ws_precommit");
@@ -2545,18 +2545,18 @@
# concurrency control: abort conflicting queries
var conflict1 := join(modified_page,page_ws).join(overlap);
if (bit(count(conflict1))) {
- var err := sprintf("conflicting update at page %s from ws %s (%d such
errors in total)",
-
str(modified_page.find(reverse(conflict1).fetch(0))),
- str(conflict1.fetch(0)), count(conflict1));
+ err := sprintf("conflicting update at page %s from ws %s (%d such
errors in total)",
+ str(modified_page.find(reverse(conflict1).fetch(0))),
+ str(conflict1.fetch(0)), count(conflict1));
if (ws_log_active) ws_log(wsid, err);
ERROR(err);
}
var conflict2 := join(modified_attr,attr_ws).join(overlap);
if (bit(count(conflict2))) {
- var err := sprintf("conflicting update at page %s from ws %s (%d such
errors in total)",
-
str(modified_attr.find(reverse(conflict2).fetch(0))),
- str(conflict2.fetch(0)), count(conflict2));
+ err := sprintf("conflicting update at page %s from ws %s (%d such
errors in total)",
+ str(modified_attr.find(reverse(conflict2).fetch(0))),
+ str(conflict2.fetch(0)), count(conflict2));
if (ws_log_active) ws_log(wsid, err);
ERROR(err);
}
@@ -3323,6 +3323,7 @@
}
if (doCommit) {
+
lock_set(pf_short);
# possibly back off if auto-caching documents gets us into meta-data
locking trouble
@@ -3417,11 +3418,18 @@
var percentage := pageFrees.find($h);
var docBAT;
+ var pre := [EMAIL PROTECTED], att := [EMAIL PROTECTED], nid := [EMAIL
PROTECTED], val := [EMAIL PROTECTED], txt := [EMAIL PROTECTED];
if (isnil(coll_oid)) {
docBAT := bat(str,bat,count(idx)); # new collection
} else { # append to existing
docBAT := [bat]([+](str(int(coll_oid)),
ws_dsk).reverse().mirror()).access(BAT_WRITE);
+ pre := oid(count(docBAT.fetch(PRE_SIZE)));
+ nid := oid(count(docBAT.fetch(NID_RID)));
+ att := oid(count(docBAT.fetch(ATTR_QN)));
+ val := oid(count(docBAT.fetch(PROP_VAL)));
+ txt := oid(count(docBAT.fetch(PROP_TEXT)));
}
+
# now shred all documents for one collection
shred_into_docBAT(docBAT,
idx.leftfetchjoin(idx_locations),
@@ -3429,8 +3437,20 @@
idx.project(timestamp_nil),
colname, coll_oid, runtime, oid_nil, percentage,
true, s, wsid);
- if (pf_commit_docmgt)
- [save](docBAT); # make persistent documents clean on disk outside the
locks
+ if (pf_commit_docmgt) {
+ # make big mmap-heaps clean outside of the locks
+ docBAT.fetch(PRE_SIZE) .msync(pre, false);
+ docBAT.fetch(PRE_LEVEL).msync(pre, false);
+ docBAT.fetch(PRE_PROP) .msync(pre, false);
+ docBAT.fetch(PRE_KIND) .msync(pre, false);
+ docBAT.fetch(PRE_NID) .msync(pre, false);
+ docBAT.fetch(NID_RID) .msync(nid, false);
+ docBAT.fetch(ATTR_OWN) .msync(att, false);
+ docBAT.fetch(ATTR_QN) .msync(att, false);
+ docBAT.fetch(ATTR_PROP).msync(att, false);
+ docBAT.fetch(PROP_TEXT).msync(txt, false);
+ docBAT.fetch(PROP_VAL) .msync(val, false);
+ }
if (isnil(coll_oid)) {
# register new bats with the logger
Index: pf_support.mx
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/runtime/pf_support.mx,v
retrieving revision 1.220
retrieving revision 1.221
diff -u -d -r1.220 -r1.221
--- pf_support.mx 1 May 2007 11:17:34 -0000 1.220
+++ pf_support.mx 2 May 2007 12:35:31 -0000 1.221
@@ -1801,7 +1801,7 @@
@= prefilter
cands := ws.fetch(PRE_SIZE).fetch(contID).mirror();
var pre_cont := ws.fetch(PRE_CONT).fetch(contID);
- if (is_constant(pre_cont) and (count(result_part_item) > 2048)) {
+ if (is_constant(pre_cont)) {
var qn_sel :=
ws.fetch(@2).fetch(bat2constant(pre_cont)).ord_uselect(@1);
if (isnil(CATCH(cands := ws_lookup(ws, contID,
qn_sel.mirror())))) {
var map_pid := ws.fetch(MAP_PID).fetch(contID);
-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Monetdb-pf-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins
