Changeset: a8cfb61a0d46 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a8cfb61a0d46
Modified Files:
sql/src/storage/bat/bat_storage.c
sql/src/storage/restrict/restrict_storage.c
Branch: default
Log Message:
Merge with Oct2010 branch.
diffs (truncated from 345 to 300 lines):
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB/src/gdk/gdk.mx
--- a/MonetDB/src/gdk/gdk.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB/src/gdk/gdk.mx Fri Oct 01 18:09:54 2010 +0200
@@ -701,7 +701,8 @@
char storage; /* storage mode (mmap/malloc). */
unsigned char copied:1, /* a copy of an existing map. */
- hashash:1; /* the string heap contains hash values */
+ hashash:1,/* the string heap contains hash values */
+ forcemap:1; /* force STORE_MMAP even if heap exists */
chr newstorage; /* new desired storage mode at re-allocation. */
chr dirty; /* specific heap dirty marker */
bat parentid; /* cache id of VIEW parent bat */
@@ -1799,7 +1800,7 @@
@item BAT *
@tab BATsave (BAT *b)
@item int
-...@tab BATmmap (BAT *b, int hb, int tb, int hh, int th )
+...@tab BATmmap (BAT *b, int hb, int tb, int hh, int th, int force )
@item int
@tab BATmadvise (BAT *b, int hb, int tb, int hh, int th )
@item int
@@ -1862,7 +1863,7 @@
#define BATaccessEnd(b,what,advice) BATaccess(b,what,advice,-1)
gdk_export size_t BATaccess(BAT *b, int what, int advice, int load);
gdk_export BAT *BATsave(BAT *b);
-gdk_export int BATmmap(BAT *b, int hb, int tb, int hh, int th);
+gdk_export int BATmmap(BAT *b, int hb, int tb, int hh, int th, int force);
gdk_export int BATmadvise(BAT *b, int hb, int tb, int hh, int th);
gdk_export int BATdelete(BAT *b);
gdk_export size_t BATvmsize(BAT *b, int dirty);
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB/src/gdk/gdk_bat.mx
--- a/MonetDB/src/gdk/gdk_bat.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB/src/gdk/gdk_bat.mx Fri Oct 01 18:09:54 2010 +0200
@@ -2503,34 +2503,42 @@
@{
@c
/* TODO niels: merge with BATsetmodes in gdk_storage */
-#define STORE_MODE(m,r,e,s) (((m) ==
STORE_MEM)?STORE_MEM:((r)&&(e))||(s)==STORE_PRIV?STORE_PRIV:STORE_MMAP)
+#define STORE_MODE(m,r,e,s,f) (((m) ==
STORE_MEM)?STORE_MEM:((r)&&(e)&&!(f))||(s)==STORE_PRIV?STORE_PRIV:STORE_MMAP)
static void
-HEAPnewstorage(BAT *b)
+HEAPnewstorage(BAT *b, int force)
{
int existing = (BBPstatus(b->batCacheid) & BBPEXISTING);
int brestrict = (b->batRestricted == BAT_WRITE);
if (b->batMaphead) {
- b->H->heap.newstorage = STORE_MODE(b->batMaphead, brestrict,
existing, b->H->heap.storage);
+ b->H->heap.newstorage = STORE_MODE(b->batMaphead, brestrict,
existing, b->H->heap.storage, force);
+ if (force)
+ b->H->heap.forcemap = 1;
}
if (b->batMaptail) {
- b->T->heap.newstorage = STORE_MODE(b->batMaptail, brestrict,
existing, b->T->heap.storage);
+ b->T->heap.newstorage = STORE_MODE(b->batMaptail, brestrict,
existing, b->T->heap.storage, force);
+ if (force)
+ b->T->heap.forcemap = 1;
}
if (b->H->vheap && b->batMaphheap) {
int hrestrict = (b->batRestricted == BAT_APPEND) &&
ATOMappendpriv(b->htype, b->H->vheap);
- b->H->vheap->newstorage = STORE_MODE(b->batMaphheap, brestrict
|| hrestrict, existing, b->H->vheap->storage);
+ b->H->vheap->newstorage = STORE_MODE(b->batMaphheap, brestrict
|| hrestrict, existing, b->H->vheap->storage, force);
+ if (force)
+ b->H->vheap->forcemap = 1;
}
if (b->T->vheap && b->batMaptheap) {
int trestrict = (b->batRestricted == BAT_APPEND) &&
ATOMappendpriv(b->ttype, b->T->vheap);
- b->T->vheap->newstorage = STORE_MODE(b->batMaptheap, brestrict
|| trestrict, existing, b->T->vheap->storage);
+ b->T->vheap->newstorage = STORE_MODE(b->batMaptheap, brestrict
|| trestrict, existing, b->T->vheap->storage, force);
+ if (force)
+ b->T->vheap->forcemap = 1;
}
}
int
-BATmmap(BAT *b, int hb, int tb, int hhp, int thp)
+BATmmap(BAT *b, int hb, int tb, int hhp, int thp, int force)
{
BATcheck(b, "BATmmap");
- IODEBUG THRprintf(GDKout, "#BATmmap(%s,%d,%d,%d,%d)\n", BATgetId(b),
hb, tb, hhp, thp);
+ IODEBUG THRprintf(GDKout, "#BATmmap(%s,%d,%d,%d,%d%s)\n", BATgetId(b),
hb, tb, hhp, thp, force ? ",force" : "");
/* Reverse back if required, as this determines which heap is saved in
the
* "hheap" file and which in the "theap" file.
@@ -2548,7 +2556,7 @@
b->batMaptail = tb;
b->batMaphheap = hhp;
b->batMaptheap = thp;
- HEAPnewstorage(b);
+ HEAPnewstorage(b, force);
b->batDirtydesc = 1;
return 0;
}
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB/src/gdk/gdk_bbp.mx
--- a/MonetDB/src/gdk/gdk_bbp.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB/src/gdk/gdk_bbp.mx Fri Oct 01 18:09:54 2010 +0200
@@ -3218,6 +3218,8 @@
assert(VIEWvhparent(b) == 0);
assert(VIEWvtparent(b) == 0);
b->H->heap.parentid = b->T->heap.parentid = 0;
+ b->H->heap.forcemap = 0;
+ b->T->heap.forcemap = 0;
b->batPersistence = TRANSIENT;
/* take care of mirror views */
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB/src/gdk/gdk_heap.mx
--- a/MonetDB/src/gdk/gdk_heap.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB/src/gdk/gdk_heap.mx Fri Oct 01 18:09:54 2010 +0200
@@ -233,11 +233,12 @@
if (fp != NULL) {
fclose(fp);
if (h->storage == STORE_MEM) {
- int newmode = h->newstorage ==
STORE_MMAP && existing ? STORE_PRIV : h->newstorage;
+ int newmode = h->newstorage ==
STORE_MMAP && existing && !h->forcemap ? STORE_PRIV : h->newstorage;
/* make sure we really MMAP */
if (must_mmap && h->newstorage ==
STORE_MEM)
newmode = STORE_MMAP;
h->newstorage = h->storage = newmode;
+ h->forcemap = 0;
}
h->base = NULL;
EXTENDDEBUG fprintf(stderr, "#HEAPextend:
converting malloced to %s mmapped heap\n", h->newstorage == STORE_MMAP ?
"shared" : "privately");
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB/src/gdk/gdk_logger.mx
--- a/MonetDB/src/gdk/gdk_logger.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB/src/gdk/gdk_logger.mx Fri Oct 01 18:09:54 2010 +0200
@@ -1830,7 +1830,7 @@
BATmode(lb, PERSISTENT);
assert(lb->P->restricted > BAT_WRITE);
if (BATcount(lb) > (BUN) REMAP_PAGE_MAXSIZE)
- BATmmap(lb, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP);
+ BATmmap(lb, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP, 0);
logbat_destroy(lb);
if (lg->debug & 1)
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB4/src/modules/plain/ascii_io.mx
--- a/MonetDB4/src/modules/plain/ascii_io.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB4/src/modules/plain/ascii_io.mx Fri Oct 01 18:09:54 2010 +0200
@@ -334,7 +334,7 @@
if (BATmirror(b))
BATseqbase(b, 0);
if (nr > (BUN) REMAP_PAGE_MAXSIZE)
- BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
if (nr > BATTINY && adt)
b = BATextend(b, nr);
if (b == NULL)
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB4/src/modules/plain/bat.mx
--- a/MonetDB4/src/modules/plain/bat.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB4/src/modules/plain/bat.mx Fri Oct 01 18:09:54 2010 +0200
@@ -1660,7 +1660,7 @@
*hhp = b->batMaphheap;
if (b->T->vheap && *thp == int_nil)
*thp = b->batMaptheap;
- if (BATmmap(*r = b, *hbns, *tbns, *hhp, *thp) == 0) {
+ if (BATmmap(*r = b, *hbns, *tbns, *hhp, *thp, 0) == 0) {
BBPfix(b->batCacheid);
return GDK_SUCCEED;
}
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB5/src/modules/kernel/bat5.mx
--- a/MonetDB5/src/modules/kernel/bat5.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB5/src/modules/kernel/bat5.mx Fri Oct 01 18:09:54 2010 +0200
@@ -1679,7 +1679,7 @@
*hhp = b->batMaphheap;
if (b->T->vheap && *thp == int_nil)
*thp = b->batMaptheap;
- if (BATmmap(*r = b, *hbns, *tbns, *hhp, *thp) == 0) {
+ if (BATmmap(*r = b, *hbns, *tbns, *hhp, *thp, 0) == 0) {
BBPfix(b->batCacheid);
return GDK_SUCCEED;
}
diff -r a2d1f5c5b38d -r a8cfb61a0d46
MonetDB5/src/modules/mal/crackers/crackers_crackmerge.mx
--- a/MonetDB5/src/modules/mal/crackers/crackers_crackmerge.mx Fri Oct 01
16:52:26 2010 +0200
+++ b/MonetDB5/src/modules/mal/crackers/crackers_crackmerge.mx Fri Oct 01
18:09:54 2010 +0200
@@ -195,7 +195,7 @@
current=CrackerIndex[m].smNode;
resTuplesLocal=0;
br=BATnew(TYPE_oid, bo->ttype,BATcount(bo)/10);
- BATmmap(br, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(br, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
br->batRestricted= BAT_READ;
while(current!=NULL){
@@ -661,7 +661,7 @@
oid endSlice,startSlice;
br=BATnew(TYPE_oid, bo->ttype,BATcount(bo)/10);
- BATmmap(br, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(br, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
br->batRestricted= BAT_READ;
m = newcrackerindexsli...@1(*bid,br->batCacheid,1000000);
diff -r a2d1f5c5b38d -r a8cfb61a0d46
MonetDB5/src/modules/mal/crackers/crackers_sortmerge.mx
--- a/MonetDB5/src/modules/mal/crackers/crackers_sortmerge.mx Fri Oct 01
16:52:26 2010 +0200
+++ b/MonetDB5/src/modules/mal/crackers/crackers_sortmerge.mx Fri Oct 01
18:09:54 2010 +0200
@@ -936,7 +936,7 @@
BATmirror(BATorder(BATmirror(b)));
timeSort+=GDKusec()-t1;
- if (BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP) != 0)
+ if (BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP, 0) != 0)
throw(MAL, "crackers.sortmerge", "Cannot mmap
BAT");
smNode=(struct sortMergeNode *)GDKmalloc(sizeof(struct
sortMergeNode));
smNode->batId=b->batCacheid;
@@ -1394,7 +1394,7 @@
b->batRestricted= BAT_WRITE;
BATmirror(BATorder(BATmirror(b)));
- if (BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP)
!= 0)
+ if (BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP,
0) != 0)
throw(MAL, "crackers.sortmerge", "Cannot mmap BAT");
smNode=(struct sortMergeNode *)GDKmalloc(sizeof(struct
sortMergeNode));
diff -r a2d1f5c5b38d -r a8cfb61a0d46 MonetDB5/src/modules/mal/tablet.mx
--- a/MonetDB5/src/modules/mal/tablet.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/MonetDB5/src/modules/mal/tablet.mx Fri Oct 01 18:09:54 2010 +0200
@@ -992,7 +992,7 @@
BATseqbase(b, 0);
BATsetaccess(b, BAT_APPEND);
if (nr > (BUN) REMAP_PAGE_MAXSIZE)
- BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
if (nr > BATTINY && adt)
b = BATextend(b, nr);
if (b == NULL)
diff -r a2d1f5c5b38d -r a8cfb61a0d46
pathfinder/modules/pftijah/serialize_pftijah.mx
--- a/pathfinder/modules/pftijah/serialize_pftijah.mx Fri Oct 01 16:52:26
2010 +0200
+++ b/pathfinder/modules/pftijah/serialize_pftijah.mx Fri Oct 01 18:09:54
2010 +0200
@@ -163,7 +163,7 @@
#if 1
/* copied this form shredder.mx, keep it here ??? */
if (newsize+newsize > (BUN) REMAP_PAGE_MAXSIZE) { /* try to use mmap() */
- BATmmap(dbat->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(dbat->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP,
0);
}
#endif
diff -r a2d1f5c5b38d -r a8cfb61a0d46 pathfinder/runtime/shredder.mx
--- a/pathfinder/runtime/shredder.mx Fri Oct 01 16:52:26 2010 +0200
+++ b/pathfinder/runtime/shredder.mx Fri Oct 01 18:09:54 2010 +0200
@@ -457,7 +457,7 @@
return GDK_SUCCEED;
}
if (newsize+newsize > (BUN) REMAP_PAGE_MAXSIZE) { /* try to use mmap() */
- BATmmap(sb->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP);
+ BATmmap(sb->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
}
if (!(sb->bat = BATextend(sb->bat, newsize))) {
GDKerror("shredBAT_extend: BATextend[\"%s\"](%d to %d) fails\n",
sb->def->name, sb->size, newsize);
@@ -504,7 +504,7 @@
}
newsize = sb->bat->T->heap.free & (REMAP_PAGE_MAXSIZE-1);
if (oldsize > newsize) { /* try to use mmap() */
- BATmmap(sb->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP,
GDK_ELIMDOUBLES(sb->bat->T->vheap)?STORE_MEM:STORE_MMAP);
+ BATmmap(sb->bat, STORE_MMAP, STORE_MMAP, STORE_MMAP,
GDK_ELIMDOUBLES(sb->bat->T->vheap)?STORE_MEM:STORE_MMAP, 0);
}
return BATcount(sb->bat)-1;
}
diff -r a2d1f5c5b38d -r a8cfb61a0d46 sql/src/storage/bat/bat_storage.c
--- a/sql/src/storage/bat/bat_storage.c Fri Oct 01 16:52:26 2010 +0200
+++ b/sql/src/storage/bat/bat_storage.c Fri Oct 01 18:09:54 2010 +0200
@@ -254,7 +254,7 @@
} else if (!isEbat(b)){
/* try to use mmap() */
if (BATcount(b)+BATcount(i) > (BUN) REMAP_PAGE_MAXSIZE) {
- BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP);
+ BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP, 1);
}
assert(b->T->heap.storage != STORE_PRIV);
BATappend(b, i, TRUE);
@@ -496,7 +496,7 @@
if (BATcount(b) > SNAPSHOT_MINSIZE)
BATmode(b, PERSISTENT);
if (BATcount(b) > (BUN) REMAP_PAGE_MAXSIZE)
- BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP);
+ BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP, 0);
bat_destroy(b);
return ok;
}
@@ -788,7 +788,7 @@
if (BATcount(b) > SNAPSHOT_MINSIZE)
BATmode(b, PERSISTENT);
if (BATcount(b) > (BUN) REMAP_PAGE_MAXSIZE)
- BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP);
+ BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP,
STORE_MMAP, 0);
bat_destroy(b);
return LOG_OK;
}
@@ -1137,7 +1137,7 @@
/* any inserts */
if (BUNlast(ins) > BUNfirst(ins)) {
if (BATcount(cur)+BATcount(ins) > (BUN) REMAP_PAGE_MAXSIZE) {
/* try to use mmap() */
- BATmmap(cur, STORE_MMAP, STORE_MMAP,
STORE_MMAP, STORE_MMAP);
+ BATmmap(cur, STORE_MMAP, STORE_MMAP,
STORE_MMAP, STORE_MMAP, 1);
}
assert(cur->T->heap.storage != STORE_PRIV);
BATappend(cur,ins,TRUE);
@@ -1320,7 +1320,7 @@
cur = newcur;
} else {
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
