Changeset: 4c7c8a446bc9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4c7c8a446bc9
Modified Files:
gdk/gdk_align.c
gdk/gdk_atoms.mx
gdk/gdk_bat.mx
gdk/gdk_bbp.mx
gdk/gdk_heap.c
gdk/gdk_private.h
gdk/gdk_search.mx
Branch: Apr2012
Log Message:
Experimental checkin: lower limit for all heaps of persistent bats.
This works on Linux, but I want to find out whether this works on
other systems as well.
This is a potential fix for bug 3067: at least on Linux, this seems to
have the desired effect.
What this change does is to lower the limit for when we use memory
mapped files for heaps (in the case of the bug, the heap for hashes)
to REMAP_PAGE_SIZE (16 kiB on Linux, 64 kiB on Windows) from something
much, much larger. This has the effect that, even though the test
described in the bug report uses lots of hash tables that are not
thrown away, they do not use swap space, and will hopefully not cause
thrashing.
diffs (169 lines):
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -470,7 +470,7 @@ BATmaterializeh(BAT *b)
HASHdestroy(b);
b->H->heap.filename = NULL;
- if (HEAPalloc(&b->H->heap, cnt, sizeof(oid)) < 0) {
+ if (HEAPalloc(&b->H->heap, cnt, sizeof(oid), b->batPersistence ==
PERSISTENT) < 0) {
b->H->heap = head;
return NULL;
}
@@ -618,7 +618,7 @@ VIEWreset(BAT *b)
if (head.filename == NULL)
goto bailout;
snprintf(head.filename, nmelen + 12, "%s.head", nme);
- if (n->htype && HEAPalloc(&head, cnt, Hsize(n)) < 0)
+ if (n->htype && HEAPalloc(&head, cnt, Hsize(n),
b->batPersistence == PERSISTENT) < 0)
goto bailout;
}
if (n->ttype) {
@@ -626,7 +626,7 @@ VIEWreset(BAT *b)
if (tail.filename == NULL)
goto bailout;
snprintf(tail.filename, nmelen + 12, "%s.tail", nme);
- if (n->ttype && HEAPalloc(&tail, cnt, Tsize(n)) < 0)
+ if (n->ttype && HEAPalloc(&tail, cnt, Tsize(n),
b->batPersistence == PERSISTENT) < 0)
goto bailout;
}
if (n->H->vheap) {
diff --git a/gdk/gdk_atoms.mx b/gdk/gdk_atoms.mx
--- a/gdk/gdk_atoms.mx
+++ b/gdk/gdk_atoms.mx
@@ -1351,7 +1351,7 @@ strHeap(Heap *d, size_t cap)
cap = MAX(cap, BATTINY);
size = GDK_STRHASHTABLE * sizeof(stridx_t) + MIN(GDK_ELIMLIMIT, cap *
GDK_VARALIGN);
- if (HEAPalloc(d, size, 1) >= 0) {
+ if (HEAPalloc(d, size, 1, 0) >= 0) {
d->free = GDK_STRHASHTABLE * sizeof(stridx_t);
memset(d->base, 0, d->free);
d->hashash = 1; /* new string heaps get the hash value (and
length) stored */
diff --git a/gdk/gdk_bat.mx b/gdk/gdk_bat.mx
--- a/gdk/gdk_bat.mx
+++ b/gdk/gdk_bat.mx
@@ -274,10 +274,10 @@ BATnewstorage(int ht, int tt, BUN cap)
bn->U->capacity = cap;
/* alloc the main heaps */
- if (ht && HEAPalloc(&bn->H->heap, cap, bn->H->width) < 0) {
+ if (ht && HEAPalloc(&bn->H->heap, cap, bn->H->width, 0) < 0) {
return NULL;
}
- if (tt && HEAPalloc(&bn->T->heap, cap, bn->T->width) < 0) {
+ if (tt && HEAPalloc(&bn->T->heap, cap, bn->T->width, 0) < 0) {
if (ht)
HEAPfree(&bn->H->heap);
return NULL;
@@ -3453,7 +3453,7 @@ BATpropcheck(BAT *b, int mode)
if (hp->filename != NULL)
snprintf(hp->filename, nmelen + 12,
"%s.%chash", nme, bv->batCacheid > 0 ? 'h' : 't');
if (hp->filename == NULL ||
- (bv->H->hash = HASHnew(hp, tpe, yy + lim,
HASHmask(lim))) == NULL) {
+ (bv->H->hash = HASHnew(hp, tpe, yy + lim,
HASHmask(lim), 0)) == NULL) {
GDKfree(hp->filename);
GDKfree(hp);
BATDEBUG THRprintf(GDKout,
"#BATpropcheck: BAT %s(%d): could not allocate hash table for key test\n",
BATgetId(b), b->batCacheid);
diff --git a/gdk/gdk_bbp.mx b/gdk/gdk_bbp.mx
--- a/gdk/gdk_bbp.mx
+++ b/gdk/gdk_bbp.mx
@@ -588,7 +588,7 @@ fixoidheapcolumn(BAT *b, const char *src
w = b->H->width; /* remember old width */
b->H->width = 1;
b->H->shift = 0;
- if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID) < 0)
+ if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID, 1) < 0)
GDKfatal("fixoidheap: allocating new %s heap "
"for BAT %d failed\n", headtail, bid);
@@ -651,7 +651,7 @@ fixoidheapcolumn(BAT *b, const char *src
b->H->width = SIZEOF_OID;
b->H->shift = 3;
assert(b->H->width == (1 << b->H->shift));
- if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID) < 0)
+ if (HEAPalloc(&b->H->heap, b->U->capacity, SIZEOF_OID, 1) < 0)
GDKfatal("fixoidheap: allocating new %s heap "
"for BAT %d failed\n", headtail, bid);
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -249,9 +249,10 @@ HEAPmargin(size_t maxsize)
/* in 64-bits space, use very large margins to accommodate reallocations */
int
-HEAPalloc(Heap *h, size_t nitems, size_t itemsize)
+HEAPalloc(Heap *h, size_t nitems, size_t itemsize, int persistent)
{
char nme[PATHLENGTH], *ext = NULL;
+ size_t minsize = persistent ? REMAP_PAGE_SIZE : GDK_mmap_minsize;
if (h->filename) {
strcpy(nme, h->filename);
@@ -268,7 +269,7 @@ HEAPalloc(Heap *h, size_t nitems, size_t
if (itemsize && nitems > (h->size / itemsize))
return -1;
- if (h->filename == NULL || (h->size < GDK_mmap_minsize)) {
+ if (h->filename == NULL || (h->size < minsize)) {
h->storage = STORE_MEM;
h->base = (char *) GDKmallocmax(h->size, &h->maxsize, 0);
ALLOCDEBUG fprintf(stderr, "#HEAPalloc " SZFMT " " SZFMT " "
PTRFMT "\n", h->size, h->maxsize, PTRFMTCAST h->base);
@@ -531,7 +532,7 @@ GDKupgradevarheap(COLrec *c, var_t v, in
int
HEAPcopy(Heap *dst, Heap *src)
{
- if (HEAPalloc(dst, src->size, 1) == 0) {
+ if (HEAPalloc(dst, src->size, 1, 0) == 0) {
dst->free = src->free;
memcpy(dst->base, src->base, src->free);
dst->hashash = src->hashash;
@@ -938,7 +939,7 @@ HEAP_initialize(Heap *heap, size_t nbyte
size_t total = 100 + nbytes + nprivate + sizeof(HEADER) +
sizeof(CHUNK);
total = roundup_8(total);
- if (HEAPalloc(heap, total, 1) < 0)
+ if (HEAPalloc(heap, total, 1, 0) < 0)
return;
heap->free = heap->size;
}
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -91,8 +91,8 @@ void GDKvminc(size_t len);
void *GDKvmrealloc(void *pold, size_t oldsize, size_t newsize, size_t oldmax,
size_t *maxsize, int emergency);
int HASHgonebad(BAT *b, ptr v);
BUN HASHmask(BUN cnt);
-Hash *HASHnew(Heap *hp, int tpe, BUN size, BUN mask);
-int HEAPalloc(Heap *h, size_t nitems, size_t itemsize);
+Hash *HASHnew(Heap *hp, int tpe, BUN size, BUN mask, int persistent);
+int HEAPalloc(Heap *h, size_t nitems, size_t itemsize, int persistent);
void HEAPcacheInit(void);
int HEAP_check(Heap *h, HeapRepair *hr);
int HEAPdelete(Heap *h, const char *o, const char *ext);
diff --git a/gdk/gdk_search.mx b/gdk/gdk_search.mx
--- a/gdk/gdk_search.mx
+++ b/gdk/gdk_search.mx
@@ -312,10 +312,10 @@ HASHclear(Hash *h)
}
Hash *
-HASHnew(Heap *hp, int tpe, BUN size, BUN mask)
+HASHnew(Heap *hp, int tpe, BUN size, BUN mask, int persistent)
{
Hash *h = NULL;
- if (HEAPalloc(hp, mask + size, sizeof(BUN)) < 0)
+ if (HEAPalloc(hp, mask + size, sizeof(BUN), persistent) < 0)
return NULL;
h = (Hash *) GDKmalloc(sizeof(Hash));
if (!h)
@@ -441,7 +441,7 @@ BAThash(BAT *b, BUN masksize)
sprintf(hp->filename, "%s.%chash", nme,
b->batCacheid > 0 ? 'h' : 't');
if (hp == NULL ||
hp->filename == NULL ||
- (h = HASHnew(hp, ATOMtype(b->htype),
BATcapacity(b), mask)) == NULL) {
+ (h = HASHnew(hp, ATOMtype(b->htype),
BATcapacity(b), mask, b->batPersistence == PERSISTENT)) == NULL) {
gdk_unset_lock(GDKhashLock(ABS(b->batCacheid) &
BBP_BATMASK), "BAThash");
if (hp != NULL) {
GDKfree(hp->filename);
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
