Changeset: 24d28c0d8af9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/24d28c0d8af9
Modified Files:
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_orderidx.c
gdk/gdk_storage.c
Branch: Jul2021
Log Message:
When extending a heap from malloc to mmap, extend file and force mmap.
diffs (152 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -535,14 +535,14 @@ typedef enum {
/* Heap storage modes */
typedef enum {
- STORE_MEM = 0, /* load into GDKmalloced memory */
- STORE_MMAP = 1, /* mmap() into virtual memory */
- STORE_PRIV = 2, /* BAT copy of copy-on-write mmap */
- STORE_CMEM = 3, /* load into malloc (not GDKmalloc) memory*/
- STORE_NOWN = 4, /* memory not owned by the BAT */
- STORE_MMAPABS = 5, /* mmap() into virtual memory from an
+ STORE_INVALID = 0, /* invalid value, used to indicate error */
+ STORE_MEM, /* load into GDKmalloced memory */
+ STORE_MMAP, /* mmap() into virtual memory */
+ STORE_PRIV, /* BAT copy of copy-on-write mmap */
+ STORE_CMEM, /* load into malloc (not GDKmalloc) memory*/
+ STORE_NOWN, /* memory not owned by the BAT */
+ STORE_MMAPABS, /* mmap() into virtual memory from an
* absolute path (not part of dbfarm) */
- STORE_INVALID /* invalid value, used to indicate error */
} storage_t;
typedef struct {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -495,8 +495,8 @@ heapinit(BAT *b, const char *buf, int *h
b->theap->size = (size_t) size;
b->theap->base = NULL;
settailname(b->theap, filename, t, width);
- b->theap->storage = (storage_t) storage;
- b->theap->newstorage = (storage_t) storage;
+ b->theap->storage = STORE_INVALID;
+ b->theap->newstorage = STORE_INVALID;
b->theap->farmid = BBPselectfarm(PERSISTENT, b->ttype, offheap);
b->theap->dirty = false;
b->theap->parentid = b->batCacheid;
@@ -535,10 +535,10 @@ vheapinit(BAT *b, const char *buf, int h
.free = (size_t) free,
.size = (size_t) size,
.base = NULL,
- .storage = (storage_t) storage,
+ .storage = STORE_INVALID,
.hashash = hashash != 0,
.cleanhash = true,
- .newstorage = (storage_t) storage,
+ .newstorage = STORE_INVALID,
.dirty = false,
.parentid = bid,
.farmid = BBPselectfarm(PERSISTENT, b->ttype, varheap),
@@ -1386,7 +1386,7 @@ heap_entry(FILE *fp, BAT *b, BUN size)
b->tseqbase,
free,
b->theap->size,
- (int) b->theap->newstorage,
+ 0,
minprop && minprop->val.oval < b->hseqbase + size ?
minprop->val.oval : oid_nil,
maxprop && maxprop->val.oval < b->hseqbase + size ?
maxprop->val.oval : oid_nil);
}
@@ -1396,8 +1396,7 @@ vheap_entry(FILE *fp, Heap *h)
{
if (h == NULL)
return 0;
- return fprintf(fp, " %zu %zu %d",
- h->free, h->size, (int) h->newstorage);
+ return fprintf(fp, " %zu %zu %d", h->free, h->size, 0);
}
static gdk_return
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -474,6 +474,10 @@ BATcheckhash(BAT *b)
strconcat_len(h->heapbckt.filename,
sizeof(h->heapbckt.filename),
nme, ".thashb", NULL);
+ h->heaplink.storage = STORE_INVALID;
+ h->heaplink.newstorage = STORE_INVALID;
+ h->heapbckt.storage = STORE_INVALID;
+ h->heapbckt.newstorage = STORE_INVALID;
/* check whether a persisted hash can be found
*/
if ((fd = GDKfdlocate(h->heapbckt.farmid, nme,
"rb+", "thashb")) >= 0) {
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -293,7 +293,12 @@ HEAPextend(Heap *h, size_t size, bool ma
}
fd = GDKfdlocate(h->farmid, nme, "wb", ext);
if (fd >= 0) {
+ gdk_return rc = GDKextendf(fd, size, nme);
close(fd);
+ if (rc != GDK_SUCCEED) {
+ failure = "h->storage == STORE_MEM &&
can_map && fd >= 0 && GDKextendf() != GDK_SUCCEED";
+ goto failed;
+ }
h->storage = h->newstorage == STORE_MMAP &&
existing && !mayshare ? STORE_PRIV : h->newstorage;
/* make sure we really MMAP */
if (must_mmap && h->newstorage == STORE_MEM)
@@ -749,7 +754,8 @@ HEAPload_intern(Heap *h, const char *nme
char *srcpath, *dstpath, *tmp;
int t0;
- h->storage = h->newstorage = h->size < GDK_mmap_minsize_persistent ?
STORE_MEM : STORE_MMAP;
+ if (h->storage == STORE_INVALID || h->newstorage == STORE_INVALID)
+ h->storage = h->newstorage = h->size < (h->farmid == 0 ?
GDK_mmap_minsize_persistent : GDK_mmap_minsize_transient) ? STORE_MEM :
STORE_MMAP;
minsize = (h->size + GDK_mmap_pagesize - 1) & ~(GDK_mmap_pagesize - 1);
if (h->storage != STORE_MEM && minsize != h->size)
diff --git a/gdk/gdk_imprints.c b/gdk/gdk_imprints.c
--- a/gdk/gdk_imprints.c
+++ b/gdk/gdk_imprints.c
@@ -321,6 +321,7 @@ BATcheckimprints(BAT *b)
strconcat_len(imprints->imprints.filename,
sizeof(imprints->imprints.filename),
nme, ".timprints", NULL);
+ imprints->imprints.storage =
imprints->imprints.newstorage = STORE_INVALID;
/* check whether a persisted imprints index
* can be found */
if ((fd =
GDKfdlocate(imprints->imprints.farmid, nme, "rb", "timprints")) >= 0) {
diff --git a/gdk/gdk_orderidx.c b/gdk/gdk_orderidx.c
--- a/gdk/gdk_orderidx.c
+++ b/gdk/gdk_orderidx.c
@@ -93,6 +93,7 @@ BATcheckorderidx(BAT *b)
strconcat_len(hp->filename,
sizeof(hp->filename),
nme, ".torderidx", NULL);
+ hp->storage = hp->newstorage = STORE_INVALID;
/* check whether a persisted orderidx can be
found */
if ((fd = GDKfdlocate(hp->farmid, nme, "rb+",
"torderidx")) >= 0) {
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -922,6 +922,7 @@ BATload_intern(bat bid, bool lock)
/* LOAD bun heap */
if (b->ttype != TYPE_void) {
+ b->theap->storage = b->theap->newstorage = STORE_INVALID;
if (HEAPload(b->theap, b->theap->filename, NULL,
b->batRestricted == BAT_READ) != GDK_SUCCEED) {
HEAPfree(b->theap, false);
return NULL;
@@ -938,6 +939,7 @@ BATload_intern(bat bid, bool lock)
/* LOAD tail heap */
if (ATOMvarsized(b->ttype)) {
+ b->tvheap->storage = b->tvheap->newstorage = STORE_INVALID;
if (HEAPload(b->tvheap, nme, "theap", b->batRestricted ==
BAT_READ) != GDK_SUCCEED) {
HEAPfree(b->theap, false);
HEAPfree(b->tvheap, false);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
