Changeset: bb73d36663f1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb73d36663f1
Modified Files:
clients/Tests/exports.stable.out
common/utils/muuid.c
gdk/gdk.h
gdk/gdk_atoms.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_posix.h
gdk/gdk_private.h
gdk/gdk_setop.c
gdk/gdk_storage.c
gdk/gdk_utils.c
monetdb5/mal/mal_parser.c
monetdb5/modules/kernel/status.c
monetdb5/modules/mal/bbp.c
monetdb5/modules/mal/mat.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/vaults/fits.c
sql/benchmarks/tpch/Tests/16-explain.stable.out
sql/benchmarks/tpch/Tests/16-plan.stable.out
sql/benchmarks/tpch/fileleak/Tests/delete_all.stable.out
sql/benchmarks/tpch/fileleak/Tests/leaks.stable.out
sql/server/rel_optimizer.c
sql/storage/bat/bat_storage.c
sql/storage/restrict/restrict_storage.c
sql/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out
Branch: default
Log Message:
Merge with Jan2014 branch.
diffs (truncated from 1373 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -253,9 +253,7 @@ BAT *BUNinplace(BAT *b, BUN p, const voi
BAT *BUNins(BAT *b, const void *left, const void *right, bit force);
BUN BUNlocate(BAT *b, const void *left, const void *right);
BAT *BUNreplace(BAT *b, const void *left, const void *right, bit force);
-size_t GDK_mem_bigsize;
size_t GDK_mem_maxsize;
-int GDK_mem_pagebits;
size_t GDK_vm_maxsize;
int GDK_vm_trim;
int GDKatomcnt;
@@ -292,7 +290,7 @@ ssize_t GDKstrFromStr(unsigned char *dst
str GDKstrdup(const char *s);
int GDKsyserror(_In_z_ _Printf_format_string_ const char *format, ...)
__attribute__((__format__(__printf__, 1, 2)));
ThreadRec GDKthreads[THREADS];
-int GDKupgradevarheap(COLrec *c, var_t v, int copyall);
+int GDKupgradevarheap(COLrec *c, var_t v, int copyall, int mayshare);
lng GDKusec(void);
BAT *GDKval;
const char *GDKversion(void);
@@ -305,7 +303,7 @@ void HEAP_free(Heap *heap, var_t block);
void HEAP_initialize(Heap *heap, size_t nbytes, size_t nprivate, int
alignment);
var_t HEAP_malloc(Heap *heap, size_t nbytes);
int HEAPcopy(Heap *dst, Heap *src);
-int HEAPextend(Heap *h, size_t size);
+int HEAPextend(Heap *h, size_t size, int mayshare);
int HEAPfree(Heap *h);
size_t HEAPmemsize(Heap *h);
size_t HEAPvmsize(Heap *h);
@@ -319,7 +317,6 @@ void MT_exiting_thread(void);
MT_Id MT_getpid(void);
size_t MT_getrss(void);
void MT_global_exit(int status) __attribute__((__noreturn__));
-char *MT_heapbase;
void MT_init(void);
int MT_join_thread(MT_Id t);
int MT_kill_thread(MT_Id t);
diff --git a/common/utils/muuid.c b/common/utils/muuid.c
--- a/common/utils/muuid.c
+++ b/common/utils/muuid.c
@@ -47,22 +47,16 @@ generateUUID(void)
uuid_unparse(uuid, out);
#else
/* try to do some pseudo interesting stuff, and stash it in the
- * format of an UUID to at least return some uniform answer */
+ * format of a UUID to at least return some uniform answer */
char out[37];
- char *p = out;
/* generate something like this:
* cefa7a9c-1dd2-11b2-8350-880020adbeef ("%08x-%04x-%04x-%04x-%012x") */
- p += snprintf(p, 5, "%04x", rand() % 65536);
- p += snprintf(p, 6, "%04x-", rand() % 65536);
- p += snprintf(p, 6, "%04x-", rand() % 65536);
- p += snprintf(p, 6, "%04x-", rand() % 65536);
- p += snprintf(p, 6, "%04x-", rand() % 65536);
- p += snprintf(p, 5, "%04x", rand() % 65536);
- p += snprintf(p, 5, "%04x", rand() % 65536);
- p += snprintf(p, 5, "%04x", rand() % 65536);
+ snprintf(out, sizeof(out), "%04x%04x-%04x-%04x-%04x-%04x%04x%04x",
+ rand() % 65536, rand() % 65536,
+ rand() % 65536, rand() % 65536,
+ rand() % 65536, rand() % 65536,
+ rand() % 65536, rand() % 65536);
#endif
- return(strdup(out));
+ return strdup(out);
}
-
-/* vim:set ts=4 sw=4 noexpandtab: */
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -990,7 +990,7 @@ typedef int (*GDKfcn) ();
* HEAPfree (Heap *h);
* @item int
* @tab
- * HEAPextend (Heap *h, size_t size);
+ * HEAPextend (Heap *h, size_t size, int mayshare);
* @item int
* @tab
* HEAPload (Heap *h, str nme,ext, int trunc);
@@ -1013,7 +1013,7 @@ typedef int (*GDKfcn) ();
* isolate you from the different ways heaps can be accessed.
*/
gdk_export int HEAPfree(Heap *h);
-gdk_export int HEAPextend(Heap *h, size_t size);
+gdk_export int HEAPextend(Heap *h, size_t size, int mayshare);
gdk_export int HEAPcopy(Heap *dst, Heap *src);
gdk_export size_t HEAPvmsize(Heap *h);
gdk_export size_t HEAPmemsize(Heap *h);
@@ -1187,7 +1187,7 @@ gdk_export bte ATOMelmshift(int sz);
if ((b)->HT->width < SIZEOF_VAR_T && \
((b)->HT->width <= 2 ? _d - GDK_VAROFFSET : _d) >=
((size_t) 1 << (8 * (b)->HT->width))) { \
/* doesn't fit in current heap, upgrade it */ \
- if (GDKupgradevarheap((b)->HT, _d, (copyall))
== GDK_FAIL) \
+ if (GDKupgradevarheap((b)->HT, _d, (copyall),
(b)->batRestricted == BAT_READ) == GDK_FAIL) \
goto bunins_failed; \
} \
_ptr = (p); \
@@ -1235,7 +1235,7 @@ gdk_export bte ATOMelmshift(int sz);
if ((b)->HT->width < SIZEOF_VAR_T && \
((b)->HT->width <= 2 ? _d - GDK_VAROFFSET : _d) >=
((size_t) 1 << (8 * (b)->HT->width))) { \
/* doesn't fit in current heap, upgrade it */ \
- if (GDKupgradevarheap((b)->HT, _d, 0) ==
GDK_FAIL) \
+ if (GDKupgradevarheap((b)->HT, _d, 0,
(b)->batRestricted == BAT_READ) == GDK_FAIL) \
goto bunins_failed; \
} \
_ptr = (p); \
@@ -1297,7 +1297,7 @@ gdk_export bte ATOMelmshift(int sz);
#define bunfastins_check(b, p, h, t) bunfastins(b, h, t)
-gdk_export int GDKupgradevarheap(COLrec *c, var_t v, int copyall);
+gdk_export int GDKupgradevarheap(COLrec *c, var_t v, int copyall, int
mayshare);
gdk_export BAT *BUNfastins(BAT *b, const void *left, const void *right);
gdk_export BAT *BUNins(BAT *b, const void *left, const void *right, bit force);
gdk_export BAT *BUNappend(BAT *b, const void *right, bit force);
@@ -1590,12 +1590,6 @@ gdk_export gdk_return BATgroup(BAT **gro
* for the expected usage pattern of the memory in a heap.
*/
-gdk_export int GDK_mem_pagebits; /* page size for non-linear mmaps */
-
-#define REMAP_PAGE_BITS GDK_mem_pagebits
-#define REMAP_PAGE_MAXBITS (REMAP_PAGE_BITS+3)
-#define REMAP_PAGE_MAXSIZE ((size_t) 1 << REMAP_PAGE_MAXBITS) /* max page
bytesize of unary BUN heap (8-byte atom) */
-
/* Buffer management advice for heaps */
#define BUF_NORMAL 0 /* No further special treatment */
#define BUF_RANDOM 1 /* Expect random page references */
@@ -2189,7 +2183,6 @@ gdk_export BAT *BATimprints(BAT *b);
/* we prefer to use vm_alloc routines on size > GDKmmap */
gdk_export void *GDKmmap(const char *path, int mode, size_t len);
-gdk_export size_t GDK_mem_bigsize; /* size after which we use anonymous VM
rather than malloc */
gdk_export size_t GDK_mem_maxsize; /* max allowed size of committed memory
*/
gdk_export size_t GDK_vm_maxsize; /* max allowed size of reserved vm */
gdk_export int GDK_vm_trim; /* allow trimming */
diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -942,7 +942,7 @@ strHeap(Heap *d, size_t cap)
cap = MAX(cap, BATTINY);
size = GDK_STRHASHTABLE * sizeof(stridx_t) + MIN(GDK_ELIMLIMIT, cap *
GDK_VARALIGN);
- if (HEAPalloc(d, size, 1) >= 0) {
+ if (HEAPalloc(d, size, 1) == 0) {
d->free = GDK_STRHASHTABLE * sizeof(stridx_t);
memset(d->base, 0, d->free);
d->hashash = 1; /* new string heaps get the hash value (and
length) stored */
@@ -1119,7 +1119,7 @@ strPut(Heap *h, var_t *dst, const char *
return 0;
}
HEAPDEBUG fprintf(stderr, "#HEAPextend in strPut %s " SZFMT " "
SZFMT "\n", h->filename, h->size, newsize);
- if (HEAPextend(h, newsize) < 0) {
+ if (HEAPextend(h, newsize, TRUE) < 0) {
return 0;
}
#ifndef NDEBUG
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -364,7 +364,7 @@ BATattach(int tt, const char *heapfile)
}
bn->batRestricted = BAT_READ;
bn->T->heap.size = (size_t) st.st_size;
- bn->T->heap.newstorage = bn->T->heap.storage = (bn->T->heap.size <
REMAP_PAGE_MAXSIZE) ? STORE_MEM : STORE_MMAP;
+ bn->T->heap.newstorage = bn->T->heap.storage = (bn->T->heap.size <
GDK_mmap_minsize) ? STORE_MEM : STORE_MMAP;
if (HEAPload(&bn->T->heap, BBP_physical(bn->batCacheid), "tail", TRUE)
< 0) {
HEAPfree(&bn->T->heap);
GDKfree(bs);
@@ -479,12 +479,14 @@ BATextend(BAT *b, BUN newcap)
hheap_size *= Hsize(b);
if (b->H->heap.base && GDKdebug & HEAPMASK)
fprintf(stderr, "#HEAPextend in BATextend %s " SZFMT " " SZFMT
"\n", b->H->heap.filename, b->H->heap.size, hheap_size);
- if (b->H->heap.base && HEAPextend(&b->H->heap, hheap_size) < 0)
+ if (b->H->heap.base &&
+ HEAPextend(&b->H->heap, hheap_size, b->batRestricted == BAT_READ) <
0)
return NULL;
theap_size *= Tsize(b);
if (b->T->heap.base && GDKdebug & HEAPMASK)
fprintf(stderr, "#HEAPextend in BATextend %s " SZFMT " " SZFMT
"\n", b->T->heap.filename, b->T->heap.size, theap_size);
- if (b->T->heap.base && HEAPextend(&b->T->heap, theap_size) < 0)
+ if (b->T->heap.base &&
+ HEAPextend(&b->T->heap, theap_size, b->batRestricted == BAT_READ) <
0)
return NULL;
HASHdestroy(b);
IMPSdestroy(b);
@@ -837,13 +839,13 @@ BATcopy(BAT *b, int ht, int tt, int writ
if (bn->hvarsized && bn->htype) {
bn->H->shift = b->H->shift;
bn->H->width = b->H->width;
- if (HEAPextend(&bn->H->heap, BATcapacity(bn) <<
bn->H->shift) < 0)
+ if (HEAPextend(&bn->H->heap, BATcapacity(bn) <<
bn->H->shift, TRUE) < 0)
goto bunins_failed;
}
if (bn->tvarsized && bn->ttype) {
bn->T->shift = b->T->shift;
bn->T->width = b->T->width;
- if (HEAPextend(&bn->T->heap, BATcapacity(bn) <<
bn->T->shift) < 0)
+ if (HEAPextend(&bn->T->heap, BATcapacity(bn) <<
bn->T->shift, TRUE) < 0)
goto bunins_failed;
}
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -63,7 +63,7 @@
* using BATins (BATdel).
*/
static BAT *
-insert_string_bat(BAT *b, BAT *n, int append)
+insert_string_bat(BAT *b, BAT *n, int append, int force)
{
BATiter ni; /* iterator */
int tt; /* tail type */
@@ -110,7 +110,9 @@ insert_string_bat(BAT *b, BAT *n, int ap
/* make sure we get alignment right */
toff = (toff + GDK_VARALIGN - 1) & ~(GDK_VARALIGN - 1);
assert(((toff >> GDK_VARSHIFT) << GDK_VARSHIFT) == toff);
- if (HEAPextend(b->T->vheap, toff + n->T->vheap->size) < 0) {
+ /* if in "force" mode, the heap may be shared when
+ * memory mapped */
+ if (HEAPextend(b->T->vheap, toff + n->T->vheap->size, force) <
0) {
toff = ~(size_t) 0;
goto bunins_failed;
}
@@ -121,7 +123,7 @@ insert_string_bat(BAT *b, BAT *n, int ap
if (b->T->width < SIZEOF_VAR_T &&
((size_t) 1 << 8 * b->T->width) < (b->T->width <= 2 ?
(b->T->vheap->size >> GDK_VARSHIFT) - GDK_VAROFFSET : (b->T->vheap->size >>
GDK_VARSHIFT))) {
/* offsets aren't going to fit */
- if (GDKupgradevarheap(b->T, (var_t) (b->T->vheap->size
>> GDK_VARSHIFT), 0) == GDK_FAIL) {
+ if (GDKupgradevarheap(b->T, (var_t) (b->T->vheap->size
>> GDK_VARSHIFT), 0, force) == GDK_FAIL) {
toff = ~(size_t) 0;
goto bunins_failed;
}
@@ -398,7 +400,7 @@ BATins(BAT *b, BAT *n, bit force)
!GDK_ELIMDOUBLES(n->T->vheap) &&
b->T->vheap->hashash == n->T->vheap->hashash &&
VIEWtparent(n) == 0)) {
- b = insert_string_bat(b, n, 0);
+ b = insert_string_bat(b, n, 0, force);
} else if (b->htype == TYPE_void) {
if (!ATOMvarsized(b->ttype) &&
BATatoms[b->ttype].atomFix == NULL &&
@@ -600,7 +602,7 @@ BATappend(BAT *b, BAT *n, bit force)
!GDK_ELIMDOUBLES(n->T->vheap) &&
b->T->vheap->hashash == n->T->vheap->hashash &&
VIEWtparent(n) == 0) {
- b = insert_string_bat(b, n, 1);
+ b = insert_string_bat(b, n, 1, force);
if (b == NULL)
return NULL;
} else if (b->htype == TYPE_void) {
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -60,19 +60,16 @@
#include "gdk_private.h"
static void *
-HEAPcreatefile(size_t *maxsz, char *fn, storage_t mode)
+HEAPcreatefile(size_t *maxsz, const char *fn)
{
- size_t size = *maxsz;
void *base = NULL;
int fd;
- size = (*maxsz + (size_t) 0xFFFF) & ~ (size_t) 0xFFFF; /* round up to
64k */
+ /* round up to mulitple of GDK_mmap_pagesize */
fd = GDKfdlocate(fn, "wb", NULL);
if (fd >= 0) {
close(fd);
- base = GDKload(fn, NULL, size, size, mode);
- if (base)
- *maxsz = size;
+ base = GDKload(fn, NULL, *maxsz, maxsz, STORE_MMAP);
}
return base;
}
@@ -118,18 +115,6 @@ HEAPalloc(Heap *h, size_t nitems, size_t
if (itemsize && nitems > (h->size / itemsize))
return -1;
- if (h->filename) {
- GDKfilepath(nme, BATDIR, h->filename, NULL);
- /* if we're going to use mmap anyway (size >=
- * GDK_mem_bigsize -- see GDKmallocmax), and the file
- * we want to use already exists and is large enough
- * for the size we want, force non-anonymous mmap */
- if (h->size >= GDK_mem_bigsize &&
- stat(nme, &st) == 0 && st.st_size >= (off_t) h->size) {
- minsize = GDK_mem_bigsize; /* force mmap */
- }
- }
-
if (h->filename == NULL || h->size < minsize) {
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
