Changeset: 9792bdc29cd0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9792bdc29cd0
Modified Files:
gdk/gdk_heap.c
gdk/gdk_posix.c
gdk/gdk_private.h
gdk/gdk_utils.c
Branch: Feb2013
Log Message:
Completely removed use of anonymous memory maps (GDKvmalloc).
Modern mallocs use anonymous memory maps (mmap/VirtualAlloc)
internally for larger allocations. We don't have to do that
ourselves.
diffs (truncated from 506 to 300 lines):
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -297,7 +297,7 @@ HEAPalloc(Heap *h, size_t nitems, size_t
if (h->filename == NULL || (h->size < minsize)) {
h->storage = STORE_MEM;
h->base = (char *) GDKmallocmax(h->size, &h->maxsize, 0);
- HEAPDEBUG fprintf(stderr, "#HEAPalloc " SZFMT " " SZFMT " "
PTRFMT "%s\n", h->size, h->maxsize, PTRFMTCAST h->base, h->base && ((ssize_t*)
h->base)[-1] < 0 ? " VM" : "");
+ HEAPDEBUG fprintf(stderr, "#HEAPalloc " SZFMT " " SZFMT " "
PTRFMT "\n", h->size, h->maxsize, PTRFMTCAST h->base);
}
if (h->filename && h->base == NULL) {
char *of = h->filename;
@@ -585,7 +585,7 @@ HEAPfree_(Heap *h, int free_file)
{
if (h->base) {
if (h->storage == STORE_MEM) { /* plain memory */
- HEAPDEBUG fprintf(stderr, "#HEAPfree " SZFMT " " SZFMT
" " PTRFMT "%s\n", h->size, h->maxsize, PTRFMTCAST h->base, h->base &&
((ssize_t*) h->base)[-1] < 0 ? " VM" : "");
+ HEAPDEBUG fprintf(stderr, "#HEAPfree " SZFMT " " SZFMT
" " PTRFMT "\n", h->size, h->maxsize, PTRFMTCAST h->base);
GDKfree(h->base);
} else { /* mapped file, or STORE_PRIV */
int ret = HEAPcacheAdd(h->base, h->maxsize,
h->filename, h->storage, free_file);
diff --git a/gdk/gdk_posix.c b/gdk/gdk_posix.c
--- a/gdk/gdk_posix.c
+++ b/gdk/gdk_posix.c
@@ -980,92 +980,6 @@ win_errno(void)
#ifndef WIN32
-#define MT_PAGESIZE(s) ((((s)-1)/MT_pagesize()+1)*MT_pagesize())
-
-#if defined(MAP_ANON) && !defined(MAP_ANONYMOUS)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-#if defined(MAP_ANONYMOUS)
-#define MMAP_FLAGS(f) f|MAP_ANONYMOUS
-#define MMAP_FD -1
-#define MMAP_OPEN_DEV_ZERO int fd = 1
-#define MMAP_CLOSE_DEV_ZERO (void)fd
-#else
-#define MMAP_FLAGS(f) f
-#define MMAP_FD fd
-#define MMAP_OPEN_DEV_ZERO int fd = open("/dev/zero", O_RDWR, MONETDB_MODE)
-#define MMAP_CLOSE_DEV_ZERO close(fd)
-#endif
-
-void *
-MT_vmalloc(size_t size, size_t *maxsize)
-{
- MMAP_OPEN_DEV_ZERO;
- char *q, *r = (char *) -1L;
-
- if (fd < 0) {
- return NULL;
- }
- size = MT_PAGESIZE(size);
- *maxsize = MT_PAGESIZE(*maxsize);
- if (*maxsize > size) {
- r = (char *) mmap(NULL, *maxsize, PROT_NONE,
MMAP_FLAGS(MAP_PRIVATE | MAP_NORESERVE), MMAP_FD, 0);
- }
- if (r == (char *) -1L) {
- *maxsize = size;
- q = (char *) mmap(NULL, size, PROT_READ | PROT_WRITE,
MMAP_FLAGS(MAP_PRIVATE), MMAP_FD, 0);
- } else {
- q = (char *) mmap(r, size, PROT_READ | PROT_WRITE,
MMAP_FLAGS(MAP_PRIVATE | MAP_FIXED), MMAP_FD, 0);
- }
- MMAP_CLOSE_DEV_ZERO;
- return (void *) ((q == (char *) -1L) ? NULL : q);
-}
-
-void
-MT_vmfree(void *p, size_t size)
-{
- size = MT_PAGESIZE(size);
- munmap(p, size);
-}
-
-void *
-MT_vmrealloc(void *voidptr, size_t oldsize, size_t newsize, size_t oldmaxsize,
size_t *newmaxsize)
-{
- char *p = (char *) voidptr;
- char *q = (char *) -1L;
-
- /* sanitize sizes */
- oldsize = MT_PAGESIZE(oldsize);
- newsize = MT_PAGESIZE(newsize);
- oldmaxsize = MT_PAGESIZE(oldmaxsize);
- *newmaxsize = MT_PAGESIZE(*newmaxsize);
- if (*newmaxsize < newsize) {
- *newmaxsize = newsize;
- }
-
- if (oldsize > newsize) {
- munmap(p + oldsize, oldsize - newsize);
- } else if (oldsize < newsize) {
- if (newsize < oldmaxsize) {
- MMAP_OPEN_DEV_ZERO;
- if (fd >= 0) {
- q = (char *) mmap(p + oldsize, newsize -
oldsize, PROT_READ | PROT_WRITE, MMAP_FLAGS(MAP_PRIVATE | MAP_FIXED), MMAP_FD,
(off_t) oldsize);
- MMAP_CLOSE_DEV_ZERO;
- }
- }
- if (q == (char *) -1L) {
- q = (char *) MT_vmalloc(newsize, newmaxsize);
- if (q != NULL) {
- memcpy(q, p, oldsize);
- MT_vmfree(p, oldmaxsize);
- return q;
- }
- }
- }
- *newmaxsize = MAX(oldmaxsize, newsize);
- return p;
-}
-
void
MT_sleep_ms(unsigned int ms)
{
@@ -1087,93 +1001,6 @@ MT_sleep_ms(unsigned int ms)
#else /* WIN32 */
-#define MT_PAGESIZE(s) (((((s)-1) >> 12) + 1) << 12)
-#define MT_SEGSIZE(s) ((((((s)-1) >> 16) & 65535) + 1) << 16)
-
-#ifndef MEM_TOP_DOWN
-#define MEM_TOP_DOWN 0
-#endif
-
-void *
-MT_vmalloc(size_t size, size_t *maxsize)
-{
- void *p, *a = NULL;
- int mode = 0;
-
- size = MT_PAGESIZE(size);
- if (*maxsize < size) {
- *maxsize = size;
- }
- *maxsize = MT_SEGSIZE(*maxsize);
- if (*maxsize < 1000000) {
- mode = MEM_TOP_DOWN; /* help NT in keeping memory
defragmented */
- }
- (void) pthread_mutex_lock(&MT_mmap_lock);
- if (*maxsize > size) {
- a = (void *) VirtualAlloc(NULL, *maxsize, MEM_RESERVE | mode,
PAGE_NOACCESS);
- if (a == NULL) {
- *maxsize = size;
- }
- }
- p = (void *) VirtualAlloc(a, size, MEM_COMMIT | mode, PAGE_READWRITE);
- (void) pthread_mutex_unlock(&MT_mmap_lock);
- if (p == NULL) {
- mnstr_printf(GDKstdout, "#VirtualAlloc(" PTRFMT "," SZFMT
",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size);
- }
- return p;
-}
-
-
-void
-MT_vmfree(void *p, size_t size)
-{
- if (VirtualFree(p, size, MEM_DECOMMIT) == 0)
- mnstr_printf(GDKstdout, "#VirtualFree(" PTRFMT "," SZFMT
",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size);
- if (VirtualFree(p, 0, MEM_RELEASE) == 0)
- mnstr_printf(GDKstdout, "#VirtualFree(" PTRFMT
",0,MEM_RELEASE): failed\n", PTRFMTCAST p);
-}
-
-void *
-MT_vmrealloc(void *v, size_t oldsize, size_t newsize, size_t oldmaxsize,
size_t *newmaxsize)
-{
- char *p = (char *) v, *a = p;
-
- /* sanitize sizes */
- oldsize = MT_PAGESIZE(oldsize);
- newsize = MT_PAGESIZE(newsize);
- oldmaxsize = MT_PAGESIZE(oldmaxsize);
- *newmaxsize = MT_PAGESIZE(*newmaxsize);
- if (*newmaxsize < newsize) {
- *newmaxsize = newsize;
- }
-
- if (oldsize > newsize) {
- size_t ret = VirtualFree(p + newsize, oldsize - newsize,
MEM_DECOMMIT);
-
- if (ret == 0)
- mnstr_printf(GDKstdout, "#VirtualFree(" PTRFMT ","
SSZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize
- newsize));
- } else if (oldsize < newsize) {
- (void) pthread_mutex_lock(&MT_mmap_lock);
- a = (char *) VirtualAlloc(p, newsize, MEM_COMMIT,
PAGE_READWRITE);
- (void) pthread_mutex_unlock(&MT_mmap_lock);
- if (a != p) {
- char *q = a;
-
- if (a == NULL) {
- q = MT_vmalloc(newsize, newmaxsize);
- }
- if (q != NULL) {
- memcpy(q, p, oldsize);
- MT_vmfree(p, oldmaxsize);
- }
- if (a == NULL)
- return q;
- }
- }
- *newmaxsize = MAX(oldmaxsize, newsize);
- return a;
-}
-
void
MT_sleep_ms(unsigned int ms)
{
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -92,9 +92,6 @@ void MT_global_exit(int status)
__attribute__((__noreturn__));
void MT_init_posix(void);
int MT_msync(void *p, size_t off, size_t len, int mode);
-void *MT_vmalloc(size_t size, size_t *maxsize);
-void MT_vmfree(void *p, size_t size);
-void *MT_vmrealloc(void *voidptr, size_t oldsize, size_t newsize, size_t
oldmaxsize, size_t *newmaxsize);
int OIDdirty(void);
int OIDinit(void);
oid *oidRead(oid *a, stream *s, size_t cnt);
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -652,15 +652,6 @@ GDKmemdump(void)
* log2 of the block size (similarly for virtual. memory blocks;
* define GDK_VM_KEEPHISTO).
*
- * - redirection to anonymous VM
- * Sometimes, fragmentation problems arise on mallocs that are not
- * tunable. In that case, it makes sense to redirect large block
- * requests (taken to be larger than GDK_mem_bigsize) to anonymous
- * virtual memory. The seamless implementation of this stores the
- * @strong{negative} block size in front of the pointer (as well as
- * the VM maxsize), so the malloc primitives can recognize these
- * redirected blocks.
- *
* 64-bits update: Some 64-bit implementations (Linux) of mallinfo is
* severely broken, as they use int-s for memory sizes!! This causes
* corruption of mallinfo stats. As we depend on those, we should keep
@@ -724,70 +715,6 @@ GDKmemfail(str s, size_t len)
} while (0)
/*
- * @- VM alloc
- * this affects both physical and logical memory resources.
- * The emergency flag can be set to force a fatal error if needed.
- * Otherwise, the caller is able to deal with the lack of memory.
- */
-static void *
-GDKvmalloc(size_t size, size_t *maxsize, int emergency)
-{
- void *ret = MT_vmalloc(size, maxsize);
-
- if (ret == NULL) {
- GDKmemfail("GDKvmalloc", size);
- ret = MT_vmalloc(size, maxsize);
- if (ret == NULL) {
- if (!emergency)
- return NULL;
- GDKfatal("GDKvmalloc: failed for " SZFMT " bytes",
- size);
- } else {
- THRprintf(GDKstdout, "#GDKvmalloc(" SZFMT "): "
- "recovery ok. Continuing..\n", size);
- }
- }
- if (ret != NULL) {
- meminc(*maxsize, "GDKvmalloc");
- }
- return ret;
-}
-
-static void *
-GDKvmrealloc(void *pold, size_t oldsize, size_t newsize,
- size_t oldmax, size_t *newmax, int emergency)
-{
- void *ret = MT_vmrealloc(pold, oldsize, newsize, oldmax, newmax);
-
- if (ret == NULL) {
- GDKmemfail("GDKvmrealloc", newsize);
- ret = MT_vmrealloc(pold, oldsize, newsize, oldmax, newmax);
- if (ret == NULL) {
- if (!emergency)
- return NULL;
- GDKfatal("GDKvmrealloc: failed for " SZFMT " bytes "
- "(from " SZFMT ")", newsize, oldsize);
- } else {
- THRprintf(GDKstdout, "#GDKvmrealloc(" SZFMT "): "
- "recovery ok. Continuing..\n", newsize);
- }
- }
- if (ret != NULL) {
- memdec(oldmax, "GDKvmrealloc");
- meminc(*newmax, "GDKvmrealloc");
- }
- return ret;
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
