Changeset: 7fe284bcdb22 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7fe284bcdb22
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_orderidx.c
gdk/gdk_storage.c
gdk/gdk_utils.c
sql/backends/monet5/sql_scenario.c
sql/storage/bat/Makefile.ag
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.SQL.py
sql/test/BugTracker-2018/Tests/local_replica_table_not_detected.Bug-6620.py
sql/test/BugTracker-2018/Tests/remote-table-where-is-null.Bug-6601.py
sql/test/BugTracker-2018/Tests/remote-table-where-not-equal.Bug-6621.py
sql/test/remote/Tests/creds.SQL.py.in
sql/test/remote/Tests/different_user.SQL.py.in
sql/test/remote/Tests/invalid_creds.SQL.py.in
sql/test/remote/Tests/ssbm.SQL.py
tools/mserver/mserver5.c
Branch: in-memory
Log Message:
Backported in-memory processing from MonetDBLite.
The in-memory code has not been tested in this version. The old mode
still works.
One change: the dbfarm directory (under which the database is created)
must exist before attempting to start the server.
diffs (truncated from 989 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -257,6 +257,7 @@ gdk_return GDKgetsem(int sem_id, int cou
gdk_return GDKgetsemval(int sem_id, int number, int *semval, str *msg);
bool GDKinit(opt *set, int setlen);
gdk_return GDKinitmmap(size_t id, size_t size, void **return_ptr, size_t
*return_size, str *msg);
+bool GDKinmemory(void);
BAT *GDKkey;
ATOMIC_TYPE volatile GDKlockcnt;
ATOMIC_TYPE volatile GDKlockcontentioncnt;
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1379,6 +1379,7 @@ gdk_export void BATmsync(BAT *b);
#define NOFARM (-1) /* indicate to GDKfilepath to create relative path */
gdk_export char *GDKfilepath(int farmid, const char *dir, const char *nme,
const char *ext);
+gdk_export bool GDKinmemory(void);
gdk_export gdk_return GDKcreatedir(const char *nme);
gdk_export void OIDXdestroy(BAT *b);
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -324,6 +324,9 @@ BBPselectfarm(int role, int type, enum h
(void) hptype; /* may use in future */
assert(role >= 0 && role < 32);
+ if (GDKinmemory())
+ return 0;
+
#ifndef PERSISTENTHASH
if (hptype == hashheap)
role = TRANSIENT;
@@ -1230,6 +1233,12 @@ BBPheader(FILE *fp)
return bbpversion;
}
+bool
+GDKinmemory(void)
+{
+ return BBPfarms[0].dirname == NULL;
+}
+
/* all errors are fatal */
void
BBPaddfarm(const char *dirname, int rolemask)
@@ -1306,78 +1315,81 @@ BBPinit(void)
{
FILE *fp = NULL;
struct stat st;
- unsigned bbpversion;
- str bbpdirstr, backupbbpdirstr;
+ unsigned bbpversion = 0;
int i;
- if(!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir")))
- GDKfatal("BBPinit: GDKmalloc failed\n");
- if(!(backupbbpdirstr = GDKfilepath(0, BAKDIR, "BBP", "dir")))
- GDKfatal("BBPinit: GDKmalloc failed\n");
-
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&GDKunloadLock, "GDKunloadLock");
ATOMIC_INIT(BBPsizeLock);
#endif
- if (BBPfarms[0].dirname == NULL) {
- BBPaddfarm(".", 1 << PERSISTENT);
- BBPaddfarm(".", 1 << TRANSIENT);
+ if (!GDKinmemory()) {
+ char *bbpdirstr, *backupbbpdirstr;
+
+ if(!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir")))
+ GDKfatal("BBPinit: GDKmalloc failed\n");
+ if(!(backupbbpdirstr = GDKfilepath(0, BAKDIR, "BBP", "dir")))
+ GDKfatal("BBPinit: GDKmalloc failed\n");
+
+ if (GDKremovedir(0, TEMPDIR) != GDK_SUCCEED)
+ GDKfatal("BBPinit: cannot remove directory %s\n",
TEMPDIR);
+
+ if (GDKremovedir(0, DELDIR) != GDK_SUCCEED)
+ GDKfatal("BBPinit: cannot remove directory %s\n",
DELDIR);
+
+ /* first move everything from SUBDIR to BAKDIR (its parent) */
+ if (BBPrecover_subdir() != GDK_SUCCEED)
+ GDKfatal("BBPinit: cannot properly recover_subdir
process %s. Please check whether your disk is full or write-protected", SUBDIR);
+
+ /* try to obtain a BBP.dir from bakdir */
+ if (stat(backupbbpdirstr, &st) == 0) {
+ /* backup exists; *must* use it */
+ if (recover_dir(0, stat(bbpdirstr, &st) == 0) !=
GDK_SUCCEED)
+ goto bailout;
+ if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
+ GDKfatal("BBPinit: cannot open recovered
BBP.dir.");
+ } else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
+ /* there was no BBP.dir either. Panic! try to use a
+ * BBP.bak */
+ if (stat(backupbbpdirstr, &st) < 0) {
+ /* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir):
+ * create a new one */
+ IODEBUG fprintf(stderr, "#BBPdir: initializing
BBP.\n"); /* BBPdir instead of BBPinit for backward compatibility of
error messages */
+ if (BBPdir(0, NULL) != GDK_SUCCEED)
+ goto bailout;
+ } else if (GDKmove(0, BATDIR, "BBP", "bak", BATDIR,
"BBP", "dir") == GDK_SUCCEED)
+ IODEBUG fprintf(stderr, "#BBPinit: reverting to
dir saved in BBP.bak.\n");
+
+ if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
+ goto bailout;
+ }
+ assert(fp != NULL);
+ GDKfree(bbpdirstr);
+ GDKfree(backupbbpdirstr);
}
- if (GDKremovedir(0, TEMPDIR) != GDK_SUCCEED)
- GDKfatal("BBPinit: cannot remove directory %s\n", TEMPDIR);
-
- if (GDKremovedir(0, DELDIR) != GDK_SUCCEED)
- GDKfatal("BBPinit: cannot remove directory %s\n", DELDIR);
-
- /* first move everything from SUBDIR to BAKDIR (its parent) */
- if (BBPrecover_subdir() != GDK_SUCCEED)
- GDKfatal("BBPinit: cannot properly recover_subdir process %s.
Please check whether your disk is full or write-protected", SUBDIR);
-
- /* try to obtain a BBP.dir from bakdir */
- if (stat(backupbbpdirstr, &st) == 0) {
- /* backup exists; *must* use it */
- if (recover_dir(0, stat(bbpdirstr, &st) == 0) != GDK_SUCCEED)
- goto bailout;
- if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
- GDKfatal("BBPinit: cannot open recovered BBP.dir.");
- } else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
- /* there was no BBP.dir either. Panic! try to use a
- * BBP.bak */
- if (stat(backupbbpdirstr, &st) < 0) {
- /* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir):
- * create a new one */
- IODEBUG fprintf(stderr, "#BBPdir: initializing
BBP.\n"); /* BBPdir instead of BBPinit for backward compatibility of
error messages */
- if (BBPdir(0, NULL) != GDK_SUCCEED)
- goto bailout;
- } else if (GDKmove(0, BATDIR, "BBP", "bak", BATDIR, "BBP",
"dir") == GDK_SUCCEED)
- IODEBUG fprintf(stderr, "#BBPinit: reverting to dir
saved in BBP.bak.\n");
-
- if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
- goto bailout;
- }
- assert(fp != NULL);
-
/* scan the BBP.dir to obtain current size */
BBPlimit = 0;
memset(BBP, 0, sizeof(BBP));
ATOMIC_SET(BBPsize, 1, BBPsizeLock);
BBP_dirty = true;
- bbpversion = BBPheader(fp);
+ if (!GDKinmemory())
+ bbpversion = BBPheader(fp);
BBPextend(0, false); /* allocate BBP records */
- ATOMIC_SET(BBPsize, 1, BBPsizeLock);
-
- BBPreadEntries(fp, bbpversion);
- fclose(fp);
+
+ if (!GDKinmemory()) {
+ ATOMIC_SET(BBPsize, 1, BBPsizeLock);
+ BBPreadEntries(fp, bbpversion);
+ fclose(fp);
+ }
if (BBPinithash(0) != GDK_SUCCEED)
GDKfatal("BBPinit: BBPinithash failed");
/* will call BBPrecover if needed */
- if (BBPprepare(false) != GDK_SUCCEED)
+ if (!GDKinmemory() && BBPprepare(false) != GDK_SUCCEED)
GDKfatal("BBPinit: cannot properly prepare process %s. Please
check whether your disk is full or write-protected", BAKDIR);
/* cleanup any leftovers (must be done after BBPrecover) */
@@ -1409,8 +1421,6 @@ BBPinit(void)
#endif
if (bbpversion < GDKLIBRARY)
TMcommit();
- GDKfree(bbpdirstr);
- GDKfree(backupbbpdirstr);
return;
bailout:
@@ -2072,7 +2082,7 @@ BBPinsert(BAT *bn)
BBP_logical(i) = BBP_bak(i);
/* Keep the physical location around forever */
- if (*BBP_physical(i) == 0) {
+ if (!GDKinmemory() && *BBP_physical(i) == 0) {
BBPgetsubdir(dirname, i);
if (*dirname) /* i.e., i >= 0100 */
diff --git a/gdk/gdk_group.c b/gdk/gdk_group.c
--- a/gdk/gdk_group.c
+++ b/gdk/gdk_group.c
@@ -1057,7 +1057,7 @@ BATgroup_internal(BAT **groups, BAT **ex
e ? BATgetId(e) : "NULL", e ? BATcount(e) : 0,
h ? BATgetId(h) : "NULL", h ? BATcount(h) : 0,
subsorted, gc ? " (g clustered)" : "");
- nme = BBP_physical(b->batCacheid);
+ nme = GDKinmemory() ? ":inmemory" : BBP_physical(b->batCacheid);
mask = MAX(HASHmask(cnt), 1 << 16);
/* mask is a power of two, so pop(mask - 1) tells us
* which power of two */
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -178,6 +178,7 @@ BATcheckhash(BAT *b)
const char *nme = BBP_physical(b->batCacheid);
int fd;
+ assert(!GDKinmemory());
b->thash = NULL;
if ((h = GDKzalloc(sizeof(*h))) != NULL &&
(h->heap.farmid = BBPselectfarm(b->batRole, b->ttype,
hashheap)) >= 0) {
@@ -360,7 +361,7 @@ BAThash_impl(BAT *b, BAT *s, const char
BUN nslots;
BUN hnil, hget;
Hash *h = NULL;
- const char *nme = BBP_physical(b->batCacheid);
+ const char *nme = GDKinmemory() ? ":inmemory" :
BBP_physical(b->batCacheid);
BATiter bi = bat_iterator(b);
ALGODEBUG t0 = GDKusec();
@@ -578,7 +579,7 @@ BAThash(BAT *b)
return GDK_FAIL;
}
#ifdef PERSISTENTHASH
- if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty)
{
+ if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty
&& !GDKinmemory()) {
MT_Id tid;
BBPfix(b->batCacheid);
if (MT_create_thread(&tid, BAThashsync, b,
@@ -673,11 +674,11 @@ HASHfree(BAT *b)
if (b) {
MT_lock_set(&GDKhashLock(b->batCacheid));
if (b->thash && b->thash != (Hash *) 1) {
- bool dirty = b->thash->heap.dirty;
+ bool rmheap = GDKinmemory() || b->thash->heap.dirty;
- HEAPfree(&b->thash->heap, dirty);
+ HEAPfree(&b->thash->heap, rmheap);
GDKfree(b->thash);
- b->thash = dirty ? NULL : (Hash *) 1;
+ b->thash = rmheap ? NULL : (Hash *) 1;
}
MT_lock_unset(&GDKhashLock(b->batCacheid));
}
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -109,14 +109,15 @@ HEAPalloc(Heap *h, size_t nitems, size_t
GDKerror("HEAPalloc: allocating more than heap can
accomodate\n");
return GDK_FAIL;
}
- if (h->size < 4 * GDK_mmap_pagesize ||
+ if (GDKinmemory() ||
+ h->size < 4 * GDK_mmap_pagesize ||
(GDKmem_cursize() + h->size < GDK_mem_maxsize &&
h->size < (h->farmid == 0 ? GDK_mmap_minsize_persistent :
GDK_mmap_minsize_transient))) {
h->storage = STORE_MEM;
h->base = GDKmalloc(h->size);
HEAPDEBUG fprintf(stderr, "#HEAPalloc %zu %p\n", h->size,
h->base);
}
- if (h->base == NULL) {
+ if (!GDKinmemory() && h->base == NULL) {
char *nme;
struct stat st;
@@ -182,12 +183,17 @@ HEAPextend(Heap *h, size_t size, bool ma
char nme[sizeof(h->filename)], *ext;
const char *failure = "None";
- strncpy(nme, h->filename, sizeof(nme));
+ if (GDKinmemory()) {
+ strncpy(nme, ":inmemory", sizeof(nme));
+ ext = "ext";
+ } else {
+ strncpy(nme, h->filename, sizeof(nme));
#ifdef STATIC_CODE_ANALYSIS
- /* help coverity */
- nme[sizeof(nme) - 1] = 0;
+ /* help coverity */
+ nme[sizeof(nme) - 1] = 0;
#endif
- ext = decompose_filename(nme);
+ ext = decompose_filename(nme);
+ }
if (size <= h->size)
return GDK_SUCCEED; /* nothing to do */
@@ -222,8 +228,8 @@ HEAPextend(Heap *h, size_t size, bool ma
/* extend a malloced heap, possibly switching over to
* file-mapped storage */
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
