Changeset: 46ba7db7f1bd for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=46ba7db7f1bd
Added Files:
sql/storage/bat/nop_logger.c
sql/storage/bat/nop_logger.h
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_group.c
gdk/gdk_hash.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_orderidx.c
gdk/gdk_storage.c
gdk/gdk_utils.c
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_scenario.c
sql/scripts/Makefile.ag
sql/storage/bat/Makefile.ag
sql/storage/bat/bat_storage.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/BugTracker-2015/Tests/schemadiff.Bug-3778.SQL.py
sql/test/BugTracker-2018/Tests/local_replica_table_not_detected.Bug-6620.py
sql/test/BugTracker-2018/Tests/remote-table-where-is-null.Bug-6601.py
sql/test/BugTracker-2018/Tests/remote-table-where-not-equal.Bug-6621.py
sql/test/remote/Tests/creds.SQL.py
sql/test/remote/Tests/different_user.SQL.py
sql/test/remote/Tests/invalid_creds.SQL.py
sql/test/remote/Tests/ssbm.SQL.py
tools/mserver/mserver5.c
Branch: default
Log Message:
Merge in in-memory branch.
diffs (truncated from 1173 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -253,6 +253,7 @@ gdk_return GDKgetsem(int sem_id, int cou
gdk_return GDKgetsemval(int sem_id, int number, int *semval);
gdk_return GDKinit(struct opt *set, int setlen);
void *GDKinitmmap(size_t id, size_t size, size_t *return_size);
+bool GDKinmemory(void);
ATOMIC_TYPE GDKlockcnt;
ATOMIC_TYPE GDKlockcontentioncnt;
MT_Lock *volatile GDKlocklist;
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1373,6 +1373,7 @@ gdk_export void BATmsync(BAT *b);
#define NOFARM (-1) /* indicate to GDKfilepath to create relative path */
gdk_export char *GDKfilepath(int farmid, const char *dir, const char *nme,
const char *ext);
+gdk_export bool GDKinmemory(void);
gdk_export gdk_return GDKcreatedir(const char *nme);
gdk_export void OIDXdestroy(BAT *b);
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -313,6 +313,9 @@ BBPselectfarm(role_t role, int type, enu
(void) type; /* may use in future */
(void) hptype; /* may use in future */
+ if (GDKinmemory())
+ return 0;
+
#ifndef PERSISTENTHASH
if (hptype == hashheap)
role = TRANSIENT;
@@ -1027,6 +1030,12 @@ BBPheader(FILE *fp)
return bbpversion;
}
+bool
+GDKinmemory(void)
+{
+ return BBPfarms[0].dirname == NULL;
+}
+
/* all errors are fatal */
gdk_return
BBPaddfarm(const char *dirname, int rolemask)
@@ -1106,98 +1115,100 @@ BBPinit(void)
{
FILE *fp = NULL;
struct stat st;
- unsigned bbpversion;
- str bbpdirstr, backupbbpdirstr;
+ unsigned bbpversion = 0;
int i;
- if(!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir"))) {
- GDKerror("BBPinit: GDKmalloc failed\n");
- return GDK_FAIL;
- }
- if(!(backupbbpdirstr = GDKfilepath(0, BAKDIR, "BBP", "dir"))) {
- GDKfree(bbpdirstr);
- GDKerror("BBPinit: GDKmalloc failed\n");
- return GDK_FAIL;
- }
-
- if (BBPfarms[0].dirname == NULL) {
+ if (GDKinmemory()) {
if (BBPaddfarm(".", 1 << PERSISTENT) != GDK_SUCCEED ||
BBPaddfarm(".", 1 << TRANSIENT) != GDK_SUCCEED) {
+ return GDK_FAIL;
+ }
+ } else {
+ str bbpdirstr, backupbbpdirstr;
+
+ if (!(bbpdirstr = GDKfilepath(0, BATDIR, "BBP", "dir"))) {
+ GDKerror("BBPinit: GDKmalloc failed\n");
+ return GDK_FAIL;
+ }
+
+ if (!(backupbbpdirstr = GDKfilepath(0, BAKDIR, "BBP", "dir"))) {
+ GDKfree(bbpdirstr);
+ GDKerror("BBPinit: GDKmalloc failed\n");
+ return GDK_FAIL;
+ }
+
+ if (GDKremovedir(0, TEMPDIR) != GDK_SUCCEED) {
GDKfree(bbpdirstr);
GDKfree(backupbbpdirstr);
+ GDKerror("BBPinit: cannot remove directory %s\n",
TEMPDIR);
+ return GDK_FAIL;
+ }
+
+ if (GDKremovedir(0, DELDIR) != GDK_SUCCEED) {
+ GDKfree(bbpdirstr);
+ GDKfree(backupbbpdirstr);
+ GDKerror("BBPinit: cannot remove directory %s\n",
DELDIR);
return GDK_FAIL;
}
- }
-
- if (GDKremovedir(0, TEMPDIR) != GDK_SUCCEED) {
- GDKfree(bbpdirstr);
- GDKfree(backupbbpdirstr);
- GDKerror("BBPinit: cannot remove directory %s\n", TEMPDIR);
- return GDK_FAIL;
- }
-
- if (GDKremovedir(0, DELDIR) != GDK_SUCCEED) {
+
+ /* first move everything from SUBDIR to BAKDIR (its parent) */
+ if (BBPrecover_subdir() != GDK_SUCCEED) {
+ GDKfree(bbpdirstr);
+ GDKfree(backupbbpdirstr);
+ GDKerror("BBPinit: cannot properly recover_subdir
process %s. Please check whether your disk is full or write-protected", SUBDIR);
+ return GDK_FAIL;
+ }
+
+ /* try to obtain a BBP.dir from bakdir */
+ if (stat(backupbbpdirstr, &st) == 0) {
+ /* backup exists; *must* use it */
+ if (recover_dir(0, stat(bbpdirstr, &st) == 0) !=
GDK_SUCCEED)
+ goto bailout;
+ if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
{
+ GDKfree(bbpdirstr);
+ GDKfree(backupbbpdirstr);
+ GDKerror("BBPinit: cannot open recovered
BBP.dir.");
+ return GDK_FAIL;
+ }
+ } else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
+ /* there was no BBP.dir either. Panic! try to use a
+ * BBP.bak */
+ if (stat(backupbbpdirstr, &st) < 0) {
+ /* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir):
+ * create a new one */
+ IODEBUG fprintf(stderr, "#BBPdir: initializing
BBP.\n"); /* BBPdir instead of BBPinit for backward compatibility of
error messages */
+ if (BBPdir(0, NULL) != GDK_SUCCEED)
+ goto bailout;
+ } else if (GDKmove(0, BATDIR, "BBP", "bak", BATDIR,
"BBP", "dir") == GDK_SUCCEED)
+ IODEBUG fprintf(stderr, "#BBPinit: reverting to
dir saved in BBP.bak.\n");
+
+ if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
+ goto bailout;
+ }
+ assert(fp != NULL);
GDKfree(bbpdirstr);
GDKfree(backupbbpdirstr);
- GDKerror("BBPinit: cannot remove directory %s\n", DELDIR);
- return GDK_FAIL;
}
- /* first move everything from SUBDIR to BAKDIR (its parent) */
- if (BBPrecover_subdir() != GDK_SUCCEED) {
- GDKfree(bbpdirstr);
- GDKfree(backupbbpdirstr);
- GDKerror("BBPinit: cannot properly recover_subdir process %s.
Please check whether your disk is full or write-protected", SUBDIR);
- return GDK_FAIL;
- }
-
- /* try to obtain a BBP.dir from bakdir */
- if (stat(backupbbpdirstr, &st) == 0) {
- /* backup exists; *must* use it */
- if (recover_dir(0, stat(bbpdirstr, &st) == 0) != GDK_SUCCEED)
- goto bailout;
- if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
- GDKfree(bbpdirstr);
- GDKfree(backupbbpdirstr);
- GDKerror("BBPinit: cannot open recovered BBP.dir.");
- return GDK_FAIL;
- }
- } else if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL) {
- /* there was no BBP.dir either. Panic! try to use a
- * BBP.bak */
- if (stat(backupbbpdirstr, &st) < 0) {
- /* no BBP.bak (nor BBP.dir or BACKUP/BBP.dir):
- * create a new one */
- IODEBUG fprintf(stderr, "#BBPdir: initializing
BBP.\n"); /* BBPdir instead of BBPinit for backward compatibility of
error messages */
- if (BBPdir(0, NULL) != GDK_SUCCEED)
- goto bailout;
- } else if (GDKmove(0, BATDIR, "BBP", "bak", BATDIR, "BBP",
"dir") == GDK_SUCCEED)
- IODEBUG fprintf(stderr, "#BBPinit: reverting to dir
saved in BBP.bak.\n");
-
- if ((fp = GDKfilelocate(0, "BBP", "r", "dir")) == NULL)
- goto bailout;
- }
- assert(fp != NULL);
- GDKfree(bbpdirstr);
- GDKfree(backupbbpdirstr);
-
/* scan the BBP.dir to obtain current size */
BBPlimit = 0;
memset(BBP, 0, sizeof(BBP));
ATOMIC_SET(&BBPsize, 1);
- bbpversion = BBPheader(fp);
- if (bbpversion == 0) {
- return GDK_FAIL;
+ if (!GDKinmemory()) {
+ bbpversion = BBPheader(fp);
+ if (bbpversion == 0)
+ return GDK_FAIL;
}
BBPextend(0, false); /* allocate BBP records */
- ATOMIC_SET(&BBPsize, 1);
-
- if (BBPreadEntries(fp, bbpversion) != GDK_SUCCEED) {
- return GDK_FAIL;
+
+ if (!GDKinmemory()) {
+ ATOMIC_SET(&BBPsize, 1);
+ if (BBPreadEntries(fp, bbpversion) != GDK_SUCCEED)
+ return GDK_FAIL;
+ fclose(fp);
}
- fclose(fp);
if (BBPinithash(0) != GDK_SUCCEED) {
GDKerror("BBPinit: BBPinithash failed");
@@ -1205,7 +1216,7 @@ BBPinit(void)
}
/* will call BBPrecover if needed */
- if (BBPprepare(false) != GDK_SUCCEED) {
+ if (!GDKinmemory() && BBPprepare(false) != GDK_SUCCEED) {
GDKerror("BBPinit: cannot properly prepare process %s. Please
check whether your disk is full or write-protected", BAKDIR);
return GDK_FAIL;
}
@@ -1236,9 +1247,8 @@ BBPinit(void)
#ifdef GDKLIBRARY_NIL_NAN
if (bbpversion <= GDKLIBRARY_NIL_NAN)
- if (fixfloatbats() != GDK_SUCCEED) {
+ if (fixfloatbats() != GDK_SUCCEED)
return GDK_FAIL;
- }
#endif
if (bbpversion < GDKLIBRARY)
TMcommit();
@@ -1909,7 +1919,7 @@ BBPinsert(BAT *bn)
BBP_logical(i) = BBP_bak(i);
/* Keep the physical location around forever */
- if (*BBP_physical(i) == 0) {
+ if (!GDKinmemory() && *BBP_physical(i) == 0) {
BBPgetsubdir(dirname, i);
if (*dirname) /* i.e., i >= 0100 */
diff --git a/gdk/gdk_group.c b/gdk/gdk_group.c
--- a/gdk/gdk_group.c
+++ b/gdk/gdk_group.c
@@ -1057,7 +1057,7 @@ BATgroup_internal(BAT **groups, BAT **ex
e ? BATgetId(e) : "NULL", e ? BATcount(e) : 0,
h ? BATgetId(h) : "NULL", h ? BATcount(h) : 0,
subsorted, gc ? " (g clustered)" : "");
- nme = BBP_physical(b->batCacheid);
+ nme = GDKinmemory() ? ":inmemory" : BBP_physical(b->batCacheid);
mask = MAX(HASHmask(cnt), 1 << 16);
/* mask is a power of two, so pop(mask - 1) tells us
* which power of two */
diff --git a/gdk/gdk_hash.c b/gdk/gdk_hash.c
--- a/gdk/gdk_hash.c
+++ b/gdk/gdk_hash.c
@@ -181,6 +181,7 @@ BATcheckhash(BAT *b)
Hash *h;
int fd;
+ assert(!GDKinmemory());
b->thash = NULL;
if ((h = GDKzalloc(sizeof(*h))) != NULL &&
(h->heap.farmid = BBPselectfarm(b->batRole,
b->ttype, hashheap)) >= 0) {
@@ -376,7 +377,7 @@ BAThash_impl(BAT *b, BAT *s, const char
BUN nslots;
BUN hnil, hget;
Hash *h = NULL;
- const char *nme = BBP_physical(b->batCacheid);
+ const char *nme = GDKinmemory() ? ":inmemory" :
BBP_physical(b->batCacheid);
BATiter bi = bat_iterator(b);
ACCELDEBUG t0 = GDKusec();
@@ -594,7 +595,7 @@ BAThash(BAT *b)
return GDK_FAIL;
}
#ifdef PERSISTENTHASH
- if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty)
{
+ if (BBP_status(b->batCacheid) & BBPEXISTING && !b->theap.dirty
&& !GDKinmemory()) {
MT_Id tid;
BBPfix(b->batCacheid);
char name[16];
@@ -694,11 +695,11 @@ HASHfree(BAT *b)
if (b && b->thash) {
MT_lock_set(&GDKhashLock(b->batCacheid));
if (b->thash && b->thash != (Hash *) 1) {
- bool dirty = b->thash->heap.dirty;
+ bool rmheap = GDKinmemory() || b->thash->heap.dirty;
- HEAPfree(&b->thash->heap, dirty);
+ HEAPfree(&b->thash->heap, rmheap);
GDKfree(b->thash);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
