Changeset: 2a06f2c12f9d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/2a06f2c12f9d
Modified Files:
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_private.h
gdk/gdk_select.c
monetdb5/mal/mal_interpreter.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/remote.c
Branch: default
Log Message:
Merge with Jul2021 branch.
diffs (truncated from 737 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -860,9 +860,6 @@ mskGetVal(BAT *b, BUN p)
* HEAPcopy (Heap *dst,*src);
* @item int
* @tab
- * HEAPdelete (Heap *dst, str o, str ext);
- * @item int
- * @tab
* HEAPwarm (Heap *h);
* @end multitable
*
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -91,7 +91,7 @@ VIEWcreate(oid seq, BAT *b)
return BATdense(seq, b->tseqbase, b->batCount);
}
- bn = BATcreatedesc(seq, b->ttype, false, TRANSIENT);
+ bn = BATcreatedesc(seq, b->ttype, false, TRANSIENT, 0);
if (bn == NULL)
return NULL;
assert(bn->theap == NULL);
@@ -193,8 +193,7 @@ BATmaterialize(BAT *b)
.parentid = b->batCacheid,
.dirty = true,
};
- strconcat_len(tail->filename, sizeof(tail->filename),
- BBP_physical(b->batCacheid), ".tail", NULL);
+ settailname(tail, BBP_physical(b->batCacheid), TYPE_oid, 0);
if (HEAPalloc(tail, cnt, sizeof(oid), 0) != GDK_SUCCEED) {
GDKfree(tail);
return GDK_FAIL;
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -62,7 +62,7 @@ BATinit_idents(BAT *bn)
}
BAT *
-BATcreatedesc(oid hseq, int tt, bool heapnames, role_t role)
+BATcreatedesc(oid hseq, int tt, bool heapnames, role_t role, uint16_t width)
{
BAT *bn;
@@ -123,8 +123,7 @@ BATcreatedesc(oid hseq, int tt, bool hea
};
const char *nme = BBP_physical(bn->batCacheid);
- strconcat_len(bn->theap->filename, sizeof(bn->theap->filename),
- nme, ".tail", NULL);
+ settailname(bn->theap, nme, tt, width);
if (ATOMneedheap(tt)) {
if ((bn->tvheap = GDKmalloc(sizeof(Heap))) == NULL) {
@@ -183,20 +182,21 @@ BATsetdims(BAT *b)
const char *
gettailname(const BAT *b)
{
- if (b->ttype != TYPE_str)
- return "tail";
- switch (b->twidth) {
- case 1:
- return "tail1";
- case 2:
- return "tail2";
+ if (b->ttype == TYPE_str) {
+ switch (b->twidth) {
+ case 1:
+ return "tail1";
+ case 2:
+ return "tail2";
#if SIZEOF_VAR_T == 8
- case 4:
- return "tail4";
+ case 4:
+ return "tail4";
#endif
- default:
- return "tail";
+ default:
+ break;
+ }
}
+ return "tail";
}
void
@@ -264,7 +264,7 @@ COLnew_intern(oid hseq, int tt, BUN cap,
if (cap > BUN_MAX)
cap = BUN_MAX;
- bn = BATcreatedesc(hseq, tt, true, role);
+ bn = BATcreatedesc(hseq, tt, true, role, width);
if (bn == NULL)
return NULL;
@@ -274,8 +274,12 @@ COLnew_intern(oid hseq, int tt, BUN cap,
if (ATOMstorage(tt) == TYPE_msk)
cap /= 8; /* 8 values per byte */
else if (tt == TYPE_str) {
- if (width != 0)
+ if (width != 0) {
+ /* power of two and not too large */
+ assert((width & (width - 1)) == 0);
+ assert(width <= sizeof(var_t));
bn->twidth = width;
+ }
settailname(bn->theap, BBP_physical(bn->batCacheid), tt,
bn->twidth);
}
@@ -2452,6 +2456,7 @@ BATassertProps(BAT *b)
int (*cmpf)(const void *, const void *);
int cmp;
const void *prev = NULL, *valp, *nilp;
+ char filename[sizeof(b->theap->filename)];
/* do the complete check within a lock */
MT_lock_set(&b->theaplock);
@@ -2495,6 +2500,22 @@ BATassertProps(BAT *b)
} else
assert(b->theap->size >> b->tshift >= b->batCapacity);
}
+ strconcat_len(filename, sizeof(filename),
+ BBP_physical(b->theap->parentid),
+ b->ttype == TYPE_str ? b->twidth == 1 ? ".tail1" :
b->twidth == 2 ? ".tail2" :
+#if SIZEOF_VAR_T == 8
+ b->twidth == 4 ? ".tail4" :
+#endif
+ ".tail" : ".tail",
+ NULL);
+ assert(strcmp(b->theap->filename, filename) == 0);
+ if (b->tvheap) {
+ strconcat_len(filename, sizeof(filename),
+ BBP_physical(b->tvheap->parentid),
+ ".theap",
+ NULL);
+ assert(strcmp(b->tvheap->filename, filename) == 0);
+ }
/* void and str imply varsized */
if (b->ttype == TYPE_void ||
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -23,12 +23,14 @@ unshare_varsized_heap(BAT *b)
{
if (ATOMvarsized(b->ttype) &&
b->tvheap->parentid != b->batCacheid) {
- Heap *h = GDKzalloc(sizeof(Heap));
+ Heap *h = GDKmalloc(sizeof(Heap));
if (h == NULL)
return GDK_FAIL;
MT_thread_setalgorithm("unshare vheap");
- h->parentid = b->batCacheid;
- h->farmid = BBPselectfarm(b->batRole, TYPE_str, varheap);
+ *h = (Heap) {
+ .parentid = b->batCacheid,
+ .farmid = BBPselectfarm(b->batRole, TYPE_str, varheap),
+ };
strconcat_len(h->filename, sizeof(h->filename),
BBP_physical(b->batCacheid), ".theap", NULL);
if (HEAPcopy(h, b->tvheap, 0) != GDK_SUCCEED) {
@@ -415,13 +417,15 @@ append_varsized_bat(BAT *b, BAT *n, stru
/* b and n do not share their vheap, so we need to copy data */
if (b->tvheap->parentid != b->batCacheid) {
/* if b shares its vheap with some other bat, unshare it */
- Heap *h = GDKzalloc(sizeof(Heap));
+ Heap *h = GDKmalloc(sizeof(Heap));
if (h == NULL) {
bat_iterator_end(&ni);
return GDK_FAIL;
}
- h->parentid = b->batCacheid;
- h->farmid = BBPselectfarm(b->batRole, b->ttype, varheap);
+ *h = (Heap) {
+ .parentid = b->batCacheid,
+ .farmid = BBPselectfarm(b->batRole, b->ttype, varheap),
+ };
strconcat_len(h->filename, sizeof(h->filename),
BBP_physical(b->batCacheid), ".theap", NULL);
if (HEAPcopy(h, b->tvheap, 0) != GDK_SUCCEED) {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -792,6 +792,34 @@ BBPcheckbats(unsigned bbpversion)
path = GDKfilepath(0, BATDIR, b->theap->filename, NULL);
if (path == NULL)
return GDK_FAIL;
+#if 1
+ /* first check string offset heap with width,
+ * then without */
+ if (MT_stat(path, &statb) < 0) {
+#ifdef GDKLIBRARY_TAILN
+ if (b->ttype == TYPE_str &&
+ b->twidth < SIZEOF_VAR_T) {
+ size_t taillen = strlen(path) - 1;
+ char tailsave = path[taillen];
+ path[taillen] = 0;
+ if (MT_stat(path, &statb) < 0) {
+ GDKsyserror("cannot stat file
%s%c or %s (expected size %zu)\n",
+ path, tailsave,
path, b->theap->free);
+ GDKfree(path);
+ return GDK_FAIL;
+ }
+ } else
+#endif
+ {
+ GDKsyserror("cannot stat file %s
(expected size %zu)\n",
+ path, b->theap->free);
+ GDKfree(path);
+ return GDK_FAIL;
+ }
+ }
+#else
+ /* first check string offset heap without widht,
+ * then with */
#ifdef GDKLIBRARY_TAILN
/* if bbpversion > GDKLIBRARY_TAILN, the offset heap can
* exist with either name .tail1 (etc) or .tail, if <=
@@ -821,6 +849,7 @@ BBPcheckbats(unsigned bbpversion)
GDKfree(path);
return GDK_FAIL;
}
+#endif
if ((size_t) statb.st_size < b->theap->free) {
GDKerror("file %s too small (expected %zu,
actual %zu)\n", path, b->theap->free, (size_t) statb.st_size);
GDKfree(path);
@@ -1264,24 +1293,40 @@ movestrbats(void)
/* not a valid BAT */
continue;
}
- if (b->ttype != TYPE_str || b->twidth == SIZEOF_VAR_T)
+ if (b->ttype != TYPE_str || b->twidth == SIZEOF_VAR_T ||
b->batCount == 0)
continue;
char *oldpath = GDKfilepath(0, BATDIR,
BBP_physical(b->batCacheid), "tail");
char *newpath = GDKfilepath(0, BATDIR, b->theap->filename,
NULL);
- struct stat st;
int ret = -1;
if (oldpath != NULL && newpath != NULL) {
- if (MT_stat(oldpath, &st) != 0 && (b->batCount == 0 ||
MT_stat(newpath, &st) == 0))
- ret = 0; /* old doesn't exist, but new does or
bat is empty: that's ok */
- else {
+ struct stat oldst, newst;
+ bool oldexist = MT_stat(oldpath, &oldst) == 0;
+ bool newexist = MT_stat(newpath, &newst) == 0;
+ if (newexist) {
+ if (oldexist) {
+ if (oldst.st_mtime > newst.st_mtime) {
+ GDKerror("both %s and %s exist
with %s unexpectedly newer: manual intervention required\n", oldpath, newpath,
oldpath);
+ ret = -1;
+ } else {
+ TRC_WARNING(GDK, "both %s and
%s exist, removing %s\n", oldpath, newpath, oldpath);
+ ret = MT_remove(oldpath);
+ }
+ } else {
+ /* already good */
+ ret = 0;
+ }
+ } else if (oldexist) {
+ TRC_DEBUG(IO_, "rename %s to %s\n", oldpath,
newpath);
ret = MT_rename(oldpath, newpath);
- if (ret < 0)
- GDKsyserror("rename %s %s\n", oldpath,
newpath);
+ } else {
+ /* neither file exists: may be ok, but
+ * will be checked later */
+ ret = 0;
}
}
GDKfree(oldpath);
GDKfree(newpath);
- if (ret < 0)
+ if (ret == -1)
return GDK_FAIL;
}
return GDK_SUCCEED;
@@ -1510,6 +1555,67 @@ BBPinit(void)
if (BBPcheckbats(bbpversion) != GDK_SUCCEED)
return GDK_FAIL;
+#ifdef GDKLIBRARY_TAILN
+ char *needstrbatmove;
+ needstrbatmove = GDKfilepath(0, BATDIR, "needstrbatmove", NULL);
+ if (bbpversion <= GDKLIBRARY_TAILN) {
+ /* create signal file that we need to rename string
+ * offset heaps */
+ int fd = MT_open(needstrbatmove, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ TRC_CRITICAL(GDK, "cannot create signal file
needstrbatmove.\n");
+ GDKfree(needstrbatmove);
+ return GDK_FAIL;
+ }
+ close(fd);
+ } else {
+ /* check signal file whether we need to rename string
+ * offset heaps */
+ int fd = MT_open(needstrbatmove, O_RDONLY);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
