Changeset: 9bf62d301f1b for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/9bf62d301f1b Branch: select-window-pushdown Log Message:
Merge with Jan2022. diffs (truncated from 3258 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -769,3 +769,4 @@ 1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3 1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3 Jun2020_SP2_release 59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_15 59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_SP3_release +38ba6e40ba9148c762fe4a4b40f4937f86e516b6 Jul2021_25 diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -31,8 +31,8 @@ once an hour. .. _MonetDB: https://dev.monetdb.org/hg/MonetDB/ __ MonetDB_ -.. _github: https://github.com/MonetDB/MonetDB -__ github_ +.. _mirror: https://github.com/MonetDB/MonetDB +__ mirror_ Building -------- @@ -92,8 +92,8 @@ WITH_XML2 Include xml2 support WITH_ZLIB Include zlib support ============== =============================================================================================== -Required packages -................. +Required and Optional Packages +.............................. On Fedora, the following packages are required: ``bison``, ``cmake``, ``gcc``, ``pkgconf``, ``python3``. diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022 --- a/gdk/ChangeLog.Jan2022 +++ b/gdk/ChangeLog.Jan2022 @@ -1,6 +1,25 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Wed Aug 10 2022 Sjoerd Mullender <[email protected]> +- A bug was fixed when upgrading a database from the Oct2020 releases + (11.39.X) or older when the write-ahead log (WAL) was not empty and + contained instructions to create new tables. + +* Tue Aug 2 2022 Sjoerd Mullender <[email protected]> +- When destroying a bat, make sure there are no files left over in + the BACKUP directory since they can cause problems when the bat id + gets reused. + +* Thu Jul 28 2022 Sjoerd Mullender <[email protected]> +- Fixed an off-by-one error in the logger which caused older log files + to stick around longer in the write-ahead log than necessary. +- When an empty BAT is committed, skip writing (and synchronizing to + disk) the heap (tail and theap) files and write 0 for their sizes to + the BBP.dir file. When reading the BBP.dir file, if an empty BAT is + encountered, set the sizes of those files to 0. This fixes potential + issues during startup of the server (BBPcheckbats reporting errors). + * Wed Jun 22 2022 Sjoerd Mullender <[email protected]> - Make sure heap files of transient bats get deleted when the bat is destroyed. If the bat was a partial view (sharing the vheap but not diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -899,8 +899,7 @@ COLcopy(BAT *b, int tt, bool writable, r /* convert number of bits to number of bytes, * and round the latter up to a multiple of * 4 (copy in units of 4 bytes) */ - bn->theap->free = (bi.count + 7) / 8; - bn->theap->free = (bn->theap->free + 3) & ~(size_t)3; + bn->theap->free = ((bi.count + 31) / 32) * 4; bn->theap->dirty |= bi.count > 0; memcpy(Tloc(bn, 0), bi.base, bn->theap->free); } else { diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -132,8 +132,10 @@ static gdk_return BBPbackup(BAT *b, bool static gdk_return BBPdir_init(void); static void BBPcallbacks(void); -static lng BBPlogno; /* two lngs of extra info in BBP.dir */ -static lng BBPtransid; +/* two lngs of extra info in BBP.dir */ +/* these two need to be atomic because of their use in AUTHcommit() */ +static ATOMIC_TYPE BBPlogno = ATOMIC_VAR_INIT(0); +static ATOMIC_TYPE BBPtransid = ATOMIC_VAR_INIT(0); #ifdef HAVE_HGE /* start out by saying we have no hge, but as soon as we've seen one, @@ -178,13 +180,13 @@ getBBPsize(void) lng getBBPlogno(void) { - return BBPlogno; + return (lng) ATOMIC_GET(&BBPlogno); } lng getBBPtransid(void) { - return BBPtransid; + return (lng) ATOMIC_GET(&BBPtransid); } @@ -433,7 +435,7 @@ heapinit(BAT *b, const char *buf, #ifdef GDKLIBRARY_HASHASH int *hashash, #endif - unsigned bbpversion, bat bid, const char *filename, int lineno) + unsigned bbpversion, const char *filename, int lineno) { int t; char type[33]; @@ -495,7 +497,7 @@ heapinit(BAT *b, const char *buf, return -1; } } else if (var != (t == TYPE_void || BATatoms[t].atomPut != NULL)) { - TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tvarsized mismatch for BAT %d on line %d\n", (int) bid, lineno); + TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tvarsized mismatch for BAT %d on line %d\n", (int) b->batCacheid, lineno); return -1; } else if (var && t != 0 ? ATOMsize(t) < width || @@ -505,7 +507,7 @@ heapinit(BAT *b, const char *buf, #endif ) : ATOMsize(t) != width) { - TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tsize mismatch for BAT %d on line %d\n", (int) bid, lineno); + TRC_CRITICAL(GDK, "inconsistent entry in BBP.dir: tsize mismatch for BAT %d on line %d\n", (int) b->batCacheid, lineno); return -1; } b->ttype = t; @@ -553,7 +555,7 @@ heapinit(BAT *b, const char *buf, } static int -vheapinit(BAT *b, const char *buf, bat bid, const char *filename, int lineno) +vheapinit(BAT *b, const char *buf, const char *filename, int lineno) { int n = 0; uint64_t free, size; @@ -567,11 +569,6 @@ vheapinit(BAT *b, const char *buf, bat b TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d", lineno); return -1; } - b->tvheap = GDKmalloc(sizeof(Heap)); - if (b->tvheap == NULL) { - TRC_CRITICAL(GDK, "cannot allocate memory for heap."); - return -1; - } if (b->batCount == 0) free = 0; if (b->ttype >= 0 && @@ -590,16 +587,134 @@ vheapinit(BAT *b, const char *buf, bat b .cleanhash = true, .newstorage = STORE_INVALID, .dirty = false, - .parentid = bid, + .parentid = b->batCacheid, .farmid = BBPselectfarm(PERSISTENT, b->ttype, varheap), }; strconcat_len(b->tvheap->filename, sizeof(b->tvheap->filename), filename, ".theap", NULL); - ATOMIC_INIT(&b->tvheap->refs, 1); + } else { + b->tvheap = NULL; } return n; } +/* read a single line from the BBP.dir file (file pointer fp) and fill + * in the structure pointed to by bn and extra information through the + * other pointers; this function does not allocate any memory; return 0 + * on end of file, 1 on success, and -1 on failure */ +static int +BBPreadBBPline(FILE *fp, unsigned bbpversion, int *lineno, BAT *bn, +#ifdef GDKLIBRARY_HASHASH + int *hashash, +#endif + char *batname, char *filename, char **options) +{ + char buf[4096]; + uint64_t batid; + uint16_t status; + unsigned int properties; + int nread, n; + char *s; + uint64_t count, capacity = 0, base = 0; + + if (fgets(buf, sizeof(buf), fp) == NULL) { + if (ferror(fp)) { + TRC_CRITICAL(GDK, "error reading BBP.dir on line %d\n", *lineno); + return -1; + } + return 0; /* end of file */ + } + (*lineno)++; + if ((s = strchr(buf, '\r')) != NULL) { + /* convert \r\n into just \n */ + if (s[1] != '\n') { + TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d", *lineno); + return -1; + } + *s++ = '\n'; + *s = 0; + } + + if (sscanf(buf, + "%" SCNu64 " %" SCNu16 " %128s %19s %u %" SCNu64 + " %" SCNu64 " %" SCNu64 + "%n", + &batid, &status, batname, filename, + &properties, + &count, &capacity, &base, + &nread) < 8) { + TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d", *lineno); + return -1; + } + + if (batid >= N_BBPINIT * BBPINIT) { + TRC_CRITICAL(GDK, "bat ID (%" PRIu64 ") too large to accomodate (max %d), on line %d.", batid, N_BBPINIT * BBPINIT - 1, *lineno); + return -1; + } + + /* convert both / and \ path separators to our own DIR_SEP */ +#if DIR_SEP != '/' + s = filename; + while ((s = strchr(s, '/')) != NULL) + *s++ = DIR_SEP; +#endif +#if DIR_SEP != '\\' + s = filename; + while ((s = strchr(s, '\\')) != NULL) + *s++ = DIR_SEP; +#endif + + bn->batCacheid = (bat) batid; + BATinit_idents(bn); + bn->batTransient = false; + bn->batCopiedtodisk = true; + switch ((properties & 0x06) >> 1) { + case 0: + bn->batRestricted = BAT_WRITE; + break; + case 1: + bn->batRestricted = BAT_READ; + break; + case 2: + bn->batRestricted = BAT_APPEND; + break; + default: + TRC_CRITICAL(GDK, "incorrect batRestricted value"); + return -1; + } + bn->batCount = (BUN) count; + bn->batInserted = bn->batCount; + /* set capacity to at least count */ + bn->batCapacity = (BUN) count <= BATTINY ? BATTINY : (BUN) count; + + if (base > (uint64_t) GDK_oid_max) { + TRC_CRITICAL(GDK, "head seqbase out of range (ID = %" PRIu64 ", seq = %" PRIu64 ") on line %d.", batid, base, *lineno); + return -1; + } + bn->hseqbase = (oid) base; + n = heapinit(bn, buf + nread, +#ifdef GDKLIBRARY_HASHASH + hashash, +#endif + bbpversion, filename, *lineno); + if (n < 0) { + return -1; + } + nread += n; + n = vheapinit(bn, buf + nread, filename, *lineno); + if (n < 0) { + return -1; + } + nread += n; + + if (buf[nread] != '\n' && buf[nread] != ' ') { + TRC_CRITICAL(GDK, "invalid format for BBP.dir on line %d", *lineno); + return -1; + } + *options = (buf[nread] == ' ') ? buf + nread + 1 : NULL; + return 1; +} + static gdk_return BBPreadEntries(FILE *fp, unsigned bbpversion, int lineno #ifdef GDKLIBRARY_HASHASH @@ -607,101 +722,114 @@ BBPreadEntries(FILE *fp, unsigned bbpver #endif ) { - bat bid = 0; - char buf[4096]; #ifdef GDKLIBRARY_HASHASH bat *hbats = NULL; _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
