Changeset: 84b3c2e2dc2d for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/84b3c2e2dc2d Modified Files: .bumpversion.cfg MonetDB.spec clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc cmake/monetdb-versions.cmake gdk/gdk_logger.c gdk/gdk_logger_old.c gdk/gdk_storage.c gdk/libbat.rc monetdb5/tools/libmonetdb5.rc sql/storage/bat/bat_logger.c sql/storage/bat/bat_storage.c testing/Mtest.py.in testing/sqltest.py Branch: default Log Message:
Merge with Jan2022 branch. diffs (truncated from 685 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -769,3 +769,4 @@ 1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3 1252291e5c0ddc91ccb16d612d04e34e6a7d3bc3 Jun2020_SP2_release 59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_15 59de1ee118d4eccc072c0cf3938f90635a7db311 Jan2022_SP3_release +38ba6e40ba9148c762fe4a4b40f4937f86e516b6 Jul2021_25 diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022 --- a/gdk/ChangeLog.Jan2022 +++ b/gdk/ChangeLog.Jan2022 @@ -1,6 +1,11 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Tue Aug 2 2022 Sjoerd Mullender <[email protected]> +- When destroying a bat, make sure there are no files left over in + the BACKUP directory since they can cause problems when the bat id + gets reused. + * Thu Jul 28 2022 Sjoerd Mullender <[email protected]> - Fixed an off-by-one error in the logger which caused older log files to stick around longer in the write-ahead log than necessary. diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -162,10 +162,10 @@ log_find(BAT *b, BAT *d, int val) static log_bid internal_find_bat(logger *lg, log_id id, int tid) { - BATiter cni = bat_iterator(lg->catalog_id); BUN p; if (BAThash(lg->catalog_id) == GDK_SUCCEED) { + BATiter cni = bat_iterator(lg->catalog_id); MT_rwlock_rdlock(&cni.b->thashlock); if (tid < 0) { HASHloop_int(cni, cni.b->thash, p, &id) { @@ -192,9 +192,10 @@ internal_find_bat(logger *lg, log_id id, } } MT_rwlock_rdunlock(&cni.b->thashlock); + bat_iterator_end(&cni); + return 0; /* not found */ } - bat_iterator_end(&cni); - return 0; + return -1; /* error creating hash */ } static void @@ -650,8 +651,9 @@ la_bat_update_count(logger *lg, log_id i } } MT_rwlock_rdunlock(&cni.b->thashlock); + return GDK_SUCCEED; } - return GDK_SUCCEED; + return GDK_FAIL; } static gdk_return @@ -660,6 +662,8 @@ la_bat_updates(logger *lg, logaction *la log_bid bid = internal_find_bat(lg, la->cid, tid); BAT *b = NULL; + if (bid < 0) + return GDK_FAIL; if (bid == 0) return GDK_SUCCEED; /* ignore bats no longer in the catalog */ @@ -751,6 +755,8 @@ la_bat_destroy(logger *lg, logaction *la { log_bid bid = internal_find_bat(lg, la->cid, tid); + if (bid < 0) + return GDK_FAIL; if (bid && log_del_bat(lg, bid) != GDK_SUCCEED) return GDK_FAIL; return GDK_SUCCEED; @@ -2622,6 +2628,10 @@ log_bat_transient(logger *lg, log_id id) log_bid bid = internal_find_bat(lg, id, -1); logformat l; + if (bid < 0) { + log_unlock(lg); + return GDK_FAIL; + } l.flag = LOG_DESTROY; l.id = id; @@ -3036,6 +3046,8 @@ log_add_bat(logger *lg, BAT *b, log_id i b == lg->seqs_val || b == lg->dseqs); assert(b->batRole == PERSISTENT); + if (bid < 0) + return GDK_FAIL; if (bid) { if (bid != b->batCacheid) { if (log_del_bat(lg, bid) != GDK_SUCCEED) diff --git a/gdk/gdk_logger_old.c b/gdk/gdk_logger_old.c --- a/gdk/gdk_logger_old.c +++ b/gdk/gdk_logger_old.c @@ -306,6 +306,7 @@ old_logger_find_bat(old_logger *lg, cons } } MT_rwlock_rdunlock(&cni.b->thashlock); + return 0; /* not found */ } } else { BATiter cni = bat_iterator_nolock(lg->catalog_oid); @@ -324,9 +325,10 @@ old_logger_find_bat(old_logger *lg, cons } } MT_rwlock_rdunlock(&cni.b->thashlock); + return 0; /* not found */ } } - return 0; + return -1; /* BAThash failed */ } static gdk_return @@ -335,6 +337,9 @@ la_bat_clear(old_logger *lg, logaction * log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid); BAT *b; + if (bid < 0) + return GDK_FAIL; + if (lg->lg->debug & 1) fprintf(stderr, "#la_bat_clear %s\n", NAME(la->name, la->tpe, la->cid)); @@ -402,6 +407,8 @@ static log_return log_read_updates(old_logger *lg, trans *tr, logformat *l, char *name, int tpe, oid id, int pax) { log_bid bid = old_logger_find_bat(lg, name, tpe, id); + if (bid < 0) + return LOG_ERR; BAT *b = BATdescriptor(bid); log_return res = LOG_OK; int ht = -1, tt = -1, tseq = 0; @@ -618,6 +625,8 @@ la_bat_updates(old_logger *lg, logaction log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid); BAT *b; + if (bid < 0) + return GDK_FAIL; if (bid == 0) return GDK_SUCCEED; /* ignore bats no longer in the catalog */ @@ -697,6 +706,8 @@ la_bat_destroy(old_logger *lg, logaction { log_bid bid = old_logger_find_bat(lg, la->name, la->tpe, la->cid); + if (bid < 0) + return GDK_FAIL; if (bid) { BUN p; @@ -1487,6 +1498,8 @@ logger_load(const char *fn, char filenam goto error; } snapshots_bid = old_logger_find_bat(lg, "snapshots_bid", 0, 0); + if (snapshots_bid < 0) + goto error; if (snapshots_bid == 0) { lg->snapshots_bid = logbat_new(TYPE_int, 1, TRANSIENT); lg->snapshots_tid = logbat_new(TYPE_int, 1, TRANSIENT); @@ -1501,6 +1514,8 @@ logger_load(const char *fn, char filenam bat snapshots_tid = old_logger_find_bat(lg, "snapshots_tid", 0, 0); bat dsnapshots = old_logger_find_bat(lg, "dsnapshots", 0, 0); + if (snapshots_tid < 0 || dsnapshots < 0) + goto error; GDKdebug &= ~CHECKMASK; lg->snapshots_bid = BATdescriptor(snapshots_bid); if (lg->snapshots_bid == NULL) { @@ -1914,6 +1929,8 @@ logger_add_bat(old_logger *lg, BAT *b, c b == lg->seqs_val || b == lg->dseqs); assert(b->batRole == PERSISTENT); + if (bid < 0) + return GDK_FAIL; if (bid) { if (bid != b->batCacheid) { if (logger_del_bat(lg, bid) != GDK_SUCCEED) diff --git a/sql/storage/bat/bat_logger.c b/sql/storage/bat/bat_logger.c --- a/sql/storage/bat/bat_logger.c +++ b/sql/storage/bat/bat_logger.c @@ -129,6 +129,14 @@ replace_bat(old_logger *old_lg, logger * } #endif +static BAT * +log_temp_descriptor(log_bid b) +{ + if (b <= 0) + return NULL; + return temp_descriptor(b); +} + #if defined CATALOG_JUN2020 || defined CATALOG_OCT2020 || defined CATALOG_JAN2022 static gdk_return tabins(logger *lg, old_logger *old_lg, bool first, int tt, int nid, ...) @@ -142,7 +150,7 @@ tabins(logger *lg, old_logger *old_lg, b va_start(va, nid); while ((cid = va_arg(va, int)) != 0) { cval = va_arg(va, void *); - if ((b = temp_descriptor(log_find_bat(lg, cid))) == NULL) { + if ((b = log_temp_descriptor(log_find_bat(lg, cid))) == NULL) { va_end(va); return GDK_FAIL; } @@ -1394,18 +1402,18 @@ upgrade(old_logger *lg) BAT *mapold = COLnew(0, TYPE_int, 256, TRANSIENT); BAT *mapnew = COLnew(0, TYPE_int, 256, TRANSIENT); - bats[0].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys_schemas_name", 0, 0)); - bats[0].idbat = temp_descriptor(old_logger_find_bat(lg, "sys_schemas_id", 0, 0)); + bats[0].nmbat = log_temp_descriptor(old_logger_find_bat(lg, "sys_schemas_name", 0, 0)); + bats[0].idbat = log_temp_descriptor(old_logger_find_bat(lg, "sys_schemas_id", 0, 0)); bats[0].parbat = NULL; - bats[0].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys_schemas", 0, 0)); - bats[1].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_name", 0, 0)); - bats[1].idbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_id", 0, 0)); - bats[1].parbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_schema_id", 0, 0)); - bats[1].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys__tables", 0, 0)); - bats[2].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_name", 0, 0)); - bats[2].idbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_id", 0, 0)); - bats[2].parbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_table_id", 0, 0)); - bats[2].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys__columns", 0, 0)); + bats[0].cands = log_temp_descriptor(old_logger_find_bat(lg, "D_sys_schemas", 0, 0)); + bats[1].nmbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__tables_name", 0, 0)); + bats[1].idbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__tables_id", 0, 0)); + bats[1].parbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__tables_schema_id", 0, 0)); + bats[1].cands = log_temp_descriptor(old_logger_find_bat(lg, "D_sys__tables", 0, 0)); + bats[2].nmbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__columns_name", 0, 0)); + bats[2].idbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__columns_id", 0, 0)); + bats[2].parbat = log_temp_descriptor(old_logger_find_bat(lg, "sys__columns_table_id", 0, 0)); + bats[2].cands = log_temp_descriptor(old_logger_find_bat(lg, "D_sys__columns", 0, 0)); if (mapold == NULL || mapnew == NULL) goto bailout; for (int i = 0; i < 3; i++) { @@ -1510,11 +1518,11 @@ upgrade(old_logger *lg) delidx = i; continue; } - BAT *b = temp_descriptor(old_logger_find_bat(lg, tables[i].fullname, 0, 0)); + BAT *b = log_temp_descriptor(old_logger_find_bat(lg, tables[i].fullname, 0, 0)); if (b == NULL) continue; if (delidx >= 0) { - BAT *d = temp_descriptor(old_logger_find_bat(lg, delname, 0, 0)); + BAT *d = log_temp_descriptor(old_logger_find_bat(lg, delname, 0, 0)); BAT *m = BATconstant(0, TYPE_msk, &(msk){false}, BATcount(b), PERSISTENT); if (m == NULL) { bat_destroy(d); @@ -1545,7 +1553,7 @@ upgrade(old_logger *lg) } if (tables[i].hasids && mapold) { BAT *b1, *b2; - BAT *cands = temp_descriptor(old_logger_find_bat(lg, delname, 0, 0)); + BAT *cands = log_temp_descriptor(old_logger_find_bat(lg, delname, 0, 0)); if (cands) { if (BATcount(cands) == 0) { bat_destroy(cands); @@ -1913,7 +1921,7 @@ bl_postversion(void *Store, void *Lg) const int *ocl; /* old eclass */ int *ncl; /* new eclass */ - te = temp_descriptor(log_find_bat(lg, 2014)); /* sys.types.eclass */ + te = log_temp_descriptor(log_find_bat(lg, 2014)); /* sys.types.eclass */ if (te == NULL) return GDK_FAIL; tne = COLnew(te->hseqbase, TYPE_int, BATcount(te), PERSISTENT); @@ -1972,7 +1980,7 @@ bl_postversion(void *Store, void *Lg) BAT *b; /* temp variable */ { /* new BOOLEAN column sys.functions.semantics */ - b = temp_descriptor(log_find_bat(lg, 2017)); /* sys.functions.id */ + b = log_temp_descriptor(log_find_bat(lg, 2017)); /* sys.functions.id */ BAT *sem = BATconstant(b->hseqbase, TYPE_bit, &(bit){1}, BATcount(b), PERSISTENT); bat_destroy(b); if (sem == NULL) @@ -2009,14 +2017,14 @@ bl_postversion(void *Store, void *Lg) } /* sys.functions i.e. deleted rows */ - BAT *del_funcs = temp_descriptor(log_find_bat(lg, 2016)); + BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016)); { /* move sql.degrees, sql.radians, sql.like and sql.ilike functions * from 09_like.sql and 10_math.sql script to sql_types list */ /* sys.functions.name */ - BAT *func_func = temp_descriptor(log_find_bat(lg, 2018)); + BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2018)); /* sys.functions.schema_id */ - BAT *func_schem = temp_descriptor(log_find_bat(lg, 2026)); + BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026)); _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
