Changeset: dce9e9015efa for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/dce9e9015efa Modified Files: clients/Tests/exports.stable.out gdk/gdk_bbp.c gdk/gdk_logger.c gdk/gdk_storage.c sql/backends/monet5/sql.c sql/backends/monet5/sql_upgrades.c sql/storage/store.c Branch: default Log Message:
Merge with Mar2025 branch. diffs (truncated from 1424 to 300 lines): diff --git a/ChangeLog.Mar2025 b/ChangeLog.Mar2025 --- a/ChangeLog.Mar2025 +++ b/ChangeLog.Mar2025 @@ -1,6 +1,15 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Fri Jun 13 2025 Joeri van Ruth <[email protected]> +- Add optional parameters omit_unlogged (bool) and omit_table_ids (str) to + sys.hot_snapshot(). If omit_unlogged is set to true, the data in UNLOGGED + tables is omitted from the snapshot. If omit_table_ids is given, it must + be a comma-separated list of table ids as found in sys.tables. The data in + each of those tables will be omitted from the snapshot. +- Empty BATs are omitted from the snapshot, the restored server will created + them if necessary. + * Thu May 8 2025 Sjoerd Mullender <[email protected]> - It is now possible to specify an idle timeout using --set idle_timeout=<seconds> (see mserver5 manual page) which gets triggered diff --git a/clients/Tests/MAL-signatures-hge.test b/clients/Tests/MAL-signatures-hge.test --- a/clients/Tests/MAL-signatures-hge.test +++ b/clients/Tests/MAL-signatures-hge.test @@ -49129,6 +49129,16 @@ unsafe pattern sql.hot_snapshot(X_0:str, SQLhot_snapshot Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit, X_3:str):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql importColumn pattern sql.importColumn(X_0:str, X_1:int, X_2:bit, X_3:str, X_4:int, X_5:oid) (X_6:bat[:any], X_7:oid) mvc_bin_import_column_wrap diff --git a/clients/Tests/MAL-signatures.test b/clients/Tests/MAL-signatures.test --- a/clients/Tests/MAL-signatures.test +++ b/clients/Tests/MAL-signatures.test @@ -37584,6 +37584,16 @@ unsafe pattern sql.hot_snapshot(X_0:str, SQLhot_snapshot Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit, X_3:str):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql importColumn pattern sql.importColumn(X_0:str, X_1:int, X_2:bit, X_3:str, X_4:int, X_5:oid) (X_6:bat[:any], X_7:oid) mvc_bin_import_column_wrap diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -215,6 +215,9 @@ gdk_return BATupdatepos(BAT *b, const oi BBPrec *BBP[N_BBPINIT]; gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); void BBPcold(bat i); +gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp); +gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf); +bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp); int BBPfix(bat b); unsigned BBPheader(FILE *fp, int *lineno, bat *bbpsize, lng *logno, bool allow_hge_upgrade); bat BBPindex(const char *nme); diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -2158,7 +2158,7 @@ BBPdir_header(FILE *f, int n, lng logno) return GDK_SUCCEED; } -static gdk_return +gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp) { FILE *obbpf = NULL, *nbbpf = NULL; @@ -2202,6 +2202,8 @@ BBPdir_first(bool subcommit, lng logno, GDKerror("cannot read BBPinfo in backup BBP.dir."); goto bailout; } + if (logno < 0) + logno = ologno; } if (n < (bat) ATOMIC_GET(&BBPsize)) @@ -2227,7 +2229,7 @@ BBPdir_first(bool subcommit, lng logno, return GDK_FAIL; } -static bat +bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp) { @@ -2266,7 +2268,8 @@ BBPdir_step(bat bid, BUN size, int n, ch goto bailout; nbat++; } - *nbatp += nbat; + if (nbatp) + *nbatp += nbat; return n == -1 ? -1 : n == bid ? 0 : n; bailout: @@ -2276,7 +2279,7 @@ BBPdir_step(bat bid, BUN size, int n, ch return -2; } -static gdk_return +gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf) { if (n > 0 && fputs(buf, nbbpf) == EOF) { diff --git a/gdk/gdk_bbp.h b/gdk/gdk_bbp.h --- a/gdk/gdk_bbp.h +++ b/gdk/gdk_bbp.h @@ -107,4 +107,10 @@ gdk_export gdk_return BBPjson_upgrade(js #define BBPswappable(b) ((b) && (b)->batCacheid && BBP_refs((b)->batCacheid) == 0) #define BBPtrimmable(b) (BBPswappable(b) && isVIEW(b) == 0 && (BBP_status((b)->batCacheid)&BBPWAITING) == 0) +/* low level support for patching BBP.dir */ +gdk_export gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp); +gdk_export bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp); +gdk_export gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf); + + #endif /* _GDK_BBP_H_ */ diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -376,7 +376,7 @@ struct offset { }; static log_return -log_read_updates(logger *lg, trans *tr, logformat *l, log_id id, BAT **cands) +log_read_updates(logger *lg, trans *tr, logformat *l, log_id id, BAT **cands, bool skip_entry) { log_return res = LOG_OK; lng nr, pnr; @@ -401,7 +401,7 @@ log_read_updates(logger *lg, trans *tr, lng offset; assert(nr <= (lng) BUN_MAX); - if (!lg->flushing && l->flag == LOG_UPDATE) { + if (!lg->flushing && !skip_entry && l->flag == LOG_UPDATE) { uid = COLnew(0, TYPE_oid, (BUN) nr, PERSISTENT); if (uid == NULL) { TRC_CRITICAL(GDK, "creating bat failed\n"); @@ -417,12 +417,12 @@ log_read_updates(logger *lg, trans *tr, if (cands) { /* This const range actually represents a segment of candidates corresponding to updated bat entries */ - if (BATcount(*cands) == 0 || lg->flushing) { + if (BATcount(*cands) == 0 || lg->flushing || skip_entry) { /* when flushing, we only need the offset and count of the last segment of inserts. */ assert((*cands)->ttype == TYPE_void); BATtseqbase(*cands, (oid) offset); BATsetcount(*cands, (BUN) nr); - } else if (!lg->flushing) { + } else if (!lg->flushing && !skip_entry) { assert(BATcount(*cands) > 0); BAT *dense = BATdense(0, (oid) offset, (BUN) nr); BAT *newcands = NULL; @@ -459,7 +459,7 @@ log_read_updates(logger *lg, trans *tr, } } - if (!lg->flushing) { + if (!lg->flushing && !skip_entry) { r = COLnew(0, tpe, (BUN) nr, PERSISTENT); if (r == NULL) { if (uid) @@ -643,7 +643,7 @@ log_read_updates(logger *lg, trans *tr, GDKfree(hv); } - if (res == LOG_OK) { + if (res == LOG_OK && !skip_entry) { if (tr_grow(tr) == GDK_SUCCEED) { tr->changes[tr->nr].type = l->flag; if (l->flag == LOG_UPDATE_BULK && offset == -1) { @@ -654,7 +654,7 @@ log_read_updates(logger *lg, trans *tr, const oid last = canditer_last(&ci); offset = (lng) first; pnr = (lng) (last - first) + 1; - if (!lg->flushing) { + if (!lg->flushing && !skip_entry) { assert(uid == NULL); uid = *cands; BBPfix((*cands)->batCacheid); @@ -1282,12 +1282,13 @@ log_open_input(logger *lg, const char *f } static log_return -log_read_transaction(logger *lg, uint32_t *updated, BUN maxupdated, time_t *t) +log_read_transaction(logger *lg, BAT *ids_to_omit, uint32_t *updated, BUN maxupdated, time_t *t) { logformat l; trans *tr = NULL; log_return err = LOG_OK; bool ok = true; + bool skip_entry = false; ATOMIC_BASE_TYPE dbg = ATOMIC_GET(&GDKdebug); time_t t0 = 0; size_t fs = 0; @@ -1332,10 +1333,14 @@ log_read_transaction(logger *lg, uint32_ else TRC_DEBUG_ENDIF(WAL, "%d %d", l.flag, l.id); } + skip_entry = (ids_to_omit && BUNfnd(ids_to_omit, &l.id) != BUN_NONE); switch (l.flag) { case LOG_UPDATE_CONST: case LOG_UPDATE_BULK: case LOG_UPDATE: + if (skip_entry) + break; + /* fall through */ case LOG_CREATE: case LOG_DESTROY: if (tr != NULL && updated && BAThash(lg->catalog_id) == GDK_SUCCEED) { @@ -1418,7 +1423,7 @@ log_read_transaction(logger *lg, uint32_ if (tr == NULL) err = LOG_EOF; else { - err = log_read_updates(lg, tr, &l, l.id, cands ? &cands : NULL); + err = log_read_updates(lg, tr, &l, l.id, cands ? &cands : NULL, skip_entry); } break; case LOG_CREATE: @@ -1483,7 +1488,7 @@ log_read_transaction(logger *lg, uint32_ } static gdk_return -log_readlog(logger *lg, const char *filename, bool *filemissing) +log_readlog(logger *lg, const char *filename, BAT *ids_to_omit, bool *filemissing) { log_return err = LOG_OK; time_t t0; @@ -1499,7 +1504,7 @@ log_readlog(logger *lg, const char *file GDKtracer_flush_buffer(); } while (err != LOG_EOF && err != LOG_ERR) { - err = log_read_transaction(lg, NULL, 0, &t0); + err = log_read_transaction(lg, ids_to_omit, NULL, 0, &t0); } log_close_input(lg); lg->input_log = NULL; @@ -1515,6 +1520,55 @@ log_readlog(logger *lg, const char *file return err == LOG_ERR ? GDK_FAIL : GDK_SUCCEED; } +static gdk_return +read_omitted_ids(const char *filename, BAT **ids_to_omit) +{ + gdk_return ret = GDK_FAIL; + BAT *ids_bat = NULL; + FILE *f = fopen(filename, "r"); + if (!f) { + if (errno != ENOENT) { + GDKsyserror("fopen %s failed\n", filename); + goto end; + } + /* file does not exist, return NULL. */ + ids_bat = NULL; + ret = GDK_SUCCEED; + goto end; + } + + ids_bat = COLnew(0, TYPE_int, 0, TRANSIENT); + if (!ids_bat) { + GDKerror("read_omitted_ids: cannot create bat"); + goto end; + } + while (1) { + int id; + if (fscanf(f, "%d", &id) == 1) { + if (BUNappend(ids_bat, &id, true) != GDK_SUCCEED) { + GDKerror("read_omitted_ids: cannot append to bat"); + goto end; + } + } else { + break; + } + } + if (ferror(f)) { + GDKsyserror("fscanf %s failed\n", filename); + goto end; + } + + ret = GDK_SUCCEED; +end: + if (f) _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
