Changeset: 08d9e284ef70 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/08d9e284ef70 Modified Files: clients/Tests/MAL-signatures-hge.test clients/Tests/MAL-signatures.test clients/Tests/exports.stable.out sql/backends/monet5/sql.c sql/backends/monet5/sql_gencode.c sql/backends/monet5/sql_statement.c sql/backends/monet5/sql_upgrades.c sql/server/rel_rel.c sql/server/rel_updates.c sql/server/sql_parser.y sql/server/sql_symbol.c sql/server/sql_symbol.h sql/storage/bat/bat_logger.c sql/storage/sql_storage.h sql/storage/store.c sql/test/emptydb/Tests/check.stable.out sql/test/emptydb/Tests/check.stable.out.int128 tools/mserver/mserver5.c Branch: nested Log Message:
merged with default diffs (truncated from 2398 to 300 lines): diff --git a/ChangeLog.Mar2025 b/ChangeLog.Mar2025 --- a/ChangeLog.Mar2025 +++ b/ChangeLog.Mar2025 @@ -1,6 +1,15 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Fri Jun 13 2025 Joeri van Ruth <[email protected]> +- Add optional parameters omit_unlogged (bool) and omit_table_ids (str) to + sys.hot_snapshot(). If omit_unlogged is set to true, the data in UNLOGGED + tables is omitted from the snapshot. If omit_table_ids is given, it must + be a comma-separated list of table ids as found in sys.tables. The data in + each of those tables will be omitted from the snapshot. +- Empty BATs are omitted from the snapshot, the restored server will created + them if necessary. + * Thu May 8 2025 Sjoerd Mullender <[email protected]> - It is now possible to specify an idle timeout using --set idle_timeout=<seconds> (see mserver5 manual page) which gets triggered diff --git a/clients/Tests/MAL-signatures-hge.test b/clients/Tests/MAL-signatures-hge.test --- a/clients/Tests/MAL-signatures-hge.test +++ b/clients/Tests/MAL-signatures-hge.test @@ -49219,6 +49219,16 @@ unsafe pattern sql.hot_snapshot(X_0:str, SQLhot_snapshot Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit, X_3:str):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql importColumn pattern sql.importColumn(X_0:str, X_1:int, X_2:bit, X_3:str, X_4:int, X_5:oid) (X_6:bat[:any], X_7:oid) mvc_bin_import_column_wrap diff --git a/clients/Tests/MAL-signatures.test b/clients/Tests/MAL-signatures.test --- a/clients/Tests/MAL-signatures.test +++ b/clients/Tests/MAL-signatures.test @@ -37674,6 +37674,16 @@ unsafe pattern sql.hot_snapshot(X_0:str, SQLhot_snapshot Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql +hot_snapshot +unsafe pattern sql.hot_snapshot(X_0:str, X_1:bit, X_2:bit, X_3:str):void +SQLhot_snapshot +Write db snapshot to the given tar(.gz/.lz4/.bz/.xz) file on either server or client, omitting some bats +sql importColumn pattern sql.importColumn(X_0:str, X_1:int, X_2:bit, X_3:str, X_4:int, X_5:oid) (X_6:bat[:any], X_7:oid) mvc_bin_import_column_wrap diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -215,6 +215,9 @@ gdk_return BATupdatepos(BAT *b, const oi BBPrec *BBP[N_BBPINIT]; gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); void BBPcold(bat i); +gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp); +gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf); +bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp); int BBPfix(bat b); unsigned BBPheader(FILE *fp, int *lineno, bat *bbpsize, lng *logno, bool allow_hge_upgrade); bat BBPindex(const char *nme); diff --git a/cmake/monetdb-defines.cmake b/cmake/monetdb-defines.cmake --- a/cmake/monetdb-defines.cmake +++ b/cmake/monetdb-defines.cmake @@ -84,6 +84,8 @@ function(monetdb_configure_defines) check_symbol_exists("strerror_r" "string.h" HAVE_STRERROR_R) check_function_exists("lockf" HAVE_LOCKF) check_symbol_exists("madvise" "sys/mman.h" HAVE_MADVISE) + check_function_exists("malloc_info" HAVE_MALLOC_INFO) + check_function_exists("mallinfo2" HAVE_MALLINFO2) check_symbol_exists("mremap" "sys/mman.h" HAVE_MREMAP) check_function_exists("nanosleep" HAVE_NANOSLEEP) check_function_exists("nl_langinfo" HAVE_NL_LANGINFO) @@ -145,6 +147,16 @@ macro(monetdb_macro_variables) set(HAVE_RTREE ${RTREE_FOUND}) set(HAVE_OPENSSL ${OPENSSL_FOUND}) + if(MALLOC_FOUND) + if(${WITH_MALLOC} STREQUAL "jemalloc") + set(WITH_JEMALLOC 1) + elseif(${WITH_MALLOC} STREQUAL "mimalloc") + set(WITH_MIMALLOC 1) + elseif(${WITH_MALLOC} STREQUAL "tcmalloc") + set(WITH_TCMALLOC 1) + endif() + endif() + if(PY3INTEGRATION) set(HAVE_LIBPY3 "${Python3_NumPy_FOUND}") set(PY3VER "${Python3_VERSION_MINOR}") diff --git a/gdk/CMakeLists.txt b/gdk/CMakeLists.txt --- a/gdk/CMakeLists.txt +++ b/gdk/CMakeLists.txt @@ -118,7 +118,8 @@ target_link_libraries(bat $<$<PLATFORM_ID:Windows>:ws2_32> $<$<BOOL:${KVM_FOUND}>:KVM::KVM> ${CMAKE_DL_LIBS} - Threads::Threads) + Threads::Threads + $<$<BOOL:${MALLOC_FOUND}>:Malloc::Malloc>) # This definition is also used in Linux target_compile_definitions(bat diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -1315,7 +1315,7 @@ BATappend_or_update(BAT *b, BAT *p, cons continue; } - bool isnil = atomcmp(new, nil) == 0; + bool isnil = !ni.nonil && atomcmp(new, nil) == 0; anynil |= isnil; MT_lock_set(&b->theaplock); if (old == NULL || diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -2158,7 +2158,7 @@ BBPdir_header(FILE *f, int n, lng logno) return GDK_SUCCEED; } -static gdk_return +gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp) { FILE *obbpf = NULL, *nbbpf = NULL; @@ -2202,6 +2202,8 @@ BBPdir_first(bool subcommit, lng logno, GDKerror("cannot read BBPinfo in backup BBP.dir."); goto bailout; } + if (logno < 0) + logno = ologno; } if (n < (bat) ATOMIC_GET(&BBPsize)) @@ -2227,7 +2229,7 @@ BBPdir_first(bool subcommit, lng logno, return GDK_FAIL; } -static bat +bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp) { @@ -2266,7 +2268,8 @@ BBPdir_step(bat bid, BUN size, int n, ch goto bailout; nbat++; } - *nbatp += nbat; + if (nbatp) + *nbatp += nbat; return n == -1 ? -1 : n == bid ? 0 : n; bailout: @@ -2276,7 +2279,7 @@ BBPdir_step(bat bid, BUN size, int n, ch return -2; } -static gdk_return +gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf) { if (n > 0 && fputs(buf, nbbpf) == EOF) { diff --git a/gdk/gdk_bbp.h b/gdk/gdk_bbp.h --- a/gdk/gdk_bbp.h +++ b/gdk/gdk_bbp.h @@ -107,4 +107,10 @@ gdk_export gdk_return BBPjson_upgrade(js #define BBPswappable(b) ((b) && (b)->batCacheid && BBP_refs((b)->batCacheid) == 0) #define BBPtrimmable(b) (BBPswappable(b) && isVIEW(b) == 0 && (BBP_status((b)->batCacheid)&BBPWAITING) == 0) +/* low level support for patching BBP.dir */ +gdk_export gdk_return BBPdir_first(bool subcommit, lng logno, FILE **obbpfp, FILE **nbbpfp); +gdk_export bat BBPdir_step(bat bid, BUN size, int n, char *buf, size_t bufsize, FILE **obbpfp, FILE *nbbpf, BATiter *bi, int *nbatp); +gdk_export gdk_return BBPdir_last(int n, char *buf, size_t bufsize, FILE *obbpf, FILE *nbbpf); + + #endif /* _GDK_BBP_H_ */ diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -376,7 +376,7 @@ struct offset { }; static log_return -log_read_updates(logger *lg, trans *tr, logformat *l, log_id id, BAT **cands) +log_read_updates(logger *lg, trans *tr, logformat *l, log_id id, BAT **cands, bool skip_entry) { log_return res = LOG_OK; lng nr, pnr; @@ -401,7 +401,7 @@ log_read_updates(logger *lg, trans *tr, lng offset; assert(nr <= (lng) BUN_MAX); - if (!lg->flushing && l->flag == LOG_UPDATE) { + if (!lg->flushing && !skip_entry && l->flag == LOG_UPDATE) { uid = COLnew(0, TYPE_oid, (BUN) nr, PERSISTENT); if (uid == NULL) { TRC_CRITICAL(GDK, "creating bat failed\n"); @@ -417,12 +417,12 @@ log_read_updates(logger *lg, trans *tr, if (cands) { /* This const range actually represents a segment of candidates corresponding to updated bat entries */ - if (BATcount(*cands) == 0 || lg->flushing) { + if (BATcount(*cands) == 0 || lg->flushing || skip_entry) { /* when flushing, we only need the offset and count of the last segment of inserts. */ assert((*cands)->ttype == TYPE_void); BATtseqbase(*cands, (oid) offset); BATsetcount(*cands, (BUN) nr); - } else if (!lg->flushing) { + } else if (!lg->flushing && !skip_entry) { assert(BATcount(*cands) > 0); BAT *dense = BATdense(0, (oid) offset, (BUN) nr); BAT *newcands = NULL; @@ -459,7 +459,7 @@ log_read_updates(logger *lg, trans *tr, } } - if (!lg->flushing) { + if (!lg->flushing && !skip_entry) { r = COLnew(0, tpe, (BUN) nr, PERSISTENT); if (r == NULL) { if (uid) @@ -643,7 +643,7 @@ log_read_updates(logger *lg, trans *tr, GDKfree(hv); } - if (res == LOG_OK) { + if (res == LOG_OK && !skip_entry) { if (tr_grow(tr) == GDK_SUCCEED) { tr->changes[tr->nr].type = l->flag; if (l->flag == LOG_UPDATE_BULK && offset == -1) { @@ -654,7 +654,7 @@ log_read_updates(logger *lg, trans *tr, const oid last = canditer_last(&ci); offset = (lng) first; pnr = (lng) (last - first) + 1; - if (!lg->flushing) { + if (!lg->flushing && !skip_entry) { assert(uid == NULL); uid = *cands; BBPfix((*cands)->batCacheid); @@ -1282,12 +1282,13 @@ log_open_input(logger *lg, const char *f } static log_return -log_read_transaction(logger *lg, uint32_t *updated, BUN maxupdated, time_t *t) +log_read_transaction(logger *lg, BAT *ids_to_omit, uint32_t *updated, BUN maxupdated, time_t *t) { logformat l; trans *tr = NULL; log_return err = LOG_OK; bool ok = true; + bool skip_entry = false; ATOMIC_BASE_TYPE dbg = ATOMIC_GET(&GDKdebug); time_t t0 = 0; size_t fs = 0; @@ -1332,10 +1333,14 @@ log_read_transaction(logger *lg, uint32_ else TRC_DEBUG_ENDIF(WAL, "%d %d", l.flag, l.id); } + skip_entry = (ids_to_omit && BUNfnd(ids_to_omit, &l.id) != BUN_NONE); switch (l.flag) { case LOG_UPDATE_CONST: case LOG_UPDATE_BULK: case LOG_UPDATE: + if (skip_entry) + break; + /* fall through */ case LOG_CREATE: case LOG_DESTROY: if (tr != NULL && updated && BAThash(lg->catalog_id) == GDK_SUCCEED) { @@ -1418,7 +1423,7 @@ log_read_transaction(logger *lg, uint32_ if (tr == NULL) err = LOG_EOF; else { - err = log_read_updates(lg, tr, &l, l.id, cands ? &cands : NULL); + err = log_read_updates(lg, tr, &l, l.id, cands ? &cands : NULL, skip_entry); } break; case LOG_CREATE: @@ -1483,7 +1488,7 @@ log_read_transaction(logger *lg, uint32_ } static gdk_return -log_readlog(logger *lg, const char *filename, bool *filemissing) +log_readlog(logger *lg, const char *filename, BAT *ids_to_omit, bool *filemissing) { log_return err = LOG_OK; time_t t0; _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
