Changeset: 3940753824ca for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/3940753824ca Modified Files: .bumpversion.cfg .hgtags MonetDB.spec clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc cmake/monetdb-versions.cmake gdk/gdk_bat.c gdk/gdk_bbp.c gdk/gdk_logger.c gdk/gdk_posix.c gdk/gdk_private.h gdk/gdk_tracer.c gdk/libbat.rc monetdb5/tools/libmonetdb5.rc sql/test/miscellaneous/Tests/groupby_error.test testing/sqllogictest.py tools/mserver/mserver5.c Branch: Jun2023 Log Message:
Merge with Sep2022 branch. diffs (truncated from 346 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -799,3 +799,5 @@ bb95192134c0d8ef8fea525204eefcd0de099a54 24385ee6130494440e09d31ec73f4aa2e2893e98 Jul2021_33 05b3d9ea9165baf1d2956d5b06ad85f817ecc6d2 Jul2021_33 05b3d9ea9165baf1d2956d5b06ad85f817ecc6d2 Jul2021_SP10_release +ed34ab4283cbfcc57fb9d7387bc9d781c91902b1 Jan2022_27 +ed34ab4283cbfcc57fb9d7387bc9d781c91902b1 Jan2022_SP7_release diff --git a/gdk/ChangeLog.Sep2022 b/gdk/ChangeLog.Sep2022 --- a/gdk/ChangeLog.Sep2022 +++ b/gdk/ChangeLog.Sep2022 @@ -1,6 +1,11 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Tue May 16 2023 Sjoerd Mullender <[email protected]> +- Warnings and informational messages are now sent to stdout instead of + stderr, which means that monetdbd will now log them with the tag MSG + instead of ERR. + * Tue Apr 25 2023 Sjoerd Mullender <[email protected]> - Fixed parsing of the BBP.dir file when BAT ids grow larger than 2**24 (i.e. 100000000 in octal). diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -363,9 +363,9 @@ recover_dir(int farmid, bool direxists) if (direxists) { /* just try; don't care about these non-vital files */ if (GDKunlink(farmid, BATDIR, "BBP", "bak") != GDK_SUCCEED) - TRC_WARNING(GDK, "unlink of BBP.bak failed\n"); + GDKwarning("unlink of BBP.bak failed\n"); if (GDKmove(farmid, BATDIR, "BBP", "dir", BATDIR, "BBP", "bak", false) != GDK_SUCCEED) - TRC_WARNING(GDK, "rename of BBP.dir to BBP.bak failed\n"); + GDKwarning("rename of BBP.dir to BBP.bak failed\n"); } return GDKmove(farmid, BAKDIR, "BBP", "dir", BATDIR, "BBP", "dir", true); } @@ -486,7 +486,7 @@ heapinit(BAT *b, const char *buf, } if (strcmp(type, "wkba") == 0) - TRC_WARNING(GDK, "type wkba (SQL name: GeometryA) is deprecated\n"); + GDKwarning("type wkba (SQL name: GeometryA) is deprecated\n"); if (properties & ~0x0F81) { TRC_CRITICAL(GDK, "unknown properties are set: incompatible database on line %d of BBP.dir\n", lineno); @@ -1442,7 +1442,7 @@ movestrbats(void) GDKerror("both %s and %s exist with %s unexpectedly newer: manual intervention required\n", oldpath, newpath, oldpath); ret = -1; } else { - TRC_WARNING(GDK, "both %s and %s exist, removing %s\n", oldpath, newpath, oldpath); + GDKwarning("both %s and %s exist, removing %s\n", oldpath, newpath, oldpath); ret = MT_remove(oldpath); } } else { @@ -3250,8 +3250,8 @@ BBPquickdesc(bat bid) const char *aname = ATOMunknown_name(b->ttype); int tt = ATOMindex(aname); if (tt < 0) { - TRC_WARNING(GDK, "atom '%s' unknown in bat '%s'.\n", - aname, BBP_physical(bid)); + GDKwarning("atom '%s' unknown in bat '%s'.\n", + aname, BBP_physical(bid)); } else { b->ttype = tt; } diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -754,8 +754,7 @@ la_bat_destroy(logger *lg, logaction *la if (bid < 0) return GDK_FAIL; if (!bid) { - GDKerror("la_bat_destroy failed to find bid for object %d (issue ignored)\n", la->cid); - GDKclrerr(); + GDKwarning("failed to find bid for object %d\n", la->cid); return GDK_SUCCEED; } if (bid && log_del_bat(lg, bid) != GDK_SUCCEED) @@ -1530,7 +1529,7 @@ cleanup_and_swap(logger *lg, int *r, con if ((lb = BATdescriptor(bids[pos])) == NULL || BATmode(lb, true/*transient*/) != GDK_SUCCEED) { - TRC_WARNING(GDK, "Failed to set bat(%d) transient\n", bids[pos]); + GDKwarning("Failed to set bat(%d) transient\n", bids[pos]); } logbat_destroy(lb); } @@ -1820,8 +1819,8 @@ log_cleanup(logger *lg, lng id) return GDK_FAIL; } if (GDKunlink(0, lg->dir, LOGFILE, log_id) != GDK_SUCCEED) { - TRC_WARNING(GDK, "#log_cleanup: failed to remove old WAL %s.%s\n", LOGFILE, log_id); - GDKclrerr(); + GDKwarning("failed to remove old WAL %s.%s\n", LOGFILE, log_id); + GDKclrerr(); /* clear error from unlink */ } return GDK_SUCCEED; } @@ -3079,7 +3078,7 @@ bm_commit(logger *lg) assert(bid); if ((lb = BATdescriptor(bid)) == NULL || BATmode(lb, false) != GDK_SUCCEED) { - TRC_WARNING(GDK, "Failed to set bat (%d%s) persistent\n", bid, !lb?" gone":""); + GDKwarning("Failed to set bat (%d%s) persistent\n", bid, !lb?" gone":""); logbat_destroy(lb); log_unlock(lg); return GDK_FAIL; diff --git a/gdk/gdk_posix.c b/gdk/gdk_posix.c --- a/gdk/gdk_posix.c +++ b/gdk/gdk_posix.c @@ -396,7 +396,7 @@ MT_mremap(const char *path, int mode, vo return old_address; } if (path && truncate(path, *new_size) < 0) - TRC_WARNING(GDK, "MT_mremap(%s): truncate failed: %s\n", + GDKwarning("truncate of %s failed: %s\n", path, GDKstrerror(errno, (char[64]){0}, 64)); #endif /* !__COVERITY__ */ return old_address; diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h --- a/gdk/gdk_private.h +++ b/gdk/gdk_private.h @@ -176,6 +176,9 @@ gdk_return GDKtracer_init(const char *db __attribute__((__visibility__("hidden"))); gdk_return GDKunlink(int farmid, const char *dir, const char *nme, const char *extension) __attribute__((__visibility__("hidden"))); +#define GDKwarning(format, ...) \ + GDKtracer_log(__FILE__, __func__, __LINE__, M_WARNING, \ + GDK, NULL, format, ##__VA_ARGS__) BUN HASHappend(BAT *b, BUN i, const void *v) __attribute__((__visibility__("hidden"))); void HASHappend_locked(BAT *b, BUN i, const void *v) diff --git a/gdk/gdk_tm.c b/gdk/gdk_tm.c --- a/gdk/gdk_tm.c +++ b/gdk/gdk_tm.c @@ -78,7 +78,7 @@ epilogue(int cnt, bat *subcommit, bool l /* check mmap modes */ MT_lock_set(&b->theaplock); if (BATcheckmodes(b, true) != GDK_SUCCEED) - TRC_WARNING(GDK, "BATcheckmodes failed\n"); + GDKwarning("BATcheckmodes failed\n"); MT_lock_unset(&b->theaplock); } } diff --git a/gdk/gdk_tracer.c b/gdk/gdk_tracer.c --- a/gdk/gdk_tracer.c +++ b/gdk/gdk_tracer.c @@ -580,7 +580,8 @@ GDKtracer_log(const char *file, const ch reinit(); if (level <= M_WARNING || (ATOMIC_GET(&GDKdebug) & FORCEMITOMASK)) { - fprintf(stderr, "#%s%s%s: %s: %s: %s%s%s\n", + fprintf(level <= M_ERROR ? stderr : stdout, + "#%s%s%s: %s: %s: %s%s%s\n", add_ts ? ts : "", add_ts ? ": " : "", MT_thread_getname(), func, level_str[level] + 2, diff --git a/sql/ChangeLog.Sep2022 b/sql/ChangeLog.Sep2022 --- a/sql/ChangeLog.Sep2022 +++ b/sql/ChangeLog.Sep2022 @@ -7,7 +7,7 @@ * Fri Mar 24 2023 Sjoerd Mullender <[email protected]> - Increased the size of a variable counting the number of changes made - to the database (e.g. in case more than a 2 billion rows are added to + to the database (e.g. in case more than 2 billion rows are added to a table). - Improved cleanup after failures such as failed memory allocations. diff --git a/sql/test/concurrent/Tests/truncate-insert-flood.py b/sql/test/concurrent/Tests/truncate-insert-flood.SQL.py rename from sql/test/concurrent/Tests/truncate-insert-flood.py rename to sql/test/concurrent/Tests/truncate-insert-flood.SQL.py --- a/sql/test/concurrent/Tests/truncate-insert-flood.py +++ b/sql/test/concurrent/Tests/truncate-insert-flood.SQL.py @@ -1,66 +1,38 @@ -import sys, os, tempfile, pymonetdb - -try: - from MonetDBtesting import process -except ImportError: - import process - from concurrent.futures import ThreadPoolExecutor +import os, pymonetdb nr_clients = 16 -with tempfile.TemporaryDirectory() as dbfarm: - os.mkdir(os.path.join(dbfarm, 'db')) +db = os.getenv("TSTDB") +port = os.getenv("MAPIPORT") - with open(os.path.join(dbfarm, 'errout'), mode='a+') as errout: - with process.server( - dbname='db', - dbfarm=os.path.join(dbfarm, 'db'), - args=[ "--set", "gdk_nr_threads=0", "--forcemito"], - stdin=process.PIPE, - stdout=process.PIPE, - stderr=errout, - mapiport='0') as db_proc: - - def client(id): - conn = pymonetdb.connect( - database='db', - port=db_proc.dbport, - autocommit=True) - cursor = conn.cursor() +def client(id): + conn = pymonetdb.connect( + database=db, + port=port, + autocommit=True) + cursor = conn.cursor() - init = f''' - drop table if exists foo_{id}; - create table foo_{id} (c1, c2, c3, c4, c5) AS VALUES - (10, 20, 30, 40, 50), - (11, 21, 31, 41, 51), - (12, 22, 32, 42, 52); - set optimizer = 'minimal_fast'; - ''' - cursor.execute(init) + init = f''' + drop table if exists foo_{id}; + create table foo_{id} (c1, c2, c3, c4, c5) AS VALUES + (10, 20, 30, 40, 50), + (11, 21, 31, 41, 51), + (12, 22, 32, 42, 52); + set optimizer = 'minimal_fast'; + ''' + cursor.execute(init) - truncate_and_insert_queries = f""" - truncate foo_{id}; - insert into foo_{id} VALUES - (10, 20, 30, 40, 50), - (11, 21, 31, 41, 51), - (12, 22, 32, 42, 52); - """ - nr_queries = 1600 - for x in range(0, nr_queries): - cursor.execute(truncate_and_insert_queries) + truncate_and_insert_queries = f""" + truncate foo_{id}; + insert into foo_{id} VALUES + (10, 20, 30, 40, 50), + (11, 21, 31, 41, 51), + (12, 22, 32, 42, 52); + """ + nr_queries = 1600 + for x in range(0, nr_queries): + cursor.execute(truncate_and_insert_queries) - with ThreadPoolExecutor(nr_clients) as pool: - pool.map(client, range(nr_clients)) - errout.flush() - - errout.seek(0) - - error = False - for line in errout: - line = line.strip() - if "builtin" not in line and "cmdline" not in line: - print("UNEXPECTED OUTPUT:", line, file=sys.stderr) - error = True - if error: - exit(1) +with ThreadPoolExecutor(nr_clients) as pool: + pool.map(client, range(nr_clients)) diff --git a/sql/test/miscellaneous/Tests/groupby_error.test b/sql/test/miscellaneous/Tests/groupby_error.test --- a/sql/test/miscellaneous/Tests/groupby_error.test +++ b/sql/test/miscellaneous/Tests/groupby_error.test @@ -100,6 +100,16 @@ NULL NULL NULL +#output differs for 1 vs. multiple threads +skipif threads=1 +query T nosort +PLAN SELECT DISTINCT col0, col1, col2, col0 FROM tab0 +---- +group by ( +| table("sys"."tab0") [ "tab0"."col0", "tab0"."col1", "tab0"."col2" ] +) [ "tab0"."col2", "tab0"."col0", "tab0"."col1" ] [ "tab0"."col0", "tab0"."col1", "tab0"."col2", "tab0"."col0" ] + +onlyif threads=1 query T nosort PLAN SELECT DISTINCT col0, col1, col2, col0 FROM tab0 ---- diff --git a/testing/sqllogictest.py b/testing/sqllogictest.py --- a/testing/sqllogictest.py +++ b/testing/sqllogictest.py @@ -664,6 +664,7 @@ class SQLLogic: def parse(self, f, approve=None, verbose=False, defines=None): self.approve = approve _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
