Changeset: ef9f119f7058 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/ef9f119f7058 Modified Files: gdk/gdk_batop.c gdk/gdk_bbp.c monetdb5/mal/mal_profiler.c tools/merovingian/daemon/connections.c tools/merovingian/daemon/controlrunner.c tools/merovingian/daemon/forkmserver.c tools/merovingian/daemon/handlers.c tools/merovingian/daemon/merovingian.c tools/merovingian/daemon/merovingian.h tools/merovingian/daemon/multiplex-funnel.c Branch: default Log Message:
Merge with Jan2022 branch. diffs (truncated from 636 to 300 lines): diff --git a/gdk/ChangeLog.Jan2022 b/gdk/ChangeLog.Jan2022 --- a/gdk/ChangeLog.Jan2022 +++ b/gdk/ChangeLog.Jan2022 @@ -1,3 +1,18 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Wed Jun 22 2022 Sjoerd Mullender <[email protected]> +- Make sure heap files of transient bats get deleted when the bat is + destroyed. If the bat was a partial view (sharing the vheap but not + the tail), the tail file wasn't deleted. +- Various changes were made to satisfy newer compilers. +- The batDirtydesc and batDirtyflushed Boolean values have been deprecated + and are no longer used. They were both holdovers from long ago. +- Various race conditions (data races) have been fixed. + +* Thu May 19 2022 Sjoerd Mullender <[email protected]> +- All accesses to the BACKUP directory need to be protected by the + same lock. The lock already existed (GDKtmLock), but wasn't used + consistently. This is now fixed. Hopefully this makes the hot snapshot + code more reliable. + diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -2797,7 +2797,6 @@ PROPdestroy_nolock(BAT *b) b->tprops = NULL; while (p) { - /* only set dirty if a saved property is changed */ n = p->next; VALclear(&p->v); GDKfree(p); diff --git a/monetdb5/ChangeLog.Jan2022 b/monetdb5/ChangeLog.Jan2022 --- a/monetdb5/ChangeLog.Jan2022 +++ b/monetdb5/ChangeLog.Jan2022 @@ -1,3 +1,6 @@ # ChangeLog file for MonetDB5 # This file is updated with Maddlog +* Wed Jun 22 2022 Sjoerd Mullender <[email protected]> +- Various race conditions (data races) have been fixed. + diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c --- a/monetdb5/mal/mal_profiler.c +++ b/monetdb5/mal/mal_profiler.c @@ -36,12 +36,13 @@ static const char *myname = 0; // avoid /* The JSON rendering can be either using '\n' separators between * each key:value pair or as a single line. * The current stethoscope implementation requires the first option and - * also the term rendering to be set to '' + * also the term rendering to be set to '' */ /* When the MAL block contains a BARRIER block we may end up with tons * of profiler events. To avoid this, we stop emitting the events - * when we reached the HIGHWATERMARK. Leaving a message in the log. */ + * when we reached the HIGHWATERMARK. Leaving a message in the log. + */ #define HIGHWATERMARK 5 @@ -66,7 +67,7 @@ static void logjsonInternal(char *logbuf len = strlen(logbuffer); if (maleventstream) { - // upon request the log record is sent over the profile stream + // upon request the log record is sent over the profile stream (void) mnstr_write(maleventstream, logbuffer, 1, len); if (flush) (void) mnstr_flush(maleventstream, MNSTR_FLUSH_DATA); @@ -170,17 +171,17 @@ logadd(struct logbuf *logbuf, const char /* JSON rendering method of performance data. * The eventparser may assume this layout for ease of parsing -EXAMPLE: -{ -"event":6 , -"thread":3, -"function":"user.s3_1", -"pc":1, -"tag":10397, -"state":"start", -"usec":0, -} -"stmt":"X_41=0@0:void := querylog.define(\"select count(*) from tables;\":str,\"default_pipe\":str,30:int);", + EXAMPLE: + { + "event":6 , + "thread":3, + "function":"user.s3_1", + "pc":1, + "tag":10397, + "state":"start", + "usec":0, + } + "stmt":"X_41=0@0:void := querylog.define(\"select count(*) from tables;\":str,\"default_pipe\":str,30:int);", */ static str prepareProfilerEvent(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int start) @@ -203,9 +204,9 @@ prepareProfilerEvent(Client cntxt, MalBl return NULL; } -/* The stream of events can be complete read by the DBA, - * all other users can only see events assigned to their account - */ + /* The stream of events can be complete read by the DBA, + * all other users can only see events assigned to their account + */ if(malprofileruser!= MAL_ADMIN && malprofileruser != cntxt->user) return NULL; diff --git a/tools/merovingian/ChangeLog.Jan2022 b/tools/merovingian/ChangeLog.Jan2022 --- a/tools/merovingian/ChangeLog.Jan2022 +++ b/tools/merovingian/ChangeLog.Jan2022 @@ -1,3 +1,18 @@ # ChangeLog file for sql/src/backends/monet5/merovingian # This file is updated with mchangelog +* Thu Jun 23 2022 Sjoerd Mullender <[email protected]> +- When multiple identical messages are written to the log, write the + first one, and combine subsequent ones in a single message. + +* Wed Jun 22 2022 Sjoerd Mullender <[email protected]> +- Fixed a leak where the log file wasn't closed when it was reopened + after a log rotation (SIGHUP signal). +- Try to deal more gracefully with "inherited" mserver5 processes. + This includes not complaining about an "impossible state", and allowing + such processes to be stopped by the monetdbd process. +- When a transient failure occurs during processing of a new connection to + the monetdbd server, sleep for half a second so that if the transient + failure occurs again, the log file doesn't get swamped with error + messages. + diff --git a/tools/merovingian/daemon/connections.c b/tools/merovingian/daemon/connections.c --- a/tools/merovingian/daemon/connections.c +++ b/tools/merovingian/daemon/connections.c @@ -96,13 +96,13 @@ openConnectionIP(int *socks, bool udp, c } #if !defined(SOCK_CLOEXEC) && defined(HAVE_FCNTL) if (fcntl(sock, F_SETFD, FD_CLOEXEC) < 0) - Mlevelfprintf(ERROR, log, "fcntl FD_CLOEXEC: %s", strerror(e)); + Mlevelfprintf(ERROR, log, "fcntl FD_CLOEXEC: %s\n", strerror(e)); #endif if (rp->ai_family == AF_INET6 && setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char *) &(int){0}, sizeof(int)) == -1) - Mlevelfprintf(ERROR, log, "setsockopt IPV6_V6ONLY: %s", strerror(e)); + Mlevelfprintf(ERROR, log, "setsockopt IPV6_V6ONLY: %s\n", strerror(e)); if (!udp) { if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, @@ -114,12 +114,12 @@ openConnectionIP(int *socks, bool udp, c #ifdef SO_EXCLUSIVEADDRUSE if (setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, (const char *) &on, sizeof on) < 0) - Mlevelfprintf(ERROR, log, "setsockopt SO_EXCLUSIVEADDRUSE: %s", strerror(e)); + Mlevelfprintf(ERROR, log, "setsockopt SO_EXCLUSIVEADDRUSE: %s\n", strerror(e)); #endif #ifdef SO_EXCLBIND if (setsockopt(sock, SOL_SOCKET, SO_EXCLBIND, (const char *) &on, sizeof on) < 0) - Mlevelfprintf(ERROR, log, "setsockopt SO_EXCLBIND: %s", strerror(e)); + Mlevelfprintf(ERROR, log, "setsockopt SO_EXCLBIND: %s\n", strerror(e)); #endif } diff --git a/tools/merovingian/daemon/controlrunner.c b/tools/merovingian/daemon/controlrunner.c --- a/tools/merovingian/daemon/controlrunner.c +++ b/tools/merovingian/daemon/controlrunner.c @@ -676,7 +676,7 @@ static void ctl_handle_client( Mlevelfprintf(INFORMATION, _mero_ctlout, "Start snapshot of database '%s' to file '%s'\n", q, dest); char *e = snapshot_database_to(q, dest); if (e != NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot database '%s' to %s failed: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot database '%s' to %s failed: %s\n", origin, q, dest, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); @@ -691,7 +691,7 @@ static void ctl_handle_client( char *dest = NULL; char *e = snapshot_default_filename(&dest, q); if (e != NULL) { - Mlevelfprintf(INFORMATION, _mero_ctlerr, "%s: snapshot database '%s': %s", + Mlevelfprintf(INFORMATION, _mero_ctlerr, "%s: snapshot database '%s': %s\n", origin, q, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); @@ -700,7 +700,7 @@ static void ctl_handle_client( Mlevelfprintf(INFORMATION, _mero_ctlout, "Start snapshot of database '%s' to file '%s'\n", q, dest); e = snapshot_database_to(q, dest); if (e != NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot database '%s' to %s failed: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot database '%s' to %s failed: %s\n", origin, q, dest, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); @@ -750,7 +750,7 @@ static void ctl_handle_client( if (wrapper) mnstr_destroy(wrapper); if (e != NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "%s: streaming snapshot database '%s' failed: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "%s: streaming snapshot database '%s' failed: %s\n", origin, q, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); @@ -767,7 +767,7 @@ static void ctl_handle_client( Mlevelfprintf(INFORMATION, _mero_ctlout, "Start restore snapshot of database '%s' from file '%s'\n", q, source); char *e = snapshot_restore_from(q, source); if (e != NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "%s: restore database '%s' from snapshot %s failed: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "%s: restore database '%s' from snapshot %s failed: %s\n", origin, q, source, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); @@ -797,7 +797,7 @@ static void ctl_handle_client( struct snapshot *snaps = NULL; char *e = snapshot_list(&nsnaps, &snaps); if (e != NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot list failed: %s", origin, getErrMsg(e)); + Mlevelfprintf(ERROR, _mero_ctlerr, "%s: snapshot list failed: %s\n", origin, getErrMsg(e)); len = snprintf(buf2, sizeof(buf2), "%s\n", getErrMsg(e)); send_client("!"); freeErr(e); @@ -1175,7 +1175,7 @@ controlRunner(void *d) do { if ((p = malloc(sizeof(int))) == NULL) { - Mlevelfprintf(ERROR, _mero_ctlerr, "malloc failed"); + Mlevelfprintf(ERROR, _mero_ctlerr, "malloc failed\n"); break; } /* limit waiting time in order to check whether we need to exit */ @@ -1221,7 +1221,7 @@ controlRunner(void *d) case ENOBUFS: case ENOMEM: /* transient failure, wait a little and continue */ - Mlevelfprintf(ERROR, _mero_ctlerr, "error during accept: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "error during accept: %s\n", strerror(errno)); sleep_ms(500); break; @@ -1230,7 +1230,7 @@ controlRunner(void *d) break; default: /* anything else */ - Mlevelfprintf(ERROR, _mero_ctlerr, "error during accept: %s", + Mlevelfprintf(ERROR, _mero_ctlerr, "error during accept: %s\n", strerror(errno)); break; } diff --git a/tools/merovingian/daemon/forkmserver.c b/tools/merovingian/daemon/forkmserver.c --- a/tools/merovingian/daemon/forkmserver.c +++ b/tools/merovingian/daemon/forkmserver.c @@ -384,8 +384,8 @@ forkMserver(const char *database, sabdb* /* fill in the rest of the dpair entry */ pthread_mutex_lock(&_mero_topdp_lock); - dp->out = pfdo[0]; - dp->err = pfde[0]; + dp->input[0].fd = pfdo[0]; + dp->input[1].fd = pfde[0]; dp->type = MEROFUN; dp->pid = getpid(); dp->flag = 0; @@ -698,9 +698,9 @@ forkMserver(const char *database, sabdb* exit(1); } else if (pid > 0) { /* parent: fine, let's add the pipes for this child */ - dp->out = pfdo[0]; + dp->input[0].fd = pfdo[0]; close(pfdo[1]); - dp->err = pfde[0]; + dp->input[1].fd = pfde[0]; close(pfde[1]); dp->type = MERODB; dp->pid = pid; diff --git a/tools/merovingian/daemon/handlers.c b/tools/merovingian/daemon/handlers.c --- a/tools/merovingian/daemon/handlers.c +++ b/tools/merovingian/daemon/handlers.c @@ -255,14 +255,14 @@ void reinitialize(void) Mlevelfprintf(INFORMATION, _mero_logfile, "%s END merovingian[%lld]: " "caught SIGHUP, closing logfile\n", mytime, (long long int)_mero_topdp->next->pid); - _mero_topdp->out = _mero_topdp->err = t; + _mero_topdp->input[0].fd = _mero_topdp->input[1].fd = t; FILE *f = _mero_logfile; if ((_mero_logfile = fdopen(t, "a")) == NULL) { /* revert to old log so that we have something */ Mlevelfprintf(ERROR, f, "%s ERR merovingian[%lld]: " - "failed to reopen logfile", + "failed to reopen logfile\n", mytime, (long long int)_mero_topdp->next->pid); - _mero_topdp->out = _mero_topdp->err = fileno(f); + _mero_topdp->input[0].fd = _mero_topdp->input[1].fd = fileno(f); _mero_logfile = f; } else { fclose(f); _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
