Changeset: f9c1199a8e5e for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/f9c1199a8e5e Modified Files: clients/Tests/exports.stable.out gdk/gdk.h sql/backends/monet5/rel_bin.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statement.c sql/server/rel_unnest.c Branch: groupjoin Log Message:
merged with default diffs (truncated from 6300 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -808,3 +808,4 @@ aba8c31c60480148cd388a09ac99b42679406258 79dbf838f04483a2d9ccce8332090ff91b18caec Jun2023_3 79dbf838f04483a2d9ccce8332090ff91b18caec Jun2023_release 573511e0e7bf2f7ab11f00b45711aab5f1aff6f2 Jun2023_5 +573511e0e7bf2f7ab11f00b45711aab5f1aff6f2 Jun2023_SP1_release diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Thu Jul 27 2023 Niels Nes <[email protected]> +- Removed the PYTHON MAP external language option, as after a fork the + synchronization primitives could be in any state, leading to deadlocks. + During the upgrade function definitions will fallback to the normal + PYTHON language option. + diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -523,9 +523,7 @@ ssize_t fltToStr(str *dst, size_t *len, const flt flt_nil; gdk_return gdk_add_callback(char *name, gdk_callback_func *f, int argc, void *argv[], int interval); gdk_return gdk_remove_callback(char *, gdk_callback_func *f); -lng getBBPlogno(void); bat getBBPsize(void); -lng getBBPtransid(void); char *get_bin_path(void); int gettimeofday(struct timeval *tv, int *ignore_zone); struct tm *gmtime_r(const time_t *restrict, struct tm *restrict); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -2432,20 +2432,32 @@ gdk_export BAT *BATsample_with_seed(BAT * on each iteration */ #define TIMEOUT_LOOP_IDX(IDX, REPEATS, TIMEOFFSET) \ for (BUN REPS = (IDX = 0, (REPEATS)); REPS > 0; REPS = 0) /* "loops" at most once */ \ - for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ - for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) + for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++) \ + if (GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET)) { \ + TIMEOFFSET = -1; \ + break; \ + } else \ + for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) /* declare and use IDX as a loop variable, initializing it to 0 and * incrementing it on each iteration */ #define TIMEOUT_LOOP_IDX_DECL(IDX, REPEATS, TIMEOFFSET) \ for (BUN IDX = 0, REPS = (REPEATS); REPS > 0; REPS = 0) /* "loops" at most once */ \ - for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ - for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) + for (BUN CTR1 = 0, END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++) \ + if (GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET)) { \ + TIMEOFFSET = -1; \ + break; \ + } else \ + for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++, IDX++) /* there is no user-visible loop variable */ #define TIMEOUT_LOOP(REPEATS, TIMEOFFSET) \ - for (BUN CTR1 = 0, REPS = (REPEATS), END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++, TIMEOFFSET = GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET) ? -1 : TIMEOFFSET) \ - for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++) + for (BUN CTR1 = 0, REPS = (REPEATS), END1 = (REPS + CHECK_QRY_TIMEOUT_STEP) >> CHECK_QRY_TIMEOUT_SHIFT; CTR1 < END1 && TIMEOFFSET >= 0; CTR1++) \ + if (GDKexiting() || (TIMEOFFSET > 0 && GDKusec() > TIMEOFFSET)) { \ + TIMEOFFSET = -1; \ + break; \ + } else \ + for (BUN CTR2 = 0, END2 = CTR1 == END1 - 1 ? REPS & CHECK_QRY_TIMEOUT_MASK : CHECK_QRY_TIMEOUT_STEP; CTR2 < END2; CTR2++) /* break out of the loop (cannot use do/while trick here) */ #define TIMEOUT_LOOP_BREAK \ diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -2008,6 +2008,12 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT BUN i; oid o; + lng timeoffset = 0; + QryCtx *qry_ctx = MT_thread_get_qry_ctx(); + if (qry_ctx != NULL) { + timeoffset = (qry_ctx->starttime && qry_ctx->querytimeout) ? (qry_ctx->starttime + qry_ctx->querytimeout) : 0; + } + if ((err = BATgroupaggrinit(b, g, e, s, &min, &max, &ngrp, &ci)) != NULL) { GDKerror("%s\n", err); return GDK_FAIL; @@ -2053,7 +2059,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT case TYPE_bte: { const bte *vals = (const bte *) bi.base; bte *avgs = Tloc(bn, 0); - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { o = canditer_next(&ci) - b->hseqbase; if (ngrp > 1) gid = gids ? gids[o] - min : o; @@ -2070,7 +2076,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT AVERAGE_ITER(bte, vals[o], avgs[gid], rems[gid], cnts[gid]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = bte_nil; bn->tnil = true; @@ -2101,7 +2107,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT case TYPE_sht: { const sht *vals = (const sht *) bi.base; sht *avgs = Tloc(bn, 0); - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { o = canditer_next(&ci) - b->hseqbase; if (ngrp > 1) gid = gids ? gids[o] - min : o; @@ -2118,7 +2124,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT AVERAGE_ITER(sht, vals[o], avgs[gid], rems[gid], cnts[gid]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = sht_nil; bn->tnil = true; @@ -2149,7 +2155,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT case TYPE_int: { const int *vals = (const int *) bi.base; int *avgs = Tloc(bn, 0); - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { o = canditer_next(&ci) - b->hseqbase; if (ngrp > 1) gid = gids ? gids[o] - min : o; @@ -2166,7 +2172,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT AVERAGE_ITER(int, vals[o], avgs[gid], rems[gid], cnts[gid]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = int_nil; bn->tnil = true; @@ -2197,7 +2203,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT case TYPE_lng: { const lng *vals = (const lng *) bi.base; lng *avgs = Tloc(bn, 0); - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { o = canditer_next(&ci) - b->hseqbase; if (ngrp > 1) gid = gids ? gids[o] - min : o; @@ -2214,7 +2220,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT AVERAGE_ITER(lng, vals[o], avgs[gid], rems[gid], cnts[gid]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = lng_nil; bn->tnil = true; @@ -2246,7 +2252,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT case TYPE_hge: { const hge *vals = (const hge *) bi.base; hge *avgs = Tloc(bn, 0); - CAND_LOOP(&ci) { + TIMEOUT_LOOP(ci.ncand, timeoffset) { o = canditer_next(&ci) - b->hseqbase; if (ngrp > 1) gid = gids ? gids[o] - min : o; @@ -2263,7 +2269,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT AVERAGE_ITER(hge, vals[o], avgs[gid], rems[gid], cnts[gid]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = hge_nil; bn->tnil = true; @@ -2294,6 +2300,7 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT #endif } bat_iterator_end(&bi); + TIMEOUT_CHECK(timeoffset, GOTO_LABEL_TIMEOUT_HANDLER(bailout)); BATsetcount(bn, ngrp); BATsetcount(rn, ngrp); BATsetcount(cn, ngrp); @@ -2307,6 +2314,12 @@ BATgroupavg3(BAT **avgp, BAT **remp, BAT *remp = rn; *cntp = cn; return GDK_SUCCEED; + + bailout: + BBPreclaim(bn); + BBPreclaim(rn); + BBPreclaim(cn); + return GDK_FAIL; } #ifdef HAVE_HGE @@ -2636,6 +2649,12 @@ BATgroupavg3combine(BAT *avg, BAT *rem, BUN i; BAT *bn, *rn, *cn; + lng timeoffset = 0; + QryCtx *qry_ctx = MT_thread_get_qry_ctx(); + if (qry_ctx != NULL) { + timeoffset = (qry_ctx->starttime && qry_ctx->querytimeout) ? (qry_ctx->starttime + qry_ctx->querytimeout) : 0; + } + if ((err = BATgroupaggrinit(avg, g, e, NULL, &min, &max, &ngrp, &ci)) != NULL) { GDKerror("%s\n", err); return NULL; @@ -2675,7 +2694,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, case TYPE_bte: { const bte *vals = (const bte *) bi.base; bte *avgs = Tloc(bn, 0); - CAND_LOOP_IDX(&ci, i) { + TIMEOUT_LOOP_IDX(i, ci.ncand, timeoffset) { if (ngrp > 1) gid = gids ? gids[i] - min : i; if (is_bte_nil(vals[i])) { @@ -2691,7 +2710,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, ocnts[i]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = bte_nil; bn->tnil = true; @@ -2716,7 +2735,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, case TYPE_sht: { const sht *vals = (const sht *) bi.base; sht *avgs = Tloc(bn, 0); - CAND_LOOP_IDX(&ci, i) { + TIMEOUT_LOOP_IDX(i, ci.ncand, timeoffset) { if (ngrp > 1) gid = gids ? gids[i] - min : i; if (is_sht_nil(vals[i])) { @@ -2732,7 +2751,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, ocnts[i]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = sht_nil; bn->tnil = true; @@ -2757,7 +2776,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, case TYPE_int: { const int *vals = (const int *) bi.base; int *avgs = Tloc(bn, 0); - CAND_LOOP_IDX(&ci, i) { + TIMEOUT_LOOP_IDX(i, ci.ncand, timeoffset) { if (ngrp > 1) gid = gids ? gids[i] - min : i; if (is_int_nil(vals[i])) { @@ -2773,7 +2792,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, ocnts[i]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = int_nil; bn->tnil = true; @@ -2798,7 +2817,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, case TYPE_lng: { const lng *vals = (const lng *) bi.base; lng *avgs = Tloc(bn, 0); - CAND_LOOP_IDX(&ci, i) { + TIMEOUT_LOOP_IDX(i, ci.ncand, timeoffset) { if (ngrp > 1) gid = gids ? gids[i] - min : i; if (is_lng_nil(vals[i])) { @@ -2814,7 +2833,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, ocnts[i]); } } - for (i = 0; i < ngrp; i++) { + TIMEOUT_LOOP_IDX(i, ngrp, timeoffset) { if (cnts[i] == 0) { avgs[i] = lng_nil; bn->tnil = true; @@ -2840,7 +2859,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, case TYPE_hge: { const hge *vals = (const hge *) bi.base; hge *avgs = Tloc(bn, 0); - CAND_LOOP_IDX(&ci, i) { + TIMEOUT_LOOP_IDX(i, ci.ncand, timeoffset) { if (ngrp > 1) gid = gids ? gids[i] - min : i; if (is_hge_nil(vals[i])) { @@ -2856,7 +2875,7 @@ BATgroupavg3combine(BAT *avg, BAT *rem, ocnts[i]); } _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
