Changeset: bfde6ef66b84 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/bfde6ef66b84 Modified Files: sql/server/rel_exp.c sql/server/rel_select.c Branch: balanced_union Log Message:
Merges with default diffs (truncated from 1816 to 300 lines): diff --git a/ChangeLog.Jun2023 b/ChangeLog.Jun2023 --- a/ChangeLog.Jun2023 +++ b/ChangeLog.Jun2023 @@ -1,6 +1,10 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Wed Aug 30 2023 Sjoerd Mullender <[email protected]> +- Do a lot more error checking, mostly for allocation failures. More is + still needed, though. + * Thu Aug 10 2023 Panagiotis Koutsourakis <[email protected]> - Improve performance of the ILIKE operator when the pattern contains only ASCII characters. In this case we do not need to treat any characters as diff --git a/buildtools/coverity_model.c b/buildtools/coverity_model.c --- a/buildtools/coverity_model.c +++ b/buildtools/coverity_model.c @@ -126,7 +126,7 @@ GDKmmap(const char *path, int mode, size } gdk_return -GDKmunmap(void *p, size_t size) +GDKmunmap(void *p, int mode, size_t size) { int failed; __coverity_free__(p); @@ -139,7 +139,7 @@ GDKmremap(const char *path, int mode, vo { void *p = GDKmmap(path, mode, new_size); if (p) { - (void) GDKmunmap(old_address, old_size); + (void) GDKmunmap(old_address, mode, old_size); } return p; } diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -299,7 +299,7 @@ size_t GDKmem_cursize(void); gdk_return GDKmergeidx(BAT *b, BAT **a, int n_ar); void *GDKmmap(const char *path, int mode, size_t len) __attribute__((__warn_unused_result__)); int GDKms(void); -gdk_return GDKmunmap(void *addr, size_t len); +gdk_return GDKmunmap(void *addr, int mode, size_t len); int GDKnr_threads; void GDKprepareExit(void); void GDKprintinfo(void); diff --git a/debian/monetdb5-sql.README.Debian b/debian/monetdb5-sql.README.Debian --- a/debian/monetdb5-sql.README.Debian +++ b/debian/monetdb5-sql.README.Debian @@ -2,7 +2,16 @@ Usage ===== -There are multiple ways to use MonetDB5-sql, but this is the most Debianic. +There are multiple ways to use MonetDB5-sql. + +The modern way is to do: + + * Enable and start MonetDB in one go:: + + $ sudo systemctl enable --now monetdbd.service + +This is the old way using System V init scripts, incompatible with the +above: * Set STARTUP to "yes" in /etc/default/monetdb5-sql @@ -10,7 +19,10 @@ There are multiple ways to use MonetDB5- $ sudo /etc/init.d/monetdb5-sql start - * Add yourself to the "monetdb" group:: +In either case, do this: + + * Add yourself to the "monetdb" group (after this, logout and back in + for this to take effect):: $ sudo addgroup `whoami` monetdb diff --git a/gdk/ChangeLog.Jun2023 b/gdk/ChangeLog.Jun2023 --- a/gdk/ChangeLog.Jun2023 +++ b/gdk/ChangeLog.Jun2023 @@ -1,3 +1,18 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Wed Aug 30 2023 Sjoerd Mullender <[email protected]> +- Only check for virtual memory limits when creating or growing bats, + not for general memory allocations. There is (still) too much code + that doesn't properly handle failing allocations, so we need to avoid + those as much as possible. This has mostly an effect if there are + virtual memory size restrictions imposed by cgroups (memory.swap.max + in cgroups v2, memory.memsw.limit_in_bytes in cgroups v1). +- The low-level commit turned out to always commit every persistent bat + in the system. There is no need for that, it should only commit bats + that were changed. This has now been fixed. +- Implemented timeout/exit checks in a bunch more operators. Long(er) + running operators occasionally check whether they're taking too long + (past a user-specified timeout) or whether the server is exiting. + This is now done in more places. + diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -1468,10 +1468,11 @@ movestrbats(void) } #endif -static void +static bool BBPtrim(bool aggressive) { int n = 0; + bool changed = false; unsigned flag = BBPUNLOADING | BBPSYNCING | BBPSAVING; if (!aggressive) flag |= BBPHOT; @@ -1502,19 +1503,23 @@ BBPtrim(bool aggressive) if (BBPfree(b) != GDK_SUCCEED) GDKerror("unload failed for bat %d", bid); n++; + changed = true; } BBPtmunlock(); } TRC_DEBUG(BAT_, "unloaded %d bats%s\n", n, aggressive ? " (also hot)" : ""); + return changed; } static void BBPmanager(void *dummy) { (void) dummy; + bool changed = true; for (;;) { int n = 0; + MT_thread_setworking("clearing HOT bits"); for (bat bid = 1, nbat = (bat) ATOMIC_GET(&BBPsize); bid < nbat; bid++) { MT_lock_set(&GDKswapLock(bid)); if (BBP_refs(bid) == 0 && BBP_lrefs(bid) != 0) { @@ -1525,12 +1530,15 @@ BBPmanager(void *dummy) } TRC_DEBUG(BAT_, "cleared HOT bit from %d bats\n", n); size_t cur = GDKvm_cursize(); - for (int i = 0, n = cur > GDK_vm_maxsize / 2 ? 1 : cur > GDK_vm_maxsize / 4 ? 10 : 100; i < n; i++) { + MT_thread_setworking("sleeping"); + for (int i = 0, n = changed && cur > GDK_vm_maxsize / 2 ? 1 : cur > GDK_vm_maxsize / 4 ? 10 : 100; i < n; i++) { MT_sleep_ms(100); if (GDKexiting()) return; } - BBPtrim(false); + MT_thread_setworking("BBPtrim"); + changed = BBPtrim(false); + MT_thread_setworking("BBPcallbacks"); BBPcallbacks(); if (GDKexiting()) return; @@ -2069,7 +2077,7 @@ BBPdir_first(bool subcommit, lng logno, * replacing the entries for the subcommitted bats */ if ((obbpf = GDKfileopen(0, SUBDIR, "BBP", "dir", "r")) == NULL && (obbpf = GDKfileopen(0, BAKDIR, "BBP", "dir", "r")) == NULL) { - GDKsyserror("subcommit attempted without backup BBP.dir."); + GDKsyserror("subcommit attempted without backup BBP.dir"); goto bailout; } /* read first three lines */ @@ -2135,7 +2143,7 @@ BBPdir_step(bat bid, BUN size, int n, ch } n = -1; if (fclose(*obbpfp) == EOF) { - GDKsyserror("Closing backup BBP.dir file failed.\n"); + GDKsyserror("Closing backup BBP.dir file failed\n"); GDKclrerr(); /* ignore error */ } *obbpfp = NULL; @@ -2173,7 +2181,7 @@ BBPdir_last(int n, char *buf, size_t buf goto bailout; } if (fclose(obbpf) == EOF) { - GDKsyserror("Closing backup BBP.dir file failed.\n"); + GDKsyserror("Closing backup BBP.dir file failed\n"); GDKclrerr(); /* ignore error */ } obbpf = NULL; @@ -3894,7 +3902,7 @@ BBPsync(int cnt, bat *restrict subcommit MT_rename(bakdir, deldir) < 0)) ret = GDK_FAIL; if (ret != GDK_SUCCEED) - GDKsyserror("rename(%s,%s) failed.\n", bakdir, deldir); + GDKsyserror("rename(%s,%s) failed\n", bakdir, deldir); TRC_DEBUG(IO_, "rename %s %s = %d\n", bakdir, deldir, (int) ret); } diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c --- a/gdk/gdk_heap.c +++ b/gdk/gdk_heap.c @@ -687,7 +687,11 @@ HEAPfree(Heap *h, bool rmheap) //heap is stored in regular C memory rather than GDK memory,so we call free() free(h->base); } else if (h->storage != STORE_NOWN) { /* mapped file, or STORE_PRIV */ - gdk_return ret = GDKmunmap(h->base, h->size); + gdk_return ret = GDKmunmap(h->base, + h->storage == STORE_PRIV ? + MMAP_COPY | MMAP_READ | MMAP_WRITE : + MMAP_READ | MMAP_WRITE, + h->size); if (ret != GDK_SUCCEED) { GDKsyserror("HEAPfree: %s was not mapped\n", diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c --- a/gdk/gdk_join.c +++ b/gdk/gdk_join.c @@ -43,7 +43,7 @@ * BATsemijoin * equi-join, but the left output is sorted, and if there are * multiple matches, only one is returned (i.e., the left output - * is also key) + * is also key, making it a candidate list) * BATthetajoin * theta-join: an extra operator must be provided encoded as an * integer (macros JOIN_EQ, JOIN_NE, JOIN_LT, JOIN_LE, JOIN_GT, @@ -301,11 +301,15 @@ nomatch(BAT **r1p, BAT **r2p, BAT *l, BA static gdk_return selectjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, struct canditer *lci, struct canditer *rci, - bool nil_matches, lng t0, bool swapped, const char *reason) + bool nil_matches, bool nil_on_miss, bool semi, bool max_one, bool min_one, + lng t0, bool swapped, const char *reason) { BATiter li = bat_iterator(l); const void *v; BAT *bn = NULL; + BAT *r1 = NULL; + BAT *r2 = NULL; + BUN bncount; assert(lci->ncand > 0); assert(lci->ncand == 1 || (li.sorted && li.revsorted)); @@ -325,7 +329,7 @@ selectjoin(BAT **r1p, BAT **r2p, BAT *l, (*ATOMcompare(li.type))(v, ATOMnilptr(li.type)) == 0) { /* NIL doesn't match anything */ bat_iterator_end(&li); - return nomatch(r1p, r2p, l, r, lci, false, false, + return nomatch(r1p, r2p, l, r, lci, nil_on_miss, false, reason, t0); } @@ -334,83 +338,116 @@ selectjoin(BAT **r1p, BAT **r2p, BAT *l, if (bn == NULL) { return GDK_FAIL; } - if (BATcount(bn) == 0) { - BBPunfix(bn->batCacheid); - return nomatch(r1p, r2p, l, r, lci, false, false, - reason, t0); + bncount = BATcount(bn); + if (bncount == 0) { + BBPreclaim(bn); + if (min_one) { + GDKerror("not enough matches"); + return GDK_FAIL; + } + if (!nil_on_miss) + return nomatch(r1p, r2p, l, r, lci, nil_on_miss, false, + reason, t0); + /* special case: return nil on RHS */ + bncount = 1; + bn = NULL; } - BAT *r1 = COLnew(0, TYPE_oid, lci->ncand * BATcount(bn), TRANSIENT); - if (r1 == NULL) { - BBPunfix(bn->batCacheid); - return GDK_FAIL; + if (bncount > 1) { + if (semi) + bncount = 1; + if (max_one) { + GDKerror("more than one match"); + goto bailout; + } } + r1 = COLnew(0, TYPE_oid, lci->ncand * bncount, TRANSIENT); + if (r1 == NULL) + goto bailout; r1->tsorted = true; r1->trevsorted = lci->ncand == 1; - r1->tseqbase = BATcount(bn) == 1 && lci->tpe == cand_dense ? o : oid_nil; - r1->tkey = BATcount(bn) == 1; + r1->tseqbase = bncount == 1 && lci->tpe == cand_dense ? o : oid_nil; + r1->tkey = bncount == 1; r1->tnil = false; r1->tnonil = true; - BAT *r2 = NULL; - if (r2p) { - r2 = COLnew(0, TYPE_oid, lci->ncand * BATcount(bn), TRANSIENT); - if (r2 == NULL) { - BBPunfix(bn->batCacheid); _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
