Changeset: e9c24cd42f20 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/e9c24cd42f20 Modified Files: ChangeLog clients/Tests/exports.stable.out gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_private.h gdk/gdk_select.c sql/backends/monet5/sql_cat.c sql/server/rel_basetable.c sql/server/rel_propagate.c sql/server/rel_rel.c sql/server/rel_rel.h sql/server/rel_schema.c sql/server/sql_mvc.c sql/server/sql_partition.c sql/server/sql_partition.h sql/storage/bat/bat_storage.c sql/storage/sql_storage.h sql/storage/store.c sql/test/merge-partitions/Tests/mergepart01.test sql/test/merge-partitions/Tests/mergepart03.test sql/test/merge-partitions/Tests/mergepart04.test sql/test/merge-partitions/Tests/mergepart05.test sql/test/merge-partitions/Tests/mergepart07.test sql/test/merge-partitions/Tests/mergepart15.test sql/test/merge-partitions/Tests/mergepart17.test sql/test/merge-partitions/Tests/mergepart18.test sql/test/merge-partitions/Tests/mergepart19.test sql/test/merge-partitions/Tests/mergepart29.SQL.py sql/test/merge-partitions/Tests/mergepart30.test sql/test/merge-partitions/Tests/mergepart31.test sql/test/merge-partitions/Tests/mergepart32.test sql/test/merge-partitions/Tests/mergepart34.test Branch: default Log Message:
Implemented min/max bound and not-null properties that SQL pushes to GDK. These properties can be used to more efficiently execute some operations, such as select. The properties are also enforced by BATappend and BATreplace. diffs (truncated from 2016 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,11 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Mon Oct 30 2023 Sjoerd Mullender <[email protected]> +- The ranges of merge partitions are now pushed down into the low + level GDK operations, giving them a handle to sometimes execute more + efficiently. + * Thu Jul 27 2023 Niels Nes <[email protected]> - Removed the PYTHON MAP external language option, as after a fork the synchronization primitives could be in any state, leading to deadlocks. diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -125,6 +125,7 @@ void BATfakeCommit(BAT *b); gdk_return BATfirstn(BAT **topn, BAT **gids, BAT *b, BAT *cands, BAT *grps, BUN n, bool asc, bool nilslast, bool distinct) __attribute__((__warn_unused_result__)); restrict_t BATgetaccess(BAT *b); ValPtr BATgetprop(BAT *b, enum prop_t idx); +ValPtr BATgetprop_nolock(BAT *b, enum prop_t idx); gdk_return BATgroup(BAT **groups, BAT **extents, BAT **histo, BAT *b, BAT *s, BAT *g, BAT *e, BAT *h) __attribute__((__warn_unused_result__)); const char *BATgroupaggrinit(BAT *b, BAT *g, BAT *e, BAT *s, oid *minp, oid *maxp, BUN *ngrpp, struct canditer *ci); gdk_return BATgroupavg(BAT **bnp, BAT **cntsp, BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, int scale); @@ -184,6 +185,8 @@ BAT *BATprojectchain(BAT **bats); gdk_return BATrangejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *rl, BAT *rh, BAT *sl, BAT *sr, bool li, bool hi, bool anti, bool symmetric, BUN estimate) __attribute__((__warn_unused_result__)); gdk_return BATreplace(BAT *b, BAT *p, BAT *n, bool force) __attribute__((__warn_unused_result__)); gdk_return BATreplacepos(BAT *b, const oid *positions, BAT *n, bool autoincr, bool force) __attribute__((__warn_unused_result__)); +void BATrmprop(BAT *b, enum prop_t idx); +void BATrmprop_nolock(BAT *b, enum prop_t idx); gdk_return BATroles(BAT *b, const char *tnme); gdk_return BATrtree(BAT *wkb, BAT *mbr); BAT *BATsample(BAT *b, BUN n); @@ -194,6 +197,8 @@ gdk_return BATsemijoin(BAT **r1p, BAT ** BAT *BATsetaccess(BAT *b, restrict_t mode) __attribute__((__warn_unused_result__)); void BATsetcapacity(BAT *b, BUN cnt); void BATsetcount(BAT *b, BUN cnt); +ValPtr BATsetprop(BAT *b, enum prop_t idx, int type, const void *v); +ValPtr BATsetprop_nolock(BAT *b, enum prop_t idx, int type, const void *v); gdk_return BATsetstrimps(BAT *b); BAT *BATslice(BAT *b, BUN low, BUN high); gdk_return BATsort(BAT **sorted, BAT **order, BAT **groups, BAT *b, BAT *o, BAT *g, bool reverse, bool nilslast, bool stable) __attribute__((__warn_unused_result__)); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -2269,10 +2269,18 @@ gdk_export void VIEWbounds(BAT *b, BAT * * levels. */ enum prop_t { - CURRENTLY_NO_PROPERTIES_DEFINED, + GDK_MIN_BOUND, /* MINimum allowed value for range partitions [min, max> */ + GDK_MAX_BOUND, /* MAXimum of the range partitions [min, max>, ie. excluding this max value */ + GDK_NOT_NULL, /* bat bound to be not null */ + /* CURRENTLY_NO_PROPERTIES_DEFINED, */ }; gdk_export ValPtr BATgetprop(BAT *b, enum prop_t idx); +gdk_export ValPtr BATgetprop_nolock(BAT *b, enum prop_t idx); +gdk_export void BATrmprop(BAT *b, enum prop_t idx); +gdk_export void BATrmprop_nolock(BAT *b, enum prop_t idx); +gdk_export ValPtr BATsetprop(BAT *b, enum prop_t idx, int type, const void *v); +gdk_export ValPtr BATsetprop_nolock(BAT *b, enum prop_t idx, int type, const void *v); /* * @- BAT relational operators diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -107,18 +107,27 @@ BATgroupaggrinit(BAT *b, BAT *g, BAT *e, max = gids[BATcount(g) - 1]; } } else { - /* we'll do a complete scan */ - gids = (const oid *) Tloc(g, 0); - for (i = 0, ngrp = BATcount(g); i < ngrp; i++) { - if (!is_oid_nil(gids[i])) { - if (gids[i] < min) - min = gids[i]; - if (gids[i] > max) - max = gids[i]; + const ValRecord *prop; + prop = BATgetprop(g, GDK_MAX_BOUND); + if (prop != NULL) { + assert(prop->vtype == TYPE_oid); + min = 0; /* just assume it starts at 0 */ + max = prop->val.oval - 1; /* bound is exclusive */ + } else { + /* we'll do a complete scan */ + gids = (const oid *) Tloc(g, 0); + for (i = 0, ngrp = BATcount(g); i < ngrp; i++) { + if (!is_oid_nil(gids[i])) { + if (gids[i] < min) + min = gids[i]; + if (gids[i] > max) + max = gids[i]; + } } + /* note: max < min is possible + * if all groups are nil (or + * BATcount(g)==0) */ } - /* note: max < min is possible if all groups - * are nil (or BATcount(g)==0) */ } } ngrp = max < min ? 0 : max - min + 1; diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -1102,6 +1102,16 @@ BUNappendmulti(BAT *b, const void *value const void *t = b->ttype == TYPE_msk ? &(msk){false} : ATOMnilptr(b->ttype); MT_lock_set(&b->theaplock); BATiter bi = bat_iterator_nolock(b); + const ValRecord *prop; + ValRecord minprop, maxprop; + const void *minbound = NULL, *maxbound = NULL; + if ((prop = BATgetprop_nolock(b, GDK_MIN_BOUND)) != NULL && + VALcopy(&minprop, prop) != NULL) + minbound = VALptr(&minprop); + if ((prop = BATgetprop_nolock(b, GDK_MAX_BOUND)) != NULL && + VALcopy(&maxprop, prop) != NULL) + maxbound = VALptr(&maxprop); + const bool notnull = BATgetprop_nolock(b, GDK_NOT_NULL) != NULL; MT_lock_unset(&b->theaplock); MT_rwlock_wrlock(&b->thashlock); if (values && b->ttype) { @@ -1116,9 +1126,33 @@ BUNappendmulti(BAT *b, const void *value const void *vbase = b->tvheap->base; for (BUN i = 0; i < count; i++) { t = ((void **) values)[i]; - gdk_return rc = tfastins_nocheckVAR(b, p, t); + bool isnil = atomcmp(t, atomnil) == 0; + gdk_return rc; + if (notnull && isnil) { + assert(0); + GDKerror("NULL value not within bounds\n"); + rc = GDK_FAIL; + } else if (minbound && + !isnil && + atomcmp(t, minbound) < 0) { + assert(0); + GDKerror("value not within bounds\n"); + rc = GDK_FAIL; + } else if (maxbound && + !isnil && + atomcmp(t, maxbound) >= 0) { + assert(0); + GDKerror("value not within bounds\n"); + rc = GDK_FAIL; + } else { + rc = tfastins_nocheckVAR(b, p, t); + } if (rc != GDK_SUCCEED) { MT_rwlock_wrunlock(&b->thashlock); + if (minbound) + VALclear(&minprop); + if (maxbound) + VALclear(&maxprop); return rc; } if (vbase != b->tvheap->base) { @@ -1140,7 +1174,7 @@ BUNappendmulti(BAT *b, const void *value if (bi.maxpos != BUN_NONE) maxvalp = BUNtvar(bi, bi.maxpos); } - if (atomcmp(t, atomnil) != 0) { + if (!isnil) { if (p == 0) { bi.minpos = bi.maxpos = 0; minvalp = maxvalp = t; @@ -1159,6 +1193,10 @@ BUNappendmulti(BAT *b, const void *value } p++; } + if (minbound) + VALclear(&minprop); + if (maxbound) + VALclear(&maxprop); if (b->thash) { p -= count; for (BUN i = 0; i < count; i++) { @@ -1507,9 +1545,10 @@ BUNdelete(BAT *b, oid o) static gdk_return BUNinplacemulti(BAT *b, const oid *positions, const void *values, BUN count, bool force, bool autoincr) { - int tt; BUN prv, nxt; const void *val; + int (*atomcmp) (const void *, const void *) = ATOMcompare(b->ttype); + const void *atomnil = ATOMnilptr(b->ttype); MT_lock_set(&b->theaplock); BUN last = BATcount(b) - 1; @@ -1532,6 +1571,16 @@ BUNinplacemulti(BAT *b, const oid *posit } else if (count > BATcount(b) / gdk_unique_estimate_keep_fraction) { b->tunique_est = 0; } + const ValRecord *prop; + ValRecord minprop, maxprop; + const void *minbound = NULL, *maxbound = NULL; + if ((prop = BATgetprop_nolock(b, GDK_MIN_BOUND)) != NULL && + VALcopy(&minprop, prop) != NULL) + minbound = VALptr(&minprop); + if ((prop = BATgetprop_nolock(b, GDK_MAX_BOUND)) != NULL && + VALcopy(&maxprop, prop) != NULL) + maxbound = VALptr(&maxprop); + const bool notnull = BATgetprop_nolock(b, GDK_NOT_NULL) != NULL; MT_lock_unset(&b->theaplock); /* load hash so that we can maintain it */ (void) BATcheckhash(b); @@ -1541,8 +1590,22 @@ BUNinplacemulti(BAT *b, const oid *posit const void *t = b->ttype && b->tvheap ? ((const void **) values)[i] : (const void *) ((const char *) values + (i << b->tshift)); - const bool isnil = ATOMlinear(b->ttype) && - ATOMcmp(b->ttype, t, ATOMnilptr(b->ttype)) == 0; + bool isnil = atomnil && atomcmp(t, atomnil) == 0; + if (notnull && isnil) { + assert(0); + GDKerror("NULL value not within bounds\n"); + MT_rwlock_wrunlock(&b->thashlock); + goto bailout; + } else if (!isnil && + ((minbound && + atomcmp(t, minbound) < 0) || + (maxbound && + atomcmp(t, maxbound) >= 0))) { + assert(0); + GDKerror("value not within bounds\n"); + MT_rwlock_wrunlock(&b->thashlock); + goto bailout; + } /* retrieve old value, but if this comes from the * logger, we need to deal with offsets that point @@ -1562,11 +1625,11 @@ BUNinplacemulti(BAT *b, const oid *posit } if (val) { - if (ATOMcmp(b->ttype, val, t) == 0) + if (atomcmp(val, t) == 0) continue; /* nothing to do */ if (!isnil && b->tnil && - ATOMcmp(b->ttype, val, ATOMnilptr(b->ttype)) == 0) { + atomcmp(val, atomnil) == 0) { /* if old value is nil and new value * isn't, we're not sure anymore about * the nil property, so we must clear @@ -1577,12 +1640,12 @@ BUNinplacemulti(BAT *b, const oid *posit } if (b->ttype != TYPE_void) { if (bi.maxpos != BUN_NONE) { - if (!isnil && ATOMcmp(b->ttype, BUNtail(bi, bi.maxpos), t) < 0) { + if (!isnil && atomcmp(BUNtail(bi, bi.maxpos), t) < 0) { /* new value is larger * than previous * largest */ bi.maxpos = p; - } else if (bi.maxpos == p && ATOMcmp(b->ttype, BUNtail(bi, bi.maxpos), t) != 0) { + } else if (bi.maxpos == p && atomcmp(BUNtail(bi, bi.maxpos), t) != 0) { /* old value is equal to * largest and new value * is smaller or nil (see @@ -1593,12 +1656,12 @@ BUNinplacemulti(BAT *b, const oid *posit } } if (bi.minpos != BUN_NONE) { - if (!isnil && ATOMcmp(b->ttype, BUNtail(bi, bi.minpos), t) > 0) { + if (!isnil && atomcmp(BUNtail(bi, bi.minpos), t) > 0) { /* new value is smaller * than previous * smallest */ bi.minpos = p; - } else if (bi.minpos == p && ATOMcmp(b->ttype, BUNtail(bi, bi.minpos), t) != 0) { + } else if (bi.minpos == p && atomcmp(BUNtail(bi, bi.minpos), t) != 0) { /* old value is equal to * smallest and new value * is larger or nil (see @@ -1654,14 +1717,14 @@ BUNinplacemulti(BAT *b, const oid *posit } if (ATOMreplaceVAR(b, &_d, t) != GDK_SUCCEED) { MT_rwlock_wrunlock(&b->thashlock); - return GDK_FAIL; + goto bailout; } if (b->twidth < SIZEOF_VAR_T && (b->twidth <= 2 ? _d - GDK_VAROFFSET : _d) >= ((size_t) 1 << (8 << b->tshift))) { /* doesn't fit in current heap, upgrade it */ if (GDKupgradevarheap(b, _d, 0, bi.count) != GDK_SUCCEED) { MT_rwlock_wrunlock(&b->thashlock); - return GDK_FAIL; + goto bailout; } } /* reinitialize iterator after possible heap upgrade */ @@ -1699,7 +1762,7 @@ BUNinplacemulti(BAT *b, const oid *posit if (ATOMfix(b->ttype, t) != GDK_SUCCEED || _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
