Changeset: 4177b2f3a4ee for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/4177b2f3a4ee Modified Files: sql/backends/monet5/rel_bin.c Branch: join-idx-improvements Log Message:
Merge with default branch. diffs (truncated from 14536 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -761,3 +761,4 @@ 8c015afafb5903ea59b0e2cffac1138a0d82e007 cab90a348501b045e19cee5cebcc44f3800bd0a8 Jul2021_21 cab90a348501b045e19cee5cebcc44f3800bd0a8 Jul2021_SP5_release 5872f047d97c98d3a848514438b8f97fa446855d Jan2022_11 +025239a5a6f122042798c0f1132a2c6298514e06 Jan2022_13 diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -848,6 +848,24 @@ fi %endif %changelog +* Fri Apr 01 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- Rebuilt. +- GH#7278: BUG when there is more than one field/filter in the having + clause + +* Fri Apr 1 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- gdk: Improved speed of BATappend to empty varsized bat: we now just copy + the heaps instead of inserting individual values. + +* Fri Apr 1 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- monetdb5: Improved parsing speed of blob values, especially on Windows. + On Windows, using the locale-aware functions isdigit and isxdigit is + comparatively very slow, so we avoid them. + +* Tue Mar 29 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- gdk: Improved speed of projection (BATproject) on varsized bats by sharing + the data heap (vheap). + * Fri Mar 25 2022 Sjoerd Mullender <[email protected]> - 11.43.11-20220325 - Rebuilt. - GH#7252: Segmentation fault on second run diff --git a/cmake/monetdb-versions.cmake b/cmake/monetdb-versions.cmake --- a/cmake/monetdb-versions.cmake +++ b/cmake/monetdb-versions.cmake @@ -40,7 +40,7 @@ set(MONETDB_VERSION "${MONETDB_VERSION_M # common/options and common/utils) set(GDK_VERSION_MAJOR "25") set(GDK_VERSION_MINOR "0") -set(GDK_VERSION_PATCH "3") +set(GDK_VERSION_PATCH "4") set(GDK_VERSION "${GDK_VERSION_MAJOR}.${GDK_VERSION_MINOR}.${GDK_VERSION_PATCH}") # version of the MAPI library (subdirectory clients/mapilib) @@ -52,7 +52,7 @@ set(MAPI_VERSION "${MAPI_VERSION_MAJOR}. # version of the MONETDB5 library (subdirectory monetdb5, not including extras or sql) set(MONETDB5_VERSION_MAJOR "32") set(MONETDB5_VERSION_MINOR "0") -set(MONETDB5_VERSION_PATCH "4") +set(MONETDB5_VERSION_PATCH "5") set(MONETDB5_VERSION "${MONETDB5_VERSION_MAJOR}.${MONETDB5_VERSION_MINOR}.${MONETDB5_VERSION_PATCH}") # version of the MONETDBE library (subdirectory tools/monetdbe) @@ -70,5 +70,5 @@ set(STREAM_VERSION "${STREAM_VERSION_MAJ # version of the SQL library (subdirectory sql) set(SQL_VERSION_MAJOR "12") set(SQL_VERSION_MINOR "0") -set(SQL_VERSION_PATCH "3") +set(SQL_VERSION_PATCH "4") set(SQL_VERSION "${SQL_VERSION_MAJOR}.${SQL_VERSION_MINOR}.${SQL_VERSION_PATCH}") diff --git a/debian/changelog b/debian/changelog --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,33 @@ +monetdb (11.43.13) unstable; urgency=low + + * Rebuilt. + * GH#7278: BUG when there is more than one field/filter in the having + clause + + -- Sjoerd Mullender <[email protected]> Fri, 01 Apr 2022 11:31:11 +0200 + +monetdb (11.43.13) unstable; urgency=low + + * gdk: Improved speed of BATappend to empty varsized bat: we now just copy + the heaps instead of inserting individual values. + + -- Sjoerd Mullender <[email protected]> Fri, 1 Apr 2022 11:31:11 +0200 + +monetdb (11.43.13) unstable; urgency=low + + * monetdb5: Improved parsing speed of blob values, especially on Windows. + On Windows, using the locale-aware functions isdigit and isxdigit is + comparatively very slow, so we avoid them. + + -- Sjoerd Mullender <[email protected]> Fri, 1 Apr 2022 11:31:11 +0200 + +monetdb (11.43.13) unstable; urgency=low + + * gdk: Improved speed of projection (BATproject) on varsized bats by sharing + the data heap (vheap). + + -- Sjoerd Mullender <[email protected]> Tue, 29 Mar 2022 11:31:11 +0200 + monetdb (11.43.11) unstable; urgency=low * Rebuilt. diff --git a/gdk/ChangeLog-Archive b/gdk/ChangeLog-Archive --- a/gdk/ChangeLog-Archive +++ b/gdk/ChangeLog-Archive @@ -1,6 +1,14 @@ # DO NOT EDIT THIS FILE -- MAINTAINED AUTOMATICALLY # This file contains past ChangeLog entries +* Fri Apr 1 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- Improved speed of BATappend to empty varsized bat: we now just copy + the heaps instead of inserting individual values. + +* Tue Mar 29 2022 Sjoerd Mullender <[email protected]> - 11.43.13-20220401 +- Improved speed of projection (BATproject) on varsized bats by sharing + the data heap (vheap). + * Fri Mar 18 2022 Sjoerd Mullender <[email protected]> - 11.43.11-20220325 - Fixed a race condition which could cause a too large size being written for a .theap file to the BBP.dir file after the correct size file had diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -692,7 +692,6 @@ gdk_export bool VALisnil(const ValRecord * bool tkey; // tail values are unique * bool tnonil; // tail has no nils * bool tsorted; // are tail values currently ordered? - * bool tvarsized; // for speed: tail type is varsized? * // Tail storage * int tloc; // byte-offset in BUN for tail elements * Heap *theap; // heap for varsized tail values @@ -726,8 +725,7 @@ typedef struct { uint16_t width; /* byte-width of the atom array */ int8_t type; /* type id. */ uint8_t shift; /* log2 of bun width */ - bool varsized:1, /* varsized/void (true) or fixedsized (false) */ - key:1, /* no duplicate values present */ + bool key:1, /* no duplicate values present */ nonil:1, /* there are no nils in the column */ nil:1, /* there is a nil in the column */ sorted:1, /* column is sorted in ascending order */ @@ -774,22 +772,34 @@ typedef enum { BAT_APPEND, /* only reads and appends allowed */ } restrict_t; +/* batDirtydesc: should be set (true) if any of the following fields + * have changed since the bat was last saved: hseqbase, batRestricted, + * batTransient, batCount, and the theap properties tkey, tseqbase, + * tsorted, trevsorted, twidth, tshift, tnonil, tnil, tnokey, tnosorted, + * tnorevsorted, tminpos, tmaxpos, and tunique_est; in addition, the + * value should be set if the BBP field BBP_logical(bid) is changed. + * + * theaplock: this lock should be held when reading or writing any of + * the fields mentioned above for batDirtydesc, and also when reading or + * writing any of the following fields: batDirtydesc, theap, tvheap, + * batInserted, batCapacity. There is no need for the lock if the bat + * cannot possibly be modified concurrently, e.g. when it is new and not + * yet returned to the interpreter or during system initialization. */ typedef struct BAT { /* static bat properties */ oid hseqbase; /* head seq base */ MT_Id creator_tid; /* which thread created it */ bat batCacheid; /* index into BBP */ + role_t batRole; /* role of the bat */ /* dynamic bat properties */ - restrict_t batRestricted; /* access privileges */ - bool batTransient; /* should the BAT persist on disk? */ + restrict_t batRestricted:2; /* access privileges */ bool + batTransient:1, /* should the BAT persist on disk? */ batCopiedtodisk:1, /* once written */ batDirtyflushed:1, /* was dirty before commit started? */ batDirtydesc:1; /* bat descriptor dirty marker */ - uint16_t /* adjacent bit fields are packed together (if they fit) */ - selcnt:10; /* how often used in equi select without hash */ - role_t batRole; /* role of the bat */ + uint16_t selcnt; /* how often used in equi select without hash */ uint16_t unused; /* value=0 for now (sneakily used by mat.c) */ int batSharecnt; /* incoming view count */ @@ -808,7 +818,6 @@ typedef struct BAT { /* macros to hide complexity of the BAT structure */ #define ttype T.type #define tkey T.key -#define tvarsized T.varsized #define tseqbase T.seq #define tsorted T.sorted #define trevsorted T.revsorted @@ -947,6 +956,7 @@ typedef struct BATiter { void *base; Heap *vh; BUN count; + BUN baseoff; uint16_t width; uint8_t shift; int8_t type; @@ -954,13 +964,24 @@ typedef struct BATiter { BUN hfree, vhfree; BUN minpos, maxpos; double unique_est; + bool key:1, + nonil:1, + nil:1, + sorted:1, + revsorted:1, + hdirty:1, + vhdirty:1, + dirtydesc:1, + copiedtodisk:1, + transient:1; + restrict_t restricted:2; +#ifndef NDEBUG + bool locked:1; +#endif union { oid tvid; bool tmsk; }; -#ifndef NDEBUG - bool locked; -#endif } BATiter; static inline BATiter @@ -973,6 +994,7 @@ bat_iterator_nolock(BAT *b) .b = b, .h = b->theap, .base = b->theap->base ? b->theap->base + (b->tbaseoff << b->tshift) : NULL, + .baseoff = b->tbaseoff, .vh = b->tvheap, .count = b->batCount, .width = b->twidth, @@ -989,6 +1011,19 @@ bat_iterator_nolock(BAT *b) .minpos = isview ? BUN_NONE : b->tminpos, .maxpos = isview ? BUN_NONE : b->tmaxpos, .unique_est = b->tunique_est, + .key = b->tkey, + .nonil = b->tnonil, + .nil = b->tnil, + .sorted = b->tsorted, + .revsorted = b->trevsorted, + /* only look at heap dirty flag if we own it */ + .hdirty = b->theap->parentid == b->batCacheid && b->theap->dirty, + /* also, if there is no vheap, it's not dirty */ + .vhdirty = b->tvheap && b->tvheap->parentid == b->batCacheid && b->tvheap->dirty, + .dirtydesc = b->batDirtydesc, + .copiedtodisk = b->batCopiedtodisk, + .transient = b->batTransient, + .restricted = b->batRestricted, #ifndef NDEBUG .locked = false, #endif @@ -1023,6 +1058,20 @@ bat_iterator(BAT *b) return bi; } +/* return a copy of a BATiter instance; needs to be released with + * bat_iterator_end */ +static inline BATiter +bat_iterator_copy(BATiter *bip) +{ + assert(bip); + assert(bip->locked); + if (bip->h) + HEAPincref(bip->h); + if (bip->vh) + HEAPincref(bip->vh); + return *bip; +} + static inline void bat_iterator_end(BATiter *bip) { @@ -1176,8 +1225,8 @@ typedef var_t stridx_t; #define BUNtmsk(bi,p) Tmsk(&(bi), (p)) #define BUNtloc(bi,p) (assert((bi).type != TYPE_msk), ((void *) ((char *) (bi).base + ((p) << (bi).shift)))) #define BUNtpos(bi,p) Tpos(&(bi),p) -#define BUNtvar(bi,p) (assert((bi).type && (bi).b->tvarsized), (void *) ((bi).vh->base+BUNtvaroff(bi,p))) -#define BUNtail(bi,p) ((bi).type?(bi).b->tvarsized?BUNtvar(bi,p):(bi).type==TYPE_msk?BUNtmsk(bi,p):BUNtloc(bi,p):BUNtpos(bi,p)) +#define BUNtvar(bi,p) (assert((bi).type && (bi).vh), (void *) ((bi).vh->base+BUNtvaroff(bi,p))) +#define BUNtail(bi,p) ((bi).type?(bi).vh?BUNtvar(bi,p):(bi).type==TYPE_msk?BUNtmsk(bi,p):BUNtloc(bi,p):BUNtpos(bi,p)) #define BATcount(b) ((b)->batCount) @@ -1252,6 +1301,10 @@ gdk_export restrict_t BATgetaccess(BAT * (b)->batDirtydesc || \ (b)->theap->dirty || \ ((b)->tvheap != NULL && (b)->tvheap->dirty)) +#define BATdirtybi(bi) (!(bi).copiedtodisk || \ + (bi).dirtydesc || \ + (bi).hdirty || \ + (bi).vhdirty) #define BATdirtydata(b) (!(b)->batCopiedtodisk || \ (b)->theap->dirty || \ ((b)->tvheap != NULL && (b)->tvheap->dirty)) @@ -1359,16 +1412,17 @@ gdk_export gdk_return BATsort(BAT **sort gdk_export void GDKqsort(void *restrict h, void *restrict t, const void *restrict base, size_t n, int hs, int ts, int tpe, bool reverse, bool nilslast); -#define BATtordered(b) ((b)->tsorted) _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
