Changeset: 8788387b1c06 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/8788387b1c06 Branch: groupjoin Log Message:
Merge with default branch. diffs (truncated from 390 to 300 lines): diff --git a/ChangeLog.Jun2023 b/ChangeLog.Jun2023 --- a/ChangeLog.Jun2023 +++ b/ChangeLog.Jun2023 @@ -1,6 +1,10 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Wed Aug 30 2023 Sjoerd Mullender <[email protected]> +- Do a lot more error checking, mostly for allocation failures. More is + still needed, though. + * Thu Aug 10 2023 Panagiotis Koutsourakis <[email protected]> - Improve performance of the ILIKE operator when the pattern contains only ASCII characters. In this case we do not need to treat any characters as diff --git a/debian/monetdb5-sql.README.Debian b/debian/monetdb5-sql.README.Debian --- a/debian/monetdb5-sql.README.Debian +++ b/debian/monetdb5-sql.README.Debian @@ -2,7 +2,16 @@ Usage ===== -There are multiple ways to use MonetDB5-sql, but this is the most Debianic. +There are multiple ways to use MonetDB5-sql. + +The modern way is to do: + + * Enable and start MonetDB in one go:: + + $ sudo systemctl enable --now monetdbd.service + +This is the old way using System V init scripts, incompatible with the +above: * Set STARTUP to "yes" in /etc/default/monetdb5-sql @@ -10,7 +19,10 @@ There are multiple ways to use MonetDB5- $ sudo /etc/init.d/monetdb5-sql start - * Add yourself to the "monetdb" group:: +In either case, do this: + + * Add yourself to the "monetdb" group (after this, logout and back in + for this to take effect):: $ sudo addgroup `whoami` monetdb diff --git a/gdk/ChangeLog.Jun2023 b/gdk/ChangeLog.Jun2023 --- a/gdk/ChangeLog.Jun2023 +++ b/gdk/ChangeLog.Jun2023 @@ -1,3 +1,18 @@ # ChangeLog file for GDK # This file is updated with Maddlog +* Wed Aug 30 2023 Sjoerd Mullender <[email protected]> +- Only check for virtual memory limits when creating or growing bats, + not for general memory allocations. There is (still) too much code + that doesn't properly handle failing allocations, so we need to avoid + those as much as possible. This has mostly an effect if there are + virtual memory size restrictions imposed by cgroups (memory.swap.max + in cgroups v2, memory.memsw.limit_in_bytes in cgroups v1). +- The low-level commit turned out to always commit every persistent bat + in the system. There is no need for that, it should only commit bats + that were changed. This has now been fixed. +- Implemented timeout/exit checks in a bunch more operators. Long(er) + running operators occasionally check whether they're taking too long + (past a user-specified timeout) or whether the server is exiting. + This is now done in more places. + diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c --- a/gdk/gdk_join.c +++ b/gdk/gdk_join.c @@ -43,7 +43,7 @@ * BATsemijoin * equi-join, but the left output is sorted, and if there are * multiple matches, only one is returned (i.e., the left output - * is also key) + * is also key, making it a candidate list) * BATthetajoin * theta-join: an extra operator must be provided encoded as an * integer (macros JOIN_EQ, JOIN_NE, JOIN_LT, JOIN_LE, JOIN_GT, @@ -3768,7 +3768,8 @@ bitmaskjoin(BAT *l, BAT *r, * semi: semi join: return one of potentially more than one matches; * only_misses: difference: list rows without match on the right; * not_in: for implementing NOT IN: if nil on right then there are no matches; - * max_one: error if there is more than one match. */ + * max_one: error if there is more than one match; + * min_one: error if there are no matches. */ static gdk_return leftjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, bool nil_on_miss, bool semi, bool only_misses, @@ -4025,6 +4026,8 @@ leftjoin(BAT **r1p, BAT **r2p, BAT *l, B doreturn: BBPreclaim(lp); BBPreclaim(rp); + if (rc == GDK_SUCCEED && (semi | only_misses)) + *r1p = virtualize(*r1p); return rc; } @@ -4080,7 +4083,7 @@ BATintersect(BAT *l, BAT *r, BAT *sl, BA false, true, false, false, max_one, false, estimate, __func__, GDK_TRACER_TEST(M_DEBUG, ALGO) ? GDKusec() : 0) == GDK_SUCCEED) - return virtualize(bn); + return bn; return NULL; } @@ -4097,7 +4100,7 @@ BATdiff(BAT *l, BAT *r, BAT *sl, BAT *sr false, false, true, not_in, false, false, estimate, __func__, GDK_TRACER_TEST(M_DEBUG, ALGO) ? GDKusec() : 0) == GDK_SUCCEED) - return virtualize(bn); + return bn; return NULL; } diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h --- a/gdk/gdk_private.h +++ b/gdk/gdk_private.h @@ -24,7 +24,7 @@ #define PERSISTENTSTRIMP 1 /* only check whether we exceed gdk_vm_maxsize when allocating heaps */ -/* #define SIZE_CHECK_IN_HEAPS_ONLY 1 */ +#define SIZE_CHECK_IN_HEAPS_ONLY 1 #include "gdk_system_private.h" diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c --- a/gdk/gdk_utils.c +++ b/gdk/gdk_utils.c @@ -667,7 +667,9 @@ MT_init(void) f = fopen(pth, "r"); } if (f != NULL) { - if (fscanf(f, "%" SCNu64, &mem) == 1 && mem < (uint64_t) _MT_pagesize * _MT_npages) { + if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 + && mem < (uint64_t) _MT_pagesize * _MT_npages) { _MT_npages = (size_t) (mem / _MT_pagesize); } success = true; @@ -678,7 +680,9 @@ MT_init(void) strcpy(q, "memory.high"); f = fopen(pth, "r"); if (f != NULL) { - if (fscanf(f, "%" SCNu64, &mem) == 1 && mem < (uint64_t) _MT_pagesize * _MT_npages) { + if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 + && mem < (uint64_t) _MT_pagesize * _MT_npages) { _MT_npages = (size_t) (mem / _MT_pagesize); } success = true; @@ -689,7 +693,9 @@ MT_init(void) strcpy(q, "memory.low"); f = fopen(pth, "r"); if (f != NULL) { - if (fscanf(f, "%" SCNu64, &mem) == 1 && mem > 0 && mem < (uint64_t) _MT_pagesize * _MT_npages) { + if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 + && mem < (uint64_t) _MT_pagesize * _MT_npages) { _MT_npages = (size_t) (mem / _MT_pagesize); } success = true; @@ -702,6 +708,7 @@ MT_init(void) f = fopen(pth, "r"); if (f != NULL) { if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 && mem < (uint64_t) GDK_vm_maxsize) { GDK_vm_maxsize = (size_t) mem; } @@ -735,6 +742,7 @@ MT_init(void) } if (f != NULL) { if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 && mem < (uint64_t) _MT_pagesize * _MT_npages) { _MT_npages = (size_t) (mem / _MT_pagesize); } @@ -756,6 +764,7 @@ MT_init(void) } if (f != NULL) { if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 && mem < (uint64_t) _MT_pagesize * _MT_npages) { _MT_npages = (size_t) (mem / _MT_pagesize); } @@ -778,6 +787,7 @@ MT_init(void) } if (f != NULL) { if (fscanf(f, "%" SCNu64, &mem) == 1 + && mem > 0 && mem < (uint64_t) GDK_vm_maxsize) { GDK_vm_maxsize = (size_t) mem; } diff --git a/sql/server/rel_partition.c b/sql/server/rel_partition.c --- a/sql/server/rel_partition.c +++ b/sql/server/rel_partition.c @@ -137,22 +137,47 @@ has_groupby(sql_rel *rel) { if (!rel) return 0; - if (is_groupby(rel->op)) - return 1; - if (is_join(rel->op) || is_semi(rel->op) || is_set(rel->op) || is_merge(rel->op)) - return has_groupby(rel->l) || has_groupby(rel->r); - if (is_simple_project(rel->op) || is_select(rel->op) || is_topn(rel->op) || is_sample(rel->op)) - return has_groupby(rel->l); - if (is_insert(rel->op) || is_update(rel->op) || is_delete(rel->op) || is_truncate(rel->op)) - return has_groupby(rel->r); - if (is_ddl(rel->op)) { - if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) + + switch (rel->op) { + case op_groupby: + return 1; + case op_join: + case op_left: + case op_right: + case op_full: + + case op_semi: + case op_anti: + + case op_union: + case op_inter: + case op_except: + + case op_merge: + return has_groupby(rel->l) || has_groupby(rel->r); + case op_project: + case op_select: + case op_topn: + case op_sample: return has_groupby(rel->l); - if (rel->flag == ddl_list || rel->flag == ddl_exception) - return has_groupby(rel->l) || has_groupby(rel->r); + case op_insert: + case op_update: + case op_delete: + case op_truncate: + return has_groupby(rel->r); + case op_ddl: + if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) + return has_groupby(rel->l); + if (rel->flag == ddl_list || rel->flag == ddl_exception) + return has_groupby(rel->l) || has_groupby(rel->r); + return 0; + case op_table: + if (IS_TABLE_PROD_FUNC(rel->flag) || rel->flag == TABLE_FROM_RELATION) + return has_groupby(rel->l); + return 0; + case op_basetable: + return 0; } - if (rel->op == op_table && (IS_TABLE_PROD_FUNC(rel->flag) || rel->flag == TABLE_FROM_RELATION)) - return has_groupby(rel->l); return 0; } @@ -162,28 +187,52 @@ rel_partition(mvc *sql, sql_rel *rel) if (mvc_highwater(sql)) return sql_error(sql, 10, SQLSTATE(42000) "Query too complex: running out of stack space"); - if (is_basetable(rel->op)) { + switch (rel->op) { + case op_basetable: + case op_sample: rel->flag = REL_PARTITION; - } else if (is_simple_project(rel->op) || is_select(rel->op) || is_groupby(rel->op) || is_topn(rel->op) || is_sample(rel->op)) { + break; + case op_project: + case op_select: + case op_groupby: + case op_topn: if (rel->l) rel_partition(sql, rel->l); - } else if (is_semi(rel->op) || is_set(rel->op) || is_merge(rel->op)) { + break; + case op_semi: + case op_anti: + + case op_union: + case op_inter: + case op_except: + + case op_merge: if (rel->l) rel_partition(sql, rel->l); if (rel->r) rel_partition(sql, rel->r); - } else if (is_insert(rel->op) || is_update(rel->op) || is_delete(rel->op) || is_truncate(rel->op)) { + break; + case op_insert: + case op_update: + case op_delete: + case op_truncate: _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
