Changeset: b7bddbe022ca for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b7bddbe022ca
Modified Files:
clients/Tests/MAL-signatures-hge.test
clients/Tests/MAL-signatures.test
monetdb5/modules/mal/pp_algebra.c
sql/backends/monet5/bin_partition_by_slice.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/rel_physical.c
sql/include/sql_relation.h
sql/server/rel_optimize_sel.c
sql/server/rel_optimizer_private.h
sql/server/rel_prop.c
sql/server/rel_statistics.c
sql/server/rel_statistics_functions.c
sql/server/rel_unnest.c
sql/test/BugTracker-2026/Tests/7975-cleanup-join-exps.test
Branch: pp_hashjoin
Log Message:
(for now only) in case of count(distinct) skip nils
detect duplicate sub-trees (for now only duplicate hash builds) and
rewrite them into a single case.
some cleanup
diffs (truncated from 897 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -48390,13 +48390,23 @@ LALGprojection
Project.
ialgebra
unique
-command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid])
(X_3:bat[:oid], X_4:bat[:any_1])
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid],
X_3:bit) (X_4:bat[:oid], X_5:bat[:any_1])
LALGgroup_unique
Unique rows per group.
ialgebra
unique
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid])
(X_3:bat[:oid], X_4:bat[:any_1])
+LALGgroup_unique_keepnil
+Unique rows per group.
+ialgebra
+unique
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bit)
(X_3:bat[:oid], X_4:bat[:any_1])
+LALGunique
+Unique rows.
+ialgebra
+unique
command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid]) (X_2:bat[:oid],
X_3:bat[:any_1])
-LALGunique
+LALGunique_keepnil
Unique rows.
igroup
group
diff --git a/clients/Tests/MAL-signatures.test
b/clients/Tests/MAL-signatures.test
--- a/clients/Tests/MAL-signatures.test
+++ b/clients/Tests/MAL-signatures.test
@@ -36595,13 +36595,23 @@ LALGprojection
Project.
ialgebra
unique
-command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid])
(X_3:bat[:oid], X_4:bat[:any_1])
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid],
X_3:bit) (X_4:bat[:oid], X_5:bat[:any_1])
LALGgroup_unique
Unique rows per group.
ialgebra
unique
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bat[:oid])
(X_3:bat[:oid], X_4:bat[:any_1])
+LALGgroup_unique_keepnil
+Unique rows per group.
+ialgebra
+unique
+command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid], X_2:bit)
(X_3:bat[:oid], X_4:bat[:any_1])
+LALGunique
+Unique rows.
+ialgebra
+unique
command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid]) (X_2:bat[:oid],
X_3:bat[:any_1])
-LALGunique
+LALGunique_keepnil
Unique rows.
igroup
group
diff --git a/monetdb5/modules/mal/pp_algebra.c
b/monetdb5/modules/mal/pp_algebra.c
--- a/monetdb5/modules/mal/pp_algebra.c
+++ b/monetdb5/modules/mal/pp_algebra.c
@@ -928,7 +928,7 @@ LOCKEDAGGRnull(Client ctx, bat *result,
INIT_ALLOCATOR;
\
TIMEOUT_LOOP_IDX_DECL(i, cnt, qry_ctx) {
\
bool new = 0, fnd = 0;
\
- if(IS_NIL)
\
+ if (skip_nil && IS_NIL)
\
continue;
\
for(; !fnd; ) {
\
NEW_VAL;
\
@@ -1039,7 +1039,7 @@ LOCKEDAGGRnull(Client ctx, bat *result,
)
static str
-LALGunique(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid)
+LALGunique(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid, bit *Skip_nil)
{
/* TODO add skip nil (or not) */
(void)ctx;
@@ -1049,6 +1049,7 @@ LALGunique(Client ctx, bat *rid, bat *ui
assert(is_bat_nil(*sid)); /* no cands jet */
(void)sid;
bool local_storage = false;
+ bit skip_nil = *Skip_nil;
BAT *u = BATdescriptor(*uid);
BAT *b = BATdescriptor(*bid);
@@ -1176,6 +1177,13 @@ error:
return err;
}
+static str
+LALGunique_keepnil(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid)
+{
+ bit f = false;
+ return LALGunique(ctx, rid, uid, bid, sid, &f);
+}
+
#define gunique_(Type, BaseType, INIT_ALLOCATOR, INIT_ITER, IS_NIL, NEW_VAL,
HASH_VAL, VAL_NOT_EQUAL, VAL_ASSIGN, ITER_DONE, NEXTK) \
if (tt == TYPE_##Type) {
\
int slots = 0;
\
@@ -1185,7 +1193,7 @@ error:
INIT_ALLOCATOR;
\
TIMEOUT_LOOP_IDX_DECL(i, cnt, qry_ctx) {
\
bool new = 0, fnd = 0;
\
- if(IS_NIL)
\
+ if (skip_nil && IS_NIL)
\
continue;
\
for(; !fnd;) {
\
NEW_VAL;
\
@@ -1299,7 +1307,7 @@ error:
)
static str
-LALGgroup_unique(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid, bat *Gid)
+LALGgroup_unique(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid, bat *Gid,
bit *Skip_nil)
{
(void)ctx;
Pipeline *p = pipeline_get_thread_private_pipeline();
@@ -1308,6 +1316,7 @@ LALGgroup_unique(Client ctx, bat *rid, b
assert(is_bat_nil(*sid)); /* no cands jet */
(void)sid;
bool local_storage = false;
+ bit skip_nil = *Skip_nil;
BAT *u = BATdescriptor(*uid);
BAT *G = BATdescriptor(*Gid);
@@ -1439,6 +1448,13 @@ error:
return err;
}
+static str
+LALGgroup_unique_keepnil(Client ctx, bat *rid, bat *uid, bat *bid, bat *sid,
bat *Gid)
+{
+ bit f = false;
+ return LALGgroup_unique(ctx, rid, uid, bid, sid, Gid, &f);
+}
+
#define group_(Type, BaseType, INIT_ALLOCATOR, INIT_ITER, NEW_VAL, HASH_VAL,
VAL_NOT_EQUAL, VAL_ASSIGN, ITER_NEXT, NEXTK) \
int slots = 0;
\
gid slot = 0;
\
@@ -4905,8 +4921,10 @@ static mel_func pp_algebra_init_funcs[]
command("ialgebra", "projection", LALGprojection, false, "Project.",
args(1,3, batargany("",1), batarg("gid", oid), batargany("b",1))),
/* COUNT DISTINCT: globle / grouped */
- command("ialgebra", "unique", LALGunique, false, "Unique rows.",
args(2,4, batarg("gid", oid), batargany("",1), batargany("b",1),
batarg("s",oid))),
- command("ialgebra", "unique", LALGgroup_unique, false, "Unique rows per
group.", args(2,5, batarg("ngid", oid), batargany("",1), batargany("b",1),
batarg("s",oid), batarg("gid",oid))),
+ command("ialgebra", "unique", LALGunique_keepnil, false, "Unique
rows.", args(2,4, batarg("gid", oid), batargany("",1), batargany("b",1),
batarg("s",oid))),
+ command("ialgebra", "unique", LALGgroup_unique_keepnil, false, "Unique
rows per group.", args(2,5, batarg("ngid", oid), batargany("",1),
batargany("b",1), batarg("s",oid), batarg("gid",oid))),
+ command("ialgebra", "unique", LALGunique, false, "Unique rows.",
args(2,5, batarg("gid", oid), batargany("",1), batargany("b",1),
batarg("s",oid), arg("skip_nil",bit))),
+ command("ialgebra", "unique", LALGgroup_unique, false, "Unique rows per
group.", args(2,6, batarg("ngid", oid), batargany("",1), batargany("b",1),
batarg("s",oid), batarg("gid",oid), arg("skip_nil",bit))),
/********** Grouped aggregates **********/
command("ilockedaggr", "min", LALGmin, false, "Min per group.",
args(1,4, batargany("",1), batarg("gid", oid), batargany("", 1), batarg("pid",
oid))),
diff --git a/sql/backends/monet5/bin_partition_by_slice.c
b/sql/backends/monet5/bin_partition_by_slice.c
--- a/sql/backends/monet5/bin_partition_by_slice.c
+++ b/sql/backends/monet5/bin_partition_by_slice.c
@@ -43,11 +43,16 @@ exp_getcard(mvc *sql, sql_rel *rel, sql_
BUN est = BUN_NONE;
lng cnt;
sql_subtype *t = exp_subtype(e);
- prop *p;
+ prop *p, *min = NULL, *max = NULL;
if ((p = find_prop(e->p, PROP_NUNIQUES)) != NULL)
est = (BUN)p->value.dval;
- else
+ else if (t->type->eclass == EC_NUM &&
+ (min = find_prop(e->p, PROP_MIN)) != NULL &&
+ (max = find_prop(e->p, PROP_MAX)) != NULL) {
+ atom *uniques = atom_sub(sql->sa, max->value.pval,
min->value.pval);
+ est = atom_get_int(uniques);
+ } else
est = get_rel_count(rel);
if (est == BUN_NONE
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1918,9 +1918,10 @@ exp_bin(backend *be, sql_exp *e, stmt *l
return NULL;
if (be->pipeline && grp)
grp = stmt_project(be, u, grp);
- if (be->pipeline && exp_aggr_is_count(e))
+ if (be->pipeline && exp_aggr_is_count(e)) {
append(l, u);
- else
+ u->q = pushBit(be->mb, u->q, true); /*
skip nils */
+ } else
append(l, stmt_project(be, u, a));
}
if (r) { /* check new ordered aggregation */
@@ -8368,6 +8369,155 @@ rel_find_refs(mvc *sql, list *refs, sql_
return refs;
}
+/* todo change into simple nid1 -> nid2 */
+static bool
+exp_pair_match(list *exps, int nid1, int nid2)
+{
+ if (list_empty(exps))
+ return false;
+ for(node *n = exps->h; n; n = n->next->next) {
+ sql_exp *e = n->data;
+ if (e->alias.label == nid1) {
+ sql_exp *e2 = n->next->data;
+ if (e2->alias.label == nid2)
+ return true;
+ }
+ }
+ return false;
+}
+
+static int exps_remap(list *lexps, list *rexps, list *remapped);
+static int
+exp_remap(sql_exp *le, sql_exp *re, list *remapped)
+{
+ list *ll, *rl;
+ if (!le || !re || le->type != re->type)
+ return -1;
+ switch(le->type) {
+ case e_column:
+ if (exp_pair_match(remapped, le->nid, re->nid))
+ return 0;
+ break;
+ case e_convert:
+ ll = le->r;
+ rl = re->r;
+ return (subtype_cmp(ll->h->data, rl->h->data) == 0 &&
+ subtype_cmp(ll->h->next->data, rl->h->next->data)
== 0 &&
+ exp_remap(le->l , re->l, remapped) == 0)?0:-1;
+ case e_cmp:
+ if (le->flag != re->flag)
+ return -1;
+ if (le->flag <= cmp_notequal) {
+ return (exp_remap(le->l, re->l, remapped) == 0 &&
+ exp_remap(le->r, re->r, remapped) == 0 &&
+ ((!le->f && !re->f) || exp_remap(le->f, re->f,
remapped) == 0))?0:-1;
+ } else if (le->flag == cmp_in || le->flag == cmp_notin) {
+ return (exp_remap(le->l, re->l, remapped) == 0 &&
+ exps_remap(le->r, re->r, remapped) ==
0)?0:-1;
+ }
+ return -1;
+ case e_atom:
+ if (le->l && re->l)
+ return atom_cmp(le->l, re->l);
+ return -1;
+ default:
+ return -1;
+ }
+ (void)remapped;
+ return 0;
+}
+
+static int
+exps_remap(list *lexps, list *rexps, list *remapped)
+{
+ if (list_length(lexps) != list_length(rexps))
+ return -1;
+ if (list_empty(lexps))
+ return 0;
+ for(node *n = lexps->h, *m = rexps->h; n && m; n = n->next, m =
m->next) {
+ sql_exp *le = n->data, *re = m->data;
+ if (exp_remap(le, re, remapped) != 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int
+rel_remap(mvc *sql, allocator *ta, sql_rel *l, sql_rel *r, list *remapped)
+{
+ if (!l || !r || l->op != r->op)
+ return -1;
+ switch(l->op) {
+ case op_project:
+ case op_select:
+ case op_buildhash:
+ if (l->op == op_project && (l->r || r->r)) /* don't match
orderings */
+ return -1;
+ if (rel_remap(sql, ta, l->l, r->l, remapped) == 0)
+ return (exps_remap(l->attr, r->attr, remapped) == 0 &&
+ exps_remap(l->exps, r->exps, remapped)
== 0)?0:-1;
+ break;
+ case op_basetable:
+ if (l->l == r->l) {
+ if (list_length(l->exps) == list_length(r->exps)) {
+ list *lexps = l->exps, *rexps = r->exps;
+ for(node *n = lexps->h, *m = rexps->h; n && m;
n = n->next, m = m->next) {
+ sql_exp *le = n->data, *re = m->data;
+ assert(le->type == e_column && re->type
== e_column);
+ if (strcmp(le->r, re->r) == 0) {
+ append(remapped, le);
+ append(remapped, re);
+ } else {
+ return -1;
+ }
+ }
+ return 0;
+ }
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
