Changeset: cd79b0ddfff3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/cd79b0ddfff3
Modified Files:
sql/backends/monet5/bin_partition_by_slice.c
sql/backends/monet5/bin_partition_by_slice.h
sql/backends/monet5/bin_partition_by_value.c
sql/backends/monet5/bin_partition_by_value.h
sql/backends/monet5/rel_physical.c
sql/include/sql_relation.h
sql/server/rel_exp.c
sql/server/rel_exp.h
sql/server/rel_optimize_sel.c
sql/server/rel_statistics.c
Branch: pp_hashjoin
Log Message:
disabled rel_push_join_down, causes crossproducts in tpcds.
improved cardinality estimation for somewhat
diffs (300 lines):
diff --git a/sql/backends/monet5/bin_partition_by_slice.c
b/sql/backends/monet5/bin_partition_by_slice.c
--- a/sql/backends/monet5/bin_partition_by_slice.c
+++ b/sql/backends/monet5/bin_partition_by_slice.c
@@ -44,7 +44,9 @@ exp_getcard(mvc *sql, sql_rel *rel, sql_
sql_subtype *t = exp_subtype(e);
prop *p;
- if ((p = find_prop(e->p, PROP_NUNIQUES)))
+ if (e->nuniques)
+ est = e->nuniques;
+ else if ((p = find_prop(e->p, PROP_NUNIQUES)))
est = (BUN)p->value.dval;
if (est == BUN_NONE
@@ -802,6 +804,60 @@ piggy_back(backend *be, sql_rel *rel, st
}
}
+bool
+rel_groupby_partition(mvc *sql, sql_rel *rel)
+{
+ bool partition = true;
+
+ for(node *n = rel->exps->h; n && partition; n = n->next ) {
+ sql_exp *e = n->data;
+
+ if (is_aggr(e->type)) {
+ sql_subfunc *sf = e->f;
+
+ /* for now only on complex aggregation */
+ if (!(strcmp(sf->func->base.name, "min") == 0 ||
strcmp(sf->func->base.name, "max") == 0 ||
+ strcmp(sf->func->base.name, "avg") == 0 ||
strcmp(sf->func->base.name, "count") == 0 ||
+ strcmp(sf->func->base.name, "sum") == 0 ||
strcmp(sf->func->base.name, "prod") == 0)) {
+ partition = false;
+ }
+ }
+ }
+ if (!partition)
+ return false;
+ if (list_empty(rel->r))
+ return false;
+ /* check size */
+ BUN est = get_rel_count(rel);
+
+ lng estimate, card = 1;
+ if (est == BUN_NONE
+#if SIZEOF_BUN == SIZEOF_LNG
+ || (ulng) est > (ulng) GDK_lng_max
+#endif
+ )
+ estimate = 85000000;
+ else
+ estimate = (lng) est;
+ if (!list_empty(rel->r)) {
+ list *l = rel->r;
+ for( node *n = l->h; n; n = n->next ) {
+ sql_exp *e = n->data;
+ lng ncard = exp_getcard(sql, rel, e);
+ card *= ncard; /* TODO check for overflow */
+ if (card > estimate || ncard >= estimate) {
+ card = estimate;
+ break;
+ }
+ }
+ }
+ if (card < estimate)
+ estimate = card;
+ if ((BUN)estimate >= 4*GDKL3_size)
+ return true;
+ return false;
+}
+
stmt *
rel2bin_groupby_pp(backend *be, sql_rel *rel, list *refs)
{
@@ -812,7 +868,7 @@ rel2bin_groupby_pp(backend *be, sql_rel
stmt *sub = NULL, *cursub;
stmt *groupby = NULL, *grp = NULL, *ext = NULL, *cnt_aggr = NULL;
bool _2phases = rel_groupby_2_phases(be->mvc, rel);
- bool value_partition = SQLrunning && rel->parallel && !_2phases &&
rel_groupby_partition(rel);
+ bool value_partition = SQLrunning && rel->parallel && !_2phases &&
rel_groupby_partition(be->mvc, rel);
int neededpp = rel->partition && get_need_pipeline(be);
int need_serialize = rel_groupby_serialize(rel); /* return if some of
the aggregates require serialization (or fallback implementation) */
sql_rel *inner = rel->l;
diff --git a/sql/backends/monet5/bin_partition_by_slice.h
b/sql/backends/monet5/bin_partition_by_slice.h
--- a/sql/backends/monet5/bin_partition_by_slice.h
+++ b/sql/backends/monet5/bin_partition_by_slice.h
@@ -15,6 +15,7 @@
#include "mal_backend.h"
extern lng exp_getcard(mvc *sql, sql_rel *rel, sql_exp *e);
+extern bool rel_groupby_partition(mvc *sql, sql_rel *rel);
extern stmt *rel2bin_groupby_pp(backend *be, sql_rel *rel, list *refs);
#endif /*_BIN_SLICE_PARTITION_H_*/
diff --git a/sql/backends/monet5/bin_partition_by_value.c
b/sql/backends/monet5/bin_partition_by_value.c
--- a/sql/backends/monet5/bin_partition_by_value.c
+++ b/sql/backends/monet5/bin_partition_by_value.c
@@ -159,38 +159,6 @@ rel2bin_slicer_pp(backend *be, stmt *sub
}
}
-bool
-rel_groupby_partition(sql_rel *rel)
-{
- bool partition = true;
-
- for(node *n = rel->exps->h; n && partition; n = n->next ) {
- sql_exp *e = n->data;
-
- if (is_aggr(e->type)) {
- sql_subfunc *sf = e->f;
-
- /* for now only on complex aggregation */
- if (!(strcmp(sf->func->base.name, "min") == 0 ||
strcmp(sf->func->base.name, "max") == 0 ||
- strcmp(sf->func->base.name, "avg") == 0 ||
strcmp(sf->func->base.name, "count") == 0 ||
- strcmp(sf->func->base.name, "sum") == 0 ||
strcmp(sf->func->base.name, "prod") == 0)) {
- partition = false;
- }
- }
- }
- if (!partition)
- return false;
- if (list_empty(rel->r))
- return false;
- /* check size */
- BUN est = get_rel_count(rel);
- if (est == BUN_NONE)
- return false;
- if (est >= GDKL3_size)
- return true;
- return false;
-}
-
/* part := part.new(nr_parts);
* mat := mat.new(type:nil, nr_parts);
*/
diff --git a/sql/backends/monet5/bin_partition_by_value.h
b/sql/backends/monet5/bin_partition_by_value.h
--- a/sql/backends/monet5/bin_partition_by_value.h
+++ b/sql/backends/monet5/bin_partition_by_value.h
@@ -18,8 +18,6 @@ extern int pp_dynamic_slices(backend *be
//extern stmt *rel2bin_slicer(backend *be, stmt *sub);
extern stmt *rel2bin_slicer_pp(backend *be, stmt *sub);
-extern bool rel_groupby_partition(sql_rel *rel);
-
extern stmt *rel2bin_partition(backend *be, sql_rel *rel, list *refs);
extern stmt *rel2bin_groupby_partition(backend *be, sql_rel *rel, list *refs,
bool neededpp);
diff --git a/sql/backends/monet5/rel_physical.c
b/sql/backends/monet5/rel_physical.c
--- a/sql/backends/monet5/rel_physical.c
+++ b/sql/backends/monet5/rel_physical.c
@@ -17,7 +17,7 @@
#include "sql_storage.h"
#include "sql_scenario.h"
#include "rel_bin.h"
-#include "bin_partition_by_value.h"
+#include "bin_partition_by_slice.h"
#define IS_ORDER_BASED_AGGR(fname, argc) (\
(argc == 2 && (strcmp((fname), "quantile") == 0
|| strcmp((fname), "quantile_avg") == 0)) || \
@@ -646,7 +646,7 @@ rel_pipeline(visitor *v, sql_rel *rel, b
}
} else if (is_groupby(rel->op)) {
bool safe = rel_groupby_partition_safe(rel);
- if (safe && rel_groupby_partition(rel)) {
+ if (safe && rel_groupby_partition(v->sql, rel)) {
sql_rel *p = rel->l = rel_build_partition(v, rel->l);
p->attr = exps_copy(v->sql, rel->r);
}
diff --git a/sql/include/sql_relation.h b/sql/include/sql_relation.h
--- a/sql/include/sql_relation.h
+++ b/sql/include/sql_relation.h
@@ -73,6 +73,7 @@ typedef struct expression {
int shared; /* shared variable */
void *p; /* properties for the optimizer */
str comment;
+ ulng nuniques;
} sql_exp;
#define TABLE_PROD_FUNC 1
diff --git a/sql/server/rel_exp.c b/sql/server/rel_exp.c
--- a/sql/server/rel_exp.c
+++ b/sql/server/rel_exp.c
@@ -176,6 +176,8 @@ exp_compare(allocator *sa, sql_exp *l, s
e->l = l;
e->r = r;
e->flag = cmptype;
+ if (cmptype == cmp_equal)
+ e->nuniques = 1;
if (!has_nil(l) && !has_nil(r))
set_has_no_nil(e);
return e;
@@ -193,6 +195,8 @@ exp_compare2(allocator *sa, sql_exp *l,
e->r = r;
e->f = f;
e->flag = cmptype;
+ if (cmptype == cmp_equal)
+ e->nuniques = 1;
if (symmetric)
set_symmetric(e);
if (!has_nil(l) && !has_nil(r) && !has_nil(f))
@@ -247,6 +251,8 @@ exp_in(allocator *sa, sql_exp *l, list *
e->card = MAX(l->card, exps_card);
e->l = l;
e->r = r;
+ if (cmptype == cmp_in)
+ e->nuniques = list_length(r);
assert( cmptype == cmp_in || cmptype == cmp_notin);
e->flag = cmptype;
if (!has_nil(l) && !have_nil(r))
@@ -1813,6 +1819,33 @@ exps_find_prop(list *exps, rel_prop kind
return NULL;
}
+sql_exp *
+predicates_find_nid(const list *exps, int nid)
+{
+ if (list_empty(exps))
+ return NULL;
+ for (node *n = exps->h ; n ; n = n->next) {
+ sql_exp *e = n->data;
+
+ if (e->type == e_cmp) {
+ if (e->flag == cmp_filter) {
+ ;
+ } else if (e->flag == cmp_con || e->flag == cmp_dis) {
+ ;
+ } else if (e->flag == cmp_in || e->flag == cmp_notin) {
+ sql_exp *l = e->l;
+ if (l->nid == nid)
+ return e;
+ } else {
+ sql_exp *l = e->l;
+ if (l->nid == nid)
+ return e;
+ }
+ }
+ }
+ return NULL;
+}
+
/* check is one of the exps can be found in this relation */
static sql_exp* rel_find_exp_and_corresponding_rel_(sql_rel *rel, sql_exp *e,
bool subexp, sql_rel **res);
diff --git a/sql/server/rel_exp.h b/sql/server/rel_exp.h
--- a/sql/server/rel_exp.h
+++ b/sql/server/rel_exp.h
@@ -209,6 +209,7 @@ extern sql_exp *exps_uses_nid(list *exps
extern sql_exp *exps_bind_column(list *exps, const char *cname, int
*ambiguous, int *multiple, int no_tname /* set if expressions should be without
a tname */);
extern sql_exp *exps_bind_column2(list *exps, const char *rname, const char
*cname, int *multiple);
extern sql_exp * list_find_exp(const list *exps, sql_exp *e);
+extern sql_exp *predicates_find_nid(const list *exps, int nid);
extern unsigned int exps_card( list *l );
extern void exps_fix_card( list *exps, unsigned int card);
diff --git a/sql/server/rel_optimize_sel.c b/sql/server/rel_optimize_sel.c
--- a/sql/server/rel_optimize_sel.c
+++ b/sql/server/rel_optimize_sel.c
@@ -3764,7 +3764,8 @@ rel_semijoin_use_fk(visitor *v, sql_rel
static inline sql_rel *
rel_push_join_down(visitor *v, sql_rel *rel)
{
- if (!rel_is_ref(rel) && ((is_left(rel->op) || rel->op == op_join ||
is_semi(rel->op)) && rel->l && rel->exps)) {
+ /* needs work, currently causes crossproducts in tpcds 65 */
+ if (0 && !rel_is_ref(rel) && ((is_left(rel->op) || rel->op == op_join
|| is_semi(rel->op)) && rel->l && rel->exps)) {
sql_rel *gb = rel->r, *ogb = gb, *l = NULL, *rell = rel->l;
if (is_simple_project(gb->op) && !rel_is_ref(gb))
@@ -3782,6 +3783,7 @@ rel_push_join_down(visitor *v, sql_rel *
int fnd = 0;
const char *rname = NULL, *name = NULL;
+ /* TODO use nids! */
/* project in between, ie find alias */
/* first find expression in expression list */
gbe = exps_uses_exp( gb->exps, gbe);
diff --git a/sql/server/rel_statistics.c b/sql/server/rel_statistics.c
--- a/sql/server/rel_statistics.c
+++ b/sql/server/rel_statistics.c
@@ -60,6 +60,10 @@ rel_propagate_column_ref_statistics(mvc
found_left = true;
if (!found_left && is_join(rel->op) &&
rel_propagate_column_ref_statistics(sql, rel->r, e))
found_right = true;
+ sql_exp *ne;
+ if ((ne = predicates_find_nid(rel->exps, e->nid)) !=
NULL) {
+ e->nuniques = ne->nuniques;
+ }
if (!found_left && !found_right)
return NULL;
@@ -180,6 +184,8 @@ rel_propagate_column_ref_statistics(mvc
atom *fval;
prop *est;
if ((found = rel_find_exp(rel, e))) {
+ if (found != e && !e->nuniques)
+ e->nuniques = found->nuniques;
if (found /*rel->op != op_table*/) { /* At the
moment don't propagate statistics for table relations */
if ((fval = find_prop_and_get(found->p,
PROP_MAX)))
set_minmax_property(sql, e,
PROP_MAX, fval);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
