Changeset: f619a55c1da9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/f619a55c1da9
Modified Files:
clients/Tests/MAL-signatures-hge.test
monetdb5/modules/mal/pp_hash.c
sql/backends/monet5/rel_physical.c
sql/backends/monet5/rel_pphash.c
sql/backends/monet5/sql_pp_statement.c
sql/backends/monet5/sql_pp_statement.h
sql/include/sql_relation.h
sql/server/rel_rel.c
sql/server/rel_unnest.c
Branch: default
Log Message:
fixes for the anti-join, ie in case right hand side is empty.
fixes for handling nil's with anti-join (where we use a hash on left)
diffs (truncated from 695 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -9681,7 +9681,8 @@ iaggr sum command iaggr.sum(X_0:bat[:flt
iaggr sum command iaggr.sum(X_0:bat[:dbl], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:flt, X_3:flt,
X_4:lng) IALGfsum_skipnil_flt Return the Kahan/Neumaier summation or nil.
-ialgebra matched command ialgebra.matched(X_0:bat[:bit],
X_1:bat[:oid]):bat[:oid] OAHASHmatched Mark matched rows, only return
rows matched rows on first match.
+ialgebra matched unsafe command ialgebra.matched(X_0:bat[:bit],
X_1:bat[:bit], X_2:bat[:oid]):bat[:oid] OAHASHmark_nils Mark rows (nil no
value, 1 nil, 0 rest), return rows.
+ialgebra matched unsafe command ialgebra.matched(X_0:bat[:bit],
X_1:bat[:oid]):bat[:oid] OAHASHmatched Mark matched rows, only return rows
matched rows on first match.
ialgebra project pattern ialgebra.project(X_0:bat[:oid],
X_1:any_1):bat[:any_1] LALGconstant Project a single value
ialgebra projection command ialgebra.projection(X_0:bat[:oid],
X_1:bat[:any_1]):bat[:any_1] LALGprojection Project.
ialgebra unique command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid],
X_2:bat[:oid], X_3:bit) (X_4:bat[:oid], X_5:bat[:any_1])
LALGgroup_unique Unique rows per group.
@@ -10105,6 +10106,7 @@ oahash combined_oprobe command oahash.co
oahash combined_oprobe command oahash.combined_oprobe(X_0:bat[:any_1],
X_1:bat[:oid], X_2:bat[:oid], X_3:bat[:any_1], X_4:bat[:lng], X_5:bit, X_6:bit)
(X_7:bat[:oid], X_8:bat[:oid], X_9:bat[:bit]) OAHASHoprobe_cmbd_single
Probe the selected `key`-s in the hash table. For a matched item, return its
OID in the 'key' column and the slot ID in the hash table
oahash combined_probe command oahash.combined_probe(X_0:bat[:any_1],
X_1:bat[:oid], X_2:bat[:oid], X_3:bat[:any_1], X_4:bit, X_5:bit)
(X_6:bat[:oid], X_7:bat[:oid]) OAHASHprobe_cmbd Probe the selected
`key`-s in the hash table. For a matched item, return its OID in the 'key'
column and the slot ID in the hash table
oahash combined_probe command oahash.combined_probe(X_0:bat[:any_1],
X_1:bat[:oid], X_2:bat[:oid], X_3:bat[:any_1], X_4:bat[:lng], X_5:bit, X_6:bit)
(X_7:bat[:oid], X_8:bat[:oid]) OAHASHprobe_cmbd_single Probe the selected
`key`-s in the hash table. For a matched item, return its OID in the 'key'
column and the slot ID in the hash table
+oahash empty command oahash.empty(X_0:bat[:any_1]):bit OAHASHempty
Hash empty.
oahash expand pattern oahash.expand(X_0:bat[:oid], X_1:bat[:oid],
X_2:bat[:lng], X_3:bit):bat[:oid] OAHASHexpand Expand the probe-side OIDs
according to their matching hash-side GIDs and frequencies. If 'leftouter' is
true, include the not matched OIDs
oahash expand_cartesian command
oahash.expand_cartesian(X_0:bat[:any_1], X_1:bat[:any_2], X_2:bit):bat[:oid]
OAHASHexpand_cart Duplicate each value in 'col' the number of times as
the count of 'rowrepeat'. For a left/right-outer join, if 'rowrepeat' is empty,
output the values in 'col' once.
oahash explode command oahash.explode(X_0:bat[:oid], X_1:bat[:lng],
X_2:bat[:any_2], X_3:bit):bat[:oid] OAHASHexplode Explode the result
vector 'frequency' times and return payload heap slot ids. If 'left_outer' is
true, fill the not 'selected' slot with oid_nil
@@ -10128,6 +10130,7 @@ oahash oprobe command oahash.oprobe(X_0:
oahash oprobe command oahash.oprobe(X_0:bat[:any_1], X_1:bat[:any_1],
X_2:bat[:lng], X_3:bit, X_4:bit) (X_5:bat[:oid], X_6:bat[:oid], X_7:bat[:bit])
OAHASHoprobe_single Probe the `key`-s in the hash table. For a matched key,
return its OID in the 'key' column and the slot ID in the hash table
oahash probe command oahash.probe(X_0:bat[:any_1], X_1:bat[:any_1], X_2:bit,
X_3:bit) (X_4:bat[:oid], X_5:bat[:oid]) OAHASHprobe Probe the `key` in the
hash table. For a matched key, return its OID in the 'key' column and the slot
ID in the hash table
oahash probe command oahash.probe(X_0:bat[:any_1], X_1:bat[:any_1],
X_2:bat[:lng], X_3:bit, X_4:bit) (X_5:bat[:oid], X_6:bat[:oid])
OAHASHprobe_single Probe the `key`-s in the hash table. For a matched key,
return its OID in the 'key' column and the slot ID in the hash table
+oahash select_unmatched command oahash.select_unmatched(X_0:bat[:bit],
X_1:lng):bat[:oid] OAHASHselect_unmatched Select unmatched rows, if cnt
== 0 return all.
optimizer aliases pattern optimizer.aliases():str OPTwrapper (empty)
optimizer aliases pattern optimizer.aliases(X_0:str, X_1:str):str
OPTwrapper Alias removal optimizer
optimizer candidates pattern optimizer.candidates():str
OPTwrapper (empty)
diff --git a/monetdb5/modules/mal/pp_hash.c b/monetdb5/modules/mal/pp_hash.c
--- a/monetdb5/modules/mal/pp_hash.c
+++ b/monetdb5/modules/mal/pp_hash.c
@@ -706,6 +706,70 @@ OAHASHmatched(Client ctx, bat *matched_f
}
static str
+OAHASHmark_nils(Client ctx, bat *res, const bat *matched, const bat *nils,
const bat *rows)
+{
+ (void)ctx;
+ BAT *m = BATdescriptor(*matched);
+ if (!m)
+ return createException(MAL, "oahash.matched", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
+ BAT *n = BATdescriptor(*nils);
+ if (!n) {
+ BBPreclaim(m);
+ return createException(MAL, "oahash.matched", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
+ }
+ BAT *r = BATdescriptor(*rows);
+ if (!r) {
+ BBPreclaim(m);
+ BBPreclaim(n);
+ return createException(MAL, "oahash.matched", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
+ }
+ BUN cnt = BATcount(r);
+ QryCtx *qry_ctx = MT_thread_get_qry_ctx();
+ qry_ctx = qry_ctx ? qry_ctx : &(QryCtx) {.endtime = 0};
+ bit *mrk = Tloc(m, 0);
+ bit *nil = Tloc(n, 0);
+ oid *row = Tloc(r, 0);
+
+ MT_lock_set(&m->theaplock);
+ oid max = BATcount(m);
+ str err = MAL_SUCCEED;
+ if (cnt && r->ttype == TYPE_oid) {
+ TIMEOUT_LOOP_IDX_DECL(i, cnt, qry_ctx) {
+ oid j = row[i];
+ if (j >= max) {
+ MT_lock_unset(&m->theaplock);
+ if (j >= BATcapacity(m) && BATextend(m,
BATgrows(m)) != GDK_SUCCEED) {
+ MT_lock_set(&m->theaplock);
+ err = MAL_MALLOC_FAIL;
+ break;
+ }
+ mrk = Tloc(m, 0);
+ MT_lock_set(&m->theaplock);
+ memset(mrk+max, bit_nil, j+1 - max);
+ max = j+1;
+ BATsetcount(m, j+1);
+ }
+ if (nil[i]) {
+ mrk[j] = TRUE;
+ } else {
+ mrk[j] = FALSE;
+ }
+ }
+ } else if (cnt) { /* handle voids */
+ assert(0);
+ }
+ BATnegateprops(m);
+ MT_lock_unset(&m->theaplock);
+ if (!err)
+ TIMEOUT_CHECK(qry_ctx, err = createException(SQL,
"oahash.build", RUNTIME_QRY_TIMEOUT));
+ BBPreclaim(m);
+ BBPreclaim(n);
+ *res = r->batCacheid;
+ BBPkeepref(r);
+ return err;
+}
+
+static str
UHASHext(Client cntxt, MalBlkPtr m, MalStkPtr s, InstrPtr p)
{
(void)cntxt;
@@ -3729,6 +3793,41 @@ error:
}
static str
+OAHASHselect_unmatched(Client ctx, bat *res, const bat *slice, const lng *cnt)
+{
+ (void)ctx;
+
+ BAT *r = NULL, *s = NULL;
+ str err = NULL;
+
+ s = BATdescriptor(*slice);
+ if (!s) {
+ err = createException(SQL, "oahash.select_unmatched",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ goto error;
+ }
+
+ BUN scnt = BATcount(s);
+ if (*cnt) {
+ bit f = FALSE;
+ r = BATthetaselect(s, NULL, (ptr)&f, "==");
+ } else {
+ r = BATdense (s->hseqbase, s->hseqbase, scnt);
+ }
+ if (!r) {
+ err = createException(SQL, "oahash.select_unmatched",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ goto error;
+ }
+ *res = r->batCacheid;
+ BBPkeepref(r);
+ BBPunfix(s->batCacheid);
+ return MAL_SUCCEED;
+error:
+ BBPreclaim(r);
+ BBPreclaim(s);
+ return err;
+}
+
+static str
OAHASHno_slices(Client ctx, int *no_slices, bat *ht_sink)
{
(void)ctx;
@@ -3793,6 +3892,22 @@ OAHASHnth_slice(Client ctx, bat *slice,
return MAL_SUCCEED;
}
+static str
+OAHASHempty(Client ctx, bit *empty, bat *ht_sink)
+{
+ (void)ctx;
+ /* return the nth slice */
+ assert(*ht_sink && !is_bat_nil(*ht_sink));
+ BAT *b = BATdescriptor(*ht_sink);
+ if (!b)
+ return createException(SQL, "oahash.empty", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
+ hash_table *h = (hash_table*)b->pl_io;
+ assert(h && h->pl_io.type == PIPELINE_IO_HASH_TABLE);
+ *empty = h->empty;
+ BBPunfix(b->batCacheid);
+ return MAL_SUCCEED;
+}
+
#define hashloop(T) \
do { \
T *v = Tloc(i, 0); \
@@ -3924,14 +4039,19 @@ static mel_func oa_hash_init_funcs[] = {
command("oahash", "explode_cartesian", OAHASHexplode_cart, false, "Duplicate
the whole 'col' the number of times as the count of 'setrepeat'. For a
left/right-ourter join, if 'col' is empty, output NULLs.", args(1,4,
batarg("fetched",oid),batargany("col",1),batarg("setrepeat",2),arg("left_outer",bit))),
command("oahash", "explode_unmatched", OAHASHexplode_unmatched, false,
"Expand the count of 'unmatched' with 'frequency'. Returns the count in a VOID
BAT.", args(1,4,
batarg("",oid),batargany("ht_sink",1),batarg("unmatched",oid),batarg("frequency",lng))),
+ command("oahash", "select_unmatched", OAHASHselect_unmatched, false, "Select
unmatched rows, if cnt == 0 return all.", args(1,3,
batarg("",oid),batarg("slice",bit),arg("cnt",lng))),
+
command("oahash", "no_slices", OAHASHno_slices, false, "Get the number of
slices for this hashtable.", args(1,2,
arg("slices",int),batargany("ht_sink",1))),
command("oahash", "nth_slice", OAHASHnth_slice, false, "Get the nth slice of
this hashtable.", args(1,3,
batarg("slice",oid),batargany("ht_sink",1),arg("slice_nr",int))),
+ command("oahash", "empty", OAHASHempty, false, "Hash empty.", args(1,2,
arg("empty", bit), batargany("hash", 1))),
pattern("oahash", "hash", OAHASHhash, false, "Compute hash.", args(1,2,
arg("hash", lng), argany("in",1))),
pattern("batoahash", "hash", OAHASHhash, false, "Compute hash.", args(1,2,
batarg("hash", lng), batargany("in",1))),
- command("ialgebra", "matched", OAHASHmatched, false, "Mark matched rows, only
return rows matched rows on first match.", args(1,3, batarg("matched_once",
oid), batarg("matched", bit), batarg("rows", oid))),
+ command("ialgebra", "matched", OAHASHmatched, true, "Mark matched rows, only
return rows matched rows on first match.", args(1,3, batarg("matched_once",
oid), batarg("matched", bit), batarg("rows", oid))),
+
+ command("ialgebra", "matched", OAHASHmark_nils, true, "Mark rows (nil no
value, 1 nil, 0 rest), return rows.", args(1,4, batarg("dummy", oid),
batarg("matched", bit), batarg("isnil", bit), batarg("rows", oid))),
{ .imp=NULL }
};
diff --git a/sql/backends/monet5/rel_physical.c
b/sql/backends/monet5/rel_physical.c
--- a/sql/backends/monet5/rel_physical.c
+++ b/sql/backends/monet5/rel_physical.c
@@ -490,6 +490,7 @@ find_cmp_exps(list **exps_hsh, list **ex
for (node *n = exps->h; n; n = n->next) {
sql_exp *e = n->data;
bit any = is_any(e);
+ bit semantic = is_semantics(e);
assert(e->type == e_cmp);
@@ -505,6 +506,8 @@ find_cmp_exps(list **exps_hsh, list **ex
set_unique(er);
if (any)
set_any(er);
+ if (semantic)
+ set_semantics(er);
append(*exps_hsh, er);
append(*exps_prb, e->l);
} else {
@@ -515,6 +518,8 @@ find_cmp_exps(list **exps_hsh, list **ex
set_unique(el);
if (any)
set_any(el);
+ if (semantic)
+ set_semantics(el);
append(*exps_hsh, el);
append(*exps_prb, e->r);
}
@@ -899,6 +904,7 @@ rel_pipeline(visitor *v, sql_rel *rel, b
need_all = true;
rel_hsh = rel_buildhash(v, rel_hsh, NULL,
list_empty(eq_exps));
rel_hsh->flag = (int)(needs_payload?op_join:op_semi);
+ rel_hsh->oahash = (rel->op == op_anti &&
rel->oahash==1);
}
rel_prb = rel_probehash(v, rel_prb, NULL);
rel->r = (rel->oahash==1) ? rel_prb : rel_hsh;
diff --git a/sql/backends/monet5/rel_pphash.c b/sql/backends/monet5/rel_pphash.c
--- a/sql/backends/monet5/rel_pphash.c
+++ b/sql/backends/monet5/rel_pphash.c
@@ -185,11 +185,11 @@ oahash_slicer(backend *be, stmt *sub)
/* Generates the parallel block to probe the hash table
*/
static stmt *
-hash_probe(backend *be, sql_rel *rel, list *jexps, list *exps_cmp_prb, const
stmt *stmts_ht, stmt *sub, bool anti, bool groupjoin, bool has_outerselect,
stmt **nulls, stmt **prb_mrk)
+hash_probe(backend *be, sql_rel *rel, list *jexps, list *exps_cmp_prb, const
stmt *stmts_ht, stmt *sub, bool anti, bool groupjoin, bool has_outerselect,
stmt **nulls, stmt **prb_mrk, stmt *match_cnt, stmt **ht_empty)
{
stmt *prb_res = NULL, *outerm = NULL;
bool has_leftouter = (rel->op == op_left || rel->op == op_full /*||
(rel->op == op_right && rel->oahash == 1)*/);
- bool perfect = stmts_ht->flag;
+ bool perfect = stmts_ht->flag, first = true;
/* stmts_ht is in the same order as the join columns */
for (node *n = exps_cmp_prb->h, *m = stmts_ht->op4.lval->h, *o =
jexps->h; n && m && o; n = n->next, m = m->next, o = o->next) {
@@ -202,6 +202,19 @@ hash_probe(backend *be, sql_rel *rel, li
bool eq = (e2->flag == cmp_equal) && !anti;
bool grpjoin = groupjoin && is_any(e2);
+ if (first && match_cnt && is_any(e2)) {
+ first = false;
+ InstrPtr q = newStmt(be->mb, aggrRef, "count"); /*
normal count */
+ /* result lng */
+ setVarType(be->mb, getArg(q, 0), TYPE_lng);
+ q = pushArgument(be->mb, q, key->nr);
+ pushInstruction(be->mb, q);
+ /* sum into match_cnt */
+ InstrPtr r = newStmt(be->mb, lockedaggrRef, "sum");
+ r->argv[0] = match_cnt->nr;
+ r = pushArgument(be->mb, r, q->argv[0]);
+ pushInstruction(be->mb, r);
+ }
assert(!perfect || (!groupjoin && !has_leftouter && !prb_res));
prb_res = perfect ?
stmt_fhash_probe(be, key, m->data,
stmts_ht->op3, single, e2->semantics, eq) :
@@ -215,6 +228,20 @@ hash_probe(backend *be, sql_rel *rel, li
if (nulls && stmt_has_null(key) && is_any(e2))
*nulls = stmt_selectnil(be, key, *nulls);
+ if (ht_empty && m && !m->next)
+ *ht_empty = stmt_hash_empty(be, m->data);
+ }
+ if (first && match_cnt) {
+ InstrPtr q = newStmt(be->mb, aggrRef, "count"); /* normal count
*/
+ /* result lng */
+ setVarType(be->mb, getArg(q, 0), TYPE_lng);
+ q = pushArgument(be->mb, q, prb_res->nr);
+ pushInstruction(be->mb, q);
+ /* sum into match_cnt */
+ InstrPtr r = newStmt(be->mb, lockedaggrRef, "sum");
+ r->argv[0] = match_cnt->nr;
+ r = pushArgument(be->mb, r, q->argv[0]);
+ pushInstruction(be->mb, r);
}
if ((has_leftouter || groupjoin) && outerm)
*prb_mrk = outerm;
@@ -374,6 +401,17 @@ rel2bin_oahash_build(backend *be, sql_re
shared_hp = oahash_prepare_bld_hp(be, exps_prj_hsh, bld_sz);
}
+ bool anti_join = (rel->oahash == 1);
+ bool need_nulls = anti_join /* anti join */, attribute_need_nulls =
false;
+ stmt *matched = NULL, *nulls = NULL;
+ for (node *n = exps_cmp_hsh->h; n && !attribute_need_nulls; n =
n->next) {
+ sql_exp *e = n->data;
+ attribute_need_nulls = (has_nil(e) && !is_semantics(e));
+ }
+ need_nulls &= attribute_need_nulls;
+ if (need_nulls || anti_join)
+ matched = stmt_bat_new(be, sql_fetch_localtype(TYPE_bit),
bld_sz);
+
stmt *sub = _start_pp(be, rel->l, true, refs, shared_ht);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
