Changeset: 7b17cec675a8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/7b17cec675a8
Modified Files:
clients/Tests/MAL-signatures-hge.test
monetdb5/modules/mal/pp_algebra.c
monetdb5/modules/mal/pp_hash.c
monetdb5/modules/mal/pp_hash.h
monetdb5/modules/mal/pp_mat.c
sql/backends/monet5/bin_partition_by_slice.c
sql/backends/monet5/rel_pphash.c
sql/backends/monet5/rel_pptopn.c
sql/backends/monet5/sql_pp_statement.c
sql/backends/monet5/sql_pp_statement.h
sql/server/rel_statistics.c
sql/storage/bat/bat_storage.c
Branch: pp_hashjoin
Log Message:
try using vkey info in LALGunique, no need to use the strings.
diffs (truncated from 411 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -50545,6 +50545,16 @@ OAHASHnew
(empty)
oahash
new
+pattern oahash.new(X_0:any_1, X_1:int, X_2:bat[:any_2], X_3:int):bat[:any_1]
+OAHASHnew
+(empty)
+oahash
+new
+pattern oahash.new(X_0:any_1, X_1:int, X_2:int):bat[:any_1]
+OAHASHnew
+(empty)
+oahash
+new
pattern oahash.new(X_0:any_1, X_1:lng):bat[:any_1]
OAHASHnew
(empty)
@@ -50554,6 +50564,16 @@ pattern oahash.new(X_0:any_1, X_1:lng, X
OAHASHnew
(empty)
oahash
+new
+pattern oahash.new(X_0:any_1, X_1:lng, X_2:bat[:any_2], X_3:int):bat[:any_1]
+OAHASHnew
+(empty)
+oahash
+new
+pattern oahash.new(X_0:any_1, X_1:lng, X_2:int):bat[:any_1]
+OAHASHnew
+(empty)
+oahash
no_slices
command oahash.no_slices(X_0:bat[:any_1]):int
OAHASHno_slices
diff --git a/monetdb5/modules/mal/pp_algebra.c
b/monetdb5/modules/mal/pp_algebra.c
--- a/monetdb5/modules/mal/pp_algebra.c
+++ b/monetdb5/modules/mal/pp_algebra.c
@@ -1049,11 +1049,12 @@ LALGunique(Client ctx, bat *rid, bat *ui
goto error;
}
+ BUN cnt = BATcount(b);
hash_table *h = (hash_table*)u->pl_io;
assert(h && h->pl_io.type == PIPELINE_IO_HASH_TABLE);
MT_lock_set(&u->theaplock);
MT_lock_set(&b->theaplock);
- if (ATOMvarsized(u->ttype) /*&& !VIEWvtparent(b)*/) {
+ if (ATOMvarsized(u->ttype) && /*!VIEWvtparent(b)*/ !h->vkey) {
local_storage = true;
MT_lock_unset(&b->theaplock);
MT_lock_unset(&u->theaplock);
@@ -1078,11 +1079,11 @@ LALGunique(Client ctx, bat *rid, bat *ui
goto error;
}
}
- } else if (ATOMvarsized(u->ttype) && BATcount(b) && BATcount(u) == 0 &&
u->tvheap->parentid == u->batCacheid) {
+ } else if (ATOMvarsized(u->ttype) && cnt && BATcount(u) == 0 &&
u->tvheap->parentid == u->batCacheid) {
MT_lock_unset(&b->theaplock);
MT_lock_unset(&u->theaplock);
BATswap_heaps(u, b, p);
- } else if (ATOMvarsized(u->ttype) && u->tvheap->parentid !=
b->tvheap->parentid) {
+ } else if (ATOMvarsized(u->ttype) && cnt && u->tvheap->parentid !=
b->tvheap->parentid) {
int i = 0;
for(i = 0; i < h->pinned_nr; i++) {
if (h->pinned[i] == b->tvheap)
@@ -1101,8 +1102,6 @@ LALGunique(Client ctx, bat *rid, bat *ui
MT_lock_unset(&u->theaplock);
}
if (h) {
- BUN cnt = BATcount(b);
-
ATOMIC_BASE_TYPE expected = 0;
BUN r = 0;
@@ -1115,6 +1114,8 @@ LALGunique(Client ctx, bat *rid, bat *ui
ht_activate(h);
/* probably need bat resize and create hash */
int tt = b->ttype;
+ if (h->vkey)
+ tt = h->type;
oid *gp = Tloc(g, 0);
QryCtx *qry_ctx = MT_thread_get_qry_ctx();
@@ -1635,7 +1636,7 @@ LALGgroup(Client ctx, bat *rid, bat *uid
err = createException(MAL, "igroup.group",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
goto error;
}
- u->pl_io = (struct
pipeline_io*)ht_create(b->ttype?b->ttype:TYPE_oid, 1, NULL);
+ u->pl_io = (struct
pipeline_io*)ht_create(b->ttype?b->ttype:TYPE_oid, 1, NULL, 0);
if (u->pl_io == NULL) {
err = createException(MAL, "igroup.group",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
goto error;
@@ -1928,7 +1929,7 @@ LALGderive(Client ctx, bat *rid, bat *ui
goto error;
}
/* Lookup parent hash */
- u->pl_io = (struct
pipeline_io*)ht_create(b->ttype?b->ttype:TYPE_oid, 1, (hash_table*)H->pl_io);
+ u->pl_io = (struct
pipeline_io*)ht_create(b->ttype?b->ttype:TYPE_oid, 1, (hash_table*)H->pl_io, 0);
if (u->pl_io == NULL) {
BBPunfix(H->batCacheid);
err = createException(MAL, "igroup.group",
SQLSTATE(HY013) MAL_MALLOC_FAIL);
diff --git a/monetdb5/modules/mal/pp_hash.c b/monetdb5/modules/mal/pp_hash.c
--- a/monetdb5/modules/mal/pp_hash.c
+++ b/monetdb5/modules/mal/pp_hash.c
@@ -86,7 +86,7 @@ ht_destroy(hash_table *ht)
}
static hash_table *
-_ht_create( int type, size_t size, hash_table *p)
+_ht_create( int type, size_t size, hash_table *p, int vkey)
{
hash_table *h = (hash_table*)GDKzalloc(sizeof(hash_table));
if (!h)
@@ -109,13 +109,17 @@ _ht_create( int type, size_t size, hash_
h->p = p;
h->pinned = NULL;
h->pinned_nr = 0; /* no more than 1024 */
- if (type == TYPE_str) {
+ h->vkey = vkey;
+ if (type == TYPE_str && !h->vkey) {
h->cmp = (fcmp)str_cmp;
h->hsh = (fhsh)str_hsh;
} else {
+ type = !vkey ? type : vkey == 1 ? TYPE_bte : vkey == 2 ?
TYPE_sht : vkey == 4 ? TYPE_int : TYPE_lng;
h->cmp = (fcmp)ATOMcompare(type);
h->hsh = (fhsh)BATatoms[type].atomHash;
h->len = (flen)BATatoms[type].atomLen;
+ if (vkey)
+ h->type = type;
}
h->processed = 0;
MT_rwlock_init(&h->rwlock, "ht_create");
@@ -129,13 +133,13 @@ _ht_create( int type, size_t size, hash_
}
hash_table *
-ht_create(int type, size_t size, hash_table *p)
+ht_create(int type, size_t size, hash_table *p, int vkey)
{
if (size < HT_MIN_SIZE)
size = HT_MIN_SIZE;
if (size > HT_MAX_SIZE)
size = HT_MAX_SIZE;
- return _ht_create(type, size, p);
+ return _ht_create(type, size, p, vkey);
}
void
@@ -405,6 +409,7 @@ OAHASHnew(Client cntxt, MalBlkPtr m, Mal
lng size = 0;
hash_table *parent = NULL;
BAT *pht = NULL;
+ int vkey = 0;
if (tt2 == TYPE_int) {
assert(0);
@@ -416,11 +421,17 @@ OAHASHnew(Client cntxt, MalBlkPtr m, Mal
/* multiply with the magic estimation while avoiding overflow */
size = size > ((dbl)INT64_MAX / 1.2 / 2.1)? INT64_MAX : (lng)(size *
1.2 * 2.1);
- if (p->argc == 4) {
- bat pid = *getArgReference_bat(s, p, 3);
- if ((pht = BATdescriptor(pid)) == NULL)
- return createException(MAL, "oahash.new",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
- parent = (hash_table*)pht->pl_io;
+ if (p->argc >= 4) {
+ int tt4 = getArgType(m, p, 3);
+ int tt5 = (p->argc == 5) ? getArgType(m, p, 4) : 0;
+
+ if (tt5 || tt4 != TYPE_int) {
+ bat pid = *getArgReference_bat(s, p, 3);
+ if ((pht = BATdescriptor(pid)) == NULL)
+ return createException(MAL, "oahash.new",
SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ parent = (hash_table*)pht->pl_io;
+ }
+ vkey = (tt5 || tt4 == TYPE_int) ? *getArgReference_int(s, p,
tt4 == TYPE_int ? 3 : 4) : false;
}
BAT *b = COLnew(0, tt, 0, TRANSIENT);
@@ -428,7 +439,7 @@ OAHASHnew(Client cntxt, MalBlkPtr m, Mal
BBPreclaim(pht);
return createException(MAL, "oahash.new", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
}
- b->pl_io = (struct pipeline_io*)ht_create(tt, (size_t)size, parent);
+ b->pl_io = (struct pipeline_io*)ht_create(tt, (size_t)size, parent,
vkey);
BBPreclaim(pht);
if (b->pl_io == NULL) {
BBPunfix(b->batCacheid);
@@ -3433,8 +3444,12 @@ OAHASHhash(Client cntxt, MalBlkPtr m, Ma
static mel_func oa_hash_init_funcs[] = {
pattern("oahash", "new", OAHASHnew, false, "", args(1,3,
batargany("ht_sink",1),argany("tt",1),arg("size",int))),
pattern("oahash", "new", OAHASHnew, false, "", args(1,4,
batargany("ht_sink",1),argany("tt",1),arg("size",int),batargany("p",2))),
+ pattern("oahash", "new", OAHASHnew, false, "", args(1,4,
batargany("ht_sink",1),argany("tt",1),arg("size",int), arg("vkey", int))),
+ pattern("oahash", "new", OAHASHnew, false, "", args(1,5,
batargany("ht_sink",1),argany("tt",1),arg("size",int),batargany("p",2),
arg("vkey", int))),
pattern("oahash", "new", OAHASHnew, false, "", args(1,3,
batargany("ht_sink",1),argany("tt",1),arg("size",lng))),
pattern("oahash", "new", OAHASHnew, false, "", args(1,4,
batargany("ht_sink",1),argany("tt",1),arg("size",lng),batargany("p",2))),
+ pattern("oahash", "new", OAHASHnew, false, "", args(1,4,
batargany("ht_sink",1),argany("tt",1),arg("size",lng), arg("vkey", int))),
+ pattern("oahash", "new", OAHASHnew, false, "", args(1,5,
batargany("ht_sink",1),argany("tt",1),arg("size",lng),batargany("p",2),
arg("vkey", int))),
command("oahash", "hashmark_init", OAHASHhashmark_init, false, "", args(1,3,
batarg("hashmark",bit),batargany("ht_sink",1),batargany("payload",2))),
pattern("hash", "ext", UHASHext, false, "", args(1,2,
batarg("ext",oid),batargany("in",1))),
diff --git a/monetdb5/modules/mal/pp_hash.h b/monetdb5/modules/mal/pp_hash.h
--- a/monetdb5/modules/mal/pp_hash.h
+++ b/monetdb5/modules/mal/pp_hash.h
@@ -132,6 +132,7 @@ typedef struct hash_table {
fhsh hsh;
flen len;
bool empty;
+ int vkey; /* vheap is unique, ie hashing on offsets */
void *vals; /* hash(ed) values */
hash_key_t *gids; /* chain of gids (k, ie mark used/-k mark used and
value filled) */
@@ -165,7 +166,7 @@ str_hsh( str v )
return h;
}
-extern hash_table *ht_create(int type, size_t size, hash_table *p);
+extern hash_table *ht_create(int type, size_t size, hash_table *p, int vkey);
extern int ht_rehash(hash_table *ht);
extern void ht_activate(hash_table *ht);
diff --git a/monetdb5/modules/mal/pp_mat.c b/monetdb5/modules/mal/pp_mat.c
--- a/monetdb5/modules/mal/pp_mat.c
+++ b/monetdb5/modules/mal/pp_mat.c
@@ -136,7 +136,7 @@ MATnew(Client cntxt, MalBlkPtr mb, MalSt
break;
BATnegateprops(b);
if (hashsize)
- b->pl_io = (struct pipeline_io*)ht_create(tt,
(size_t)(hashsize), pmat?(hash_table*)pmat->bat[i]->pl_io:NULL);
+ b->pl_io = (struct pipeline_io*)ht_create(tt,
(size_t)(hashsize), pmat?(hash_table*)pmat->bat[i]->pl_io:NULL, 0);
}
if (i < mat->nr) {
mat_destroy(mat);
diff --git a/sql/backends/monet5/bin_partition_by_slice.c
b/sql/backends/monet5/bin_partition_by_slice.c
--- a/sql/backends/monet5/bin_partition_by_slice.c
+++ b/sql/backends/monet5/bin_partition_by_slice.c
@@ -63,8 +63,8 @@ exp_getcard(mvc *sql, sql_rel *rel, sql_
if (c) {
int de = mvc_is_duplicate_eliminated(sql, c);
- if (de && de < cnt)
- cnt = de;
+ if (de && (1<<(de*8)) < cnt)
+ cnt = (1<<(de*8));
}
}
/* for now only based on type info, later use propagated cardinality
estimation */
@@ -261,7 +261,7 @@ rel_groupby_prepare_pp(list **aggrresult
estimate =
85000000;
}
- stmt *s =
stmt_oahash_new(be, t, estimate, curhash, 0); /* pushed already */
+ stmt *s =
stmt_oahash_new(be, t, estimate, curhash, 0, 0); /* pushed already */
if (s == NULL)
return NULL;
assert(!e->shared);
@@ -320,8 +320,13 @@ rel_groupby_prepare_pp(list **aggrresult
assert(0);
estimate = 85000000;
}
+ int vkey = 0;
+ if (a) {
+ sql_column *c = exp_find_column(rel, a,
-1);
+ vkey = c ?
mvc_is_duplicate_eliminated(be->mvc, c) : 0;
+ }
- stmt *s = stmt_oahash_new(be, t, estimate, 0,
0); /* pushed already */
+ stmt *s = stmt_oahash_new(be, t, estimate, 0,
0, vkey); /* pushed already */
if (s == NULL)
return NULL;
assert(!e->shared);
@@ -351,7 +356,7 @@ rel_groupby_prepare_pp(list **aggrresult
if (card > INT_MAX)
card = INT_MAX;
- stmt *s = stmt_oahash_new(be, t,
nrparts?PARTITION_NRPARTS:card, curhash, nrparts);
+ stmt *s = stmt_oahash_new(be, t,
nrparts?PARTITION_NRPARTS:card, curhash, nrparts, 0);
if (s == NULL)
return NULL;
curhash = s->nr;
@@ -402,7 +407,7 @@ rel_groupby_prepare_pp(list **aggrresult
else
estimate =
(BUN) est;
- stmt *s =
stmt_oahash_new(be, t, estimate, grphash, 0);
+ stmt *s =
stmt_oahash_new(be, t, estimate, grphash, 0, 0);
if (s == NULL)
return NULL;
assert(!e->shared);
@@ -466,7 +471,7 @@ rel_groupby_prepare_pp(list **aggrresult
if ((BUN)estimate > est)
estimate = est;
- stmt *s = stmt_oahash_new(be, t,
nrparts?PARTITION_NRPARTS:estimate, curhash, nrparts);
+ stmt *s = stmt_oahash_new(be, t,
nrparts?PARTITION_NRPARTS:estimate, curhash, nrparts, 0);
if (s == NULL)
return NULL;
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
