Changeset: 0566e1f2e779 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/0566e1f2e779
Added Files:
sql/backends/monet5/dict.c
sql/backends/monet5/dict.h
Modified Files:
sql/backends/monet5/CMakeLists.txt
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/include/sql_catalog.h
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
sql/storage/sql_storage.h
Branch: dict
Log Message:
v1 dictionary compression
diffs (truncated from 642 to 300 lines):
diff --git a/sql/backends/monet5/CMakeLists.txt
b/sql/backends/monet5/CMakeLists.txt
--- a/sql/backends/monet5/CMakeLists.txt
+++ b/sql/backends/monet5/CMakeLists.txt
@@ -151,6 +151,7 @@ target_sources(sql
sql_rank.c sql_rank.h
sql_subquery.c sql_subquery.h
opt_backend.h
+ dict.c dict.h
${MONETDB_CURRENT_SQL_SOURCES}
PUBLIC
${sql_public_headers})
diff --git a/sql/backends/monet5/dict.c b/sql/backends/monet5/dict.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/dict.c
@@ -0,0 +1,178 @@
+
+#include "monetdb_config.h"
+#include "sql.h"
+#include "mal.h"
+#include "mal_client.h"
+
+#include "dict.h"
+
+str
+DICTcompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ (void)mb;
+ /* always assume one result */
+ str msg = MAL_SUCCEED;
+ const char *sname = *getArgReference_str(stk, pci, 1);
+ const char *tname = *getArgReference_str(stk, pci, 2);
+ const char *cname = *getArgReference_str(stk, pci, 3);
+ backend *be = NULL;
+ sql_trans *tr = NULL;
+
+ if (!sname || !tname || !cname)
+ throw(SQL, "sql.dict_compress", SQLSTATE(3F000) "dict compress:
invalid column name");
+ if ((msg = getBackendContext(cntxt, &be)) != MAL_SUCCEED)
+ return msg;
+ tr = be->mvc->session->tr;
+
+ sql_schema *s = find_sql_schema(tr, sname);
+ assert(s);
+ sql_table *t = find_sql_table(tr, s, tname);
+ assert(t);
+ sql_column *c = find_sql_column(t, cname);
+ assert(c);
+
+ sqlstore *store = tr->store;
+ BAT *b = store->storage_api.bind_col(tr, c, RDONLY);
+
+ /* for now use all rows */
+ BAT *u = BATunique(b, NULL);
+ if (!u)
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+
+ BUN cnt = BATcount(u);
+ /* create hash on u */
+ int tt = (cnt<256)?TYPE_bte:(cnt<(64*1024))?TYPE_sht:TYPE_int;
+ if (cnt > 2L*1024*1024*1024) {
+ bat_destroy(u);
+ bat_destroy(b);
+ throw(SQL, "sql.dict_compress", SQLSTATE(3F000) "dict compress:
too many values");
+ }
+ BAT *uv = BATproject(u, b); /* get values */
+ bat_destroy(u);
+ if (!uv) {
+ bat_destroy(b);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ BAT *uu = COLcopy(uv, uv->ttype, true, PERSISTENT);
+ if (!uu) {
+ bat_destroy(uv);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ bat_destroy(uv);
+ u = uu;
+
+ BAT *o = COLnew(0, tt, BATcount(b), PERSISTENT);
+ if (!o || BAThash(u) != GDK_SUCCEED) {
+ bat_destroy(u);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ BUN p, q;
+ BATiter bi = bat_iterator(b);
+ BATiter ui = bat_iterator_nolock(u);
+ if (tt == TYPE_bte) {
+ bte *op = (bte*)Tloc(o, 0);
+ BATloop(b, p, q) {
+ BUN up = 0;
+ HASHloop(ui, ui.b->thash, up, BUNtail(bi, p)) {
+ op[p] = up;
+ }
+ }
+ BATsetcount(o, BATcount(b));
+ o->tsorted = (u->tsorted && b->tsorted);
+ o->trevsorted = false;
+ o->tnil = b->tnil;
+ o->tnonil = b->tnonil;
+ o->tkey = b->tkey;
+ if (sql_trans_alter_storage(tr, c, "DICT") != LOG_OK ||
store->storage_api.col_dict(tr, c, o, u) != LOG_OK)
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
"alter_storage failed");
+ } else if (tt == TYPE_sht) {
+ sht *op = (sht*)Tloc(o, 0);
+ BATloop(b, p, q) {
+ BUN up = 0;
+ HASHloop(ui, ui.b->thash, up, BUNtail(bi, p)) {
+ op[p] = up;
+ }
+ }
+ BATsetcount(o, BATcount(b));
+ o->tsorted = (u->tsorted && b->tsorted);
+ o->trevsorted = false;
+ o->tnil = b->tnil;
+ o->tnonil = b->tnonil;
+ o->tkey = b->tkey;
+ if (sql_trans_alter_storage(tr, c, "DICT") != LOG_OK ||
store->storage_api.col_dict(tr, c, o, u) != LOG_OK)
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
"alter_storage failed");
+ } else {
+ printf("implement int cases \n");
+ }
+ bat_iterator_end(&bi);
+ bat_destroy(b);
+ bat_destroy(u);
+ bat_destroy(o);
+ return MAL_SUCCEED;
+}
+
+
+str
+DICTdecompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ /* b = project(o:bat[:bte], u) */
+ /* b = project(o:bat[:sht], u) */
+ /* b = project(o:bat[:int], u) */
+ (void)cntxt;
+ (void)mb;
+ bat *r = getArgReference_bat(stk, pci, 0);
+ bat O = *getArgReference_bat(stk, pci, 1);
+ bat U = *getArgReference_bat(stk, pci, 2);
+
+ BAT *o = BATdescriptor(O);
+ BAT *u = BATdescriptor(U);
+ if (!o || !u) {
+ bat_destroy(o);
+ bat_destroy(o);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ BAT *b = COLnew(0, u->ttype, BATcount(o), TRANSIENT);
+
+ BUN p, q;
+ BATiter oi = bat_iterator(o);
+ BATiter ui = bat_iterator_nolock(u);
+ //if (ATOMvarsized(u->ttype)) {
+ if (o->ttype == TYPE_bte) {
+ bte *op = Tloc(o, 0);
+ BATloop(o, p, q) {
+ BUN up = op[p];
+ if (BUNappend(b, BUNtail(ui, up), false) != GDK_SUCCEED) {
+ bat_iterator_end(&oi);
+ bat_destroy(b);
+ bat_destroy(o);
+ bat_destroy(u);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ }
+ } else if (o->ttype == TYPE_sht) {
+ sht *op = Tloc(o, 0);
+ BATloop(o, p, q) {
+ BUN up = op[p];
+ if (BUNappend(b, BUNtail(ui, up), false) != GDK_SUCCEED) {
+ bat_iterator_end(&oi);
+ bat_destroy(b);
+ bat_destroy(o);
+ bat_destroy(u);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
+ }
+ }
+ } else if (o->ttype == TYPE_int) {
+ assert(0);
+ } else {
+ bat_iterator_end(&oi);
+ bat_destroy(b);
+ bat_destroy(o);
+ bat_destroy(u);
+ throw(SQL, "sql.dict_compress", SQLSTATE(HY013) "unknown offset
type");
+ }
+ bat_iterator_end(&oi);
+ BBPkeepref(*r = b->batCacheid);
+ bat_destroy(o);
+ bat_destroy(u);
+ return MAL_SUCCEED;
+}
diff --git a/sql/backends/monet5/dict.h b/sql/backends/monet5/dict.h
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/dict.h
@@ -0,0 +1,11 @@
+
+#ifndef _DICT_H
+#define _DICT_H
+
+#include "sql.h"
+
+extern str DICTcompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
+extern str DICTdecompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
+
+#endif /* _DICT_H */
+
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1576,6 +1576,10 @@ stmt_col( backend *be, sql_column *c, st
sc = stmt_project_delta(be, sc, u);
if (del)
sc = stmt_project(be, del, sc);
+ if (c->storage_type) {
+ stmt *v = stmt_bat(be, c, RD_DICT, part);
+ sc = stmt_dict(be, sc, v);
+ }
} else if (del) { /* always handle the deletes */
sc = stmt_project(be, del, sc);
}
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -5037,6 +5037,7 @@ finalize:
#include "sql_subquery.h"
#include "sql_statistics.h"
#include "sql_transaction.h"
+#include "dict.h"
#include "mel.h"
static mel_func sql_init_funcs[] = {
pattern("sql", "shutdown", SQLshutdown_wrap, false, "", args(1,3,
arg("",str),arg("delay",bte),arg("force",bit))),
@@ -5167,6 +5168,8 @@ static mel_func sql_init_funcs[] = {
pattern("sql", "prepared_statements_args",
SQLsession_prepared_statements_args, false, "Available prepared statements'
arguments in the current session", args(9,9,
batarg("statementid",int),batarg("type",str),batarg("digits",int),batarg("scale",int),batarg("inout",bte),batarg("number",int),batarg("schema",str),batarg("table",str),batarg("column",str))),
pattern("sql", "copy_rejects", COPYrejects, false, "", args(4,4,
batarg("rowid",lng),batarg("fldid",int),batarg("msg",str),batarg("inp",str))),
pattern("sql", "copy_rejects_clear", COPYrejects_clear, true, "", noargs),
+ pattern("sql", "dict_compress", DICTcompress, false, "compress a sql column",
args(0, 3, arg("schema", str), arg("table", str), arg("column", str))),
+ pattern("sql", "dict_decompress", DICTdecompress, false, "decompress a
dictionary compressed (sub)column", args(1, 3, batargany("", 1), batargany("o",
0), batargany("u", 1))),
command("calc", "dec_round", bte_dec_round_wrap, false, "round off the value
v to nearests multiple of r", args(1,3, arg("",bte),arg("v",bte),arg("r",bte))),
pattern("batcalc", "dec_round", bte_bat_dec_round_wrap, false, "round off the
value v to nearests multiple of r", args(1,3,
batarg("",bte),batarg("v",bte),arg("r",bte))),
pattern("batcalc", "dec_round", bte_bat_dec_round_wrap, false, "round off the
value v to nearests multiple of r", args(1,4,
batarg("",bte),batarg("v",bte),arg("r",bte),batarg("s",oid))),
diff --git a/sql/backends/monet5/sql_statement.c
b/sql/backends/monet5/sql_statement.c
--- a/sql/backends/monet5/sql_statement.c
+++ b/sql/backends/monet5/sql_statement.c
@@ -579,6 +579,12 @@ stmt_bat(backend *be, sql_column *c, int
q = newStmtArgs(mb, sqlRef, bindRef, 9);
if (q == NULL)
return NULL;
+ if (c->storage_type && access != RD_DICT) {
+ sql_trans *tr = be->mvc->session->tr;
+ sqlstore *store = tr->store;
+ BAT *b = store->storage_api.bind_col(tr, c, QUICK);
+ tt = b->ttype;
+ }
if (access == RD_UPD_ID) {
q = pushReturn(mb, q, newTmpVariable(mb, newBatType(tt)));
} else {
@@ -2257,6 +2263,39 @@ stmt_left_project(backend *be, stmt *op1
}
stmt *
+stmt_dict(backend *be, stmt *op1, stmt *op2)
+{
+ MalBlkPtr mb = be->mb;
+ InstrPtr q = NULL;
+
+ if (op1->nr < 0 || op2->nr < 0)
+ return NULL;
+
+ q = newStmt(mb, sqlRef, "dict_decompress");
+ q = pushArgument(mb, q, op1->nr);
+ q = pushArgument(mb, q, op2->nr);
+
+ if (q) {
+ stmt *s = stmt_create(be->mvc->sa, st_join);
+ if (s == NULL) {
+ freeInstruction(q);
+ return NULL;
+ }
+
+ s->op1 = op1;
+ s->op2 = op2;
+ s->flag = cmp_project;
+ s->key = 0;
+ s->nrcols = MAX(op1->nrcols,op2->nrcols);
+ s->nr = getDestVar(q);
+ s->q = q;
+ s->tname = op1->tname;
+ s->cname = op1->cname;
+ return s;
+ }
+ return NULL;
+}
+stmt *
stmt_join2(backend *be, stmt *l, stmt *ra, stmt *rb, int cmp, int anti, int
symmetric, int swapped)
{
InstrPtr q = select2_join2(be, l, ra, rb, cmp, NULL, anti, symmetric,
swapped, st_join2, 1/*reduce semantics*/);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
