Changeset: 663fd2a422fa for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/663fd2a422fa
Modified Files:
sql/backends/monet5/vaults/parquet/parquet.c
sql/backends/monet5/vaults/parquet/parquet.h
Branch: parquet
Log Message:
Attempt to generate MAL code for Parquet
Currently crashes later in the code generator
diffs (198 lines):
diff --git a/sql/backends/monet5/vaults/parquet/parquet.c
b/sql/backends/monet5/vaults/parquet/parquet.c
--- a/sql/backends/monet5/vaults/parquet/parquet.c
+++ b/sql/backends/monet5/vaults/parquet/parquet.c
@@ -17,6 +17,7 @@
#include "mal_debugger.h"
#include "mal_linker.h"
#include "sql_types.h"
+#include "sql_statement.h"
#include <unistd.h>
@@ -66,7 +67,7 @@ static char* parquet_type_map(GArrowType
case GARROW_TYPE_UINT64:
case GARROW_TYPE_INT64:
- return "BIGINT";
+ return "bigint";
case GARROW_TYPE_FLOAT:
case GARROW_TYPE_HALF_FLOAT:
@@ -126,6 +127,11 @@ static char* parquet_type_map(GArrowType
return NULL;
}
+// This function is called while the relational plan is being built (the sym
to rel step).
+// It needs to figure out the types of the columns.
+// If it goes well you can see the result by running
+//
+// PLAN SELECT * FROM 'data.parquet';
static str
parquet_add_types(mvc *sql, sql_subfunc *f, char *filename, list *res_exps,
char *tname)
{
@@ -159,6 +165,8 @@ parquet_add_types(mvc *sql, sql_subfunc
if(st) {
sql_subtype *t = sql_bind_subtype(sql->sa, st, 0, 0);
+ if (!t)
+ throw(SQL, SQLSTATE(42000), "Cannot resolve
type '%s'", st);
// list_append(types, t);
list_append(res_exps, exp_column(sql->sa, NULL, name,
t, CARD_MULTI, 1, 0, 0));
@@ -177,39 +185,130 @@ parquet_add_types(mvc *sql, sql_subfunc
return MAL_SUCCEED;
}
-static void *
-parquet_load(void *BE, sql_subfunc *f, char *filename)
+static stmt*
+parquet_emit_plan(backend *be, sql_subfunc *f, char *filename)
{
- (void)BE;
- (void)f;
- (void)filename;
- return NULL;
+ // We cannot use stmt_unop() because our f is bound to a generic
sql_func,
+ // not to one that is specific to Parquet and contains a reference
+ // to PARQUETload.
+ // This means we have to emit the MAL code itself and create a stmt that
+ // reflects it.
+ mvc *mvc = be->mvc;
+ MalBlkPtr mb = be->mb;
+
+ list *return_types = f->res;
+
+ // This is the statement we append to the MAL block:
+ int nargs = list_length(return_types) + 1;
+ InstrPtr p = newStmtArgs(mb, "parquet", "load_table", nargs);
+ if (p == NULL)
+ return sql_error(mvc, 10, SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ // Add the return variables
+ for (node *n = return_types->h; n; n = n->next) {
+ sql_exp *e = n->data;
+ sql_subtype *subtype = &e->tpe;
+ int data_type = subtype->type->localtype;
+ int bat_type = newBatType(data_type);
+ if (n == return_types->h) {
+ // The first return position has already been created by
+ // newStmtArgs above
+ setArgType(mb, p, 0, bat_type);
+ } else {
+ // The other return positions are created by us
+ int var = newTmpVariable(mb, bat_type);
+ p = pushReturn(mb, p, var);
+ }
+ }
+ // Then add the filename
+ p = pushStr(mb, p, filename);
+ // And add the MAL statement to the block
+ pushInstruction(mb, p);
+
+ // Later on, the rest of the SQL compiler will need to know which
+ // MAL variables we stored the result BATs in.
+ // That information goes into the stmt we return.
+ //
+ // I'm not sure about the official way to do this, most stmt_* functions
+ // assume that we already have stmts for the individual columns.
+ // We don't so we apply voodoo voodoo wave dead chicken.
+ list *result_column_stmts = sa_list(mvc->sa);
+ if (!result_column_stmts)
+ return NULL;
+ int i = 0;
+ for (node *n = return_types->h; n; n = n->next) {
+ sql_exp *e = n->data;
+ sql_subtype *subtype = &e->tpe;
+ stmt *s = stmt_blackbox_result(be, p, i++, subtype);
+ result_column_stmts = list_append(result_column_stmts, s);
+ }
+
+ return stmt_list(be, result_column_stmts);
}
+// This function is called while the MAL plan is being built (the rel to bin
step).
+// If it goes well you can see the result by running
+//
+// EXPLAIN SELECT * FROM 'data.parquet';
+static void *
+parquet_generate_plan(void *BE, sql_subfunc *f, char *filename)
+{
+ backend *be = (backend*)BE;
+
+ // So, basically, our task is to append MAL statements to the MAL block
'mb'.
+ // We will return a 'stmt', which is a kind of summary of the
statements we
+ // produced. In particular, the stmt contains the list of MAL variables
in
+ // which the generated statements leave the result BATs.
+ stmt *s = parquet_emit_plan(be, f, filename);
+
+ // For technical reasons we need to return the stmt as a void pointer.
+ return (void*)s;
+}
+
+// This function is called when the module is loaded.
+// It registers the parquet loader.
static str
-Parquetprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+PARQUETprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void)cntxt;
(void)mb;
(void)stk;
(void)pci;
- fl_register("parquet", &parquet_add_types, &parquet_load);
+ fl_register("parquet", &parquet_add_types, &parquet_generate_plan);
return MAL_SUCCEED;
}
+// This function is called if the module is ever unloaded.
+// Currently, it does nothing.
static str
-Parquetepilogue(void *ret)
+PARQUETepilogue(void *ret)
{
(void)ret;
return MAL_SUCCEED;
}
+// This function is called when the parquet.load_table operator in the MAL
+// plan is executed.
+static str
+PARQUETload(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ (void)cntxt;
+ (void)mb;
+ (void)stk;
+ (void)pci;
+ throw(MAL, "parquet.load_table", SQLSTATE(42000) "Not implemented yet");
+}
+
+
#include "sql_scenario.h"
#include "mel.h"
static mel_func parquet_init_funcs[] = {
- pattern("parquet", "prelude", Parquetprelude, false, "", noargs),
- command("parquet", "epilogue", Parquetepilogue, false, "", noargs),
+ pattern("parquet", "prelude", PARQUETprelude, false, "", noargs),
+ command("parquet", "epilogue", PARQUETepilogue, false, "", noargs),
+ pattern("parquet", "load_table", PARQUETload, true, "load data from the
parquet file", args(1,2,
+ batvarargany("", 0),
+ arg("filename", str),
+ )),
{ .imp=NULL }
};
diff --git a/sql/backends/monet5/vaults/parquet/parquet.h
b/sql/backends/monet5/vaults/parquet/parquet.h
--- a/sql/backends/monet5/vaults/parquet/parquet.h
+++ b/sql/backends/monet5/vaults/parquet/parquet.h
@@ -10,11 +10,4 @@ typedef struct parquet_file {
} parquet_file;
-/* Opens the file, and returns a struct containing the reader. */
-//parquet_file *parquet_open_file(char* filename);
-
-//parquet_table_metadata parquet_get_table_metadata(parquet_file *file);
-
-//void parquet_init(void);
-
#endif
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
