Changeset: b2a0e04a707f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/b2a0e04a707f
Modified Files:
sql/backends/monet5/vaults/parquet/parquet.c
sql/server/rel_dump.c
sql/server/rel_file_loader.h
sql/server/rel_select.c
sql/server/rel_select.h
Branch: parquet
Log Message:
small steps
op_table needs
a table functions expression (->r)
and a output expression list in ->exps
These are now handled both by fl_add_types (needs cleanup)
(refactor of table returning functions needed as well)
diffs (274 lines):
diff --git a/sql/backends/monet5/vaults/parquet/parquet.c
b/sql/backends/monet5/vaults/parquet/parquet.c
--- a/sql/backends/monet5/vaults/parquet/parquet.c
+++ b/sql/backends/monet5/vaults/parquet/parquet.c
@@ -5,6 +5,7 @@
#include "monetdb_config.h"
#include "rel_file_loader.h"
+#include "rel_exp.h"
#include "parquet.h"
#include "mal_instruction.h"
@@ -75,7 +76,7 @@ static char* parquet_type_map(GArrowType
return "DOUBLE";
case GARROW_TYPE_STRING:
- return "STRING";
+ return "varchar";
case GARROW_TYPE_BINARY:
case GARROW_TYPE_FIXED_SIZE_BINARY:
@@ -126,51 +127,56 @@ static char* parquet_type_map(GArrowType
}
static str
-parquet_add_types(mvc *sql, sql_subfunc *f, char *filename)
+parquet_add_types(mvc *sql, sql_subfunc *f, char *filename, list *res_exps,
char *tname)
{
- parquet_file *file = parquet_open_file(filename);
+ parquet_file *file = parquet_open_file(filename);
if(file->reader == NULL) {
- throw(SQL, SQLSTATE(42000), "parquet" RUNTIME_FILE_NOT_FOUND);
+ throw(SQL, SQLSTATE(42000), "parquet" RUNTIME_FILE_NOT_FOUND);
+ }
+
+ GError *table_error = NULL;
+ GArrowTable *table =
gparquet_arrow_file_reader_read_table(file->reader, &table_error);
+
+ if(table_error) {
+ throw(SQL, SQLSTATE(42000), "parquet" RUNTIME_LOAD_ERROR); //
TODO: different error.
}
- GError *table_error = NULL;
- GArrowTable *table = gparquet_arrow_file_reader_read_table(file->reader,
&table_error);
+ guint n_columns = garrow_table_get_n_columns(table);
- if(table_error) {
- throw(SQL, SQLSTATE(42000), "parquet" RUNTIME_LOAD_ERROR); // TODO:
different error.
- }
+ list *types = sa_list(sql->sa);
+ list *col_names = sa_list(sql->sa);
+
+ if (!tname)
+ tname = "parquet";
- guint n_columns = garrow_table_get_n_columns(table);
-
- list *types = sa_list(sql->sa);
- list *col_names = sa_list(sql->sa);
+ for(int col = 0; col < (int)n_columns; col++) {
+ GArrowChunkedArray *array = garrow_table_get_column_data(table,
col);
+ GArrowType type = garrow_chunked_array_get_value_type(array);
+ char *name = "column name";
+ char* st = parquet_type_map(type);
- for(int col = 0; col < (int)n_columns; col++) {
- GArrowChunkedArray *array = garrow_table_get_column_data(table, col);
- GArrowType type = garrow_chunked_array_get_value_type(array);
- char* st = parquet_type_map(type);
+ printf("%s\n", st);
- printf("%s\n", st);
-
- if(st) {
- sql_subtype *t = sql_bind_subtype(sql->sa, st, 8, 0);
+ if(st) {
+ sql_subtype *t = sql_bind_subtype(sql->sa, st, 0, 0);
- sa_list_append(sql->sa, types, t);
- }
- else {
- throw(SQL, SQLSTATE(42000), "parquet" RUNTIME_LOAD_ERROR); // TODO:
this should throw a 'unsupported column type' error.
- }
- }
+ list_append(types, t);
+ list_append(res_exps, exp_column(sql->sa, tname, name,
t, CARD_MULTI, 1, 0, 0));
+ }
+ else {
+ throw(SQL, SQLSTATE(42000), "parquet"
RUNTIME_LOAD_ERROR); // TODO: this should throw a 'unsupported column type'
error.
+ }
+ }
- (void)table;
+ (void)table;
/* cleanup tbl */
- f->res = types;
- f->colnames = col_names;
+ f->res = types;
+ f->colnames = col_names;
/* close file */
GDKfree(file);
- return "";
+ return MAL_SUCCEED;
}
static int
diff --git a/sql/server/rel_dump.c b/sql/server/rel_dump.c
--- a/sql/server/rel_dump.c
+++ b/sql/server/rel_dump.c
@@ -1930,7 +1930,7 @@ rel_read(mvc *sql, char *r, int *pos, li
if (!mvc_bind_schema(sql, sname))
return sql_error(sql, ERR_NOTFOUND,
SQLSTATE(3F000) "No such schema '%s'\n", sname);
- if (!(tudf = find_table_function(sql, sname,
tname, list_empty(inputs) ? NULL : inputs, list_empty(inputs) ? NULL :
exp_types(sql->sa, inputs), F_UNION)))
+ if (!(tudf = find_table_function(sql, sname,
tname, list_empty(inputs) ? NULL : inputs, list_empty(inputs) ? NULL :
exp_types(sql->sa, inputs), F_UNION, NULL, NULL)))
return NULL;
sf = tudf->f;
if (tudf->type != e_func || sf->func->type !=
F_UNION)
diff --git a/sql/server/rel_file_loader.h b/sql/server/rel_file_loader.h
--- a/sql/server/rel_file_loader.h
+++ b/sql/server/rel_file_loader.h
@@ -13,7 +13,7 @@
#include "sql_mvc.h"
/* TODO think of set of file names */
-typedef str (*fl_add_types_fptr)(mvc *sql, sql_subfunc *f, char *filename);
+typedef str (*fl_add_types_fptr)(mvc *sql, sql_subfunc *f, char *filename,
list *res_exps, char *name);
typedef int (*fl_load_fptr)(mvc *sql, sql_subfunc *f, char *filename);
typedef struct file_loader_t {
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -532,7 +532,7 @@ nary_function_arg_types_2str(mvc *sql, l
*/
static char *
-file_loader_add_table_column_types(mvc *sql, sql_subfunc *f, sql_exp *e)
+file_loader_add_table_column_types(mvc *sql, sql_subfunc *f, sql_exp *e, list
*res_exps, char *tname)
{
if (!exp_is_atom(e))
return "Filename missing";
@@ -548,7 +548,7 @@ file_loader_add_table_column_types(mvc *
file_loader_t *fl = fl_find(ext);
/* TODO add errors on missing file loader */
if (fl) {
- str err = fl->add_types(sql, f, filename); /* TODO check for
errors */
+ str err = fl->add_types(sql, f, filename, res_exps, tname); /*
TODO check for errors */
if (err)
return err;
}
@@ -556,7 +556,7 @@ file_loader_add_table_column_types(mvc *
}
sql_exp *
-find_table_function(mvc *sql, char *sname, char *fname, list *exps, list *tl,
sql_ftype type)
+find_table_function(mvc *sql, char *sname, char *fname, list *exps, list *tl,
sql_ftype type, sql_rel **rel, char *tname)
{
bool found = false;
list *ff = NULL;
@@ -567,13 +567,17 @@ find_table_function(mvc *sql, char *snam
if ((f = bind_func_(sql, sname, fname, tl, type, false, &found))) {
list *nexps = exps;
if (list_empty(tl) || f->func->vararg || (nexps =
check_arguments_and_find_largest_any_type(sql, NULL, exps, f, 1))) {
+ list *res_exps = sa_list(sql->sa);
if (list_length(exps) == 1 && f && f->func->varres &&
strlen(f->func->mod) == 0 && strlen(f->func->imp) == 0 && strcmp(fname,
"file_loader") == 0) {
sql_exp *file = exps->h->data;
- char *err =
file_loader_add_table_column_types(sql, f, file);
+ char *err =
file_loader_add_table_column_types(sql, f, file, res_exps, tname);
if (err)
return sql_error(sql, ERR_NOTFOUND,
SQLSTATE(42000) "SELECT: file_loader function type resolutions failed '%s'",
err);
}
- return exp_op(sql->sa, nexps, f);
+ sql_exp *e = exp_op(sql->sa, nexps, f);
+ *rel = rel_table_func(sql->sa, NULL, e, res_exps,
TABLE_PROD_FUNC);
+ *rel = rel_project(sql->sa, *rel, res_exps);
+ return e;
}
found = false;
}
@@ -698,39 +702,43 @@ rel_named_table_function(sql_query *quer
}
}
- if (!(e = find_table_function(sql, sname, fname, list_empty(exps) ?
NULL : exps, tl, F_UNION)))
- return NULL;
- rel = sq;
-
+ rel = NULL;
if (ast->data.lval->t->type == type_symbol &&
ast->data.lval->t->data.sym)
tname = ast->data.lval->t->data.sym->data.lval->h->data.sval;
else
tname = make_label(sql->sa, ++sql->label);
- /* column or table function */
- sf = e->f;
- if (e->type != e_func || sf->func->type != F_UNION)
- return sql_error(sql, 02, SQLSTATE(42000) "SELECT: '%s' does
not return a table", exp_func_name(e));
-
- if (sq) {
- for (node *n = sq->exps->h, *m = sf->func->ops->h ; n && m ; n
= n->next, m = m->next) {
- sql_exp *e = (sql_exp*) n->data;
- sql_arg *a = (sql_arg*) m->data;
- if (!exp_subtype(e) && rel_set_type_param(sql,
&(a->type), sq, e, 0) < 0)
- return NULL;
- }
- }
-
- /* for each column add table.column name */
- exps = new_exp_list(sql->sa);
- for (m = sf->func->res->h; m; m = m->next) {
- sql_arg *a = m->data;
- sql_exp *e = exp_column(sql->sa, tname, a->name, &a->type,
CARD_MULTI, 1, 0, 0);
-
- set_basecol(e);
- append(exps, e);
- }
- rel = rel_table_func(sql->sa, rel, e, exps,
(sq)?TABLE_FROM_RELATION:TABLE_PROD_FUNC);
+ if (!(e = find_table_function(sql, sname, fname, list_empty(exps) ?
NULL : exps, tl, F_UNION, &rel, tname)))
+ return NULL;
+
+ if (!rel) {
+ rel = sq;
+
+ /* column or table function */
+ sf = e->f;
+ if (e->type != e_func || sf->func->type != F_UNION)
+ return sql_error(sql, 02, SQLSTATE(42000) "SELECT: '%s'
does not return a table", exp_func_name(e));
+
+ if (sq) {
+ for (node *n = sq->exps->h, *m = sf->func->ops->h ; n
&& m ; n = n->next, m = m->next) {
+ sql_exp *e = (sql_exp*) n->data;
+ sql_arg *a = (sql_arg*) m->data;
+ if (!exp_subtype(e) && rel_set_type_param(sql,
&(a->type), sq, e, 0) < 0)
+ return NULL;
+ }
+ }
+
+ /* for each column add table.column name */
+ exps = new_exp_list(sql->sa);
+ for (m = sf->func->res->h; m; m = m->next) {
+ sql_arg *a = m->data;
+ sql_exp *e = exp_column(sql->sa, tname, a->name,
&a->type, CARD_MULTI, 1, 0, 0);
+
+ set_basecol(e);
+ append(exps, e);
+ }
+ rel = rel_table_func(sql->sa, rel, e, exps,
(sq)?TABLE_FROM_RELATION:TABLE_PROD_FUNC);
+ }
if (ast->data.lval->t->type == type_symbol &&
ast->data.lval->t->data.sym &&
ast->data.lval->t->data.sym->data.lval->h->next->data.lval) {
rel = rel_table_optname(sql, rel, ast->data.lval->t->data.sym,
refs);
} else if (refs) { /* if this relation is under a FROM clause, check
for duplicate names */
@@ -6322,7 +6330,7 @@ rel_loader_function(sql_query *query, sy
}
sql_exp *e = NULL;
- if (!(e = find_table_function(sql, sname, fname, exps, tl, F_LOADER)))
+ if (!(e = find_table_function(sql, sname, fname, exps, tl, F_LOADER,
NULL, NULL)))
return NULL;
sql_subfunc *sf = e->f;
if (sq) {
diff --git a/sql/server/rel_select.h b/sql/server/rel_select.h
--- a/sql/server/rel_select.h
+++ b/sql/server/rel_select.h
@@ -30,7 +30,7 @@ extern sql_exp *rel_binop_(mvc *sql, sql
extern sql_exp *rel_nop_(mvc *sql, sql_rel *rel, sql_exp *l, sql_exp *r,
sql_exp *r2, sql_exp *r3, char *sname, char *fname, int card);
extern sql_rel *rel_with_query(sql_query *query, symbol *q);
extern sql_rel *table_ref(sql_query *query, symbol *tableref, int lateral,
list *refs);
-extern sql_exp *find_table_function(mvc *sql, char *sname, char *fname, list
*exps, list *tl, sql_ftype type);
+extern sql_exp *find_table_function(mvc *sql, char *sname, char *fname, list
*exps, list *tl, sql_ftype type, sql_rel **rel, char *tname);
extern sql_rel *rel_loader_function(sql_query* query, symbol* s, list *fexps,
sql_subfunc **loader_function);
extern list *check_arguments_and_find_largest_any_type(mvc *sql, sql_rel *rel,
list *exps, sql_subfunc *sf, int maybe_zero_or_one);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
