Changeset: 6e47fe588566 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6e47fe588566
Modified Files:
monetdb5/extras/jaql/Tests/All
monetdb5/extras/jaql/Tests/plan00.mal
monetdb5/extras/jaql/Tests/plan00.stable.out
monetdb5/extras/jaql/Tests/transform00.mal
monetdb5/extras/jaql/Tests/transform00.stable.err
monetdb5/extras/jaql/Tests/transform00.stable.out
monetdb5/extras/jaql/jaql.c
monetdb5/extras/jaql/jaql.h
monetdb5/extras/jaql/jaqlgencode.c
monetdb5/extras/jaql/parser/jaql.l
monetdb5/extras/jaql/parser/jaql.y
Branch: jacqueline
Log Message:
jaql: implemented first step towards transform
added parser support for arithmetic
added parser support for construction of JSON objects/arrays
dropped old manual parsing method of transform args
implemented printtree of j_transform nodes
implemented simple case of static value as transform expression in gencode
added test
diffs (truncated from 924 to 300 lines):
diff --git a/monetdb5/extras/jaql/Tests/All b/monetdb5/extras/jaql/Tests/All
--- a/monetdb5/extras/jaql/Tests/All
+++ b/monetdb5/extras/jaql/Tests/All
@@ -1,5 +1,6 @@
expand00
filter00
+transform00
sort00
top00
variable00
diff --git a/monetdb5/extras/jaql/Tests/plan00.mal
b/monetdb5/extras/jaql/Tests/plan00.mal
--- a/monetdb5/extras/jaql/Tests/plan00.mal
+++ b/monetdb5/extras/jaql/Tests/plan00.mal
@@ -4,9 +4,16 @@ jaql.x("plan test -> filter each emp emp
jaql.x("plan e -> filter $.mgr or $.income > 30000;");
jaql.x("plan emps = [ {id:1, dept:1, income:12000},{id:2, dept:1,
income:13000} ];");
jaql.x("plan [ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ] ->
filter $.dept == 1;");
-#jaql.x("plan [ 1, 2, 3 ] -> transform {\"value\":$};");
+
jaql.x("plan [ [3,65,8,72], [5,98,2,65] ] -> expand each arr;");
jaql.x("plan [ {\"name\":\"Jon Doe\", \"movie_ids\":[3,65,8,72]},
{\"name\":\"Jane Dean\",\"movie_ids\":[5,98,2]} ] -> expand $.movie_ids;");
-#jaql.x("plan A -> expand -> transform $ * 2;");
+jaql.x("plan A -> expand -> transform $ * 2;");
jaql.x("plan A -> expand $.ids;");
+jaql.x("plan [1,2,3] -> transform {\"value\":$};");
+jaql.x("plan [1,2,3] -> transform $ * 2;");
+jaql.x("plan [1,2,3] -> transform 2 * $;");
+jaql.x("plan [1,2,3] -> transform 2 * 2;");
+jaql.x("plan [1,2,3] -> transform 2 * ($ * 2);");
+jaql.x("plan [1,2,3] -> transform 2 * 2.0 * $;");
+jaql.x("plan [1,2,3] -> transform ($ * 2.0) / 4;");
diff --git a/monetdb5/extras/jaql/Tests/plan00.stable.out
b/monetdb5/extras/jaql/Tests/plan00.stable.out
--- a/monetdb5/extras/jaql/Tests/plan00.stable.out
+++ b/monetdb5/extras/jaql/Tests/plan00.stable.out
@@ -23,11 +23,17 @@ function user.main():void;
jaql.x("plan e -> filter $.mgr or $.income > 30000;");
jaql.x("plan emps = [ {id:1, dept:1, income:12000},{id:2, dept:1,
income:13000} ];");
jaql.x("plan [ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ]
-> filter $.dept == 1;");
-#jaql.x("plan [ 1, 2, 3 ] -> transform {\"value\":$};");
jaql.x("plan [ [3,65,8,72], [5,98,2,65] ] -> expand each arr;");
jaql.x("plan [ {\"name\":\"Jon Doe\", \"movie_ids\":[3,65,8,72]},
{\"name\":\"Jane Dean\",\"movie_ids\":[5,98,2]} ] -> expand $.movie_ids;");
-#jaql.x("plan A -> expand -> transform $ * 2;");
+ jaql.x("plan A -> expand -> transform $ * 2;");
jaql.x("plan A -> expand $.ids;");
+ jaql.x("plan [1,2,3] -> transform {\"value\":$};");
+ jaql.x("plan [1,2,3] -> transform $ * 2;");
+ jaql.x("plan [1,2,3] -> transform 2 * $;");
+ jaql.x("plan [1,2,3] -> transform 2 * 2;");
+ jaql.x("plan [1,2,3] -> transform 2 * ($ * 2);");
+ jaql.x("plan [1,2,3] -> transform 2 * 2.0 * $;");
+ jaql.x("plan [1,2,3] -> transform ($ * 2.0) / 4;");
end main;
B => A
test as $ -> filter: $.mgr == true => <result>
@@ -37,7 +43,15 @@ e as $ -> filter: ( $.mgr == true || $.i
[ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ] as $ -> filter:
$.dept == 1 => <result>
[ [3,65,8,72], [5,98,2,65] ] as arr -> expand: arr => <result>
[ {"name":"Jon Doe", "movie_ids":[3,65,8,72]}, {"name":"Jane
Dean","movie_ids":[5,98,2]} ] as $ -> expand: $.movie_ids => <result>
+A as $ -> expand: $ as $ -> transform: ( $ * 2 )
A as $ -> expand: $.ids => <result>
+[1,2,3] as $ -> transform: { "value": $ }
+[1,2,3] as $ -> transform: ( $ * 2 )
+[1,2,3] as $ -> transform: ( $ * 2 )
+[1,2,3] as $ -> transform: 4
+[1,2,3] as $ -> transform: ( ( $ * 2 ) * 2 )
+[1,2,3] as $ -> transform: ( ( $ * 2.000000 ) * 2 )
+[1,2,3] as $ -> transform: ( ( $ * 2.000000 ) / 4 )
# 21:19:49 >
# 21:19:49 > "Done."
diff --git a/monetdb5/extras/jaql/Tests/transform00.mal
b/monetdb5/extras/jaql/Tests/transform00.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/transform00.mal
@@ -0,0 +1,1 @@
+jaql.x("[1,2,3] -> transform 2 * 2;");
diff --git a/monetdb5/extras/jaql/Tests/transform00.stable.err
b/monetdb5/extras/jaql/Tests/transform00.stable.err
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/transform00.stable.err
@@ -0,0 +1,31 @@
+stderr of test 'transform00` in directory 'extras/jaql` itself:
+
+
+# 21:27:12 >
+# 21:27:12 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm"
"--set" "mapi_open=true" "--set" "mapi_port=35341" "--set" "monet_prompt="
"--trace" "--forcemito" "--set" "mal_listing=2" "--dbname=mTests_extras_jaql"
"transform00.mal"
+# 21:27:12 >
+
+# builtin opt gdk_dbname = demo
+# builtin opt gdk_dbfarm =
/ufs/fabian/scratch/ssd/monetdb/jacqueline/program-x86_64/var/lib/monetdb5/dbfarm
+# builtin opt gdk_debug = 0
+# builtin opt gdk_alloc_map = no
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt gdk_dbfarm =
/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 35341
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbname = mTests_extras_jaql
+
+# 21:27:13 >
+# 21:27:13 > "Done."
+# 21:27:13 >
+
diff --git a/monetdb5/extras/jaql/Tests/transform00.stable.out
b/monetdb5/extras/jaql/Tests/transform00.stable.out
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/transform00.stable.out
@@ -0,0 +1,27 @@
+stdout of test 'transform00` in directory 'extras/jaql` itself:
+
+
+# 21:27:12 >
+# 21:27:12 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm"
"--set" "mapi_open=true" "--set" "mapi_port=35341" "--set" "monet_prompt="
"--trace" "--forcemito" "--set" "mal_listing=2" "--dbname=mTests_extras_jaql"
"transform00.mal"
+# 21:27:12 >
+
+# MonetDB 5 server v11.8.0 "jacqueline-4b6d0a64f745+"
+# Serving database 'mTests_extras_jaql', using 8 threads
+# Compiled for x86_64-pc-linux-gnu/64bit with 64bit OIDs dynamically linked
+# Found 15.662 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2012 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://sofia.ins.cwi.nl:35341/
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+# MonetDB/DataCell module not loaded: MALException:jaql.context:JAQL
environment not found
+function user.main():void;
+ jaql.x("[1,2,3] -> transform 2 * 2;");
+end main;
+[ 4, 4, 4 ]
+
+# 21:27:13 >
+# 21:27:13 > "Done."
+# 21:27:13 >
+
diff --git a/monetdb5/extras/jaql/jaql.c b/monetdb5/extras/jaql/jaql.c
--- a/monetdb5/extras/jaql/jaql.c
+++ b/monetdb5/extras/jaql/jaql.c
@@ -66,6 +66,30 @@ make_json(char *s)
return res;
}
+tree *
+make_json_object(tree *obj)
+{
+ tree *res = GDKzalloc(sizeof(tree));
+
+ assert(obj != NULL && obj->type == j_pair);
+ res->type = j_json_obj;
+ res->tval1 = obj;
+
+ return res;
+}
+
+tree *
+make_json_array(tree *arr)
+{
+ tree *res = GDKzalloc(sizeof(tree));
+
+ assert(arr != NULL); /* arr can be about everything */
+ res->type = j_json_arr;
+ res->tval1 = arr;
+
+ return res;
+}
+
/* append naction as next pipe after oaction */
tree *
append_jaql_pipe(tree *oaction, tree *naction)
@@ -102,6 +126,38 @@ make_jaql_filter(tree *var, tree *pred)
res->type = j_filter;
res->tval1 = var;
res->tval2 = pred;
+ /* FIXME: check pred's var usage, like _check_exp_var */
+
+ return res;
+}
+
+/* recursive helper to check variable usages for validity */
+static tree *
+_check_exp_var(char *var, tree *t)
+{
+ tree *res = NULL;
+
+ if (t == NULL)
+ return res;
+
+ if (t->type == j_var) {
+ if (strcmp(var, t->sval) != 0) {
+ char buf[128];
+ res = GDKzalloc(sizeof(tree));
+ snprintf(buf, sizeof(buf), "transform: unknown
variable: %s",
+ t->sval);
+ res->type = j_error;
+ res->sval = GDKstrdup(buf);
+ }
+ return res;
+ }
+
+ if ((res = _check_exp_var(var, t->tval1)) != NULL)
+ return res;
+ if ((res = _check_exp_var(var, t->tval2)) != NULL)
+ return res;
+ if ((res = _check_exp_var(var, t->tval3)) != NULL)
+ return res;
return res;
}
@@ -111,55 +167,20 @@ make_jaql_filter(tree *var, tree *pred)
tree *
make_jaql_transform(tree *var, tree *tmpl)
{
- tree *res = GDKzalloc(sizeof(tree));
- char buf[8096];
- char *p, *q, *r;
- size_t l;
- tree *t;
+ tree *res;
- /* process tmpl, to turn into a C-like printf string and extract
- * variable references */
- for (p = tmpl->sval, q = buf; *p != '\0' && q - buf < 8096; p++, q++)
- {
- l = strlen(var->sval);
- if (*p == *var->sval && strncmp(p, var->sval, l) == 0 &&
- !(p[l] >= 'A' && p[l] <= 'Z') &&
- !(p[l] >= 'a' && p[l] <= 'z') &&
- !(p[l] >= '0' && p[l] <= '9') && p[l] != '_')
- {
- p += l;
- if (res->tval3 == NULL) {
- t = res->tval3 = make_varname(var->sval);
- } else {
- t = t->next = make_varname(var->sval);
- }
- *q++ = '%';
- *q++ = 's';
+ assert(var != NULL && var->type == j_var);
+ assert(tmpl != NULL);
- while (*p == '.') {
- r = p;
- for (; *p != '\0'; p++) {
- if (*p != '_' &&
- !(p[l] >= 'A' && p[l]
<= 'Z') &&
- !(p[l] >= 'a' && p[l]
<= 'z') &&
- !(p[l] >= '0' && p[l]
<= '9'))
- {
- *q = *p;
- *p = '\0';
- append_varname(t, r);
- *p = *q;
- }
- }
- }
- /* we don't support x[n] yet */
- }
- *q = *p;
+ /* traverse down tmpl, searching for all variable references to
+ * check if they refer to var */
+ if ((res = _check_exp_var(var->sval, tmpl)) != NULL) {
+ freetree(var);
+ freetree(tmpl);
+ return(res);
}
- *q = '\0';
- GDKfree(tmpl->sval);
- tmpl->sval = GDKstrdup(buf);
-
+ res = GDKzalloc(sizeof(tree));
res->type = j_transform;
res->tval1 = var;
res->tval2 = tmpl;
@@ -351,6 +372,67 @@ append_varname(tree *var, char *ident)
return var;
}
+/* constructs a JSON pair '"name": val' where val can be an expression
+ * or a literal value
+ * if name is NULL, val must be a variable, and the pair constructed
+ * will be named after the (last part of the) variable */
+tree *
+make_pair(char *name, tree *val)
+{
+ tree *res = GDKzalloc(sizeof(tree));
+
+ assert(val != NULL);
+ assert(name != NULL || val->type == j_var);
+
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
