Changeset: a6f24c4f95df for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a6f24c4f95df
Modified Files:
buildtools/autogen/autogen/codegen.py
buildtools/conf/rules.mk
configure.ag
monetdb5/extras/Makefile.ag
monetdb5/extras/jaql/30_jaql.mal
monetdb5/extras/jaql/Makefile.ag
monetdb5/extras/jaql/expand.mal
monetdb5/extras/jaql/filter.mal
monetdb5/extras/jaql/jaql.c
monetdb5/extras/jaql/jaql.h
monetdb5/extras/jaql/jaql.l
monetdb5/extras/jaql/jaql.mal
monetdb5/extras/jaql/jaql.y
monetdb5/extras/jaql/jaqlgencode.c
monetdb5/extras/jaql/jaqlgencode.h
monetdb5/extras/jaql/jaqltest.mal
monetdb5/extras/jaql/json.c
monetdb5/extras/jaql/json.h
monetdb5/extras/jaql/json.mal
monetdb5/extras/jaql/jsontest.mal
monetdb5/extras/jaql/transform.mal
Branch: jacqueline
Log Message:
Initial branch commit: jacqueline
Codename "Jacqueline": MonetDB JSON/JAQL
This first commit adds a rough preliminary implementation of JAQL. It
uses the JSON shredding and serialising functionality also introduced in
this commit. At this stage, code generation takes place for expand and
filter JAQL core functions, which mostly successfully execute on the MAL
engine (returning a JSON result).
diffs (truncated from 2992 to 300 lines):
diff --git a/buildtools/autogen/autogen/codegen.py
b/buildtools/autogen/autogen/codegen.py
--- a/buildtools/autogen/autogen/codegen.py
+++ b/buildtools/autogen/autogen/codegen.py
@@ -77,7 +77,7 @@ end_code_extract = { 'mx': e_mx, 'mx.in'
# direct rules
code_gen = {'y': [ '.tab.c', '.tab.h' ],
'tab.c': [ '.tab.o' ],
- 'l': [ '.yy.c' ],
+ 'l': [ '.yy.c', '.yy.h' ],
'yy.c': [ '.yy.o' ],
'mt': [ '.symbols.h', '.c' ],
'brg': [ '.c' ],
diff --git a/buildtools/conf/rules.mk b/buildtools/conf/rules.mk
--- a/buildtools/conf/rules.mk
+++ b/buildtools/conf/rules.mk
@@ -51,6 +51,14 @@ MX = $(top_builddir)/buildtools/Mx/Mx
echo '#include <'"$(CONFIG_H)"'>' > $*.yy.c
grep -v '^#include.*[<"]'"$(CONFIG_H)"'[">]' $*.yy.c.tmp >> $*.yy.c
$(RM) $*.yy.c.tmp
+ [ -f $(LEX_OUTPUT_ROOT).h ] && $(RM) $(LEX_OUTPUT_ROOT).h
+ $(RM) waiting
+
+%.yy.h: %.l
+ touch waiting.$$$$ && until ln waiting.$$$$ waiting 2>/dev/null; do
sleep 1; done && rm waiting.$$$$
+ $(LEX) $(LFLAGS) $(AM_LFLAGS) $< || { $(RM) waiting ; exit 1 ; }
+ if [ -f $(LEX_OUTPUT_ROOT).h ]; then $(MV) $(LEX_OUTPUT_ROOT).h $*.yy.h
; fi
+ [ -f $(LEX_OUTPUT_ROOT).c ] && $(RM) $(LEX_OUTPUT_ROOT).c
$(RM) waiting
%.def: %.syms
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1389,6 +1389,7 @@ fi
AC_SUBST(RUBYGEM)
AM_CONDITIONAL(HAVE_RUBYGEM, test x"$have_rubygem" != xno -a
x"$have_rubygem_dir" != xno)
+AM_PROG_LEX
AC_PROG_YACC
case "$YACC" in
bison*)
diff --git a/monetdb5/extras/Makefile.ag b/monetdb5/extras/Makefile.ag
--- a/monetdb5/extras/Makefile.ag
+++ b/monetdb5/extras/Makefile.ag
@@ -15,5 +15,5 @@
# Copyright August 2008-2012 MonetDB B.V.
# All Rights Reserved.
-SUBDIRS = compiler ENABLE_CRACKERS?crackers HAVE_RAPTOR?rdf
HAVE_SPHINXCLIENT?sphinx
+SUBDIRS = compiler ENABLE_CRACKERS?crackers HAVE_RAPTOR?rdf
HAVE_SPHINXCLIENT?sphinx jaql
diff --git a/monetdb5/extras/jaql/30_jaql.mal b/monetdb5/extras/jaql/30_jaql.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/30_jaql.mal
@@ -0,0 +1,19 @@
+# The contents of this file are subject to the MonetDB Public License
+# Version 1.1 (the "License"); you may not use this file except in
+# compliance with the License. You may obtain a copy of the License at
+# http://www.monetdb.org/Legal/MonetDBLicense
+#
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+# License for the specific language governing rights and limitations
+# under the License.
+#
+# The Original Code is the MonetDB Database System.
+#
+# The Initial Developer of the Original Code is CWI.
+# Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+# Copyright August 2008-2012 MonetDB B.V.
+# All Rights Reserved.
+
+include json;
+include jaql;
diff --git a/monetdb5/extras/jaql/Makefile.ag b/monetdb5/extras/jaql/Makefile.ag
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Makefile.ag
@@ -0,0 +1,64 @@
+# The contents of this file are subject to the MonetDB Public License
+# Version 1.1 (the "License"); you may not use this file except in
+# compliance with the License. You may obtain a copy of the License at
+# http://www.monetdb.org/Legal/MonetDBLicense
+#
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+# License for the specific language governing rights and limitations
+# under the License.
+#
+# The Original Code is the MonetDB Database System.
+#
+# The Initial Developer of the Original Code is CWI.
+# Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+# Copyright August 2008-2012 MonetDB B.V.
+# All Rights Reserved.
+
+INCLUDES = ../../mal \
+ ../../optimizer \
+ ../../../common/options \
+ ../../../common/stream \
+ ../../../gdk
+
+AM_YFLAGS = -d
+AM_LFLAGS = -o$(LEX_OUTPUT_ROOT).c --header-file=$(LEX_OUTPUT_ROOT).h
+
+# flex' output isn't up to the strictness of our flags
+X_CFLAGS = #$(X_CFLAGS:-Werror=) <- recursive
+
+lib__json = {
+ MODULE
+ DIR = libdir/monetdb5
+ SOURCES = json.c json.h
+
+ LIBS = ../../tools/libmonetdb5
+}
+
+lib__jaql = {
+ MODULE
+ DIR = libdir/monetdb5
+ SOURCES = jaql.y jaql.l jaql.c jaql.h jaqlgencode.c jaqlgencode.h
+
+ LIBS = ../../tools/libmonetdb5
+}
+
+headers_json_mal = {
+ HEADERS = mal
+ DIR = libdir/monetdb5
+ SOURCES = json.mal
+}
+
+headers_jaql_mal = {
+ HEADERS = mal
+ DIR = libdir/monetdb5
+ SOURCES = jaql.mal
+}
+
+headers_autoload = {
+ HEADERS = mal
+ DIR = libdir/monetdb5/autoload
+ SOURCES = 30_jaql.mal
+}
+
+EXTRA_DIST = json.mal jaql.mal 30_jaql.mal
diff --git a/monetdb5/extras/jaql/expand.mal b/monetdb5/extras/jaql/expand.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/expand.mal
@@ -0,0 +1,73 @@
+# [ [3,65,8,72], [5,98,2,65] ] -> expand each arr
+# '[ [3,65,8,72], [5,98,2,65] ]' | as arr -> expand: arr => <result>
+# j_json( [ [3,65,8,72], [5,98,2,65] ] ) j_expand( j_var( arr ) , j_var( arr )
) j_output()
+
+include json;
+out := io.stdout();
+
+# j_json( [ [3,65,8,72], [5,98,2,65] ] )
+(x1,x2,x3,x4,x5,x6,x7) := json.shred("[ [3,65,8,72], [5,98,2,65,[]] ]");
+json.print(out, x1,x2,x3,x4,x5,x6,x7);
+
+# j_var( arr )
+o4 := algebra.selectH(x1, 0@0); # semantic: first is outermost array
+o0 := algebra.semijoin(x5, o4);
+o2 := bat.reverse(o0);
+o1 := algebra.semijoin(x1, o2);
+
+# j_var( arr )
+o3 := algebra.uselect(o1, 'a'); # ids from elems (all arrays match)
+
+# j_expand( o1 , o3 )
+# construct json with these elements in the outermost array
+o6 := algebra.semijoin(x5, o3);
+o7 := algebra.project(0@0, o6); # semantic: first is outermost array
+# remove nested arrays, collapse all of their content to top-level array
+o8 := algebra.sunion(o0, o6);
+o9 := algebra.sdifference(x5, o8);
+o5 := bat.insert(o9, o7);
+
+# j_output()
+out := io.stdout();
+json.print(out, x1,x2,x3,x4,o5,x6,x7);
+
+
+# [ {"name":"Jon Doe", "movie_ids":[3,65,8,72]}, {"name":"Jane Dean",
"movie_ids":[5,98,2]} ] -> expand $.movie_ids;
+# jaql.x("[ {\"name\":\"Jon Doe\", \"movie_ids\":[3,65,8,72]},
{\"name\":\"Jane Dean\",\"movie_ids\":[5,98,2]} ] -> expand $.movie_ids;");
+# [ {"name":"Jon Doe", "movie_ids":[3,65,8,72]}, {"name":"Jane
Dean","movie_ids":[5,98,2]} ] as $ -> expand: $.movie_ids => <result>
+# j_json( [ {"name":"Jon Doe", "movie_ids":[3,65,8,72]}, {"name":"Jane
Dean","movie_ids":[5,98,2]} ] ) j_expand( j_var( $ ) , j_var( $. j_var(
movie_ids ) ) ) j_output()
+
+# j_json( [ {"name":"Jon Doe", "movie_ids":[3,65,8,72]}, {"name":"Jane Dean",
"movie_ids":[5,98,2]} ] )
+(x1,x2,x3,x4,x5,x6,x7) := json.shred("[ {\"name\":\"Jon Doe\",
\"movie_ids\":[3,65,8,72]}, {\"name\":\"Jane Dean\",\"movie_ids\":[5,98,2]} ]
-> expand $.movie_ids;");
+json.print(out, x1,x2,x3,x4,x5,x6,x7);
+
+# j_var( arr )
+o4 := algebra.selectH(x1, 0@0); # semantic: first is outermost array
+o0 := algebra.semijoin(x5, o4);
+o2 := bat.reverse(o0);
+o1 := algebra.semijoin(x1, o2);
+# = identical
+
+# j_var( $. j_var( movie_ids ) )
+# semantic: deref means object
+f0 := algebra.uselect(o1, 'o');
+f1 := algebra.semijoin(x6, f0);
+f2 := bat.reverse(f1);
+f3 := algebra.semijoin(x7, f2);
+f4 := algebra.uselect(f3, "movie_ids");
+f5 := algebra.semijoin(x1, f4);
+o3 := algebra.uselect(f5, 'a'); # only arrays match
+# = produces elems from x1 (identical interface)
+
+# j_expand( o1 , o3 )
+# construct json with these elements in the outermost array
+o6 := algebra.semijoin(x5, o3);
+o7 := algebra.project(0@0, o6); # semantic: first is outermost array
+# remove nested arrays, collapse all of their content to top-level array
+o8 := algebra.sunion(o0, o6);
+o9 := algebra.sdifference(x5, o8);
+o5 := bat.insert(o9, o7);
+# = identical
+
+# j_output()
+json.print(out, x1,x2,x3,x4,o5,x6,x7);
diff --git a/monetdb5/extras/jaql/filter.mal b/monetdb5/extras/jaql/filter.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/filter.mal
@@ -0,0 +1,47 @@
+# [ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ] -> filter
$.dept == 1;
+# [ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ] as $ ->
filter: $.dept == 1 => <result>
+# j_json( [ {id:1, dept:1, income:12000},{id:2, dept:1, income:13000} ] )
j_filter( j_var( $ ) , j_pred( j_var( $. j_var( dept ) ) , == , 1 ) ) j_output()
+
+
+include json;
+
+# j_json( [ {id:1, dept:1, income:12000},{id:2, dept:2, income:13000} ] ) = x0
+(x1,x2,x3,x4,x5,x6,x7) := json.shred("[ {\"id\":1, \"dept\":1,
\"income\":12000},{\"id\":2, \"dept\":2, \"income\":13000} ]");
+
+# j_var( $ ) over x0
+o9 := algebra.selectH(x1, 0@0);
+o0 := algebra.semijoin(x5, o9);
+o2 := bat.reverse(o0);
+o1 := algebra.semijoin(x1, o2);
+
+# j_var( $. j_var( dept ) -- deref must be object
+# (usage of var should be semantic check)
+o4 := algebra.select(o1, 'o');
+v0 := algebra.semijoin(x6, o4);
+v1 := bat.reverse(v0);
+n0 := algebra.semijoin(x7, v1);
+n1 := algebra.uselect(n0, "dept");
+n2 := algebra.semijoin(v1, n1);
+v2 := bat.reverse(n2); # ids from elems that have matching "dept"
+
+# j_pred( n1 , == , 1 )
+# semantic: we request 1 (int), so only look in integers, probably too
+# limited
+i0 := algebra.semijoin(x3, n1);
+i1 := algebra.uselect(i0, 1:lng); # argument
+n3 := algebra.semijoin(v1, i1);
+v3 := bat.reverse(n3); # ids from elems that have matching "dept == 1"
+
+# j_filter( o1 , i1 ) = x9
+# filter operates on object here, so can simply remove all objects that
+# don't match from the array
+o5 := bat.reverse(o0);
+o6 := algebra.kdifference(o5, v3);
+o7 := bat.reverse(o6);
+o8 := algebra.difference(x5, o7);
+# construct result by replacing the original x5 with o8
+x5 := o8;
+
+# j_output()
+out := io.stdout();
+json.print(out, x1,x2,x3,x4,x5,x6,x7);
diff --git a/monetdb5/extras/jaql/jaql.c b/monetdb5/extras/jaql/jaql.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/jaql.c
@@ -0,0 +1,582 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+ */
+
+/*
+ * JAQL is a query language for JavaScript Object Notation or JSON.
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
