Changeset: e13fcc82b4d9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e13fcc82b4d9
Modified Files:
monetdb5/extras/jaql/jaqlscenario.c
monetdb5/extras/jaql/jaqltests/Tests/documents.stable.err
monetdb5/extras/jaql/jaqltree.h
monetdb5/extras/jaql/parser/jaql.l
Branch: Oct2012
Log Message:
jaql: implemented multi-line input
The parser now requests (more) data from the stream, like the SQL
scanner does. This triggers the "more>" prompt in mclient when the
parser needs more data to complete the query.
diffs (261 lines):
diff --git a/monetdb5/extras/jaql/jaqlscenario.c
b/monetdb5/extras/jaql/jaqlscenario.c
--- a/monetdb5/extras/jaql/jaqlscenario.c
+++ b/monetdb5/extras/jaql/jaqlscenario.c
@@ -145,14 +145,19 @@ freeVariables(Client c, MalBlkPtr mb, Ma
str
JAQLreader(Client c)
{
- if (MCreadClient(c) > 0)
- return MAL_SUCCEED;
+ /* dummy stub, the scanner reads for us
+ * TODO: pre-fill the buf if we have single line mode */
- c->mode = FINISHING;
- if (c->fdin) {
- c->fdin->buf[c->fdin->pos] = 0;
- } else {
+ if (c->fdin == NULL)
throw(MAL, "jaql.reader", RUNTIME_IO_EOF);
+
+ /* "activate" the stream by sending a prompt (client sync) */
+ if (c->fdin->eof != 0) {
+ if (mnstr_flush(c->fdout) < 0) {
+ c->mode = FINISHING;
+ } else {
+ c->fdin->eof = 0;
+ }
}
return MAL_SUCCEED;
@@ -183,37 +188,36 @@ JAQLparser(Client c)
oldstop = c->curprg->def->stop;
j->vtop = oldvtop;
j->explain = j->plan = j->planf = j->debug = j->trace = j->mapimode = 0;
- j->buf = in->buf + in->pos;
+ j->buf = NULL;
+ j->scanstreamin = in;
+ j->scanstreamout = out;
+ j->scanstreameof = 0;
j->pos = 0;
j->p = NULL;
jaqlparse(j);
-
- /* now the parsing is done we should advance the stream */
- in->pos = in->len;
+
+ /* stop if it seems nothing is going to come any more */
+ if (j->scanstreameof == 1) {
+ c->mode = FINISHING;
+ freetree(j->p);
+ j->p = NULL;
+ return MAL_SUCCEED;
+ }
+
+ /* parsing is done */
+ in->pos = j->pos;
c->yycur = 0;
if (j->err[0] != '\0') {
/* tell the client */
mnstr_printf(out, "!%s\n", j->err);
- /* read away anything left */
- while (j->buf[j->pos + (j->tokstart - j->scanbuf)] != '\0') {
- freetree(j->p);
- jaqlparse(j);
- }
j->err[0] = '\0';
return MAL_SUCCEED;
}
- if (j->p == NULL) { /* there was nothing to parse, EOF */
- /* read away anything left */
- while (j->buf[j->pos + (j->tokstart - j->scanbuf)] != '\0') {
- freetree(j->p);
- jaqlparse(j);
- }
- j->err[0] = '\0';
+ if (j->p == NULL) /* there was nothing to parse, EOF */
return MAL_SUCCEED;
- }
if (!j->plan && !j->planf) {
Symbol prg = c->curprg;
@@ -246,12 +250,6 @@ JAQLparser(Client c)
}
}
- /* read away anything left */
- while (j->buf[j->pos + (j->tokstart - j->scanbuf)] != '\0') {
- freetree(j->p);
- jaqlparse(j);
- }
- j->err[0] = '\0';
return MAL_SUCCEED;
}
diff --git a/monetdb5/extras/jaql/jaqltests/Tests/documents.stable.err
b/monetdb5/extras/jaql/jaqltests/Tests/documents.stable.err
--- a/monetdb5/extras/jaql/jaqltests/Tests/documents.stable.err
+++ b/monetdb5/extras/jaql/jaqltests/Tests/documents.stable.err
@@ -29,11 +29,13 @@ stderr of test 'documents` in directory
# 09:19:01 > "mclient" "-ljaql" "-ftest" "-Eutf-8" "-i" "-e" "--host=Phoebe"
"--port=31818"
# 09:19:01 >
-MAPI = monetdb@Phoebe:33035
-QUERY = load("doctest");
+MAPI = monetdb@Phoebe:34968
+QUERY = # shouldn't exist
+ load("doctest");
ERROR = !no such JSON object with name: doctest
-MAPI = monetdb@Phoebe:33035
-QUERY = load("doctest");
+MAPI = monetdb@Phoebe:34968
+QUERY = # hence, shouldn't exist any more
+ load("doctest");
ERROR = !no such JSON object with name: doctest
# 09:19:01 >
diff --git a/monetdb5/extras/jaql/jaqltree.h b/monetdb5/extras/jaql/jaqltree.h
--- a/monetdb5/extras/jaql/jaqltree.h
+++ b/monetdb5/extras/jaql/jaqltree.h
@@ -40,10 +40,18 @@ typedef struct _jc {
size_t start;
size_t pos;
char *scanbuf;
+ void *scanstreamin;
+ void *scanstreamout;
char *tokstart;
char err[1024];
void *scanner;
- char explain:1, debug:2, trace:3, plan:4, planf:5, mapimode:6;
+ char explain:1,
+ debug:2,
+ trace:3,
+ plan:4,
+ planf:5,
+ mapimode:6,
+ scanstreameof:7;
jvar *vars;
int j1, j2, j3, j4, j5, j6, j7, startoid;
char ro1:1, ro2:2, ro3:3, ro4:4, ro5:5, ro6:6, ro7:7;
diff --git a/monetdb5/extras/jaql/parser/jaql.l
b/monetdb5/extras/jaql/parser/jaql.l
--- a/monetdb5/extras/jaql/parser/jaql.l
+++ b/monetdb5/extras/jaql/parser/jaql.l
@@ -19,8 +19,13 @@
%{
#include <stdio.h>
+#include <stream.h>
#include "jaqltree.h"
#include "jaql.tab.h"
+
+/* prompt: more data needed (stolen from mapi.h to avoid conflicts) */
+#define PROMPT2 "\001\002\n"
+
#ifdef _MSC_VER
#define snprintf _snprintf
#define fileno _fileno
@@ -62,8 +67,8 @@ extern void jaqlerror(jc* j, char const
#define stdout (FILE *)0
static void readinput(jc *j, char *buf, int *res, size_t max_size) {
- if (j->buf[j->pos] != '\0') {
- *res = (int)strlen(j->buf + j->pos);
+ if (j->buf != NULL && j->buf[j->pos] != '\0') {
+ *res = (int)strlen(j->buf + j->pos); /* FIXME: cache this */
if ((size_t)*res > max_size)
*res = (int)max_size;
memcpy(buf, j->buf + j->pos, *res);
@@ -71,6 +76,40 @@ static void readinput(jc *j, char *buf,
j->pos += *res;
j->scanbuf = buf;
} else {
+ if (j->expect_json != ';' && j->scanstreamin != NULL) {
+ bstream *bs = (bstream *)j->scanstreamin;
+
+ /* try and read some more data */
+ if (bstream_next(bs) < 0) {
+ /* read failed, force shutdown next iteration */
+ j->scanstreameof = 1;
+ *res = YY_NULL;
+ return;
+ }
+
+ /* request more if we appear to be at the end of current
+ * block */
+ if (j->buf != NULL && bs->eof == 1) {
+ stream *s = (stream *)j->scanstreamout;
+ if (mnstr_write(s, PROMPT2, sizeof(PROMPT2) -
1, 1) == 1) {
+ mnstr_flush(s);
+ bs->eof = 0;
+ }
+ if (bstream_next(bs) < 0) {
+ /* read failed, force shutdown next
iteration */
+ j->scanstreameof = 1;
+ *res = YY_NULL;
+ return;
+ }
+ }
+
+ /* did we get some query text */
+ if (bs->eof == 0) {
+ j->buf = bs->buf;
+ readinput(j, buf, res, max_size);
+ return;
+ }
+ }
*res = YY_NULL;
}
}
@@ -82,12 +121,14 @@ static void readinput(jc *j, char *buf,
%%
/* if the parser expects a JSON bit, give it */
if (yyextra->expect_json == '[') {
- yyextra->esc_depth = 1;
+ yyextra->esc_depth = 0;
yyextra->expect_json = 0;
+ unput('[');
BEGIN(ARR);
} else if (yyextra->expect_json == '{') {
- yyextra->esc_depth = 1;
+ yyextra->esc_depth = 0;
yyextra->expect_json = 0;
+ unput('{');
BEGIN(OBJ);
}
@@ -150,7 +191,7 @@ null return NIL;
<ARR>{
"]" {
if (--yyextra->esc_depth == 0) {
- yylval->j_json = GDKstrdup(yytext - 1);
+ yylval->j_json = GDKstrdup(yytext);
BEGIN(INITIAL);
return ARRAY;
} else {
@@ -174,7 +215,7 @@ null return NIL;
<OBJ>{
"}" {
if (--yyextra->esc_depth == 0) {
- yylval->j_json = GDKstrdup(yytext - 1);
+ yylval->j_json = GDKstrdup(yytext);
BEGIN(INITIAL);
return OBJECT;
} else {
@@ -195,13 +236,12 @@ null return NIL;
}
"}" return '}';
"{" return '{';
-";" BEGIN(SCOLON);
+";" { yyextra->expect_json = ';'; BEGIN(SCOLON); }
<SCOLON>{
";" /* ignore superfluous semi-colons */;
[ \t\r\n]+ /* ignore whitespace */;
- ("#"|"//").*\n /* ignore comments */;
- <<EOF>> { BEGIN(INITIAL); return ';'; }
- . { unput(yytext[0]); BEGIN(INITIAL); return ';'; }
+ <<EOF>> { yyextra->expect_json = 0; BEGIN(INITIAL); return ';'; }
+ . { unput(yytext[0]); yyextra->expect_json = 0; BEGIN(INITIAL); return
';'; }
}
[ \t\r\n]+ /* ignore whitespace */;
("#"|"//").*\n /* ignore comments */;
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
