Changeset: 6fafc2da9b47 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/6fafc2da9b47
Modified Files:
sql/backends/monet5/vaults/json/json.c
sql/scripts/40_json.sql
Branch: nested
Log Message:
read new line delimited json files
diffs (132 lines):
diff --git a/sql/backends/monet5/vaults/json/json.c
b/sql/backends/monet5/vaults/json/json.c
--- a/sql/backends/monet5/vaults/json/json.c
+++ b/sql/backends/monet5/vaults/json/json.c
@@ -68,6 +68,7 @@ json_open(const char *fname, allocator *
return res;
}
+
static void
json_close(JSONFileHandle *jfh)
{
@@ -93,6 +94,7 @@ read_json_file(JSONFileHandle *jfh)
return content;
}
+
static str
append_terms(allocator *sa, JSON *jt, BAT *b)
{
@@ -191,8 +193,10 @@ json_load(void *BE, sql_subfunc *f, char
return s;
}
+
int TYPE_json;
+
static str
JSONprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
@@ -203,6 +207,7 @@ JSONprelude(Client cntxt, MalBlkPtr mb,
return MAL_SUCCEED;
}
+
static str
JSONepilogue(void *ret)
{
@@ -252,12 +257,79 @@ JSONread_json(Client cntxt, MalBlkPtr mb
return msg;
}
+
+static str
+JSONread_nd_json(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ (void) cntxt; (void) mb;
+ char *msg = MAL_SUCCEED;
+ char *fname = *(str*)getArgReference(stk, pci, pci->retc);
+ allocator *sa = sa_create(NULL);
+ JSONFileHandle *jfh = json_open(fname, sa);
+ if (!jfh) {
+ sa_destroy(sa);
+ msg = createException(SQL, "json.read_nd_json", "Failed to open
file %s", fname);
+ return msg;
+ }
+ char *content = read_json_file(jfh);
+ json_close(jfh);
+ BAT *b = COLnew(0, TYPE_json, 0, TRANSIENT);
+ if (content) {
+ if (b) {
+ size_t cnt = 0;
+ char *head = content;
+ char *tail = content;
+ while (cnt < (jfh->size + 1)) {
+ if (head[0] == '\n' || (head[0] == '\r' &&
head[1] == '\n')) {
+ int skip = 1;
+ if (head[0] == '\r' && head[1] == '\n')
+ skip = 2;
+ head[0] = '\0';
+ JSON *jt = JSONparse(tail);
+ if (jt) {
+ // must be valid json obj str
+ if (BUNappend(b, tail, false)
!= GDK_SUCCEED) {
+ msg =
createException(SQL, "json.read_nd_json", "BUNappend failed!");
+ break;
+ }
+ } else {
+ msg =
createException(SQL, "json.read_nd_json", "Invalid json object, JSONparse
failed!");
+ break;
+ }
+ tail = head + skip;
+ while (tail[0] == '\n') // multiple
newlines e.g. \n\n
+ tail ++;
+ head = tail;
+ }
+ head ++;
+ cnt ++;
+ }
+ } else {
+ msg = createException(SQL, "json.read_nd_json", "Failed
to allocate bat");
+ }
+ } else {
+ msg = createException(SQL, "json.read_nd_json", "Failed to read
file %s", fname);
+ }
+ if (msg == MAL_SUCCEED) {
+ bat *res = getArgReference_bat(stk, pci, 0);
+ *res = b->batCacheid;
+ BBPkeepref(b);
+ } else {
+ BBPreclaim(b);
+ }
+
+ sa_destroy(sa);
+ return msg;
+}
+
+
#include "mel.h"
static mel_func json_init_funcs[] = {
pattern("json", "prelude", JSONprelude, false, "", noargs),
command("json", "epilogue", JSONepilogue, false, "", noargs),
- pattern("json", "read_json", JSONread_json, false, "Reads json file
into a table", args(1,2, batarg("", json), arg("filename", str))),
+ pattern("json", "read_json", JSONread_json, false, "Reads json file",
args(1,2, batarg("", json), arg("filename", str))),
+ pattern("json", "read_nd_json", JSONread_nd_json, false, "Reads new
line delimited json objects", args(1,2, batarg("", json), arg("filename",
str))),
{ .imp=NULL }
};
diff --git a/sql/scripts/40_json.sql b/sql/scripts/40_json.sql
--- a/sql/scripts/40_json.sql
+++ b/sql/scripts/40_json.sql
@@ -104,3 +104,9 @@ create aggregate json.tojsonarray( x str
GRANT EXECUTE ON AGGREGATE json.tojsonarray( string ) TO PUBLIC;
create aggregate json.tojsonarray( x double ) returns string external name
aggr.jsonaggr;
GRANT EXECUTE ON AGGREGATE json.tojsonarray( double ) TO PUBLIC;
+
+
+create function sys.read_nd_json(fname string)
+returns table(json JSON)
+external name json.read_nd_json;
+GRANT EXECUTE ON FUNCTION json.read_nd_json(string) TO PUBLIC;
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
