Changeset: 779ca8caf99f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=779ca8caf99f
Added Files:
monetdb5/extras/rdf/rdf_shredder.c
Modified Files:
monetdb5/extras/rdf/Makefile.ag
Branch: rdf
Log Message:
Stop using .mx file for rdf_shredder
Change to rdf_shredder.c.
diffs (truncated from 847 to 300 lines):
diff --git a/monetdb5/extras/rdf/Makefile.ag b/monetdb5/extras/rdf/Makefile.ag
--- a/monetdb5/extras/rdf/Makefile.ag
+++ b/monetdb5/extras/rdf/Makefile.ag
@@ -31,7 +31,7 @@ lib__rdf = {
#MODULE
NOINST
#DIR = libdir/monetdb5
- SOURCES = rdf.h rdfschema.h rdf_shredder.mx rdfalgebra.c rdfschema.c
+ SOURCES = rdf.h rdfschema.h rdf_shredder.c rdfalgebra.c rdfschema.c
#SEP = _
# LIBS = ./hashmap/librdfhash
# ../../tools/libmonetdb5 \
diff --git a/monetdb5/extras/rdf/rdf_shredder.c
b/monetdb5/extras/rdf/rdf_shredder.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/rdf/rdf_shredder.c
@@ -0,0 +1,830 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+ */
+
+/*
+ * @a L.Sidirourgos, Minh-Duc Pham
+ *
+ * @+ Shredder for RDF Documents
+ */
+#include "monetdb_config.h"
+#include "mal_exception.h"
+#include "url.h"
+#include "tokenizer.h"
+#include <gdk.h>
+#include <rdf.h>
+#include <raptor2.h>
+
+typedef struct graphBATdef {
+ graphBATType batType; /* BAT type */
+ str name; /* name of the BAT */
+ int headType; /* type of left column */
+ int tailType; /* type of right column */
+} graphBATdef;
+
+static BUN batsz = 10000000;
+
+/* this list should be kept alligned with the graphBATType enum */
+#if STORE == TRIPLE_STORE
+ static graphBATdef graphdef[N_GRAPH_BAT] = {
+ {S_sort, "_s_sort", TYPE_void, TYPE_oid},
+ {P_sort, "_p_sort", TYPE_void, TYPE_oid},
+ {O_sort, "_o_sort", TYPE_void, TYPE_oid},
+
+ {P_PO, "_p_po", TYPE_void, TYPE_oid},
+ {O_PO, "_o_po", TYPE_void, TYPE_oid},
+ {P_OP, "_p_op", TYPE_void, TYPE_oid},
+ {O_OP, "_o_op", TYPE_void, TYPE_oid},
+
+ {S_SO, "_s_so", TYPE_void, TYPE_oid},
+ {O_SO, "_o_so", TYPE_void, TYPE_oid},
+ {S_OS, "_s_os", TYPE_void, TYPE_oid},
+ {O_OS, "_o_os", TYPE_void, TYPE_oid},
+
+ {S_SP, "_s_sp", TYPE_void, TYPE_oid},
+ {P_SP, "_p_sp", TYPE_void, TYPE_oid},
+ {S_PS, "_s_ps", TYPE_void, TYPE_oid},
+ {P_PS, "_p_ps", TYPE_void, TYPE_oid},
+
+ {MAP_LEX, "_map_lex", TYPE_void, TYPE_str}
+ };
+
+#elif STORE == MLA_STORE
+static graphBATdef graphdef[N_GRAPH_BAT] = {
+ {S_sort, "_s_sort", TYPE_void, TYPE_oid},
+ {P_sort, "_p_sort", TYPE_void, TYPE_oid},
+ {O_sort, "_o_sort", TYPE_void, TYPE_oid},
+ {MAP_LEX, "_map_lex", TYPE_void, TYPE_str}
+};
+#endif /* STORE */
+
+typedef struct parserData {
+ /**PROPERTIES */
+ str location; /* rdf data file location */
+ oid tcount; /* triple count */
+ raptor_parser *rparser; /* the parser object */
+ /**ERROR HANDLING */
+ int exception; /* raise an exception */
+ int warning; /* number of warning msgs */
+ int error; /* number of error msgs */
+ int fatal; /* number of fatal msgs */
+ const char *exceptionMsg; /* exception msgs */
+ const char *warningMsg; /* warning msgs */
+ const char *errorMsg; /* error msgs */
+ const char *fatalMsg; /* fatal msgs */
+ int line; /* locator for errors */
+ int column; /* locator for errors */
+ /**GRAPH DATA */
+ BAT **graph; /* BATs for the result
+ shredded RDF graph */
+} parserData;
+
+/*
+ * @-
+ * The (fatal) errors and warnings produced by the raptor parser are handled
+ * by the next three message handler functions.
+ */
+
+static void
+fatalHandler (void *user_data, raptor_log_message* message)
+{
+ parserData *pdata = ((parserData *) user_data);
+ pdata->fatalMsg = GDKstrdup(message->text);
+ mnstr_printf(GDKout, "rdflib: fatal:%s\n", pdata->fatalMsg);
+ pdata->fatal++;
+
+ /* check for a valid locator object and only then use it */
+ if (message->locator != NULL) {
+ pdata->line = message->locator->line;
+ pdata->column = message->locator->column;
+ mnstr_printf(GDKout, "rdflib: fatal: at line %d column %d\n",
pdata->line, pdata->column);
+ }
+
+}
+
+
+static void
+errorHandler (void *user_data, raptor_log_message* message)
+{
+ parserData *pdata = ((parserData *) user_data);
+ pdata->errorMsg = GDKstrdup(message->text);
+ mnstr_printf(GDKout, "rdflib: error:%s\n", pdata->errorMsg);
+ pdata->error++;
+
+ /* check for a valid locator object and only then use it */
+ if (message->locator != NULL) {
+ pdata->line = message->locator->line;
+ pdata->column = message->locator->column;
+ mnstr_printf(GDKout, "rdflib: error: at line %d column %d\n",
pdata->line, pdata->column);
+ }
+
+}
+
+
+static void
+warningHandler (void *user_data, raptor_log_message* message)
+{
+ parserData *pdata = ((parserData *) user_data);
+ pdata->warningMsg = GDKstrdup(message->text);
+ mnstr_printf(GDKout, "rdflib: warning:%s\n", pdata->warningMsg);
+ pdata->warning++;
+
+ /* check for a valid locator object and only then use it */
+ if (message->locator != NULL) {
+ pdata->line = message->locator->line;
+ pdata->column = message->locator->column;
+ mnstr_printf(GDKout, "rdflib: warning: at line %d column %d\n",
pdata->line, pdata->column);
+ }
+
+}
+
+static void
+raptor_exception(parserData *pdata, const char* msg){
+ pdata->exception++;
+ pdata->exceptionMsg = GDKstrdup(msg);
+ raptor_parser_parse_abort (pdata->rparser);
+}
+/*
+static void
+rdf_BUNappend_unq(parserData* pdata, BAT *b, void* value, BUN* bun){
+
+ *bun = BUNfnd(BATmirror(b),(ptr) (str)value);
+ if (*bun == BUN_NONE) {
+ if (BATcount(b) > 4 * b->T->hash->mask) {
+ HASHdestroy(b);
+ BAThash(BATmirror(b), 2*BATcount(b));
+ }
+ *bun = (BUN) b->batCount;
+ b = BUNappend(b, (ptr) (str)value, TRUE);
+ if (b == NULL) {
+ pdata->exception++;
+ pdata->exceptionMsg = "could not append to a BAT";
+ raptor_parser_parse_abort (pdata->rparser);
+ }
+ }
+}
+*/
+
+static void
+rdf_tknzr_insert(void* value, BUN* bun)
+{
+ str t = (str) value;
+ TKNZRappend(bun,&t);
+}
+
+/*
+static void
+rdf_insert(parserData* pdata, BAT *b, void* value, BUN* bun){
+
+ #ifdef _TKNZR_H
+ rdf_tknzr_insert(value, bun);
+ #else
+ rdf_BUNappend_unq(pdata, b, value, bun);
+ #endif
+}
+*/
+
+static void
+rdf_BUNappend(parserData* pdata, BAT *b, BUN* bun){
+ b = BUNappend(b, bun, TRUE);
+ if (b == NULL) {
+ pdata->exception++;
+ pdata->exceptionMsg = "could not append to a BAT with
rdf_BUNappend";
+ raptor_parser_parse_abort (pdata->rparser);
+ }
+
+}
+
+/* For inserting an literal object value of RDF triple */
+
+static void
+rdf_BUNappend_unq_ForObj(parserData* pdata, BAT *b, void* objStr, ObjectType
objType, BUN* bun){
+
+ *bun = BUNfnd(BATmirror(b),(ptr) (str)objStr);
+ if (*bun == BUN_NONE) {
+ if (b->T->hash && BATcount(b) > 4 * b->T->hash->mask) {
+ HASHdestroy(b);
+ BAThash(BATmirror(b), 2*BATcount(b));
+ }
+
+ *bun = (BUN) (RDF_MIN_LITERAL + (b)->batCount);
+
+ /* Add the type here by changing 2 bits at position 62, 63 of
oid */
+ if ( objType == DATETIME){
+ printf("Datetime appears here \n Before: " BUNFMT "\n",
*bun);
+ *bun |= (BUN)1 << (sizeof(BUN)*8 - 3);
+ printf("After: " BUNFMT "\n", *bun);
+ }
+ else if ( objType == NUMERIC){
+ printf("Numeric value appears here \n Before: " BUNFMT
"\n", *bun);
+ *bun |= (BUN)2 << (sizeof(BUN)*8 - 3);
+ printf("After: " BUNFMT "\n", *bun);
+ }
+ else { /* objType == STRING */
+ printf("String value appears here \n Before: " BUNFMT
"\n", *bun);
+ *bun |= (BUN)3 << (sizeof(BUN)*8 - 3);
+ printf("After: " BUNFMT "\n", *bun);
+ }
+
+ //b = BUNappend(b, (ptr) (str)objStr, TRUE);
+ b = BUNins(b, (ptr) bun, (ptr) (str)objStr, TRUE);
+
+ if (b == NULL) {
+
+ pdata->exception++;
+ pdata->exceptionMsg = "could not append in Object bat";
+ raptor_parser_parse_abort (pdata->rparser);
+ }
+ } else {
+ *bun = (b)->hseqbase + *bun;
+ }
+
+}
+
+
+/*
+* Get the specific type of the object value in an RDF triple
+* The URI object can be recoginized by raptor parser.
+* If the object value is not an URI ==> it is a literal, and
+* specifically, a numeric, a dateTime or a string.
+* This function will find the specific type of Object value
+*/
+
+static ObjectType
+getObjectType(unsigned char* objStr){
+ ObjectType obType;
+ if (strstr((const char*) objStr, "XMLSchema#date") != NULL){
+ obType = DATETIME;
+ printf("%s: DateTime \n", objStr);
+ }
+ else if (strstr((const char*) objStr, "XMLSchema#float") != NULL
+ || strstr((const char*) objStr, "XMLSchema#integer") != NULL
+ )
+ {
+ obType = NUMERIC;
+ printf("%s: Numeric \n", objStr);
+ }
+ else {
+ obType = STRING;
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
