Changeset: 299d55b0c92b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=299d55b0c92b
Added Files:
monetdb5/extras/rdf/ontmetadata/loadOntologyListSAMPLE.sql
monetdb5/extras/rdf/ontmetadata/ontList.csv
Modified Files:
monetdb5/extras/rdf/ontmetadata/createMonetdbOntologyTable.sql
monetdb5/extras/rdf/ontmetadata/loadOntologyToMonetDB.sh
monetdb5/extras/rdf/rdf.h
monetdb5/extras/rdf/rdf_shredder.c
monetdb5/extras/rdf/rdflabels.c
monetdb5/extras/rdf/rdflabels.h
monetdb5/extras/rdf/rdfparser.h
monetdb5/extras/rdf/rdfschema.c
sql/backends/monet5/sql.mx
Branch: rdf
Log Message:
Build histogram for the use of ontologies in rdf dataset.
Also, add/modify command for creating ontology list table.
diffs (truncated from 405 to 300 lines):
diff --git a/monetdb5/extras/rdf/ontmetadata/createMonetdbOntologyTable.sql
b/monetdb5/extras/rdf/ontmetadata/createMonetdbOntologyTable.sql
--- a/monetdb5/extras/rdf/ontmetadata/createMonetdbOntologyTable.sql
+++ b/monetdb5/extras/rdf/ontmetadata/createMonetdbOntologyTable.sql
@@ -9,3 +9,7 @@ create table OntAttributes(
mURI string,
mAttr string
);
+
+create table OntList(
+ mOnt string
+);
diff --git a/monetdb5/extras/rdf/ontmetadata/loadOntologyListSAMPLE.sql
b/monetdb5/extras/rdf/ontmetadata/loadOntologyListSAMPLE.sql
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/rdf/ontmetadata/loadOntologyListSAMPLE.sql
@@ -0,0 +1,1 @@
+COPY NUMONT RECORDS INTO OntList FROM 'OntListFile' USING DELIMITERS '|',
'\n';
diff --git a/monetdb5/extras/rdf/ontmetadata/loadOntologyToMonetDB.sh
b/monetdb5/extras/rdf/ontmetadata/loadOntologyToMonetDB.sh
--- a/monetdb5/extras/rdf/ontmetadata/loadOntologyToMonetDB.sh
+++ b/monetdb5/extras/rdf/ontmetadata/loadOntologyToMonetDB.sh
@@ -25,3 +25,13 @@ sed -i "s:AttFile:${PWD}/ontAttribute.gr
mclient < loadtmp.sql
+
+
+NUMONT=`cat ontList.csv | wc -l`
+
+cp loadOntologyListSAMPLE.sql loadtmp.sql
+sed -i "s:NUMONT:$NUMONT:g" loadtmp.sql
+sed -i "s:OntListFile:${PWD}/ontList.csv:g" loadtmp.sql
+
+
+mclient < loadtmp.sql
diff --git a/monetdb5/extras/rdf/ontmetadata/ontList.csv
b/monetdb5/extras/rdf/ontmetadata/ontList.csv
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/rdf/ontmetadata/ontList.csv
@@ -0,0 +1,73 @@
+<http://www.facebook.com/2008
+<http://facebook.com/2008
+<http://developers.facebook.com/schema
+<https://www.facebook.com/2008
+<http://purl.org/dc/elements/1.1
+<http://purl.org/dc/terms
+<http://purl.org/goodrelations/v1
+<http://purl.org/rss/1.0/modules
+<http://purl.org/stuff
+<http://www.purl.org/stuff
+<http://ogp.me/ns
+<https://ogp.me/ns
+<http://www.w3.org/1999/02/22-rdf-syntax-ns
+<http://www.w3.org/2000/01/rdf-schema
+<http://www.w3.org/2004/02/skos/core
+<http://www.w3.org/2002/07/owl
+<http://www.w3.org/2006/vcard/ns
+<http://www.w3.org/2001/vcard-rdf/3.0
+<http://www.w3.org/2003/01/geo/wgs84_pos
+<http://www.w3.org/1999/xhtml/vocab
+<http://search.yahoo.com/searchmonkey
+<https://search.yahoo.com/searchmonkey
+<http://search.yahoo.co.jp/searchmonkey
+<http://g.yahoo.com/searchmonkey
+<http://opengraphprotocol.org/schema
+<https://opengraphprotocol.org/schema
+<http://opengraph.org/schema
+<https://opengraph.org/schema
+<http://creativecommons.org/ns
+<http://rdf.data-vocabulary.org
+<http://rdfs.org/sioc/ns
+<http://xmlns.com/foaf/0.1
+<http://mixi-platform.com/ns
+<http://commontag.org/ns
+<http://semsl.org/ontology
+<http://schema.org
+<http://openelectiondata.org/0.1
+<http://search.aol.com/rdf
+<http://www.loc.gov/loc.terms/relators
+<http://dbpedia.org/ontology
+<http://dbpedia.org/resource
+<http://dbpedia.org/property
+<http://www.aktors.org/ontology/portal
+<http://purl.org/ontology/bibo
+<http://purl.org/ontology/mo
+<http://www.geonames.org/ontology
+<http://purl.org/vocab/frbr/core
+<http://www.w3.org/2001/XMLSchema
+<http://www.w3.org/2006/time
+<http://purl.org/NET/c4dm/event.owl
+<http://www.openarchives.org/ore/terms
+<http://purl.org/vocab/bio/0.1
+<http://www.holygoat.co.uk/owl/redwood/0.1/tags
+<http://rdfs.org/ns/void
+<http://www.w3.org/2006/http
+<http://purl.uniprot.org/core
+<http://umbel.org/umbel
+<http://purl.org/stuff/rev
+<http://purl.org/linked-data/cube
+<http://www.w3.org/ns/org
+<http://purl.org/vocab/vann
+<http://data.ordnancesurvey.co.uk/ontology/admingeo
+<http://www.w3.org/2007/05/powder-s
+<http://usefulinc.com/ns/doap
+<http://lod.taxonconcept.org/ontology/txn.owl
+<http://xmlns.com/wot/0.1
+<http://purl.org/net/compass
+<http://www.w3.org/2004/03/trix/rdfg-1
+<http://purl.org/NET/c4dm/timeline.owl
+<http://purl.org/dc/dcam
+<http://swrc.ontoware.org/ontology
+<http://zeitkunst.org/bibtex/0.1/bibtex.owl
+<http://purl.org/ontology/po
diff --git a/monetdb5/extras/rdf/rdf.h b/monetdb5/extras/rdf/rdf.h
--- a/monetdb5/extras/rdf/rdf.h
+++ b/monetdb5/extras/rdf/rdf.h
@@ -44,7 +44,7 @@
#define _RDF_DEBUG
rdf_export str
-RDFParser(BAT **graph, str *location, str *graphname, str *schemam);
+RDFParser(BAT **graph, str *location, str *graphname, str *schemam, bat
*ontbatid);
rdf_export str
RDFleftfetchjoin_sortedestimate(int *result, int *lid, int *rid, lng
*estimate);
diff --git a/monetdb5/extras/rdf/rdf_shredder.c
b/monetdb5/extras/rdf/rdf_shredder.c
--- a/monetdb5/extras/rdf/rdf_shredder.c
+++ b/monetdb5/extras/rdf/rdf_shredder.c
@@ -235,7 +235,27 @@ getObjectType(unsigned char* objStr, BUN
return obType;
}
-
+#if BUILD_ONTOLOGIES_HISTO
+static
+void printHistogram(parserData *pdata){
+ str value;
+ int total = 0;
+ int i;
+ int numOnt = (int)BATcount(pdata->ontBat);
+ BATiter onti;
+
+ (void) value;
+ //BATprint(pdata->ontBat);
+ onti = bat_iterator(pdata->ontBat);
+ for (i = 0; i < numOnt; i++){
+ value = (str)BUNtail(onti, i);
+ printf("%s: %d \n", value, pdata->numOntInstances[i]);
+ total += pdata->numOntInstances[i];
+ }
+ total += pdata->numNonOnt;
+ printf("Number of non-ontology data: %d / %d (%f) \n",
pdata->numNonOnt, total, (float)pdata->numNonOnt/total);
+}
+#endif
/*
@@ -250,6 +270,12 @@ tripleHandler(void* user_data, const rap
#if CHECK_NUM_DBPONTOLOGY
const char* pos = NULL;
#endif
+#if BUILD_ONTOLOGIES_HISTO
+ char* ontpart;
+ char* ptrEndOnt;
+ int ontlen = 0;
+ BUN bunOnt;
+#endif
parserData *pdata = ((parserData *) user_data);
BUN bun = BUN_NONE;
BUN realNumValue = BUN_NONE;
@@ -291,7 +317,34 @@ tripleHandler(void* user_data, const rap
if ( (pos = strstr((str)predicateStr ,
"http://dbpedia.org/ontology";)) != NULL){
pdata->numOntologyTriples++;
}
+
+ //Store all ontology prefixes in a BAT,
+ //Extract the prefix from each predicateStr by locating
the last # (if available), or /) --> strrchr
+ //Then, look for this prefix in the BAT, must prepare
hash for the BAT
#endif
+ #if BUILD_ONTOLOGIES_HISTO
+ ptrEndOnt = NULL;
+ ptrEndOnt = strrchr((str)predicateStr, '#');
+ if (ptrEndOnt == NULL) ptrEndOnt =
strrchr((str)predicateStr, '/');
+ if (ptrEndOnt == NULL)
+ raptor_exception(pdata, "Could not get the
ontology");
+
+ ontlen = (int) (ptrEndOnt - (str)predicateStr);
+
+ ontpart = substring((char*)predicateStr, 1, ontlen);
+ //printf("Ontpart of %s is %s \n", (str)predicateStr,
(str)ontpart);
+ (void) ontpart;
+
+ //Check whether ontpart appear in the ontBat
+ bunOnt = BUNfnd(BATmirror(pdata->ontBat),(ptr)
(str)ontpart);
+ if (bunOnt == BUN_NONE){
+ pdata->numNonOnt++;
+ }
+ else{
+ pdata->numOntInstances[(int)bunOnt]++;
+ }
+ #endif
+
//rdf_insert(pdata, graph[MAP_LEX], (str) predicateStr,
&bun);
rdf_tknzr_insert((str) predicateStr, &bun);
rdf_BUNappend(pdata, graph[P_sort], &bun);
@@ -370,11 +423,12 @@ create_BAT(int ht, int tt, int size)
}
static parserData*
-parserData_create (str location, BAT** graph)
+parserData_create (str location, BAT** graph, bat *ontbatid)
{
int i;
parserData *pdata = (parserData *) GDKmalloc(sizeof(parserData));
+
if (pdata == NULL) return NULL;
pdata->tcount = 0;
@@ -425,10 +479,45 @@ parserData_create (str location, BAT** g
/* Reset the dense property of graph[MAP_LEX] */
pdata->graph[MAP_LEX]->hdense = FALSE;
+ (void) ontbatid;
+ #if BUILD_ONTOLOGIES_HISTO
+ if (ontbatid == NULL){
+ printf("There is no ontology list loaded \n");
+ return NULL;
+ }
+ else{
+ if ((pdata->ontBat = BATdescriptor(*ontbatid)) == NULL) {
+ return NULL;
+ }
+ (void)BATprepareHash(BATmirror(pdata->ontBat));
+ if (!(pdata->ontBat->T->hash)){
+ return NULL;
+ }
+
+ printf("Ontology list contains "BUNFMT" ontologies \n ",
BATcount(pdata->ontBat));
+
+ pdata->numOntInstances = GDKmalloc(sizeof(int) *
BATcount(pdata->ontBat));
+ for (i = 0; i < (int)BATcount(pdata->ontBat); i++){
+ pdata->numOntInstances[i] = 0;
+ }
+
+ pdata->numNonOnt = 0;
+ }
+ #endif
+
return pdata;
}
+static
+void freeParserData(parserData *pdata){
+
+ #if BUILD_ONTOLOGIES_HISTO
+ BBPunfix(pdata->ontBat->batCacheid);
+ GDKfree(pdata->numOntInstances);
+ #endif
+ GDKfree(pdata);
+}
/*
* @-
@@ -629,14 +718,14 @@ clean(parserData *pdata){
if (pdata->graph[iret] != NULL)
BBPreclaim(pdata->graph[iret]);
}
-
- GDKfree(pdata);
+ freeParserData(pdata);
}
}
+
/* Main RDF parser function that drives raptor */
str
-RDFParser (BAT **graph, str *location, str *graphname, str *schema)
+RDFParser (BAT **graph, str *location, str *graphname, str *schema, bat
*ontbatid)
{
raptor_parser *rparser;
parserData *pdata;
@@ -666,7 +755,7 @@ RDFParser (BAT **graph, str *location, s
#endif
/* Init pdata */
- pdata = parserData_create(*location,graph);
+ pdata = parserData_create(*location,graph, ontbatid);
if (pdata == NULL) {
#ifdef _TKNZR_H
TKNZRclose(&iret);
@@ -801,13 +890,16 @@ RDFParser (BAT **graph, str *location, s
#endif
printf("Total number of error %d , fatal %d , warning %d",
pdata->error, pdata->fatal, pdata->warning);
#endif
+ #if BUILD_ONTOLOGIES_HISTO
+ printHistogram(pdata);
+ #endif
/* post processing step */
ret = post_processing(pdata);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
