Changeset: ffc5b6715cb6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ffc5b6715cb6
Modified Files:
monetdb5/extras/rdf/rdf_shredder.c
monetdb5/extras/rdf/rdfschema.c
monetdb5/extras/rdf/rdfschema.h
Branch: rdf
Log Message:
Re-design the relational tables for storing rdf triples
diffs (truncated from 583 to 300 lines):
diff --git a/monetdb5/extras/rdf/rdf_shredder.c
b/monetdb5/extras/rdf/rdf_shredder.c
--- a/monetdb5/extras/rdf/rdf_shredder.c
+++ b/monetdb5/extras/rdf/rdf_shredder.c
@@ -206,9 +206,10 @@ char *substring(char *string, int positi
}
static
-char isInt(char *input){
+char isInt(char *input, int len){
- int i, len = strlen(input);
+ int i;
+ //int len = strlen(input);
//printf("... Checking value %s with len %d \n", input, len);
for(i = 0; i < len; i++)
{
@@ -224,6 +225,26 @@ char isInt(char *input){
return 0;
}
+/*
+static
+char isIntWithQuote(char *input, int len){ // "123123" ==> INT
+
+ int i;
+ if (len < 3) return 0;
+ for(i = 1; i < len-1; i++)
+ {
+ if(isdigit(input[i]) == 0){ // May also check
ispunct(string[i]) != 0
+ //printf("NOT A DIGIT \n");
+ break;
+ }
+ }
+ //printf("i is %d \n",i);
+ if(i == len)
+ return 1;
+ else
+ return 0;
+}
+*/
/*
* Get the specific type of the object value in an RDF triple
@@ -239,11 +260,14 @@ getObjectType(unsigned char* objStr, BUN
unsigned char* endpart;
char* valuepart;
const char* pos = NULL;
+ int len = 0;
+ int subLen = 0;
*realNumValue = BUN_NONE;
+ len = strlen((str)objStr);
- if (strlen((str)objStr) > 20){
- endpart = objStr + (strlen((str)objStr) - 19); /*
XMLSchema#dateTime> */
+ if (len > 20){
+ endpart = objStr + (len - 19); /* XMLSchema#dateTime> */
/* printf("Original: %s --> substring: %s \n", (str)objStr,
(str)endpart); */
if ( (pos = strstr((str)endpart , "XMLSchema#date>")) != NULL
|| (pos = strstr((str)endpart, "XMLSchema#dateTime>")) != NULL ){
@@ -252,9 +276,10 @@ getObjectType(unsigned char* objStr, BUN
}
else if ((pos = strstr((str) endpart, "XMLSchema#int>")) !=
NULL || (pos = strstr((str)endpart, "XMLSchema#integer>")) != NULL){
obType = INTEGER;
- valuepart = substring((char*)objStr, 2 , (int) (pos -
(str)objStr - 28));
+ subLen = (int) (pos - (str)objStr - 28);
+ valuepart = substring((char*)objStr, 2 , subLen);
/* printf("%s: Integer \n. Length of value %d ==> value
%s \n", objStr, (int) (pos - (str)objStr - 28), valuepart); */
- if (isInt(valuepart) == 1){ /* Check whether the
real value is an integer */
+ if (isInt(valuepart, subLen) == 1){ /* Check
whether the real value is an integer */
*realNumValue = (BUN) atoi(valuepart);
/* printf("Real value is: " BUNFMT " \n",
*realNumValue); */
}
@@ -275,7 +300,7 @@ getObjectType(unsigned char* objStr, BUN
/* printf("%s: String \n", objStr); */
}
}
- else
+ else // There is no XMLschema
obType = STRING;
return obType;
diff --git a/monetdb5/extras/rdf/rdfschema.c b/monetdb5/extras/rdf/rdfschema.c
--- a/monetdb5/extras/rdf/rdfschema.c
+++ b/monetdb5/extras/rdf/rdfschema.c
@@ -517,13 +517,16 @@ CSPropTypes* initCSPropTypes(CSset* freq
csPropTypes[id].lstPropTypes[j].propCover = 0;
csPropTypes[id].lstPropTypes[j].numType =
MULTIVALUES + 1;
csPropTypes[id].lstPropTypes[j].defaultType =
STRING;
+ csPropTypes[id].lstPropTypes[j].isMVProp = 0;
csPropTypes[id].lstPropTypes[j].lstTypes =
(char*)GDKmalloc(sizeof(char) * csPropTypes[id].lstPropTypes[j].numType);
csPropTypes[id].lstPropTypes[j].lstFreq =
(int*)GDKmalloc(sizeof(int) * csPropTypes[id].lstPropTypes[j].numType);
+ csPropTypes[id].lstPropTypes[j].lstFreqWithMV =
(int*)GDKmalloc(sizeof(int) * csPropTypes[id].lstPropTypes[j].numType);
csPropTypes[id].lstPropTypes[j].colIdxes =
(int*)GDKmalloc(sizeof(int) * csPropTypes[id].lstPropTypes[j].numType);
csPropTypes[id].lstPropTypes[j].TableTypes =
(char*)GDKmalloc(sizeof(char) * csPropTypes[id].lstPropTypes[j].numType);
for (k = 0; k <
csPropTypes[id].lstPropTypes[j].numType; k++){
csPropTypes[id].lstPropTypes[j].lstFreq[k] = 0;
+
csPropTypes[id].lstPropTypes[j].lstFreqWithMV[k] = 0;
csPropTypes[id].lstPropTypes[j].TableTypes[k] = 0;
csPropTypes[id].lstPropTypes[j].colIdxes[k] = -1;
}
@@ -540,6 +543,18 @@ CSPropTypes* initCSPropTypes(CSset* freq
}
static
+char isMultiValueCol(PropTypes pt){
+ double tmpRatio;
+
+ tmpRatio = (double) (pt.propCover / (pt.numSingleType + pt.numMVType));
+
+ if ((pt.numMVType > 0) && (tmpRatio > IS_MULVALUE_THRESHOLD)){
+ return 1;
+ }
+ else return 0;
+}
+
+static
void genCSPropTypesColIdx(CSPropTypes* csPropTypes, int numMergedCS, CSset*
freqCSset){
int i, j, k;
int tmpMaxFreq;
@@ -551,31 +566,63 @@ void genCSPropTypesColIdx(CSPropTypes* c
for (i = 0; i < numMergedCS; i++){
curTypeColIdx = 0;
for(j = 0; j < csPropTypes[i].numProp; j++){
- tmpMaxFreq = csPropTypes[i].lstPropTypes[j].lstFreq[0];
- defaultIdx = 0;
- for (k = 0; k < csPropTypes[i].lstPropTypes[j].numType;
k++){
- if (csPropTypes[i].lstPropTypes[j].lstFreq[k] >
tmpMaxFreq){
- tmpMaxFreq =
csPropTypes[i].lstPropTypes[j].lstFreq[k];
- defaultIdx = k;
+ if (isMultiValueCol(csPropTypes[i].lstPropTypes[j])){
+ //if this property is a Multi-valued prop
+
csPropTypes[i].lstPropTypes[j].TableTypes[MULTIVALUES] = MAINTBL;
+
csPropTypes[i].lstPropTypes[j].colIdxes[MULTIVALUES] = j;
+ csPropTypes[i].lstPropTypes[j].isMVProp = 1;
+
+ //Find the default type for this MV col
+ tmpMaxFreq =
csPropTypes[i].lstPropTypes[j].lstFreqWithMV[0];
+ defaultIdx = 0;
+ //find the default type of the multi-valued prop
+ for (k = 0; k < MULTIVALUES; k++){
+
csPropTypes[i].lstPropTypes[j].TableTypes[k] = MVTBL;
+ if
(csPropTypes[i].lstPropTypes[j].lstFreqWithMV[k] > tmpMaxFreq){
+ tmpMaxFreq =
csPropTypes[i].lstPropTypes[j].lstFreqWithMV[k];
+ defaultIdx = k;
+ }
+
}
- if (csPropTypes[i].lstPropTypes[j].lstFreq[k] <
csPropTypes[i].lstPropTypes[j].propFreq * 0.1){
- //non-frequent type goes to PSO
-
csPropTypes[i].lstPropTypes[j].TableTypes[k] = PSOTBL;
+ /* One type is set to be the default type (in
the mv table) */
+ csPropTypes[i].lstPropTypes[j].defaultType =
defaultIdx;
+
+ /* Count the number of column for MV table
needed */
+
+ }
+ else{
+ csPropTypes[i].lstPropTypes[j].isMVProp = 0;
+
+
+ tmpMaxFreq =
csPropTypes[i].lstPropTypes[j].lstFreq[0];
+ defaultIdx = 0;
+ for (k = 0; k < MULTIVALUES; k++){
+ if
(csPropTypes[i].lstPropTypes[j].lstFreq[k] > tmpMaxFreq){
+ tmpMaxFreq =
csPropTypes[i].lstPropTypes[j].lstFreq[k];
+ defaultIdx = k;
+ }
+ if
(csPropTypes[i].lstPropTypes[j].lstFreq[k] <
csPropTypes[i].lstPropTypes[j].propFreq * INFREQ_TYPE_THRESHOLD){
+ //non-frequent type goes to PSO
+
csPropTypes[i].lstPropTypes[j].TableTypes[k] = PSOTBL;
+ }
+ else
+
csPropTypes[i].lstPropTypes[j].TableTypes[k] =TYPETBL;
}
- else
-
csPropTypes[i].lstPropTypes[j].TableTypes[k] =TYPETBL;
- }
- /* One type is set to be the default type (in the main
table) */
- csPropTypes[i].lstPropTypes[j].TableTypes[defaultIdx] =
MAINTBL;
- csPropTypes[i].lstPropTypes[j].colIdxes[defaultIdx] = j;
- csPropTypes[i].lstPropTypes[j].defaultType =
defaultIdx;
-
- /* Count the number of column needed */
- for (k = 0; k < csPropTypes[i].lstPropTypes[j].numType;
k++){
- if
(csPropTypes[i].lstPropTypes[j].TableTypes[k] == TYPETBL){
-
csPropTypes[i].lstPropTypes[j].colIdxes[k] = curTypeColIdx;
- curTypeColIdx++;
- }
+ /* One type is set to be the default type (in
the main table) */
+
csPropTypes[i].lstPropTypes[j].TableTypes[defaultIdx] = MAINTBL;
+
csPropTypes[i].lstPropTypes[j].colIdxes[defaultIdx] = j;
+ csPropTypes[i].lstPropTypes[j].defaultType =
defaultIdx;
+
+ //Multi-valued prop go to PSO
+
csPropTypes[i].lstPropTypes[j].TableTypes[MULTIVALUES] = PSOTBL;
+
+ /* Count the number of column needed */
+ for (k = 0; k <
csPropTypes[i].lstPropTypes[j].numType; k++){
+ if
(csPropTypes[i].lstPropTypes[j].TableTypes[k] == TYPETBL){
+
csPropTypes[i].lstPropTypes[j].colIdxes[k] = curTypeColIdx;
+ curTypeColIdx++;
+ }
+ }
}
}
csPropTypes[i].numNonDefTypes = curTypeColIdx;
@@ -598,6 +645,9 @@ void updatePropSupport(CSPropTypes* csPr
}
}
}
+
+
+
#endif /* #if COLORINGPROP */
static
void printCSPropTypes(CSPropTypes* csPropTypes, int numMergedCS, CSset*
freqCSset, int freqThreshold){
@@ -612,8 +662,6 @@ void printCSPropTypes(CSPropTypes* csPro
int numNonMVCS = 0;
char tmpIsMVCS = 0;
char tmpIsMVCSFilter = 0;
- double threshold = 1.1;
- double tmpRatio;
strcpy(filename, "csPropTypes");
sprintf(tmpStr, "%d", freqThreshold);
@@ -632,9 +680,8 @@ void printCSPropTypes(CSPropTypes* csPro
tmpIsMVCS = 1;
numMVCols++;
}
- tmpRatio = (double)
(csPropTypes[i].lstPropTypes[j].propCover /
(csPropTypes[i].lstPropTypes[j].numSingleType +
csPropTypes[i].lstPropTypes[j].numMVType));
-
- if ((csPropTypes[i].lstPropTypes[j].numMVType > 0) &&
(tmpRatio > threshold)){
+
+ if (isMultiValueCol(csPropTypes[i].lstPropTypes[j])){
tmpIsMVCSFilter = 1;
numMVColsFilter++;
}
@@ -645,7 +692,7 @@ void printCSPropTypes(CSPropTypes* csPro
csPropTypes[i].lstPropTypes[j].numNull,
csPropTypes[i].lstPropTypes[j].numSingleType,
csPropTypes[i].lstPropTypes[j].numMVType);
fprintf(fout, " ");
for (k = 0; k < csPropTypes[i].lstPropTypes[j].numType;
k++){
- fprintf(fout, " Type %d (%d) | ", k,
csPropTypes[i].lstPropTypes[j].lstFreq[k]);
+ fprintf(fout, " Type %d (%d)(+MV: %d) | ", k,
csPropTypes[i].lstPropTypes[j].lstFreq[k],csPropTypes[i].lstPropTypes[j].lstFreqWithMV[k]);
}
fprintf(fout, "\n");
fprintf(fout, " ");
@@ -684,8 +731,8 @@ void printCSPropTypes(CSPropTypes* csPro
* csPropTypes[tbIdx] contains properties {1,3,4,5,7} with types for each
property and frequency of each <property, type>
* */
static
-void addPropTypes(char *buffTypes, oid* buffP, int numP, int* buffCover, int
csId, int* csTblIdxMapping, CSPropTypes* csPropTypes){
- int i,j;
+void addPropTypes(char *buffTypes, oid* buffP, int numP, int* buffCover, int
**buffTypesCoverMV, int csId, int* csTblIdxMapping, CSPropTypes* csPropTypes){
+ int i,j,k;
int tblId = csTblIdxMapping[csId];
//printf("Add %d prop from CS %d to table %d \n", numP, csId, tblId);
@@ -704,6 +751,16 @@ void addPropTypes(char *buffTypes, oid*
csPropTypes[tblId].lstPropTypes[j].propFreq++;
csPropTypes[tblId].lstPropTypes[j].propCover +=
buffCover[i];
csPropTypes[tblId].lstPropTypes[j].lstFreq[(int)buffTypes[i]]++;
+
csPropTypes[tblId].lstPropTypes[j].lstFreqWithMV[(int)buffTypes[i]]++;
+
+ if (buffTypes[i] == MULTIVALUES){
+ //Add the number of triples per type (e.g.,
int, string
+ //in this multi-valued prop to the freq of each
type
+ for (k = 0; k < MULTIVALUES; k++){
+
csPropTypes[tblId].lstPropTypes[j].lstFreqWithMV[k] += buffTypesCoverMV[i][k];
+ }
+ }
+
#if STAT_ANALYZE
if (buffTypes[i] == MULTIVALUES){
csPropTypes[tblId].lstPropTypes[j].numMVType++;
@@ -734,6 +791,7 @@ void freeCSPropTypes(CSPropTypes* csProp
for (j = 0; j < csPropTypes[i].numProp; j++){
GDKfree(csPropTypes[i].lstPropTypes[j].lstTypes);
GDKfree(csPropTypes[i].lstPropTypes[j].lstFreq);
+ GDKfree(csPropTypes[i].lstPropTypes[j].lstFreqWithMV);
GDKfree(csPropTypes[i].lstPropTypes[j].colIdxes);
GDKfree(csPropTypes[i].lstPropTypes[j].TableTypes);
}
@@ -1471,8 +1529,8 @@ str printFreqCSSet(CSset *freqCSset, BAT
int i;
int j;
int *freq;
- FILE *fout, *fout2;
- char filename[100], filename2[100];
+ FILE *fout;
+ char filename[100];
char tmpStr[20];
#if SHOWPROPERTYNAME
@@ -1514,13 +1572,9 @@ str printFreqCSSet(CSset *freqCSset, BAT
strcpy(filename, "freqCSFullInfo");
sprintf(tmpStr, "%d", freqThreshold);
strcat(filename, tmpStr);
- strcpy(filename2, "max");
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
