Changeset: a12fd542f915 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a12fd542f915
Modified Files:
monetdb5/extras/rdf/rdf.h
monetdb5/extras/rdf/rdflabels.c
monetdb5/extras/rdf/rdfschema.c
sql/backends/monet5/sql_rdf.c
Branch: rdf
Log Message:
Add option for measuring size of emergent tables
diffs (202 lines):
diff --git a/monetdb5/extras/rdf/rdf.h b/monetdb5/extras/rdf/rdf.h
--- a/monetdb5/extras/rdf/rdf.h
+++ b/monetdb5/extras/rdf/rdf.h
@@ -86,6 +86,7 @@ RDFmerge_join(bat *ret1, bat *ret2, bat
#define EVERYTHING_AS_OID 1 /*We do not store type-specific column but oid
only*/
#define STORE_ALL_EXCEPTION_IN_PSO 1 /* All the exceptions such as
non-default type values are stored in
PSO table.*/
+#define COMPUTE_STORAGE_SIZE_ONLY 1 /* Only need to compute the size of the
emergent tables*/
#define batsz 10000000
#define smallbatsz 100000
diff --git a/monetdb5/extras/rdf/rdflabels.c b/monetdb5/extras/rdf/rdflabels.c
--- a/monetdb5/extras/rdf/rdflabels.c
+++ b/monetdb5/extras/rdf/rdflabels.c
@@ -2579,6 +2579,9 @@ void exportLabels(CSset* freqCSset, CSre
// Print and Export
printf("exportLabels: printUML \n");
+ #if COMPUTE_STORAGE_SIZE_ONLY == 1
+ if(0)
+ #endif
printUML2(cstablestat, csPropTypes, freqThreshold, csRelFinalFKs, mapi,
mbat, numTables, mTblIdxFreqIdxMapping, csTblIdxMapping, freqCSset);
printf("exportLabels: Done \n");
diff --git a/monetdb5/extras/rdf/rdfschema.c b/monetdb5/extras/rdf/rdfschema.c
--- a/monetdb5/extras/rdf/rdfschema.c
+++ b/monetdb5/extras/rdf/rdfschema.c
@@ -4103,9 +4103,13 @@ str mergeFreqCSByS1(CSset *freqCSset, CS
}
#if OUTPUT_FREQID_PER_LABEL
- fclose(fout);
+ fclose(fout);
TKNZRclose(&ret);
- #endif
+ #if COMPUTE_STORAGE_SIZE_ONLY
+ remove("freqIdPerLabel.txt");
+ #endif
+ #endif
+
freeLabelStat(labelStat);
@@ -11866,6 +11870,7 @@ str RDFdistTriplesToCSs(int *ret, bat *s
}
#endif
+#if COMPUTE_STORAGE_SIZE_ONLY == 0
#if BUILDTOKENZIER_TO_MAPID
/*
* Since the of s,p,o oids are converted to table-based oids,
@@ -11997,6 +12002,7 @@ str buildTKNZRMappingBat(BAT *lmap, BAT
}
#endif /* BUILDTOKENZIER_TO_MAPID*/
+#endif /* COMPUTE_STORAGE_SIZE_ONLY == 0*/
str
RDFreorganize(int *ret, CStableStat *cstablestat, CSPropTypes **csPropTypes,
bat *sbatid, bat *pbatid, bat *obatid, bat *mapbatid, bat *ontbatid, int
*freqThreshold, int *mode){
@@ -12166,7 +12172,10 @@ RDFreorganize(int *ret, CStableStat *cst
curT = clock();
printf (" Export label process took %f seconds.\n", ((float)(curT -
tmpLastT))/CLOCKS_PER_SEC);
tmpLastT = curT;
-
+
+ #if COMPUTE_STORAGE_SIZE_ONLY
+ if (0)
+ #endif
printFinalStructure(cstablestat, *csPropTypes,
numTables,*freqThreshold, mapbatid);
#if DETECT_INCORRECT_TYPE_SUBJECT
@@ -12201,7 +12210,7 @@ RDFreorganize(int *ret, CStableStat *cst
computeMetricsQForRefinedTable(freqCSset,
*csPropTypes,mfreqIdxTblIdxMapping,mTblIdxFreqIdxMapping,numTables);
#endif
-
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
#if DUMP_CSSET
{
str schema = "rdf";
@@ -12215,6 +12224,7 @@ RDFreorganize(int *ret, CStableStat *cst
TKNZRclose(ret);
}
#endif
+ #endif
if (*mode == EXPLOREONLY){
printf("Only explore the schema information \n");
@@ -12335,9 +12345,11 @@ RDFreorganize(int *ret, CStableStat *cst
printf("Number of subject removed is: %d \n", numSubjRemoved);
#endif
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
#if BUILDTOKENZIER_TO_MAPID
buildTKNZRMappingBat(lmap, rmap);
#endif
+ #endif
origobat = getOriginalUriOBat(obat); //Return obat without
type-specific information for URI & BLANKNODE
//This is to get the same oid
as the subject oid for a same URI, BLANKNODE
diff --git a/sql/backends/monet5/sql_rdf.c b/sql/backends/monet5/sql_rdf.c
--- a/sql/backends/monet5/sql_rdf.c
+++ b/sql/backends/monet5/sql_rdf.c
@@ -791,6 +791,7 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
int **colOrder = NULL;
(void) tmptbnameex;
+ (void) respotbl;
beginT = clock();
@@ -869,6 +870,7 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
cstablestat->obat, TYPE_bat);
// Add regular triple to cstables and cstablesEx
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
#if TRIPLEBASED_TABLE
printf("Create triple-based relational table ...");
respotbl = mvc_create_table(m, sch, "triples", tt_table, 0,
@@ -891,6 +893,7 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
printf("Done\n");
#endif
+ #endif
printf("Starting creating SQL Table -- \n");
@@ -957,20 +960,29 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
tmpbat = cstablestat->lstcstable[i].colBats[j];
isRightPropBAT(tmpbat);
+ tmpNumMVCols =
cstablestat->lstcstable[i].lstMVTables[j].numCol;
+
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
mvc_create_column(m, cstables[i], tmpcolname,
&tpes[tmpbat->ttype]);
+ #else
+ if (tmpNumMVCols == 0)
+ mvc_create_column(m, cstables[i], tmpcolname,
&tpes[tmpbat->ttype]);
+ #endif
+
//For multi-values table
- tmpNumMVCols =
cstablestat->lstcstable[i].lstMVTables[j].numCol;
if (tmpNumMVCols != 0){
getMvTblSQLname(tmpmvtbname, i, j,
cstablestat->lstcstable[i].tblname, cstablestat->lstcstable[i].lstProp[j],
mapi, mbat);
csmvtables[i][j] = mvc_create_table(m, sch,
tmpmvtbname, tt_table, 0, SQL_PERSIST, 0, 3);
totalNoTablesCreated++;
-
+
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
//One column for key
sprintf(tmpcolname, "mvKey");
tmpbat =
cstablestat->lstcstable[i].lstMVTables[j].keyBat;
isRightPropBAT(tmpbat);
mvc_create_column(m, csmvtables[i][j],
tmpcolname, &tpes[tmpbat->ttype]);
+ #endif
//One column for subj oid
sprintf(tmpcolname, "mvsubj");
@@ -1044,14 +1056,23 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
isRightPropBAT(tmpbat);
//printf("Column %d of tableId %d: Batid is %d \n",j,
i, tmpbat->batCacheid);
//BATprint(tmpbat);
+ tmpNumMVCols =
cstablestat->lstcstable[i].lstMVTables[j].numCol;
+
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0
store_funcs.append_col(m->session->tr,
mvc_bind_column(m,
cstables[i],tmpcolname ),
tmpbat, TYPE_bat);
+ #else
+ if (tmpNumMVCols == 0)
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m,
cstables[i],tmpcolname ),
+ tmpbat, TYPE_bat);
+ #endif
//For multi-values table
- tmpNumMVCols =
cstablestat->lstcstable[i].lstMVTables[j].numCol;
if (tmpNumMVCols != 0){
+ #if COMPUTE_STORAGE_SIZE_ONLY == 0 /* Do not
need to store this extra info*/
//One column for key
sprintf(tmpcolname, "mvKey");
tmpbat =
cstablestat->lstcstable[i].lstMVTables[j].keyBat;
@@ -1059,6 +1080,7 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
store_funcs.append_col(m->session->tr,
mvc_bind_column(m,
csmvtables[i][j],tmpcolname),
tmpbat, TYPE_bat);
+ #endif
//One column for subj Bat
sprintf(tmpcolname, "mvsubj");
@@ -1133,6 +1155,9 @@ SQLrdfreorganize(Client cntxt, MalBlkPtr
printf("Number of non-default-type columns: %d (%f ex-types per prop)
\n ", totalNumNonDefCols, (float)totalNumNonDefCols/totalNumDefCols);
printf("Generating script for FK creation ...");
+ #if COMPUTE_STORAGE_SIZE_ONLY == 1
+ if (0)
+ #endif
addPKandFKs(cstablestat, csPropTypes, *schema, mapi, mbat);
printf("done\n");
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
