Changeset: 19b59ba70e65 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19b59ba70e65
Modified Files:
monetdb5/modules/mal/mosaic.c
monetdb5/modules/mal/mosaic.h
monetdb5/modules/mal/mosaic_delta.c
monetdb5/modules/mal/mosaic_dictionary.c
monetdb5/modules/mal/mosaic_frame.c
monetdb5/modules/mal/mosaic_hdr.c
monetdb5/modules/mal/mosaic_linear.c
monetdb5/modules/mal/mosaic_literal.c
monetdb5/modules/mal/mosaic_prefix.c
monetdb5/modules/mal/mosaic_runlength.c
Branch: mosaic
Log Message:
Reuse compression information
Add a global optimizer that re-uses explorative compression
information in each step.
diffs (truncated from 1120 to 300 lines):
diff --git a/monetdb5/modules/mal/mosaic.c b/monetdb5/modules/mal/mosaic.c
--- a/monetdb5/modules/mal/mosaic.c
+++ b/monetdb5/modules/mal/mosaic.c
@@ -281,15 +281,75 @@ inheritCOL( BAT *bn, COLrec *cn, BAT *b,
MOSsetCnt(TASK->blk,0);\
TASK->dst = ((char*) TASK->blk)+ MosaicBlkSize;
+/* The compression orchestration is dealt with here.
+ * We assume that the estimates for each scheme returns
+ * the number of elements it applies to. Moreover, we
+ * assume that the compression factor holds for any subsequence.
+ * This allows us to avoid expensive estimate calls when a small
+ * sequence is found with high compression factor.
+ */
+static int
+MOSoptimizer(Client cntxt, MOStask task)
+{
+ int cand = MOSAIC_NONE;
+ float ratio = 1.0, fac = 1.0;
+
+ // select candidate amongst those
+ if ( task->filter[MOSAIC_RLE]){
+ fac = MOSestimate_runlength(cntxt,task);
+ if (fac > ratio){
+ cand = MOSAIC_RLE;
+ ratio = fac;
+ }
+ }
+ if ( task->filter[MOSAIC_DICT]){
+ fac = MOSestimate_dictionary(cntxt,task);
+ if (fac > ratio){
+ cand = MOSAIC_DICT;
+ ratio = fac;
+ }
+ }
+ if ( task->filter[MOSAIC_DELTA]){
+ fac = MOSestimate_delta(cntxt,task);
+ if ( fac > ratio ){
+ cand = MOSAIC_DELTA;
+ ratio = fac;
+ }
+ }
+ if ( task->filter[MOSAIC_LINEAR]){
+ fac = MOSestimate_linear(cntxt,task);
+ if ( fac >ratio){
+ cand = MOSAIC_LINEAR;
+ ratio = fac;
+ }
+ }
+ if ( task->filter[MOSAIC_FRAME]){
+ fac = MOSestimate_frame(cntxt,task);
+ if (fac > ratio){
+ cand = MOSAIC_FRAME;
+ ratio = fac;
+ }
+ }
+ if ( task->filter[MOSAIC_PREFIX]){
+ fac = MOSestimate_prefix(cntxt,task);
+ if ( fac > ratio ){
+ cand = MOSAIC_PREFIX;
+ ratio = fac;
+ }
+ if ( fac < 0.0)
+ task->filter[MOSAIC_PREFIX] = 0;
+ }
+ //mnstr_printf(cntxt->fdout,"#cand %d factor %f\n",cand,ratio);
+ return cand;
+}
+
str
MOScompressInternal(Client cntxt, bat *ret, bat *bid, MOStask task, int
inplace, int debug)
{
BAT *bsrc; // the source BAT
BAT *bcompress; // the BAT that will contain the compressed version
- BUN cutoff =0;
str msg = MAL_SUCCEED;
int cand;
- float ratio= 1.0, fac= 1.0;
*ret = 0;
@@ -397,67 +457,9 @@ MOScompressInternal(Client cntxt, bat *r
// always start with an EOL block
MOSsetTag(task->blk,MOSAIC_EOL);
- cutoff = task->elm > 1000? task->elm - 1000: task->elm;
while(task->start < task->stop ){
// default is to extend the non-compressed block
- cand = MOSAIC_NONE;
- fac = 1.0;
- ratio = 1.0;
-
- // cutoff the filters, especially dictionary tests are expensive
- if( cutoff && cutoff < task->start){
- if( task->hdr->blks[MOSAIC_PREFIX] == 0)
- task->filter[MOSAIC_PREFIX] = 0;
- if( task->hdr->blks[MOSAIC_DICT] == 0)
- task->filter[MOSAIC_DICT] = 0;
- cutoff = 0;
- }
-
- // select candidate amongst those
- if ( task->filter[MOSAIC_RLE]){
- fac = MOSestimate_runlength(cntxt,task);
- if (fac > ratio){
- cand = MOSAIC_RLE;
- ratio = fac;
- }
- }
- if ( task->filter[MOSAIC_DICT]){
- fac = MOSestimate_dictionary(cntxt,task);
- if (fac > ratio){
- cand = MOSAIC_DICT;
- ratio = fac;
- }
- }
- if ( task->filter[MOSAIC_FRAME]){
- fac = MOSestimate_frame(cntxt,task);
- if (fac > ratio){
- cand = MOSAIC_FRAME;
- ratio = fac;
- }
- }
- if ( task->filter[MOSAIC_DELTA]){
- fac = MOSestimate_delta(cntxt,task);
- if ( fac > ratio ){
- cand = MOSAIC_DELTA;
- ratio = fac;
- }
- }
- if ( task->filter[MOSAIC_PREFIX]){
- fac = MOSestimate_prefix(cntxt,task);
- if ( fac > ratio ){
- cand = MOSAIC_PREFIX;
- ratio = fac;
- }
- if ( fac < 0.0)
- task->filter[MOSAIC_PREFIX] = 0;
- }
- if ( task->filter[MOSAIC_LINEAR]){
- fac = MOSestimate_linear(cntxt,task);
- if ( fac >ratio){
- cand = MOSAIC_LINEAR;
- ratio = fac;
- }
- }
+ cand = MOSoptimizer(cntxt,task);
// wrapup previous block
switch(cand){
@@ -490,42 +492,42 @@ MOScompressInternal(Client cntxt, bat *r
}
// apply the compression to a chunk
switch(cand){
+ case MOSAIC_RLE:
+ MOScompress_runlength(cntxt,task);
+ MOSupdateHeader(cntxt,task);
+ MOSadvance_runlength(cntxt,task);
+ MOSnewBlk(task);
+ break;
case MOSAIC_DICT:
MOScompress_dictionary(cntxt,task);
MOSupdateHeader(cntxt,task);
MOSadvance_dictionary(cntxt,task);
MOSnewBlk(task);
break;
+ case MOSAIC_DELTA:
+ MOScompress_delta(cntxt,task);
+ MOSupdateHeader(cntxt,task);
+ MOSadvance_delta(cntxt,task);
+ MOSnewBlk(task);
+ break;
+ case MOSAIC_LINEAR:
+ MOScompress_linear(cntxt,task);
+ MOSupdateHeader(cntxt,task);
+ MOSadvance_linear(cntxt,task);
+ MOSnewBlk(task);
+ break;
case MOSAIC_FRAME:
MOScompress_frame(cntxt,task);
MOSupdateHeader(cntxt,task);
MOSadvance_frame(cntxt,task);
MOSnewBlk(task);
break;
- case MOSAIC_DELTA:
- MOScompress_delta(cntxt,task);
- MOSupdateHeader(cntxt,task);
- MOSadvance_delta(cntxt,task);
- MOSnewBlk(task);
- break;
case MOSAIC_PREFIX:
MOScompress_prefix(cntxt,task);
MOSupdateHeader(cntxt,task);
MOSadvance_prefix(cntxt,task);
MOSnewBlk(task);
break;
- case MOSAIC_LINEAR:
- MOScompress_linear(cntxt,task);
- MOSupdateHeader(cntxt,task);
- MOSadvance_linear(cntxt,task);
- MOSnewBlk(task);
- break;
- case MOSAIC_RLE:
- MOScompress_runlength(cntxt,task);
- MOSupdateHeader(cntxt,task);
- MOSadvance_runlength(cntxt,task);
- MOSnewBlk(task);
- break;
default :
// continue to use the last block header.
MOScompress_literal(cntxt,task);
@@ -704,26 +706,6 @@ MOSdecompressInternal(Client cntxt, bat
while(task->blk){
switch(MOSgetTag(task->blk)){
- case MOSAIC_DICT:
- MOSdecompress_dictionary(cntxt,task);
- MOSskip_dictionary(cntxt,task);
- break;
- case MOSAIC_FRAME:
- MOSdecompress_frame(cntxt,task);
- MOSskip_frame(cntxt,task);
- break;
- case MOSAIC_DELTA:
- MOSdecompress_delta(cntxt,task);
- MOSskip_delta(cntxt,task);
- break;
- case MOSAIC_PREFIX:
- MOSdecompress_prefix(cntxt,task);
- MOSskip_prefix(cntxt,task);
- break;
- case MOSAIC_LINEAR:
- MOSdecompress_linear(cntxt,task);
- MOSskip_linear(cntxt,task);
- break;
case MOSAIC_NONE:
MOSdecompress_literal(cntxt,task);
MOSskip_literal(cntxt,task);
@@ -732,6 +714,26 @@ MOSdecompressInternal(Client cntxt, bat
MOSdecompress_runlength(cntxt,task);
MOSskip_runlength(cntxt,task);
break;
+ case MOSAIC_DICT:
+ MOSdecompress_dictionary(cntxt,task);
+ MOSskip_dictionary(cntxt,task);
+ break;
+ case MOSAIC_DELTA:
+ MOSdecompress_delta(cntxt,task);
+ MOSskip_delta(cntxt,task);
+ break;
+ case MOSAIC_LINEAR:
+ MOSdecompress_linear(cntxt,task);
+ MOSskip_linear(cntxt,task);
+ break;
+ case MOSAIC_FRAME:
+ MOSdecompress_frame(cntxt,task);
+ MOSskip_frame(cntxt,task);
+ break;
+ case MOSAIC_PREFIX:
+ MOSdecompress_prefix(cntxt,task);
+ MOSskip_prefix(cntxt,task);
+ break;
default: assert(0);
}
}
@@ -780,6 +782,8 @@ MOSdecompressInternal(Client cntxt, bat
case TYPE_dbl:
error = task->hdr->checksum.sumdbl !=
task->hdr->checksum2.sumdbl;
break;
+ case TYPE_str:
+ break;
default:
mnstr_printf(cntxt->fdout,"#unknown compression
compatibility\n");
}
@@ -1534,6 +1538,8 @@ MOSanalyseReport(Client cntxt, BAT *b, B
if( j<i ) continue;
for(j=0, bit=1; j < MOSAIC_METHODS-1; j++){
task->filter[j]= (pattern[i] & bit)>0;
+ task->range[j]= 0;
+ task->factor[j]= 0.0;
bit *=2;
}
MOScompressInternal(cntxt, &ret, &bid, task, 0, 0);
@@ -1687,11 +1693,17 @@ MOSanalyse(Client cntxt, MalBlkPtr mb, M
mx = 0;
for( j = 0; j < top; j++){
if( properties[j] && !strstr(properties[j],"mosaic"))
- for( k = 0; k< MOSAIC_METHODS; k++)
+ for( k = 0; k< MOSAIC_METHODS; k++){
task->filter[k]=
strstr(properties[j],MOSfiltername[k]) != 0;
+ task->range[k] = 0;
+ task->factor[k] = 0.0;
+ }
else
- for( k = 0; k< MOSAIC_METHODS; k++)
+ for( k = 0; k< MOSAIC_METHODS; k++){
task->filter[k]= 1;
+ task->range[k] = 0;
+ task->factor[k] = 0.0;
+ }
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
