Changeset: 14276605e07d for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=14276605e07d
Modified Files:
clients/Tests/exports.stable.out
monetdb5/modules/mosaic/TODO_MOSAIC.txt
monetdb5/modules/mosaic/mosaic.c
monetdb5/modules/mosaic/mosaic_capped.c
monetdb5/modules/mosaic/mosaic_capped.h
monetdb5/modules/mosaic/mosaic_delta.c
monetdb5/modules/mosaic/mosaic_delta.h
monetdb5/modules/mosaic/mosaic_frame.c
monetdb5/modules/mosaic/mosaic_frame.h
monetdb5/modules/mosaic/mosaic_linear.c
monetdb5/modules/mosaic/mosaic_linear.h
monetdb5/modules/mosaic/mosaic_prefix.c
monetdb5/modules/mosaic/mosaic_prefix.h
monetdb5/modules/mosaic/mosaic_raw.c
monetdb5/modules/mosaic/mosaic_raw.h
monetdb5/modules/mosaic/mosaic_runlength.c
monetdb5/modules/mosaic/mosaic_runlength.h
monetdb5/modules/mosaic/mosaic_utility.h
monetdb5/modules/mosaic/mosaic_var.c
monetdb5/modules/mosaic/mosaic_var.h
Branch: mosaic
Log Message:
Refactoring:
Apply type-first-technique-last in (de)compress logic.
Remove the oid-type specific handling.
diffs (truncated from 2266 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1523,34 +1523,10 @@ void MOSadvance_var(MOStask task);
void MOSblk(MosaicBlk blk);
str MOScompress(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str MOScompressInternal(BAT *bsrc, const char *compressions);
-void MOScompress_capped(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_delta(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_frame(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_linear(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_prefix(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_raw(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_runlength(MOStask task, MosaicBlkRec *estimate);
-void MOScompress_var(MOStask task, MosaicBlkRec *estimate);
str MOScreateCappedInfo(MOStask task);
void MOScreateframeDictionary(MOStask task);
void MOScreatevar(MOStask task);
str MOSdecompress(bat *ret, const bat *bid);
-void MOSdecompress_capped(MOStask task);
-void MOSdecompress_delta(MOStask task);
-void MOSdecompress_frame(MOStask task);
-void MOSdecompress_linear(MOStask task);
-void MOSdecompress_prefix(MOStask task);
-void MOSdecompress_raw(MOStask task);
-void MOSdecompress_runlength(MOStask task);
-void MOSdecompress_var(MOStask task);
-str MOSestimate_capped(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_delta(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_frame(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_linear(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_prefix(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_raw(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_runlength(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
-str MOSestimate_var(MOStask task, MosaicEstimation *current, const
MosaicEstimation *previous);
char *MOSfiltername[];
void MOSinitHeader(MOStask task);
void MOSinitializeScan(MOStask task, BAT *b);
@@ -1569,8 +1545,6 @@ void MOSlayout_runlength(MOStask task, B
void MOSlayout_var(MOStask task, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties);
void MOSlayout_var_hdr(MOStask task, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties);
BUN MOSlimit(void);
-void MOSpostEstimate_capped(MOStask task);
-void MOSpostEstimate_var(MOStask task);
str MOSprepareEstimate_capped(MOStask task);
str MOSprepareEstimate_var(MOStask task);
str MOSprojection(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
diff --git a/monetdb5/modules/mosaic/TODO_MOSAIC.txt
b/monetdb5/modules/mosaic/TODO_MOSAIC.txt
--- a/monetdb5/modules/mosaic/TODO_MOSAIC.txt
+++ b/monetdb5/modules/mosaic/TODO_MOSAIC.txt
@@ -1,7 +1,5 @@
####technical depth####
Split the MOSadvance-like functions to type specific variant.
-See if compress, decompress and layout can be refactored similar to the
algebraic operators.
-Factor out boilerplate code mosaic code: resulting mosaic boilerplate is for
testing purposes.
Create test for swapped join.
Add lots of tests with candidate lists for all
Add test for float/double
@@ -27,7 +25,6 @@ Get rid of (most of) MOStask and/or don'
Make sure the parameters.bits fields are always char.
Use GDK naming conventions, e.g. o for stuff coming out if canditer_next.
Make block headers specific by having type parametrized blockheader struct's.
-atomic base type of an oid bat is lng. So we can remove that one.
It is a bit cleaner if MOSadvance call's are done outside of select functions.
create a common header/source file for mosaic for common tasks e.g. to house a
generic IS_NIL macro, IPtpe, etc.
Improve the administration of the various hard coded compression techniques,
e.g. MOSskipit skip (2*)
diff --git a/monetdb5/modules/mosaic/mosaic.c b/monetdb5/modules/mosaic/mosaic.c
--- a/monetdb5/modules/mosaic/mosaic.c
+++ b/monetdb5/modules/mosaic/mosaic.c
@@ -141,6 +141,7 @@ MOSlayout(BAT *b, BAT *btech, BAT *bcoun
BUNappend(bproperties, "", false) != GDK_SUCCEED)
throw(MAL,"mosaic.layout", MAL_MALLOC_FAIL);
+/*
while(task->start< task->stop){
switch(MOSgetTag(task->blk)){
case MOSAIC_RAW:
@@ -187,6 +188,7 @@ MOSlayout(BAT *b, BAT *btech, BAT *bcoun
assert(0);
}
}
+*/
return MAL_SUCCEED;
}
@@ -222,138 +224,156 @@ MOSprepareEstimate(MOStask task) {
return MAL_SUCCEED;
}
-/* The compression orchestration is dealt with here.
- * We assume that the estimates for each scheme returns
- * the number of elements it applies to. Moreover, we
- * assume that the compression factor holds for any subsequence.
- * This allows us to avoid expensive estimate calls when a small
- * sequence is found with high compression factor.
- */
-static str
-MOSoptimizerCost(MOStask task, MosaicEstimation* current, const
MosaicEstimation* previous) {
- str result = MAL_SUCCEED;
- MosaicEstimation estimations[MOSAICINDEX];
- const int size = sizeof(estimations) / sizeof(MosaicEstimation);
- for (int i = 0; i < size; i++) {
- estimations[i].uncompressed_size = previous->uncompressed_size;
- estimations[i].compressed_size = previous->compressed_size;
- estimations[i].compression_strategy =
previous->compression_strategy;
- estimations[i].nr_var_encoded_blocks =
previous->nr_var_encoded_blocks;
- estimations[i].nr_var_encoded_elements =
previous->nr_var_encoded_elements;
- estimations[i].nr_capped_encoded_elements =
previous->nr_capped_encoded_elements;
- estimations[i].nr_capped_encoded_blocks =
previous->nr_capped_encoded_blocks;
- estimations[i].must_be_merged_with_previous = false;
- estimations[i].is_applicable = false;
- }
+#define do_estimate(NAME, TPE, DUMMY_ARGUMENT)\
+{\
+ str msg = MOSestimate_##NAME##_##TPE(task, &estimations[MOSAIC_LINEAR],
previous);\
+ if (msg != MAL_SUCCEED) return msg;\
+}
+
+
+#define do_postEstimate(NAME, TPE, DUMMY_ARGUMENT)
MOSpostEstimate_##NAME##_##TPE(task);
- // select candidate amongst those
- if (task->filter[MOSAIC_RAW]){
- if( (result = MOSestimate_raw(task, &estimations[MOSAIC_RAW],
previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_RLE]){
- if( (result = MOSestimate_runlength(task,
&estimations[MOSAIC_RLE], previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_CAPPED]){
- if( (result = MOSestimate_capped(task,
&estimations[MOSAIC_CAPPED], previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_VAR]){
- if( (result = MOSestimate_var(task, &estimations[MOSAIC_VAR],
previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_DELTA]){
- if( (result = MOSestimate_delta(task,
&estimations[MOSAIC_DELTA], previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_LINEAR]){
- if( (result = MOSestimate_linear(task,
&estimations[MOSAIC_LINEAR], previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_FRAME]){
- if( (result = MOSestimate_frame(task,
&estimations[MOSAIC_FRAME], previous))) {
- return result;
- }
- }
- if (task->filter[MOSAIC_PREFIX]){
- if( (result = MOSestimate_prefix(task,
&estimations[MOSAIC_PREFIX], previous))) {
- return result;
- }
- }
+#define MOSestimate_AND_MOSoptimizerCost_DEF(TPE) \
+static str MOSestimate_inner_##TPE(MOStask task, MosaicEstimation* current,
const MosaicEstimation* previous) {\
+\
+ MosaicEstimation estimations[MOSAICINDEX];\
+ const int size = sizeof(estimations) / sizeof(MosaicEstimation);\
+ for (int i = 0; i < size; i++) {\
+ estimations[i].uncompressed_size = previous->uncompressed_size;\
+ estimations[i].compressed_size = previous->compressed_size;\
+ estimations[i].compression_strategy =
previous->compression_strategy;\
+ estimations[i].nr_var_encoded_blocks =
previous->nr_var_encoded_blocks;\
+ estimations[i].nr_var_encoded_elements =
previous->nr_var_encoded_elements;\
+ estimations[i].nr_capped_encoded_elements =
previous->nr_capped_encoded_elements;\
+ estimations[i].nr_capped_encoded_blocks =
previous->nr_capped_encoded_blocks;\
+ estimations[i].must_be_merged_with_previous = false;\
+ estimations[i].is_applicable = false;\
+ }\
+\
+ /* select candidate amongst those*/\
+ if (task->filter[MOSAIC_RAW]){\
+ DO_OPERATION_IF_ALLOWED(estimate, raw, TPE);\
+ }\
+ if (task->filter[MOSAIC_RLE]){\
+ DO_OPERATION_IF_ALLOWED(estimate, runlength, TPE);\
+ }\
+ if (task->filter[MOSAIC_CAPPED]){\
+ DO_OPERATION_IF_ALLOWED(estimate, capped, TPE);\
+ }\
+ if (task->filter[MOSAIC_VAR]){\
+ DO_OPERATION_IF_ALLOWED(estimate, var, TPE);\
+ }\
+ if (task->filter[MOSAIC_DELTA]){\
+ DO_OPERATION_IF_ALLOWED(estimate, delta, TPE);\
+ }\
+ if (task->filter[MOSAIC_LINEAR]){\
+ DO_OPERATION_IF_ALLOWED(estimate, linear, TPE);\
+ }\
+ if (task->filter[MOSAIC_FRAME]){\
+ DO_OPERATION_IF_ALLOWED(estimate, frame, TPE);\
+ }\
+ if (task->filter[MOSAIC_PREFIX]){\
+ DO_OPERATION_IF_ALLOWED(estimate, prefix, TPE);\
+ }\
+\
+ flt best_factor = 0.0;\
+ current->is_applicable = false;\
+\
+ for (int i = 0; i < size; i++) {\
+ flt factor = getFactor(estimations[i]);\
+\
+ if (estimations[i].is_applicable && best_factor < factor) {\
+ *current = estimations[i];\
+ best_factor = factor;\
+ }\
+ }\
+\
+ if (current->compression_strategy.tag == MOSAIC_RAW)
DO_OPERATION_IF_ALLOWED(postEstimate, raw, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_RLE)
DO_OPERATION_IF_ALLOWED(postEstimate, runlength, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_CAPPED)
DO_OPERATION_IF_ALLOWED(postEstimate, capped, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_VAR)
DO_OPERATION_IF_ALLOWED(postEstimate, var, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_DELTA)
DO_OPERATION_IF_ALLOWED(postEstimate, delta, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_LINEAR)
DO_OPERATION_IF_ALLOWED(postEstimate, linear, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_FRAME)
DO_OPERATION_IF_ALLOWED(postEstimate, frame, TPE);\
+ if (current->compression_strategy.tag == MOSAIC_PREFIX)
DO_OPERATION_IF_ALLOWED(postEstimate, prefix, TPE);\
+\
+ return MAL_SUCCEED;\
+}\
+static str MOSestimate_##TPE(MOStask task, BAT* estimates, size_t*
compressed_size) {\
+ str result = MAL_SUCCEED;\
+\
+ *compressed_size = 0;\
+\
+ MosaicEstimation previous = {\
+ .is_applicable = false,\
+ .uncompressed_size = 0,\
+ .compressed_size = 0,\
+ .nr_var_encoded_elements = 0,\
+ .nr_var_encoded_blocks = 0,\
+ .nr_capped_encoded_elements = 0,\
+ .nr_capped_encoded_blocks = 0,\
+ .compression_strategy = {.tag = MOSAIC_EOL, .cnt = 0},\
+ .must_be_merged_with_previous = false\
+ };\
+\
+ MosaicEstimation current;\
+ MosaicBlkRec* cursor = Tloc(estimates,0);\
+\
+ while(task->start < task->stop ){\
+ /* default is to extend the non-compressed block with a single
element*/\
+ if ( (result = MOSestimate_inner_##TPE(task, ¤t,
&previous)) ) {\
+ return result;\
+ }\
+\
+ if (!current.is_applicable) {\
+ throw(MAL,"mosaic.compress", "Cannot compress BAT with
given compression techniques.");\
+ }\
+\
+ if (current.must_be_merged_with_previous) {\
+ --cursor;\
+ assert(cursor->tag == previous.compression_strategy.tag
&& cursor->cnt == previous.compression_strategy.cnt);\
+ task->start -= previous.compression_strategy.cnt;\
+ }\
+ else BATcount(estimates)++;\
+\
+ *cursor = current.compression_strategy;\
+ ++cursor;\
+ previous = current;\
+ task->start += current.compression_strategy.cnt;\
+ }\
+\
+ (*compressed_size) = current.compressed_size;\
+\
+ return MAL_SUCCEED;\
+}
- flt best_factor = 0.0;
- current->is_applicable = false;
-
- for (int i = 0; i < size; i++) {
- flt factor = getFactor(estimations[i]);
+MOSestimate_AND_MOSoptimizerCost_DEF(bte)
+MOSestimate_AND_MOSoptimizerCost_DEF(sht)
+MOSestimate_AND_MOSoptimizerCost_DEF(int)
+MOSestimate_AND_MOSoptimizerCost_DEF(lng)
+MOSestimate_AND_MOSoptimizerCost_DEF(flt)
+MOSestimate_AND_MOSoptimizerCost_DEF(dbl)
+#ifdef HAVE_HGE
+MOSestimate_AND_MOSoptimizerCost_DEF(hge)
+#endif
- if (estimations[i].is_applicable && best_factor < factor) {
- *current = estimations[i];
- best_factor = factor;
- }
- }
-
- if (current->compression_strategy.tag == MOSAIC_CAPPED)
MOSpostEstimate_capped(task);
- if (current->compression_strategy.tag == MOSAIC_VAR)
MOSpostEstimate_var(task);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
