Changeset: 1f0edad08957 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1f0edad08957
Modified Files:
monetdb5/modules/mosaic/TODO_MOSAIC.txt
monetdb5/modules/mosaic/mosaic.c
monetdb5/modules/mosaic/mosaic.h
monetdb5/modules/mosaic/mosaic_delta.c
monetdb5/modules/mosaic/mosaic_delta.h
monetdb5/modules/mosaic/mosaic_delta_templates.h
monetdb5/modules/mosaic/mosaic_dict.c
monetdb5/modules/mosaic/mosaic_dict.h
monetdb5/modules/mosaic/mosaic_dict256.c
monetdb5/modules/mosaic/mosaic_dict256.h
monetdb5/modules/mosaic/mosaic_dictionary_templates.h
monetdb5/modules/mosaic/mosaic_frame.c
monetdb5/modules/mosaic/mosaic_frame.h
monetdb5/modules/mosaic/mosaic_frame_templates.h
monetdb5/modules/mosaic/mosaic_hdr.c
monetdb5/modules/mosaic/mosaic_hdr.h
monetdb5/modules/mosaic/mosaic_linear.c
monetdb5/modules/mosaic/mosaic_linear.h
monetdb5/modules/mosaic/mosaic_linear_templates.h
monetdb5/modules/mosaic/mosaic_prefix.c
monetdb5/modules/mosaic/mosaic_prefix.h
monetdb5/modules/mosaic/mosaic_prefix_templates.h
monetdb5/modules/mosaic/mosaic_raw.c
monetdb5/modules/mosaic/mosaic_raw.h
monetdb5/modules/mosaic/mosaic_raw_templates.h
monetdb5/modules/mosaic/mosaic_runlength.c
monetdb5/modules/mosaic/mosaic_runlength.h
monetdb5/modules/mosaic/mosaic_runlength_templates.h
monetdb5/modules/mosaic/mosaic_template.h
monetdb5/modules/mosaic/mosaic_utility.h
Branch: mosaic-scratch
Log Message:
Work in progress on layout.
diffs (truncated from 1245 to 300 lines):
diff --git a/monetdb5/modules/mosaic/TODO_MOSAIC.txt
b/monetdb5/modules/mosaic/TODO_MOSAIC.txt
--- a/monetdb5/modules/mosaic/TODO_MOSAIC.txt
+++ b/monetdb5/modules/mosaic/TODO_MOSAIC.txt
@@ -5,7 +5,6 @@ Fix/check/test MOSlayout.
Add the compression on OID heap for string valued columns too.
Figure out why mix compression don't seem to work that well.
benchmark using TPC-H
-Refactor macro-ised functions to real functions in header files and apply the
include trick
Add negative values to tests
Add test for huge
create mix compression test cases particularly for dict/dict256 combo.
@@ -27,6 +26,8 @@ Create a plug-in estimation model framew
####code issues####
Straighten out semantics MOSselect. Make them exact to BATselect.
+Refactor macro-ised functions to real functions in header files and apply the
include trick
+Refactor the structs MosaicEstimation,
GlobalDictionaryInfo/EstimationParameters, MOStask.
See if the iteration algorithms can be improved.
Use GDK naming conventions, e.g. o for stuff coming out if canditer_next.
Update copyright in 2020.
diff --git a/monetdb5/modules/mosaic/mosaic.c b/monetdb5/modules/mosaic/mosaic.c
--- a/monetdb5/modules/mosaic/mosaic.c
+++ b/monetdb5/modules/mosaic/mosaic.c
@@ -145,102 +145,6 @@ MOSinit(MOStask* task, BAT *b) {
task->padding = NULL;
}
-str
-MOSlayout(BAT *b, BAT *btech, BAT *bcount, BAT *binput, BAT *boutput, BAT
*bproperties)
-{
- MOStask task = {0};
- unsigned i;
- char buf[BUFSIZ];
- lng zero=0;
-
- if( b->tmosaic == NULL) {
- throw(MAL,"mosaic.layout","Compression heap missing");
- }
-
- MOSinit(&task,b);
- MOSinitializeScan(&task, b);
- // safe the general properties
-
- snprintf(buf,BUFSIZ,"%g", (task.hdr)->ratio);
- if( BUNappend(btech, "ratio", false) != GDK_SUCCEED ||
- BUNappend(bcount, &zero, false) != GDK_SUCCEED ||
- BUNappend(binput, &zero, false) != GDK_SUCCEED ||
- BUNappend(bproperties, buf, false) != GDK_SUCCEED ||
- BUNappend(boutput, &zero , false) != GDK_SUCCEED)
- throw(MAL,"mosaic.layout", MAL_MALLOC_FAIL);
- for(i=0; i < MOSAIC_METHODS; i++){
- lng zero = 0;
- snprintf(buf,BUFSIZ,"%s blocks", MOSmethods[i].name);
- if( BUNappend(btech, buf, false) != GDK_SUCCEED ||
- BUNappend(bcount, &(task.hdr)->blks[i], false) !=
GDK_SUCCEED ||
- BUNappend(binput, &(task.hdr)->elms[i], false) !=
GDK_SUCCEED ||
- BUNappend(boutput, &zero , false) != GDK_SUCCEED ||
- BUNappend(bproperties, "", false) != GDK_SUCCEED)
- throw(MAL,"mosaic.layout", MAL_MALLOC_FAIL);
- }
- if( (task.hdr)->blks[MOSAIC_DICT256])
-
MOSlayout_dict256_hdr(&task,btech,bcount,binput,boutput,bproperties);
- if( (task.hdr)->blks[MOSAIC_DICT])
-
MOSlayout_dict_hdr(&task,btech,bcount,binput,boutput,bproperties);
-
- if( BUNappend(btech, "========", false) != GDK_SUCCEED ||
- BUNappend(bcount, &zero, false) != GDK_SUCCEED ||
- BUNappend(binput, &zero, false) != GDK_SUCCEED ||
- BUNappend(boutput, &zero , false) != GDK_SUCCEED ||
- BUNappend(bproperties, "", false) != GDK_SUCCEED)
- throw(MAL,"mosaic.layout", MAL_MALLOC_FAIL);
-
-/*
- while(task->start< task->stop){
- switch(MOSgetTag(task->blk)){
- case MOSAIC_RAW:
- ALGODEBUG mnstr_printf(GDKstdout, "#MOSlayout_raw\n");
-
MOSlayout_raw(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_raw(task);
- break;
- case MOSAIC_RLE:
- ALGODEBUG mnstr_printf(GDKstdout,
"#MOSlayout_runlength\n");
-
MOSlayout_runlength(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_runlength(task);
- break;
- case MOSAIC_DICT256:
- ALGODEBUG mnstr_printf(GDKstdout,
"#MOSlayout_dict256\n");
-
MOSlayout_dict256(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_dict256(task);
- break;
- case MOSAIC_DICT:
- ALGODEBUG mnstr_printf(GDKstdout, "#MOSlayout_dict\n");
-
MOSlayout_dict(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_dict(task);
- break;
- case MOSAIC_DELTA:
- ALGODEBUG mnstr_printf(GDKstdout, "#MOSlayout_delta\n");
-
MOSlayout_delta(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_delta(task);
- break;
- case MOSAIC_LINEAR:
- ALGODEBUG mnstr_printf(GDKstdout,
"#MOSlayout_linear\n");
-
MOSlayout_linear(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_linear(task);
- break;
- case MOSAIC_FRAME:
- ALGODEBUG mnstr_printf(GDKstdout, "#MOSlayout_frame\n");
-
MOSlayout_frame(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_frame(task);
- break;
- case MOSAIC_PREFIX:
- ALGODEBUG mnstr_printf(GDKstdout,
"#MOSlayout_prefix\n");
-
MOSlayout_prefix(task,btech,bcount,binput,boutput,bproperties);
- MOSadvance_prefix(task);
- break;
- default:
- assert(0);
- }
- }
-*/
- return MAL_SUCCEED;
-}
-
/*
* Compression is focussed on a single column.
* Multiple compression MOSmethods are applied at the same time.
@@ -303,6 +207,61 @@ MOSprepareDictionaryContext(MOStask* tas
#undef TPE
#endif
+static str
+layout_insert_record(MosaicLayout* layout, lng* bsn, str* tech, lng* count,
lng* input, lng* output, str* properties) {
+ if(
+ BUNappend(layout->bsn , bsn, false) != GDK_SUCCEED ||
+ BUNappend(layout->tech , tech, false) != GDK_SUCCEED ||
+ BUNappend(layout->count , count, false) != GDK_SUCCEED
||
+ BUNappend(layout->input , input, false) != GDK_SUCCEED
||
+ BUNappend(layout->properties, properties, false) != GDK_SUCCEED
||
+ BUNappend(layout->output , output, false) != GDK_SUCCEED
+ ) throw(MAL,"mosaic.layout_insert_record", MAL_MALLOC_FAIL);
+
+ return MAL_SUCCEED;
+}
+
+str
+MOSlayout(BAT *b, BAT *bbsn, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties) {
+ str msg = MAL_SUCCEED;
+
+ if( b->tmosaic == NULL)
+ throw(MAL,"mosaic.layout","Compression heap missing");
+
+ MOStask task = {0};
+ MOSinit(&task,b);
+ MOSinitializeScan(&task, b);
+
+ MosaicLayout layout = {
+ .bsn = bbsn,
+ .tech = btech,
+ .count = bcount,
+ .input = binput,
+ .output = boutput,
+ .properties = bproperties
+ };
+
+ if ((msg = MOSlayout_hdr(&task, &layout)) != MAL_SUCCEED) {
+ return msg;
+ }
+
+ switch(ATOMbasetype(task.type)){
+ case TYPE_bte: return MOSlayout_bte(&task, &layout);
+ case TYPE_sht: return MOSlayout_sht(&task, &layout);
+ case TYPE_int: return MOSlayout_int(&task, &layout);
+ case TYPE_lng: return MOSlayout_lng(&task, &layout);
+ case TYPE_flt: return MOSlayout_flt(&task, &layout);
+ case TYPE_dbl: return MOSlayout_dbl(&task, &layout);
+#ifdef HAVE_HGE
+ case TYPE_hge: return MOSlayout_hge(&task, &layout);
+#endif
+ default:
+ // Unknown type. Should not happen.
+ assert(0);
+ }
+
+ return MAL_SUCCEED;
+}
static
str MOSestimate(MOStask* task, BAT* estimates, size_t* compressed_size) {
diff --git a/monetdb5/modules/mosaic/mosaic.h b/monetdb5/modules/mosaic/mosaic.h
--- a/monetdb5/modules/mosaic/mosaic.h
+++ b/monetdb5/modules/mosaic/mosaic.h
@@ -199,6 +199,17 @@ typedef struct _MosaicEstimation {
BUN* max_compression_length;
} MosaicEstimation;
+
+
+typedef struct _MosaicLayout {
+ BAT *bsn;
+ BAT *tech;
+ BAT *count;
+ BAT *input;
+ BAT *output;
+ BAT *properties;
+} MosaicLayout;
+
#define GET_PADDING(blk, METHOD, TPE) (((MOSBlockHeaderTpe(METHOD, TPE)*)
(blk))->padding)
#define ALIGN_BLOCK_HEADER(task, METHOD, TPE)\
@@ -237,7 +248,7 @@ mal_export str MOSselect2nil(bat *ret, c
mal_export str MOSthetaselect(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
mal_export str MOSprojection(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
mal_export str MOSjoin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
-mal_export str MOSlayout(BAT *b, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties);
+mal_export str MOSlayout(BAT *b, BAT *bbsn, BAT *btech, BAT *bcount, BAT
*binput, BAT *boutput, BAT *bproperties);
mal_export str MOSAnalysis(BAT *b, BAT *btech, BAT *output, BAT *factor, BAT
*compress, BAT *decompress, str compressions);
#endif /* _MOSLIST_H */
diff --git a/monetdb5/modules/mosaic/mosaic_delta.c
b/monetdb5/modules/mosaic/mosaic_delta.c
--- a/monetdb5/modules/mosaic/mosaic_delta.c
+++ b/monetdb5/modules/mosaic/mosaic_delta.c
@@ -39,30 +39,6 @@ bool MOStypes_delta(BAT* b) {
}
#define BitVectorSize(CNT, BITS) wordaligned(((CNT) * (BITS) / CHAR_BIT) + (
((CNT) * (BITS)) % CHAR_BIT != 0 ), BitVectorChunk)
-void
-MOSlayout_delta(MOStask* task, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties)
-{
- MosaicBlk blk = task->blk;
- lng cnt = MOSgetCnt(blk), input=0, output= 0;
-
- input = cnt * ATOMsize(task->type);
- switch(ATOMbasetype(task->type)){
- case TYPE_bte: output = wordaligned(MosaicBlkSize + sizeof(bte) +
MOSgetCnt(blk)-1,bte); break ;
- case TYPE_sht: output = wordaligned(MosaicBlkSize + sizeof(sht) +
MOSgetCnt(blk)-1,sht); break ;
- case TYPE_int: output = wordaligned(MosaicBlkSize + sizeof(int) +
MOSgetCnt(blk)-1,int); break ;
- case TYPE_lng: output = wordaligned(MosaicBlkSize + sizeof(lng) +
MOSgetCnt(blk)-1,lng); break ;
-#ifdef HAVE_HGE
- case TYPE_hge: output = wordaligned(MosaicBlkSize + sizeof(hge) +
MOSgetCnt(blk)-1,hge); break ;
-#endif
- }
- if( BUNappend(btech, "delta", false) != GDK_SUCCEED ||
- BUNappend(bcount, &cnt, false) != GDK_SUCCEED ||
- BUNappend(binput, &input, false) != GDK_SUCCEED ||
- BUNappend(boutput, &output, false) != GDK_SUCCEED ||
- BUNappend(bproperties, "", false) != GDK_SUCCEED)
- return;
-}
-
#define METHOD delta
#define METHOD_TEMPLATES_INCLUDE MAKE_TEMPLATES_INCLUDE_FILE(METHOD)
@@ -86,6 +62,27 @@ MOSlayout_delta(MOStask* task, BAT *btec
#endif
#undef COMPRESSION_DEFINITION
+#define LAYOUT_DEFINITION
+#include METHOD_TEMPLATES_INCLUDE
+#define TPE bte
+#include METHOD_TEMPLATES_INCLUDE
+#undef TPE
+#define TPE sht
+#include METHOD_TEMPLATES_INCLUDE
+#undef TPE
+#define TPE int
+#include METHOD_TEMPLATES_INCLUDE
+#undef TPE
+#define TPE lng
+#include METHOD_TEMPLATES_INCLUDE
+#undef TPE
+#ifdef HAVE_HGE
+#define TPE hge
+#include METHOD_TEMPLATES_INCLUDE
+#undef TPE
+#endif
+#undef LAYOUT_DEFINITION
+
#define TPE bte
#include "mosaic_select_template.h"
#include "mosaic_projection_template.h"
diff --git a/monetdb5/modules/mosaic/mosaic_delta.h
b/monetdb5/modules/mosaic/mosaic_delta.h
--- a/monetdb5/modules/mosaic/mosaic_delta.h
+++ b/monetdb5/modules/mosaic/mosaic_delta.h
@@ -20,7 +20,6 @@
#include "gdk_bitvector.h"
bool MOStypes_delta(BAT* b);
-mal_export void MOSlayout_delta(MOStask* task, BAT *btech, BAT *bcount, BAT
*binput, BAT *boutput, BAT *bproperties);
#define MosaicBlkHeader_DEF_delta(TPE)\
typedef struct {\
diff --git a/monetdb5/modules/mosaic/mosaic_delta_templates.h
b/monetdb5/modules/mosaic/mosaic_delta_templates.h
--- a/monetdb5/modules/mosaic/mosaic_delta_templates.h
+++ b/monetdb5/modules/mosaic/mosaic_delta_templates.h
@@ -196,3 +196,16 @@ MOSprojectionloop_SIGNATURE(delta, TPE,
task->src = (char*) bt;
}
#endif
+
+#ifdef LAYOUT_DEFINITION
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
