Changeset: 882720b4baf5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=882720b4baf5
Added Files:
monetdb5/modules/mosaic/mosaic_delta_template.h
monetdb5/modules/mosaic/mosaic_frame_template.h
monetdb5/modules/mosaic/mosaic_linear_template.h
monetdb5/modules/mosaic/mosaic_prefix_template.h
monetdb5/modules/mosaic/mosaic_raw_template.h
monetdb5/modules/mosaic/mosaic_runlength_template.h
Modified Files:
monetdb5/modules/mosaic/Makefile.ag
monetdb5/modules/mosaic/mosaic.h
monetdb5/modules/mosaic/mosaic_delta.c
monetdb5/modules/mosaic/mosaic_frame.c
monetdb5/modules/mosaic/mosaic_linear.c
monetdb5/modules/mosaic/mosaic_prefix.c
monetdb5/modules/mosaic/mosaic_prefix.h
monetdb5/modules/mosaic/mosaic_raw.c
monetdb5/modules/mosaic/mosaic_runlength.c
Branch: mosaic
Log Message:
Refactor various compression macro's to template functions.
diffs (truncated from 1783 to 300 lines):
diff --git a/monetdb5/modules/mosaic/Makefile.ag
b/monetdb5/modules/mosaic/Makefile.ag
--- a/monetdb5/modules/mosaic/Makefile.ag
+++ b/monetdb5/modules/mosaic/Makefile.ag
@@ -25,15 +25,15 @@ lib_mosaic = {
mosaic_select.h mosaic_select_template.h
mosaic_select_template2.h \
mosaic_projection.h \
mosaic_join.h \
- mosaic_raw.c mosaic_raw.h \
- mosaic_runlength.c mosaic_runlength.h \
+ mosaic_raw.c mosaic_raw.h mosaic_raw_template.h \
+ mosaic_runlength.c mosaic_runlength.h
mosaic_runlength_template.h \
mosaic_dict.c mosaic_dict.h \
mosaic_dict256.c mosaic_dict256.h \
mosaic_dictionary.h mosaic_dictionary_impl.h
mosaic_dictionary_prepare_context_impl.h \
- mosaic_delta.c mosaic_delta.h \
- mosaic_linear.c mosaic_linear.h \
- mosaic_frame.c mosaic_frame.h \
- mosaic_prefix.c mosaic_prefix.h \
+ mosaic_delta.c mosaic_delta.h mosaic_delta_template.h \
+ mosaic_linear.c mosaic_linear.h mosaic_linear_template.h \
+ mosaic_frame.c mosaic_frame.h mosaic_frame_template.h \
+ mosaic_prefix.c mosaic_prefix.h mosaic_prefix_template.h \
mosaic_private.h
}
diff --git a/monetdb5/modules/mosaic/mosaic.h b/monetdb5/modules/mosaic/mosaic.h
--- a/monetdb5/modules/mosaic/mosaic.h
+++ b/monetdb5/modules/mosaic/mosaic.h
@@ -71,7 +71,7 @@ typedef Heap *mosaic; // compressed data
#define Deltahge uhge
#endif
-#define DeltaTpe(TPE) Delta##TPE
+#define DeltaTpe(TPE) CONCAT2(Delta, TPE)
/* Use standard unsigned integer operations
* to avoid undefined behavior due to overflow's
@@ -91,7 +91,7 @@ typedef Heap *mosaic; // compressed data
#define IPhge uhge
#endif
-#define IPTpe(TPE) IP##TPE
+#define IPTpe(TPE) CONCAT2(IP, TPE)
#define METHOD_NOT_AVAILABLE -1
typedef struct MOSAICHEADER{
diff --git a/monetdb5/modules/mosaic/mosaic_delta.c
b/monetdb5/modules/mosaic/mosaic_delta.c
--- a/monetdb5/modules/mosaic/mosaic_delta.c
+++ b/monetdb5/modules/mosaic/mosaic_delta.c
@@ -39,34 +39,6 @@ bool MOStypes_delta(BAT* b) {
}
#define BitVectorSize(CNT, BITS) wordaligned(((CNT) * (BITS) / CHAR_BIT) + (
((CNT) * (BITS)) % CHAR_BIT != 0 ), BitVectorChunk)
-#define MOSadvance_DEF(TPE)\
-MOSadvance_SIGNATURE(delta, TPE)\
-{\
- MOSBlockHeaderTpe(delta, TPE)* parameters = (MOSBlockHeaderTpe(delta,
TPE)*) (task)->blk;\
- BUN cnt = MOSgetCnt(task->blk);\
-\
- assert(cnt > 0);\
- assert(MOSgetTag(task->blk) == MOSAIC_DELTA);\
-\
- task->start += (oid) cnt;\
-\
- char* blk = (char*)task->blk;\
- blk += sizeof(MOSBlockHeaderTpe(delta, TPE));\
- blk += BitVectorSize(cnt, parameters->bits);\
- blk += GET_PADDING(task->blk, delta, TPE);\
-\
- task->blk = (MosaicBlk) blk;\
-\
-}
-
-MOSadvance_DEF(bte)
-MOSadvance_DEF(sht)
-MOSadvance_DEF(int)
-MOSadvance_DEF(lng)
-#ifdef HAVE_HGE
-MOSadvance_DEF(hge)
-#endif
-
void
MOSlayout_delta(MOStask* task, BAT *btech, BAT *bcount, BAT *binput, BAT
*boutput, BAT *bproperties)
{
@@ -91,155 +63,22 @@ MOSlayout_delta(MOStask* task, BAT *btec
return;
}
-#define determineDeltaParameters(PARAMETERS, SRC, LIMIT, TPE) \
-do {\
- TPE *val = SRC;\
- bte bits = 1;\
- unsigned int i;\
- DeltaTpe(TPE) unsigned_delta = 0;\
- TPE prev_val;\
- (PARAMETERS).init = *val;\
-\
- for(i = 1; i < LIMIT; i++){\
- prev_val = *val++;\
- DeltaTpe(TPE) current_unsigned_delta;\
- if (*val > prev_val) {\
- current_unsigned_delta = GET_DELTA(TPE, *val,
prev_val);\
- }\
- else {\
- current_unsigned_delta = GET_DELTA(TPE, prev_val,
*val);\
- }\
-\
- if (current_unsigned_delta > unsigned_delta) {\
- bte current_bits = bits;\
- while (current_unsigned_delta > ((DeltaTpe(TPE))(-1))
>> (sizeof(DeltaTpe(TPE)) * CHAR_BIT - current_bits) ) {\
- /*keep track of number of BITS necessary to
store the difference*/\
- current_bits++;\
- }\
- int current_bits_with_sign_bit = current_bits + 1;\
- if ( (current_bits_with_sign_bit >= (int) ((sizeof(TPE)
* CHAR_BIT) / 2))\
- /*If we can from here on not compress better
then the half of the original data type, we give up. */\
- || (current_bits_with_sign_bit > (int)
sizeof(BitVectorChunk) * CHAR_BIT) ) {\
- /*TODO: this extra condition should be removed
once bitvector is extended to int64's*/\
- break;\
- }\
- bits = current_bits;\
- unsigned_delta = current_unsigned_delta;\
- }\
- }\
-\
- /*Add the additional sign bit to the bit count.*/\
- bits++;\
- (PARAMETERS).rec.cnt = i;\
- (PARAMETERS).bits = bits;\
-} while(0)
-
-#define MOSestimate_DEF(TPE) \
-MOSestimate_SIGNATURE(delta, TPE)\
-{\
- (void) previous;\
- current->is_applicable = true;\
- current->compression_strategy.tag = MOSAIC_DELTA;\
- TPE *src = getSrc(TPE, task);\
- BUN limit = task->stop - task->start > MOSAICMAXCNT? MOSAICMAXCNT:
task->stop - task->start;\
- MOSBlockHeaderTpe(delta, TPE) parameters;\
- determineDeltaParameters(parameters, src, limit, TPE);\
- assert(parameters.rec.cnt > 0);/*Should always compress.*/\
- current->uncompressed_size += (BUN) (parameters.rec.cnt * sizeof(TPE));\
- current->compressed_size += 2 * sizeof(MOSBlockHeaderTpe(delta, TPE)) +
wordaligned((parameters.rec.cnt * parameters.bits) / CHAR_BIT, lng);\
- current->compression_strategy.cnt = (unsigned int) parameters.rec.cnt;\
-\
- if (parameters.rec.cnt > *current->max_compression_length ) {\
- *current->max_compression_length = parameters.rec.cnt;\
- }\
-\
- return MAL_SUCCEED;\
-}
-
-MOSestimate_DEF(bte)
-MOSestimate_DEF(sht)
-MOSestimate_DEF(int)
-MOSestimate_DEF(lng)
+#define TPE bte
+#include "mosaic_delta_template.h"
+#undef TPE
+#define TPE sht
+#include "mosaic_delta_template.h"
+#undef TPE
+#define TPE int
+#include "mosaic_delta_template.h"
+#undef TPE
+#define TPE lng
+#include "mosaic_delta_template.h"
+#undef TPE
#ifdef HAVE_HGE
-MOSestimate_DEF(hge)
-#endif
-
-#define MOSpostEstimate_DEF(TPE)\
-MOSpostEstimate_SIGNATURE(delta, TPE)\
-{\
- (void) task;\
-}
-
-MOSpostEstimate_DEF(bte)
-MOSpostEstimate_DEF(sht)
-MOSpostEstimate_DEF(int)
-MOSpostEstimate_DEF(lng)
-#ifdef HAVE_HGE
-MOSpostEstimate_DEF(hge)
-#endif
-
-// rather expensive simple value non-compressed store
-#define MOScompress_DEF(TPE)\
-MOScompress_SIGNATURE(delta, TPE)\
-{\
- ALIGN_BLOCK_HEADER(task, delta, TPE);\
-\
- MosaicBlk blk = task->blk;\
- MOSsetTag(blk,MOSAIC_DELTA);\
- MOSsetCnt(blk, 0);\
- TPE *src = getSrc(TPE, task);\
- BUN i = 0;\
- BUN limit = estimate->cnt;\
- MOSBlockHeaderTpe(delta, TPE)* parameters = (MOSBlockHeaderTpe(delta,
TPE)*) (task)->blk;\
- determineDeltaParameters(*parameters, src, limit, TPE);\
- BitVector base = MOScodevectorDelta(task, TPE);\
- task->dst = (char*) base;\
- TPE pv = parameters->init; /*previous value*/\
- /*Initial delta is zero.*/\
- setBitVector(base, 0, parameters->bits, (BitVectorChunk) 0);\
- DeltaTpe(TPE) sign_mask = (DeltaTpe(TPE)) ((IPTpe(TPE)) 1) <<
(parameters->bits - 1);\
-\
- for(i = 1; i < MOSgetCnt(task->blk); i++) {\
- /*TODO: assert that delta's actually does not cause an
overflow. */\
- TPE cv = *++src; /*current value*/\
- DeltaTpe(TPE) delta = (DeltaTpe(TPE)) (cv > pv ? (IPTpe(TPE))
(cv - pv) : (IPTpe(TPE)) ((sign_mask) | (IPTpe(TPE)) (pv - cv)));\
- setBitVector(base, i, parameters->bits, (BitVectorChunk)
/*TODO: fix this once we have increased capacity of bitvector*/ delta);\
- pv = cv;\
- }\
- task->dst += BitVectorSize(i, parameters->bits);\
-}
-
-MOScompress_DEF(bte)
-MOScompress_DEF(sht)
-MOScompress_DEF(int)
-MOScompress_DEF(lng)
-#ifdef HAVE_HGE
-MOScompress_DEF(hge)
-#endif
-
-#define MOSdecompress_DEF(TPE) \
-MOSdecompress_SIGNATURE(delta, TPE)\
-{\
- MOSBlockHeaderTpe(delta, TPE)* parameters = (MOSBlockHeaderTpe(delta,
TPE)*) (task)->blk;\
- BUN lim = MOSgetCnt(task->blk);\
- ((TPE*)task->src)[0] = parameters->init; /*previous value*/\
- BitVector base = (BitVector) MOScodevectorDelta(task, TPE);\
- DeltaTpe(TPE) sign_mask = (DeltaTpe(TPE)) ((IPTpe(TPE)) 1) <<
(parameters->bits - 1);\
- DeltaTpe(TPE) acc = (DeltaTpe(TPE)) parameters->init /*unsigned
accumulating value*/;\
- BUN i;\
- for(i = 0; i < lim; i++) {\
- DeltaTpe(TPE) delta = getBitVector(base, i, parameters->bits);\
- ((TPE*)task->src)[i] = ACCUMULATE(acc, delta, sign_mask, TPE);\
- }\
- task->src += i * sizeof(TPE);\
-}
-
-MOSdecompress_DEF(bte)
-MOSdecompress_DEF(sht)
-MOSdecompress_DEF(int)
-MOSdecompress_DEF(lng)
-#ifdef HAVE_HGE
-MOSdecompress_DEF(hge)
+#define TPE hge
+#include "mosaic_delta_template.h"
+#undef TPE
#endif
#define scan_loop_delta(TPE, CANDITER_NEXT, TEST) \
diff --git a/monetdb5/modules/mosaic/mosaic_delta_template.h
b/monetdb5/modules/mosaic/mosaic_delta_template.h
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mosaic/mosaic_delta_template.h
@@ -0,0 +1,133 @@
+MOSadvance_SIGNATURE(delta, TPE)
+{
+ MOSBlockHeaderTpe(delta, TPE)* parameters = (MOSBlockHeaderTpe(delta,
TPE)*) (task)->blk;
+ BUN cnt = MOSgetCnt(task->blk);
+
+ assert(cnt > 0);
+ assert(MOSgetTag(task->blk) == MOSAIC_DELTA);
+
+ task->start += (oid) cnt;
+
+ char* blk = (char*)task->blk;
+ blk += sizeof(MOSBlockHeaderTpe(delta, TPE));
+ blk += BitVectorSize(cnt, parameters->bits);
+ blk += GET_PADDING(task->blk, delta, TPE);
+
+ task->blk = (MosaicBlk) blk;
+
+}
+
+static inline void CONCAT2(determineDeltaParameters, TPE)
+(MOSBlockHeaderTpe(delta, TPE)* parameters, TPE* src, BUN limit) {
+ TPE *val = src;
+ bte bits = 1;
+ unsigned int i;
+ DeltaTpe(TPE) unsigned_delta = 0;
+ TPE prev_val;
+ parameters->init = *val;
+
+ for(i = 1; i < limit; i++){
+ prev_val = *val++;
+ DeltaTpe(TPE) current_unsigned_delta;
+ if (*val > prev_val) {
+ current_unsigned_delta = GET_DELTA(TPE, *val, prev_val);
+ }
+ else {
+ current_unsigned_delta = GET_DELTA(TPE, prev_val, *val);
+ }
+
+ if (current_unsigned_delta > unsigned_delta) {
+ bte current_bits = bits;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
