Changeset: af90a750b234 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=af90a750b234
Modified Files:
monetdb5/modules/mosaic/mosaic.c
monetdb5/modules/mosaic/mosaic_frame.c
monetdb5/modules/mosaic/mosaic_frame.h
Branch: mosaic
Log Message:
Fix memory corruption error in frame compression.
diffs (184 lines):
diff --git a/monetdb5/modules/mosaic/mosaic.c b/monetdb5/modules/mosaic/mosaic.c
--- a/monetdb5/modules/mosaic/mosaic.c
+++ b/monetdb5/modules/mosaic/mosaic.c
@@ -512,7 +512,7 @@ MOScompressInternal(BAT* bsrc, const cha
break;
case MOSAIC_FRAME:
ALGODEBUG mnstr_printf(GDKstdout,
"MOScompress_frame\n");
- MOScompress_frame(task);
+ MOScompress_frame(task, estimate);
MOSupdateHeader(task);
MOSadvance_frame(task);
MOSnewBlk(task);
diff --git a/monetdb5/modules/mosaic/mosaic_frame.c
b/monetdb5/modules/mosaic/mosaic_frame.c
--- a/monetdb5/modules/mosaic/mosaic_frame.c
+++ b/monetdb5/modules/mosaic/mosaic_frame.c
@@ -51,17 +51,45 @@ bool MOStypes_frame(BAT* b) {
// we use longs as the basis for bit vectors
#define chunk_size(Task, Cnt) wordaligned(MosaicBlkSize + (Cnt *
Task->hdr->framebits)/8 + (((Cnt * Task->hdr->framebits) %8) != 0), lng)
+typedef struct _FrameParameters_t {
+ MosaicBlkRec base;
+ int bits;
+ union {
+ bte minbte;
+ sht minsht;
+ int minint;
+ lng minlng;
+ oid minoid;
+#ifdef HAVE_HGE
+ hge minhge;
+#endif
+ } min;
+ union {
+ bte maxbte;
+ sht maxsht;
+ int maxint;
+ lng maxlng;
+ oid maxoid;
+#ifdef HAVE_HGE
+ hge maxhge;
+#endif
+ } max;
+
+} MosaicBlkHeader_frame_t;
+
+#define toEndOfBitVector(CNT, BITS) wordaligned(((CNT) * (BITS) / CHAR_BIT) +
( ((CNT) * (BITS)) % CHAR_BIT != 0 ), lng)
+
void
MOSadvance_frame(MOStask task)
{
- int *dst = (int*) (((char*) task->blk) + MosaicBlkSize);
- long cnt = MOSgetCnt(task->blk);
- long bytes;
+ MosaicBlkHeader_frame_t* parameters = (MosaicBlkHeader_frame_t*)
(task)->blk;
+ int *dst = (int*) (((char*) task->blk) +
wordaligned(sizeof(MosaicBlkHeader_frame_t), unsigned int));
+ long cnt = parameters->base.cnt;
+ long bytes = toEndOfBitVector(cnt, parameters->bits);
assert(cnt > 0);
task->start += (oid) cnt;
- bytes = (cnt * task->hdr->framebits)/8 + (((cnt *
task->hdr->framebits) %8) != 0) + sizeof(ulng);
- task->blk = (MosaicBlk) (((char*) dst) + wordaligned(bytes, lng));
+ task->blk = (MosaicBlk) (((char*) dst) + bytes);
}
void
@@ -88,33 +116,7 @@ MOSskip_frame(MOStask task)
task->blk = 0; // ENDOFLIST
}
-typedef struct _FrameParameters_t {
- MosaicBlkRec base;
- int bits;
- union {
- bte minbte;
- sht minsht;
- int minint;
- lng minlng;
- oid minoid;
-#ifdef HAVE_HGE
- hge minhge;
-#endif
- } min;
- union {
- bte maxbte;
- sht maxsht;
- int maxint;
- lng maxlng;
- oid maxoid;
-#ifdef HAVE_HGE
- hge maxhge;
-#endif
- } max;
-
-} MosaicBlkHeader_frame_t;
-
-#define MOScodevectorFrame(Task) (((char*) (Task)->blk)+
wordaligned(sizeof(MosaicBlkHeader_frame_t), BitVector))
+#define MOScodevectorFrame(Task) (((char*) (Task)->blk)+
wordaligned(sizeof(MosaicBlkHeader_frame_t), unsigned int))
/* Use ternary operator because (in theory) we have to be careful not to get
overflow's*/\
#define GET_DELTA_FOR_SIGNED_TYPE(DELTA_TPE, max, min) (min < 0? max <
0?(DELTA_TPE) (max - min) : (DELTA_TPE)(max) + (DELTA_TPE)(-1 * min) :
(DELTA_TPE) (max - min))
@@ -129,17 +131,19 @@ do {\
min = *val;\
/*TODO: add additional loop to find best bit wise upper bound*/\
for(i = 0; i < LIMIT; i++, val++){\
+ TPE current_max = max;\
+ TPE current_min = min;\
bool evaluate_bits = false;\
- if (*val > max) {\
- max = *val;\
+ if (*val > current_max) {\
+ current_max = *val;\
evaluate_bits = true;\
}\
- if (*val < min) {\
- min = *val;\
+ if (*val < current_min) {\
+ current_min = *val;\
evaluate_bits = true;\
}\
if (evaluate_bits) {\
- DELTA_TPE width = GET_DELTA(DELTA_TPE, max, min);\
+ DELTA_TPE width = GET_DELTA(DELTA_TPE, current_max,
current_min);\
int current_bits = bits;\
while (width > ((DELTA_TPE)(-1)) >> (sizeof(DELTA_TPE)
* CHAR_BIT - current_bits) ) {/*keep track of number of BITS necessary to store
difference*/\
current_bits++;\
@@ -150,6 +154,8 @@ do {\
/*If we can from here on not compress better
then the half of the original data type, we give up. */\
break;\
}\
+ max = current_max;\
+ min = current_min;\
bits = current_bits;\
}\
}\
@@ -198,7 +204,7 @@ do {\
TPE *src = getSrc(TPE, (TASK));\
TPE delta;\
BUN i = 0;\
- BUN limit = (TASK)->stop - (TASK)->start > MOSAICMAXCNT? MOSAICMAXCNT:
(TASK)->stop - (TASK)->start;\
+ BUN limit = estimate->cnt;\
BitVector base;\
MosaicBlkHeader_frame_t* parameters = (MosaicBlkHeader_frame_t*)
((TASK))->blk;\
determineFrameParameters(*parameters, src, limit, TPE, DELTA_TPE,
GET_DELTA);\
@@ -209,23 +215,23 @@ do {\
delta = *src - parameters->min.min##TPE;\
setBitVector(base, i, parameters->bits, (unsigned int) /*TODO:
fix this once we have increased capacity of bitvector*/ delta);\
}\
- (TASK)->dst += wordaligned((i * parameters->bits / CHAR_BIT) + ( (i *
parameters->bits) % CHAR_BIT ) != 0, lng);\
+ (TASK)->dst += toEndOfBitVector(i, parameters->bits);\
} while(0)
void
-MOScompress_frame(MOStask task)
+MOScompress_frame(MOStask task, MosaicBlkRec* estimate)
{
MosaicBlk blk = task->blk;
MOSsetTag(blk,MOSAIC_FRAME);
MOSsetCnt(blk, 0);
- switch(ATOMbasetype(task->type)){
+ switch(ATOMbasetype(task->type)) {
case TYPE_bte: FRAMEcompress(task, bte, ulng,
GET_DELTA_FOR_SIGNED_TYPE); break;
case TYPE_sht: FRAMEcompress(task, sht, ulng,
GET_DELTA_FOR_SIGNED_TYPE); break;
case TYPE_int: FRAMEcompress(task, int, ulng,
GET_DELTA_FOR_SIGNED_TYPE); break;
case TYPE_lng: FRAMEcompress(task, lng, ulng,
GET_DELTA_FOR_SIGNED_TYPE); break;
- case TYPE_oid: FRAMEcompress(task, oid, oid,
GET_DELTA_FOR_UNSIGNED_TYPE); break;
+ case TYPE_oid: FRAMEcompress(task, oid, oid,
GET_DELTA_FOR_UNSIGNED_TYPE); break;
#ifdef HAVE_HGE
case TYPE_hge: FRAMEcompress(task, hge, uhge,
GET_DELTA_FOR_SIGNED_TYPE); break;
#endif
diff --git a/monetdb5/modules/mosaic/mosaic_frame.h
b/monetdb5/modules/mosaic/mosaic_frame.h
--- a/monetdb5/modules/mosaic/mosaic_frame.h
+++ b/monetdb5/modules/mosaic/mosaic_frame.h
@@ -24,7 +24,7 @@ mal_export void MOSlayout_frame(MOStask
mal_export void MOSadvance_frame(MOStask task);
mal_export void MOSskip_frame(MOStask task);
mal_export str MOSestimate_frame(MOStask task, MosaicEstimation* current,
const MosaicEstimation* previous);
-mal_export void MOScompress_frame(MOStask task);
+mal_export void MOScompress_frame(MOStask task, MosaicBlkRec* estimate);
mal_export void MOSdecompress_frame(MOStask task);
mal_export str MOSselect_frame( MOStask task, void *low, void *hgh, bit *li,
bit *hi, bit *anti);
mal_export str MOSthetaselect_frame( MOStask task, void *val, str oper);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
