Changeset: 59688ca0184f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=59688ca0184f
Modified Files:
monetdb5/modules/mal/Tests/mosaic_mix.mal
monetdb5/modules/mal/mosaic_dictionary.c
monetdb5/modules/mal/mosaic_prefix.c
Branch: mosaic
Log Message:
Fix bit stuffing.
diffs (207 lines):
diff --git a/monetdb5/modules/mal/Tests/mosaic_mix.mal
b/monetdb5/modules/mal/Tests/mosaic_mix.mal
--- a/monetdb5/modules/mal/Tests/mosaic_mix.mal
+++ b/monetdb5/modules/mal/Tests/mosaic_mix.mal
@@ -96,7 +96,7 @@ bat.append(b,9);
b:= bat.append(b,b);
io.print(b);
-x:= mosaic.compress(b,"runlength prefix delta dictionary linear");
+x:= mosaic.compress(b);
mosaic.dump(x);
z:= mosaic.decompress(x);
io.print(z);
diff --git a/monetdb5/modules/mal/mosaic_dictionary.c
b/monetdb5/modules/mal/mosaic_dictionary.c
--- a/monetdb5/modules/mal/mosaic_dictionary.c
+++ b/monetdb5/modules/mal/mosaic_dictionary.c
@@ -255,16 +255,17 @@ MOSestimate_dictionary(Client cntxt, MOS
}
// insert a series of values into the compressor block using dictionary
-#define dictcompress()\
-cid = (i * hdr->bits)/64;\
-lshift= 64 -((i * hdr->bits) % 64) ;\
-if ( lshift > hdr->bits){\
- base[cid]= base[cid] | (((unsigned long)j) << (lshift-hdr->bits));\
-}else{ \
- rshift= 64 - ((i+1) * hdr->bits) % 64;\
- base[cid]= base[cid] | (((unsigned long)j) >> (hdr->bits-lshift));\
- base[cid+1]= 0 | (((unsigned long)j) << rshift);\
-}
+#define dictcompress(Vector,I,Bits,Value)\
+{int cid,lshift,rshift;\
+ cid = (I * Bits)/64;\
+ lshift= 63 -((I * Bits) % 64) ;\
+ if ( lshift >= Bits){\
+ Vector[cid]= Vector[cid] | (((unsigned long) Value) << (lshift-
Bits));\
+ }else{ \
+ rshift= 63 - ((I+1) * Bits) % 64;\
+ Vector[cid]= Vector[cid] | (((unsigned long) Value) >>
(Bits-lshift));\
+ Vector[cid+1]= 0 | (((unsigned long) Value) << rshift);\
+}}
#define DICTcompress(TPE)\
{ TPE *val = (TPE*)task->src;\
@@ -279,7 +280,7 @@ if ( lshift > hdr->bits){\
break;\
else {\
MOSincCnt(blk,1);\
- dictcompress();\
+ dictcompress(base,i,hdr->bits,j);\
}\
}\
assert(i);\
@@ -321,18 +322,18 @@ MOScompress_dictionary(Client cntxt, MOS
base[0]=0;
for(i =0; i<limit; i++, val++){
MOSfind(j,*val,0,hdr->dictsize);
- //mnstr_printf(cntxt->fdout,"compress %d %d
bits %d\n",*val,j,hdr->bits);
+ //mnstr_printf(cntxt->fdout,"compress
["BUNFMT"] val %d index %d bits %d\n",i, *val,j,hdr->bits);
if( j == hdr->dictsize || dict[j] != *val )
break;
else {
MOSincCnt(blk,1);
cid = (i * hdr->bits)/64;
- lshift= 64 -((i * hdr->bits) % 64) ;
- if ( lshift > hdr->bits){
+ lshift= 63 -((i * hdr->bits) % 64) ;
+ if ( lshift >= hdr->bits){
base[cid]= base[cid] |
(((unsigned long)j) << (lshift-hdr->bits));
//mnstr_printf(cntxt->fdout,"[%d] shift %d rbits %d \n",cid, lshift, hdr->bits);
}else{
- rshift= 64 - ((i+1) *
hdr->bits) % 64;
+ rshift= 63 - ((i+1) *
hdr->bits) % 64;
base[cid]= base[cid] |
(((unsigned long)j) >> (hdr->bits-lshift));
base[cid+1]= 0 | (((unsigned
long)j) << rshift);
//mnstr_printf(cntxt->fdout,"[%d] shift %d %d val %o %o\n", cid, lshift, rshift,
@@ -349,14 +350,14 @@ MOScompress_dictionary(Client cntxt, MOS
// the inverse operator, extend the src
#define dictdecompress(I)\
cid = (I * hdr->bits)/64;\
-lshift= 64 -((I * hdr->bits) % 64) ;\
-if ( lshift >hdr->bits){\
+lshift= 63 -((I * hdr->bits) % 64) ;\
+if ( lshift >= hdr->bits){\
j = (base[cid]>> (lshift-hdr->bits)) & ((unsigned long)hdr->mask);\
}else{ \
- rshift= 64 - ((I+1) * hdr->bits) % 64;\
+ rshift= 63 - ((I+1) * hdr->bits) % 64;\
m1 = (base[cid] & ( ((unsigned long)hdr->mask) >> (hdr->bits-lshift)));\
m2 = base[cid+1] >>rshift;\
- j= 0 | (m1 <<(hdr->bits-lshift)) | m2;\
+ j= ((m1 <<(hdr->bits-lshift)) | m2) & 0377;\
}
#define DICTdecompress(TPE)\
@@ -376,7 +377,7 @@ MOSdecompress_dictionary(Client cntxt, M
MosaicBlk blk = task->blk;
MosaicHdr hdr = task->hdr;
BUN i;
- bte j,m1,m2;
+ unsigned int j,m1,m2;
int cid, lshift, rshift;
unsigned long *base;
(void) cntxt;
@@ -399,15 +400,15 @@ MOSdecompress_dictionary(Client cntxt, M
for(i = 0; i < lim; i++){
cid = (i * hdr->bits)/64;
- lshift= 64 -((i * hdr->bits) % 64) ;
- if ( lshift >hdr->bits){
+ lshift= 63 -((i * hdr->bits) % 64) ;
+ if ( lshift >= hdr->bits){
j = (base[cid]>> (lshift-hdr->bits)) &
((unsigned long)hdr->mask);
//mnstr_printf(cntxt->fdout,"[%d]
lshift %d m %d\n", cid, lshift,m1);
}else{
- rshift= 64 - ((i+1) * hdr->bits) % 64;
+ rshift= 63 - ((i+1) * hdr->bits) % 64;
m1 = (base[cid] & ( ((unsigned
long)hdr->mask) >> (hdr->bits-lshift)));
m2 = (base[cid+1] >>rshift);
- j= 0 | (m1 <<(hdr->bits-lshift)) | m2;
+ j= ((m1 <<(hdr->bits-lshift)) | m2) &
0377;\
//mnstr_printf(cntxt->fdout,"[%d] shift
%d %d cid %lo %lo val %o %o\n", cid, lshift, rshift,base[cid],base[cid+1], m1,
m2);
}
((int*)task->src)[i] = dict[j];
diff --git a/monetdb5/modules/mal/mosaic_prefix.c
b/monetdb5/modules/mal/mosaic_prefix.c
--- a/monetdb5/modules/mal/mosaic_prefix.c
+++ b/monetdb5/modules/mal/mosaic_prefix.c
@@ -151,7 +151,6 @@ MOSestimate_prefix(Client cntxt, MOStask
int bits, size;
lng store;
(void) cntxt;
- (void) bits;
size = ATOMsize(task->type);
if( ATOMstorage(task->type == TYPE_str))
@@ -249,15 +248,15 @@ MOSestimate_prefix(Client cntxt, MOStask
}
#define compress(Vector,I, Bits, Value) \
-{ int cell, lshift, rshift;\
- cell = (I * Bits)/64;\
- lshift= 64 -((I * Bits) % 64) ;\
- if ( lshift > Bits){\
- Vector[cell]= Vector[cell] | (((unsigned long)Value) <<
(lshift-Bits));\
+{ int cid, lshift, rshift;\
+ cid = (I * Bits)/64;\
+ lshift= 63 -((I * Bits) % 64) ;\
+ if ( lshift >= Bits){\
+ Vector[cid]= Vector[cid] | (((unsigned long)Value) <<
(lshift-Bits));\
}else{ \
- rshift= 64 - ((I+1) * Bits) % 64;\
- Vector[cell]= Vector[cell] | (((unsigned long)Value) >>
(Bits-lshift));\
- Vector[cell+1]= 0 | (((unsigned long)Value) << rshift);\
+ rshift= 63 - ((I+1) * Bits) % 64;\
+ Vector[cid]= Vector[cid] | (((unsigned long)Value) >>
(Bits-lshift));\
+ Vector[cid+1]= 0 | (((unsigned long)Value) << rshift);\
}\
}
@@ -379,12 +378,12 @@ MOScompress_prefix(Client cntxt, MOStask
m = *w & (~mask); // residu
//mnstr_printf(cntxt->fdout,"compress %d residu
%d %o\n",*w,m,m);
cell = (i * rbits)/64;
- lshift= 64 -((i * rbits) % 64) ;
- if ( lshift > rbits){
+ lshift= 63 -((i * rbits) % 64) ;
+ if ( lshift >= rbits){
base[cell]= base[cell] | (((long)m) <<
(lshift-rbits));
//mnstr_printf(cntxt->fdout,"[%d] shift
%d rbits %d cell %o\n",cell, lshift, rbits, base[cell]);
}else{
- rshift= 64 - ((i+1) * rbits) % 64;
+ rshift= 63 - ((i+1) * rbits) % 64;
base[cell]= base[cell] | (((long)m) >>
(rbits-lshift));
base[cell+1]= 0 | (((long)m) <<
rshift);
//mnstr_printf(cntxt->fdout,"[%d] shift
%d %d cell %o %o val %o %o\n", cell, lshift, rshift,
@@ -440,12 +439,12 @@ MOScompress_prefix(Client cntxt, MOStask
#define decompress(Vector,I)\
{ int cell = (I * rbits)/64, lshift,rshift;\
- lshift= 64 -((I * rbits) % 64) ;\
- if ( lshift > (int)rbits){\
+ lshift= 63 -((I * rbits) % 64) ;\
+ if ( lshift >= (int)rbits){\
m1 = (Vector[cell]>> (lshift-rbits)) & ((unsigned long) m);\
v = val | m1;\
}else{ \
- rshift= 64 - ((I+1) * rbits) % 64;\
+ rshift= 63 - ((I+1) * rbits) % 64;\
m1 =(Vector[cell] & (((unsigned long)m) >> (rbits-lshift)));\
m2 = Vector[cell+1] >>rshift;\
v= val | (m1 <<(rbits-lshift)) | m2;\
@@ -519,13 +518,13 @@ MOSdecompress_prefix(Client cntxt, MOSta
//mnstr_printf(cntxt->fdout,"decompress rbits %d mask
%o val %d\n",rbits,m,val);
for(i = 0; i < lim; i++){
cell = (i * rbits)/64;
- lshift= 64 -((i * rbits) % 64) ;
- if ( lshift >rbits){
+ lshift= 63 -((i * rbits) % 64) ;
+ if ( lshift >= rbits){
m1 = (base[cell]>> (lshift-rbits)) &
((long)m);
//mnstr_printf(cntxt->fdout,"[%d]cell
%o lshift %d m %d\n", cell, base[cell],lshift,m1);
v = val | m1;
}else{
- rshift= 64 - ((i+1) * rbits) % 64;
+ rshift= 63 - ((i+1) * rbits) % 64;
m1 =(base[cell] & ( ((long)m) >>
(rbits-lshift)));
m2 = base[cell+1] >>rshift;
v= val | (m1 <<(rbits-lshift)) | m2;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
