Changeset: 050594303a14 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=050594303a14
Modified Files:
monetdb5/modules/mal/Tests/mosaic_rle.mal
monetdb5/modules/mal/mosaic.c
monetdb5/modules/mal/mosaic.h
monetdb5/modules/mal/mosaic_delta.c
monetdb5/modules/mal/mosaic_dict.c
monetdb5/modules/mal/mosaic_linear.c
monetdb5/modules/mal/mosaic_none.c
monetdb5/modules/mal/mosaic_rle.c
monetdb5/modules/mal/mosaic_zone.c
sql/test/mosaic/Tests/All
sql/test/mosaic/Tests/compression.sql
sql/test/mosaic/Tests/compressionRLE.sql
sql/test/mosaic/Tests/compressionSUM.sql
Branch: mosaic
Log Message:
Fixed type alignment and rle errors
SQL tests should run with sequential-pipe for the time being
diffs (truncated from 496 to 300 lines):
diff --git a/monetdb5/modules/mal/Tests/mosaic_rle.mal
b/monetdb5/modules/mal/Tests/mosaic_rle.mal
--- a/monetdb5/modules/mal/Tests/mosaic_rle.mal
+++ b/monetdb5/modules/mal/Tests/mosaic_rle.mal
@@ -62,3 +62,102 @@ x:= mosaic.compress(b,"rle");
#mosaic.dump(x);
z:= mosaic.decompress(x);
io.print(z);
+# OID list compressions
+
+
+o := 19531025:int;
+
+f:= bat.new(:oid,:flt);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.314);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.316);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+bat.append(f,0.317);
+
+
+io.print(f);
+xf:= mosaic.compress(f,"rle");
+zf:= mosaic.decompress(xf);
+io.print(zf);
diff --git a/monetdb5/modules/mal/mosaic.c b/monetdb5/modules/mal/mosaic.c
--- a/monetdb5/modules/mal/mosaic.c
+++ b/monetdb5/modules/mal/mosaic.c
@@ -171,7 +171,7 @@ MOScompressInternal(Client cntxt, int *r
lng percentage=0, perc;
int filter[MOSAIC_METHODS];
- if( properties && !strstr(properties,"compress"))
+ if( properties && !strstr(properties,"compressed"))
for( i = 0; i< MOSAIC_METHODS; i++)
filter[i]= strstr(properties,filtername[i]) != 0;
else
diff --git a/monetdb5/modules/mal/mosaic.h b/monetdb5/modules/mal/mosaic.h
--- a/monetdb5/modules/mal/mosaic.h
+++ b/monetdb5/modules/mal/mosaic.h
@@ -78,11 +78,14 @@ typedef lng *MosaicBlk;
#define MOSzone (((lng)MOSAIC_ZONE) <<56)
#define MOSeol (((lng)MOSAIC_EOL) <<56)
-#define wordaligned(SZ) \
- ((SZ) + ((SZ) % sizeof(int)? sizeof(int) - ((SZ)%sizeof(int)) : 0))
+/* Memory word alignement is type and platform dependent.
+ * We use an encoding that fits the column type requirements
+ */
+#define wordaligned(SZ,TYPE) \
+ ((SZ) + ((SZ) % sizeof(int)? sizeof(int) - ((SZ)%sizeof(TYPE)) : 0))
-#define MosaicHdrSize wordaligned(sizeof(struct MOSAICHEADER))
-#define MosaicBlkSize wordaligned(sizeof(lng))
+#define MosaicHdrSize wordaligned(sizeof(struct MOSAICHEADER),lng)
+#define MosaicBlkSize wordaligned(sizeof(lng),lng)
typedef struct MOSTASK{
diff --git a/monetdb5/modules/mal/mosaic_delta.c
b/monetdb5/modules/mal/mosaic_delta.c
--- a/monetdb5/modules/mal/mosaic_delta.c
+++ b/monetdb5/modules/mal/mosaic_delta.c
@@ -38,18 +38,18 @@ MOSadvance_delta(MOStask task)
{
MosaicBlk blk = task->blk;
switch(task->type){
- case TYPE_sht: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(sht) + MOScnt(blk)-1)); break ;
- case TYPE_int: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(int) + MOScnt(blk)-1)); break ;
- case TYPE_oid: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(oid) + MOScnt(blk)-1)); break ;
- case TYPE_wrd: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(wrd) + MOScnt(blk)-1)); break ;
- case TYPE_lng: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(lng) + MOScnt(blk)-1)); break ;
+ case TYPE_sht: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(sht) + MOScnt(blk)-1,sht)); break ;
+ case TYPE_int: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(int) + MOScnt(blk)-1,int)); break ;
+ case TYPE_oid: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(oid) + MOScnt(blk)-1,oid)); break ;
+ case TYPE_wrd: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(wrd) + MOScnt(blk)-1,wrd)); break ;
+ case TYPE_lng: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(lng) + MOScnt(blk)-1,lng)); break ;
default:
if( task->type == TYPE_timestamp)
- task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(timestamp) + MOScnt(blk)-1));
+ task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(timestamp) + MOScnt(blk)-1,timestamp));
if( task->type == TYPE_date)
- task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(date) + MOScnt(blk)-1));
+ task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(date) + MOScnt(blk)-1,date));
if( task->type == TYPE_daytime)
- task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(daytime) + MOScnt(blk)-1));
+ task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(daytime) + MOScnt(blk)-1,daytime));
}
}
diff --git a/monetdb5/modules/mal/mosaic_dict.c
b/monetdb5/modules/mal/mosaic_dict.c
--- a/monetdb5/modules/mal/mosaic_dict.c
+++ b/monetdb5/modules/mal/mosaic_dict.c
@@ -62,6 +62,8 @@ MOSdump_dict(Client cntxt, MOStask task)
default:
if( task->type == TYPE_date){
}
+ if( task->type == TYPE_daytime){
+ }
if( task->type == TYPE_timestamp){
}
}
@@ -72,16 +74,18 @@ void
MOSadvance_dict(MOStask task)
{
switch(task->type){
- case TYPE_sht: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(sht)+ wordaligned(sizeof(bte) *
MOScnt(task->blk))); break;
- case TYPE_int: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(int)+ wordaligned(sizeof(bte) *
MOScnt(task->blk))); break;
- case TYPE_oid: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(oid)+ wordaligned(sizeof(bte) *
MOScnt(task->blk))); break;
- case TYPE_lng: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(lng)+ wordaligned(sizeof(bte) *
MOScnt(task->blk))); break;
- case TYPE_wrd: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(wrd)+ wordaligned(sizeof(bte) *
MOScnt(task->blk))); break;
+ case TYPE_sht: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(sht)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),sht)); break;
+ case TYPE_int: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(int)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),int)); break;
+ case TYPE_oid: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(oid)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),oid)); break;
+ case TYPE_lng: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(lng)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),lng)); break;
+ case TYPE_wrd: task->blk = (MosaicBlk)( ((char*)task->blk) + 2*
MosaicBlkSize + dictsize * sizeof(wrd)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),wrd)); break;
default:
if( task->type == TYPE_timestamp)
- task->blk = (MosaicBlk)( ((char*)task->blk) +
2* MosaicBlkSize + dictsize * sizeof(timestamp)+ wordaligned(sizeof(bte) *
MOScnt(task->blk)));
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
2* MosaicBlkSize + dictsize * sizeof(timestamp)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),timestamp));
if( task->type == TYPE_date)
- task->blk = (MosaicBlk)( ((char*)task->blk) +
2* MosaicBlkSize + dictsize * sizeof(date)+ wordaligned(sizeof(bte) *
MOScnt(task->blk)));
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
2* MosaicBlkSize + dictsize * sizeof(date)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),date));
+ if( task->type == TYPE_daytime)
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
2* MosaicBlkSize + dictsize * sizeof(date)+ wordaligned(sizeof(bte) *
MOScnt(task->blk),daytime));
}
}
@@ -163,8 +167,10 @@ MOSestimate_dict(Client cntxt, MOStask t
break;\
}\
if ( j == *size){\
- if ( *size == dictsize)\
+ if ( *size == dictsize){\
+ task->dst += wordaligned(MOScnt(blk) %2,TPE);\
break;\
+ }\
dict[j] = *val;\
*size = *size+1;\
*task->dst++ = (char) j;\
@@ -206,7 +212,7 @@ MOScompress_dict(Client cntxt, MOStask t
if ( j == *size){
if ( *size == dictsize){
// align on word boundary
- task->dst +=
wordaligned(MOScnt(blk) %2);
+ task->dst +=
wordaligned(MOScnt(blk) %2,lng);
break;
}
dict[j] = *val;
diff --git a/monetdb5/modules/mal/mosaic_linear.c
b/monetdb5/modules/mal/mosaic_linear.c
--- a/monetdb5/modules/mal/mosaic_linear.c
+++ b/monetdb5/modules/mal/mosaic_linear.c
@@ -81,22 +81,22 @@ void
MOSadvance_linear(MOStask task)
{
switch(task->type){
- case TYPE_bte: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(bte))); break;
- case TYPE_bit: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(bit))); break;
- case TYPE_sht: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(sht))); break;
- case TYPE_int: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(int))); break;
- case TYPE_oid: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(oid))); break;
- case TYPE_lng: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(lng))); break;
- case TYPE_wrd: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(wrd))); break;
- case TYPE_flt: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(flt))); break;
- case TYPE_dbl: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(dbl))); break;
+ case TYPE_bte: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(bte),bte)); break;
+ case TYPE_bit: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(bit),bit)); break;
+ case TYPE_sht: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(sht),sht)); break;
+ case TYPE_int: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(int),int)); break;
+ case TYPE_oid: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(oid),oid)); break;
+ case TYPE_lng: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(lng),lng)); break;
+ case TYPE_wrd: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(wrd),wrd)); break;
+ case TYPE_flt: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(flt),flt)); break;
+ case TYPE_dbl: task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(dbl),dbl)); break;
default:
if( task->type == TYPE_daytime)
- task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(daytime)) );
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(daytime), daytime) );
if( task->type == TYPE_date)
- task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(date)) );
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(date),date) );
if( task->type == TYPE_timestamp)
- task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(lng)) );
+ task->blk = (MosaicBlk)( ((char*)task->blk) +
MosaicBlkSize + wordaligned(2 * sizeof(timestamp),timestamp) );
}
}
diff --git a/monetdb5/modules/mal/mosaic_none.c
b/monetdb5/modules/mal/mosaic_none.c
--- a/monetdb5/modules/mal/mosaic_none.c
+++ b/monetdb5/modules/mal/mosaic_none.c
@@ -38,22 +38,22 @@ MOSadvance_none(MOStask task)
{
MosaicBlk blk = task->blk;
switch(task->type){
- case TYPE_bte: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(bte)* MOScnt(blk))); break ;
- case TYPE_bit: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(bit)* MOScnt(blk))); break ;
- case TYPE_sht: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(sht)* MOScnt(blk))); break ;
- case TYPE_int: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(int)* MOScnt(blk))); break ;
- case TYPE_oid: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(oid)* MOScnt(blk))); break ;
- case TYPE_lng: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(lng)* MOScnt(blk))); break ;
- case TYPE_wrd: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(wrd)* MOScnt(blk))); break ;
- case TYPE_flt: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(flt)* MOScnt(blk))); break ;
- case TYPE_dbl: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(dbl)* MOScnt(blk))); break;
+ case TYPE_bte: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(bte)* MOScnt(blk),bte)); break ;
+ case TYPE_bit: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(bit)* MOScnt(blk),bit)); break ;
+ case TYPE_sht: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(sht)* MOScnt(blk),sht)); break ;
+ case TYPE_int: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(int)* MOScnt(blk),int)); break ;
+ case TYPE_oid: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(oid)* MOScnt(blk),oid)); break ;
+ case TYPE_lng: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(lng)* MOScnt(blk),lng)); break ;
+ case TYPE_wrd: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(wrd)* MOScnt(blk),wrd)); break ;
+ case TYPE_flt: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(flt)* MOScnt(blk),flt)); break ;
+ case TYPE_dbl: task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(dbl)* MOScnt(blk),dbl)); break;
default:
if( task->type == TYPE_date)
- task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(date)* MOScnt(blk)));
+ task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(date)* MOScnt(blk),date));
if( task->type == TYPE_daytime)
- task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(daytime)* MOScnt(blk)));
+ task->blk = (MosaicBlk)( ((char*) task->blk) +
MosaicBlkSize + wordaligned(sizeof(daytime)* MOScnt(blk),daytime));
if( task->type == TYPE_timestamp)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
