Changeset: 105cc0b8a397 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=105cc0b8a397
Added Files:
monetdb5/modules/mal/Tests/mosaic_prefix.mal
monetdb5/modules/mal/mosaic_prefix.c
monetdb5/modules/mal/mosaic_prefix.h
Modified Files:
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/Tests/All
monetdb5/modules/mal/mosaic.c
monetdb5/modules/mal/mosaic.h
Branch: mosaic
Log Message:
Checkpoint of prefix compression
diffs (truncated from 980 to 300 lines):
diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag
--- a/monetdb5/modules/mal/Makefile.ag
+++ b/monetdb5/modules/mal/Makefile.ag
@@ -71,6 +71,7 @@ lib_mal = {
mosaic_delta.c mosaic_delta.h \
mosaic_linear.c mosaic_linear.h \
mosaic_variance.c mosaic_variance.h \
+ mosaic_prefix.c mosaic_prefix.h \
calc.c batcalc.c
}
diff --git a/monetdb5/modules/mal/Tests/All b/monetdb5/modules/mal/Tests/All
--- a/monetdb5/modules/mal/Tests/All
+++ b/monetdb5/modules/mal/Tests/All
@@ -75,6 +75,7 @@ mosaic_dictionary
mosaic_zone
mosaic_delta
mosaic_linear
+mosaic_prefix
mosaic_literal_double
diff --git a/monetdb5/modules/mal/Tests/mosaic_prefix.mal
b/monetdb5/modules/mal/Tests/mosaic_prefix.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/Tests/mosaic_prefix.mal
@@ -0,0 +1,33 @@
+b:= bat.new(:oid,:int);
+#bat.append(b,50);
+#bat.append(b,51);
+#bat.append(b,51);
+#bat.append(b,51);
+#bat.append(b,51);
+#bat.append(b,51);
+#bat.append(b,51);
+#bat.append(b,60);
+#bat.append(b,67);
+#bat.append(b,67);
+#bat.append(b,67);
+#bat.append(b,67);
+#bat.append(b,67);
+#bat.append(b,67);
+#bat.append(b,67);
+bat.append(b,255);
+bat.append(b,256);
+bat.append(b,256);
+bat.append(b,256);
+bat.append(b,256);
+bat.append(b,256);
+bat.append(b,255);
+bat.append(b,255);
+bat.append(b,255);
+bat.append(b,255);
+bat.append(b,255);
+
+io.print(b);
+x:= mosaic.compress(b,"prefix");
+#mosaic.dump(x);
+z:= mosaic.decompress(x);
+io.print(z);
diff --git a/monetdb5/modules/mal/mosaic.c b/monetdb5/modules/mal/mosaic.c
--- a/monetdb5/modules/mal/mosaic.c
+++ b/monetdb5/modules/mal/mosaic.c
@@ -33,8 +33,9 @@
#include "mosaic_delta.h"
#include "mosaic_linear.h"
#include "mosaic_variance.h"
+#include "mosaic_prefix.h"
-static char
*filtername[]={"literal","runlength","dictionary","delta","linear","variance","zone","EOL"};
+static char
*filtername[]={"literal","runlength","dictionary","delta","linear","variance","prefix","zone","EOL"};
static void
MOSinit(MOStask task, BAT *b){
@@ -100,6 +101,10 @@ MOSdumpInternal(Client cntxt, BAT *b){
MOSdump_delta(cntxt,task);
MOSadvance_delta(cntxt,task);
break;
+ case MOSAIC_PREFIX:
+ MOSdump_prefix(cntxt,task);
+ MOSadvance_prefix(cntxt,task);
+ break;
case MOSAIC_VARIANCE:
MOSdump_variance(cntxt,task);
MOSadvance_variance(cntxt,task);
@@ -339,6 +344,7 @@ MOScompressInternal(Client cntxt, int *r
factor = fac;
}
}
+/*
if ( filter[MOSAIC_VARIANCE]){
fac = MOSestimate_variance(cntxt,task);
if (fac > factor){
@@ -346,7 +352,6 @@ MOScompressInternal(Client cntxt, int *r
factor = fac;
}
}
-/*
if ( filter[MOSAIC_ZONE]){
fac = MOSestimate_zone(cntxt,task);
if (fac > factor){
@@ -362,6 +367,13 @@ MOScompressInternal(Client cntxt, int *r
factor = fac;
}
}
+ if ( filter[MOSAIC_PREFIX]){
+ fac = MOSestimate_prefix(cntxt,task);
+ if ( fac > factor ){
+ cand = MOSAIC_PREFIX;
+ factor = fac;
+ }
+ }
if ( filter[MOSAIC_LINEAR]){
fac = MOSestimate_linear(cntxt,task);
if ( fac >factor){
@@ -376,6 +388,7 @@ MOScompressInternal(Client cntxt, int *r
case MOSAIC_DICT:
case MOSAIC_DELTA:
case MOSAIC_LINEAR:
+ case MOSAIC_PREFIX:
// close the non-compressed part
if( (MOSgetTag(task->blk) == MOSAIC_NONE ||
MOSgetTag(task->blk) == MOSAIC_ZONE) && MOSgetCnt(task->blk) ){
MOSupdateHeader(cntxt,task);
@@ -430,6 +443,15 @@ MOScompressInternal(Client cntxt, int *r
MOSsetTag(task->blk,MOSAIC_EOL);
task->dst = ((char*) task->blk)+ MosaicBlkSize;
break;
+ case MOSAIC_PREFIX:
+ MOScompress_prefix(cntxt,task);
+ MOSupdateHeader(cntxt,task);
+ //prepare new block header
+ task->elm -= MOSgetCnt(task->blk);
+ MOSadvance_prefix(cntxt,task);
+ MOSsetTag(task->blk,MOSAIC_EOL);
+ task->dst = ((char*) task->blk)+ MosaicBlkSize;
+ break;
case MOSAIC_LINEAR:
MOScompress_linear(cntxt,task);
MOSupdateHeader(cntxt,task);
@@ -613,6 +635,10 @@ MOSdecompressInternal(Client cntxt, int
MOSdecompress_delta(cntxt,task);
MOSskip_delta(cntxt,task);
break;
+ case MOSAIC_PREFIX:
+ MOSdecompress_prefix(cntxt,task);
+ MOSskip_prefix(cntxt,task);
+ break;
case MOSAIC_LINEAR:
MOSdecompress_linear(cntxt,task);
MOSskip_linear(cntxt,task);
@@ -829,6 +855,9 @@ MOSsubselect(Client cntxt, MalBlkPtr mb,
case MOSAIC_DELTA:
MOSsubselect_delta(cntxt,task,low,hgh,li,hi,anti);
break;
+ case MOSAIC_PREFIX:
+ MOSsubselect_prefix(cntxt,task,low,hgh,li,hi,anti);
+ break;
case MOSAIC_LINEAR:
MOSsubselect_linear(cntxt,task,low,hgh,li,hi,anti);
break;
@@ -953,6 +982,9 @@ str MOSthetasubselect(Client cntxt, MalB
case MOSAIC_DELTA:
MOSthetasubselect_delta(cntxt,task,low,*oper);
break;
+ case MOSAIC_PREFIX:
+ MOSthetasubselect_prefix(cntxt,task,low,*oper);
+ break;
case MOSAIC_LINEAR:
MOSthetasubselect_linear(cntxt,task,low,*oper);
break;
@@ -1093,6 +1125,9 @@ str MOSleftfetchjoin(Client cntxt, MalBl
case MOSAIC_DELTA:
MOSleftfetchjoin_delta(cntxt, task);
break;
+ case MOSAIC_PREFIX:
+ MOSleftfetchjoin_prefix(cntxt, task);
+ break;
case MOSAIC_LINEAR:
MOSleftfetchjoin_linear(cntxt, task);
break;
@@ -1218,6 +1253,9 @@ MOSjoin(Client cntxt, MalBlkPtr mb, MalS
case MOSAIC_DELTA:
MOSjoin_delta(cntxt, task);
break;
+ case MOSAIC_PREFIX:
+ MOSjoin_prefix(cntxt, task);
+ break;
case MOSAIC_LINEAR:
MOSjoin_linear(cntxt, task);
break;
diff --git a/monetdb5/modules/mal/mosaic.h b/monetdb5/modules/mal/mosaic.h
--- a/monetdb5/modules/mal/mosaic.h
+++ b/monetdb5/modules/mal/mosaic.h
@@ -39,15 +39,16 @@
#define MIN_INPUT_COUNT 1
/* The compressor kinds currently hardwired */
-#define MOSAIC_METHODS 7
+#define MOSAIC_METHODS 8
#define MOSAIC_NONE 0 // no compression at all
#define MOSAIC_RLE 1 // use run-length encoding
#define MOSAIC_DICT 2 // local dictionary encoding
#define MOSAIC_DELTA 3 // use delta encoding
#define MOSAIC_LINEAR 4 // use an encoding for a linear sequence
#define MOSAIC_VARIANCE 5 // adaptive dictionary over
deltas
-#define MOSAIC_ZONE 6 // adaptive zone map over
non-compressed data
-#define MOSAIC_EOL 7 // marker for the last block
+#define MOSAIC_PREFIX 6 // prefix/postfix bitwise compression
+#define MOSAIC_ZONE 7 // adaptive zone map over
non-compressed data
+#define MOSAIC_EOL 9 // marker for the last block
//Compression should have a significant reduction to apply.
#define COMPRESS_THRESHOLD 50 //percent
diff --git a/monetdb5/modules/mal/mosaic_prefix.c
b/monetdb5/modules/mal/mosaic_prefix.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/mosaic_prefix.c
@@ -0,0 +1,715 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * * Copyright August 2008-2014 MonetDB B.V.
+ * All Rights Reserved.
+ */
+
+/*
+ * (c)2014 author Martin Kersten
+ * Blocked bit_ prefix compression
+ */
+
+#include "monetdb_config.h"
+#include "mosaic.h"
+#include "mosaic_prefix.h"
+
+/* Beware, the dump routines use the compressed part of the task */
+void
+MOSdump_prefix(Client cntxt, MOStask task)
+{
+ MosaicBlk blk= task->blk;
+ void *val = (void*)(((char*) blk) + MosaicBlkSize);
+
+ mnstr_printf(cntxt->fdout,"#rle "BUNFMT" ", MOSgetCnt(blk));
+ switch(task->type){
+ case TYPE_bte:
+ mnstr_printf(cntxt->fdout,"bte %hhd", *(bte*) val); break;
+ case TYPE_sht:
+ mnstr_printf(cntxt->fdout,"sht %hd", *(sht*) val); break;
+ case TYPE_int:
+ mnstr_printf(cntxt->fdout,"int %d", *(int*) val); break;
+ case TYPE_oid:
+ mnstr_printf(cntxt->fdout,"oid "OIDFMT, *(oid*) val); break;
+ case TYPE_lng:
+ mnstr_printf(cntxt->fdout,"lng "LLFMT, *(lng*) val); break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ mnstr_printf(cntxt->fdout,"hge %.40g", (dbl) *(hge*) val);
break;
+#endif
+ case TYPE_wrd:
+ mnstr_printf(cntxt->fdout,"wrd "SZFMT, *(wrd*) val); break;
+ case TYPE_flt:
+ mnstr_printf(cntxt->fdout,"flt %f", *(flt*) val); break;
+ case TYPE_dbl:
+ mnstr_printf(cntxt->fdout,"flt %f", *(dbl*) val); break;
+ case TYPE_str:
+ mnstr_printf(cntxt->fdout,"str TBD"); break;
+ default:
+ if( task->type == TYPE_date)
+ mnstr_printf(cntxt->fdout,"date %d ", *(int*) val);
+ if( task->type == TYPE_daytime)
+ mnstr_printf(cntxt->fdout,"daytime %d ", *(int*) val);
+ if( task->type == TYPE_timestamp)
+ mnstr_printf(cntxt->fdout,"int "LLFMT, *(lng*) val);
+ }
+ mnstr_printf(cntxt->fdout,"\n");
+}
+
+void
+MOSadvance_prefix(Client cntxt, MOStask task)
+{
+ int bits, bytes;
+ (void) cntxt;
+
+ task->start += MOSgetCnt(task->blk);
+ switch(ATOMsize(task->type)){
+ case 4:
+ { int *dst = (int*) (((char*) task->blk) + MosaicBlkSize);
+ int mask = *dst++;
+ int val = *dst++;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
