Changeset: c907ccb28174 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c907ccb28174
Modified Files:
monetdb5/modules/mal/Tests/xidlist.stable.out
monetdb5/modules/mal/xid.c
Branch: xid
Log Message:
xid.c: store both original & compressed counts in compressed column
diffs (truncated from 580 to 300 lines):
diff --git a/monetdb5/modules/mal/Tests/xidlist.stable.out
b/monetdb5/modules/mal/Tests/xidlist.stable.out
--- a/monetdb5/modules/mal/Tests/xidlist.stable.out
+++ b/monetdb5/modules/mal/Tests/xidlist.stable.out
@@ -153,7 +153,7 @@ end main;
[ 78@0, 1280@0 ]
[ 79@0, 1290@0 ]
#xid, 83, tail compress, 80,56, clk 3 usec
-column first 57, size 0,
+column first 56, size 0,
r+:0 3
p:5
p:7
diff --git a/monetdb5/modules/mal/xid.c b/monetdb5/modules/mal/xid.c
--- a/monetdb5/modules/mal/xid.c
+++ b/monetdb5/modules/mal/xid.c
@@ -31,25 +31,35 @@
#define XIDRANGEINC 3
#define XIDRANGEDEC 4
#define XIDRANGEEQ 5
-/* must be < 2^XID_TAG_BITS i.e. < 1<<XID_TAG_BITS */
+/* must be <= XID_TAG_MAX */
typedef oid xid;
#define SIZEOF_XID SIZEOF_OID
#define XIDFMT OIDFMT
#define XID_TAG_BITS 3
+#define XID_TAG_MAX ((((xid)1) << XID_TAG_BITS) - 1)
#define XID_VAL_BITS ((8 * SIZEOF_XID) - XID_TAG_BITS)
#define XID_VAL_MAX ((((xid)1) << XID_VAL_BITS) - 1)
+#define XID_CNT_MAX GDK_oid_max
-typedef struct XIDCOLUMN{
- xid tag:XID_TAG_BITS, value:XID_VAL_BITS;
+typedef union XIDCOLUMN{
+ struct {
+ xid tag:XID_TAG_BITS,
+ val:XID_VAL_BITS;
+ } x;
+ xid count;
} *XIDcolumn;
+#define XID_IDX_ORIG 0
+#define XID_IDX_COMP 1
+#define XID_IDX_BASE 2
+
static str
XIDencode(BUN *rtrn, XIDcolumn col, oid *p, oid *q)
{
oid o;
xid v, prev=0;
- BUN i=1,scnt =0;
+ BUN i=XID_IDX_BASE, scnt=0;
//xid point=0, range=0,set=0;
o = *(oid*) p++;
@@ -59,8 +69,8 @@ XIDencode(BUN *rtrn, XIDcolumn col, oid
if (o > XID_VAL_MAX)
throw(MAL, "xid.compress", "value too large in XIDencode()");
v = (xid) o;
- col[i].value = v;
- col[i].tag = XIDPOINT;
+ col[i].x.val = v;
+ col[i].x.tag = XIDPOINT;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " " XIDFMT "\n",i,v);
for ( ; p<q ; p++) {
@@ -70,25 +80,25 @@ XIDencode(BUN *rtrn, XIDcolumn col, oid
if (o > XID_VAL_MAX)
throw(MAL, "xid.compress", "value too large in
XIDencode()");
v = (xid) o;
- switch ( col[i].tag ) {
+ switch ( col[i].x.tag ) {
case XIDSETINC:
/* works only for strictly increasing sets (sequences);
* otherwise, we loose order information
*/
- if ( v > prev && v < (xid) col[i-1].value +
XID_VAL_BITS ) {
- col[i].value |= ( ((xid)1)<< (v -
col[i-1].value));
+ if ( v > prev && v < (xid) col[i-1].x.val +
XID_VAL_BITS ) {
+ col[i].x.val |= ( ((xid)1)<< (v -
col[i-1].x.val));
scnt++;
prev= v;
- //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v - col[i-1].value));
+ //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v - col[i-1].x.val));
} else {
if (scnt == 1) {
- col[i].tag = XIDPOINT;
- col[i].value = prev;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = prev;
scnt =0;
}
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
//point++;
}
@@ -97,125 +107,125 @@ XIDencode(BUN *rtrn, XIDcolumn col, oid
/* works only for strictly decreasing sets (sequences);
* otherwise, we loose order information
*/
- if ( v < prev && v > (xid) col[i-1].value -
XID_VAL_BITS ) {
- col[i].value |= ( ((xid)1)<< (col[i-1].value -
v));
+ if ( v < prev && v > (xid) col[i-1].x.val -
XID_VAL_BITS ) {
+ col[i].x.val |= ( ((xid)1)<< (col[i-1].x.val -
v));
scnt++;
prev= v;
- //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v - col[i-1].value));
+ //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v - col[i-1].x.val));
} else {
if (scnt == 1) {
- col[i].tag = XIDPOINT;
- col[i].value = prev;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = prev;
scnt =0;
}
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
//point++;
}
break;
case XIDRANGEEQ:
- if ( (xid) col[i-1].value == v && col[i].value <
XID_VAL_MAX){
- col[i].value++;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ if ( (xid) col[i-1].x.val == v && col[i].x.val <
XID_VAL_MAX){
+ col[i].x.val++;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
} else {
/* fall back to point if spread to large */
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
//point++;
}
break;
case XIDRANGEINC:
- if ( (xid) col[i].value + 1 == v){
- col[i].value = v;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ if ( (xid) col[i].x.val + 1 == v){
+ col[i].x.val = v;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
} else {
/* fall back to point if spread to large */
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
//point++;
}
break;
case XIDRANGEDEC:
- if ( (xid) col[i].value - 1 == v){
- col[i].value = v;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ if ( (xid) col[i].x.val - 1 == v){
+ col[i].x.val = v;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
} else {
/* fall back to point if spread to large */
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
//point++;
}
break;
case XIDPOINT:
- if ( (xid) col[i].value == v){
+ if ( (xid) col[i].x.val == v){
/* make a new inc-range */
- col[i].tag = XIDRANGEEQ;
- col[++i].tag = XIDRANGEEQ;
- col[i].value = 2;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ col[i].x.tag = XIDRANGEEQ;
+ col[++i].x.tag = XIDRANGEEQ;
+ col[i].x.val = 2;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
//range++;
} else
- if ( (xid) col[i].value + 1 == v){
+ if ( (xid) col[i].x.val + 1 == v){
/* make a new inc-range */
- col[i].tag = XIDRANGEINC;
- col[++i].tag = XIDRANGEINC;
- col[i].value = v;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ col[i].x.tag = XIDRANGEINC;
+ col[++i].x.tag = XIDRANGEINC;
+ col[i].x.val = v;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
//range++;
} else
- if ( (xid) col[i].value - 1 == v){
+ if ( (xid) col[i].x.val - 1 == v){
/* make a new dec-range */
- col[i].tag = XIDRANGEDEC;
- col[++i].tag = XIDRANGEDEC;
- col[i].value = v;
- //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].value, v);
+ col[i].x.tag = XIDRANGEDEC;
+ col[++i].x.tag = XIDRANGEDEC;
+ col[i].x.val = v;
+ //mnstr_printf(GDKout,"xidrange " BUNFMT " "
XIDFMT " " XIDFMT "\n",i, (xid) col[i-1].x.val, v);
//range++;
} else
- if ( v > (xid) col[i].value && v < (xid) col[i].value +
XID_VAL_BITS ){
+ if ( v > (xid) col[i].x.val && v < (xid) col[i].x.val +
XID_VAL_BITS ){
/* make a new set */
i++;
- col[i].tag = XIDSETINC;
+ col[i].x.tag = XIDSETINC;
scnt = 1;
prev = v;
- col[i].value = ((xid)1) << (v -col[i-1].value);
- //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v- col[i-1].value));
+ col[i].x.val = ((xid)1) << (v -col[i-1].x.val);
+ //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v- col[i-1].x.val));
//set++;
} else
- if ( v < (xid) col[i].value && v > (xid) col[i].value -
XID_VAL_BITS ){
+ if ( v < (xid) col[i].x.val && v > (xid) col[i].x.val -
XID_VAL_BITS ){
/* make a new set */
i++;
- col[i].tag = XIDSETDEC;
+ col[i].x.tag = XIDSETDEC;
scnt = 1;
prev = v;
- col[i].value = ((xid)1) << (col[i-1].value - v);
- //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v- col[i-1].value));
+ col[i].x.val = ((xid)1) << (col[i-1].x.val - v);
+ //mnstr_printf(GDKout,"xidset " BUNFMT " "
XIDFMT "\n",i,(xid) (v- col[i-1].x.val));
//set++;
} else {
i++;
- col[i].tag = XIDPOINT;
- col[i].value = v;
+ col[i].x.tag = XIDPOINT;
+ col[i].x.val = v;
//mnstr_printf(GDKout,"xidpoint " BUNFMT " "
XIDFMT "\n",i,v);
}
break;
default:
- throw(MAL, "xid.compress", "unknown col.tag in
XIDencode()");
+ throw(MAL, "xid.compress", "unknown col.x.tag in
XIDencode()");
}
}
//mnstr_printf(GDKout,"stats point " XIDFMT " range " XIDFMT " set "
XIDFMT "\n",point,range,set);
- *rtrn = i;
+ *rtrn = i - XID_IDX_BASE;
return MAL_SUCCEED;
}
static str
-XIDcompressCol( Client cntxt, InstrPtr pci, BAT *b, COLrec *c, BAT *bn, COLrec
*cn, str s, int t, oid *p, oid *q )
+XIDcompressCol( Client cntxt, InstrPtr pci, COLrec *c, COLrec *cn, BUN cnt,
str s, int t, oid *p, oid *q )
{
XIDcolumn col;
lng clk;
@@ -229,14 +239,12 @@ XIDcompressCol( Client cntxt, InstrPtr p
msg = XIDencode(&i,col,p,q);
if (msg != MAL_SUCCEED)
return msg;
- if (i+1 > XID_VAL_MAX)
- throw(MAL, "xid.compress", "value too large");
- col[0].value = (xid)(i+1); /* keep compression size */
+ col[XID_IDX_ORIG].count = (xid)cnt; /* keep original size */
+ col[XID_IDX_COMP].count = (xid)(i+1); /* keep compression size
*/
mnstr_printf(cntxt->fdout,"#xid, %d, %s compress, " BUNFMT ","
BUNFMT ", %4.2f clk " LLFMT " usec\n",
- getArg(pci,0), s, BATcount(b), i,
i/(BATcount(b)/100.0), GDKusec()-clk);
+ getArg(pci,0), s, cnt, i, i/(cnt/100.0), GDKusec()-clk);
cn->heap.xidcompressed = 1;
- bn->batDirty =1;
}
cn->sorted = c->sorted;
cn->revsorted = c->revsorted;
@@ -255,47 +263,60 @@ XIDcompress(Client cntxt, MalBlkPtr mb,
bat *ret = (bat*) getArgReference(stk,pci,0);
bat *bid = (bat*) getArgReference(stk,pci,1);
BAT *b, *bn;
+ BUN cnt;
str msg = MAL_SUCCEED;
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
