Changeset: 4dd9558ebba4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4dd9558ebba4
Modified Files:
gdk/gdk.h
gdk/gdk_bat.c
monetdb5/modules/mal/xid.c
monetdb5/modules/mal/xid.mal
Branch: xid
Log Message:
Reduce the code base to pure columns
Focus on void-head oid-tail BATs only.
diffs (truncated from 346 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -646,7 +646,7 @@ typedef struct {
unsigned int copied:1, /* a copy of an existing map. */
hashash:1,/* the string heap contains hash values */
forcemap:1, /* force STORE_MMAP even if heap exists */
- xidcompressed:1; /* compress heaps */
+ compressed:1; /* compress heaps */
storage_t storage; /* storage mode (mmap/malloc). */
storage_t newstorage; /* new desired storage mode at re-allocation. */
bte dirty; /* specific heap dirty marker */
@@ -1719,10 +1719,10 @@ gdk_export void GDKqsort_rev(void *h, vo
#define BATtkey(b) (b->tkey != FALSE || BATtdense(b))
/* set some properties that are trivial to deduce */
-/* but not for xid-compressed BATs */
+/* but not for compressed BATs */
#define COLsettrivprop(b, col) \
do { \
- if ((col)->type == TYPE_oid && (col)->heap.xidcompressed) \
+ if ((col)->type == TYPE_oid && (col)->heap.compressed) \
break; \
if ((col)->type == TYPE_void) { \
if ((col)->seq == oid_nil) { \
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -2877,8 +2877,8 @@ BATassertHeadProps(BAT *b)
return;
}
- if (b->htype == TYPE_oid && b->H->heap.xidcompressed) {
- /* no further property checking on xid-compressed BATs */
+ if (b->htype == TYPE_oid && b->H->heap.compressed) {
+ /* no further property checking on compressed BATs */
return;
}
@@ -3086,8 +3086,8 @@ BATderiveHeadProps(BAT *b, int expensive
assert(b != NULL);
if (b == NULL)
return;
- if (b->htype == TYPE_oid && b->H->heap.xidcompressed) {
- /* avoid property deriving on xid-compressed BATs */
+ if (b->htype == TYPE_oid && b->H->heap.compressed) {
+ /* avoid property deriving on compressed BATs */
return;
}
assert((b->hkey & BOUND2BTRUE) == 0);
diff --git a/monetdb5/modules/mal/xid.c b/monetdb5/modules/mal/xid.c
--- a/monetdb5/modules/mal/xid.c
+++ b/monetdb5/modules/mal/xid.c
@@ -18,7 +18,8 @@
*/
/*
* author Martin Kersten, Stefan Manegold
- * Light-weight compress oid columns to reduce temporary storage footprint.
+ * Light-weight (de)compress oid columns to reduce storage footprint.
+ * It works on void-headed columns and compressed tails only
*/
#include "monetdb_config.h"
@@ -314,7 +315,7 @@ XIDcompressCol( Client cntxt, MalBlkPtr
getArg(pci,0), s, cnt, i, i/(cnt/100.0),
GDKusec()/1000000, GDKusec()-clk);
MT_lock_unset(&mal_profileLock, "columndump");
- cn->heap.xidcompressed = 1;
+ cn->heap.compressed = 1;
}
cn->sorted = c->sorted;
cn->revsorted = c->revsorted;
@@ -400,22 +401,26 @@ XIDcompress(Client cntxt, MalBlkPtr mb,
BAT *b, *bn;
BUN cnt;
str msg = MAL_SUCCEED;
- int ht, tt, htx, ttx;
- oid hmin = 0, tmin = 0, hmax = oid_nil, tmax = oid_nil;
- bit skip_h, skip_t;
+ int ttx;
+ oid tmin = 0, tmax = oid_nil;
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "xid.compress", INTERNAL_BAT_ACCESS);
- assert(ATOMtype(b->htype) == TYPE_oid || ATOMtype(b->ttype) ==
TYPE_oid);
- if (b->H->heap.xidcompressed || b->T->heap.xidcompressed) {
+
+ if ( b->ttype == TYPE_void){
+ // void columns are already compressed
+ BBPkeepref(*ret = b->batCacheid);
+ return MAL_SUCCEED;
+ }
+ assert(ATOMtype(b->htype) == TYPE_void || ATOMtype(b->ttype) ==
TYPE_oid);
+
+ if (b->T->heap.compressed) {
BBPreleaseref(b->batCacheid);
- throw(MAL, "xid.compress", "cannot compress already
%s%scompressed BAT",
- b->H->heap.xidcompressed?"head-":"",
b->T->heap.xidcompressed?"tail-":"");
+ throw(MAL, "xid.compress", "cannot compress already compressed
column");
}
+
cnt = BATcount(b);
- ht = b->htype;
- tt = b->ttype;
- if ( isVIEWCOMBINE(b) || cnt < MIN_INPUT_COUNT || (ht == TYPE_void &&
tt != TYPE_oid) || (ht != TYPE_oid && tt == TYPE_void) ) {
+ if ( isVIEWCOMBINE(b) || cnt < MIN_INPUT_COUNT ){
/* no need to compress */
MT_lock_set(&mal_profileLock, "compress");
mnstr_printf(cntxt->fdout,"#xid view (count = "BUNFMT")
variable %d\n",cnt, getArg(pci,1));
@@ -425,27 +430,22 @@ XIDcompress(Client cntxt, MalBlkPtr mb,
}
if (cnt > XID_CNT_MAX) {
BBPreleaseref(b->batCacheid);
- throw(MAL, "xid.compress", "original count too large");
+ throw(MAL, "xid.compress", "original count too large to
compress");
}
/* "compress" materialized into non-materialized dense OIDs */
- htx = (BAThdense(b) ? TYPE_void : ht);
- ttx = (BATtdense(b) ? TYPE_void : tt);
- if (htx == TYPE_oid)
- getMinMax( b->H, cnt, (oid*)Hloc(b,BUNfirst(b)), &hmin, &hmax );
+ ttx = (BATtdense(b) ? TYPE_void : b->ttype);
if (ttx == TYPE_oid)
getMinMax( b->T, cnt, (oid*)Tloc(b,BUNfirst(b)), &tmin, &tmax );
- skip_h = (ATOMtype(ht) != TYPE_oid || (htx == TYPE_oid && hmax >
XID_CNT_MAX));
- skip_t = (ATOMtype(tt) != TYPE_oid || (ttx == TYPE_oid && tmax >
XID_CNT_MAX));
- if ((skip_h || ht == TYPE_void) && (skip_t || tt == TYPE_void)) {
+ if (tmax > XID_CNT_MAX ) {
/* cannot (yet?) compress */
MT_lock_set(&mal_profileLock, "compress");
- mnstr_printf(cntxt->fdout,"#xid NIL or too large OID " OIDFMT
":" OIDFMT", variable %d\n", hmax, tmax, getArg(pci,1));
+ mnstr_printf(cntxt->fdout,"#xid NIL or too large OID " OIDFMT",
variable %d\n", tmax, getArg(pci,1));
MT_lock_unset(&mal_profileLock, "compress");
BBPkeepref(*ret = b->batCacheid);
return MAL_SUCCEED;
}
- if ( (!skip_h && htx == TYPE_oid && VIEWhparent(b)) || (!skip_t && ttx
== TYPE_oid && VIEWtparent(b)) ) {
+ if ( VIEWtparent(b)) {
/* no need to compress */
MT_lock_set(&mal_profileLock, "compress");
mnstr_printf(cntxt->fdout,"#xid view (count = "BUNFMT")
variable %d\n",cnt, getArg(pci,1));
@@ -454,38 +454,25 @@ XIDcompress(Client cntxt, MalBlkPtr mb,
return MAL_SUCCEED;
}
- bn = BATnew( (skip_h ? TYPE_void : htx), (skip_t ? TYPE_void : ttx),
cnt+XID_IDX_BASE );
+ bn = BATnew( TYPE_void, ttx, cnt+XID_IDX_BASE );
if (bn == NULL) {
BBPreleaseref(b->batCacheid);
throw(MAL,"xid.compress", MAL_MALLOC_FAIL);
}
+ BATseqbase(bn,0);
- if (!skip_h) {
- msg = XIDcompressCol( cntxt, mb, pci, b->H, bn->H, cnt, "head",
htx, (oid*)Hloc(b,BUNfirst(b)), (oid*)Hloc(b,BUNlast(b)), hmin );
- if (msg != MAL_SUCCEED) {
- BBPreleaseref(b->batCacheid);
- BBPreleaseref(bn->batCacheid);
- return msg;
- }
- }
- if (!skip_t) {
- msg = XIDcompressCol( cntxt, mb, pci, b->T, bn->T, cnt, "tail",
ttx, (oid*)Tloc(b,BUNfirst(b)), (oid*)Tloc(b,BUNlast(b)), tmin );
- if (msg != MAL_SUCCEED) {
- BBPreleaseref(b->batCacheid);
- BBPreleaseref(bn->batCacheid);
- return msg;
- }
+ msg = XIDcompressCol( cntxt, mb, pci, b->T, bn->T, cnt, "tail", ttx,
(oid*)Tloc(b,BUNfirst(b)), (oid*)Tloc(b,BUNlast(b)), tmin );
+ if (msg != MAL_SUCCEED) {
+ BBPreleaseref(b->batCacheid);
+ BBPreleaseref(bn->batCacheid);
+ return msg;
}
- if (bn->H->heap.xidcompressed || bn->T->heap.xidcompressed) {
+ if (bn->T->heap.compressed) {
/* keep original count in case we inherit uncompressed head or
tail */
BATsetcount(bn, cnt);
bn->batDirty = 1;
- if (skip_h || (htx == TYPE_oid && !bn->H->heap.xidcompressed)) {
- /* inherit original uncompressed head as view */
- bn = inheritCOL( bn, bn->H, b, b->H, VIEWhparent(b) );
- }
- if (skip_t || (ttx == TYPE_oid && !bn->T->heap.xidcompressed)) {
+ if (!bn->T->heap.compressed) {
/* inherit original uncompressed tail as view */
bn = inheritCOL( bn, bn->T, b, b->T, VIEWtparent(b) );
}
@@ -602,7 +589,7 @@ XIDdecompressCol( Client cntxt, MalBlkPt
str msg = MAL_SUCCEED;
oid min = 0;
- if ( c->heap.xidcompressed )
+ if ( c->heap.compressed )
{
col = (XIDcolumn) c->heap.base;
lim = (BUN) col[XID_IDX_COMP].count;
@@ -619,7 +606,7 @@ XIDdecompressCol( Client cntxt, MalBlkPt
if (cnt != cap)
throw(MAL, "xid.decompress", "decompressed %s count
does not match original count: "BUNFMT" != "BUNFMT"", s, cnt, cap);
- cn->heap.xidcompressed = 0;
+ cn->heap.compressed = 0;
}
cn->sorted = c->sorted;
cn->revsorted = c->revsorted;
@@ -644,8 +631,10 @@ XIDdecompress(Client cntxt, MalBlkPtr mb
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "xid.decompress", INTERNAL_BAT_ACCESS);
- assert(ATOMtype(b->htype) == TYPE_oid || ATOMtype(b->ttype) ==
TYPE_oid);
- if (!b->H->heap.xidcompressed && !b->T->heap.xidcompressed) {
+
+ assert(ATOMtype(b->htype) == TYPE_void || ATOMtype(b->ttype) ==
TYPE_oid);
+
+ if (!b->T->heap.compressed) {
BBPkeepref(*ret = b->batCacheid);
return MAL_SUCCEED;
}
@@ -653,20 +642,12 @@ XIDdecompress(Client cntxt, MalBlkPtr mb
BBPreleaseref(b->batCacheid);
throw(MAL, "xid.decompress", "cannot decompress VIEWCOMBINE");
}
- if (b->H->heap.xidcompressed && VIEWhparent(b)) {
- BBPreleaseref(b->batCacheid);
- throw(MAL, "xid.decompress", "cannot decompress head-VIEW");
- }
- if (b->T->heap.xidcompressed && VIEWtparent(b)) {
+ if (b->T->heap.compressed && VIEWtparent(b)) {
BBPreleaseref(b->batCacheid);
throw(MAL, "xid.decompress", "cannot decompress tail-VIEW");
}
- if ( b->H->heap.xidcompressed ) {
- col = (XIDcolumn) b->H->heap.base;
- cap = (BUN) col[XID_IDX_ORIG].count;
- } else
- if ( b->T->heap.xidcompressed ) {
+ if ( b->T->heap.compressed ) {
col = (XIDcolumn) b->T->heap.base;
cap = (BUN) col[XID_IDX_ORIG].count;
} else {
@@ -674,21 +655,14 @@ XIDdecompress(Client cntxt, MalBlkPtr mb
}
assert(cap == BATcount(b));
- bn = BATnew((b->H->heap.xidcompressed ? b->htype : TYPE_void),
(b->T->heap.xidcompressed ? b->ttype : TYPE_void), cap);
+ bn = BATnew( TYPE_void, (b->T->heap.compressed ? b->ttype : TYPE_void),
cap);
if ( bn == NULL) {
BBPreleaseref(b->batCacheid);
throw(MAL, "xid.decompress", MAL_MALLOC_FAIL);
}
+ BATseqbase(bn,0);
- if (b->H->heap.xidcompressed || b->htype == TYPE_void) {
- msg = XIDdecompressCol( cntxt, mb, pci, b->H, bn->H, "head",
cap, (oid*)Hloc(bn,BUNfirst(bn)) );
- if (msg != MAL_SUCCEED) {
- BBPreleaseref(b->batCacheid);
- BBPreleaseref(bn->batCacheid);
- return msg;
- }
- }
- if (b->T->heap.xidcompressed || b->ttype == TYPE_void) {
+ if (b->T->heap.compressed) {
msg = XIDdecompressCol( cntxt, mb, pci, b->T, bn->T, "tail",
cap, (oid*)Tloc(bn,BUNfirst(bn)) );
if (msg != MAL_SUCCEED) {
BBPreleaseref(b->batCacheid);
@@ -699,11 +673,7 @@ XIDdecompress(Client cntxt, MalBlkPtr mb
BATsetcount(bn, cap);
bn->batDirty = 1;
- if (!b->H->heap.xidcompressed && b->htype != TYPE_void) {
- /* inherit original uncompressed head as view */
- bn = inheritCOL( bn, bn->H, b, b->H, VIEWhparent(b) );
- }
- if (!b->T->heap.xidcompressed && b->ttype != TYPE_void) {
+ if (!b->T->heap.compressed && b->ttype != TYPE_void) {
/* inherit original uncompressed tail as view */
bn = inheritCOL( bn, bn->T, b, b->T, VIEWtparent(b) );
}
@@ -760,7 +730,7 @@ XIDdump(void *ret, bat *bid)
if ((b = BATdescriptor(*bid)) == NULL)
throw(MAL, "xid.dump", INTERNAL_BAT_ACCESS);
assert(ATOMtype(b->htype) == TYPE_oid || ATOMtype(b->ttype) ==
TYPE_oid);
- if ( b->T->heap.xidcompressed){
+ if ( b->T->heap.compressed){
col = (XIDcolumn) b->T->heap.base;
lim = (BUN) col[XID_IDX_COMP].count;
@@ -773,19 +743,6 @@ XIDdump(void *ret, bat *bid)
return msg;
}
}
- if ( b->H->heap.xidcompressed){
- col = (XIDcolumn) b->H->heap.base;
- lim = (BUN) col[XID_IDX_COMP].count;
-
- MT_lock_set(&mal_profileLock, "columndump");
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
