Changeset: 0ccdb1f64e3d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0ccdb1f64e3d
Modified Files:
gdk/Makefile.ag
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_private.h
gdk/gdk_storage.c
monetdb5/modules/mal/mosaic.c
monetdb5/modules/mal/mosaic.h
monetdb5/modules/mal/mosaic_delta.c
monetdb5/modules/mal/mosaic_linear.c
monetdb5/modules/mal/mosaic_literal.c
monetdb5/modules/mal/mosaic_prefix.c
monetdb5/modules/mal/mosaic_runlength.c
sql/backends/monet5/sql.c
Branch: mosaic
Log Message:
Switch to explicit mosaic heap in BAT descriptor
This allows for compressed mirrors.
diffs (truncated from 1294 to 300 lines):
diff --git a/gdk/Makefile.ag b/gdk/Makefile.ag
--- a/gdk/Makefile.ag
+++ b/gdk/Makefile.ag
@@ -30,6 +30,7 @@ lib_gdk = {
gdk_join.c \
gdk_unique.c \
gdk_firstn.c \
+ gdk_mosaic.c \
bat.feps bat1.feps bat2.feps \
libbat.rc
LIBS = ../common/options/libmoptions \
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -655,8 +655,7 @@ typedef struct {
unsigned int copied:1, /* a copy of an existing map. */
hashash:1,/* the string heap contains hash values */
- forcemap:1, /* force STORE_MMAP even if heap exists */
- compressed:1; /* compress heaps */
+ forcemap:1; /* force STORE_MMAP even if heap exists */
storage_t storage; /* storage mode (mmap/malloc). */
storage_t newstorage; /* new desired storage mode at re-allocation. */
bte dirty; /* specific heap dirty marker */
@@ -890,6 +889,7 @@ typedef struct {
Heap *vheap; /* space for the varsized data. */
Hash *hash; /* hash table */
Imprints *imprints; /* column imprints index */
+ Heap *mosaic; /* compressed representation */
PROPrec *props; /* list of dynamic properties stored in the bat
descriptor */
} COLrec;
@@ -2011,6 +2011,9 @@ gdk_export oid OIDnew(oid inc);
*/
gdk_export gdk_return BAThash(BAT *b, BUN masksize);
+/* support routines for the mosaic approach */
+gdk_export gdk_return MOSheapAlloc(BAT *b, BUN cap);
+
/*
* @- Column Imprints Functions
*
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -439,6 +439,7 @@ BATmaterializet(BAT *b)
/* cleanup possible ACC's */
HASHdestroy(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
b->T->heap.filename = NULL;
if (HEAPalloc(&b->T->heap, cnt, sizeof(oid)) != GDK_SUCCEED) {
@@ -781,6 +782,7 @@ VIEWdestroy(BAT *b)
if (b->T->hash)
HASHremove(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
VIEWunlink(b);
if (b->htype && !b->H->heap.parentid) {
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -449,6 +449,7 @@ BATextend(BAT *b, BUN newcap)
return GDK_FAIL;
HASHdestroy(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
return GDK_SUCCEED;
}
@@ -492,6 +493,7 @@ BATclear(BAT *b, int force)
/* kill all search accelerators */
HASHdestroy(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
/* we must dispose of all inserted atoms */
if ((b->batDeleted == b->batInserted || force) &&
@@ -592,6 +594,7 @@ BATfree(BAT *b)
b->T->props = NULL;
HASHdestroy(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
if (b->htype)
HEAPfree(&b->H->heap, 0);
else
@@ -1179,6 +1182,7 @@ BUNins(BAT *b, const void *h, const void
}
}
IMPSdestroy(b); /* no support for inserts in imprints yet */
+ MOSheapDestroy(b);
return GDK_SUCCEED;
bunins_failed:
return GDK_FAIL;
@@ -1272,6 +1276,7 @@ BUNappend(BAT *b, const void *t, bit for
IMPSdestroy(b); /* no support for inserts in imprints yet */
+ MOSheapDestroy(b);
/* first adapt the hashes; then the user-defined accelerators.
* REASON: some accelerator updates (qsignature) use the hashes!
@@ -1450,6 +1455,7 @@ BUNdelete_(BAT *b, BUN p, bit force)
b->batCount--;
b->batDirty = 1; /* bat is dirty */
IMPSdestroy(b); /* no support for inserts in imprints yet */
+ MOSheapDestroy(b);
return p;
}
@@ -2757,7 +2763,7 @@ BATassertHeadProps(BAT *b)
p = BUNfirst(b);
q = BUNlast(b);
- assert(b->H->heap.compressed || b->H->heap.free >= headsize(b,
BUNlast(b)));
+ assert(b->H->heap.free >= headsize(b, BUNlast(b)));
if (b->htype != TYPE_void) {
assert(b->batCount <= b->batCapacity);
assert(b->H->heap.size >= b->H->heap.free);
@@ -2822,7 +2828,7 @@ BATassertHeadProps(BAT *b)
PROPDEBUG { /* only do a scan if property checking is requested */
/* if compressed, don't look at the data */
- if (!b->H->heap.compressed && (b->hsorted || b->hrevsorted ||
!b->hkey)) {
+ if ((b->hsorted || b->hrevsorted || !b->hkey)) {
/* if sorted (either way), or we don't have to
* prove uniqueness, we can do a simple
* scan */
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -690,6 +690,7 @@ BATappend(BAT *b, BAT *n, bit force)
}
IMPSdestroy(b); /* imprints do not support updates yet */
+ MOSheapDestroy(b);
/* a hash is useless for void bats */
if (b->H->hash)
HASHremove(BATmirror(b));
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -752,9 +752,6 @@ heapinit(COLrec *col, const char *buf, i
col->dense = (properties & 0x0200) != 0;
col->nonil = (properties & 0x0400) != 0;
col->nil = (properties & 0x0800) != 0;
- if ((col->heap.compressed = (properties & 0x1000) != 0) != 0 &&
- bbpversion <= GDKLIBRARY_NOCOMPRESS)
- GDKfatal("BBPinit: inconsistent entry in BBP.dir: compression
flag set in version without compression\n");
col->nosorted = (BUN) nosorted;
col->norevsorted = (BUN) norevsorted;
col->seq = base < 0 ? oid_nil : (oid) base;
@@ -1228,8 +1225,7 @@ heap_entry(FILE *fp, COLrec *col)
(((unsigned short) col->key & 0x01) << 8) |
((unsigned short) col->dense << 9) |
((unsigned short) col->nonil << 10) |
- ((unsigned short) col->nil << 11) |
- ((unsigned short) col->heap.compressed << 12),
+ ((unsigned short) col->nil << 11) ,
col->nokey[0],
col->nokey[1],
col->nosorted,
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -167,6 +167,8 @@ void BBPdump(void); /* never called: fo
__attribute__((__visibility__("hidden")));
__hidden void IMPSdestroy(BAT *b)
__attribute__((__visibility__("hidden")));
+__hidden void MOSheapDestroy(BAT *b)
+ __attribute__((__visibility__("hidden")));
__hidden int IMPSgetbin(int tpe, bte bits, const char *restrict bins, const
void *restrict v)
__attribute__((__visibility__("hidden")));
#ifndef NDEBUG
diff --git a/gdk/gdk_storage.c b/gdk/gdk_storage.c
--- a/gdk/gdk_storage.c
+++ b/gdk/gdk_storage.c
@@ -987,6 +987,7 @@ BATdelete(BAT *b)
b = loaded;
HASHdestroy(b);
IMPSdestroy(b);
+ MOSheapDestroy(b);
}
assert(!b->H->heap.base || !b->T->heap.base || b->H->heap.base !=
b->T->heap.base);
if (b->batCopiedtodisk || (b->H->heap.storage != STORE_MEM)) {
diff --git a/monetdb5/modules/mal/mosaic.c b/monetdb5/modules/mal/mosaic.c
--- a/monetdb5/modules/mal/mosaic.c
+++ b/monetdb5/modules/mal/mosaic.c
@@ -37,16 +37,18 @@
char
*MOSfiltername[]={"literal","runlength","dictionary","delta","linear","frame","prefix","EOL"};
BUN MOSblocklimit = 100000;
+str MOScompressInternal(Client cntxt, bat *ret, bat *bid, MOStask task, int
debug);
+
static void
MOSinit(MOStask task, BAT *b){
char *base;
if( isVIEW(b))
b= BATdescriptor(VIEWtparent(b));
assert(b);
- base = b->T->heap.base;
+ base = b->T->mosaic->base;
assert(base);
task->type = b->ttype;
- task->b = b;
+ task->bsrc = b;
task->hdr = (MosaicHdr) base;
base += MosaicHdrSize;
task->blk = (MosaicBlk) base;
@@ -62,11 +64,10 @@ static void
MOSdumpTask(Client cntxt,MOStask task)
{
int i;
- dbl perc = task->size/100.0;
mnstr_printf(cntxt->fdout,"# ");
- mnstr_printf(cntxt->fdout,"clk " LLFMT"\tsizes
%10lld\t%10lld\t%3.0f%%\t%10.3fx\t",
- task->timer,task->size,task->xsize, task->xsize/perc,
task->xsize ==0 ? 0:(flt)task->size/task->xsize);
+ mnstr_printf(cntxt->fdout,"clk "LLFMT"\tsizes "BUNFMT"\t%3.0fx\t",
+ task->timer, task->bsrc->T->mosaic->free, (flt)
task->bsrc->T->heap.free/task->bsrc->T->mosaic->free);
for ( i=0; i < MOSAIC_METHODS -1; i++)
if( task->filter[i])
mnstr_printf(cntxt->fdout, "%s["LLFMT ","LLFMT "]\t" ,
MOSfiltername[i], task->hdr->blks[i], task->hdr->elms[i]);
@@ -91,7 +92,7 @@ MOSlayout(Client cntxt, BAT *b, BAT *bte
for( i = 0; i< MOSAIC_METHODS; i++)
task->filter[i]= strstr(compressionscheme,
MOSfiltername[i]) != 0;
bid = b->batCacheid;
- MOScompressInternal(cntxt, &ret, &bid, task,0,FALSE);
+ MOScompressInternal(cntxt, &ret, &bid, task,FALSE);
if( ret == 0)
throw(MAL,"mosaic.layout","Compression failed");
bn = BATdescriptor(ret);
@@ -221,7 +222,7 @@ MOSdump(Client cntxt, MalBlkPtr mb, MalS
if ((b = BATdescriptor(bid)) == NULL)
throw(MAL,"mosaic.dump",INTERNAL_BAT_ACCESS);
- if ( !b->T->heap.compressed){
+ if ( !b->T->mosaic){
BBPunfix(bid);
return MAL_SUCCEED;
}
@@ -348,20 +349,19 @@ MOSoptimizer(Client cntxt, MOStask task,
}
str
-MOScompressInternal(Client cntxt, bat *ret, bat *bid, MOStask task, int
inplace, int debug)
+MOScompressInternal(Client cntxt, bat *ret, bat *bid, MOStask task, int debug)
{
- BAT *bsrc; // the source BAT
- BAT *bcompress; // the BAT that will contain the compressed version
+ BAT *bsrc; // the BAT to be augmented with a compressed
heap
str msg = MAL_SUCCEED;
int cand;
int tpe, typewidth;
*ret = 0;
- if ((bcompress = BATdescriptor(*bid)) == NULL)
+ if ((bsrc = BATdescriptor(*bid)) == NULL)
throw(MAL, "mosaic.compress", INTERNAL_BAT_ACCESS);
- switch( tpe =ATOMbasetype(bcompress->ttype)){
+ switch( tpe =ATOMbasetype(bsrc->ttype)){
case TYPE_bit:
case TYPE_bte:
case TYPE_sht:
@@ -379,84 +379,51 @@ MOScompressInternal(Client cntxt, bat *r
break;
default:
// don't compress them
- BBPkeepref(*ret = bcompress->batCacheid);
+ BBPkeepref(*ret = bsrc->batCacheid);
return msg;
}
- if (bcompress->T->heap.compressed) {
- BBPkeepref(*ret = bcompress->batCacheid);
- return msg; // don't compress twice
- }
-
- if ( isVIEWCOMBINE(bcompress) || BATcount(bcompress) < MIN_INPUT_COUNT
){
+ if ( isVIEWCOMBINE(bsrc) || BATcount(bsrc) < MIN_INPUT_COUNT ){
/* no need to compress */
- BBPkeepref(*ret = bcompress->batCacheid);
+ BBPkeepref(*ret = bsrc->batCacheid);
return msg;
}
#ifdef _DEBUG_MOSAIC_
mnstr_printf(cntxt->fdout,"#compress bat %d \n",*bid);
#endif
- bsrc = BATcopy(bcompress, TYPE_void, bcompress->ttype, TRUE,TRANSIENT);
- if (bsrc == NULL) {
- BBPunfix(bcompress->batCacheid);
- throw(MAL,"mosaic.compress", MAL_MALLOC_FAIL);
- }
- BATseqbase(bsrc,bcompress->hseqbase);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
