Changeset: ecc593e3ebc2 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ecc593e3ebc2
Modified Files:
monetdb5/modules/mal/pp_hash.c
monetdb5/modules/mal/pp_hash.h
sql/backends/monet5/rel_physical.c
sql/backends/monet5/rel_pphash.c
sql/backends/monet5/sql_pp_statement.c
sql/backends/monet5/sql_pp_statement.h
sql/server/rel_unnest.c
Branch: manti
Log Message:
merged from default
diffs (truncated from 2113 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -9681,6 +9681,7 @@ iaggr sum command iaggr.sum(X_0:bat[:flt
iaggr sum command iaggr.sum(X_0:bat[:dbl], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:flt, X_3:flt,
X_4:lng) IALGfsum_skipnil_flt Return the Kahan/Neumaier summation or nil.
+ialgebra matched command ialgebra.matched(X_0:bat[:bit],
X_1:bat[:oid]):bat[:oid] OAHASHmatched Mark matched rows, only return
rows matched rows on first match.
ialgebra project pattern ialgebra.project(X_0:bat[:oid],
X_1:any_1):bat[:any_1] LALGconstant Project a single value
ialgebra projection command ialgebra.projection(X_0:bat[:oid],
X_1:bat[:any_1]):bat[:any_1] LALGprojection Project.
ialgebra unique command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid],
X_2:bat[:oid], X_3:bit) (X_4:bat[:oid], X_5:bat[:any_1])
LALGgroup_unique Unique rows per group.
diff --git a/clients/Tests/MAL-signatures.test
b/clients/Tests/MAL-signatures.test
--- a/clients/Tests/MAL-signatures.test
+++ b/clients/Tests/MAL-signatures.test
@@ -7322,6 +7322,7 @@ iaggr sum command iaggr.sum(X_0:bat[:flt
iaggr sum command iaggr.sum(X_0:bat[:dbl], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:dbl, X_3:dbl,
X_4:lng) IALGfsum_skipnil Return the Kahan/Neumaier summation or nil.
iaggr sum command iaggr.sum(X_0:bat[:flt], X_1:bit) (X_2:flt, X_3:flt,
X_4:lng) IALGfsum_skipnil_flt Return the Kahan/Neumaier summation or nil.
+ialgebra matched command ialgebra.matched(X_0:bat[:bit],
X_1:bat[:oid]):bat[:oid] OAHASHmatched Mark matched rows, only return
rows matched rows on first match.
ialgebra project pattern ialgebra.project(X_0:bat[:oid],
X_1:any_1):bat[:any_1] LALGconstant Project a single value
ialgebra projection command ialgebra.projection(X_0:bat[:oid],
X_1:bat[:any_1]):bat[:any_1] LALGprojection Project.
ialgebra unique command ialgebra.unique(X_0:bat[:any_1], X_1:bat[:oid],
X_2:bat[:oid], X_3:bit) (X_4:bat[:oid], X_5:bat[:any_1])
LALGgroup_unique Unique rows per group.
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1452,10 +1452,7 @@ typedef enum {
STORE_MEM, /* load into GDKmalloced memory */
STORE_MMAP, /* mmap() into virtual memory */
STORE_PRIV, /* BAT copy of copy-on-write mmap */
- STORE_CMEM, /* load into malloc (not GDKmalloc) memory*/
STORE_NOWN, /* memory not owned by the BAT */
- STORE_MMAPABS, /* mmap() into virtual memory from an
- * absolute path (not part of dbfarm) */
} storage_t;
typedef struct {
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -60,7 +60,7 @@ BUNtpos(BATiter *bi, BUN p)
if (bi->vh) {
oid o;
assert(!is_oid_nil(bi->tseq));
- if (((ccand_t *) bi->vh)->type == CAND_NEGOID) {
+ if (((ccand_t *) bi->vh->base)->type == CAND_NEGOID) {
BUN nexc = (bi->vhfree - sizeof(ccand_t)) / SIZEOF_OID;
o = bi->tseq + p;
if (nexc > 0) {
@@ -90,9 +90,14 @@ BUNtpos(BATiter *bi, BUN p)
uint32_t m = candmask_pop(msk[i]);
if (o + m > p) {
m = msk[i];
- for (i = 0; i < 32; i++) {
- if (m & (1U << i) && ++o == p)
- break;
+ for (int j = 0; j < 32; j++) {
+ if (m & (1U << j)) {
+ if (o == p) {
+ o = i * 32 + j;
+ break;
+ }
+ o++;
+ }
}
break;
}
diff --git a/gdk/gdk_cand.c b/gdk/gdk_cand.c
--- a/gdk/gdk_cand.c
+++ b/gdk/gdk_cand.c
@@ -701,7 +701,9 @@ canditer_init(struct canditer *ci, BAT *
cnt -= 32;
}
}
- ci->ncand = count_mask_bits(ci, 0, cnt);
+ ci->ncand = s->ttype == TYPE_msk
+ ? s->batCount
+ : count_mask_bits(ci, 0, cnt);
return;
default:
MT_UNREACHABLE();
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -49,6 +49,7 @@
#include "gdk_private.h"
#include "mutils.h"
+__attribute__((__access__(read_write, 2)))
static void *
HEAPcreatefile(int farmid, size_t *maxsz, const char *fn)
{
@@ -64,10 +65,19 @@ HEAPcreatefile(int farmid, size_t *maxsz
fn = path;
}
/* round up to multiple of GDK_mmap_pagesize */
- fd = GDKfdlocate(NOFARM, fn, "wb", NULL);
+ fd = GDKfdlocate(NOFARM, fn, "w+b", NULL);
if (fd >= 0) {
+ size_t size = (*maxsz + GDK_mmap_pagesize - 1) &
~(GDK_mmap_pagesize - 1);
+ if (size == 0)
+ size = GDK_mmap_pagesize;
+ if (GDKextendf(fd, size, fn) != GDK_SUCCEED) {
+ close(fd);
+ return NULL;
+ }
close(fd);
- base = GDKload(NOFARM, fn, NULL, *maxsz, maxsz, STORE_MMAP);
+ base = GDKmmap(fn, MMAP_READ | MMAP_WRITE, size);
+ if (base != NULL)
+ *maxsz = size;
}
return base;
}
@@ -355,40 +365,34 @@ HEAPextend(Heap *h, size_t size, bool ma
* to STORE_PRIV (copy-on-write memory mapped files),
* but if the heap file doesn't exist yet, the BAT is
* new and we can use STORE_MMAP */
- int fd = GDKfdlocate(h->farmid, nme, "rb", ext);
+ int fd = GDKfdlocate(h->farmid, nme, "r+b", ext);
if (fd >= 0) {
assert(h->hasfile);
+ gdk_return rc = GDKextendf(fd, size, nme);
close(fd);
- fd = GDKfdlocate(h->farmid, nme, "wb", ext);
- if (fd >= 0) {
- gdk_return rc = GDKextendf(fd, size,
nme);
- close(fd);
- if (rc != GDK_SUCCEED) {
- failure = "h->storage ==
STORE_MEM && can_map && fd >= 0 && GDKextendf() != GDK_SUCCEED";
- goto failed;
- }
- h->storage = h->newstorage ==
STORE_MMAP && !mayshare ? STORE_PRIV : h->newstorage;
- /* make sure we really MMAP */
- if (must_mmap && h->newstorage ==
STORE_MEM)
- h->storage = STORE_MMAP;
- h->newstorage = h->storage;
+ if (rc != GDK_SUCCEED) {
+ failure = "h->storage == STORE_MEM &&
can_map && fd >= 0 && GDKextendf() != GDK_SUCCEED";
+ goto failed;
+ }
+ h->storage = h->newstorage == STORE_MMAP &&
!mayshare ? STORE_PRIV : h->newstorage;
+ /* make sure we really MMAP */
+ if (must_mmap && h->newstorage == STORE_MEM)
+ h->storage = STORE_MMAP;
+ h->newstorage = h->storage;
- h->base = NULL;
- TRC_DEBUG(HEAP, "Converting malloced to
%s mmapped heap %s\n", h->newstorage == STORE_MMAP ? "shared" : "privately",
h->filename);
- /* try to allocate a memory-mapped based
- * heap */
- if (HEAPload(h, nme, ext, false) ==
GDK_SUCCEED) {
- /* copy data to heap and free
old
- * memory */
- memcpy(h->base, bak.base,
bak.free);
- HEAPfree(&bak, false);
- return GDK_SUCCEED;
- }
- failure = "h->storage == STORE_MEM &&
can_map && fd >= 0 && HEAPload() != GDK_SUCCEED";
- /* we failed */
- } else {
- failure = "h->storage == STORE_MEM &&
can_map && fd < 0";
+ h->base = NULL;
+ TRC_DEBUG(HEAP, "Converting malloced to %s
mmapped heap %s\n", h->newstorage == STORE_MMAP ? "shared" : "privately",
h->filename);
+ /* try to allocate a memory-mapped based
+ * heap */
+ if (HEAPload(h, nme, ext, false) ==
GDK_SUCCEED) {
+ /* copy data to heap and free old
+ * memory */
+ memcpy(h->base, bak.base, bak.free);
+ HEAPfree(&bak, false);
+ return GDK_SUCCEED;
}
+ failure = "h->storage == STORE_MEM && can_map
&& fd >= 0 && HEAPload() != GDK_SUCCEED";
+ /* we failed */
} else {
/* no pre-existing heap file, so create a new
* one */
@@ -633,9 +637,6 @@ HEAPfree(Heap *h, bool rmheap)
if (h->storage == STORE_MEM) { /* plain memory */
TRC_DEBUG(HEAP, "HEAPfree %s %zu %p\n", h->filename,
h->size, h->base);
GDKfree(h->base);
- } else if (h->storage == STORE_CMEM) {
- //heap is stored in regular C memory rather than GDK
memory,so we call free()
- free(h->base);
} else if (h->storage != STORE_NOWN) { /* mapped file, or
STORE_PRIV */
gdk_return ret = GDKmunmap(h->base,
h->storage == STORE_PRIV ?
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -2667,7 +2667,7 @@ log_load(const char *fn, logger *lg, cha
goto error;
}
if (!earlyexit) {
- /* in case or process-wal-and-exit, do NOT run
+ /* in case of process-wal-and-exit, do NOT run
* upgrade code, and therefore do NOT update WAL
* version number */
if (lg->postfuncp && (*lg->postfuncp) (lg->funcdata,
lg) != GDK_SUCCEED)
diff --git a/gdk/gdk_private.h b/gdk/gdk_private.h
--- a/gdk/gdk_private.h
+++ b/gdk/gdk_private.h
@@ -148,7 +148,8 @@ FILE *GDKfileopen(int farmid, const char
__attribute__((__malloc__))
__attribute__((__malloc__(fclose, 1)));
char *GDKload(int farmid, const char *nme, const char *ext, size_t size,
size_t *maxsize, storage_t mode)
- __attribute__((__visibility__("hidden")));
+ __attribute__((__visibility__("hidden")))
+ __attribute__((__access__(read_write, 5)));
gdk_return GDKmove(int farmid, const char *dir1, const char *nme1, const char
*ext1, const char *dir2, const char *nme2, const char *ext2, bool report)
__attribute__((__warn_unused_result__))
__attribute__((__visibility__("hidden")));
diff --git a/gdk/gdk_rtree.c b/gdk/gdk_rtree.c
--- a/gdk/gdk_rtree.c
+++ b/gdk/gdk_rtree.c
@@ -159,7 +159,7 @@ RTREEexistsonfile(BAT *b)
if (!b->theap) return false;
int farmid = b->theap->farmid;
- int fd = GDKfdlocate(farmid, filename, "r", "bsrt");
+ int fd = GDKfdlocate(farmid, filename, "rb", "bsrt");
//Do we have the rtree on file?
if (fd == -1) {
diff --git a/gdk/gdk_sample.c b/gdk/gdk_sample.c
--- a/gdk/gdk_sample.c
+++ b/gdk/gdk_sample.c
@@ -32,14 +32,9 @@
/* this is a straightforward implementation of a binary tree */
struct oidtreenode {
- union {
- struct { /* use as a binary tree */
- oid o;
- struct oidtreenode *left;
- struct oidtreenode *right;
- };
- uint64_t r; /* temporary storage for random numbers */
- };
+ oid o;
+ struct oidtreenode *left;
+ struct oidtreenode *right;
};
static bool
@@ -69,115 +64,177 @@ OIDTreeMaybeInsert(struct oidtreenode *t
/* inorder traversal, gives us a sorted BAT */
static void
-OIDTreeToBAT(struct oidtreenode *node, BAT *bn)
+OIDTreeToBAT(struct oidtreenode *node, oid *oids, BUN *n)
{
if (node->left != NULL)
- OIDTreeToBAT(node->left, bn);
- ((oid *) bn->theap->base)[bn->batCount++] = node->o;
+ OIDTreeToBAT(node->left, oids, n);
+ oids[*n] = node->o;
+ (*n)++;
if (node->right != NULL )
- OIDTreeToBAT(node->right, bn);
-}
-
-/* Antiset traversal, give us all values but the ones in the tree */
-static void
-OIDTreeToBATAntiset(struct oidtreenode *node, BAT *bn, oid start, oid stop)
-{
- oid noid;
-
- if (node->left != NULL)
- OIDTreeToBATAntiset(node->left, bn, start, node->o);
- else
- for (noid = start; noid < node->o; noid++)
- ((oid *) bn->theap->base)[bn->batCount++] = noid;
-
- if (node->right != NULL)
- OIDTreeToBATAntiset(node->right, bn, node->o + 1, stop);
- else
- for (noid = node->o+1; noid < stop; noid++)
- ((oid *) bn->theap->base)[bn->batCount++] = noid;
+ OIDTreeToBAT(node->right, oids, n);
}
static BAT *
-do_batsample(oid hseq, BUN cnt, BUN n, random_state_engine rse, MT_Lock *lock)
+do_batsample(oid hseq, BUN cnt, BUN n, random_state_engine rse)
{
BAT *bn;
- BUN slen;
- BUN rescnt;
- struct oidtreenode *tree = NULL;
+ BUN rescnt; /* counter used for several purposes */
ERRORcheck(n > BUN_MAX, "sample size larger than BUN_MAX\n", NULL);
- /* empty sample size */
if (n == 0) {
- bn = BATdense(0, 0, 0);
- } else if (cnt <= n) {
+ /* empty sample size */
+ return BATdense(0, 0, 0);
+ }
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
