Changeset: 1ee6709546f5 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/1ee6709546f5
Modified Files:
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_batop.c
gdk/gdk_calc.c
gdk/gdk_firstn.c
gdk/gdk_group.c
gdk/gdk_join.c
gdk/gdk_logger.c
gdk/gdk_private.h
gdk/gdk_sample.c
gdk/gdk_select.c
monetdb5/modules/atoms/json.c
monetdb5/modules/mal/batMask.c
monetdb5/modules/mal/mkey.c
sql/backends/monet5/generator/generator.c
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_table.c
testing/Mtest.py.in
Branch: default
Log Message:
Use BAT iterators throughout the BAT join code.
diffs (truncated from 4878 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -2311,6 +2311,7 @@ typedef struct BATiter {
Heap *vh;
BUN count;
BUN baseoff;
+ oid hseq;
oid tseq;
BUN hfree, vhfree;
BUN nokey[2];
@@ -2331,11 +2332,12 @@ typedef struct BATiter {
copiedtodisk:1,
transient:1,
ascii:1;
- bat ustr;
restrict_t restricted:2;
#ifndef NDEBUG
bool locked:1;
#endif
+ bat id;
+ bat ustr;
union {
oid tvid;
bool tmsk;
@@ -2350,6 +2352,7 @@ bat_iterator_nolock(BAT *b)
const bool isview = VIEWtparent(b) != 0;
return (BATiter) {
.b = b,
+ .id = b->batCacheid,
.h = b->theap,
.base = b->theap->base
? b->theap->base + (b->tbaseoff << b->tshift)
@@ -2360,6 +2363,7 @@ bat_iterator_nolock(BAT *b)
.width = b->twidth,
.shift = b->tshift,
.type = b->ttype,
+ .hseq = b->hseqbase,
.tseq = b->tseqbase,
/* don't use b->theap->free in case b is a slice */
.hfree = b->ttype
@@ -2487,6 +2491,15 @@ bat_iterator_end(BATiter *bip)
*bip = (BATiter) {0};
}
+ATTR_PURE
+static inline bat
+VIEWtparentbi(const BATiter *bi)
+{
+ return bi->h == NULL || bi->h->parentid == bi->id
+ ? 0
+ : bi->h->parentid;
+}
+
gdk_export gdk_return HEAP_initialize(
Heap *heap, /* nbytes -- Initial size of the heap. */
size_t nbytes, /* alignment -- for objects on the heap. */
@@ -2914,29 +2927,35 @@ BUNtail(BATiter *bi, BUN p)
* works with any TYPE_void or TYPE_oid bat */
ATTR_PURE
static inline oid
-BUNtoid(BAT *b, BUN p)
+BUNtoid(BATiter *bi, BUN p)
{
- assert(ATOMtype(b->ttype) == TYPE_oid);
+ assert(ATOMtype(bi->type) == TYPE_oid);
/* BATcount is the number of valid entries, so with
* exceptions, the last value can well be larger than
- * b->tseqbase + BATcount(b) */
- assert(p < BATcount(b));
- assert(b->ttype == TYPE_void || b->tvheap == NULL);
- if (is_oid_nil(b->tseqbase)) {
- if (b->ttype == TYPE_void)
+ * bi->seqbase + BATcount(b) */
+ assert(p < bi->count);
+ assert(bi->type == TYPE_void || bi->vh == NULL);
+ if (is_oid_nil(bi->tseq)) {
+ if (bi->type == TYPE_void)
return oid_nil;
- MT_lock_set(&b->theaplock);
- oid o = ((const oid *) b->theap->base)[p + b->tbaseoff];
- MT_lock_unset(&b->theaplock);
- return o;
+ return ((const oid *) bi->base)[p];
+ }
+ if (bi->type == TYPE_oid || bi->vh == NULL) {
+ return bi->tseq + p;
}
- if (b->ttype == TYPE_oid || b->tvheap == NULL) {
- return b->tseqbase + p;
- }
- /* b->tvheap != NULL, so we know there will be no parallel
- * modifications (so no locking) */
- BATiter bi = bat_iterator_nolock(b);
- return * (const oid *) BUNtpos(&bi, p);
+ /* bi->vh != NULL, so we know there will be no parallel
+ * modifications */
+ return * (const oid *) BUNtpos(bi, p);
+}
+
+ATTR_PURE
+static inline oid
+fetchtoid(BAT *b, BUN p)
+{
+ BATiter bi = bat_iterator(b);
+ oid o = BUNtoid(&bi, p);
+ bat_iterator_end(&bi);
+ return o;
}
gdk_export BUN BATcount_no_nil(BAT *b, BAT *s);
@@ -3025,6 +3044,7 @@ BATtdense(const BAT *b)
return !is_oid_nil(b->tseqbase) &&
(b->tvheap == NULL || b->tvheap->free == 0);
}
+
ATTR_PURE
static inline bool
BATtdensebi(const BATiter *bi)
@@ -3043,6 +3063,13 @@ BATtvoid(const BAT *b)
ATTR_PURE
static inline bool
+BATtvoidbi(const BATiter *bi)
+{
+ return bi->type == TYPE_void || BATtdensebi(bi);
+}
+
+ATTR_PURE
+static inline bool
BATtkey(const BAT *b)
{
return b->tkey || BATtdense(b);
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -86,7 +86,7 @@ BATgroupaggrinit2(bool pipeline,
} else if (e == NULL) {
if (g->tmaxpos != BUN_NONE) {
min = 0;
- max = BUNtoid(g, g->tmaxpos);
+ max = fetchtoid(g, g->tmaxpos);
} else {
min = oid_nil; /* note that oid_nil > 0! (unsigned) */
max = 0;
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -1055,7 +1055,7 @@ BATappend2(BAT *b, BAT *n, BAT *s, bool
if (b->ttype != TYPE_void && b->tsorted && BATtdense(b) &&
(!BATtdensebi(&ni) ||
ci.tpe != cand_dense ||
- 1 + *(oid *) BUNtloc(&bi, last) != BUNtoid(n, ci.seq -
hseq))) {
+ 1 + *(oid *) BUNtloc(&bi, last) != fetchtoid(n, ci.seq -
hseq))) {
b->tseqbase = oid_nil;
}
b->tnonil &= ni.nonil;
@@ -1438,7 +1438,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
if (positions) {
updid = autoincr ? pos++ : *positions++;
} else {
- updid = BUNtoid(p, i);
+ updid = fetchtoid(p, i);
}
if (updid < b->hseqbase ||
@@ -1484,7 +1484,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
}
} else {
for (j = 1; j < ni.count - i;
j++)
- if (BUNtoid(p, i + j)
!= updid + j)
+ if (fetchtoid(p, i + j)
!= updid + j)
break;
}
if (j > 1) {
@@ -1695,7 +1695,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
if (positions) {
updid = autoincr ? pos++ : *positions++;
} else {
- updid = BUNtoid(p, i);
+ updid = fetchtoid(p, i);
}
if (updid < b->hseqbase ||
@@ -1756,6 +1756,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
}
bi = bat_iterator_nolock(b);
+ MT_lock_set(&b->theaplock);
/* we copy all of n, so if there are nils in n we get
* nils in b (and else we don't know) */
b->tnil = ni.nil;
@@ -1763,6 +1764,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
* there are no nils in b afterward if there weren't
* any in either b or n to begin with */
b->tnonil &= ni.nonil;
+ MT_lock_unset(&b->theaplock);
/* if there is no hash, we don't start the loop, if
* there is only a persisted hash, it will get destroyed
* in the first iteration, after which there is no hash
@@ -1788,7 +1790,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
* the new min/max of b if those of n
* are smaller/larger than the old */
if (bi.minpos != BUN_NONE) {
- if (v <= BUNtoid(b, bi.minpos))
+ if (v <= fetchtoid(b, bi.minpos))
bi.minpos = pos;
else if (pos <= bi.minpos && bi.minpos
< pos + ni.count)
bi.minpos = BUN_NONE;
@@ -1805,7 +1807,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
v--;
}
if (bi.maxpos != BUN_NONE) {
- if (v >= BUNtoid(b, bi.maxpos))
+ if (v >= fetchtoid(b, bi.maxpos))
bi.maxpos = pos + ni.count - 1;
else if (pos <= bi.maxpos && bi.maxpos
< pos + ni.count)
bi.maxpos = BUN_NONE;
@@ -1863,7 +1865,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
/* assert(!autoincr) */
updid = *positions++;
} else {
- updid = BUNtoid(p, i);
+ updid = fetchtoid(p, i);
}
if (updid < b->hseqbase ||
@@ -1905,7 +1907,7 @@ BATappend_or_update(BAT *b, BAT *p, cons
}
} else {
for (j = 1; j < ni.count - i;
j++) {
- if (BUNtoid(p, i + j)
!= updid + j)
+ if (fetchtoid(p, i + j)
!= updid + j)
break;
}
}
@@ -2229,47 +2231,47 @@ BATslice(BAT *b, BUN l, BUN h)
#define BAT_ORDERED(TPE) \
do { \
- const TPE *restrict vals = Tloc(b, 0); \
- for (BUN q = BATcount(b), p = 1; p < q; p++) { \
+ const TPE *restrict vals = bi->base; \
+ for (BUN q = bi->count, p = 1; p < q; p++) { \
if (vals[p - 1] > vals[p]) { \
- b->tnosorted = p; \
- TRC_DEBUG(ALGO, "Fixed nosorted(" BUNFMT ") for
" ALGOBATFMT " (" LLFMT " usec)\n", p, ALGOBATPAR(b), GDKusec() - t0); \
+ bi->nosorted = p; \
+ TRC_DEBUG(ALGO, "Fixed nosorted(" BUNFMT ") for
" ALGOBATFMT " (" LLFMT " usec)\n", p, ALGOBATIPAR(bi), GDKusec() - t0); \
goto doreturn; \
} else if (vals[p - 1] < vals[p]) { \
- if (!b->trevsorted && b->tnorevsorted == 0) { \
- b->tnorevsorted = p; \
- TRC_DEBUG(ALGO, "Fixed norevsorted("
BUNFMT ") for " ALGOBATFMT "\n", p, ALGOBATPAR(b)); \
+ if (!bi->revsorted && bi->norevsorted == 0) { \
+ bi->norevsorted = p; \
+ TRC_DEBUG(ALGO, "Fixed norevsorted("
BUNFMT ") for " ALGOBATFMT "\n", p, ALGOBATIPAR(bi)); \
} \
- } else if (!b->tkey && b->tnokey[1] == 0) { \
- b->tnokey[0] = p - 1; \
- b->tnokey[1] = p; \
- TRC_DEBUG(ALGO, "Fixed nokey(" BUNFMT ","
BUNFMT") for " ALGOBATFMT "\n", p - 1, p, ALGOBATPAR(b)); \
+ } else if (!bi->key && bi->nokey[1] == 0) { \
+ bi->nokey[0] = p - 1; \
+ bi->nokey[1] = p; \
+ TRC_DEBUG(ALGO, "Fixed nokey(" BUNFMT ","
BUNFMT") for " ALGOBATFMT "\n", p - 1, p, ALGOBATIPAR(bi)); \
} \
} \
} while (0)
#define BAT_ORDERED_FP(TPE) \
do { \
- const TPE *restrict vals = Tloc(b, 0); \
+ const TPE *restrict vals = bi->base; \
TPE prev = vals[0]; \
bool prevnil = is_##TPE##_nil(prev); \
- for (BUN q = BATcount(b), p = 1; p < q; p++) { \
+ for (BUN q = bi->count, p = 1; p < q; p++) { \
TPE next = vals[p]; \
int cmp = prevnil ? -!(prevnil = is_##TPE##_nil(next))
: (prevnil = is_##TPE##_nil(next)) ? 1 : (prev > next) - (prev < next); \
prev = next; \
if (cmp > 0) { \
- b->tnosorted = bi.nosorted = p; \
- TRC_DEBUG(ALGO, "Fixed nosorted(" BUNFMT ") for
" ALGOBATFMT " (" LLFMT " usec)\n", p, ALGOBATPAR(b), GDKusec() - t0); \
+ bi->nosorted = p; \
+ TRC_DEBUG(ALGO, "Fixed nosorted(" BUNFMT ") for
" ALGOBATFMT " (" LLFMT " usec)\n", p, ALGOBATIPAR(bi), GDKusec() - t0); \
goto doreturn; \
} else if (cmp < 0) { \
- if (!b->trevsorted && b->tnorevsorted == 0) { \
- b->tnorevsorted = bi.norevsorted = p; \
- TRC_DEBUG(ALGO, "Fixed norevsorted("
BUNFMT ") for " ALGOBATFMT "\n", p, ALGOBATPAR(b)); \
+ if (!bi->revsorted && bi->norevsorted == 0) { \
+ bi->norevsorted = p; \
+ TRC_DEBUG(ALGO, "Fixed norevsorted("
BUNFMT ") for " ALGOBATFMT "\n", p, ALGOBATIPAR(bi)); \
} \
- } else if (!b->tkey && b->tnokey[1] == 0) { \
- b->tnokey[0] = bi.nokey[0] = p - 1; \
- b->tnokey[1] = bi.nokey[1] = p; \
- TRC_DEBUG(ALGO, "Fixed nokey(" BUNFMT ","
BUNFMT") for " ALGOBATFMT "\n", p - 1, p, ALGOBATPAR(b)); \
+ } else if (!bi->key && bi->nokey[1] == 0) { \
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
