Changeset: e03f1325052a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e03f1325052a
Modified Files:
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_project.c
gdk/gdk_string.c
Branch: ascii-flag
Log Message:
Implemented a new property "tascii" to indicate string bat is all ASCII.
I.e. only single byte codepoints in the UTF-8 encoding.
diffs (222 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -726,7 +726,8 @@ typedef struct {
nonil:1, /* there are no nils in the column */
nil:1, /* there is a nil in the column */
sorted:1, /* column is sorted in ascending order */
- revsorted:1; /* column is sorted in descending order */
+ revsorted:1, /* column is sorted in descending order */
+ ascii:1; /* string column is fully ASCII (7 bit) */
BUN nokey[2]; /* positions that prove key==FALSE */
BUN nosorted; /* position that proves sorted==FALSE */
BUN norevsorted; /* position that proves revsorted==FALSE */
@@ -820,6 +821,7 @@ typedef struct BAT {
#define tseqbase T.seq
#define tsorted T.sorted
#define trevsorted T.revsorted
+#define tascii T.ascii
#define torderidx T.orderidx
#define twidth T.width
#define tshift T.shift
@@ -1079,7 +1081,8 @@ typedef struct BATiter {
hdirty:1,
vhdirty:1,
copiedtodisk:1,
- transient:1;
+ transient:1,
+ ascii:1;
restrict_t restricted:2;
#ifndef NDEBUG
bool locked:1;
@@ -1126,6 +1129,7 @@ bat_iterator_nolock(BAT *b)
.nil = b->tnil,
.sorted = b->tsorted,
.revsorted = b->trevsorted,
+ .ascii = b->tascii,
/* only look at heap dirty flag if we own it */
.hdirty = b->theap->parentid == b->batCacheid &&
b->theap->dirty,
/* also, if there is no vheap, it's not dirty */
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -117,6 +117,7 @@ VIEWcreate(oid seq, BAT *b)
bn->tshift = b->tshift;
bn->tnonil = b->tnonil;
bn->tnil = b->tnil;
+ bn->tascii = b->tascii;
bn->tnokey[0] = b->tnokey[0];
bn->tnokey[1] = b->tnokey[1];
bn->tnosorted = b->tnosorted;
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -82,6 +82,7 @@ BATcreatedesc(oid hseq, int tt, bool hea
.tnil = false,
.tsorted = ATOMlinear(tt),
.trevsorted = ATOMlinear(tt),
+ .tascii = tt == TYPE_str,
.tseqbase = oid_nil,
.tminpos = BUN_NONE,
.tmaxpos = BUN_NONE,
@@ -895,6 +896,7 @@ COLcopy(BAT *b, int tt, bool writable, r
memcpy(bn->tvheap->base, bi.vh->base,
bi.vhfree);
bn->tvheap->free = bi.vhfree;
bn->tvheap->dirty = true;
+ bn->tascii = bi.ascii;
}
/* make sure we use the correct capacity */
@@ -2624,6 +2626,17 @@ BATmode(BAT *b, bool transient)
#define assert(test) ((void) ((test) || (TRC_CRITICAL_ENDIF(CHECK_,
"Assertion `%s' failed\n", #test), 0)))
#endif
+static void
+assert_ascii(const char *s)
+{
+ if (!strNil(s)) {
+ while (*s) {
+ assert((*s & 0x80) == 0);
+ s++;
+ }
+ }
+}
+
/* Assert that properties are set correctly.
*
* A BAT can have a bunch of properties set. Mostly, the property
@@ -2652,6 +2665,9 @@ BATmode(BAT *b, bool transient)
* and one before are not ordered correctly).
* nokey Pair of BUN positions that proof not all values are
* distinct (i.e. values at given locations are equal).
+ * ascii Only valid for TYPE_str columns: all strings in the column
+ * are ASCII, i.e. the UTF-8 encoding for all characters is a
+ * single byte.
*
* Note that the functions BATtseqbase and BATkey also set more
* properties than you might suspect. When setting properties on a
@@ -2763,6 +2779,8 @@ BATassertProps(BAT *b)
assert(is_oid_nil(b->tseqbase) || b->ttype == TYPE_oid || b->ttype ==
TYPE_void);
/* a column cannot both have and not have NILs */
assert(!b->tnil || !b->tnonil);
+ /* only string columns can be ASCII */
+ assert(!b->tascii || ATOMstorage(b->ttype) == TYPE_str);
if (b->ttype == TYPE_void) {
assert(b->tshift == 0);
assert(b->twidth == 0);
@@ -2914,6 +2932,8 @@ BATassertProps(BAT *b)
assert(!b->tnonil || !isnil);
assert(b->ttype != TYPE_flt ||
!isinf(*(flt*)valp));
assert(b->ttype != TYPE_dbl ||
!isinf(*(dbl*)valp));
+ if (b->tascii)
+ assert_ascii(valp);
if (minbound && !isnil) {
cmp = cmpf(minbound, valp);
assert(cmp <= 0);
@@ -2994,6 +3014,8 @@ BATassertProps(BAT *b)
assert(!isnil || !notnull);
assert(b->ttype != TYPE_flt ||
!isinf(*(flt*)valp));
assert(b->ttype != TYPE_dbl ||
!isinf(*(dbl*)valp));
+ if (b->tascii)
+ assert_ascii(valp);
if (minbound && !isnil) {
cmp = cmpf(minbound, valp);
assert(cmp <= 0);
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -91,6 +91,7 @@ insert_string_bat(BAT *b, BATiter *ni, s
HEAPdecref(b->tvheap, bid == b->batCacheid);
HEAPincref(ni->vh);
b->tvheap = ni->vh;
+ b->tascii = ni->ascii;
MT_lock_unset(&b->theaplock);
BBPretain(ni->vh->parentid);
if (bid != b->batCacheid)
@@ -148,6 +149,7 @@ insert_string_bat(BAT *b, BATiter *ni, s
memcpy(b->tvheap->base + toff, ni->vh->base,
ni->vhfree);
b->tvheap->free = toff + ni->vhfree;
b->tvheap->dirty = true;
+ b->tascii &= ni->ascii;
MT_lock_unset(&b->theaplock);
}
}
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -519,7 +519,7 @@ heapinit(BAT *b, const char *buf,
if (strcmp(type, "wkba") == 0)
GDKwarning("type wkba (SQL name: GeometryA) is deprecated\n");
- if (properties & ~0x0F81) {
+ if (properties & ~0x1F81) {
TRC_CRITICAL(GDK, "unknown properties are set: incompatible
database on line %d of BBP.dir\n", lineno);
return -1;
}
@@ -557,6 +557,7 @@ heapinit(BAT *b, const char *buf,
b->tkey = (properties & 0x0100) != 0;
b->tnonil = (properties & 0x0400) != 0;
b->tnil = (properties & 0x0800) != 0;
+ b->tascii = (properties & 0x1000) != 0;
b->tnosorted = (BUN) nosorted;
b->tnorevsorted = (BUN) norevsorted;
b->tunique_est = 0.0;
@@ -2281,7 +2282,8 @@ heap_entry(FILE *fp, BATiter *bi, BUN si
((unsigned short) bi->key << 8) |
((unsigned short) BATtdensebi(bi) << 9) |
((unsigned short) bi->nonil << 10) |
- ((unsigned short) bi->nil << 11),
+ ((unsigned short) bi->nil << 11) |
+ ((unsigned short) bi->ascii << 12),
bi->nokey[0] >= size || bi->nokey[1] >= size ? 0 :
bi->nokey[0],
bi->nokey[0] >= size || bi->nokey[1] >= size ? 0 :
bi->nokey[1],
bi->nosorted >= size ? 0 : bi->nosorted,
diff --git a/gdk/gdk_project.c b/gdk/gdk_project.c
--- a/gdk/gdk_project.c
+++ b/gdk/gdk_project.c
@@ -818,6 +818,7 @@ BATproject2(BAT *restrict l, BAT *restri
bn->ttype = r1i.type;
bn->twidth = r1i.width;
bn->tshift = r1i.shift;
+ bn->tascii = r1i.ascii;
}
if (!BATtdensebi(&r1i) || (r2 && !BATtdensebi(&r2i)))
@@ -1076,8 +1077,9 @@ BATprojectchain(BAT **bats)
}
if (stringtrick) {
bn->tnil = false;
- bn->tnonil = b->tnonil;
+ bn->tnonil = bi.nonil;
bn->tkey = false;
+ bn->tascii = bi.ascii;
assert(bn->tvheap == NULL);
bn->tvheap = bi.vh;
HEAPincref(bi.vh);
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -206,6 +206,7 @@ strPut(BAT *b, var_t *dst, const void *V
/* fill should solve initialization problems within valgrind */
memset(h->base, 0, h->size);
#endif
+ b->tascii = true;
}
off = strHash(v);
@@ -312,6 +313,15 @@ strPut(BAT *b, var_t *dst, const void *V
}
*bucket = (stridx_t) pos; /* set bucket to the new string */
+ if (b->tascii && !strNil(v)) {
+ for (const uint8_t *p = (const uint8_t *) v; *p; p++) {
+ if (*p >= 128) {
+ b->tascii = false;
+ break;
+ }
+ }
+ }
+
return *dst;
}
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
