Changeset: 822bcb77f1da for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=822bcb77f1da
Modified Files:
monetdb5/modules/atoms/str.c
sql/backends/monet5/UDF/udf/udf.c
Branch: Nov2019
Log Message:
Backported recent string function fixes from Jun2020 into Nov2019
diffs (truncated from 513 to 300 lines):
diff --git a/monetdb5/modules/atoms/str.c b/monetdb5/modules/atoms/str.c
--- a/monetdb5/modules/atoms/str.c
+++ b/monetdb5/modules/atoms/str.c
@@ -3156,68 +3156,123 @@ UTF8_strtail(const char *s, int pos)
return (str) s;
}
+static inline str
+UTF8_strncpy(char *restrict dst, const char *restrict s, int n)
+{
+ UTF8_assert(s);
+ while (*s && n) {
+ if ((*s & 0xF8) == 0xF0) {
+ /* 4 byte UTF-8 sequence */
+ *dst++ = *s++;
+ *dst++ = *s++;
+ *dst++ = *s++;
+ *dst++ = *s++;
+ } else if ((*s & 0xF0) == 0xE0) {
+ /* 3 byte UTF-8 sequence */
+ *dst++ = *s++;
+ *dst++ = *s++;
+ *dst++ = *s++;
+ } else if ((*s & 0xE0) == 0xC0) {
+ /* 2 byte UTF-8 sequence */
+ *dst++ = *s++;
+ *dst++ = *s++;
+ } else {
+ /* 1 byte UTF-8 "sequence" */
+ *dst++ = *s++;
+ }
+ n--;
+ }
+ *dst = '\0';
+ return dst;
+}
+
+static inline str
+UTF8_offset(char *restrict s, int n)
+{
+ UTF8_assert(s);
+ while (*s && n) {
+ if ((*s & 0xF8) == 0xF0) {
+ /* 4 byte UTF-8 sequence */
+ s += 4;
+ } else if ((*s & 0xF0) == 0xE0) {
+ /* 3 byte UTF-8 sequence */
+ s += 3;
+ } else if ((*s & 0xE0) == 0xC0) {
+ /* 2 byte UTF-8 sequence */
+ s += 2;
+ } else {
+ /* 1 byte UTF-8 "sequence" */
+ s++;
+ }
+ n--;
+ }
+ return s;
+}
+
static str
convertCase(BAT *from, BAT *to, str *res, const char *src, const char *malfunc)
{
- BATiter toi = bat_iterator(to);
- BATiter fromi = bat_iterator(from);
size_t len = strlen(src);
char *dst;
const char *end = src + len;
- BUN UTF8_CONV_r;
bool lower_to_upper = from == UTF8_toUpperFrom;
- if (strNil(src)) {
+ if (GDK_STRNIL(src)) {
*res = GDKstrdup(str_nil);
} else {
- *res = GDKmalloc(len + 1);
- if (*res != NULL) {
- dst = *res;
- while (src < end) {
- int c;
+ if (BAThash(from) != GDK_SUCCEED ||
+ (*res = GDKmalloc(len + 1)) == NULL) {
+ throw(MAL, malfunc, SQLSTATE(HY013) MAL_MALLOC_FAIL);
+ }
+ Hash *h = from->thash;
+ dst = *res;
+ while (src < end) {
+ int c;
- UTF8_GETCHAR(c, src);
- if ((c & 0x80) == 0) {
- /* for ASCII characters we don't need
to do a hash
- * lookup */
- if (lower_to_upper) {
- if ('a' <= c && c <= 'z')
- c += 'A' - 'a';
- } else {
- if ('A' <= c && c <= 'Z')
- c += 'a' - 'A';
- }
+ UTF8_GETCHAR(c, src);
+ if ((c & 0x80) == 0) {
+ /* for ASCII characters we don't need to do a
hash
+ * lookup */
+ if (lower_to_upper) {
+ if ('a' <= c && c <= 'z')
+ c += 'A' - 'a';
} else {
- /* use hash, even though BAT is sorted
*/
- HASHfnd_int(UTF8_CONV_r, fromi, &c);
- if (UTF8_CONV_r != BUN_NONE)
- c = *(int*) BUNtloc(toi,
UTF8_CONV_r);
+ if ('A' <= c && c <= 'Z')
+ c += 'a' - 'A';
}
- if (dst + UTF8_CHARLEN(c) > *res + len) {
- /* doesn't fit, so allocate more space;
- * also allocate enough for the rest of
the
- * source */
- size_t off = dst - *res;
+ } else {
+ /* use hash, even though BAT is sorted */
+ for (BUN hb = HASHget(h, hash_int(h, &c));
+ hb != HASHnil(h);
+ hb = HASHgetlink(h, hb)) {
+ if (c == ((int *)
from->theap.base)[hb]) {
+ c = ((int *)
to->theap.base)[hb];
+ break;
+ }
+ }
+ }
+ if (dst + UTF8_CHARLEN(c) > *res + len) {
+ /* doesn't fit, so allocate more space;
+ * also allocate enough for the rest of the
+ * source */
+ size_t off = dst - *res;
- dst = GDKrealloc(*res, (len += 4 + (end
- src)) + 1);
- if (dst == NULL) {
- /* if realloc fails, original
buffer is still
- * allocated, so free it */
- GDKfree(*res);
- goto hashfnd_failed;
- }
- *res = dst;
- dst = *res + off;
+ dst = GDKrealloc(*res, (len += 4 + (end - src))
+ 1);
+ if (dst == NULL) {
+ /* if realloc fails, original buffer is
still
+ * allocated, so free it */
+ GDKfree(*res);
+ throw(MAL, malfunc, SQLSTATE(HY013)
MAL_MALLOC_FAIL);
}
- UTF8_PUTCHAR(c, dst);
+ *res = dst;
+ dst = *res + off;
}
- *dst = 0;
+ UTF8_PUTCHAR(c, dst);
}
+ *dst = 0;
}
if (*res != NULL)
return MAL_SUCCEED;
- hashfnd_failed:
- throw(MAL, malfunc, SQLSTATE(HY013) MAL_MALLOC_FAIL);
illegal:
throw(MAL, malfunc, SQLSTATE(42000) "Illegal Unicode code point");
}
@@ -3301,7 +3356,7 @@ STRConcat(str *res, const str *val1, con
const char *s1 = *val1;
const char *s2 = *val2;
- if (strNil(s1) || strNil(s2)) {
+ if (GDK_STRNIL(s1) || GDK_STRNIL(s2)) {
*res = GDKstrdup(str_nil);
} else {
l1 = strlen(s1);
@@ -3323,7 +3378,7 @@ STRLength(int *res, const str *arg1)
size_t l;
const char *s = *arg1;
- if (strNil(s)) {
+ if (GDK_STRNIL(s)) {
*res = int_nil;
return MAL_SUCCEED;
}
@@ -3341,7 +3396,7 @@ STRBytes(int *res, const str *arg1)
size_t l;
const char *s = *arg1;
- if (strNil(s)) {
+ if (GDK_STRNIL(s)) {
*res = int_nil;
return MAL_SUCCEED;
}
@@ -3357,7 +3412,7 @@ STRTail(str *res, const str *arg1, const
int off = *offset;
const char *s = *arg1;
- if (strNil(s) || is_int_nil(off)) {
+ if (GDK_STRNIL(s) || is_int_nil(off)) {
*res = GDKstrdup(str_nil);
} else {
if (off < 0) {
@@ -3382,7 +3437,7 @@ STRSubString(str *res, const str *arg1,
int off = *offset, l = *length;
const char *s = *arg1;
- if (strNil(s) || is_int_nil(off) || is_int_nil(l)) {
+ if (GDK_STRNIL(s) || is_int_nil(off) || is_int_nil(l)) {
*res = GDKstrdup(str_nil);
if (*res == NULL)
throw(MAL, "str.substring", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
@@ -3447,7 +3502,7 @@ STRWChrAt(int *res, const str *arg1, con
/* 64bit: should have lng arg */
const char *s = *arg1;
- if (strNil(s) || is_int_nil(*at) || *at < 0) {
+ if (GDK_STRNIL(s) || is_int_nil(*at) || *at < 0) {
*res = int_nil;
return MAL_SUCCEED;
}
@@ -3469,7 +3524,7 @@ STRPrefix(bit *res, const str *arg1, con
const char *s = *arg1;
const char *prefix = *arg2;
- if (strNil(s) || strNil(prefix)) {
+ if (GDK_STRNIL(s) || GDK_STRNIL(prefix)) {
*res = bit_nil;
return MAL_SUCCEED;
}
@@ -3485,7 +3540,7 @@ STRSuffix(bit *res, const str *arg1, con
const char *s = *arg1;
const char *suffix = *arg2;
- if (strNil(s) || strNil(suffix)) {
+ if (GDK_STRNIL(s) || GDK_STRNIL(suffix)) {
*res = bit_nil;
return MAL_SUCCEED;
}
@@ -3519,7 +3574,7 @@ STRstrSearch(int *res, const str *haysta
const char *s = *haystack;
const char *s2 = *needle;
- if (strNil(s) || strNil(s2)) {
+ if (GDK_STRNIL(s) || GDK_STRNIL(s2)) {
*res = int_nil;
return MAL_SUCCEED;
}
@@ -3539,7 +3594,7 @@ STRReverseStrSearch(int *res, const str
const char *s = *arg1;
const char *s2 = *arg2;
- if (strNil(s) || strNil(s2)) {
+ if (GDK_STRNIL(s) || GDK_STRNIL(s2)) {
*res = int_nil;
return MAL_SUCCEED;
}
@@ -3564,11 +3619,11 @@ STRsplitpart(str *res, str *haystack, st
{
size_t len;
int f = *field;
- char *p;
+ char *p = NULL;
const char *s = *haystack;
const char *s2 = *needle;
- if (strNil(s) || strNil(s2) || is_int_nil(*field)) {
+ if (GDK_STRNIL(s) || GDK_STRNIL(s2) || is_int_nil(*field)) {
*res = GDKstrdup(str_nil);
if (*res == NULL)
throw(MAL, "str.splitpart", SQLSTATE(HY013)
MAL_MALLOC_FAIL);
@@ -3580,10 +3635,11 @@ STRsplitpart(str *res, str *haystack, st
}
len = strlen(s2);
-
- while ((p = strstr(s, s2)) != NULL && f > 1) {
- s = p + len;
- f--;
+ if (len) {
+ while ((p = strstr(s, s2)) != NULL && f > 1) {
+ s = p + len;
+ f--;
+ }
}
if (f != 1) {
@@ -3744,10 +3800,10 @@ STRRtrim(str *res, const str *arg1)
/* return a list of codepoints in s */
static int *
-trimchars(const char *s, size_t *n)
+trimchars(const char *s, size_t *n, size_t len_s)
{
size_t len = 0;
- int *chars = GDKmalloc(strlen(s) * sizeof(int));
+ int *chars = GDKmalloc(len_s * sizeof(int));
int c;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
