Changeset: 26b0c251bdcb for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=26b0c251bdcb
Modified Files:
common/stream/stream.c
Branch: Aug2018
Log Message:
Use similar code to pcre.c to convert UTF-8 to wide characters.
diffs (160 lines):
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -235,86 +235,84 @@ mnstr_init(void)
/* convert a string from UTF-8 to wide characters; the return value is
* freshly allocated */
static wchar_t *
-utf8towchar(const char *s)
-{
- wchar_t *ws;
+utf8towchar(const char *src)
+{
+ wchar_t *dest;
size_t i = 0;
size_t j = 0;
-
- ws = malloc((strlen(s) + 1) * sizeof(wchar_t));
- if (ws == NULL)
- return NULL;
- while (s[j]) {
- if ((s[j] & 0x80) == 0) {
- ws[i++] = s[j++];
- } else if ((s[j] & 0xC0) == 0x80) {
- free(ws);
- return NULL;
- } else if ((s[j] & 0xE0) == 0xC0) {
- ws[i] = (s[j++] & 0x1F) << 6;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i++] |= s[j++] & 0x3F;
- } else if ((s[j] & 0xF0) == 0xE0) {
- ws[i] = (s[j++] & 0x0F) << 12;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i] |= (s[j++] & 0x3F) << 6;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i++] |= s[j++] & 0x3F;
- } else if ((s[j] & 0xF8) == 0xF0) {
-#if SIZEOF_WCHAR_T == 2
- ws[i] = (s[j++] & 0x07) << 8;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
+ uint32_t c;
+
+ /* count how many wchar_t's we need, while also checking for
+ * correctness of the input */
+ while (src[j]) {
+ i++;
+ if ((src[j+0] & 0x80) == 0) {
+ j += 1;
+ } else if ((src[j+0] & 0xE0) == 0xC0
+ && (src[j+1] & 0xC0) == 0x80
+ && (src[j+0] & 0x1E) != 0) {
+ j += 2;
+ } else if ((src[j+0] & 0xF0) == 0xE0
+ && (src[j+1] & 0xC0) == 0x80
+ && (src[j+2] & 0xC0) == 0x80
+ && ((src[j+0] & 0x0F) != 0
+ || (src[j+1] & 0x20) != 0)) {
+ j += 3;
+ } else if ((src[j+0] & 0xF8) == 0xF0
+ && (src[j+1] & 0xC0) == 0x80
+ && (src[j+2] & 0xC0) == 0x80
+ && (src[j+3] & 0xC0) == 0x80) {
+ c = (src[j+0] & 0x07) << 18
+ | (src[j+1] & 0x3F) << 12
+ | (src[j+2] & 0x3F) << 6
+ | (src[j+3] & 0x3F);
+ if (c < 0x10000
+ || c > 0x10FFFF
+ || (c & 0x1FF800) == 0x00D800)
return NULL;
- }
- ws[i] |= (s[j++] & 0x3F) << 2;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i] |= (s[j] & 0x30) >> 4;
- ws[i] -= 0x0040;
- ws[i++] |= 0xD800;
- ws[i] = 0xDC00 | ((s[j++] & 0x0F) << 6);
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i++] |= s[j++] & 0x3F;
-#else
- ws[i] = (s[j++] & 0x07) << 18;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i] |= (s[j++] & 0x3F) << 12;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i] |= (s[j++] & 0x3F) << 6;
- if ((s[j] & 0xC0) != 0x80) {
- free(ws);
- return NULL;
- }
- ws[i++] |= s[j++] & 0x3F;
+#if SIZEOF_WCHAR_T == 2
+ i++;
#endif
+ j += 4;
} else {
- free(ws);
return NULL;
}
}
- ws[i] = L'\0';
- return ws;
+ dest = malloc((i + 1) * sizeof(wchar_t));
+ if (dest == NULL)
+ return NULL;
+ /* go through the source string again, this time we can skip
+ * the correctness tests */
+ i = j = 0;
+ while (src[j]) {
+ if ((src[j+0] & 0x80) == 0) {
+ dest[i++] = src[j+0];
+ j += 1;
+ } else if ((src[j+0] & 0xE0) == 0xC0) {
+ dest[i++] = (src[j+0] & 0x1F) << 6
+ | (src[j+1] & 0x3F);
+ j += 2;
+ } else if ((src[j+0] & 0xF0) == 0xE0) {
+ dest[i++] = (src[j+0] & 0x0F) << 12
+ | (src[j+1] & 0x3F) << 6
+ | (src[j+2] & 0x3F);
+ j += 3;
+ } else if ((src[j+0] & 0xF8) == 0xF0) {
+ c = (src[j+0] & 0x07) << 18
+ | (src[j+1] & 0x3F) << 12
+ | (src[j+2] & 0x3F) << 6
+ | (src[j+3] & 0x3F);
+#if SIZEOF_WCHAR_T == 2
+ dest[i++] = 0xD800 | ((c - 0x10000) >> 10);
+ dest[i++] = 0xDE00 | (c & 0x3FF);
+#else
+ dest[i++] = c;
+#endif
+ j += 4;
+ }
+ }
+ dest[i] = 0;
+ return dest;
}
#else
static char *
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
