Changeset: bd6efd91626b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bd6efd91626b
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapiclient/mclient.c
common/stream/stream.c
configure.ag
monetdb5/ChangeLog
monetdb5/modules/atoms/str.c
monetdb5/modules/atoms/str.h
monetdb5/modules/atoms/str.mal
sql/backends/monet5/sql.c
Branch: default
Log Message:
Removed str.iconv and str.codeset.
We can't have non-UTF-8 strings internally, so don't allow
converting. The C versions were used to convert file names. We now
instead use the (new) rule that the file names specified in the stream
functions open_*stream() are UTF-8 encoded, and those functions
convert the names to the correct locale.
diffs (truncated from 619 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -40990,10 +40990,6 @@ command str.ascii(s:str):int
address STRascii;
comment Return unicode of head of string
-command str.codeset():str
-address STRcodeset;
-comment Return the locale's codeset
-
command str.endsWith(s:str,suffix:str):bit
address STRSuffix;
comment Suffix check.
@@ -41002,10 +40998,6 @@ command str.insert(s:str,start:int,l:int
address STRinsert;
comment Insert a string into another
-command str.iconv(org:str,fromCs:str,toCs:str):str
-address STRIconv;
-comment String codeset conversion
-
command str.locate(s1:str,s2:str,start:int):int
address STRlocate2;
comment Locate the start position of a string
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -51929,10 +51929,6 @@ command str.ascii(s:str):int
address STRascii;
comment Return unicode of head of string
-command str.codeset():str
-address STRcodeset;
-comment Return the locale's codeset
-
command str.endsWith(s:str,suffix:str):bit
address STRSuffix;
comment Suffix check.
@@ -51941,10 +51937,6 @@ command str.insert(s:str,start:int,l:int
address STRinsert;
comment Insert a string into another
-command str.iconv(org:str,fromCs:str,toCs:str):str
-address STRIconv;
-comment String codeset conversion
-
command str.locate(s1:str,s2:str,start:int):int
address STRlocate2;
comment Locate the start position of a string
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1762,7 +1762,6 @@ str SRVsetServers(Client cntxt, MalBlkPt
str STRBytes(int *res, const str *arg1);
str STRConcat(str *res, const str *val1, const str *val2);
str STRFromWChr(str *res, const int *at);
-str STRIconv(str *res, const str *o, const str *fp, const str *tp);
str STRLength(int *res, const str *arg1);
str STRLower(str *res, const str *arg1);
str STRLpad(str *res, const str *arg1, const int *len);
@@ -1826,7 +1825,6 @@ str STRbatstrSearchcst(bat *ret, const b
str STRbatstringLength(bat *ret, const bat *l);
str STRbatsubstring(bat *ret, const bat *l, const bat *r, const bat *t);
str STRbatsubstringcst(bat *ret, const bat *bid, const int *start, const int
*length);
-str STRcodeset(str *res);
str STRinsert(str *ret, const str *s, const int *start, const int *l, const
str *s2);
str STRlikewrap(bit *ret, const str *s, const str *pat, const str *esc);
str STRlikewrap2(bit *ret, const str *s, const str *pat);
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -2152,7 +2152,7 @@ enum hmyesno { UNKNOWN, YES, NO };
#define READBLOCK 8192
static int
-doFile(Mapi mid, const char *file, int useinserts, int interactive, int
save_history)
+doFile(Mapi mid, FILE *fp, int useinserts, int interactive, int save_history)
{
char *line = NULL;
char *oldbuf = NULL, *buf = NULL;
@@ -2163,7 +2163,6 @@ doFile(Mapi mid, const char *file, int u
int lineno = 1;
enum hmyesno hassysfuncs = UNKNOWN;
enum hmyesno hasschemsys = UNKNOWN;
- FILE *fp;
char *prompt = NULL;
int prepno = 0;
#ifdef HAVE_ICONV
@@ -2171,20 +2170,14 @@ doFile(Mapi mid, const char *file, int u
#endif
(void) save_history; /* not used if no readline */
- if (strcmp(file, "-") == 0) {
- fp = stdin;
- if (isatty(fileno(fp))) {
- interactive = 1;
- setPrompt();
- prompt = promptbuf;
+ if (isatty(fileno(fp))) {
+ interactive = 1;
+ setPrompt();
+ prompt = promptbuf;
#ifdef HAVE_LIBREADLINE
- init_readline(mid, language, save_history);
+ init_readline(mid, language, save_history);
#endif
- fromConsole = stdin;
- }
- } else if ((fp = fopen(file, "r")) == NULL) {
- fprintf(stderr, "%s: cannot open\n", file);
- return 1;
+ fromConsole = stdin;
}
if (!interactive && !echoquery)
@@ -2706,14 +2699,28 @@ doFile(Mapi mid, const char *file, int u
#endif
continue;
}
- case '<':
+ case '<': {
+ stream *s;
/* read commands from file */
while (isascii((int) line[length - 1])
&& isspace((int) line[length - 1]))
line[--length] = 0;
for (line += 2; *line && isascii((int)
*line) && isspace((int) *line); line++)
;
- doFile(mid, line, 0, 0, 0);
+ /* use open_rastream to
+ * convert filename from UTF-8
+ * to locale */
+ if ((s = open_rastream(line)) == NULL ||
+ mnstr_errnr(s)) {
+ fprintf(stderr, "%s: cannot
open\n", line);
+ close_stream(s);
+ } else {
+ FILE *fp = getFile(s);
+ mnstr_destroy(s);
+ doFile(mid, fp, 0, 0, 0);
+ fclose(fp);
+ }
continue;
+ }
case '>':
/* redirect output to file */
while (isascii((int) line[length - 1])
&& isspace((int) line[length - 1]))
@@ -3428,14 +3435,21 @@ main(int argc, char **argv)
if (optind < argc) {
/* execute from file(s) */
while (optind < argc) {
- c |= doFile(mid, argv[optind], useinserts, interactive,
save_history);
+ FILE *fp = fopen(argv[optind], "r");
+ if (fp == NULL) {
+ fprintf(stderr, "%s: cannot open\n",
argv[optind]);
+ c |= 1;
+ } else {
+ c |= doFile(mid, fp, useinserts, interactive,
save_history);
+ fclose(fp);
+ }
optind++;
}
} else if (command && mapi_get_active(mid))
c = doFileBulk(mid, NULL);
if (!has_fileargs && command == NULL)
- c = doFile(mid, "-", useinserts, interactive, save_history);
+ c = doFile(mid, stdin, useinserts, interactive, save_history);
mapi_destroy(mid);
mnstr_destroy(stdout_stream);
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -91,6 +91,15 @@
#include <bzlib.h>
#endif
+#ifdef HAVE_ICONV
+#ifdef HAVE_ICONV_H
+#include <iconv.h>
+#endif
+#ifdef HAVE_LANGINFO_H
+#include <langinfo.h>
+#endif
+#endif
+
#ifndef SHUT_RD
#define SHUT_RD 0
#define SHUT_WR 1
@@ -189,6 +198,126 @@ mnstr_init(void)
/* #define STREAM_DEBUG 1 */
/* #define BSTREAM_DEBUG 1 */
+#ifdef HAVE__WFOPEN
+/* convert a string from UTF-8 to wide characters; the return value is
+ * freshly allocated */
+static wchar_t *
+utf8towchar(const char *s)
+{
+ wchar_t *ws;
+ size_t i = 0;
+ size_t j = 0;
+
+ ws = malloc((strlen(s) + 1) * sizeof(wchar_t));
+ if (ws == NULL)
+ return NULL;
+ while (s[j]) {
+ if ((s[j] & 0x80) == 0) {
+ ws[i++] = s[j++];
+ } else if ((s[j] & 0xC0) == 0x80) {
+ free(ws);
+ return NULL;
+ } else if ((s[j] & 0xE0) == 0xC0) {
+ ws[i] = (s[j++] & 0x1F) << 6;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i++] |= s[j++] & 0x3F;
+ } else if ((s[j] & 0xF0) == 0xE0) {
+ ws[i] = (s[j++] & 0x0F) << 12;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i] |= (s[j++] & 0x3F) << 6;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i++] |= s[j++] & 0x3F;
+ } else if ((s[j] & 0xF8) == 0xF0) {
+#if SIZEOF_WCHAR_T == 2
+ ws[i] = (s[j++] & 0x07) << 8;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i] |= (s[j++] & 0x3F) << 2;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i] |= (s[j] & 0x30) >> 4;
+ ws[i] -= 0x0040;
+ ws[i++] |= 0xD800;
+ ws[i] = 0xDC00 | ((s[j++] & 0x0F) << 6);
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i++] |= s[j++] & 0x3F;
+#else
+ ws[i] = (s[j++] & 0x07) << 18;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i] |= (s[j++] & 0x3F) << 12;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i] |= (s[j++] & 0x3F) << 6;
+ if ((s[j] & 0xC0) != 0x80) {
+ free(ws);
+ return NULL;
+ }
+ ws[i++] |= s[j++] & 0x3F;
+#endif
+ } else {
+ free(ws);
+ return NULL;
+ }
+ }
+ ws[i] = L'\0';
+ return ws;
+}
+#else
+static char *
+cvfilename(const char *filename)
+{
+#if defined(HAVE_NL_LANGINFO) && defined(HAVE_ICONV)
+ char *code_set = nl_langinfo(CODESET);
+
+ if (code_set != NULL && strcmp(code_set, "UTF-8") != 0) {
+ iconv_t cd = iconv_open("UTF-8", code_set);
+
+ if (cd != (iconv_t) -1) {
+ size_t len = strlen(filename);
+ size_t size = 4 * len;
+ ICONV_CONST char *from = (ICONV_CONST char *) filename;
+ char *r = malloc(size);
+
+ if (r &&
+ iconv(cd, &from, &len, &r, &size) != (size_t) -1) {
+ iconv_close(cd);
+ return r;
+ } else if (r)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
