Changeset: 85cd30b5e7f6 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=85cd30b5e7f6
Added Files:
common/stream/text_stream.c
Modified Files:
common/stream/Makefile.ag
common/stream/stream.h
common/stream/xz_stream.c
Branch: makelibstreamgreatagain
Log Message:
Add text_stream and use it in xz_stream
It does not convert line endings yet, but the bom conversions, which were
broken by
the previous commit, work again.
diffs (172 lines):
diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag
--- a/common/stream/Makefile.ag
+++ b/common/stream/Makefile.ag
@@ -24,7 +24,7 @@ lib_stream = {
url_stream.c socket_stream.c \
memio.c callback.c blackhole.c fwf.c \
iconv_stream.c \
- misc.c \
+ misc.c text_stream.c \
stream.h stream_internal.h stream_socket.h
VERSION = $(STREAM_VERSION)
LIBS = WIN32?$(SOCKET_LIBS) \
diff --git a/common/stream/stream.h b/common/stream/stream.h
--- a/common/stream/stream.h
+++ b/common/stream/stream.h
@@ -272,4 +272,7 @@ stream_export stream *stream_blackhole_c
stream_export stream *stream_fwf_create(stream *restrict s, size_t num_fields,
size_t *restrict widths, char filler); // sql.c
+
+stream_export stream *create_text_stream(stream *s);
+
#endif /*_STREAM_H_*/
diff --git a/common/stream/text_stream.c b/common/stream/text_stream.c
new file mode 100644
--- /dev/null
+++ b/common/stream/text_stream.c
@@ -0,0 +1,126 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdb_config.h"
+#include "stream.h"
+#include "stream_internal.h"
+
+/* When reading, text streams convert \r\n to \n regardless of operating
system,
+ * and they drop the leading UTF-8 BOM marker if found.
+ * When writing on Windows, \n is translated back to \r\n.
+ *
+ * Currently, skipping the BOM happens when opening
+ */
+
+#define UTF8BOM "\xEF\xBB\xBF" /* UTF-8 encoding of Unicode
BOM */
+#define UTF8BOMLENGTH 3 /* length of above */
+
+
+typedef struct text_stream_state {
+ char putback_buf[UTF8BOMLENGTH];
+ int putback_start;
+ int putback_end;
+} state;
+
+
+static void
+text_destroy(stream *s)
+{
+ if (s == NULL)
+ return;
+
+ free(s->stream_data.p);
+ destroy_stream(s);
+}
+
+static ssize_t
+text_read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt)
+{
+ return s->inner->read(s->inner, buf, elmsize, cnt);
+}
+
+static ssize_t
+text_read_putback(stream *restrict s, void *restrict buf, size_t elmsize,
size_t cnt)
+{
+ state *st = (state*) s->stream_data.p;
+ char *p = buf;
+ size_t size = elmsize * cnt;
+ size_t i;
+
+ for (i = 0; i < size; i++) {
+ if (st->putback_start == st->putback_end) {
+ s->read = text_read;
+ break;
+ }
+ *p++ = st->putback_buf[st->putback_start++];
+ }
+
+ return i;
+}
+
+
+static ssize_t
+skip_bom(stream *s)
+{
+ state *st = (state*) s->stream_data.p;
+ stream *inner = s->inner;
+
+ ssize_t nread = mnstr_read(inner, st->putback_buf, 1, UTF8BOMLENGTH);
+ if (nread < 0)
+ return nread;
+
+ if (nread == UTF8BOMLENGTH && memcmp(st->putback_buf, UTF8BOM, nread)
== 0) {
+ // Bingo! Skip it!
+ s->isutf8 = true;
+ return 3;
+ }
+
+
+ // We have consumed some bytes that have to be returned.
+ // skip_bom left them in the putback_buf.
+ // Switch to a read function that returns them.
+ s->read = text_read_putback;
+ st->putback_start = 0;
+ st->putback_end = nread;
+
+ return nread;
+}
+
+
+stream *
+create_text_stream(stream *inner)
+{
+ assert(inner->readonly);
+ state *st = malloc(sizeof(state));
+ struct stream *s = create_wrapper_stream(NULL, inner);
+ if (st == NULL)
+ goto bail;
+ if (s == NULL)
+ goto bail;
+
+ *st = (state) { .putback_start = 0, .putback_end = 0, };
+ s->stream_data.p = st;
+
+ s->binary = false;
+ s->destroy = text_destroy;
+
+ // bool isutf8; /* known to be UTF-8 due to BOM */
+ // ssize_t (*read)(stream *restrict s, void *restrict buf, size_t
elmsize, size_t cnt);
+ // ssize_t (*write)(stream *restrict s, const void *restrict buf,
size_t elmsize, size_t cnt);
+ // void (*close)(stream *s);
+ // void (*destroy)(stream *s);
+ // int (*flush)(stream *s);
+
+ if (skip_bom(s) < 0)
+ goto bail;
+ return s;
+bail:
+ free(st);
+ destroy_stream(s);
+ return NULL;
+}
diff --git a/common/stream/xz_stream.c b/common/stream/xz_stream.c
--- a/common/stream/xz_stream.c
+++ b/common/stream/xz_stream.c
@@ -347,12 +347,8 @@ open_xzwstream(const char *restrict file
stream *
open_xzrastream(const char *filename)
{
- stream *s;
-
- if ((s = open_xzstream(filename, "r")) == NULL)
- return NULL;
- s->binary = false;
- return s;
+ stream *s = open_xzstream(filename, "r");
+ return create_text_stream(s);
}
stream *
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
