Changeset: 5b8a628a93cc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5b8a628a93cc
Modified Files:
common/stream/Tests/read_bz2.stable.out
common/stream/Tests/read_gz.stable.out
common/stream/Tests/read_lz4.stable.out
common/stream/Tests/read_tests.py
common/stream/Tests/read_uncompressed.stable.out
common/stream/Tests/read_xz.stable.out
monetdb5/mal/mal_prelude.c
Branch: default
Log Message:
Merge with Oct2020 branch.
diffs (183 lines):
diff --git a/common/stream/Tests/read_tests.py
b/common/stream/Tests/read_tests.py
--- a/common/stream/Tests/read_tests.py
+++ b/common/stream/Tests/read_tests.py
@@ -59,6 +59,7 @@ def gen_docs():
# Whole file
yield 'sherlock.txt', Doc(text)
yield 'sherlock_bom.txt', Doc(text, prepend_bom=True)
+ yield 'sherlock_cr.txt', Doc(text.replace(b'\n', b'\r'))
# Empty file
yield 'empty.txt', Doc(b'')
@@ -97,6 +98,8 @@ def gen_tests():
continue
yield TestCase(name, doc, compr, ["rstream", "blocksize:2"], doc)
yield TestCase(name, doc, compr, ["rastream", "blocksize:2"],
doc.without_bom().to_unix())
+ yield TestCase(name, doc, compr, ["rstream", "blocksize:1"], doc)
+ yield TestCase(name, doc, compr, ["rastream", "blocksize:1"],
doc.without_bom().to_unix())
yield TestCase(name, doc, compr, ["rstream", "blocksize:1000000"],
doc)
yield TestCase(name, doc, compr, ["rastream",
"blocksize:1000000"], doc.without_bom().to_unix())
diff --git a/common/stream/text_stream.c b/common/stream/text_stream.c
--- a/common/stream/text_stream.c
+++ b/common/stream/text_stream.c
@@ -70,6 +70,7 @@ static void
put_byte(inner_state_t *ist, char byte)
{
*ist->dst_win.start++ = byte;
+ assert(ist->dst_win.count > 0);
ist->dst_win.count--;
}
@@ -85,11 +86,14 @@ text_pump_in(inner_state_t *ist, pump_ac
{
bool crlf_pending = ist->crlf_pending;
- // not the most efficient loop but easy to verify
while (ist->src_win.count > 0 && ist->dst_win.count > 0) {
char c = take_byte(ist);
switch (c) {
case '\r':
+ if (crlf_pending) {
+ // put the previous one, which is
clearly not followed by an \n
+ put_byte(ist, '\r');
+ }
crlf_pending = true;
continue;
case '\n':
@@ -97,10 +101,20 @@ text_pump_in(inner_state_t *ist, pump_ac
crlf_pending = false;
continue;
default:
- if (crlf_pending)
+ if (crlf_pending) {
put_byte(ist, '\r');
- put_byte(ist, c);
- crlf_pending = false;
+ crlf_pending = false;
+ // if dst_win.count was 1, there is no
room for another put_byte().
+ if (ist->dst_win.count > 0) {
+ put_byte(ist, c);
+ } else {
+ // no room anymore for char c,
put it back!
+ ist->src_win.start--;
+ ist->src_win.count++;
+ }
+ } else {
+ put_byte(ist, c);
+ }
continue;
}
}
diff --git a/monetdb5/mal/mal_prelude.c b/monetdb5/mal/mal_prelude.c
--- a/monetdb5/mal/mal_prelude.c
+++ b/monetdb5/mal/mal_prelude.c
@@ -25,11 +25,13 @@
#define MAX_MAL_MODULES 128
static int mel_modules = 0;
-static str mel_module_name[MAX_MAL_MODULES] = {0};
-static mel_atom *mel_module_atoms[MAX_MAL_MODULES] = {0};
-static mel_func *mel_module_funcs[MAX_MAL_MODULES] = {0};
-static mel_init mel_module_inits[MAX_MAL_MODULES] = {0};
-static const char*mel_module_code[MAX_MAL_MODULES] = {0};
+static struct mel_module {
+ char *name;
+ mel_atom *atoms;
+ mel_func *funcs;
+ mel_init inits;
+ const char *code;
+} mel_module[MAX_MAL_MODULES];
int
mal_startup(void)
@@ -48,11 +50,11 @@ void
mal_module2(str name, mel_atom *atoms, mel_func *funcs, mel_init initfunc,
const char *code)
{
assert (mel_modules < MAX_MAL_MODULES);
- mel_module_name[mel_modules] = name;
- mel_module_atoms[mel_modules] = atoms;
- mel_module_funcs[mel_modules] = funcs;
- mel_module_inits[mel_modules] = initfunc;
- mel_module_code[mel_modules] = code;
+ mel_module[mel_modules].name = name;
+ mel_module[mel_modules].atoms = atoms;
+ mel_module[mel_modules].funcs = funcs;
+ mel_module[mel_modules].inits = initfunc;
+ mel_module[mel_modules].code = code;
mel_modules++;
}
@@ -60,11 +62,11 @@ void
mal_module(str name, mel_atom *atoms, mel_func *funcs)
{
assert (mel_modules < MAX_MAL_MODULES);
- mel_module_name[mel_modules] = name;
- mel_module_atoms[mel_modules] = atoms;
- mel_module_funcs[mel_modules] = funcs;
- mel_module_inits[mel_modules] = NULL;
- mel_module_code[mel_modules] = NULL;
+ mel_module[mel_modules].name = name;
+ mel_module[mel_modules].atoms = atoms;
+ mel_module[mel_modules].funcs = funcs;
+ mel_module[mel_modules].inits = NULL;
+ mel_module[mel_modules].code = NULL;
mel_modules++;
}
@@ -415,8 +417,8 @@ malPrelude(Client c, int listing, int no
(void) listing;
/* Add all atom definitions */
for(i = 0; i<mel_modules; i++) {
- if (mel_module_atoms[i]) {
- msg = addAtom(mel_module_atoms[i]);
+ if (mel_module[i].atoms) {
+ msg = addAtom(mel_module[i].atoms);
if (msg)
return msg;
}
@@ -424,32 +426,32 @@ malPrelude(Client c, int listing, int no
/* Add the signatures, where we now have access to all atoms */
for(i = 0; i<mel_modules; i++) {
- if (!malLibraryEnabled(mel_module_name[i]))
+ if (!malLibraryEnabled(mel_module[i].name))
continue;
- if (mel_module_funcs[i]) {
- msg = addFunctions(mel_module_funcs[i]);
- if (!msg && mel_module_code[i]) /* some modules may
also have some function definitions */
- msg = malIncludeString(c, mel_module_name[i],
(str)mel_module_code[i], listing, NULL);
+ if (mel_module[i].funcs) {
+ msg = addFunctions(mel_module[i].funcs);
+ if (!msg && mel_module[i].code) /* some modules may
also have some function definitions */
+ msg = malIncludeString(c, mel_module[i].name,
(str)mel_module[i].code, listing, NULL);
if (msg)
return msg;
/* skip sql should be last to startup and mapi if
configured without mapi server */
- if (strcmp(mel_module_name[i], "sql") == 0 ||
(no_mapi_server && strcmp(mel_module_name[i], "mapi") == 0))
+ if (strcmp(mel_module[i].name, "sql") == 0 ||
(no_mapi_server && strcmp(mel_module[i].name, "mapi") == 0))
continue;
- if (!mel_module_inits[i]) {
- msg = initModule(c, mel_module_name[i]);
+ if (!mel_module[i].inits) {
+ msg = initModule(c, mel_module[i].name);
if (msg)
return msg;
}
}
- if (mel_module_inits[i]) {
- msg = mel_module_inits[i]();
+ if (mel_module[i].inits) {
+ msg = mel_module[i].inits();
if (msg)
return msg;
/* skip sql should be last to startup and mapi if
configured without mapi server */
- if (strcmp(mel_module_name[i], "sql") == 0 ||
(no_mapi_server && strcmp(mel_module_name[i], "mapi") == 0))
+ if (strcmp(mel_module[i].name, "sql") == 0 ||
(no_mapi_server && strcmp(mel_module[i].name, "mapi") == 0))
continue;
- msg = initModule(c, mel_module_name[i]);
+ msg = initModule(c, mel_module[i].name);
if (msg)
return msg;
}
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
