Changeset: ae7bcc22e7ce for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ae7bcc22e7ce
Removed Files:
sql/test/copy/Tests/crlf_normalization.stable.err
sql/test/copy/Tests/crlf_normalization.stable.out
Modified Files:
MonetDB.spec
sql/server/rel_select.c
sql/server/rel_updates.c
sql/test/copy/Tests/All
Branch: default
Log Message:
Merged with Oct2020
diffs (115 lines):
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -302,8 +302,8 @@ This package contains the files needed t
Summary: MonetDB ODBC driver
Group: Applications/Databases
Requires: %{name}-client%{?_isa} = %{version}-%{release}
-Requires(post): unixODBC
-Requires(postun): unixODBC
+Requires(post): %{_bindir}/odbcinst
+Requires(postun): %{_bindir}/odbcinst
%description client-odbc
MonetDB is a database management system that is developed from a
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1496,7 +1496,7 @@ copyfrom(sql_query *query, dlist *qname,
char *tname = qname_schema_object(qname);
sql_table *t = NULL, *nt = NULL;
const char *tsep = seps->h->data.sval;
- const char *rsep = seps->h->next->data.sval;
+ char *rsep = seps->h->next->data.sval; // not const, might need
adjusting
const char *ssep =
(seps->h->next->next)?seps->h->next->next->data.sval:NULL;
const char *ns = (null_string)?null_string:"null";
lng nr = (nr_offset)?nr_offset->h->data.l_val:-1;
@@ -1506,10 +1506,15 @@ copyfrom(sql_query *query, dlist *qname,
assert(!nr_offset || nr_offset->h->type == type_lng);
assert(!nr_offset || nr_offset->h->next->type == type_lng);
- if (strstr(rsep, "\r\n") != NULL)
+ if (strcmp(rsep, "\r\n") == 0) {
+ // silently fix it
+ rsep[0] = '\n';
+ rsep[1] = '\0';
+ } else if (strstr(rsep, "\r\n") != NULL) {
return sql_error(sql, 02, SQLSTATE(42000)
"COPY INTO: record separator contains '\\r\\n'
but "
- "in the input stream, '\\r\\n' is being
normalized into '\\n'");
+ "that will never match, use '\\n' instead");
+ }
t = find_table_or_view_on_scope(sql, NULL, sname, tname, "COPY INTO",
false);
if (insert_allowed(sql, t, tname, "COPY INTO", "copy into") == NULL)
diff --git a/sql/test/copy/Tests/All b/sql/test/copy/Tests/All
--- a/sql/test/copy/Tests/All
+++ b/sql/test/copy/Tests/All
@@ -15,3 +15,4 @@ incorrect_columns
columns
no_escape
no_escape2
+crlf_normalization
diff --git a/sql/test/copy/Tests/crlf_normalization.SQL.py
b/sql/test/copy/Tests/crlf_normalization.SQL.py
new file mode 100644
--- /dev/null
+++ b/sql/test/copy/Tests/crlf_normalization.SQL.py
@@ -0,0 +1,58 @@
+import os, re, sys, tempfile
+try:
+ from MonetDBtesting import process
+except ImportError:
+ import process
+
+# Use a Python test because we're testing LF / CR LF handling and we don't
+# want editors or version control systems messing with our line endings
+
+def r_escape(s):
+ return "r'" + s.replace("'", "''") + "' "
+
+def testdata(prefix,line_sep):
+ prefix = "crlf_test_" + prefix + "_"
+ lines = [ b"1,one", b"3,two", b"5,three" ]
+ text = bytes(line_sep, "ascii").join([*lines, b''])
+ f = tempfile.NamedTemporaryFile(delete=False, prefix=prefix, suffix=".csv")
+ name = f.name
+ f.write(text)
+ f.close()
+ return name, text
+
+def run_test(name, data_delimiter, copy_delimiter):
+ file_name, test_data = testdata(name, data_delimiter)
+ script = f"""
+ DROP TABLE IF EXISTS foo;
+ CREATE TABLE foo(i INT, t TEXT);
+ COPY INTO foo FROM {r_escape(file_name)}
+ USING DELIMITERS ',', '{copy_delimiter}';
+ SELECT i, LENGTH(t) FROM foo;
+ """
+ with process.client('sql', stdin=process.PIPE, stdout=process.PIPE,
stderr=process.PIPE) as c:
+ out, err = c.communicate(script)
+ reduced = "\n".join(re.sub(r"\s+", "", line) for line in
out.splitlines() if line.startswith("["))
+ expected = "[3]\n[1,3]\n[3,3]\n[5,5]"
+ if reduced != expected:
+ print("TEST: ", name, file=sys.stderr)
+ print("\nLINE DELIMITER: ", repr(data_delimiter), sep='',
file=sys.stderr)
+ print("\nFILE CONTENTS: ", repr(test_data), sep='',
file=sys.stderr)
+ print("\nSCRIPT:\n", script, sep='', file=sys.stderr)
+ print("\nEXPECTED:\n", expected, sep='', file=sys.stderr)
+ print("\nGOT:\n", reduced, sep='', file=sys.stderr)
+ print("\nFULL STDERR:\n", err, sep='', file=sys.stderr)
+ print("\nFULL OUTPUT:\n", out, sep='', file=sys.stderr)
+ raise SystemExit("Test failed")
+ os.remove(file_name)
+
+# Load unix endings while asking for Unix endings.
+run_test("unix", "\n", r"\n")
+
+# Load dos endings while asking for dos endings
+run_test("dos", "\n", r"\n")
+
+# Load dos endings while asking for unix endings
+run_test("dos_as_unix", "\r\n", r"\n")
+
+# Load unix endings while asking for dos endings
+run_test("unix_as_dos", "\n", r"\r\n")
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
