Changeset: 6b863eccdaae for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6b863eccdaae
Modified Files:
common/stream/Tests/read_iconv.py
common/stream/Tests/read_tests.py
common/stream/Tests/testdata.py
common/stream/Tests/write_iconv.py
common/stream/Tests/write_tests.py
common/stream/text_stream.c
Branch: makelibstreamgreatagain
Log Message:
Properly convert line endings
diffs (truncated from 1153 to 300 lines):
diff --git a/common/stream/Tests/read_iconv.py
b/common/stream/Tests/read_iconv.py
--- a/common/stream/Tests/read_iconv.py
+++ b/common/stream/Tests/read_iconv.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-from testdata import Doc
+from testdata import Doc, TestFile
import os
import subprocess
@@ -9,8 +9,11 @@ import sys
def run_streamcat(text, enc):
content = bytes(text, enc)
- d = Doc(f'read_iconv_{enc}.txt', content)
- filename = d.write_tmp()
+ name = f'read_iconv_{enc}.txt'
+
+ tf = TestFile(name, None)
+ filename = tf.write(content)
+
cmd = ['streamcat', 'read', filename, 'rstream', f'iconv:{enc}']
print(f"Input with encoding '{enc}' is {repr(content)}")
# print(cmd)
diff --git a/common/stream/Tests/read_tests.py
b/common/stream/Tests/read_tests.py
--- a/common/stream/Tests/read_tests.py
+++ b/common/stream/Tests/read_tests.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import testdata
+from testdata import Doc, TestFile
import hashlib
import json
@@ -11,135 +12,119 @@ import sys
BOM = b'\xEF\xBB\xBF'
-COMPRESSIONS = [None, "gz", "bz2", "xz", "lz4"]
-def compr_name(name, compression):
- if compression:
- return name + '.' + compression
- else:
- return name
+
+class TestCase:
+ def __init__(self, name, doc, compression, openers, expected):
+ self.tf = TestFile(name, compression)
+ self.name = self.tf.name
+ self.doc = doc
+ self.compression = compression
+ self.openers = openers
+ self.expected = expected
+
+ def run(self):
+ doc = self.doc
+ openers = self.openers
+ filename = self.tf.write(doc.content)
+
+ test = f"read {openers} {self.name}"
+
+ if not isinstance(openers, list):
+ openers = [openers]
-def gen_compr_variants(name, content, limit):
- for compr in COMPRESSIONS:
- yield testdata.Doc(compr_name(name, compr), content, limit, compr)
+ print()
+ print(f"Test: {test}")
+
+ cmd = ['streamcat', 'read', filename, *openers]
+ results = subprocess.run(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if results.returncode != 0 or results.stderr:
+ print(
+ f"\tFAIL: streamcat returned with exit code
{results.returncode}:\n{results.stderr or ''}")
+ return False
-def gen_bom_compr_variants(name, content, limit):
- yield from gen_compr_variants(name + ".txt", content, limit)
- yield from gen_compr_variants(name + "_bom.txt", BOM + content, limit)
+ output = results.stdout or b""
+ complaint = self.expected.verify(output)
-def broken_boms():
- limit = 2000
- for compr in COMPRESSIONS:
- yield testdata.Doc(compr_name('brokenbom1.txt', compr), BOM[:1] +
testdata.SHERLOCK, limit, compr)
- yield testdata.Doc(compr_name('brokenbom2.txt', compr), BOM[:2] +
testdata.SHERLOCK, limit, compr)
+ if complaint:
+ print(f"\tFAIL: {complaint}")
+ return False
+ else:
+ print(f"\tOK")
+ os.remove(filename)
+ return True
+
def gen_docs():
- input = testdata.SHERLOCK
+ # We use a document with DOS line endings.
+ # This way we can verify that rastream replaces them with \n
+ # and rstream leaves them alone.
+ text = Doc(testdata.SHERLOCK).to_dos().content
# Whole file
- yield from gen_bom_compr_variants('sherlock', input, None)
+ yield 'sherlock.txt', Doc(text)
+ yield 'sherlock_bom.txt', Doc(text, prepend_bom=True)
# Empty file
- yield from gen_bom_compr_variants('empty', b'', None)
+ yield 'empty.txt', Doc(b'')
+ yield 'empty_bom.txt', Doc(b'', prepend_bom=True)
- # First 16 lines
- head = b'\n'.join(input.split(b'\n')[:16]) + b'\n'
- yield from gen_bom_compr_variants('small', head, None)
+ # First few lines
+ small = b'\n'.join(text.split(b'\n')[:16]) + b'\n'
+ yield 'small.txt', Doc(small)
+ yield 'small_bom.txt', Doc(small, prepend_bom=True)
# Buffer size boundary cases
for base_size in [1024, 2048, 4096, 8192, 16384]:
for delta in [-1, 0, 1]:
size = base_size + delta
- yield from gen_bom_compr_variants(f'block{size}', input, size)
+ yield f'block{size}.txt', Doc(text, truncate=size)
+ yield f'block{size}_bom.txt', Doc(text, prepend_bom=True,
truncate=size)
# \r at end of first block, \n at start of next
- head = (1023 * b'a') + b'\r\n' + (20 * b'b') + b'\r\n' + (20 * b'c')
+ abc = (1023 * b'a') + b'\r\n' + (20 * b'b') + b'\r\n' + (20 * b'c')
# word of wisdom: you have to test your tests
- assert head[:1024].endswith(b'\r')
- assert head[1024:].startswith(b'\n')
- yield from gen_compr_variants('crlf1024.txt', head, None)
-
- yield from broken_boms()
-
-def test_read(opener, text_mode, doc):
- filename = doc.write_tmp()
-
- test = f"read {opener} {doc.name}"
-
- if not isinstance(opener, list):
- opener = [opener]
-
- print()
- print(f"Test: {test}")
+ assert abc[:1024].endswith(b'\r')
+ assert abc[1024:].startswith(b'\n')
+ yield 'crlf1024.txt', Doc(abc)
- cmd = ['streamcat', 'read', filename, *opener]
- results = subprocess.run(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- if results.returncode != 0 or results.stderr:
- print(
- f"\tFAIL: streamcat returned with exit code
{results.returncode}:\n{results.stderr or ''}")
- return False
-
- output = results.stdout or b""
- complaint = doc.verify(output, text_mode)
-
- if complaint:
- print(f"\tFAIL: {complaint}")
- return False
- else:
- print(f"\tOK")
- os.remove(filename)
- return True
+ yield 'brokenbom1.txt', Doc(BOM[:1] + small)
+ yield 'brokenbom2.txt', Doc(BOM[:2] + small)
-def test_reads(doc):
- failures = 0
-
- # rstream does not strip BOM
- failures += not test_read('rstream', False, doc)
-
- # rastream does strip the BOM
- failures += not test_read('rastream', True, doc)
-
- return failures
-
-
-def test_nonstd_reads(doc):
- failures = 0
-
- failures += not test_read(['rstream', 'blocksize:2'], False, doc)
- failures += not test_read(['rastream', 'blocksize:2'], True, doc)
-
- failures += not test_read(['rstream', 'blocksize:1000000'], False, doc)
- failures += not test_read(['rastream', 'blocksize:1000000'], True, doc)
-
- return failures
+def gen_tests():
+ for compr in testdata.COMPRESSIONS:
+ for name, doc in gen_docs():
+ yield TestCase(name, doc, compr, "rstream", doc)
+ yield TestCase(name, doc, compr, "rastream",
doc.without_bom().to_unix())
+ for name, doc in gen_docs():
+ if not name.startswith('sherlock') or name.startswith('empty'):
+ continue
+ yield TestCase(name, doc, compr, ["rstream", "blocksize:2"], doc)
+ yield TestCase(name, doc, compr, ["rastream", "blocksize:2"],
doc.without_bom().to_unix())
+ yield TestCase(name, doc, compr, ["rstream", "blocksize:1000000"],
doc)
+ yield TestCase(name, doc, compr, ["rastream",
"blocksize:1000000"], doc.without_bom().to_unix())
def all_tests(filename_filter):
failures = 0
- for d in gen_docs():
- if not filename_filter(d.name):
+ for t in gen_tests():
+ if not filename_filter(t.name):
continue
- failures += test_reads(d)
+ failures += t.run()
- for d in gen_docs():
- if not d.name.startswith('sherlock') or d.name.startswith('empty'):
- continue
- if not filename_filter(d.name):
- continue
- failures += test_nonstd_reads(d)
return failures
if __name__ == "__main__":
# generate test data for manual testing
if len(sys.argv) == 1:
- for d in gen_docs():
- print(d.name)
+ for name, d in gen_docs():
+ print(name)
elif len(sys.argv) == 2:
- for d in gen_docs():
- if d.name == sys.argv[1]:
- d.write(sys.stdout.buffer)
+ for name, d in gen_docs():
+ if name == sys.argv[1]:
+ sys.stdout.buffer.write(d.content)
else:
print("Usage: python3 read_tests.py [TESTDATANAME]", file=sys.stderr)
sys.exit(1)
diff --git a/common/stream/Tests/testdata.py b/common/stream/Tests/testdata.py
--- a/common/stream/Tests/testdata.py
+++ b/common/stream/Tests/testdata.py
@@ -12,52 +12,197 @@ import sys
import tempfile
+LF = b'\n'
+CRLF = b'\r\n'
BOM = b'\xEF\xBB\xBF'
SRCDIR = os.environ.get(
'TSTSRCDIR',
os.path.dirname(os.path.abspath(sys.argv[0]))
)
+# The functions we pass this to will pick their own default if None:
TMPDIR = os.environ.get('TSTTRGDIR')
-# Often used for testfile contents. Uses DOS line endings, which is
-# important because we can check whether they are transformed or left alone.
-TESTDATA = os.path.join(SRCDIR, '1661-0.txt.gz')
-SHERLOCK = gzip.open(TESTDATA, 'rb').read()
-assert SHERLOCK.find(b'\x0d\x0a') >= 0
+SHERLOCK = gzip.open(os.path.join(SRCDIR, '1661-0.txt.gz'),
'rb').read().replace(CRLF, LF)
+COMPRESSIONS = [None, "gz", "bz2", "xz", "lz4"]
class Doc:
- def __init__(self, name, content, length_limit=None, compression=None):
- if length_limit:
- content = content[:length_limit]
+ """Contents to be read or written. The constructor has several options
+ to make it easy to construct certain variants. The verify method tries
+ to give a human friendly description of the differences found.
+ """
+
+ def __init__(self, content, prepend_bom=False, dos_line_endings=False,
truncate=None):
+ assert isinstance(content, bytes)
+ if prepend_bom:
+ content = BOM + content
+ if dos_line_endings:
+ content = content.replace(LF, CRLF)
+ if truncate != None:
+ assert truncate >= 0
+ content = content[:truncate]
+ self.content = content
+
+ def with_bom(self):
+ assert not self.content.startswith(BOM)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
