Changeset: 47c6d10bec4e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=47c6d10bec4e
Modified Files:
common/stream/Tests/read_bz2.stable.out
common/stream/Tests/read_gz.stable.out
common/stream/Tests/read_lz4.stable.out
common/stream/Tests/read_tests.py
common/stream/Tests/read_uncompressed.stable.out
common/stream/Tests/read_xz.stable.out
Branch: makelibstreamgreatagain
Log Message:
Test if incomplete BOM marks are left alone
diffs (155 lines):
diff --git a/common/stream/Tests/read_bz2.stable.out
b/common/stream/Tests/read_bz2.stable.out
--- a/common/stream/Tests/read_bz2.stable.out
+++ b/common/stream/Tests/read_bz2.stable.out
@@ -228,6 +228,18 @@ Test: read rstream crlf1024.txt.bz2
Test: read rastream crlf1024.txt.bz2
OK
+Test: read rstream brokenbom1.txt.bz2
+ OK
+
+Test: read rastream brokenbom1.txt.bz2
+ OK
+
+Test: read rstream brokenbom2.txt.bz2
+ OK
+
+Test: read rastream brokenbom2.txt.bz2
+ OK
+
Test: read ['rstream', 'blocksize:2'] sherlock.txt.bz2
OK
diff --git a/common/stream/Tests/read_gz.stable.out
b/common/stream/Tests/read_gz.stable.out
--- a/common/stream/Tests/read_gz.stable.out
+++ b/common/stream/Tests/read_gz.stable.out
@@ -228,6 +228,18 @@ Test: read rstream crlf1024.txt.gz
Test: read rastream crlf1024.txt.gz
OK
+Test: read rstream brokenbom1.txt.gz
+ OK
+
+Test: read rastream brokenbom1.txt.gz
+ OK
+
+Test: read rstream brokenbom2.txt.gz
+ OK
+
+Test: read rastream brokenbom2.txt.gz
+ OK
+
Test: read ['rstream', 'blocksize:2'] sherlock.txt.gz
OK
diff --git a/common/stream/Tests/read_lz4.stable.out
b/common/stream/Tests/read_lz4.stable.out
--- a/common/stream/Tests/read_lz4.stable.out
+++ b/common/stream/Tests/read_lz4.stable.out
@@ -228,6 +228,18 @@ Test: read rstream crlf1024.txt.lz4
Test: read rastream crlf1024.txt.lz4
OK
+Test: read rstream brokenbom1.txt.lz4
+ OK
+
+Test: read rastream brokenbom1.txt.lz4
+ OK
+
+Test: read rstream brokenbom2.txt.lz4
+ OK
+
+Test: read rastream brokenbom2.txt.lz4
+ OK
+
Test: read ['rstream', 'blocksize:2'] sherlock.txt.lz4
OK
diff --git a/common/stream/Tests/read_tests.py
b/common/stream/Tests/read_tests.py
--- a/common/stream/Tests/read_tests.py
+++ b/common/stream/Tests/read_tests.py
@@ -11,19 +11,26 @@ import sys
BOM = b'\xEF\xBB\xBF'
+COMPRESSIONS = [None, "gz", "bz2", "xz", "lz4"]
+def compr_name(name, compression):
+ if compression:
+ return name + '.' + compression
+ else:
+ return name
def gen_compr_variants(name, content, limit):
- yield testdata.Doc(name, content, limit, None)
- yield testdata.Doc(name + ".gz", content, limit, "gz")
- yield testdata.Doc(name + ".bz2", content, limit, "bz2")
- yield testdata.Doc(name + ".xz", content, limit, "xz")
- yield testdata.Doc(name + ".lz4", content, limit, "lz4")
-
+ for compr in COMPRESSIONS:
+ yield testdata.Doc(compr_name(name, compr), content, limit, compr)
def gen_bom_compr_variants(name, content, limit):
yield from gen_compr_variants(name + ".txt", content, limit)
yield from gen_compr_variants(name + "_bom.txt", BOM + content, limit)
+def broken_boms():
+ limit = 2000
+ for compr in COMPRESSIONS:
+ yield testdata.Doc(compr_name('brokenbom1.txt', compr), BOM[:1] +
testdata.SHERLOCK, limit, compr)
+ yield testdata.Doc(compr_name('brokenbom2.txt', compr), BOM[:2] +
testdata.SHERLOCK, limit, compr)
def gen_docs():
input = testdata.SHERLOCK
@@ -51,6 +58,8 @@ def gen_docs():
assert head[1024:].startswith(b'\n')
yield from gen_compr_variants('crlf1024.txt', head, None)
+ yield from broken_boms()
+
def test_read(opener, text_mode, doc):
filename = doc.write_tmp()
diff --git a/common/stream/Tests/read_uncompressed.stable.out
b/common/stream/Tests/read_uncompressed.stable.out
--- a/common/stream/Tests/read_uncompressed.stable.out
+++ b/common/stream/Tests/read_uncompressed.stable.out
@@ -228,6 +228,18 @@ Test: read rstream crlf1024.txt
Test: read rastream crlf1024.txt
OK
+Test: read rstream brokenbom1.txt
+ OK
+
+Test: read rastream brokenbom1.txt
+ OK
+
+Test: read rstream brokenbom2.txt
+ OK
+
+Test: read rastream brokenbom2.txt
+ OK
+
Test: read ['rstream', 'blocksize:2'] sherlock.txt
OK
diff --git a/common/stream/Tests/read_xz.stable.out
b/common/stream/Tests/read_xz.stable.out
--- a/common/stream/Tests/read_xz.stable.out
+++ b/common/stream/Tests/read_xz.stable.out
@@ -228,6 +228,18 @@ Test: read rstream crlf1024.txt.xz
Test: read rastream crlf1024.txt.xz
OK
+Test: read rstream brokenbom1.txt.xz
+ OK
+
+Test: read rastream brokenbom1.txt.xz
+ OK
+
+Test: read rstream brokenbom2.txt.xz
+ OK
+
+Test: read rastream brokenbom2.txt.xz
+ OK
+
Test: read ['rstream', 'blocksize:2'] sherlock.txt.xz
OK
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
