This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
The following commit(s) were added to refs/heads/master by this push:
new 7f17e52 Add MboxoFactory.
7f17e52 is described below
commit 7f17e52cf8adaca472f62bbcb350ff8961bfea8e
Author: Sebb <[email protected]>
AuthorDate: Wed Aug 26 22:52:53 2020 +0100
Add MboxoFactory.
---
runall.py | 7 +++-
tests/mboxo_patch.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++
tests/test-generators.py | 15 +++++++-
tests/test-parsing.py | 16 +++++++-
4 files changed, 130 insertions(+), 5 deletions(-)
diff --git a/runall.py b/runall.py
index ab8d9cf..6018d2a 100755
--- a/runall.py
+++ b/runall.py
@@ -15,6 +15,8 @@ if __name__ == '__main__':
help="Root directory of Apache Pony Mail")
parser.add_argument('--load', dest='load', type=str, nargs='+',
help="Load only specific yaml spec files instead of
all test specs")
+ parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
+ help = 'Skip Mboxo processing')
parser.add_argument('--fof', dest='failonfail', action='store_true',
help="Stop running more tests if an error is
encountered")
args = parser.parse_args()
@@ -41,7 +43,10 @@ if __name__ == '__main__':
tests_total += 1
print("Running '%s' tests from %s..." % (test_type, spec_file))
try:
- rv = subprocess.check_output((PYTHON3, 'tests/test-%s.py'
% test_type, '--rootdir', args.rootdir, '--load', spec_file))
+ if args.nomboxo:
+ rv = subprocess.check_output((PYTHON3,
'tests/test-%s.py' % test_type, '--rootdir', args.rootdir, '--load', spec_file,
'--nomboxo'))
+ else:
+ rv = subprocess.check_output((PYTHON3,
'tests/test-%s.py' % test_type, '--rootdir', args.rootdir, '--load', spec_file))
tests_success += 1
except subprocess.CalledProcessError as e:
rv = e.output
diff --git a/tests/mboxo_patch.py b/tests/mboxo_patch.py
new file mode 100644
index 0000000..a577567
--- /dev/null
+++ b/tests/mboxo_patch.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Byte stream reader to process mboxo style mailbox files.
+These are not currently handled by the Python email package.
+
+It replaces any occurrence of b'\n>From ' with b'\nFrom '
+
+The class handles matching across read boundaries.
+
+To use:
+
+from mboxo_patch import MboxoFactory
+...
+messages = mailbox.mbox(filename, MboxoFactory)
+
+N.B.
+To simplify the code, the MboxoReader class changes the
+size parameter to 7 if (and only if): 0 <= size < 7
+The return byte buffer can thus be larger than expected.
+However this is only a theoretical possibility
+as the mailbox code uses a size of 8192 (or None)
+
+"""
+import mailbox
+
+FROM_MANGLED =b'\n>From '
+FROM_MANGLED_LEN=len(FROM_MANGLED)
+FROM_UNMANGLED=b'\nFrom '
+# We want to match the 7 bytes b'\n>From ' in the input stream
+# However this can be split over multiple reads.
+# The split can occur anywhere after the leading b'\n'
+# and the trailing b' '. If we match any of these
+# we keep the trailing part of the buffer for next time
+# The following are all the possible prefixes for a split:
+FROMS=(FROM_MANGLED[:-1],
+ FROM_MANGLED[:-2],
+ FROM_MANGLED[:-3],
+ FROM_MANGLED[:-4],
+ FROM_MANGLED[:-5],
+ FROM_MANGLED[:-6],
+ )
+
+class MboxoReader(mailbox._PartialFile): # pylint: disable=W0212
+ def __init__(self, f, start=None, stop=None):
+ self.remain=0 # number of bytes to keep for next read
+ super().__init__(f._file, start=f._start, stop=f._stop) # pylint:
disable=W0212
+
+ # Override the read method to provide mboxo filtering
+ def _read(self, size, read_method):
+ # get the next chunk, resetting if necessary
+ if self.remain != 0:
+ super().seek(whence=1, offset=-self.remain)
+ # if size is None or negative, then read returns everything.
+ # in which case there is no need to wory about matching across reads
+ limited_read = size and size >= 0
+ # ensure we get enough to match successfully when refilling
+ if limited_read and size < FROM_MANGLED_LEN:
+ size = FROM_MANGLED_LEN
+ buff = super()._read(size, read_method)
+ bufflen=len(buff)
+ # did we get anything new?
+ if limited_read and bufflen > self.remain:
+ # is there a potential cross-boundary match?
+ if buff.endswith(FROMS):
+ # yes, work out what to keep
+ # N.B. rindex will fail if it cannot find the LF;
+ # this should be impossible
+ self.remain=bufflen - buff.rindex(b'\n')
+ else:
+ # don't need to keep anything back
+ self.remain=0
+ else:
+ # EOF
+ self.remain=0
+ # we cannot use -0 to mean end of array...
+ end = bufflen if self.remain == 0 else -self.remain
+ # exclude the potential split match from the return
+ return buff[:end].replace(FROM_MANGLED, FROM_UNMANGLED)
+
+class MboxoFactory(mailbox.mboxMessage):
+ def __init__(self, message=None):
+ super().__init__(message=MboxoReader(message))
diff --git a/tests/test-generators.py b/tests/test-generators.py
index 700d1d4..bc9b530 100755
--- a/tests/test-generators.py
+++ b/tests/test-generators.py
@@ -17,6 +17,10 @@ fake_args = collections.namedtuple('fakeargs', ['verbose',
'ibody'])(False, None
def generate_specs(args):
+ if not args.nomboxo:
+ # Temporary patch to fix Python email package limitation
+ # It must be removed when the Python package is fixed
+ from mboxo_patch import MboxoFactory, MboxoReader
import archiver
if args.generators:
generator_names = args.generators
@@ -34,7 +38,7 @@ def generate_specs(args):
sys.stderr.write("Generating specs for type '%s'...\n" % gen_type)
gen_spec = []
- mbox = mailbox.mbox(args.mboxfile, None, create=False)
+ mbox = mailbox.mbox(args.mboxfile, None if args.nomboxo else
MboxoFactory, create=False)
for key in mbox.keys():
message_raw = mbox.get_bytes(key) # True raw format, as opposed
to calling .as_bytes()
message = mbox.get(key)
@@ -53,6 +57,10 @@ def generate_specs(args):
def run_tests(args):
+ if not args.nomboxo:
+ # Temporary patch to fix Python email package limitation
+ # It must be removed when the Python package is fixed
+ from mboxo_patch import MboxoFactory, MboxoReader
import archiver
import logging
verbose_logger = logging.getLogger()
@@ -75,7 +83,7 @@ def run_tests(args):
continue
test_args = collections.namedtuple('testargs', ['parse_html',
'generator'])(parse_html, gen_type)
archie = interfacer.Archiver(archiver, test_args)
- mbox = mailbox.mbox(mboxfile, None, create=False)
+ mbox = mailbox.mbox(mboxfile, None if args.nomboxo else
MboxoFactory, create=False)
no_messages = len(mbox.keys())
no_tests = len(tests)
if no_messages != no_tests:
@@ -83,6 +91,7 @@ def run_tests(args):
(gen_type, mboxfile, no_tests, no_messages))
for test in tests:
tests_run += 1
+ # TODO does get_bytes take account of MboxoFactory?
message_raw = mbox.get_bytes(test['index']) # True raw
format, as opposed to calling .as_bytes()
message = mbox.get(test['index'])
msgid =(message.get('message-id') or '').strip()
@@ -118,6 +127,8 @@ def main():
help='List-ID header override if needed')
parser.add_argument('--rootdir', dest='rootdir', type=str, required=True,
help="Root directory of Apache Pony Mail")
+ parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
+ help = 'Skip Mboxo processing')
args = parser.parse_args()
if args.rootdir:
diff --git a/tests/test-parsing.py b/tests/test-parsing.py
index eac8d43..5495384 100755
--- a/tests/test-parsing.py
+++ b/tests/test-parsing.py
@@ -17,6 +17,10 @@ fake_args = collections.namedtuple('fakeargs', ['verbose',
'ibody'])(False, None
def generate_specs(args):
+ if not args.nomboxo:
+ # Temporary patch to fix Python email package limitation
+ # It must be removed when the Python package is fixed
+ from mboxo_patch import MboxoFactory, MboxoReader
import archiver
cli_args = collections.namedtuple('testargs', ['parse_html'])(args.html)
archie = interfacer.Archiver(archiver, cli_args)
@@ -25,7 +29,7 @@ def generate_specs(args):
items = {}
for mboxfile in args.mboxfile:
tests = []
- mbox = mailbox.mbox(mboxfile, None, create=False)
+ mbox = mailbox.mbox(mboxfile, None if args.nomboxo else MboxoFactory,
create=False)
for key in mbox.keys():
message_raw = mbox.get_bytes(key) # True raw format, as opposed
to calling .as_bytes()
message = mbox.get(key)
@@ -47,6 +51,10 @@ def generate_specs(args):
def run_tests(args):
+ if not args.nomboxo:
+ # Temporary patch to fix Python email package limitation
+ # It must be removed when the Python package is fixed
+ from mboxo_patch import MboxoFactory, MboxoReader
import archiver
import logging
verbose_logger = logging.getLogger()
@@ -62,7 +70,7 @@ def run_tests(args):
archie = interfacer.Archiver(archiver, test_args)
for mboxfile, tests in yml['parsing'].items():
- mbox = mailbox.mbox(mboxfile, None, create=False)
+ mbox = mailbox.mbox(mboxfile, None if args.nomboxo else MboxoFactory,
create=False)
no_messages = len(mbox.keys())
no_tests = len(tests)
if no_messages != no_tests:
@@ -70,6 +78,7 @@ def run_tests(args):
('TBA', mboxfile, no_tests, no_messages))
for test in tests:
tests_run += 1
+ # TODO does get_bytes take account of MboxoFactory?
message_raw = mbox.get_bytes(test['index']) # True raw format, as
opposed to calling .as_bytes()
message = mbox.get(test['index'])
msgid =(message.get('message-id') or '').strip()
@@ -111,8 +120,11 @@ def main():
help="Root directory of Apache Pony Mail")
parser.add_argument('--html', dest='html', action='store_true',
help="Enable HTML parsing if generating test specs")
+ parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
+ help = 'Skip Mboxo processing')
args = parser.parse_args()
+ print(args.nomboxo,file=sys.stderr)
if args.rootdir:
tools_dir = os.path.join(args.rootdir, 'tools')
else:
