This was already done for patches but cover letters and comments were
not handled correctly, resulting in errors while parsing archives. While
we're here, we slightly modify how these exceptions are handle. Rather
than simply ignoring them, as we were doing, we raise a custom
exception. This allows us to specifically identify these types of
exceptions, print a log and still skip them (which we want, as seen in
commit d2eb1f6d2).
While we're here, we change from separate create-save calls to a
combined create-save call for both created CoverLetter and Comment
objects. We were already doing this for patches.
Signed-off-by: Stephen Finucane <step...@that.guru>
Cc: Daniel Axtens <d...@axtens.net>
---
patchwork/management/commands/parsearchive.py | 7 ++-
patchwork/management/commands/parsemail.py | 3 ++
patchwork/parser.py | 50 ++++++++++++-------
3 files changed, 39 insertions(+), 21 deletions(-)
diff --git a/patchwork/management/commands/parsearchive.py
b/patchwork/management/commands/parsearchive.py
index 3ef91f51..381ee94d 100644
--- a/patchwork/management/commands/parsearchive.py
+++ b/patchwork/management/commands/parsearchive.py
@@ -12,6 +12,7 @@ from django.core.management.base import BaseCommand
from patchwork import models
from patchwork.parser import parse_mail
+from patchwork.parser import DuplicateMailError
logger = logging.getLogger(__name__)
@@ -89,9 +90,11 @@ class Command(BaseCommand):
else:
dropped += 1
except ValueError:
- # TODO(stephenfin): Perhaps we should store the broken patch
- # somewhere for future reference?
errors += 1
+ logger.warning('Invalid mail')
+ except DuplicateMailError as exc:
+ errors += 1
+ logger.warning('Duplicate mail for message ID %s', exc.msgid)
if verbosity == 1 and (i % 10) == 0:
self.stdout.write('%06d/%06d\r' % (i, count), ending='')
diff --git a/patchwork/management/commands/parsemail.py
b/patchwork/management/commands/parsemail.py
index a7ec97ff..9c7cdc0b 100644
--- a/patchwork/management/commands/parsemail.py
+++ b/patchwork/management/commands/parsemail.py
@@ -11,6 +11,7 @@ from django.core.management import base
from django.utils import six
from patchwork.parser import parse_mail
+from patchwork.parser import DuplicateMailError
logger = logging.getLogger(__name__)
@@ -65,6 +66,8 @@ class Command(base.BaseCommand):
result = parse_mail(mail, options['list_id'])
if result is None:
logger.warning('Nothing added to database')
+ except DuplicateMailError as exc:
+ logger.warning('Duplicate mail for message ID %s', exc.msgid)
except Exception:
logger.exception('Error when parsing incoming email',
extra={'mail': mail.as_string()})
diff --git a/patchwork/parser.py b/patchwork/parser.py
index 4ede8255..611db39c 100644
--- a/patchwork/parser.py
+++ b/patchwork/parser.py
@@ -41,6 +41,12 @@ SERIES_DELAY_INTERVAL = 10
logger = logging.getLogger(__name__)
+class DuplicateMailError(Exception):
+
+ def __init__(self, msgid):
+ self.msgid = msgid
+
+
def normalise_space(value):
whitespace_re = re.compile(r'\s+')
return whitespace_re.sub(' ', value).strip()
@@ -1014,8 +1020,7 @@ def parse_mail(mail, list_id=None):
state=find_state(mail))
logger.debug('Patch saved')
except IntegrityError:
- logger.error("Duplicate mail for message ID %s" % msgid)
- return None
+ raise DuplicateMailError(msgid=msgid)
# if we don't have a series marker, we will never have an existing
# series to match against.
@@ -1121,15 +1126,18 @@ def parse_mail(mail, list_id=None):
logger.error("Multiple SeriesReferences for %s"
" in project %s!" % (msgid, project.name))
- cover_letter = CoverLetter(
- msgid=msgid,
- project=project,
- name=name[:255],
- date=date,
- headers=headers,
- submitter=author,
- content=message)
- cover_letter.save()
+ try:
+ cover_letter = CoverLetter.objects.create(
+ msgid=msgid,
+ project=project,
+ name=name[:255],
+ date=date,
+ headers=headers,
+ submitter=author,
+ content=message)
+ except IntegrityError:
+ raise DuplicateMailError(msgid=msgid)
+
logger.debug('Cover letter saved')
series.add_cover_letter(cover_letter)
@@ -1145,14 +1153,18 @@ def parse_mail(mail, list_id=None):
author = get_or_create_author(mail)
- comment = Comment(
- submission=submission,
- msgid=msgid,
- date=date,
- headers=headers,
- submitter=author,
- content=message)
- comment.save()
+
+ try:
+ comment = Comment.objects.create(
+ submission=submission,
+ msgid=msgid,
+ date=date,
+ headers=headers,
+ submitter=author,
+ content=message)
+ except IntegrityError:
+ raise DuplicateMailError(msgid=msgid)
+
logger.debug('Comment saved')
return comment
--
2.17.1
_______________________________________________
Patchwork mailing list
Patchwork@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/patchwork
