This is an automated email from the ASF dual-hosted git repository. humbedooh pushed a commit to branch humbedooh/archiver-date-estimate in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit ddeab4dabb5cbbedec62b581c0ff1c8ba624a41e Author: Daniel Gruno <[email protected]> AuthorDate: Sat Sep 18 10:38:50 2021 -0500 Look for dates on envelope and in headers if Date is missing or invalid This expands the way we deal with missing or ínvalid date headers, by: - looking for an envelope From header, and if possible, using the date from there - if no valid From envelope, search the Received headers for a useful date - if still nothing, fall back to current time and date. This should address #56. I've tested with emails missing all three fields (sets to NOW), the Date field only (uses the From envelope header), and missing both Date and FROM (uses Received headers then). --- tools/archiver.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tools/archiver.py b/tools/archiver.py index 75dd7cb..95ab722 100755 --- a/tools/archiver.py +++ b/tools/archiver.py @@ -53,6 +53,7 @@ import time import traceback import typing import uuid +import datetime import elasticsearch import formatflowed @@ -482,15 +483,31 @@ class Archiver(object): # N.B. Also used by import-mbox.py message_date = email.utils.parsedate_tz( str(msg_metadata.get("archived-at")) ) - if not message_date: - epoch = time.time() + print("No message date could be derived from the Date: header, looking elsewhere.") + # See if we have a "From" header line in the raw email, we can use + first_line = raw_msg.split(b"\n", 1)[0].decode("us-ascii") + if first_line.startswith("From "): + # If we have one, the date must be the third element when splitting by single space. + env_from_date = first_line.split(" ", 2)[-1] # Split twice, grab last element. + message_date = email.utils.parsedate_tz(env_from_date) + if message_date: + print("Found date in envelope FROM header: %s" % env_from_date) + # Otherwise, look for a Received: header we can scan + if not message_date: + for recv_from in msg.get_all('received', []): # We may have multiple of these, not all have "from". + m = re.match(r"from[^;]+?;\s+(.+?)(?:$|[\r\n])", recv_from) + if m: + message_date = email.utils.parsedate_tz(m.group(1)) + if message_date: + print("Found date in Received header: %s" % m.group(1)) + break + if not message_date: + print("Could not find any valid dates in email headers, using current time") + epoch = time.time() + else: + epoch = email.utils.mktime_tz(message_date) notes.append(["BADDATE: Email date missing or invalid, setting to %u" % epoch]) - print( - "Date (%s) seems totally wrong, using current UNIX epoch instead." - % message_date - ) - else: epoch = email.utils.mktime_tz(message_date) # message_date calculations are all done, prepare the index entry
