This commit broke the functioning of the --generator option for the archiver.
Please fix. On Mon, 10 Aug 2020 at 17:13, <humbed...@apache.org> wrote: > > This is an automated email from the ASF dual-hosted git repository. > > humbedooh pushed a commit to branch master > in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git > > > The following commit(s) were added to refs/heads/master by this push: > new 95beb51 Pull main operation into main(), linting/PEP conformity > fixes > 95beb51 is described below > > commit 95beb51158b58bcb9fdb1371af7699b72598ac34 > Author: Daniel Gruno <humbed...@apache.org> > AuthorDate: Mon Aug 10 18:13:05 2020 +0200 > > Pull main operation into main(), linting/PEP conformity fixes > > Also prepping for adding ES 7.x support. > import-mbox will need to be updated shortly > --- > tools/archiver.py | 123 > +++++++++++++++++++++++++----------------------------- > 1 file changed, 58 insertions(+), 65 deletions(-) > > diff --git a/tools/archiver.py b/tools/archiver.py > index b97db76..2c50f19 100755 > --- a/tools/archiver.py > +++ b/tools/archiver.py > @@ -62,23 +62,19 @@ import uuid > import json > import certifi > import urllib.parse > +import argparse > +import netaddr > > # Fetch config > config = PonymailConfig() > auth = None > -parseHTML = False > -iBody = None # N.B. Also used by import-mbox.py > -args=None > -dumpDir = None > -aURL = None > > if config.has_section('mailman') and config.has_option('mailman', 'plugin'): > from zope.interface import implementer > from mailman.interfaces.archiver import IArchiver > from mailman.interfaces.archiver import ArchivePolicy > logger = logging.getLogger("mailman.archiver") > -elif __name__ == '__main__': > - import argparse > + > > if config.has_option('archiver', 'baseurl'): > aURL = config.get('archiver', 'baseurl') > @@ -102,8 +98,7 @@ def parse_attachment(part): > if cdtype == "attachment" or cdtype == 'inline': > fd = part.get_payload(decode=True) > # Allow for empty string > - if fd is None: > - return None, None > + if fd is None: return None, None > filename = part.get_filename() > if filename: > print("Found attachment: %s" % filename) > @@ -160,7 +155,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > > """ Intercept index calls and fix up consistency argument """ > def index(self, **kwargs): > - if ES_MAJOR in [5,6]: > + if ES_MAJOR in [5,6,7]: > if kwargs.pop('consistency', None): # drop the key if present > if self.wait_for_active_shards: # replace with wait if > defined > kwargs['wait_for_active_shards'] = > self.wait_for_active_shards > @@ -168,10 +163,11 @@ class Archiver(object): # N.B. Also used by > import-mbox.py > **kwargs > ) > > - def __init__(self, parseHTML=False): > + def __init__(self, generator='full', parse_html=False, dump_dir=None): > """ Just initialize ES. """ > - self.html = parseHTML > - if parseHTML: > + self.html = parse_html > + self.generator = generator > + if parse_html: > import html2text > self.html2text = html2text.html2text > self.dbname = config.get("elasticsearch", "dbname") > @@ -180,7 +176,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > self.consistency = config.get('elasticsearch', 'write', > fallback='quorum') > if ES_MAJOR == 2: > pass > - elif ES_MAJOR in [5,6]: > + elif ES_MAJOR in [5,6,7]: > self.wait_for_active_shards = config.get('elasticsearch', > 'wait', fallback=1) > else: > raise Exception("Unexpected elasticsearch version ", ES_VERSION) > @@ -209,14 +205,14 @@ class Archiver(object): # N.B. Also used by > import-mbox.py > } > ) > # If we have a dump dir, we can risk failing the connection. > - if dumpDir: > + if dump_dir: > try: > self.es = Elasticsearch(dbs, > max_retries=5, > retry_on_timeout=True > ) > except: > - print("ES connection failed, but dumponfail specified, > dumping to %s" % dumpDir) > + print("ES connection failed, but dumponfail specified, > dumping to %s" % dump_dir) > else: > self.es = Elasticsearch(dbs, > max_retries=5, > @@ -233,13 +229,12 @@ class Archiver(object): # N.B. Also used by > import-mbox.py > contents[part_meta['hash']] = part_file > return attachments, contents > > - > - def msgbody(self, msg): > + def msgbody(self, msg, verbose=False, ignore_body=None): > body = None > firstHTML = None > for part in msg.walk(): > # can be called from importer > - if args and args.verbose: > + if verbose: > print("Content-Type: %s" % part.get_content_type()) > """ > Find the first body part and the first HTML part > @@ -251,12 +246,12 @@ class Archiver(object): # N.B. Also used by > import-mbox.py > if not body and part.get_content_type() == 'text/enriched': > body = part.get_payload(decode=True) > elif self.html and not firstHTML and part.get_content_type() > == 'text/html': > - firstHTML = part.get_payload(decode=True) > + first_html = part.get_payload(decode=True) > except Exception as err: > print(err) > > # this requires a GPL lib, user will have to install it themselves > - if firstHTML and (not body or len(body) <= 1 or (iBody and > str(body).find(str(iBody)) != -1)): > + if firstHTML and (not body or len(body) <= 1 or (ignore_body and > str(body).find(str(ignore_body)) != -1)): > body = self.html2text(firstHTML.decode("utf-8", 'ignore') if > type(firstHTML) is bytes else firstHTML) > > # See issue#463 > @@ -273,7 +268,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > return body > > # N.B. this is also called by import-mbox.py > - def compute_updates(self, lid, private, msg): > + def compute_updates(self, args, lid, private, msg): > """Determine what needs to be sent to the archiver. > > :param lid: The list id > @@ -324,7 +319,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > # mdate calculations are all done, prepare the index entry > epoch = email.utils.mktime_tz(mdate) > mdatestring = time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(epoch)) > - body = self.msgbody(msg) > + body = self.msgbody(msg, verbose=args.verbose, > ignore_body=args.ibody) > try: > if 'content-type' in msg_metadata and > msg_metadata['content-type'].find("flowed") != -1: > body = convertToWrapped(body, character_set="utf-8") > @@ -352,8 +347,9 @@ class Archiver(object): # N.B. Also used by import-mbox.py > except Exception as err: > if logger: > # N.B. use .get just in case there is no message-id > - logger.warning("Could not generate MID: %s. MSGID: %s", > err, msg_metadata.get('message-id','?')) > + logger.warning("Could not generate MID: %s. MSGID: %s", > err, msg_metadata.get('message-id', '?')) > mid = pmid > + > if 'in-reply-to' in msg_metadata: > try: > try: > @@ -383,7 +379,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > > return ojson, contents, msg_metadata, irt > > - def archive_message(self, mlist, msg, raw_msg): > + def archive_message(self, args, mlist, msg, raw_message): > """Send the message to the archiver. > > :param mlist: The IMailingList object. > @@ -402,7 +398,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py > elif hasattr(mlist, 'archive_policy') and mlist.archive_policy is > not ArchivePolicy.public: > private = True > > - ojson, contents, msg_metadata, irt = self.compute_updates(lid, > private, msg) > + ojson, contents, msg_metadata, irt = self.compute_updates(args, lid, > private, msg) > if not ojson: > _id = msg.get('message-id') or msg.get('Subject') or > msg.get("Date") > raise Exception("Could not parse message %s for %s" % (_id,lid)) > @@ -438,23 +434,23 @@ class Archiver(object): # N.B. Also used by > import-mbox.py > consistency = self.consistency, > body = { > "message-id": msg_metadata['message-id'], > - "source": self.mbox_source(raw_msg) > + "source": self.mbox_source(raw_message) > } > ) > # If we have a dump dir and ES failed, push to dump dir instead as a > JSON object > # We'll leave it to another process to pick up the slack. > except Exception as err: > - if dumpDir: > + if args.dump: > print("Pushing to ES failed, but dumponfail specified, > dumping JSON docs") > uid = uuid.uuid4() > - mboxPath = os.path.join(dumpDir, "%s.json" % uid) > + mboxPath = os.path.join(args.dump, "%s.json" % uid) > with open(mboxPath, "w") as f: > json.dump({ > 'id': ojson['mid'], > 'mbox': ojson, > 'mbox_source': { > "message-id": msg_metadata['message-id'], > - "source": self.mbox_source(raw_msg) > + "source": self.mbox_source(raw_message) > }, > 'attachments': contents > },f , indent = 2) > @@ -488,8 +484,8 @@ class Archiver(object): # N.B. Also used by import-mbox.py > if dm: > cid = dm.group(1) > mid = dm.group(2) > - if self.es.exists(index = self.dbname, doc_type = 'account', > id = cid): > - doc = self.es.get(index = self.dbname, doc_type = > 'account', id = cid) > + if self.es.exists(index=self.dbname, doc_type='account', > id=cid): > + doc = self.es.get(index=self.dbname, doc_type='account', > id=cid) > if doc: > oldrefs.append(cid) > # N.B. no index is supplied, so ES will generate one > @@ -571,7 +567,8 @@ class Archiver(object): # N.B. Also used by import-mbox.py > """ > return None > > -if __name__ == '__main__': > + > +def main(): > parser = argparse.ArgumentParser(description='Command line options.') > parser.add_argument('--lid', dest='lid', type=str, nargs=1, > help='Alternate specific list ID') > @@ -593,16 +590,15 @@ if __name__ == '__main__': > help='Try to convert HTML to text if no text/plain > message is found') > parser.add_argument('--dry', dest='dry', action='store_true', > help='Do not save emails to elasticsearch, only test > parsing') > + parser.add_argument('--ignorebody', dest='ibody', type=str, nargs=1, > + help='Optional email bodies to treat as empty (in > conjunction with --html2text)') > parser.add_argument('--dumponfail', dest='dump', > - help='If pushing to ElasticSearch fails, dump > documents in JSON format to this directory and fail silently.') > + help='If pushing to ElasticSearch fails, dump > documents in JSON format to this directory and ' > + 'fail silently.') > parser.add_argument('--generator', dest='generator', > help='Override the generator.') > args = parser.parse_args() > > - if args.html2text: > - parseHTML = True > - if args.dump: > - dumpDir = args.dump > if args.verbose: > logging.basicConfig(stream=sys.stdout, level=logging.INFO) > else: > @@ -610,39 +606,34 @@ if __name__ == '__main__': > # Also eliminates: 'Undecodable raw error response from server:' > warning message > logging.getLogger("elasticsearch").setLevel(logging.ERROR) > > - if args.generator: > - archiver_generator = args.generator > - > - archie = Archiver(parseHTML = parseHTML) > + archie = Archiver(generator=args.generator or archiver_generator, > parse_html=args.html2text) > # use binary input so parser can use appropriate charset > input_stream = sys.stdin.buffer > > try: > - msgstring = input_stream.read() > + raw_message = input_stream.read() > try: > - msg = email.message_from_bytes(msgstring) > + msg = email.message_from_bytes(raw_message) > except Exception as err: > print("STDIN parser exception: %s" % err) > > # We're reading from STDIN, so let's fake an MM3 call > ispublic = True > - ignorefrom = None > - allowfrom = None > > if args.altheader: > - altheader = args.altheader[0] > - if altheader in msg: > + alt_header = args.altheader[0] > + if alt_header in msg: > try: > - msg.replace_header('List-ID', msg.get(altheader)) > + msg.replace_header('List-ID', msg.get(alt_header)) > except: > - msg.add_header('list-id', msg.get(altheader)) > + msg.add_header('list-id', msg.get(alt_header)) > elif 'altheader' in sys.argv: > - altheader = sys.argv[len(sys.argv)-1] > - if altheader in msg: > + alt_header = sys.argv[len(sys.argv)-1] > + if alt_header in msg: > try: > - msg.replace_header('List-ID', msg.get(altheader)) > + msg.replace_header('List-ID', msg.get(alt_header)) > except: > - msg.add_header('list-id', msg.get(altheader)) > + msg.add_header('list-id', msg.get(alt_header)) > > # Set specific LID? > if args.lid and len(args.lid[0]) > 3: > @@ -654,21 +645,21 @@ if __name__ == '__main__': > > #Ignore based on --ignore flag? > if args.ignorefrom: > - ignorefrom = args.ignorefrom[0] > - if fnmatch.fnmatch(msg.get("from"), ignorefrom) or > (msg.get("list-id") and fnmatch.fnmatch(msg.get("list-id"), ignorefrom)): > + ignore_from = args.ignorefrom[0] > + if fnmatch.fnmatch(msg.get("from"), ignore_from) or > (msg.get("list-id") and fnmatch.fnmatch(msg.get("list-id"), ignore_from)): > print("Ignoring message as instructed by --ignore flag") > sys.exit(0) > > # Check CIDR if need be > if args.allowfrom: > - from netaddr import IPNetwork, IPAddress > - c = IPNetwork(args.allowfrom[0]) > + > + c = netaddr.IPNetwork(args.allowfrom[0]) > good = False > for line in msg.get_all('received') or []: > m = re.search(r"from .+\[(.+)\]", line) > if m: > try: > - ip = IPAddress(m.group(1)) > + ip = netaddr.IPAddress(m.group(1)) > if ip in c: > good = True > msg.add_header("ip-whitelisted", "yes") > @@ -681,19 +672,18 @@ if __name__ == '__main__': > # Replace date header with $now? > if args.makedate: > msg.replace_header('date', email.utils.formatdate()) > - > + is_public = True > if args.private: > - ispublic = False > + is_public = False > if 'list-id' in msg: > if not msg.get('archived-at'): > msg.add_header('archived-at', email.utils.formatdate()) > - list_data = namedtuple('importmsg', ['list_id', > 'archive_public'])(list_id = msg.get('list-id'), archive_public=ispublic) > + list_data = namedtuple('importmsg', ['list_id', > 'archive_public'])(list_id=msg.get('list-id'), > + > archive_public=is_public) > > try: > - lid, mid = archie.archive_message(list_data, msg, msgstring) > + lid, mid = archie.archive_message(args, list_data, msg, > raw_message) > print("%s: Done archiving to %s as %s!" % > (email.utils.formatdate(), lid, mid)) > - if aURL: > - print("Published as: %s/thread.html/%s" % (aURL, > urllib.parse.quote(mid))) > except Exception as err: > if args.verbose: > traceback.print_exc() > @@ -711,3 +701,6 @@ if __name__ == '__main__': > print("Could not parse email: %s (@ %s)" % (err, line)) > sys.exit(-1) > > + > +if __name__ == '__main__': > + main() >
