This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
The following commit(s) were added to refs/heads/master by this push:
new 3991c2e Move parsing options to constructor
3991c2e is described below
commit 3991c2e8080a0640dd046be0bd4da0c0d2c6adc3
Author: Sebb <[email protected]>
AuthorDate: Sat Aug 22 21:31:10 2020 +0100
Move parsing options to constructor
---
tools/archiver.py | 33 ++++++++++++++-------------------
tools/import-mbox.py | 41 +++++++++++++++--------------------------
2 files changed, 29 insertions(+), 45 deletions(-)
diff --git a/tools/archiver.py b/tools/archiver.py
index 77c478b..bb57f00 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -56,7 +56,6 @@ import uuid
import formatflowed
import netaddr
-import yaml
import plugins.ponymailconfig
import plugins.generators
@@ -67,8 +66,6 @@ import elasticsearch
config = plugins.ponymailconfig.PonymailConfig()
# Set some vars before we begin
-archiver_generator = config.get("archiver", "generator", fallback="full")
-# Fall back to full hashing if nothing is set.
logger = None
@@ -250,32 +247,32 @@ class Archiver(object): # N.B. Also used by
import-mbox.py
"x-mailman-rule-misses",
]
- def __init__(self, generator=archiver_generator, parse_html=False,
dump_dir=None):
+ def __init__(self, generator=None, parse_html=False, ignore_body=None,
verbose=False):
""" Just initialize ES. """
self.html = parse_html
- self.generator = generator
- self.dump_dir = dump_dir
+ # Fall back to full hashing if nothing is set.
+ self.generator = generator or config.get("archiver", "generator",
fallback="full")
self.cropout = config.get("debug", "cropout")
- if parse_html:
+ self.verbose = verbose
+ self.ignore_body = ignore_body
+ if self.html:
import html2text
self.html2text = html2text.html2text
def message_body(
- self, msg: email.message.Message, verbose=False, ignore_body=None
+ self, msg: email.message.Message
) -> typing.Optional[Body]:
"""
Fetches the proper text body from an email as an archiver.Body
object
:param msg: The email or part of it to examine for proper body
- :param verbose: Verbose output while parsing
- :param ignore_body: Optional bodies to ignore while parsing
:return: archiver.Body object
"""
body = None
first_html = None
for part in msg.walk():
# can be called from importer
- if verbose:
+ if self.verbose:
print("Content-Type: %s" % part.get_content_type())
"""
Find the first body part and the first HTML part
@@ -300,7 +297,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
if first_html and (
body is None
or len(body) <= 1
- or (ignore_body and str(body).find(str(ignore_body)) != -1)
+ or (self.ignore_body and str(body).find(str(self.ignore_body)) !=
-1)
):
body = first_html
body.assign(self.html2text(str(body)))
@@ -310,14 +307,12 @@ class Archiver(object): # N.B. Also used by
import-mbox.py
# N.B. this is also called by import-mbox.py
def compute_updates(
self,
- args,
lid: typing.Optional[str],
private: bool,
msg: email.message.Message,
raw_msg: bytes,
) -> typing.Tuple[typing.Optional[dict], dict, dict, typing.Optional[str]]:
"""Determine what needs to be sent to the archiver.
- :param args: Command line arguments for the archiver
:param lid: The list id
:param private: Whether privately archived email or not (bool)
:param msg: The message object
@@ -384,7 +379,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
epoch = email.utils.mktime_tz(message_date)
# message_date calculations are all done, prepare the index entry
date_as_string = time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(epoch))
- body = self.message_body(msg, verbose=args.verbose,
ignore_body=args.ibody)
+ body = self.message_body(msg)
attachments, contents = message_attachments(msg)
irt = ""
@@ -464,7 +459,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
private = True
ojson, contents, msg_metadata, irt = self.compute_updates(
- args, lid, private, msg, raw_message
+ lid, private, msg, raw_message
)
sha3 = hashlib.sha3_256(raw_message).hexdigest()
if not ojson:
@@ -475,14 +470,14 @@ class Archiver(object): # N.B. Also used by
import-mbox.py
print("**** Dry run, not saving message to database *****")
return lid, ojson["mid"]
- if self.dump_dir:
+ if args.dump:
try:
elastic = plugins.elastic.Elastic()
except elasticsearch.exceptions.ElasticsearchException as e:
print(e)
print(
"ES connection failed, but dumponfail specified, dumping
to %s"
- % self.dump_dir
+ % args.dump
)
else:
elastic = plugins.elastic.Elastic()
@@ -732,7 +727,7 @@ def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
archie = Archiver(
- generator=args.generator or archiver_generator,
parse_html=args.html2text
+ generator=args.generator, parse_html=args.html2text,
ignore_body=args.ibody, verbose=args.verbose
)
# use binary input so parser can use appropriate charset
input_stream = sys.stdin.buffer
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 1e31c32..482a2b8 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -67,7 +67,6 @@ fileToLID = {}
interactive = False
extension = ".mbox"
piperWeirdness = False
-parseHTML = False
resendTo = None
timeout = 600
fromFilter = None
@@ -75,17 +74,9 @@ dedup = False
dedupped = 0
noMboxo = False # Don't skip MBoxo patch
-# Fetch config and set up ES
-es = Elastic()
-# We need the index name for bulk actions
-dbname = es.getdbname()
-
rootURL = ""
-
def bulk_insert(name, json, xes, dtype, wc="quorum"):
- if args.dry:
- return
sys.stderr.flush()
@@ -127,10 +118,9 @@ class SlurpThread(Thread):
ml = ""
mboxfile = ""
filename = ""
- if args.generator:
- archie = archiver.Archiver(generator=args.generator,
parse_html=parseHTML)
- else:
- archie = archiver.Archiver(parse_html=parseHTML)
+ archie = archiver.Archiver(
+ generator=args.generator, parse_html=args.html2text,
ignore_body=args.ibody, verbose=args.verbose
+ )
while len(lists) > 0:
self.printid("%u elements left to slurp" % len(lists))
@@ -258,7 +248,7 @@ class SlurpThread(Thread):
continue
json, contents, _msgdata, _irt = archie.compute_updates(
- args, list_override, private, message, message_raw
+ list_override, private, message, message_raw
)
# Not sure this can ever happen
@@ -352,7 +342,7 @@ class SlurpThread(Thread):
id=key,
body={"source": contents[key]},
)
- if len(ja) >= 40:
+ if len(ja) >= 40 and not args.dry:
bulk_insert(self.name, ja, es, "mbox")
ja = []
@@ -382,11 +372,11 @@ class SlurpThread(Thread):
goodies += count
baddies += bad
- if len(ja) > 0:
+ if len(ja) > 0 and not args.dry:
bulk_insert(self.name, ja, es, "mbox")
ja = []
- if len(jas) > 0:
+ if len(jas) > 0 and not args.dry:
bulk_insert(self.name, jas, es, "source")
jas = []
self.printid("Done, %u elements left to slurp" % len(lists))
@@ -557,8 +547,6 @@ if args.dedup:
dedup = args.dedup
if args.ext:
extension = args.ext[0]
-if args.html2text:
- parseHTML = True
if args.fromfilter:
fromFilter = args.fromfilter[0]
if args.nomboxo:
@@ -575,9 +563,6 @@ if args.timeout:
timeout = args.timeout[0]
baddies = 0
-# No point continuing if the index does not exist
-print("Checking that the database index %s exists ... " % dbname)
-
# elasticsearch logs lots of warnings on retries/connection failure
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
@@ -592,6 +577,14 @@ if args.verbose:
if args.dry:
print("Dry-run; continuing to check input data")
else:
+ # Fetch config and set up ES
+ es = Elastic()
+ # We need the index name for bulk actions
+ dbname = es.getdbname()
+
+ # No point continuing if the index does not exist
+ print("Checking that the database index %s exists ... " % dbname)
+
# Need to check the index before starting bulk operations
try:
if not es.indices.exists(index=es.db_mbox):
@@ -602,10 +595,6 @@ else:
print("Error: unable to check if the index %s exists!: %s" %
(es.db_mbox, err))
sys.exit(1)
-if args.generator:
- archiver.archiver_generator = args.generator
-
-
def glob_dir(d):
dirs = [f for f in listdir(d) if isdir(join(d, f))]
mboxes = [f for f in glob.glob(join(d, "*" + extension)) if isfile(f)]
