This is an automated email from the ASF dual-hosted git repository. humbedooh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 6a944258e33fa820ee82558de9813faadd903ddf Author: Daniel Gruno <[email protected]> AuthorDate: Tue Sep 7 16:26:07 2021 -0500 Add gravatar at point of archiving It makes no sense generating this over and over. Let's generate once at archiving and re-use from database. --- server/endpoints/stats.py | 3 --- server/plugins/messages.py | 3 +++ tools/archiver.py | 5 +++++ tools/migrate.py | 7 +++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py index d7d044b..171280b 100644 --- a/server/endpoints/stats.py +++ b/server/endpoints/stats.py @@ -43,9 +43,6 @@ async def process( session, query_defuzzed, query_limit=server.config.database.max_hits, shorten=True, ) - for msg in results: - msg["gravatar"] = plugins.messages.gravatar(msg) - wordcloud = None if server.config.ui.wordcloud: wordcloud = await plugins.messages.wordcloud(session, query_defuzzed) diff --git a/server/plugins/messages.py b/server/plugins/messages.py index 740fd42..291c232 100644 --- a/server/plugins/messages.py +++ b/server/plugins/messages.py @@ -329,6 +329,9 @@ async def query( continue doc["id"] = doc["mid"] if plugins.aaa.can_access_email(session, doc): + # Calculate gravatars if not present in source + if not metadata_only and "gravatar" not in doc: + doc["gravatar"] = gravatar(doc) if not session.credentials: doc = anonymize(doc) if shorten: diff --git a/tools/archiver.py b/tools/archiver.py index 8df5dcb..f9fc455 100755 --- a/tools/archiver.py +++ b/tools/archiver.py @@ -568,11 +568,16 @@ class Archiver(object): # N.B. Also used by import-mbox.py irt = "" document_id = id_set[0] + # Pre-calculate gravatar + mailaddr = email.utils.parseaddr(msg_metadata["from"])[1] + ghash = hashlib.md5(mailaddr.encode("utf-8")).hexdigest() + notes.append(["ARCHIVE: Email archived as %s at %u" % (document_id, time.time())]) output_json = { "from_raw": msg_metadata["from"], "from": msg_metadata["from"], + "gravatar": ghash, "to": msg_metadata["to"], "subject": msg_metadata["subject"], "message-id": msg_metadata["message-id"], diff --git a/tools/migrate.py b/tools/migrate.py index 160810a..6c9bad8 100644 --- a/tools/migrate.py +++ b/tools/migrate.py @@ -29,6 +29,7 @@ else: import argparse import base64 +import email.utils import hashlib import multiprocessing import os @@ -180,6 +181,12 @@ def process_document(old_es, doc, old_dbname, dbname_source, dbname_mbox, do_dki source["_source"]["permalinks"] = doc["_source"]["permalinks"] doc["_source"]["dbid"] = hashlib.sha3_256(source_text).hexdigest() + # Add in gravatar + header_from = doc["_source"]["from"] + mailaddr = email.utils.parseaddr(header_from)[1] + ghash = hashlib.md5(mailaddr.encode("utf-8")).hexdigest() + doc["_source"]["gravatar"] = ghash + # Append migration details to notes field in doc notes = doc["_source"].get("_notes", []) # We want a list, not a single string
