This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
The following commit(s) were added to refs/heads/master by this push:
new 5fba68f Add thread info capabilities to the archiver
new 98bd28e Merge pull request #43 from sbp/thread-info
5fba68f is described below
commit 5fba68f8c10313059310a30b9aa2f659b59e57ab
Author: Sean B. Palmer <[email protected]>
AuthorDate: Wed Jun 2 13:17:00 2021 +0100
Add thread info capabilities to the archiver
---
tools/archiver.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++++
tools/mappings.yaml | 8 ++++
tools/rethread.py | 65 +++++++++++++++++++++++++++++++++
3 files changed, 176 insertions(+)
diff --git a/tools/archiver.py b/tools/archiver.py
index 71a04e9..580f93a 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -258,6 +258,97 @@ class Body:
return self.string
+def message_identifiers(header, reverse=False):
+ if "<" not in header:
+ return []
+ parts = header.split("<")
+ identifier_junks = parts[1:]
+ identifiers = []
+ for identifier_junk in identifier_junks:
+ identifier = identifier_junk.split(">").pop(0)
+ identifiers.append("<" + identifier + ">")
+ if reverse is True:
+ identifiers = list(reversed(identifiers))
+ return identifiers
+
+
+def get_parent_identifiers(ojson):
+ identifiers = []
+ for irt in message_identifiers(ojson.get("in-reply-to", ""), reverse=True):
+ identifiers.append(irt)
+ for ref in message_identifiers(ojson.get("references", ""), reverse=True):
+ identifiers.append(ref)
+ return identifiers
+
+
+def get_by_message_id(elastic, msgid):
+ data = elastic.es.search(index=elastic.db_mbox, body={
+ "query": {
+ "bool": {
+ "must": {"term": {"message-id": msgid}}
+ }
+ }
+ })
+ if data["hits"]["total"]["value"] == 1:
+ return data["hits"]["hits"][0]["_source"]
+ return None
+
+
+def get_parent_info(elastic, ojson):
+ parent_identifiers = get_parent_identifiers(ojson)
+ if not parent_identifiers:
+ return None
+ for parent_identifier in parent_identifiers:
+ parent_info = get_by_message_id(elastic, parent_identifier)
+ if parent_info is not None:
+ return parent_info
+ return None
+
+
+def get_previous_mid(elastic, forum, ojson):
+ latest = ojson.get("epoch", 1) - 1
+ data = elastic.es.search(index=elastic.db_mbox, body={
+ "query": {
+ "bool": {
+ "must": [
+ {"range": {"epoch": {"lte": latest}}},
+ {"term": {"forum": forum}},
+ {"term": {"top": True}}
+ ]
+ }
+ },
+ "sort": [{"epoch": "desc"}],
+ "size": 1,
+ "_source": "mid",
+ })
+ for hit in data["hits"]["hits"]:
+ return hit["_source"]["mid"]
+ return None
+
+
+def add_thread_properties(elastic, mid, ojson, size):
+ forum = ojson.get("list", "").strip("<>").replace(".", "@", 1)
+
+ parent_info = get_parent_info(elastic, ojson)
+
+ if parent_info is None:
+ top = True
+ thread = mid
+ previous = get_previous_mid(elastic, forum, ojson)
+ else:
+ top = False
+ thread = parent_info.get("thread")
+ previous = parent_info["mid"]
+
+ ojson["forum"] = forum
+ ojson["previous"] = previous
+ ojson["size"] = size
+ ojson["thread"] = thread
+ ojson["top"] = top
+
+ return ojson
+
+
class Archiver(object): # N.B. Also used by import-mbox.py
"""The general archiver class. Compatible with MailMan3 archiver
classes."""
@@ -555,6 +646,18 @@ class Archiver(object): # N.B. Also used by import-mbox.py
else:
elastic = Elastic()
+ if config.get("archiver", "threadinfo"):
+ try:
+ ojson = add_thread_properties(elastic, ojson["mid"], ojson,
len(raw_message))
+ except Exception as err:
+ print("Could not add thread info", err)
+ if logger:
+ logger.info("Could not add thread info %s" % (err,))
+ else:
+ print("Added thread info successfully", ojson["mid"])
+ if logger:
+ logger.info("Added thread info successfully %s" %
(ojson["mid"],))
+
try:
if contents:
for key in contents:
diff --git a/tools/mappings.yaml b/tools/mappings.yaml
index 49b205a..e80d89c 100644
--- a/tools/mappings.yaml
+++ b/tools/mappings.yaml
@@ -59,6 +59,8 @@ mbox:
type: keyword
epoch:
type: long
+ forum:
+ type: keyword
from:
type: text
from_raw:
@@ -77,6 +79,8 @@ mbox:
type: keyword
permalinks:
type: keyword
+ previous:
+ type: keyword
private:
type: boolean
references:
@@ -84,8 +88,12 @@ mbox:
subject:
fielddata: true
type: text
+ thread:
+ type: keyword
to:
type: text
+ top:
+ type: boolean
_notes:
type: text
_archived_at:
diff --git a/tools/rethread.py b/tools/rethread.py
new file mode 100644
index 0000000..c5440cc
--- /dev/null
+++ b/tools/rethread.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import archiver
+from elasticsearch.helpers import scan
+from plugins.elastic import Elastic
+
+
+def first_pass(elastic: Elastic) -> None:
+ hits = scan(
+ client=elastic.es,
+ index=elastic.db_mbox,
+ query={"sort": {"epoch": "asc"}},
+ )
+ for hit in hits:
+ pid = hit["_id"]
+ ojson = hit["_source"]
+ parent_info = archiver.get_parent_info(elastic, ojson)
+ ojson["top"] = parent_info is None
+ ojson["forum"] = ojson.get("list", "").strip("<>").replace(".", "@", 1)
+ source = elastic.es.get(
+ elastic.db_source, ojson["dbid"], _source="source"
+ )["_source"]["source"]
+ ojson["size"] = len(source)
+ ojson["previous"] = ""
+ ojson["thread"] = pid if (parent_info is None) else ""
+ elastic.index(index=elastic.db_mbox, id=pid, body=ojson)
+
+
+def second_pass(elastic: Elastic) -> None:
+ hits = scan(client=elastic.es, index=elastic.db_mbox, query={})
+ for hit in hits:
+ pid = hit["_id"]
+ ojson = hit["_source"]
+ if ojson["thread"] != "":
+ continue
+ if ojson["top"] is True:
+ ojson["previous"] = archiver.get_previous_mid(
+ elastic, ojson["forum"], ojson
+ )
+ ojson["thread"] = pid
+ elastic.index(index=elastic.db_mbox, id=pid, body=ojson)
+ else:
+ tree = []
+ while ojson["thread"] == "":
+ tree.append(ojson)
+ ojson_parent = archiver.get_parent_info(elastic, ojson)
+ if ojson_parent is None:
+ ojson["previous"] = None
+ print("Error:", ojson["mid"], "has no parent")
+ break
+ ojson["previous"] = ojson_parent["mid"]
+ ojson = ojson_parent
+ for info in tree:
+ info["thread"] = ojson["thread"]
+ elastic.index(index=elastic.db_mbox, id=info["mid"], body=info)
+
+
+def main() -> None:
+ elastic: Elastic = Elastic()
+ first_pass(elastic)
+ second_pass(elastic)
+
+
+if __name__ == "__main__":
+ main()
