This is an automated email from the ASF dual-hosted git repository. sebb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit ddb0af055952cb3d6ba36f57caf0d07f12a9d1d0 Author: Sebb <[email protected]> AuthorDate: Mon Nov 22 22:50:13 2021 +0000 Don't fetch fields that aren't needed --- server/endpoints/stats.py | 14 +++++++++----- server/plugins/messages.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/server/endpoints/stats.py b/server/endpoints/stats.py index dbf6e4c..b25775c 100644 --- a/server/endpoints/stats.py +++ b/server/endpoints/stats.py @@ -59,15 +59,11 @@ async def process( query_since = query_defuzzed.copy() query_since['must'].append({"range" : { "epoch": { "gt": epoch}}}) results = await plugins.messages.query( - session, query_since, query_limit=1, metadata_only=True + session, query_since, query_limit=1, source_fields=[''] # don't need any fields ) if len(results) == 0: return {"changed" : False} - results = await plugins.messages.query( - session, query_defuzzed, query_limit=server.config.database.max_hits - ) - # statsOnly: Whether to only send statistical info (for n-grams etc), and not the # thread struct and message bodies # Param: quick @@ -76,6 +72,14 @@ async def process( # i.e. omit thread_struct, top 10 participants and word-cloud emailsOnly = 'emailsOnly' in indata + source_fields = None + if statsOnly: + source_fields = ['epoch'] + + results = await plugins.messages.query( + session, query_defuzzed, query_limit=server.config.database.max_hits, source_fields=source_fields + ) + wordcloud = None if server.config.ui.wordcloud and not emailsOnly and not statsOnly: wordcloud = await plugins.messages.wordcloud(session, query_defuzzed) diff --git a/server/plugins/messages.py b/server/plugins/messages.py index baf6e1a..f255f18 100644 --- a/server/plugins/messages.py +++ b/server/plugins/messages.py @@ -318,7 +318,8 @@ async def query( query_limit=10000, hide_deleted=True, metadata_only=False, - epoch_order="desc" + epoch_order="desc", + source_fields=None ): """ Advanced query and grab for stats.py @@ -334,6 +335,8 @@ async def query( } if metadata_only: # Only doc IDs and AAA fields. es_query["_source"] = ["deleted", "private", "mid", "dbid", "list_raw"] + elif source_fields: + es_query["_source"] = source_fields else: es_query["_source"] = { "excludes": ["body"] } async for hit in session.database.scan( @@ -344,14 +347,15 @@ async def query( # If email was delete/hidden and we're not doing an admin query, ignore it if hide_deleted and doc.get("deleted", False): continue - doc["id"] = doc["mid"] if plugins.aaa.can_access_email(session, doc): + if "mid" in doc: # might be missing when using source_fields + doc["id"] = doc["mid"] # Calculate gravatars if not present in source - if not metadata_only and "gravatar" not in doc: + if not metadata_only and not source_fields and "gravatar" not in doc: doc["gravatar"] = gravatar(doc) if not session.credentials: doc = anonymize(doc) - if not metadata_only: + if "body_short" in doc: # The body_short field is set to SHORT_BODY_MAX_LEN+1 if the body is longer # than SHORT_BODY_MAX_LEN, so we know if it has been truncated if len(doc["body_short"] or "") > SHORT_BODY_MAX_LEN:
