This is an automated email from the ASF dual-hosted git repository. sebb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 475e3a7adfb6f1812111cef42452b8c60719e644 Author: Sebb <[email protected]> AuthorDate: Mon Dec 13 17:37:14 2021 +0000 Rename to agree with elastic/Elastic class Also, the prefix makes it easier to distinguish db index references --- server/endpoints/email.py | 2 +- server/endpoints/gravatar.py | 2 +- server/endpoints/mgmt.py | 18 +++++++++--------- server/plugins/auditlog.py | 6 +++--- server/plugins/background.py | 20 ++++++++++---------- server/plugins/database.py | 24 ++++++++++++------------ server/plugins/messages.py | 10 +++++----- server/plugins/session.py | 12 ++++++------ 8 files changed, 47 insertions(+), 47 deletions(-) diff --git a/server/endpoints/email.py b/server/endpoints/email.py index 1ff6fdd..22f2343 100644 --- a/server/endpoints/email.py +++ b/server/endpoints/email.py @@ -67,7 +67,7 @@ async def process( try: assert session.database, "Database not connected!" attachment = await session.database.get( - index=session.database.dbs.attachment, id=indata.get("file") + index=session.database.dbs.db_attachment, id=indata.get("file") ) if attachment: blob = base64.decodebytes(attachment["_source"].get("source").encode("utf-8")) diff --git a/server/endpoints/gravatar.py b/server/endpoints/gravatar.py index 6602596..c0b1cf9 100644 --- a/server/endpoints/gravatar.py +++ b/server/endpoints/gravatar.py @@ -44,7 +44,7 @@ async def fetch_gravatar(gid): async def gravatar_exists_in_db(session, gid): res = await session.database.search( - index=session.database.dbs.mbox, + index=session.database.dbs.db_mbox, size=1, body={"query": {"bool": {"must": [{"term": {"gravatar": gid}}]}}}, ) diff --git a/server/endpoints/mgmt.py b/server/endpoints/mgmt.py index 6ac0d67..28335ea 100644 --- a/server/endpoints/mgmt.py +++ b/server/endpoints/mgmt.py @@ -58,14 +58,14 @@ async def process( email["deleted"] = True if server.config.ui.fully_delete and email["id"] and email["dbid"]: # Full on GDPR blast? await session.database.delete( - index=session.database.dbs.mbox, id=email["id"], + index=session.database.dbs.db_mbox, id=email["id"], ) await session.database.delete( - index=session.database.dbs.source, id=email["dbid"], + index=session.database.dbs.db_source, id=email["dbid"], ) else: # Standard behavior: hide the email from everyone. await session.database.update( - index=session.database.dbs.mbox, body={"doc": email}, id=email["id"], + index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"], ) lid = email.get("list_raw", "??") await plugins.auditlog.add_entry(session, action="delete", target=doc, lid=lid, log=f"Removed email {doc} from {lid} archives") @@ -80,7 +80,7 @@ async def process( if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email): email["deleted"] = True await session.database.update( - index=session.database.dbs.mbox, body={"doc": email}, id=email["id"], + index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"], ) lid = email.get("list_raw", "??") await plugins.auditlog.add_entry(session, action="hide", target=doc, lid=lid, log=f"Hid email {doc} from {lid} archives") @@ -95,7 +95,7 @@ async def process( if email and isinstance(email, dict) and plugins.aaa.can_access_email(session, email): email["deleted"] = False await session.database.update( - index=session.database.dbs.mbox, body={"doc": email}, id=email["id"], + index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"], ) lid = email.get("list_raw", "??") await plugins.auditlog.add_entry(session, action="unhide", target=doc, lid=lid, log=f"Unhid email {doc} from {lid} archives") @@ -110,14 +110,14 @@ async def process( try: assert session.database, "Database not connected!" attachment = await session.database.get( - index=session.database.dbs.attachment, id=doc + index=session.database.dbs.db_attachment, id=doc ) except plugins.database.DBError: pass # attachment not found if attachment and isinstance(attachment, dict): await session.database.delete( - index=session.database.dbs.attachment, id=attachment["_id"], + index=session.database.dbs.db_attachment, id=attachment["_id"], ) lid = "<system>" await plugins.auditlog.add_entry(session, action="delatt", target=doc, lid=lid, log=f"Removed attachment {doc} from the archives") @@ -164,7 +164,7 @@ async def process( # Save edited email await session.database.update( - index=session.database.dbs.mbox, body={"doc": email}, id=email["id"], + index=session.database.dbs.db_mbox, body={"doc": email}, id=email["id"], ) # Fetch source, mark as deleted (modified) and save IF anything but just privacy changed @@ -176,7 +176,7 @@ async def process( source = source["_source"] source["deleted"] = True await session.database.update( - index=session.database.dbs.source, body={"doc": source}, id=docid, + index=session.database.dbs.db_source, body={"doc": source}, id=docid, ) await plugins.auditlog.add_entry(session, action="edit", target=doc, lid=lid, diff --git a/server/plugins/auditlog.py b/server/plugins/auditlog.py index ad94d16..955ba82 100644 --- a/server/plugins/auditlog.py +++ b/server/plugins/auditlog.py @@ -50,11 +50,11 @@ async def view( assert session.database, "No database connection could be found!" if not filter: res = await session.database.search( - index=session.database.dbs.auditlog, size=num_entries, from_=page * num_entries, sort="date:desc", + index=session.database.dbs.db_auditlog, size=num_entries, from_=page * num_entries, sort="date:desc", ) else: res = await session.database.search( - index=session.database.dbs.auditlog, size=num_entries, from_=page * num_entries, sort="date:desc", + index=session.database.dbs.db_auditlog, size=num_entries, from_=page * num_entries, sort="date:desc", body={ "query": {"bool": {"must": [{"terms": {"action": filter}}]}} }, @@ -79,7 +79,7 @@ async def add_entry(session: plugins.session.SessionObject, action: str, target: assert session.credentials, "No session credentials could be found!" assert session.database, "Session not connected to database!" await session.database.index( - index=session.database.dbs.auditlog, + index=session.database.dbs.db_auditlog, body={ "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(time.time())), "action": action, diff --git a/server/plugins/background.py b/server/plugins/background.py index 34296a3..72c626d 100644 --- a/server/plugins/background.py +++ b/server/plugins/background.py @@ -63,13 +63,13 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict: # Fetch aggregations of all private emails # Do this first, so mixed lists are not marked private - s = Search(using=db.client, index=db.dbs.mbox).filter( + s = Search(using=db.client, index=db.dbs.db_mbox).filter( "term", private=True ) s.aggs.bucket("per_list", "terms", field="list_raw", size=limit) res = await db.search( - index=db.dbs.mbox, body=s.to_dict(), size=0 + index=db.dbs.db_mbox, body=s.to_dict(), size=0 ) for ml in res["aggregations"]["per_list"]["buckets"]: @@ -80,13 +80,13 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict: } # Fetch aggregations of all public emails - s = Search(using=db.client, index=db.dbs.mbox).filter( + s = Search(using=db.client, index=db.dbs.db_mbox).filter( "term", private=False ) s.aggs.bucket("per_list", "terms", field="list_raw", size=limit) res = await db.search( - index=db.dbs.mbox, body=s.to_dict(), size=0 + index=db.dbs.db_mbox, body=s.to_dict(), size=0 ) for ml in res["aggregations"]["per_list"]["buckets"]: @@ -97,12 +97,12 @@ async def get_lists(database: plugins.configuration.DBConfig) -> dict: } # Get 90 day activity, if any - s = Search(using=db.client, index=db.dbs.mbox) + s = Search(using=db.client, index=db.dbs.db_mbox) s = s.filter('range', date = {'gte': ACTIVITY_TIMESPAN}) s.aggs.bucket("per_list", "terms", field="list_raw", size=limit) res = await db.search( - index=db.dbs.mbox, body=s.to_dict(), size=0 + index=db.dbs.db_mbox, body=s.to_dict(), size=0 ) for ml in res["aggregations"]["per_list"]["buckets"]: @@ -125,7 +125,7 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict: # Fetch aggregations of all public emails s = ( - Search(using=db, index=db.dbs.mbox) + Search(using=db, index=db.dbs.db_mbox) .query("match", private=False) .filter("range", date={"lt": "now+1d", "gt": "now-14d"}) ) @@ -137,7 +137,7 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict: ) res = await db.search( - index=db.dbs.mbox, body=s.to_dict(), size=0 + index=db.dbs.db_mbox, body=s.to_dict(), size=0 ) no_emails = res["hits"]["total"]["value"] @@ -153,12 +153,12 @@ async def get_public_activity(database: plugins.configuration.DBConfig) -> dict: thread_count = 0 s = ( - Search(using=db.client, index=db.dbs.mbox) + Search(using=db.client, index=db.dbs.db_mbox) .query("match", private=False) .filter("range", date={"lt": "now+1d", "gt": "now-14d"}) ) async for docs in db.scan( - index=db.dbs.mbox, + index=db.dbs.db_mbox, query=s.to_dict(), _source_includes=[ "message-id", diff --git a/server/plugins/database.py b/server/plugins/database.py index cf1f436..eccb18b 100644 --- a/server/plugins/database.py +++ b/server/plugins/database.py @@ -33,13 +33,13 @@ class Timeout (elasticsearch.exceptions.ConnectionTimeout): class DBNames: def __init__(self, dbprefix): - self.mbox = f"{dbprefix}-mbox" - self.source = f"{dbprefix}-source" - self.attachment = f"{dbprefix}-attachment" - self.account = f"{dbprefix}-account" - self.session = f"{dbprefix}-session" - self.notification = f"{dbprefix}-notification" - self.auditlog = f"{dbprefix}-auditlog" + self.db_mbox = f"{dbprefix}-mbox" + self.db_source = f"{dbprefix}-source" + self.db_attachment = f"{dbprefix}-attachment" + self.db_account = f"{dbprefix}-account" + self.db_session = f"{dbprefix}-session" + self.db_notification = f"{dbprefix}-notification" + self.db_auditlog = f"{dbprefix}-auditlog" DBError = elasticsearch.ElasticsearchException @@ -71,7 +71,7 @@ class Database: async def search(self, index="", **kwargs): if not index: - index = self.dbs.mbox + index = self.dbs.db_mbox try: res = await self.client.search(index=index, **kwargs) return res @@ -80,19 +80,19 @@ class Database: async def get(self, index="", **kwargs): if not index: - index = self.dbs.mbox + index = self.dbs.db_mbox res = await self.client.get(index=index, **kwargs) return res async def delete(self, index="", **kwargs): if not index: - index = self.dbs.session + index = self.dbs.db_session res = await self.client.delete(index=index, **kwargs) return res async def index(self, index="", **kwargs): if not index: - index = self.dbs.session + index = self.dbs.db_session res = await self.client.index(index=index, **kwargs) return res @@ -108,7 +108,7 @@ class Database: async def update(self, index="", **kwargs): if not index: - index = self.dbs.session + index = self.dbs.db_session res = await self.client.update(index=index, **kwargs) return res diff --git a/server/plugins/messages.py b/server/plugins/messages.py index 9c63458..42cb948 100644 --- a/server/plugins/messages.py +++ b/server/plugins/messages.py @@ -216,9 +216,9 @@ async def get_email( source=False, ): assert session.database, DATABASE_NOT_CONNECTED - doctype = session.database.dbs.mbox + doctype = session.database.dbs.db_mbox if source: - doctype = session.database.dbs.source + doctype = session.database.dbs.db_source # Older indexes may need a match instead of a strict terms agg in order to find # emails in DBs that may have been incorrectly analyzed. aggtype = "match" @@ -287,7 +287,7 @@ async def get_email( async def get_source(session: plugins.session.SessionObject, permalink: str = None, raw=False): assert session.database, DATABASE_NOT_CONNECTED - doctype = session.database.dbs.source + doctype = session.database.dbs.db_source try: doc = await session.database.get(index=doctype, id=permalink) except plugins.database.DBError: @@ -457,7 +457,7 @@ async def get_activity_span(session, query_defuzzed): fuzz_private_only = dict(query_defuzzed) fuzz_private_only["filter"] = [{"term": {"private": True}}] res = await session.database.search( - index=session.database.dbs.mbox, + index=session.database.dbs.db_mbox, size=0, body={ "query": {"bool": fuzz_private_only}, @@ -486,7 +486,7 @@ async def get_activity_span(session, query_defuzzed): # Get oldest and youngest doc in single scan, as well as a monthly histogram res = await session.database.search( - index=session.database.dbs.mbox, + index=session.database.dbs.db_mbox, size=0, body={"query": {"bool": query_defuzzed}, "aggs": { diff --git a/server/plugins/session.py b/server/plugins/session.py index 4b2cbc2..ed44730 100644 --- a/server/plugins/session.py +++ b/server/plugins/session.py @@ -143,14 +143,14 @@ async def get_session( if session_id and session.database: try: session_doc = await session.database.get( - session.database.dbs.session, id=session_id + session.database.dbs.db_session, id=session_id ) last_update = session_doc["_source"]["updated"] session.cookie = session_id # Check that this cookie ain't too old. If it is, delete it and return bare-bones session object if (now - last_update) > FOAL_MAX_SESSION_AGE: session.database.delete( - index=session.database.dbs.session, id=session_id + index=session.database.dbs.db_session, id=session_id ) return session @@ -158,7 +158,7 @@ async def get_session( cid = session_doc["_source"]["cid"] if cid: account_doc = await session.database.get( - session.database.dbs.account, id=cid + session.database.dbs.db_account, id=cid ) creds = account_doc["_source"]["credentials"] internal = account_doc["_source"]["internal"] @@ -212,7 +212,7 @@ async def save_session(session: SessionObject): """Save a session object in the ES database""" assert session.database, DATABASE_NOT_CONNECTED await session.database.index( - index=session.database.dbs.session, + index=session.database.dbs.db_session, id=session.cookie, body={ "cookie": session.cookie, @@ -225,7 +225,7 @@ async def save_session(session: SessionObject): async def remove_session(session: SessionObject): """Remove a session object in the ES database""" assert session.database, DATABASE_NOT_CONNECTED - await session.database.delete(index=session.database.dbs.session, id=session.cookie) + await session.database.delete(index=session.database.dbs.db_session, id=session.cookie) async def save_credentials(session: SessionObject): @@ -233,7 +233,7 @@ async def save_credentials(session: SessionObject): assert session.database, DATABASE_NOT_CONNECTED assert session.credentials, "Session object without credentials, cannot save!" await session.database.index( - index=session.database.dbs.account, + index=session.database.dbs.db_account, id=session.cid, body={ "cid": session.cid,
