mikebridge commented on code in PR #41549: URL: https://github.com/apache/superset/pull/41549#discussion_r3574282556
########## superset/commands/deletion_retention/audit.py: ########## @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Write-ahead purge audit record. + +Every purge — time-based or force — writes an immutable record that +**survives** the entity it names, on a **dedicated session** outside the +purge transaction so it neither entangles with the ``DBEventLogger`` +(which shares ``db.session`` and commits mid-request) nor vanishes if the +purge rolls back. The record is written ``pending`` *before* the purge and +flipped to ``confirmed`` *after* it commits, so a crash leaves at most a +``pending`` row, never a missing one. ``pending`` rows are reconciled on the +next run (the purge is convergent). + +The dedicated ``purge_audit_log`` table is content-free (no name or PII; only +action, actor, UTC time, entity type, UUID, and affected referrers) and is never +removed by the purge cascade. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta +from typing import Any, cast + +import sqlalchemy as sa +from sqlalchemy import Column, DateTime, Integer, String, Text +from sqlalchemy.orm import Session, sessionmaker + +from superset import db + +logger = logging.getLogger(__name__) + + +def _dedicated_session() -> Session: + """A fresh session on its own connection, independent of the request / + task ``db.session``. The audit write must commit on its own so it survives + a rolled-back or crashed purge.""" + return sessionmaker(bind=db.engine)() + + +STATUS_PENDING = "pending" +STATUS_CONFIRMED = "confirmed" +STATUS_FAILED = "failed" +STATUS_BLOCKED = "blocked" + +_PENDING_STALE_AFTER = timedelta(hours=1) + +TRIGGER_RETENTION = "retention" +TRIGGER_FORCE = "force" + +ACTOR_SYSTEM = "system" + + +class PurgeAuditLog(db.Model): + """Immutable, content-free record of a purge.""" + + __tablename__ = "purge_audit_log" + + id = Column(Integer, primary_key=True) Review Comment: Addressed in 5b4b2460fc: PurgeAuditLog now uses a UUIDType primary key with uuid4 as its default. ########## superset/commands/deletion_retention/force_purge.py: ########## @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Compliance force-purge of a single entity by UUID. + +Immediate, irreversible removal of one entity regardless of the retention +window or whether it is currently soft-deleted or live. Runs the same cascade +as the time-based task with ``enforce_window=False`` — identical dependent +handling with legacy hard-delete semantics: M:N join rows hard-deleted, +a referencing live chart's loose ``datasource_id`` left dangling (the chart is +never modified). Idempotent: a UUID that resolves to nothing is a no-op. +""" + +from __future__ import annotations + +import logging +from typing import Any, cast + +from superset import db +from superset.commands.deletion_retention import audit +from superset.commands.deletion_retention.purge_cascade import ( + cascade_hard_delete, + CascadeResult, + dashboard_slice_count, + suspend_version_capture, +) +from superset.models.helpers import skip_visibility_filter, SoftDeleteMixin + +logger = logging.getLogger(__name__) Review Comment: Addressed in 5b4b2460fc: the module logger now has an explicit logging.Logger annotation. ########## superset/commands/deletion_retention/force_purge.py: ########## @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Compliance force-purge of a single entity by UUID. + +Immediate, irreversible removal of one entity regardless of the retention +window or whether it is currently soft-deleted or live. Runs the same cascade +as the time-based task with ``enforce_window=False`` — identical dependent +handling with legacy hard-delete semantics: M:N join rows hard-deleted, +a referencing live chart's loose ``datasource_id`` left dangling (the chart is +never modified). Idempotent: a UUID that resolves to nothing is a no-op. +""" + +from __future__ import annotations + +import logging +from typing import Any, cast + +from superset import db +from superset.commands.deletion_retention import audit +from superset.commands.deletion_retention.purge_cascade import ( + cascade_hard_delete, + CascadeResult, + dashboard_slice_count, + suspend_version_capture, +) +from superset.models.helpers import skip_visibility_filter, SoftDeleteMixin + +logger = logging.getLogger(__name__) + + +class ForcePurgeCommand: + """Force-purge the entity identified by *uuid*, if any.""" + + def __init__(self, uuid: str, actor: str = "operator") -> None: + self._uuid = uuid + self._actor = actor Review Comment: Addressed in 5b4b2460fc: _uuid and _actor now have explicit str annotations. ########## superset/commands/deletion_retention/window.py: ########## @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Resolve the soft-delete retention window.""" + +from __future__ import annotations + +import logging + +from flask import current_app + +from superset.key_value.shared_entries import get_shared_value +from superset.key_value.types import SharedKey + +logger = logging.getLogger(__name__) + +_DEFAULT_RETENTION_DAYS = 30 Review Comment: Addressed in 5b4b2460fc: logger and _DEFAULT_RETENTION_DAYS now have explicit annotations. ########## tests/integration_tests/deletion_retention/_base.py: ########## @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared base + self-contained builders for deletion-retention tests. + +These tests do not depend on the example datasets — each builds its own +``Database`` + ``SqlaTable`` so they run on a bare (schema-only) test DB. +Everything created is torn down (bypassing the soft-delete visibility +filter) so a leftover soft-deleted row never trips a later test. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Any + +import sqlalchemy as sa + +from superset import db +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn +from superset.constants import SKIP_VISIBILITY_FILTER_CLASSES +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from tests.integration_tests.base_tests import SupersetTestCase + +_PREFIX = "retention_it_" + + +def _bypass(model: type[Any]) -> dict[str, Any]: + return {"execution_options": {SKIP_VISIBILITY_FILTER_CLASSES: {model}}} + + +class DeletionRetentionTestBase(SupersetTestCase): + """Builds an isolated database + dataset and cleans up after itself.""" + + def setUp(self) -> None: + super().setUp() + self._cleanup() + self.database = Database( + database_name=f"{_PREFIX}db", sqlalchemy_uri="sqlite://" + ) Review Comment: Addressed in 5b4b2460fc: self.database now has an explicit Database annotation. ########## tests/integration_tests/deletion_retention/_base.py: ########## @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared base + self-contained builders for deletion-retention tests. + +These tests do not depend on the example datasets — each builds its own +``Database`` + ``SqlaTable`` so they run on a bare (schema-only) test DB. +Everything created is torn down (bypassing the soft-delete visibility +filter) so a leftover soft-deleted row never trips a later test. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Any + +import sqlalchemy as sa + +from superset import db +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn +from superset.constants import SKIP_VISIBILITY_FILTER_CLASSES +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from tests.integration_tests.base_tests import SupersetTestCase + +_PREFIX = "retention_it_" + + +def _bypass(model: type[Any]) -> dict[str, Any]: + return {"execution_options": {SKIP_VISIBILITY_FILTER_CLASSES: {model}}} + + +class DeletionRetentionTestBase(SupersetTestCase): + """Builds an isolated database + dataset and cleans up after itself.""" + + def setUp(self) -> None: + super().setUp() + self._cleanup() + self.database = Database( + database_name=f"{_PREFIX}db", sqlalchemy_uri="sqlite://" + ) + db.session.add(self.database) + db.session.commit() + self.dataset = self.make_dataset("ds") Review Comment: Addressed in 5b4b2460fc: self.dataset now has an explicit SqlaTable annotation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
