This is an automated email from the ASF dual-hosted git repository. mobuchowski pushed a commit to branch openlineage-do-not-redact-proxy in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1663fd409e937badf1bbace407920bed58826e3b Author: Maciej Obuchowski <[email protected]> AuthorDate: Mon Aug 14 18:19:16 2023 +0200 openlineage: do not try to redact Proxy objects from deprecated config Signed-off-by: Maciej Obuchowski <[email protected]> --- airflow/providers/openlineage/utils/utils.py | 60 ++++++++++++++--------- tests/providers/openlineage/plugins/test_utils.py | 6 +++ 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py index 20b9afef49..3ff7610c4f 100644 --- a/airflow/providers/openlineage/utils/utils.py +++ b/airflow/providers/openlineage/utils/utils.py @@ -38,6 +38,7 @@ from airflow.providers.openlineage.plugins.facets import ( AirflowMappedTaskRunFacet, AirflowRunFacet, ) +from airflow.utils.context import AirflowContextDeprecationWarning from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key if TYPE_CHECKING: @@ -345,29 +346,40 @@ class OpenLineageRedactor(SecretsMasker): if depth > max_depth: return item try: - if name and should_hide_value_for_key(name): - return self._redact_all(item, depth, max_depth) - if attrs.has(type(item)): - # TODO: fixme when mypy gets compatible with new attrs - for dict_key, subval in attrs.asdict(item, recurse=False).items(): # type: ignore[arg-type] - if _is_name_redactable(dict_key, item): - setattr( - item, - dict_key, - self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth), - ) - return item - elif is_json_serializable(item) and hasattr(item, "__dict__"): - for dict_key, subval in item.__dict__.items(): - if _is_name_redactable(dict_key, item): - setattr( - item, - dict_key, - self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth), - ) - return item - else: - return super()._redact(item, name, depth, max_depth) + # It's impossible to check the type of variable in a dict without accessing it, and + # this already causes warning - so suppress it + with suppress(AirflowContextDeprecationWarning): + if type(item).__name__ == "Proxy": + # Those are deprecated values in _DEPRECATION_REPLACEMENTS + # in airflow.utils.context.Context + return "<<non-redactable: Proxy>>" + if name and should_hide_value_for_key(name): + return self._redact_all(item, depth, max_depth) + if attrs.has(type(item)): + # TODO: fixme when mypy gets compatible with new attrs + for dict_key, subval in attrs.asdict( + item, recurse=False # type: ignore[arg-type] + ).items(): + if _is_name_redactable(dict_key, item): + setattr( + item, + dict_key, + self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth), + ) + return item + elif is_json_serializable(item) and hasattr(item, "__dict__"): + for dict_key, subval in item.__dict__.items(): + if type(subval).__name__ == "Proxy": + return "<<non-redactable: Proxy>>" + if _is_name_redactable(dict_key, item): + setattr( + item, + dict_key, + self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth), + ) + return item + else: + return super()._redact(item, name, depth, max_depth) except Exception as e: log.warning( "Unable to redact %s. Error was: %s: %s", @@ -375,7 +387,7 @@ class OpenLineageRedactor(SecretsMasker): type(e).__name__, str(e), ) - return item + return item def is_json_serializable(item): diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index afbc40ee4c..59abb43e09 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -171,6 +171,9 @@ def test_redact_with_exclusions(monkeypatch): def __init__(self): self.password = "passwd" + class Proxy: + pass + def default(self, o): if isinstance(o, NotMixin): return o.__dict__ @@ -180,6 +183,9 @@ def test_redact_with_exclusions(monkeypatch): monkeypatch.setattr(JSONEncoder, "default", default) assert redactor.redact(NotMixin()).password == "***" + assert redactor.redact(Proxy()) == "<<non-redactable: Proxy>>" + assert redactor.redact({"a": "a", "b": Proxy()}) == {"a": "a", "b": "<<non-redactable: Proxy>>"} + class Mixined(RedactMixin): _skip_redact = ["password"]
