amoghrajesh commented on code in PR #76575: URL: https://github.com/apache/airflow/pull/76575#discussion_r4546564398
########## dev/registry/fill_durable_versions.py: ########## @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Date each durable module in modules.json against the provider tags that first shipped it. + +Runs on the host rather than in Breeze with the rest of the module extraction, because it reads +the ``providers-<id>/<version>`` git tags and Breeze does not mount ``.git``. + +Usage: + python dev/registry/fill_durable_versions.py --modules dev/registry/modules.json +""" + +from __future__ import annotations + +import argparse +import functools +import json +import subprocess +import sys +from pathlib import Path + +from packaging.version import InvalidVersion, Version + +AIRFLOW_ROOT = Path(__file__).parent.parent.parent + +# Patterns matching the mechanism, never its registry declaration, and earliest match wins. Review Comment: Swapped. ########## dev/registry/tests/test_fill_durable_versions.py: ########## @@ -0,0 +1,281 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import json +import subprocess +from unittest import mock + +import pytest +from fill_durable_versions import ( + DURABLE_TOKENS, + GitCommandError, + build_source_path, + durable_group, + find_first_releases, + get_released_tags, + main, + run_git, +) + +POD_PATH = "providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py" +POD_CLASS = "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator" + + [email protected](autouse=True) +def clear_tag_cache(): + get_released_tags.cache_clear() + yield + get_released_tags.cache_clear() + + +def fake_git(tags=(), matches=(), token=None, paths=(), allow_grep=True): + """Stand in for run_git. The grep argv is checked, since without `-lE` git prints + `tag:path:line` and nothing gets dated.""" + + def run(*args, **kwargs): + if args[0] == "tag": + return list(tags) + if args[0] == "grep": + if not allow_grep: + raise AssertionError("grep ran even though there were no tags to search") + assert args[1] == "-lE", f"grep must list file names only, got {args[1]!r}" + if token is not None: + assert args[2] == token + assert list(args[3 : 3 + len(tags)]) == list(tags) + assert args[3 + len(tags)] == "--" + if paths: + assert list(args[4 + len(tags) :]) == sorted(paths) + assert kwargs["allowed_returncodes"] == (0, 1) + return list(matches) + raise AssertionError(f"unexpected git call: {args}") + + return run + + +class TestRunGit: + @mock.patch("fill_durable_versions.subprocess.run", autospec=True) + def test_unexpected_exit_code_is_raised_with_stderr(self, run): + run.return_value = subprocess.CompletedProcess( + args=[], returncode=128, stdout="", stderr="fatal: not a git repository" + ) + with pytest.raises(GitCommandError, match="fatal: not a git repository"): + run_git("tag", "--list") + + @mock.patch("fill_durable_versions.subprocess.run", autospec=True) + def test_allowed_exit_code_is_an_empty_answer(self, run): + run.return_value = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="") + assert run_git("grep", "-lE", "x", allowed_returncodes=(0, 1)) == [] + + +class TestBuildSourcePath: + @pytest.mark.parametrize( + ("provider_id", "qualified_class", "expected"), + [ + ("cncf-kubernetes", POD_CLASS, POD_PATH), + ( + "amazon", + "airflow.providers.amazon.aws.operators.glue.GlueJobOperator", + "providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py", + ), + ], + ) + def test_multi_part_provider_id_becomes_a_directory(self, provider_id, qualified_class, expected): + assert build_source_path(provider_id, qualified_class) == expected + + +class TestDurableGroup: + def test_marker_and_mixin_get_different_tokens(self): + marker = durable_group( + { + "durable_mechanism": "marker", + "durable_gating_provider": "cncf-kubernetes", + "durable_gating_class": POD_CLASS, + } + ) + mixin = durable_group( + { + "durable_mechanism": "resumable_mixin", + "durable_gating_provider": "apache-livy", + "durable_gating_class": "airflow.providers.apache.livy.operators.livy.LivyOperator", + } + ) + assert marker == ("cncf-kubernetes", DURABLE_TOKENS["marker"], POD_PATH) + assert mixin[1] == "execute_resumable" + + def test_marker_token_also_catches_the_durable_parameter(self): Review Comment: Now runs `re.search` on real lines: AgentOperator 0.1.0's `durable: bool = False,` and KPO 10.20.0's `task_state_store` line match, `deferrable: bool = False,` doesn't. Fails with the old token. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
