potiuk commented on a change in pull request #19737: URL: https://github.com/apache/airflow/pull/19737#discussion_r754323580
########## File path: docker_tests/docker_tests_utils.py ########## @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import subprocess +from pathlib import Path +from typing import List + +docker_image = os.environ.get('DOCKER_IMAGE') +SOURCE_ROOT = Path(__file__).resolve().parents[1] + +if not docker_image: + raise Exception("The DOCKER_IMAGE environment variable is required") + + +def run_command(cmd: List[str], print_output_on_error: bool = True, **kwargs): + print(f"$ {' '.join(c for c in cmd)}") Review comment: I like this as this will give an easy way to copy&paste the command to run it manually ########## File path: docker_tests/ci_image.py ########## @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess +import unittest + +from docker_tests.docker_tests_utils import ( + display_dependency_conflict_message, + docker_image, + run_bash, + run_command, +) + + +class TestFiles(unittest.TestCase): Review comment: Thos could be ptyest tests (functions) indeed (followiung @uranusjr comments). No need to have classes. ########## File path: docker_tests/prod_image.py ########## @@ -0,0 +1,202 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import subprocess +import tempfile +from pathlib import Path + +import pytest + +from docker_tests.docker_tests_utils import ( + SOURCE_ROOT, + display_dependency_conflict_message, + docker_image, + run_bash, + run_command, + run_python, +) + + +class TestCommands: + def test_without_command(self): + """Checking the image without a command. It should return non-zero exit code.""" + with pytest.raises(subprocess.CalledProcessError) as ctx: + run_command(["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image]) + assert 2 == ctx.value.returncode + + def test_airflow_command(self): + """Checking 'airflow' command It should return non-zero exit code.""" + with pytest.raises(subprocess.CalledProcessError) as ctx: + run_command(["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image, "airflow"]) + assert 2 == ctx.value.returncode + + def test_airflow_version(self): + """Checking 'airflow version' command It should return zero exit code.""" + output = run_command( + ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image, "airflow", "version"] + ) + assert "2." in output + + def test_python_version(self): + """Checking 'python --version' command It should return zero exit code.""" + output = run_command( + ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image, "python", "--version"] + ) + assert "Python 3." in output + + def test_bash_version(self): + """Checking 'bash --version' command It should return zero exit code.""" + output = run_command( + ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image, "bash", "--version"] + ) + assert "GNU bash," in output + + +class TestPythonPackages: + def test_required_providers_are_installed(self): + lines = ( + d.strip() + for d in (SOURCE_ROOT / "scripts" / "ci" / "installed_providers.txt").read_text().splitlines() + ) + lines = (d for d in lines) + packages_to_install = {f"apache-airflow-providers-{d.replace('.', '-')}" for d in lines} + assert len(packages_to_install) != 0 + + output = run_bash("airflow providers list --output json", stderr=subprocess.DEVNULL) + providers = json.loads(output) + packages_installed = {d['package_name'] for d in providers} + assert len(packages_installed) != 0 + + assert packages_to_install == packages_installed Review comment: We should add message on what to do in case they are different I think (add missing packages to installed_providers.txt or smth) ########## File path: docker_tests/docker_tests_utils.py ########## @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import subprocess +from pathlib import Path +from typing import List + +docker_image = os.environ.get('DOCKER_IMAGE') Review comment: There is no need to use env variable, I think for python tool like that it would be much bettter to use `click` and pass the image as command line option. ########## File path: scripts/ci/images/ci_run_docker_tests.sh ########## @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +interactive="false" +initialize_only="false" +declare -a tests_to_run +declare -a pytest_args + +tests_to_run=() + +function parse_tests_to_run() { + if [[ $# != 0 ]]; then + if [[ $1 == "--help" || $1 == "-h" ]]; then + echo + echo "Running Docker tests" + echo + echo " $0 TEST [TEST ...] - runs tests (from docker_tests folder)" + echo " $0 [-i|--interactive] - Activates virtual environment ready to run tests and drops you in" + echo " $0 [--initialize] - Initialize virtual environment and exit" + echo " $0 [--help] - Prints this help message" + echo + exit + elif [[ $1 == "--interactive" || $1 == "-i" ]]; then + echo + echo "Entering interactive environment for docker testing" + echo + interactive="true" + elif [[ $1 == "--initialize" ]]; then + echo + echo "Initializing environment for docker testing" + echo + initialize_only="true" + else + tests_to_run=("${@}") + fi + pytest_args=( + "--pythonwarnings=ignore::DeprecationWarning" + "--pythonwarnings=ignore::PendingDeprecationWarning" + "-n" "auto" + ) + else + echo "You must select the tests to run." + exit 1 + fi +} + +function create_virtualenv() { + HOST_PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') + readonly HOST_PYTHON_VERSION + + local virtualenv_path="${BUILD_CACHE_DIR}/.docker_venv/host_python_${HOST_PYTHON_VERSION}" + + mkdir -pv "${BUILD_CACHE_DIR}/.docker_venv/" + if [[ ! -d ${virtualenv_path} ]]; then + echo + echo "Creating virtualenv at ${virtualenv_path}" + echo + python3 -m venv "${virtualenv_path}" + fi + + . "${virtualenv_path}/bin/activate" + + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + + local constraints=( + --constraint + "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${HOST_PYTHON_VERSION}.txt"; + ) + if [[ -n ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} ]]; then + # Disable constraints when building in CI with specific version of sources + # In case there will be conflicting constraints + constraints=() + fi + + pip install pytest pytest-xdist "${constraints[@]}" +} + +function run_tests() { + pytest "${pytest_args[@]}" "${tests_to_run[@]}" +} + +cd "${AIRFLOW_SOURCES}" || exit 1 + +set +u +parse_tests_to_run "${@}" +set -u + +create_virtualenv + +if [[ ${interactive} == "true" ]]; then + echo + echo "Activating the virtual environment for docker testing" + echo + echo "You can run testing via 'pytest docker tests/....'" Review comment: ```suggestion echo "You can run testing via 'pytest docker_tests/....'" ``` ########## File path: scripts/ci/images/ci_run_docker_tests.sh ########## @@ -0,0 +1,121 @@ +#!/usr/bin/env bash Review comment: I think if we are getting ride of bash we shoudl get rid of it entirely. I think this could be very easily (and better) written as pure-stdib code that will prepare and activate the env with dependencies stored as `docker_test/requirements.txt' . In our case it would be: `pytest, pytest-xdist, click` (the last one if we convert the script above to use click). This could be an 'entrypoint' to run tests - both from CI (installing such a small set of requirements can be done in CI and even cached easily) as well for a "fresh" user who does not have the venv installed - adding pytest-xdist automatically to such venv and running tests with it is good reason to have such auto-installation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
