This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8878998cf6081e45e4af6954d3f8294d78642285 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Oct 11 16:40:31 2020 +0200 Split tests to more sub-types (#11402) We seem to have a problem with running all tests at once - most likely due to some resource problems in our CI, therefore it makes sense to split the tests into more batches. This is not yet full implementation of selective tests but it is going in this direction by splitting to Core/Providers/API/CLI tests. The full selective tests approach will be implemented as part of #10507 issue. This split is possible thanks to #10422 which moved building image to a separate workflow - this way each image is only built once and it is uploaded to a shared registry, where it is quickly downloaded from rather than built by all the jobs separately - this way we can have many more jobs as there is very little per-job overhead before the tests start runnning. (cherry picked from commit 5bc5994c2c1a8f73a644e29e98d3d132c6097ab2) --- .github/workflows/ci.yml | 27 +++- BREEZE.rst | 17 ++- TESTING.rst | 57 +++++++- breeze | 47 ++++++- breeze-complete | 12 +- scripts/ci/docker-compose/base.yml | 7 +- scripts/ci/libraries/_docker.env | 4 + scripts/ci/libraries/_initialization.sh | 42 +++--- scripts/ci/libraries/_parameters.sh | 28 ++-- scripts/ci/libraries/_push_pull_remove_images.sh | 14 ++ scripts/ci/testing/ci_run_airflow_testing.sh | 11 +- scripts/in_container/entrypoint_ci.sh | 151 ++++++++++++++++----- scripts/in_container/run_ci_tests.sh | 59 ++++++-- tests/bats/test_breeze_complete.bats | 8 ++ tests/{ => core}/test_config_templates.py | 0 tests/{ => core}/test_configuration.py | 0 tests/{ => core}/test_core.py | 0 tests/{ => core}/test_example_dags_system.py | 0 .../test_impersonation_tests.py} | 0 tests/{ => core}/test_logging_config.py | 0 tests/{ => core}/test_sentry.py | 0 tests/{ => core}/test_sqlalchemy_config.py | 3 +- tests/jobs/test_local_task_job.py | 2 +- .../task/task_runner/test_standard_task_runner.py | 6 +- 24 files changed, 390 insertions(+), 105 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e14f56..8c42d33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}" GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - TEST_TYPES: '["Core", "Integration", "Heisentests"]' + TEST_TYPES: '["Core", "Providers", "API", "CLI", "Integration", "Other", "WWW", "Heisentests"]' # You can switch between building the image in "Build Images" workflow or building them in CI workflow # Separately for each job. @@ -79,7 +79,7 @@ jobs: postgresVersions: ${{ steps.versions.outputs.postgres-versions }} defaultPostgresVersion: ${{ steps.versions.outputs.default-postgres-version }} mysqlVersions: ${{ steps.versions.outputs.mysql-versions }} - defaultMysqlVersion: ${{ steps.versions.outputs.default-mysql-version }} + defaultMySQLVersion: ${{ steps.versions.outputs.default-mysql-version }} helmVersions: ${{ steps.versions.outputs.helm-versions }} defaultHelmVersion: ${{ steps.versions.outputs.default-helm-version }} kindVersions: ${{ steps.versions.outputs.kind-versions }} @@ -274,6 +274,12 @@ jobs: postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }} exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} + # Additionally include all postgres-only tests but only for default versions + # in case the regular tests are excluded in the future + include: + - python_version: ${{ needs.build-info.outputs.defaultPythonVersion }} + postgres-version: ${{ needs.build-info.outputs.defaultPostgresVersion }} + test-type: "Postgres" fail-fast: false env: BACKEND: postgres @@ -321,6 +327,12 @@ jobs: mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }} exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} + # Additionally include all mysql-only tests but only for default versions + # in case the regular tests are excluded in the future + include: + - python_version: ${{ needs.build-info.outputs.defaultPythonVersion }} + postgres-version: ${{ needs.build-info.outputs.defaultMySQLVersion }} + test-type: "MySQL" fail-fast: false env: BACKEND: mysql @@ -405,9 +417,16 @@ jobs: runs-on: ubuntu-latest continue-on-error: true needs: [build-info, trigger-tests, ci-images] + strategy: + matrix: + include: + - backend: mysql + - backend: postgres + - backend: sqlite env: - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - BACKEND: postgres + BACKEND: ${{ matrix.backend }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} RUN_TESTS: true TEST_TYPE: Quarantined diff --git a/BREEZE.rst b/BREEZE.rst index 5409792..3f3aca1 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1862,7 +1862,12 @@ This is the current syntax for `./breeze <./breeze>`_: Flags: - Run 'breeze flags' to see all applicable flags. + --test-type TEST_TYPE + Type of the test to run. One of: + + All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL + + Default: All #################################################################################################### @@ -2208,6 +2213,16 @@ This is the current syntax for `./breeze <./breeze>`_: Default: latest. **************************************************************************************************** + Flags for running tests + + --test-type TEST_TYPE + Type of the test to run. One of: + + All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL + + Default: All + + **************************************************************************************************** Increase verbosity of the scripts -v, --verbose diff --git a/TESTING.rst b/TESTING.rst index fd8a9f7..f6b9964 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -131,23 +131,70 @@ Running Tests for a Specified Target Using Breeze from the Host --------------------------------------------------------------- If you wish to only run tests and not to drop into shell, apply the -``tests`` command. You can add extra targets and pytest flags after the ``tests`` command. +``tests`` command. You can add extra targets and pytest flags after the ``--`` command. Note that +often you want to run the tests with a clean/reset db, so usually you want to add ``--db-reset`` flag +to breeze. .. code-block:: bash - ./breeze tests tests/hooks/test_druid_hook.py tests/tests_core.py --logging-level=DEBUG + ./breeze tests tests/hooks/test_druid_hook.py tests/tests_core.py --db-reset -- --logging-level=DEBUG -You can run the whole test suite with a 'tests' test target: +You can run the whole test suite without adding the test target: .. code-block:: bash - ./breeze tests tests + ./breeze tests --db-reset You can also specify individual tests or a group of tests: .. code-block:: bash - ./breeze tests tests/test_core.py::TestCore + ./breeze tests --db-reset tests/test_core.py::TestCore + + +Running Tests of a specified type from the Host +----------------------------------------------- + +You can also run tests for a specific test type. For the stability and performance point of view +we separated tests to different test types so that they can be run separately. + +You can select the test type by adding ``--test-type TEST_TYPE`` before the test command. There are two +kinds of test types: + +* Per-directories types are added to select subset of the tests based on sub-directories in ``tests`` folder. + Example test types there - Core, Providers, CLI. The only action that happens when you choose the right + test folders are pre-selected. For those types of tests it is only useful to choose the test type + when you do not specify test to run. + +Runs all core tests: + +.. code-block:: bash + + ./breeze --test-type Core --db-reset tests + +Runs all provider tests: + +.. code-block:: bash + + ./breeze --test-type Providers --db-reset tests + +* Special kinds of tests - Integration, Heisentests, Quarantined, Postgres, MySQL which are marked with pytest + marks and for those you need to select the type using test-type switch. If you want to run such tests + using breeze, you need to pass appropriate ``--test-type`` otherwise the test will be skipped. + Similarly to the per-directory tests if you do not specify the test or tests to run, + all tests of a given type are run + +Run quarantined test_task_command.py test: + +.. code-block:: bash + + ./breeze --test-type Quarantined tests tests/cli/commands/test_task_command.py --db-reset + +Run all Quarantined tests: + +.. code-block:: bash + + ./breeze --test-type Quarantined tests --db-reset Airflow Integration Tests diff --git a/breeze b/breeze index e7a2ca8..b4e0996 100755 --- a/breeze +++ b/breeze @@ -120,7 +120,7 @@ function breeze::setup_default_breeze_constants() { # If set to true, the test connections will be created export LOAD_DEFAULT_CONNECTIONS="false" - # If set to true, the sample dags will be created + # If set to true, the sample dags will be used export LOAD_EXAMPLES="false" # If set to true, RBAC UI will not be used for 1.10 version @@ -1073,6 +1073,7 @@ function breeze::parse_arguments() { export SKIP_BUILDING_PROD_IMAGE="true" export MOUNT_LOCAL_SOURCES="false" export SKIP_CHECK_REMOTE_IMAGE="true" + export FAIL_ON_GITHUB_DOCKER_PULL_ERROR="true" shift 2 ;; --init-script) @@ -1105,6 +1106,12 @@ function breeze::parse_arguments() { echo shift ;; + --test-type) + export TEST_TYPE="${2}" + echo "Selected test type: ${TEST_TYPE}" + echo + shift 2 + ;; --) shift break @@ -1255,12 +1262,8 @@ function breeze::parse_arguments() { ;; tests) last_subcommand="${1}" - if [[ $# -lt 2 ]]; then - run_help="true" - else - shift - fi command_to_run="run_tests" + shift ;; toggle-suppress-cheatsheet) last_subcommand="${1}" @@ -1404,6 +1407,11 @@ function breeze::prepare_formatted_versions() { FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" | tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/") readonly FORMATTED_DEFAULT_PROD_EXTRAS + + FORMATTED_TEST_TYPES=$(echo "${_breeze_allowed_test_types=""}" | + tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/") + readonly FORMATTED_TEST_TYPES + } ####################################################################################################### @@ -1737,7 +1745,7 @@ ${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>] '${CMDNAME} tests tests Flags: -$(breeze::flag_footer) +$(breeze::flag_tests) " readonly DETAILED_USAGE_TESTS export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET=" @@ -2359,6 +2367,25 @@ function breeze::flag_start_airflow() { " } +##################################################################################################### +# +# Prints flags that control tests +# +# Outputs: +# Flag information. +####################################################################################################### +function breeze::flag_tests() { + echo " +--test-type TEST_TYPE + Type of the test to run. One of: + +${FORMATTED_TEST_TYPES} + + Default: ${_breeze_default_test_type:=} + +" +} + ####################################################################################################### # # Prints all flags @@ -2421,6 +2448,10 @@ $(breeze::print_star_line) $(breeze::flag_pull_push_docker_images) $(breeze::print_star_line) + Flags for running tests +$(breeze::flag_tests) + +$(breeze::print_star_line) Increase verbosity of the scripts $(breeze::flag_verbosity) @@ -2561,6 +2592,8 @@ function breeze::check_and_save_all_params() { parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version" parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version" + parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type" + # Can't verify those - they can be anything, so let's just save them parameters::save_to_file DOCKERHUB_USER parameters::save_to_file DOCKERHUB_REPO diff --git a/breeze-complete b/breeze-complete index 2c57a79..9449f5e 100644 --- a/breeze-complete +++ b/breeze-complete @@ -33,6 +33,7 @@ _breeze_allowed_kind_versions="v0.8.0" _breeze_allowed_mysql_versions="5.6 5.7" _breeze_allowed_postgres_versions="9.6 10" _breeze_allowed_kind_operations="start stop restart status deploy test shell" +_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL" # shellcheck disable=SC2034 { @@ -44,6 +45,7 @@ _breeze_allowed_kind_operations="start stop restart status deploy test shell" _breeze_default_kind_version=$(echo "${_breeze_allowed_kind_versions}" | awk '{print $1}') _breeze_default_postgres_version=$(echo "${_breeze_allowed_postgres_versions}" | awk '{print $1}') _breeze_default_mysql_version=$(echo "${_breeze_allowed_mysql_versions}" | awk '{print $1}') + _breeze_default_test_type=$(echo "${_breeze_allowed_test_types}" | awk '{print $1}') } _breeze_allowed_install_airflow_versions=$(cat <<-EOF @@ -141,6 +143,7 @@ dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-comma runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags install-wheels no-rbac-ui +test-type: " _breeze_commands=" @@ -163,7 +166,8 @@ _breeze_extra_arg_commands=" docker-compose kind-cluster static-check -tests" +tests +" _breeze_help_commands=" flags @@ -248,6 +252,12 @@ function breeze_complete::get_known_values_breeze() { kind-cluster) _breeze_known_values="${_breeze_allowed_kind_operations}" ;; + tests) + _breeze_known_values="$(find tests -name '*.py')" + ;; + --test-type) + _breeze_known_values="${_breeze_allowed_test_types}" + ;; *) _breeze_known_values="" ;; diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index f89aefd..8a0781e 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -39,9 +39,7 @@ services: - ENABLE_KIND_CLUSTER - ENABLED_INTEGRATIONS - RUN_INTEGRATION_TESTS - - ONLY_RUN_LONG_RUNNING_TESTS - - ONLY_RUN_QUARANTINED_TESTS - - ONLY_RUN_HEISEN_TESTS + - TEST_TYPE - GITHUB_TOKEN - GITHUB_REPOSITORY - ISSUE_ID @@ -64,6 +62,9 @@ services: - HOST_OS - PYTHONDONTWRITEBYTECODE - INIT_SCRIPT_FILE + - GITHUB_REGISTRY_PULL_IMAGE_TAG + - POSTGRES_VERSION + - MYSQL_VERSION volumes: # Pass docker to inside of the container so that Kind and Moto tests can use it. - /var/run/docker.sock:/var/run/docker.sock diff --git a/scripts/ci/libraries/_docker.env b/scripts/ci/libraries/_docker.env index 986b7b4..86461d0 100644 --- a/scripts/ci/libraries/_docker.env +++ b/scripts/ci/libraries/_docker.env @@ -24,9 +24,13 @@ HOST_OS HOST_HOME HOST_AIRFLOW_SOURCES PYTHON_MAJOR_MINOR_VERSION +BACKEND VERSION_SUFFIX_FOR_PYPI VERSION_SUFFIX_FOR_SVN PRINT_INFO_FROM_SCRIPTS CI LOAD_DEFAULT_CONNECTIONS LOAD_EXAMPLES +GITHUB_REGISTRY_PULL_IMAGE_TAG +POSTGRES_VERSION +MYSQL_VERSION diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 576e80c..dd40a76 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -34,6 +34,12 @@ function initialization::create_directories() { export FILES_DIR="${AIRFLOW_SOURCES}/files" readonly FILES_DIR + # Create an empty .pypirc file that you can customise. It is .gitignored so it will never + # land in the repository - it is only added to the "build image" of production image + # So you can keep your credentials safe as long as you do not push the build image. + # The final image does not contain it. + touch "${AIRFLOW_SOURCES}/.pypirc" + # Directory where all the build cache is stored - we keep there status of all the docker images # As well as hashes of the important files, but also we generate build scripts there that are # Used to execute the commands for breeze @@ -95,7 +101,7 @@ function initialization::initialize_base_variables() { CURRENT_POSTGRES_VERSIONS+=("9.6" "10") export CURRENT_POSTGRES_VERSIONS - # Currently supported versions of MySQL + # Currently supported versions of MySQL CURRENT_MYSQL_VERSIONS+=("5.6" "5.7") export CURRENT_MYSQL_VERSIONS @@ -111,8 +117,10 @@ function initialization::initialize_base_variables() { # If set to true, the database will be initialized, a user created and webserver and scheduler started export START_AIRFLOW=${START_AIRFLOW:="false"} + # If set to true, the sample dags will be used export LOAD_EXAMPLES=${LOAD_EXAMPLES:="false"} + # If set to true, the test connections will be created export LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS:="false"} # If set to true, RBAC UI will not be used for 1.10 version @@ -126,9 +134,6 @@ function initialization::initialize_base_variables() { # otherwise it will use files/airflow-breeze-config/init.sh export INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE:=""} - # If set to true, RBAC mode is enabled - export RBAC_UI=${RBAC_UI:="false"} - # Read airflow version from the version.py AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g") export AIRFLOW_VERSION @@ -170,7 +175,6 @@ function initialization::initialize_available_integrations() { export AVAILABLE_INTEGRATIONS="cassandra kerberos mongo openldap presto rabbitmq redis" } - # Needs to be declared outside of function for MacOS FILES_FOR_REBUILD_CHECK=() @@ -188,7 +192,6 @@ function initialization::initialize_files_for_rebuild_check() { ) } - # Needs to be declared outside of function for MacOS # extra flags passed to docker run for PROD image @@ -257,6 +260,8 @@ function initialization::initialize_force_variables() { # Can be set to true to skip if the image is newer in registry export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"} + # Should be set to true if you expect image frm GitHub to be present and downloaded + export FAIL_ON_GITHUB_DOCKER_PULL_ERROR=${FAIL_ON_GITHUB_DOCKER_PULL_ERROR:="false"} } # Determine information about the host @@ -425,11 +430,9 @@ function initialization::initialize_git_variables() { function initialization::initialize_github_variables() { # Defaults for interacting with GitHub - export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} - GITHUB_REPOSITORY_LOWERCASE="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" - export GITHUB_REPOSITORY_LOWERCASE - export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"} export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"} + + export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"} export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"} export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"} export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"} @@ -439,8 +442,10 @@ function initialization::initialize_github_variables() { # Used only in CI environment export GITHUB_TOKEN="${GITHUB_TOKEN=""}" export GITHUB_USERNAME="${GITHUB_USERNAME=""}" +} - +function initialization::initialize_test_variables() { + export TEST_TYPE=${TEST_TYPE:="All"} } # Common environment that is initialized by both Breeze and CI scripts @@ -459,6 +464,7 @@ function initialization::initialize_common_environment() { initialization::initialize_kubernetes_variables initialization::initialize_git_variables initialization::initialize_github_variables + initialization::initialize_test_variables } function initialization::set_default_python_version_if_empty() { @@ -480,7 +486,6 @@ Basic variables: PYTHON_MAJOR_MINOR_VERSION: ${PYTHON_MAJOR_MINOR_VERSION} DB_RESET: ${DB_RESET} START_AIRFLOW: ${START_AIRFLOW} - RBAC_UI: ${RBAC_UI} DockerHub variables: @@ -498,6 +503,7 @@ Force variables: FORCE_BUILD_IMAGES: ${FORCE_BUILD_IMAGES} FORCE_ANSWER_TO_QUESTIONS: ${FORCE_ANSWER_TO_QUESTIONS} SKIP_CHECK_REMOTE_IMAGE: ${SKIP_CHECK_REMOTE_IMAGE} + FAIL_ON_GITHUB_DOCKER_PULL_ERROR: ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} Host variables: @@ -562,6 +568,10 @@ Initialization variables: INSTALL_WHEELS: ${INSTALL_WHEELS} DISABLE_RBAC: ${DISABLE_RBAC} +Test variables: + + TEST_TYPE: ${TEST_TYPE} + EOF } @@ -633,7 +643,6 @@ function initialization::make_constants_read_only() { readonly POSTGRES_HOST_PORT readonly MYSQL_HOST_PORT - readonly HOST_USER_ID readonly HOST_GROUP_ID readonly HOST_AIRFLOW_SOURCES @@ -660,7 +669,6 @@ function initialization::make_constants_read_only() { readonly VERBOSE readonly START_AIRFLOW - readonly RBAC_UI readonly PRODUCTION_IMAGE @@ -716,7 +724,6 @@ function initialization::make_constants_read_only() { readonly GITHUB_TOKEN readonly GITHUB_USERNAME - readonly FORWARD_CREDENTIALS readonly USE_GITHUB_REGISTRY @@ -740,21 +747,18 @@ function initialization::make_constants_read_only() { } - # converts parameters to json array function initialization::parameters_to_json() { echo -n "[" local separator="" local var - for var in "${@}" - do + for var in "${@}"; do echo -n "${separator}\"${var}\"" separator="," done echo "]" } - # output parameter name and value - both to stdout and to be set by GitHub Actions function initialization::ga_output() { echo "::set-output name=${1}::${2}" diff --git a/scripts/ci/libraries/_parameters.sh b/scripts/ci/libraries/_parameters.sh index 3f2f02d..d655853 100644 --- a/scripts/ci/libraries/_parameters.sh +++ b/scripts/ci/libraries/_parameters.sh @@ -17,21 +17,22 @@ # under the License. # Reads environment variable passed as first parameter from the .build cache file -function parameters::read_from_file { +function parameters::read_from_file() { cat "${BUILD_CACHE_DIR}/.$1" 2>/dev/null || true } # Saves environment variable passed as first parameter to the .build cache file -function parameters::save_to_file { +function parameters::save_to_file() { # shellcheck disable=SC2005 - echo "$(eval echo "\$$1")" > "${BUILD_CACHE_DIR}/.$1" + echo "$(eval echo "\$$1")" >"${BUILD_CACHE_DIR}/.$1" } # check if parameter set for the variable is allowed (should be on the _breeze_allowed list) -# and if it is, it saves it to .build cache file. In case the parameter is wrong, the -# saved variable is removed (so that bad value is not used again in case it comes from there) -# and exits with an error -function parameters::check_and_save_allowed_param { +# parameters: +# $1 - name of the variable +# $2 - descriptive name of the parameter +# $3 - flag used to set te parameter +function parameters::check_allowed_param() { _VARIABLE_NAME="${1}" _VARIABLE_DESCRIPTIVE_NAME="${2}" _FLAG="${3}" @@ -44,9 +45,7 @@ function parameters::check_and_save_allowed_param { echo >&2 echo >&2 "Switch to supported value with ${_FLAG} flag." - if [[ -n ${!_VARIABLE_NAME} && \ - -f "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}" && \ - ${!_VARIABLE_NAME} == $(cat "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}" ) ]]; then + if [[ -n ${!_VARIABLE_NAME} && -f "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}" && ${!_VARIABLE_NAME} == $(cat "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}") ]]; then echo >&2 echo >&2 "Removing ${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}. Next time you run it, it should be OK." echo >&2 @@ -54,5 +53,12 @@ function parameters::check_and_save_allowed_param { fi exit 1 fi - parameters::save_to_file "${_VARIABLE_NAME}" +} +# check if parameter set for the variable is allowed (should be on the _breeze_allowed list) +# and if it is, it saves it to .build cache file. In case the parameter is wrong, the +# saved variable is removed (so that bad value is not used again in case it comes from there) +# and exits with an error +function parameters::check_and_save_allowed_param() { + parameters::check_allowed_param "${@}" + parameters::save_to_file "${1}" } diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 5810303..fae0807 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -63,6 +63,20 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() { echo docker pull "${IMAGE_TO_PULL}" EXIT_VALUE="$?" + if [[ ${EXIT_VALUE} != "0" && ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} == "true" ]]; then + >&2 echo + >&2 echo "ERROR! Exiting on docker pull error" + >&2 echo + >&2 echo "If you have authorisation problems, you might want to run:" + >&2 echo + >&2 echo "docker login ${IMAGE_TO_PULL%%\/*}" + >&2 echo + >&2 echo "You need to use generate token as the password, not your personal password." + >&2 echo "You can generete one at https://github.com/settings/tokens"; + >&2 echo "Make sure to choose 'read:packages' scope". + >&2 echo + exit ${EXIT_VALUE} + fi echo return ${EXIT_VALUE} fi diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 8e6f8f5..5c24e48 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -27,11 +27,6 @@ ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""} if [[ ${TEST_TYPE:=} == "Integration" ]]; then export ENABLED_INTEGRATIONS="${AVAILABLE_INTEGRATIONS}" export RUN_INTEGRATION_TESTS="${AVAILABLE_INTEGRATIONS}" -elif [[ ${TEST_TYPE:=} == "Long" ]]; then - export ONLY_RUN_LONG_RUNNING_TESTS="true" -elif [[ ${TEST_TYPE:=} == "Quarantined" ]]; then - export ONLY_RUN_QUARANTINED_TESTS="true" - # Do not fail in quarantined tests fi for _INT in ${ENABLED_INTEGRATIONS} @@ -81,7 +76,7 @@ function run_airflow_testing_in_docker() { break fi done - if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} == "true" ]]; then + if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then if [[ ${exit_code} == "1" ]]; then echo echo "Some Quarantined tests failed. but we recorded it in an issue" @@ -137,7 +132,3 @@ readonly RUN_INTEGRATION_TESTS run_airflow_testing_in_docker "${@}" - -if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then - export ONLY_RUN_QUARANTINED_TESTS="true" -fi diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index f59e58e..02ecab2 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -19,6 +19,15 @@ if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then set -x fi +function disable_rbac_if_requested() { + if [[ ${DISABLE_RBAC:="false"} == "true" ]]; then + export AIRFLOW__WEBSERVER__RBAC="False" + else + export AIRFLOW__WEBSERVER__RBAC="True" + fi +} + + # shellcheck source=scripts/in_container/_in_container_script_init.sh . /opt/airflow/scripts/in_container/_in_container_script_init.sh @@ -189,17 +198,102 @@ if [[ "${GITHUB_ACTIONS}" == "true" ]]; then "--pythonwarnings=ignore::DeprecationWarning" "--pythonwarnings=ignore::PendingDeprecationWarning" "--junitxml=${RESULT_LOG_FILE}" + # timeouts in seconds for individual tests + "--setup-timeout=20" + "--execution-timeout=60" + "--teardown-timeout=20" + # Only display summary for non-expected case + # f - failed + # E - error + # X - xpessed (passed even if expected to fail) + # The following cases are not displayed: + # s - skipped + # x - xfailed (expected to fail and failed) + # p - passed + # P - passed with output + "-rfEX" ) else - EXTRA_PYTEST_ARGS=() + EXTRA_PYTEST_ARGS=( + "-rfEX" + ) fi -declare -a TESTS_TO_RUN -TESTS_TO_RUN=("tests") +declare -a SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ + ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS + +# Finds all directories that are not on the list of tests +# - so that we do not skip any in the future if new directories are added +function find_all_other_tests() { + local all_tests_dirs + all_tests_dirs=$(find "tests" -type d) + all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/tests$/d" ) + all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/tests\/dags/d" ) + local path + for path in "${ALL_PRESELECTED_TESTS[@]}" + do + escaped_path="${path//\//\\\/}" + all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/${escaped_path}/d" ) + done + for path in ${all_tests_dirs} + do + ALL_OTHER_TESTS+=("${path}") + done +} if [[ ${#@} -gt 0 && -n "$1" ]]; then - TESTS_TO_RUN=("${@}") + SELECTED_TESTS=("${@}") +else + CLI_TESTS=("tests/cli") + API_TESTS=("tests/api" "tests/api_connexion") + PROVIDERS_TESTS=("tests/providers") + CORE_TESTS=( + "tests/core" + "tests/executors" + "tests/jobs" + "tests/models" + "tests/serialization" + "tests/ti_deps" + "tests/utils" + ) + WWW_TESTS=("tests/www") + ALL_TESTS=("tests") + ALL_PRESELECTED_TESTS=( + "${CLI_TESTS[@]}" + "${API_TESTS[@]}" + "${PROVIDERS_TESTS[@]}" + "${CORE_TESTS[@]}" + "${WWW_TESTS[@]}" + ) + + if [[ ${TEST_TYPE:=""} == "CLI" ]]; then + SELECTED_TESTS=("${CLI_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "API" ]]; then + SELECTED_TESTS=("${API_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "Providers" ]]; then + SELECTED_TESTS=("${PROVIDERS_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "Core" ]]; then + SELECTED_TESTS=("${CORE_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then + SELECTED_TESTS=("${WWW_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "Other" ]]; then + find_all_other_tests + SELECTED_TESTS=("${ALL_OTHER_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "All" || ${TEST_TYPE} == "Quarantined" || \ + ${TEST_TYPE} == "Postgres" || ${TEST_TYPE} == "MySQL" || \ + ${TEST_TYPE} == "Heisentests" || ${TEST_TYPE} == "Long" || \ + ${TEST_TYPE} == "Integration" ]]; then + SELECTED_TESTS=("${ALL_TESTS[@]}") + else + >&2 echo + >&2 echo "Wrong test type ${TEST_TYPE}" + >&2 echo + exit 1 + fi + fi +readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ + ALL_TESTS ALL_PRESELECTED_TESTS if [[ -n ${RUN_INTEGRATION_TESTS=} ]]; then # Integration tests @@ -207,43 +301,38 @@ if [[ -n ${RUN_INTEGRATION_TESTS=} ]]; then do EXTRA_PYTEST_ARGS+=("--integration" "${INT}") done - EXTRA_PYTEST_ARGS+=( - # timeouts in seconds for individual tests - "--setup-timeout=20" - "--execution-timeout=60" - "--teardown-timeout=20" - # Do not display skipped tests - "-rfExFpP" - ) - -elif [[ ${ONLY_RUN_LONG_RUNNING_TESTS:=""} == "true" ]]; then +elif [[ ${TEST_TYPE:=""} == "Long" ]]; then EXTRA_PYTEST_ARGS+=( "-m" "long_running" "--include-long-running" - "--verbosity=1" - "--setup-timeout=30" - "--execution-timeout=120" - "--teardown-timeout=30" ) -elif [[ ${ONLY_RUN_QUARANTINED_TESTS:=""} == "true" ]]; then +elif [[ ${TEST_TYPE:=""} == "Heisentests" ]]; then EXTRA_PYTEST_ARGS+=( - "-m" "quarantined" - "--include-quarantined" - "--verbosity=1" - "--setup-timeout=10" - "--execution-timeout=50" - "--teardown-timeout=10" + "-m" "heisentests" + "--include-heisentests" ) -else - # Core tests +elif [[ ${TEST_TYPE:=""} == "Postgres" ]]; then EXTRA_PYTEST_ARGS+=( - "--setup-timeout=10" - "--execution-timeout=30" - "--teardown-timeout=10" + "--backend" + "postgres" + ) +elif [[ ${TEST_TYPE:=""} == "MySQL" ]]; then + EXTRA_PYTEST_ARGS+=( + "--backend" + "mysql" + ) +elif [[ ${TEST_TYPE:=""} == "Quarantined" ]]; then + EXTRA_PYTEST_ARGS+=( + "-m" "quarantined" + "--include-quarantined" ) fi -ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}") +echo +echo "Running tests ${SELECTED_TESTS[*]}" +echo + +ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${SELECTED_TESTS[@]}") if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}" diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh index b505979..7494eec 100755 --- a/scripts/in_container/run_ci_tests.sh +++ b/scripts/in_container/run_ci_tests.sh @@ -30,12 +30,59 @@ RES=$? set +x if [[ "${RES}" == "0" && ${CI:="false"} == "true" ]]; then echo "All tests successful" - bash <(curl -s https://codecov.io/bash) + cp .coverage /files +elif [[ "${RES}" != "0" ]]; then + EXTRA_ARGS="" + if [[ ${BACKEND} == "postgres" ]]; then + EXTRA_ARGS="--postgres-version ${POSTGRES_VERSION} " + elif [[ ${BACKEND} == "mysql" ]]; then + EXTRA_ARGS="--mysql-version ${MYSQL_VERSION} " + fi + + >&2 echo "***********************************************************************************************" + >&2 echo "*" + >&2 echo "* ERROR! Some tests failed, unfortunately. Those might be transient errors," + >&2 echo "* but usually you have to fix something." + >&2 echo "* See the above log for details." + >&2 echo "*" + >&2 echo "***********************************************************************************************" + >&2 echo "* You can easily reproduce the failed tests on your dev machine/" + >&2 echo "*" + >&2 echo "* When you have the source branch checked out locally:" + >&2 echo "*" + >&2 echo "* Run all tests:" + >&2 echo "*" + >&2 echo "* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --test-type ${TEST_TYPE} tests" + >&2 echo "*" + >&2 echo "* Enter docker shell:" + >&2 echo "*" + >&2 echo "* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --test-type ${TEST_TYPE} shell" + >&2 echo "*" + if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} != "" ]]; then + >&2 echo "* When you do not have sources:" + >&2 echo "*" + >&2 echo "* Run all tests:" + >&2 echo "*" + >&2 echo "* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --test-type ${TEST_TYPE} tests" + >&2 echo "*" + >&2 echo "* Enter docker shell:" + >&2 echo "*" + >&2 echo "* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --test-type ${TEST_TYPE} shell" + >&2 echo "*" + fi + >&2 echo "*" + >&2 echo "* NOTE! Once you are in the docker shell, you can run failed test with:" + >&2 echo "*" + >&2 echo "* pytest [TEST_NAME]" + >&2 echo "*" + >&2 echo "* You can copy the test name from the output above" + >&2 echo "*" + >&2 echo "***********************************************************************************************" fi MAIN_GITHUB_REPOSITORY="apache/airflow" -if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} = "true" ]]; then +if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then if [[ ${GITHUB_REPOSITORY} == "${MAIN_GITHUB_REPOSITORY}" ]]; then if [[ ${RES} == "1" || ${RES} == "0" ]]; then echo @@ -47,15 +94,7 @@ if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} = "true" ]]; then echo "Pytest exited with ${RES} result. NOT Updating Quarantine Issue!" echo fi - else - echo - echo "GitHub repository '${GITHUB_REPOSITORY}'. NOT Updating Quarantine Issue!" - echo fi -else - echo - echo "Regular tests. NOT Updating Quarantine Issue!" - echo fi if [[ ${CI:=} == "true" ]]; then diff --git a/tests/bats/test_breeze_complete.bats b/tests/bats/test_breeze_complete.bats index 2fee270..163db61 100644 --- a/tests/bats/test_breeze_complete.bats +++ b/tests/bats/test_breeze_complete.bats @@ -254,3 +254,11 @@ assert_equal "${_breeze_default_postgres_version}" "${POSTGRES_VERSION}" } + +@test "Test default test type same as TEST_TYPE" { + load bats_utils + #shellcheck source=breeze-complete + source "${AIRFLOW_SOURCES}/breeze-complete" + + assert_equal "${_breeze_default_test_type}" "${TEST_TYPE}" +} diff --git a/tests/test_config_templates.py b/tests/core/test_config_templates.py similarity index 100% rename from tests/test_config_templates.py rename to tests/core/test_config_templates.py diff --git a/tests/test_configuration.py b/tests/core/test_configuration.py similarity index 100% rename from tests/test_configuration.py rename to tests/core/test_configuration.py diff --git a/tests/test_core.py b/tests/core/test_core.py similarity index 100% rename from tests/test_core.py rename to tests/core/test_core.py diff --git a/tests/test_example_dags_system.py b/tests/core/test_example_dags_system.py similarity index 100% rename from tests/test_example_dags_system.py rename to tests/core/test_example_dags_system.py diff --git a/tests/test_impersonation.py b/tests/core/test_impersonation_tests.py similarity index 100% rename from tests/test_impersonation.py rename to tests/core/test_impersonation_tests.py diff --git a/tests/test_logging_config.py b/tests/core/test_logging_config.py similarity index 100% rename from tests/test_logging_config.py rename to tests/core/test_logging_config.py diff --git a/tests/test_sentry.py b/tests/core/test_sentry.py similarity index 100% rename from tests/test_sentry.py rename to tests/core/test_sentry.py diff --git a/tests/test_sqlalchemy_config.py b/tests/core/test_sqlalchemy_config.py similarity index 96% rename from tests/test_sqlalchemy_config.py rename to tests/core/test_sqlalchemy_config.py index b908ce2..6fa7ac9 100644 --- a/tests/test_sqlalchemy_config.py +++ b/tests/core/test_sqlalchemy_config.py @@ -76,7 +76,8 @@ class TestSqlAlchemySettings(unittest.TestCase): mock_scoped_session, mock_setup_event_handlers): config = { - ('core', 'sql_alchemy_connect_args'): 'tests.test_sqlalchemy_config.SQL_ALCHEMY_CONNECT_ARGS', + ('core', 'sql_alchemy_connect_args'): + 'tests.core.test_sqlalchemy_config.SQL_ALCHEMY_CONNECT_ARGS', ('core', 'sql_alchemy_pool_enabled'): 'False' } with conf_vars(config): diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index 0955825..72e3628 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -165,7 +165,7 @@ class LocalTaskJobTest(unittest.TestCase): delta = (time2 - time1).total_seconds() self.assertAlmostEqual(delta, job.heartrate, delta=0.05) - @pytest.mark.xfail(condition=True, reason="This test might be flaky in postgres/mysql") + @pytest.mark.quarantined def test_mark_success_no_kill(self): """ Test that ensures that mark_success in the UI doesn't cause diff --git a/tests/task/task_runner/test_standard_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py index 2350807..e3fceec 100644 --- a/tests/task/task_runner/test_standard_task_runner.py +++ b/tests/task/task_runner/test_standard_task_runner.py @@ -69,7 +69,11 @@ class TestStandardTaskRunner(unittest.TestCase): @classmethod def tearDownClass(cls): - clear_db_runs() + try: + clear_db_runs() + except Exception: # noqa pylint: disable=broad-except + # It might happen that we lost connection to the server here so we need to ignore any errors here + pass def test_start_and_terminate(self): local_task_job = mock.Mock()
