This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a13ccbbdec Make latest botocore tests green (#41626)
a13ccbbdec is described below
commit a13ccbbdec8e59f30218f604fca8cbb999fcb757
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Aug 21 11:17:55 2024 +0200
Make latest botocore tests green (#41626)
The latest botocore tests are conflicting with a few requirements
and until apache-beam upcoming version is released we need to do
some manual exclusions. Those exclusions should make latest botocore
test green again.
---
Dockerfile.ci | 9 +++++----
pyproject.toml | 3 +++
scripts/docker/entrypoint_ci.sh | 9 +++++----
tests/always/test_example_dags.py | 20 ++++++--------------
tests/providers/opensearch/conftest.py | 11 +++++++++--
tests/providers/opensearch/hooks/test_opensearch.py | 3 ++-
.../opensearch/operators/test_opensearch.py | 3 +++
7 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 2ebe82c6ef..25151df8e9 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1013,16 +1013,17 @@ function check_boto_upgrade() {
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run
Amazon tests with them${COLOR_RESET}"
echo
# shellcheck disable=SC2086
- ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs
yandexcloud || true
+ ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs
yandexcloud opensearch-py || true
# We need to include few dependencies to pass pip check with other
dependencies:
# * oss2 as dependency as otherwise jmespath will be bumped (sync with
alibaba provider)
- # * gcloud-aio-auth limit is needed to be included as it bumps
cryptography (sync with google provider)
+ # * cryptography is kept for snowflake-connector-python limitation (sync
with snowflake provider)
# * requests needs to be limited to be compatible with apache beam (sync
with apache-beam provider)
# * yandexcloud requirements for requests does not match those of
apache.beam and latest botocore
# Both requests and yandexcloud exclusion above might be removed after
# https://github.com/apache/beam/issues/32080 is addressed
- # When you remove yandexcloud from the above list, also remove it from
"test_example_dags.py"
- # in "tests/always".
+ # This is already addressed and planned for 2.59.0 release.
+ # When you remove yandexcloud and opensearch from the above list, you
can also remove the
+ # optional providers_dependencies exclusions from "test_example_dags.py"
in "tests/always".
set -x
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3
botocore \
diff --git a/pyproject.toml b/pyproject.toml
index 8562a137d2..3c09126751 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -406,6 +406,9 @@ combine-as-imports = true
"tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py" =
["E402"]
"tests/providers/google/cloud/triggers/test_vertex_ai.py" = ["E402"]
"tests/providers/openai/hooks/test_openai.py" = ["E402"]
+"tests/providers/opensearch/conftest.py" = ["E402"]
+"tests/providers/opensearch/hooks/test_opensearch.py" = ["E402"]
+"tests/providers/opensearch/operators/test_opensearch.py" = ["E402"]
"tests/providers/openai/operators/test_openai.py" = ["E402"]
"tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"]
"tests/providers/qdrant/operators/test_qdrant.py" = ["E402"]
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 06b1437c2f..b1d6f4cf70 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -250,16 +250,17 @@ function check_boto_upgrade() {
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run
Amazon tests with them${COLOR_RESET}"
echo
# shellcheck disable=SC2086
- ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs
yandexcloud || true
+ ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs
yandexcloud opensearch-py || true
# We need to include few dependencies to pass pip check with other
dependencies:
# * oss2 as dependency as otherwise jmespath will be bumped (sync with
alibaba provider)
- # * gcloud-aio-auth limit is needed to be included as it bumps
cryptography (sync with google provider)
+ # * cryptography is kept for snowflake-connector-python limitation (sync
with snowflake provider)
# * requests needs to be limited to be compatible with apache beam (sync
with apache-beam provider)
# * yandexcloud requirements for requests does not match those of
apache.beam and latest botocore
# Both requests and yandexcloud exclusion above might be removed after
# https://github.com/apache/beam/issues/32080 is addressed
- # When you remove yandexcloud from the above list, also remove it from
"test_example_dags.py"
- # in "tests/always".
+ # This is already addressed and planned for 2.59.0 release.
+ # When you remove yandexcloud and opensearch from the above list, you
can also remove the
+ # optional providers_dependencies exclusions from "test_example_dags.py"
in "tests/always".
set -x
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3
botocore \
diff --git a/tests/always/test_example_dags.py
b/tests/always/test_example_dags.py
index f81bbf82ca..b9e45647ec 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import os
+import re
import sys
from glob import glob
from importlib import metadata as importlib_metadata
@@ -38,8 +39,11 @@ OPTIONAL_PROVIDERS_DEPENDENCIES: dict[str, dict[str, str |
None]] = {
# Some examples or system tests may depend on additional packages
# that are not included in certain CI checks.
# The format of the dictionary is as follows:
- # key: the prefix of the file to be excluded,
+ # key: the regexp matching the file to be excluded,
# value: a dictionary containing package distributions with an optional
version specifier, e.g., >=2.3.4
+ ".*example_bedrock_retrieve_and_generate.py": {"opensearch-py": None},
+ ".*example_opensearch.py": {"opensearch-py": None},
+ r".*example_yandexcloud.*\.py": {"yandexcloud": None},
}
IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = (
# Certain examples or system tests may trigger
AirflowProviderDeprecationWarnings.
@@ -123,13 +127,6 @@ def example_not_excluded_dags(xfail_db_exception: bool =
False):
for prefix in PROVIDERS_PREFIXES
for provider in suspended_providers_folders
]
- temporary_excluded_upgrade_boto_providers_folders = [
- AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
- for prefix in PROVIDERS_PREFIXES
- # TODO - remove me when https://github.com/apache/beam/issues/32080 is
addressed
- # and we bring back yandex to be run in case of upgrade boto
- for provider in ["yandex"]
- ]
current_python_excluded_providers_folders = [
AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
for prefix in PROVIDERS_PREFIXES
@@ -145,18 +142,13 @@ def example_not_excluded_dags(xfail_db_exception: bool =
False):
if candidate.startswith(tuple(suspended_providers_folders)):
param_marks.append(pytest.mark.skip(reason="Suspended
provider"))
- if os.environ.get("UPGRADE_BOTO", "false") == "true" and
candidate.startswith(
- tuple(temporary_excluded_upgrade_boto_providers_folders)
- ):
- param_marks.append(pytest.mark.skip(reason="Temporary excluded
upgrade boto provider"))
-
if
candidate.startswith(tuple(current_python_excluded_providers_folders)):
param_marks.append(
pytest.mark.skip(reason=f"Not supported for Python
{CURRENT_PYTHON_VERSION}")
)
for optional, dependencies in
OPTIONAL_PROVIDERS_DEPENDENCIES.items():
- if candidate.endswith(optional):
+ if re.match(optional, candidate):
for distribution_name, specifier in dependencies.items():
result, reason =
match_optional_dependencies(distribution_name, specifier)
if not result:
diff --git a/tests/providers/opensearch/conftest.py
b/tests/providers/opensearch/conftest.py
index 47a447188e..934bbd642a 100644
--- a/tests/providers/opensearch/conftest.py
+++ b/tests/providers/opensearch/conftest.py
@@ -19,12 +19,19 @@ from __future__ import annotations
from typing import Any
import pytest
-from opensearchpy import OpenSearch
+from airflow.hooks.base import BaseHook
from airflow.models import Connection
-from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook
from airflow.utils import db
+try:
+ from opensearchpy import OpenSearch
+
+ from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook
+except ImportError:
+ OpenSearch = None # type: ignore[assignment, misc]
+ OpenSearchHook = BaseHook # type: ignore[assignment,misc]
+
# TODO: FIXME - those Mocks have overrides that are not used but they also do
not make Mypy Happy
# mypy: disable-error-code="override"
diff --git a/tests/providers/opensearch/hooks/test_opensearch.py
b/tests/providers/opensearch/hooks/test_opensearch.py
index 84360ae73f..43075e8532 100644
--- a/tests/providers/opensearch/hooks/test_opensearch.py
+++ b/tests/providers/opensearch/hooks/test_opensearch.py
@@ -18,8 +18,9 @@ from __future__ import annotations
from unittest import mock
-import opensearchpy
import pytest
+
+opensearchpy = pytest.importorskip("opensearchpy")
from opensearchpy import Urllib3HttpConnection
from airflow.exceptions import AirflowException
diff --git a/tests/providers/opensearch/operators/test_opensearch.py
b/tests/providers/opensearch/operators/test_opensearch.py
index 706112fef6..63ad7eafe4 100644
--- a/tests/providers/opensearch/operators/test_opensearch.py
+++ b/tests/providers/opensearch/operators/test_opensearch.py
@@ -17,6 +17,9 @@
from __future__ import annotations
import pytest
+
+opensearchpy = pytest.importorskip("opensearchpy")
+
from opensearchpy import Document, Keyword, Text
from airflow.models import DAG
