This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 06a200d5fa Bump boto min versions (#40052)
06a200d5fa is described below
commit 06a200d5fad3d80219d1d545929ebec3cc1afb55
Author: D. Ferruzzi <[email protected]>
AuthorDate: Tue Jun 4 21:17:19 2024 -0700
Bump boto min versions (#40052)
* bump boto min versions
* Revert "Doc-only: mention minimum boto3 1.34.52 for AWS provider when
using Batch `ecs_properties_override` (#39983)"
This reverts commit 99dd24b436c1a8c5f736e9784c60d469a5b2bed7.
* remove warnings that specific botocore version is required since min
version is now higher
---
airflow/providers/amazon/aws/operators/batch.py | 20 +----------------
airflow/providers/amazon/aws/operators/bedrock.py | 26 ++++++-----------------
airflow/providers/amazon/provider.yaml | 16 +++++++-------
generated/provider_dependencies.json | 14 ++++++------
4 files changed, 22 insertions(+), 54 deletions(-)
diff --git a/airflow/providers/amazon/aws/operators/batch.py
b/airflow/providers/amazon/aws/operators/batch.py
index 6125c37140..849fc19346 100644
--- a/airflow/providers/amazon/aws/operators/batch.py
+++ b/airflow/providers/amazon/aws/operators/batch.py
@@ -30,8 +30,6 @@ from datetime import timedelta
from functools import cached_property
from typing import TYPE_CHECKING, Any, Sequence
-import botocore
-
from airflow.configuration import conf
from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
from airflow.models import BaseOperator
@@ -47,11 +45,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
BatchCreateComputeEnvironmentTrigger,
BatchJobTrigger,
)
-from airflow.providers.amazon.aws.utils import (
- get_botocore_version,
- trim_none_values,
- validate_execute_complete_event,
-)
+from airflow.providers.amazon.aws.utils import trim_none_values,
validate_execute_complete_event
from airflow.providers.amazon.aws.utils.task_log_fetcher import
AwsTaskLogFetcher
from airflow.utils.types import NOTSET
@@ -72,7 +66,6 @@ class BatchOperator(BaseOperator):
:param overrides: DEPRECATED, use container_overrides instead with the
same value.
:param container_overrides: the `containerOverrides` parameter for boto3
(templated)
:param ecs_properties_override: the `ecsPropertiesOverride` parameter for
boto3 (templated)
- **NOTE** This requires `boto3` version 1.34.52+
:param node_overrides: the `nodeOverrides` parameter for boto3 (templated)
:param share_identifier: The share identifier for the job. Don't specify
this parameter if the job queue
doesn't have a scheduling policy.
@@ -330,17 +323,6 @@ class BatchOperator(BaseOperator):
try:
response = self.hook.client.submit_job(**trim_none_values(args))
- except botocore.exceptions.ParamValidationError as error:
- if (
- 'Unknown parameter in input: "ecsPropertiesOverride"' in
str(error)
- ) and self.ecs_properties_override:
- self.log.error(
- "You are attempting to use ecsPropertiesOverride and the
botocore API returned an "
- "error message which may indicate the need to update
botocore to do this. \n"
- "Support for using ecsPropertiesOverride was added in
botocore 1.34.52 and you are using botocore %s",
- ".".join(map(str, get_botocore_version())),
- )
- raise
except Exception as e:
self.log.error(
"AWS Batch job failed submission - job definition: %s - on
queue %s",
diff --git a/airflow/providers/amazon/aws/operators/bedrock.py
b/airflow/providers/amazon/aws/operators/bedrock.py
index 3efd4ff8a5..021ee92dbc 100644
--- a/airflow/providers/amazon/aws/operators/bedrock.py
+++ b/airflow/providers/amazon/aws/operators/bedrock.py
@@ -20,7 +20,6 @@ import json
from time import sleep
from typing import TYPE_CHECKING, Any, Sequence
-import botocore
from botocore.exceptions import ClientError
from airflow.configuration import conf
@@ -38,7 +37,7 @@ from airflow.providers.amazon.aws.triggers.bedrock import (
BedrockKnowledgeBaseActiveTrigger,
BedrockProvisionModelThroughputCompletedTrigger,
)
-from airflow.providers.amazon.aws.utils import get_botocore_version,
validate_execute_complete_event
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
from airflow.utils.helpers import prune_dict
from airflow.utils.timezone import utcnow
@@ -799,24 +798,11 @@ class
BedrockRaGOperator(AwsBaseOperator[BedrockAgentRuntimeHook]):
def execute(self, context: Context) -> Any:
self.validate_inputs()
- try:
- result = self.hook.conn.retrieve_and_generate(
- input={"text": self.input},
- retrieveAndGenerateConfiguration=self.build_rag_config(),
- **self.rag_kwargs,
- )
- except botocore.exceptions.ParamValidationError as error:
- if (
- 'Unknown parameter in retrieveAndGenerateConfiguration:
"externalSourcesConfiguration"'
- in str(error)
- ) and (self.source_type == "EXTERNAL_SOURCES"):
- self.log.error(
- "You are attempting to use External Sources and the BOTO
API returned an "
- "error message which may indicate the need to update
botocore to do this. \n"
- "Support for External Sources was added in botocore
1.34.90 and you are using botocore %s",
- ".".join(map(str, get_botocore_version())),
- )
- raise
+ result = self.hook.conn.retrieve_and_generate(
+ input={"text": self.input},
+ retrieveAndGenerateConfiguration=self.build_rag_config(),
+ **self.rag_kwargs,
+ )
self.log.info(
"\nPrompt: %s\nResponse: %s\nCitations: %s",
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index 9789ccf6a0..83515935e8 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -92,8 +92,8 @@ dependencies:
# We should update minimum version of boto3 and here regularly to avoid
`pip` backtracking with the number
# of candidates to consider. Make sure to configure boto3 version here as
well as in all the tools below
# in the `devel-dependencies` section to be the same minimum version.
- - boto3>=1.34.0
- - botocore>=1.34.0
+ - boto3>=1.34.90
+ - botocore>=1.34.90
- inflection>=0.5.1
# Allow a wider range of watchtower versions for flexibility among users
- watchtower>=3.0.0,<4
@@ -119,7 +119,7 @@ additional-extras:
# boto3 have native async support and we move away from aio aiobotocore
- name: aiobotocore
dependencies:
- - aiobotocore[boto3]>=2.10.0
+ - aiobotocore[boto3]>=2.13.0
- name: cncf.kubernetes
dependencies:
- apache-airflow-providers-cncf-kubernetes>=7.2.0
@@ -131,13 +131,13 @@ additional-extras:
- python3-saml>=1.16.0
devel-dependencies:
- - aiobotocore>=2.7.0
+ - aiobotocore>=2.13.0
- aws_xray_sdk>=2.12.0
- moto[cloudformation,glue]>=5.0.0
- - mypy-boto3-appflow>=1.33.0
- - mypy-boto3-rds>=1.33.0
- - mypy-boto3-redshift-data>=1.33.0
- - mypy-boto3-s3>=1.33.0
+ - mypy-boto3-appflow>=1.34.0
+ - mypy-boto3-rds>=1.34.90
+ - mypy-boto3-redshift-data>=1.34.0
+ - mypy-boto3-s3>=1.34.90
- s3fs>=2023.10.0
- openapi-schema-validator>=0.6.2
- openapi-spec-validator>=0.7.1
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index d96c77cf4c..4a2191f6a2 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -32,8 +32,8 @@
"apache-airflow-providers-http",
"apache-airflow>=2.7.0",
"asgiref>=2.3.0",
- "boto3>=1.34.0",
- "botocore>=1.34.0",
+ "boto3>=1.34.90",
+ "botocore>=1.34.90",
"inflection>=0.5.1",
"jmespath>=0.7.0",
"jsonpath_ng>=1.5.3",
@@ -42,13 +42,13 @@
"watchtower>=3.0.0,<4"
],
"devel-deps": [
- "aiobotocore>=2.7.0",
+ "aiobotocore>=2.13.0",
"aws_xray_sdk>=2.12.0",
"moto[cloudformation,glue]>=5.0.0",
- "mypy-boto3-appflow>=1.33.0",
- "mypy-boto3-rds>=1.33.0",
- "mypy-boto3-redshift-data>=1.33.0",
- "mypy-boto3-s3>=1.33.0",
+ "mypy-boto3-appflow>=1.34.0",
+ "mypy-boto3-rds>=1.34.90",
+ "mypy-boto3-redshift-data>=1.34.0",
+ "mypy-boto3-s3>=1.34.90",
"openapi-schema-validator>=0.6.2",
"openapi-spec-validator>=0.7.1",
"s3fs>=2023.10.0"
