This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 547e6e80f3 Fix Azure Batch errors revealed by added typing to azure
batch lib (#27601)
547e6e80f3 is described below
commit 547e6e80f342ee6ed454732477700a85cfa4df8b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Nov 10 22:26:10 2022 +0100
Fix Azure Batch errors revealed by added typing to azure batch lib (#27601)
The Azure batch library added typing and it revealed a few
potential (or even not that potential) bugs that needed to be
fixed.
* implicit optional is added for azure.batch as they use it
* some of the parameters are required rather than optional and we
needed to change them from optional in our Hooks
* some fields were wrongly scoped
---
airflow/providers/microsoft/azure/CHANGELOG.rst | 1 +
airflow/providers/microsoft/azure/hooks/batch.py | 15 +++++++--------
airflow/providers/microsoft/azure/operators/batch.py | 2 +-
setup.cfg | 7 +++++++
4 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst
b/airflow/providers/microsoft/azure/CHANGELOG.rst
index 284068b36e..0fc23d1a40 100644
--- a/airflow/providers/microsoft/azure/CHANGELOG.rst
+++ b/airflow/providers/microsoft/azure/CHANGELOG.rst
@@ -35,6 +35,7 @@ Breaking changes
* In AzureFileShareHook, if both ``extra__azure_fileshare__foo`` and ``foo``
existed in connection extra
dict, the prefixed version would be used; now, the non-prefixed version will
be preferred.
* ``Remove deprecated classes (#27417)``
+* In Azure Batch ``vm_size`` and ``vm_node_agent_sku_id`` parameters are
required.
4.3.0
.....
diff --git a/airflow/providers/microsoft/azure/hooks/batch.py
b/airflow/providers/microsoft/azure/hooks/batch.py
index f846a69ec5..e16e118fa9 100644
--- a/airflow/providers/microsoft/azure/hooks/batch.py
+++ b/airflow/providers/microsoft/azure/hooks/batch.py
@@ -103,13 +103,13 @@ class AzureBatchHook(BaseHook):
def configure_pool(
self,
pool_id: str,
- vm_size: str | None = None,
+ vm_size: str,
+ vm_node_agent_sku_id: str,
vm_publisher: str | None = None,
vm_offer: str | None = None,
sku_starts_with: str | None = None,
vm_sku: str | None = None,
vm_version: str | None = None,
- vm_node_agent_sku_id: str | None = None,
os_family: str | None = None,
os_version: str | None = None,
display_name: str | None = None,
@@ -212,8 +212,8 @@ class AzureBatchHook(BaseHook):
self.log.info("Attempting to create a pool: %s", pool.id)
self.connection.pool.add(pool)
self.log.info("Created pool: %s", pool.id)
- except batch_models.BatchErrorException as e:
- if e.error.code != "PoolExists":
+ except batch_models.BatchErrorException as err:
+ if not err.error or err.error.code != "PoolExists":
raise
else:
self.log.info("Pool %s already exists", pool.id)
@@ -302,7 +302,7 @@ class AzureBatchHook(BaseHook):
self.connection.job.add(job)
self.log.info("Job %s created", job.id)
except batch_models.BatchErrorException as err:
- if err.error.code != "JobExists":
+ if not err.error or err.error.code != "JobExists":
raise
else:
self.log.info("Job %s already exists", job.id)
@@ -347,7 +347,7 @@ class AzureBatchHook(BaseHook):
self.connection.task.add(job_id=job_id, task=task)
except batch_models.BatchErrorException as err:
- if err.error.code != "TaskExists":
+ if not err.error or err.error.code != "TaskExists":
raise
else:
self.log.info("Task %s already exists", task.id)
@@ -369,8 +369,7 @@ class AzureBatchHook(BaseHook):
fail_tasks = [
task
for task in tasks
- if task.executionInfo.result
- ==
batch_models.TaskExecutionInformation.TaskExecutionResult.failure
+ if task.executionInfo.result ==
batch_models.TaskExecutionResult.failure
]
return fail_tasks
for task in incomplete_tasks:
diff --git a/airflow/providers/microsoft/azure/operators/batch.py
b/airflow/providers/microsoft/azure/operators/batch.py
index 6d9b341811..0a14993c58 100644
--- a/airflow/providers/microsoft/azure/operators/batch.py
+++ b/airflow/providers/microsoft/azure/operators/batch.py
@@ -109,12 +109,12 @@ class AzureBatchOperator(BaseOperator):
batch_job_id: str,
batch_task_command_line: str,
batch_task_id: str,
+ vm_node_agent_sku_id: str,
vm_publisher: str | None = None,
vm_offer: str | None = None,
sku_starts_with: str | None = None,
vm_sku: str | None = None,
vm_version: str | None = None,
- vm_node_agent_sku_id: str | None = None,
os_family: str | None = None,
os_version: str | None = None,
batch_pool_display_name: str | None = None,
diff --git a/setup.cfg b/setup.cfg
index d798c970e0..3c085f72e6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -186,3 +186,10 @@ ignore_errors = True
# Most of them don't but even if they do, it does not matter
[mypy-google.cloud.*]
no_implicit_optional = False
+
+
+#Let's assume all azure packages have no implicit optional
+[mypy-azure.batch.*]
+no_implicit_optional = False
+[mypy-azure.batch.models.*]
+no_implicit_optional = False
