Trollgeir commented on a change in pull request #5319:
[AIRFLOW-4564][AIRFLOW-4523][AIRFLOW-4577] Azure Container Instance bugfixes
and improvements
URL: https://github.com/apache/airflow/pull/5319#discussion_r298870498
##########
File path: airflow/contrib/operators/azure_container_instances_operator.py
##########
@@ -72,59 +81,96 @@ class AzureContainerInstancesOperator(BaseOperator):
:param environment_variables: key,value pairs containing environment
variables which will be passed to the running container
:type environment_variables: dict
- :param volumes: list of volumes to be mounted to the container.
+ :param secured_variables: names of environmental variables that should not
+ be exposed outside the container (typically passwords).
+ :type secured_variables: [str]
+ :param volumes: list of ``Volume`` tuples to be mounted to the container.
Currently only Azure Fileshares are supported.
:type volumes: list[<conn_id, account_name, share_name, mount_path,
read_only>]
:param memory_in_gb: the amount of memory to allocate to this container
:type memory_in_gb: double
:param cpu: the number of cpus to allocate to this container
:type cpu: double
+ :param gpu: the number of gpus with a SKU (possible values
+ include: 'K80', 'P100', 'V100') to allocate to this container
+ :type gpu: tuple(<count>, <sku>)
:param command: the command to run inside the container
- :type command: str
-
- :Example:
-
- >>> a = AzureContainerInstancesOperator(
- 'azure_service_principal',
- 'azure_registry_user',
- 'my-resource-group',
- 'my-container-name-{{ ds }}',
- 'myprivateregistry.azurecr.io/my_container:latest',
- 'westeurope',
- {'EXECUTION_DATE': '{{ ds }}'},
- [('azure_wasb_conn_id',
- 'my_storage_container',
- 'my_fileshare',
- '/input-data',
- True),],
- memory_in_gb=14.0,
- cpu=4.0,
- command='python /app/myfile.py',
- task_id='start_container'
- )
+ :type command: [str]
+ :param container_timeout: max time allowed for the execution of
+ the container instance.
+ :type container_timeout: datetime.timedelta
+
+ **Example**::
+
+ AzureContainerInstancesOperator(
+ "azure_service_principal",
+ "azure_registry_user",
+ "my-resource-group",
+ "my-container-name-{{ ds }}",
+ "myprivateregistry.azurecr.io/my_container:latest",
+ "westeurope",
+ {"MODEL_PATH": "my_value",
+ "POSTGRES_LOGIN": "{{
macros.connection('postgres_default').login }}"
+ "POSTGRES_PASSWORD": "{{
macros.connection('postgres_default').password }}",
+ "JOB_GUID": "{{ ti.xcom_pull(task_ids='task1',
key='guid') }}" },
+ ['POSTGRES_PASSWORD'],
+ [("azure_wasb_conn_id",
+ "my_storage_container",
+ "my_fileshare",
+ "/input-data",
+ True),],
+ memory_in_gb=14.0,
+ cpu=4.0,
+ gpu=(1, 'K80'),
+ command=["/bin/echo", "world"],
+ container_timeout=timedelta(hours=2),
+ task_id="start_container"
+ )
"""
- template_fields = ('name', 'environment_variables')
+ template_fields = ('name', 'image', 'command', 'environment_variables')
@apply_defaults
- def __init__(self, ci_conn_id, registry_conn_id, resource_group, name,
image, region,
- environment_variables=None, volumes=None, memory_in_gb=None,
cpu=None,
- command=None, remove_on_error=True, fail_if_exists=True,
*args, **kwargs):
+ def __init__(self,
+ ci_conn_id,
+ registry_conn_id,
+ resource_group,
+ name,
+ image,
+ region,
+ environment_variables=None,
+ secured_variables=None,
+ volumes=None,
+ memory_in_gb=None,
+ cpu=None,
+ gpu=None,
+ command=None,
+ remove_on_error=True,
+ fail_if_exists=True,
+ container_timeout=None,
Review comment:
Using your suggestion. Handling the cleanup procedures with the `on_kill()`
method.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
