Repository: incubator-airflow Updated Branches: refs/heads/master 06bd4d3c1 -> 46c86a5cd
[AIRFLOW-1368] Automatically remove Docker container on exit Closes #2411 from nathanielvarona/docker-operator Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/46c86a5c Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/46c86a5c Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/46c86a5c Branch: refs/heads/master Commit: 46c86a5cd2b69f4f1853280b442e8810b178e6c7 Parents: 06bd4d3 Author: Nathaniel Varona <[email protected]> Authored: Fri Sep 22 10:15:02 2017 -0700 Committer: Chris Riccomini <[email protected]> Committed: Fri Sep 22 10:15:23 2017 -0700 ---------------------------------------------------------------------- airflow/operators/docker_operator.py | 14 ++++++++++---- scripts/ci/requirements.txt | 2 +- setup.py | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/airflow/operators/docker_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index 3011f1c..75a7a39 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -18,7 +18,7 @@ from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory -from docker import Client, tls +from docker import APIClient, tls import ast @@ -78,6 +78,8 @@ class DockerOperator(BaseOperator): :type xcom_push: bool :param xcom_all: Push all the stdout or just the last line. The default is False (last line). :type xcom_all: bool + :param auto_remove: Automatically remove the container when it exits + :type auto_remove: bool """ template_fields = ('command',) template_ext = ('.sh', '.bash',) @@ -105,6 +107,7 @@ class DockerOperator(BaseOperator): working_dir=None, xcom_push=False, xcom_all=False, + auto_remove=False, *args, **kwargs): @@ -129,6 +132,7 @@ class DockerOperator(BaseOperator): self.working_dir = working_dir self.xcom_push_flag = xcom_push self.xcom_all = xcom_all + self.auto_remove = auto_remove self.cli = None self.container = None @@ -147,7 +151,7 @@ class DockerOperator(BaseOperator): ) self.docker_url = self.docker_url.replace('tcp://', 'https://') - self.cli = Client(base_url=self.docker_url, version=self.api_version, tls=tls_config) + self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config) if ':' not in self.image: image = self.image + ':latest' @@ -170,8 +174,10 @@ class DockerOperator(BaseOperator): command=self.get_command(), cpu_shares=cpu_shares, environment=self.environment, - host_config=self.cli.create_host_config(binds=self.volumes, - network_mode=self.network_mode), + host_config=self.cli.create_host_config( + binds=self.volumes, + network_mode=self.network_mode, + auto_remove=self.auto_remove), image=image, mem_limit=self.mem_limit, user=self.user, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/scripts/ci/requirements.txt ---------------------------------------------------------------------- diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt index d612d6f..0429787 100644 --- a/scripts/ci/requirements.txt +++ b/scripts/ci/requirements.txt @@ -28,7 +28,7 @@ cryptography datadog dill distributed -docker-py +docker filechunkio flake8 flask http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index a97abfd..46f1b1b 100644 --- a/setup.py +++ b/setup.py @@ -125,6 +125,8 @@ doc = [ 'Sphinx-PyPI-upload>=0.2.1' ] docker = ['docker-py>=1.6.0'] +docker = ['docker>=2.4.0'] +druid = ['pydruid>=0.2.1'] emr = ['boto3>=1.0.0'] gcp_api = [ 'httplib2',
