This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new 301a0b8 Fix quarantined tests - TestCliWebServer (#9596)
301a0b8 is described below
commit 301a0b8d7196244956f53970991d91b780dfdf2e
Author: Kamil BreguĊa <mik-...@users.noreply.github.com>
AuthorDate: Wed Jul 1 19:10:58 2020 +0200
Fix quarantined tests - TestCliWebServer (#9596)
---
airflow/bin/cli.py | 24 ++--
requirements/requirements-python2.7.txt | 9 +-
requirements/requirements-python3.5.txt | 10 +-
requirements/requirements-python3.6.txt | 14 +--
requirements/requirements-python3.7.txt | 12 +-
requirements/requirements-python3.8.txt | 6 +-
requirements/setup-2.7.md5 | 2 +-
requirements/setup-3.5.md5 | 2 +-
requirements/setup-3.6.md5 | 2 +-
requirements/setup-3.7.md5 | 2 +-
requirements/setup-3.8.md5 | 2 +-
setup.py | 4 +-
tests/cli/test_cli.py | 7 +-
tests/insert_extras.py | 2 +-
tests/test_core.py | 193 ++++++++++++++++++++------------
15 files changed, 167 insertions(+), 124 deletions(-)
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 8334af6..51cf49d 100644
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -800,8 +800,8 @@ class GunicornMonitor(LoggingMixin):
respectively. Gunicorn guarantees that on TTOU workers are terminated
gracefully and that the oldest worker is terminated.
- :param gunicorn_master_proc: handle for the main Gunicorn process
- :param num_workers_expected: Number of workers to run the Gunicorn web
server
+ :param gunicorn_master_pid: pid of the main Gunicorn process
+ :param num_workers_expected: Number of workers to run the Gunicorn web
server
:param master_timeout: Number of seconds the webserver waits before
killing gunicorn master that
doesn't respond
:param worker_refresh_interval: Number of seconds to wait before
refreshing a batch of workers.
@@ -813,7 +813,7 @@ class GunicornMonitor(LoggingMixin):
"""
def __init__(
self,
- gunicorn_master_proc,
+ gunicorn_master_pid,
num_workers_expected,
master_timeout,
worker_refresh_interval,
@@ -821,7 +821,7 @@ class GunicornMonitor(LoggingMixin):
reload_on_plugin_change
):
super(GunicornMonitor, self).__init__()
- self.gunicorn_master_proc = gunicorn_master_proc
+ self.gunicorn_master_proc = psutil.Process(gunicorn_master_pid)
self.num_workers_expected = num_workers_expected
self.master_timeout = master_timeout
self.worker_refresh_interval = worker_refresh_interval
@@ -936,14 +936,15 @@ class GunicornMonitor(LoggingMixin):
"""
Starts monitoring the webserver.
"""
+ self.log.debug("Start monitoring gunicorn")
try: # pylint: disable=too-many-nested-blocks
self._wait_until_true(
lambda: self.num_workers_expected ==
self._get_num_workers_running(),
timeout=self.master_timeout
)
while True:
- if self.gunicorn_master_proc.poll() is not None:
- sys.exit(self.gunicorn_master_proc.returncode)
+ if not self.gunicorn_master_proc.is_running():
+ sys.exit(1)
self._check_workers()
# Throttle loop
time.sleep(1)
@@ -1122,15 +1123,16 @@ def webserver(args):
gunicorn_master_proc = None
- def kill_proc(dummy_signum, dummy_frame):
+ def kill_proc(signum, _):
+ log.info("Received signal: %s. Closing gunicorn.", signum)
gunicorn_master_proc.terminate()
gunicorn_master_proc.wait()
sys.exit(0)
- def monitor_gunicorn(gunicorn_master_proc):
+ def monitor_gunicorn(gunicorn_master_pid):
# These run forever until SIG{INT, TERM, KILL, ...} signal is sent
GunicornMonitor(
- gunicorn_master_proc=gunicorn_master_proc,
+ gunicorn_master_pid=gunicorn_master_pid,
num_workers_expected=num_workers,
master_timeout=conf.getint('webserver',
'web_server_master_timeout'),
worker_refresh_interval=conf.getint('webserver',
'worker_refresh_interval', fallback=30),
@@ -1167,7 +1169,7 @@ def webserver(args):
time.sleep(0.1)
gunicorn_master_proc = psutil.Process(gunicorn_master_proc_pid)
- monitor_gunicorn(gunicorn_master_proc)
+ monitor_gunicorn(gunicorn_master_proc.pid)
stdout.close()
stderr.close()
@@ -1177,7 +1179,7 @@ def webserver(args):
signal.signal(signal.SIGINT, kill_proc)
signal.signal(signal.SIGTERM, kill_proc)
- monitor_gunicorn(gunicorn_master_proc)
+ monitor_gunicorn(gunicorn_master_proc.pid)
@cli_utils.action_logging
diff --git a/requirements/requirements-python2.7.txt
b/requirements/requirements-python2.7.txt
index e1d8154..6973e5a 100644
--- a/requirements/requirements-python2.7.txt
+++ b/requirements/requirements-python2.7.txt
@@ -1,4 +1,4 @@
-# Editable install with no version control (apache-airflow==1.10.10)
+# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==1.13.1
@@ -157,12 +157,12 @@ hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
hvac==0.10.4
-identify==1.4.20
+identify==1.4.21
idna==2.10
ijson==2.6.1
imagesize==1.2.0
importlib-metadata==1.7.0
-importlib-resources==2.0.1
+importlib-resources==3.0.0
inflection==0.3.1
ipaddress==1.0.23
ipdb==0.13.3
@@ -258,7 +258,7 @@ pytest-rerunfailures==9.0
pytest-timeout==1.4.1
pytest-xdist==1.32.0
pytest==4.6.11
-python-daemon==2.1.2
+python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
@@ -310,7 +310,6 @@ sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
-termcolor==1.1.0
testpath==0.4.4
text-unidecode==1.3
textwrap3==0.9.2
diff --git a/requirements/requirements-python3.5.txt
b/requirements/requirements-python3.5.txt
index 20c9b75..ac06c01 100644
--- a/requirements/requirements-python3.5.txt
+++ b/requirements/requirements-python3.5.txt
@@ -1,4 +1,4 @@
-# Editable install with no version control (apache-airflow==1.10.10)
+# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==1.13.1
@@ -143,12 +143,11 @@ hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
hvac==0.10.4
-identify==1.4.20
+identify==1.4.21
idna==2.10
-ijson==2.6.1
imagesize==1.2.0
importlib-metadata==1.7.0
-importlib-resources==2.0.1
+importlib-resources==3.0.0
inflection==0.5.0
ipdb==0.13.3
ipython-genutils==0.2.0
@@ -244,7 +243,7 @@ pytest-rerunfailures==9.0
pytest-timeout==1.4.1
pytest-xdist==1.32.0
pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
@@ -297,7 +296,6 @@ sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
-termcolor==1.1.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
diff --git a/requirements/requirements-python3.6.txt
b/requirements/requirements-python3.6.txt
index 275cd7c..615b800 100644
--- a/requirements/requirements-python3.6.txt
+++ b/requirements/requirements-python3.6.txt
@@ -1,4 +1,4 @@
-# Editable install with no version control (apache-airflow==1.10.10)
+# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
@@ -62,9 +62,9 @@ beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
-boto3==1.14.13
+boto3==1.14.14
boto==2.49.0
-botocore==1.17.13
+botocore==1.17.14
cached-property==1.5.1
cachetools==4.1.1
cassandra-driver==3.20.2
@@ -145,12 +145,11 @@ hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
hvac==0.10.4
-identify==1.4.20
+identify==1.4.21
idna==2.10
-ijson==2.6.1
imagesize==1.2.0
importlib-metadata==1.7.0
-importlib-resources==2.0.1
+importlib-resources==3.0.0
inflection==0.5.0
ipdb==0.13.3
ipython-genutils==0.2.0
@@ -246,7 +245,7 @@ pytest-rerunfailures==9.0
pytest-timeout==1.4.1
pytest-xdist==1.32.0
pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
@@ -301,7 +300,6 @@ sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
-termcolor==1.1.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
diff --git a/requirements/requirements-python3.7.txt
b/requirements/requirements-python3.7.txt
index 91eb574..7e9d467 100644
--- a/requirements/requirements-python3.7.txt
+++ b/requirements/requirements-python3.7.txt
@@ -1,4 +1,4 @@
-# Editable install with no version control (apache-airflow==1.10.10)
+# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
@@ -62,9 +62,9 @@ beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
-boto3==1.14.13
+boto3==1.14.14
boto==2.49.0
-botocore==1.17.13
+botocore==1.17.14
cached-property==1.5.1
cachetools==4.1.1
cassandra-driver==3.20.2
@@ -145,9 +145,8 @@ hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
hvac==0.10.4
-identify==1.4.20
+identify==1.4.21
idna==2.10
-ijson==2.6.1
imagesize==1.2.0
importlib-metadata==1.7.0
inflection==0.5.0
@@ -245,7 +244,7 @@ pytest-rerunfailures==9.0
pytest-timeout==1.4.1
pytest-xdist==1.32.0
pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
@@ -300,7 +299,6 @@ sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
-termcolor==1.1.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
diff --git a/requirements/requirements-python3.8.txt
b/requirements/requirements-python3.8.txt
index 56f161d..747ae42 100644
--- a/requirements/requirements-python3.8.txt
+++ b/requirements/requirements-python3.8.txt
@@ -1,4 +1,4 @@
-# Editable install with no version control (apache-airflow==1.10.10)
+# Editable install with no version control (apache-airflow==1.10.11)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
@@ -145,7 +145,7 @@ hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
hvac==0.10.4
-identify==1.4.20
+identify==1.4.21
idna==2.10
imagesize==1.2.0
importlib-metadata==1.7.0
@@ -243,7 +243,7 @@ pytest-rerunfailures==9.0
pytest-timeout==1.4.1
pytest-xdist==1.32.0
pytest==5.4.3
-python-daemon==2.1.2
+python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
diff --git a/requirements/setup-2.7.md5 b/requirements/setup-2.7.md5
index 7cb2b1c..7302c51 100644
--- a/requirements/setup-2.7.md5
+++ b/requirements/setup-2.7.md5
@@ -1 +1 @@
-9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py
+da591fb5f6ed08129068e227610706cb /opt/airflow/setup.py
diff --git a/requirements/setup-3.5.md5 b/requirements/setup-3.5.md5
index 7cb2b1c..7302c51 100644
--- a/requirements/setup-3.5.md5
+++ b/requirements/setup-3.5.md5
@@ -1 +1 @@
-9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py
+da591fb5f6ed08129068e227610706cb /opt/airflow/setup.py
diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5
index 7cb2b1c..7302c51 100644
--- a/requirements/setup-3.6.md5
+++ b/requirements/setup-3.6.md5
@@ -1 +1 @@
-9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py
+da591fb5f6ed08129068e227610706cb /opt/airflow/setup.py
diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5
index 7cb2b1c..7302c51 100644
--- a/requirements/setup-3.7.md5
+++ b/requirements/setup-3.7.md5
@@ -1 +1 @@
-9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py
+da591fb5f6ed08129068e227610706cb /opt/airflow/setup.py
diff --git a/requirements/setup-3.8.md5 b/requirements/setup-3.8.md5
index 7cb2b1c..7302c51 100644
--- a/requirements/setup-3.8.md5
+++ b/requirements/setup-3.8.md5
@@ -1 +1 @@
-9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py
+da591fb5f6ed08129068e227610706cb /opt/airflow/setup.py
diff --git a/setup.py b/setup.py
index fc14c3c..906c705 100644
--- a/setup.py
+++ b/setup.py
@@ -189,7 +189,7 @@ azure_blob_storage = [
'azure-storage-blob<12.0',
]
azure_container_instances = [
- 'azure-mgmt-containerinstance>=1.5.0'
+ 'azure-mgmt-containerinstance>=1.5.0,<2'
]
azure_cosmos = [
'azure-cosmos>=3.0.1,<4',
@@ -580,7 +580,7 @@ INSTALL_REQUIREMENTS = [
'pendulum==1.4.4',
'psutil>=4.2.0, <6.0.0',
'pygments>=2.0.1, <3.0',
- 'python-daemon>=2.1.1, <2.2',
+ 'python-daemon>=2.1.1',
'python-dateutil>=2.3, <3',
'python-nvd3~=0.15.0',
'python-slugify>=3.0.0,<5.0',
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 5748057..d3e9fe2 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -821,9 +821,8 @@ class TestShowInfo(unittest.TestCase):
class TestGunicornMonitor(unittest.TestCase):
def setUp(self):
- self.gunicorn_master_proc = mock.Mock(pid=2137)
self.monitor = cli.GunicornMonitor(
- gunicorn_master_proc=self.gunicorn_master_proc,
+ gunicorn_master_pid=1,
num_workers_expected=4,
master_timeout=60,
worker_refresh_interval=60,
@@ -922,7 +921,7 @@ class
TestGunicornMonitorGeneratePluginState(unittest.TestCase):
self._prepare_test_file("{}/file3.txt".format(tempdir), 300)
monitor = cli.GunicornMonitor(
- gunicorn_master_proc=mock.MagicMock(),
+ gunicorn_master_pid=1,
num_workers_expected=4,
master_timeout=60,
worker_refresh_interval=60,
@@ -974,7 +973,7 @@ class TestCLIGetNumReadyWorkersRunning(unittest.TestCase):
self.child = mock.MagicMock()
self.process = mock.MagicMock()
self.monitor = cli.GunicornMonitor(
- gunicorn_master_proc=self.gunicorn_master_proc,
+ gunicorn_master_pid=1,
num_workers_expected=4,
master_timeout=60,
worker_refresh_interval=60,
diff --git a/tests/insert_extras.py b/tests/insert_extras.py
index 871f24e..b38b543 100644
--- a/tests/insert_extras.py
+++ b/tests/insert_extras.py
@@ -58,7 +58,7 @@ def insert_documentation(file_path, content, header, footer):
if __name__ == '__main__':
install_file_path = os.path.join(AIRFLOW_SOURCES_DIR, 'INSTALL')
contributing_file_path = os.path.join(AIRFLOW_SOURCES_DIR,
'CONTRIBUTING.rst')
- extras = wrap(", ".join(EXTRAS_REQUIREMENTS.keys()), 100)
+ extras = wrap(", ".join(sorted(EXTRAS_REQUIREMENTS.keys())) , 100)
extras = [line + "\n" for line in extras]
insert_documentation(install_file_path, extras, INSTALL_HEADER,
INSTALL_FOOTER)
insert_documentation(contributing_file_path, extras, RST_HEADER,
RST_FOOTER)
diff --git a/tests/test_core.py b/tests/test_core.py
index 8e55aa5..5530f2a 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -27,7 +27,6 @@ import os
import re
import signal
-import pytest
import sqlalchemy
import subprocess
import tempfile
@@ -38,7 +37,7 @@ from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from numpy.testing import assert_array_almost_equal
from six.moves.urllib.parse import urlencode
-from time import sleep
+from time import sleep, time
from bs4 import BeautifulSoup
@@ -56,11 +55,13 @@ from airflow.operators.dummy_operator import DummyOperator
from airflow.hooks.base_hook import BaseHook
from airflow.hooks.sqlite_hook import SqliteHook
from airflow.bin import cli
+from airflow.utils.file import TemporaryDirectory
from airflow.www import app as application
from airflow.settings import Session
from airflow.utils import timezone
from airflow.utils.timezone import datetime
from airflow.utils.state import State
+from airflow.utils.timeout import timeout
from airflow.utils.dates import days_ago, infer_time_unit, round_time,
scale_time_units
from airflow.exceptions import AirflowException
from airflow.configuration import (
@@ -1649,23 +1650,35 @@ class CliTests(unittest.TestCase):
os.remove('variables2.json')
os.remove('variables3.json')
- def _wait_pidfile(self, pidfile):
- while True:
- try:
- with open(pidfile) as f:
- return int(f.read())
- except Exception:
- sleep(1)
+
+class TestCliWebServer(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.parser = cli.get_parser()
+
+ def setUp(self):
+ self._check_processes()
+ self._clean_pidfiles()
+
+ def tearDown(self):
+ self._check_processes()
+ self._clean_pidfiles()
def _check_processes(self):
- try:
- # Confirm that webserver hasn't been launched.
- # pgrep returns exit status 1 if no process matched.
- self.assertEqual(1, subprocess.Popen(["pgrep", "-f", "-c",
"airflow webserver"]).wait())
- self.assertEqual(1, subprocess.Popen(["pgrep", "-c",
"gunicorn"]).wait())
- except: # noqa: E722
+ # Confirm that webserver hasn't been launched.
+ # pgrep returns exit status 1 if no process matched.
+ exit_code_pgrep_webserver = subprocess.Popen(["pgrep", "-c", "-f",
"airflow webserver"]).wait()
+ exit_code_pgrep_gunicorn = subprocess.Popen(["pgrep", "-c", "-f",
"gunicorn"]).wait()
+ if exit_code_pgrep_webserver != 1 or exit_code_pgrep_gunicorn != 1:
subprocess.Popen(["ps", "-ax"]).wait()
- raise
+ if exit_code_pgrep_webserver != 1:
+ subprocess.Popen(["pkill", "-9", "-f", "airflow
webserver"]).wait()
+ if exit_code_pgrep_gunicorn != 1:
+ subprocess.Popen(["pkill", "-9", "-f", "gunicorn"]).wait()
+
+ raise AssertionError(
+ "Background processes are running that prevent the test from
passing successfully."
+ )
def _clean_pidfiles(self):
pidfile_webserver = cli.setup_locations("webserver")[0]
@@ -1675,70 +1688,106 @@ class CliTests(unittest.TestCase):
if os.path.exists(pidfile_monitor):
os.remove(pidfile_monitor)
- @pytest.mark.quarantined
- def test_cli_webserver_foreground(self):
- self._check_processes()
- self._clean_pidfiles()
- try:
-
- # Confirm that webserver hasn't been launched.
- # pgrep returns exit status 1 if no process matched.
- self.assertEqual(1, subprocess.Popen(["pgrep", "-c",
"airflow"]).wait())
- self.assertEqual(1, subprocess.Popen(["pgrep", "-c",
"gunicorn"]).wait())
+ def _wait_pidfile(self, pidfile):
+ start_time = time()
+ while True:
+ try:
+ with open(pidfile) as file:
+ return int(file.read())
+ except Exception: # pylint: disable=broad-except
+ if start_time - time() > 60:
+ raise
+ sleep(1)
+ def test_cli_webserver_foreground(self):
+ with mock.patch.dict(
+ "os.environ",
+ AIRFLOW__CORE__DAGS_FOLDER="/dev/null",
+ AIRFLOW__CORE__LOAD_EXAMPLES="False",
+ AIRFLOW__WEBSERVER__WORKERS="1"
+ ):
# Run webserver in foreground and terminate it.
- p = subprocess.Popen(["airflow", "webserver"])
- p.terminate()
- p.wait()
-
- # Assert that no process remains.
- self.assertEqual(1, subprocess.Popen(["pgrep", "-c",
"airflow"]).wait())
- self.assertEqual(1, subprocess.Popen(["pgrep", "-c",
"gunicorn"]).wait())
- finally:
- self._check_processes()
-
- @unittest.skipIf("TRAVIS" in os.environ and bool(os.environ["TRAVIS"]),
- "Skipping test due to lack of required file permission")
- @pytest.mark.quarantined
- def test_cli_webserver_foreground_with_pid(self):
- # Run webserver in foreground with --pid option
- pidfile = tempfile.mkstemp()[1]
- p = subprocess.Popen(["airflow", "webserver", "--pid", pidfile])
+ proc = subprocess.Popen(["airflow", "webserver"])
+ self.assertEqual(None, proc.poll())
- # Check the file specified by --pid option exists
- self._wait_pidfile(pidfile)
+ # Wait for process
+ sleep(10)
# Terminate webserver
- p.terminate()
- p.wait()
+ proc.terminate()
+ with timeout(60):
+ # -15 - the server was stopped before it started
+ # 0 - the server terminated correctly
+ self.assertIn(proc.wait(), (-15, 0))
+
+ def test_cli_webserver_foreground_with_pid(self):
+ with TemporaryDirectory(prefix='tmp-pid') as tmpdir:
+ pidfile = "{}/pidfile".format(tmpdir)
+ with mock.patch.dict(
+ "os.environ",
+ AIRFLOW__CORE__DAGS_FOLDER="/dev/null",
+ AIRFLOW__CORE__LOAD_EXAMPLES="False",
+ AIRFLOW__WEBSERVER__WORKERS="1"
+ ):
+ proc = subprocess.Popen(["airflow", "webserver", "--pid",
pidfile])
+ self.assertEqual(None, proc.poll())
+
+ # Check the file specified by --pid option exists
+ self._wait_pidfile(pidfile)
+
+ # Terminate webserver
+ proc.terminate()
+ with timeout(60):
+ self.assertEqual(0, proc.wait())
- @unittest.skipIf("TRAVIS" in os.environ and bool(os.environ["TRAVIS"]),
- "Skipping test due to lack of required file permission")
- @pytest.mark.quarantined
def test_cli_webserver_background(self):
import psutil
- self._check_processes()
- self._clean_pidfiles()
- try:
- pidfile_webserver = cli.setup_locations("webserver")[0]
- pidfile_monitor = cli.setup_locations("webserver-monitor")[0]
-
- # Run webserver as daemon in background. Note that the wait method
is not called.
- subprocess.Popen(["airflow", "webserver", "-D"])
-
- pid_monitor = self._wait_pidfile(pidfile_monitor)
- self._wait_pidfile(pidfile_webserver)
-
- # Assert that gunicorn and its monitor are launched.
- self.assertEqual(0, subprocess.Popen(["pgrep", "-f", "-c",
"airflow webserver"]).wait())
- self.assertEqual(0, subprocess.Popen(["pgrep", "-c",
"gunicorn"]).wait())
-
- # Terminate monitor process.
- proc = psutil.Process(pid_monitor)
- proc.terminate()
- proc.wait()
- finally:
- self._check_processes()
+ with TemporaryDirectory(prefix="gunicorn") as tmpdir, \
+ mock.patch.dict(
+ "os.environ",
+ AIRFLOW__CORE__DAGS_FOLDER="/dev/null",
+ AIRFLOW__CORE__LOAD_EXAMPLES="False",
+ AIRFLOW__WEBSERVER__WORKERS="1"):
+ pidfile_webserver = "{}/pidflow-webserver.pid".format(tmpdir)
+ pidfile_monitor = "{}/pidflow-webserver-monitor.pid".format(tmpdir)
+ stdout = "{}/airflow-webserver.out".format(tmpdir)
+ stderr = "{}/airflow-webserver.err".format(tmpdir)
+ logfile = "{}/airflow-webserver.log".format(tmpdir)
+ try:
+ # Run webserver as daemon in background. Note that the wait
method is not called.
+ proc = subprocess.Popen([
+ "airflow",
+ "webserver",
+ "--daemon",
+ "--pid", pidfile_webserver,
+ "--stdout", stdout,
+ "--stderr", stderr,
+ "--log-file", logfile,
+ ])
+ self.assertEqual(None, proc.poll())
+
+ pid_monitor = self._wait_pidfile(pidfile_monitor)
+ self._wait_pidfile(pidfile_webserver)
+
+ # Assert that gunicorn and its monitor are launched.
+ self.assertEqual(
+ 0, subprocess.Popen(["pgrep", "-f", "-c", "airflow
webserver --daemon"]).wait()
+ )
+ self.assertEqual(0, subprocess.Popen(["pgrep", "-c", "-f",
"gunicorn: master"]).wait())
+
+ # Terminate monitor process.
+ proc = psutil.Process(pid_monitor)
+ proc.terminate()
+ with timeout(120):
+ self.assertIn(proc.wait(), (0, None))
+
+ self._check_processes()
+ except Exception:
+ # List all logs
+ subprocess.Popen(["ls", "-lah", tmpdir]).wait()
+ # Dump all logs
+ subprocess.Popen(["bash", "-c", "ls {}/* | xargs -n 1 -t
cat".format(tmpdir)]).wait()
+ raise
# Patch for causing webserver timeout
@mock.patch("airflow.bin.cli.GunicornMonitor._get_num_workers_running",
return_value=0)
