potiuk opened a new issue, #35865: URL: https://github.com/apache/airflow/issues/35865
### Body Sometimes (rather rarely) the "check_processes" in `tests/cli/commands/test_webserver_command.py::TestCliWebServer::test_cli_webserver_background` detects some remaining processes. This is likely a side effect of other test. Example failure: https://github.com/ache/airflow/actions/runs/6996129264/job/19031743272?pr=35862 ``` ________________ TestCliWebServer.test_cli_webserver_background ________________ self = <tests.cli.commands.test_webserver_command.TestCliWebServer object at 0x7f2352c76e20> tmp_path = PosixPath('/tmp/pytest-of-root/pytest-0/test_cli_webserver_background0') @pytest.mark.execution_timeout(210) def test_cli_webserver_background(self, tmp_path): with mock.patch.dict( "os.environ", AIRFLOW__CORE__DAGS_FOLDER="/dev/null", AIRFLOW__CORE__LOAD_EXAMPLES="False", AIRFLOW__WEBSERVER__WORKERS="1", ): pidfile_webserver = tmp_path / "pidflow-webserver.pid" pidfile_monitor = tmp_path / "pidflow-webserver-monitor.pid" stdout = tmp_path / "airflow-webserver.out" stderr = tmp_path / "airflow-webserver.err" logfile = tmp_path / "airflow-webserver.log" try: # Run webserver as daemon in background. Note that the wait method is not called. proc = subprocess.Popen( [ "airflow", "webserver", "--daemon", "--pid", os.fspath(pidfile_webserver), "--stdout", os.fspath(stdout), "--stderr", os.fspath(stderr), "--log-file", os.fspath(logfile), ] ) assert proc.poll() is None pid_monitor = self._wait_pidfile(pidfile_monitor) console.print(f"[blue]Monitor started at {pid_monitor}") pid_webserver = self._wait_pidfile(pidfile_webserver) console.print(f"[blue]Webserver started at {pid_webserver}") console.print("[blue]Running airflow webserver process:") # Assert that the webserver and gunicorn processes are running (by name rather than pid). assert self._find_process(r"airflow webserver", print_found_process=True) console.print("[blue]Waiting for gunicorn processes:") # wait for gunicorn to start for i in range(30): if self._find_process(r"^gunicorn"): > raise AssertionError( "Background processes are running that prevent the test from passing successfully." ) E AssertionError: Background processes are running that prevent the test from passing successfully. tests/cli/commands/_common_cli_classes.py:72: AssertionError ----------------------------- Captured stdout call ----------------------------- Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... ____________ _____________ ____ |__( )_________ __/__ /________ __ ____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / / ___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ / _/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/ Running the Gunicorn Server with: Workers: 1 sync Host: 0.0.0.0:8080 Timeout: 120 Logfiles: - - Access Logformat: ================================================================= [2023-11-26T14:43:32.521+0000] {dagbag.py:538} INFO - Filling up the DagBag from /dev/null Please make sure to build the frontend in static/ directory and restart the server [2023-11-26T14:43:32.822+0000] {override.py:872} WARNING - No user yet created, use flask fab command to do it. Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... [2023-11-26T14:43:36.046+0000] {options.py:83} WARNING - The swagger_ui directory could not be found. Please install connexion with extra install: pip install connexion[swagger-ui] or provide the path to your local installation by passing swagger_path=<your path> [2023-11-26T14:43:36.046+0000] {options.py:83} WARNING - The swagger_ui directory could not be found. Please install connexion with extra install: pip install connexion[swagger-ui] or provide the path to your local installation by passing swagger_path=<your path> Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver-monitor.pid to be created ... Monitor started at 254 Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Waiting for pidfile /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid to be created ... Webserver started at 258 Running airflow webserver process: { 'cmdline': [ '/usr/local/bin/python', '/usr/local/bin/airflow', 'webserver', '--daemon', '--pid', '/tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/pidflow-webserver.pid', '--stdout', '/tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/airflow-webserver.out', '--stderr', '/tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/airflow-webserver.err', '--log-file', '/tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/airflow-webserver.log' ], 'pid': 254, 'name': 'airflow' } Waiting for gunicorn processes: Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Waiting for gunicorn to start ... Running gunicorn processes: { 'cmdline': [ 'gunicorn: master [airflow-webserver]', '', '', '', '', '', ........... LONG LIST HERE ........ '' ], 'pid': 258, 'name': 'gunicorn: master [airflow-webserver]' } Here list of processes ends Forcefully killing all gunicorn processes Exception occurred. Dumping all logs. Dumping /tmp/pytest-of-root/pytest-0/test_cli_webserver_background0/airflow-webserver.err (size: 2775) Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/psutil/_psposix.py", line 121, in wait_pid retpid, status = os.waitpid(pid, flags) ChildProcessError: [Errno 10] No child processes During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/bin/airflow", line 33, in <module> sys.exit(load_entry_point('apache-airflow', 'console_scripts', 'airflow')()) File "/opt/airflow/airflow/__main__.py", line 57, in main args.func(args) File "/opt/airflow/airflow/cli/cli_config.py", line 49, in command return func(*args, **kwargs) File "/opt/airflow/airflow/utils/cli.py", line 114, in wrapper return f(*args, **kwargs) File "/opt/airflow/airflow/utils/providers_configuration_loader.py", line 55, in wrapped_function return func(*args, **kwargs) File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 484, in webserver run_command_with_daemon_option( File "/opt/airflow/airflow/cli/commands/daemon_utils.py", line 81, in run_command_with_daemon_option callback() File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 487, in <lambda> callback=lambda: start_and_monitor_gunicorn(args), File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 471, in start_and_monitor_gunicorn monitor_gunicorn(gunicorn_master_proc) File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 447, in monitor_gunicorn GunicornMonitor( File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 217, in start self._check_workers() File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 241, in _check_workers sleep(1) File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 444, in <lambda> signal.signal(signal.SIGTERM, lambda signum, _: kill_proc(signum, gunicorn_master_proc)) File "/opt/airflow/airflow/cli/commands/webserver_command.py", line 432, in kill_proc gunicorn_master_proc.wait(timeout=30) File "/usr/local/lib/python3.8/site-packages/psutil/__init__.py", line 1294, in wait self._exitcode = self._proc.wait(timeout) File "/usr/local/lib/python3.8/site-packages/psutil/_pslinux.py", line 1650, in wrapper return fun(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/psutil/_pslinux.py", line 1873, in wait return _psposix.wait_pid(self.pid, timeout, self._name) File "/usr/local/lib/python3.8/site-packages/psutil/_psposix.py", line 132, in wait_pid interval = sleep(interval) File "/usr/local/lib/python3.8/site-packages/psutil/_psposix.py", line 114, in sleep raise TimeoutExpired(timeout, pid=pid, name=proc_name) psutil.TimeoutExpired: timeout after 30 seconds (pid=258) ``` ### Committer - [X] I acknowledge that I am a maintainer/committer of the Apache Airflow project. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
