Tyhoning commented on issue #53797:
URL: https://github.com/apache/airflow/issues/53797#issuecomment-3199973726
> ### Apache Airflow version
> 3.0.4
>
> ### I encountered the same issue while using version 3.0.4
>
> ### What happened?
> I can run 【 airflow dags test sleep_dag 】 successfully using the command
line, but running this DAG on the web UI will failed.
>
> Dag Code
from airflow.sdk import dag
from airflow.providers.standard.operators.bash import BashOperator
from datetime import datetime
@dag(start_date=datetime(2025, 1, 1), schedule="@hourly", catchup=False)
def sleep_dag():
t1 = BashOperator(
task_id="sleep_1_minutes",
bash_command="echo 10101010",
)
sleep_dag().
>
> airflow-scheduler.log
> ```
httpcore.ConnectError: [Errno 111] Connection refused
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File
"/root/af_env/lib64/python3.9/site-packages/airflow/executors/local_executor.py",
line 92, in _run_worker
_execute_work(log, workload)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/executors/local_executor.py",
line 120, in _execute_work
supervise(
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 1793, in supervise
process = ActivitySubprocess.start(
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 854, in start
proc._on_child_started(ti=what, dag_rel_path=dag_rel_path,
bundle_info=bundle_info)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 865, in _on_child_started
ti_context = self.client.task_instances.start(ti.id, self.pid,
start_date)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/api/client.py", line
151, in start
resp = self.client.patch(f"task-instances/{id}/run",
content=body.model_dump_json())
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
1218, in patch
return self.request(
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 336, in wrapped_f
return copy(f, *args, **kw)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 475, in __call__
do = self.iter(retry_state=retry_state)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 376, in iter
result = action(retry_state)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 418, in exc_check
raise retry_exc.reraise()
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 185, in reraise
raise self.last_attempt.result()
File "/usr/lib64/python3.9/concurrent/futures/_base.py", line 439, in
result
return self.__get_result()
File "/usr/lib64/python3.9/concurrent/futures/_base.py", line 391, in
__get_result
raise self._exception
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 478, in __call__
result = fn(*args, **kwargs)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/api/client.py", line
714, in request
return super().request(*args, **kwargs)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
825, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
914, in send
response = self._send_handling_auth(
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
942, in _send_handling_auth
response = self._send_handling_redirects(
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
979, in _send_handling_redirects
response = self._send_single_request(request)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
1014, in _send_single_request
response = transport.handle_request(request)
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
250, in handle_request
resp = self._pool.handle_request(req)
File "/usr/lib64/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
118, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError: [Errno 111] Connection refused
2025-08-19 09:00:03,908 INFO - Received executor event with state failed for
task instance TaskInstanceKey(dag_id='sleep_dag', task_id='sleep_1_minutes',
run_id='scheduled__2025-08-19T01:00:00+00:00', try_number=1, map_index=-1)
2025-08-19 09:00:03,911 INFO - TaskInstance Finished: dag_id=sleep_dag,
task_id=sleep_1_minutes, run_id=scheduled__2025-08-19T01:00:00+00:00,
map_index=-1, run_start_date=None, run_end_date=None, run_duration=None,
state=queued, executor=LocalExecutor(parallelism=1), executor_state=failed,
try_number=1, max_tries=0, pool=default_pool, queue=default, priority_weight=1,
operator=BashOperator, queued_dttm=2025-08-19 01:00:00.247432+00:00,
scheduled_dttm=2025-08-19 01:00:00.235021+00:00,queued_by_job_id=6, pid=None
2025-08-19 09:00:03,913 ERROR - DAG 'sleep_dag' for task instance
<TaskInstance: sleep_dag.sleep_1_minutes scheduled__2025-08-19T01:00:00+00:00
[queued]> not found in serialized_dag table
2025-08-19 09:00:03,913 ERROR - Executor LocalExecutor(parallelism=1)
reported that the task instance <TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T01:00:00+00:00 [queued]> finished with state failed, but
the task instance's state attribute is queued. Learn more:
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally
2025-08-19 09:00:03,918 INFO - Marking task as FAILED. dag_id=sleep_dag,
task_id=sleep_1_minutes, run_id=scheduled__2025-08-19T01:00:00+00:00,
logical_date=20250819T010000, start_date=, end_date=20250819T010003
2025-08-19 09:00:04,943 INFO - Marking run <DagRun sleep_dag @ 2025-08-19
01:00:00+00:00: scheduled__2025-08-19T01:00:00+00:00, state:running, queued_at:
2025-08-19 01:00:00.203613+00:00. run_type: scheduled> failed
2025-08-19 09:00:04,944 INFO - DagRun Finished: dag_id=sleep_dag,
logical_date=2025-08-19 01:00:00+00:00,
run_id=scheduled__2025-08-19T01:00:00+00:00, run_start_date=2025-08-19
01:00:00.219100+00:00, run_end_date=2025-08-19 01:00:04.944064+00:00,
run_duration=4.724964, state=failed, run_type=scheduled,
data_interval_start=2025-08-19 01:00:00+00:00, data_interval_end=2025-08-19
01:00:00+00:00,
2025-08-19 09:00:04,948 INFO - Setting next_dagrun for sleep_dag to
2025-08-19 02:00:00+00:00, run_after=2025-08-19 02:00:00+00:00
2025-08-19 09:00:16,141 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:04:59,001 INFO - checking for stale bundle versions locally
2025-08-19 09:04:59,002 INFO - DAG bundles loaded: dags-folder, example_dags
2025-08-19 09:05:16,162 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:10:16,182 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:15:16,202 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:20:16,222 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:25:16,241 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:30:16,262 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:34:59,019 INFO - checking for stale bundle versions locally
2025-08-19 09:34:59,020 INFO - DAG bundles loaded: dags-folder, example_dags
2025-08-19 09:35:16,282 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:40:16,301 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:45:16,322 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:50:16,341 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 09:55:16,361 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:00:01,002 INFO - Setting next_dagrun for sleep_dag to
2025-08-19 03:00:00+00:00, run_after=2025-08-19 03:00:00+00:00
2025-08-19 10:00:01,041 INFO - 1 tasks up for execution:
<TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T02:00:00+00:00 [scheduled]>
2025-08-19 10:00:01,042 INFO - DAG sleep_dag has 0/16 running and queued
tasks
2025-08-19 10:00:01,042 INFO - Setting the following tasks to queued state:
<TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T02:00:00+00:00 [scheduled]>
2025-08-19 10:00:01,044 INFO - Trying to enqueue tasks: [<TaskInstance:
sleep_dag.sleep_1_minutes scheduled__2025-08-19T02:00:00+00:00 [scheduled]>]
for executor: LocalExecutor(parallelism=1)
2025-08-19 10:00:10,216 ERROR - uhoh
Traceback (most recent call last):
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
101, in map_httpcore_exceptions
yield
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
250, in handle_request
resp = self._pool.handle_request(req)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection_pool.py",
line 256, in handle_request
raise exc from None
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection_pool.py",
line 236, in handle_request
response = connection.handle_request(
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
101, in handle_request
raise exc
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
78, in handle_request
stream = self._connect(request)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
124, in _connect
stream = self._network_backend.connect_tcp(**kwargs)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_backends/sync.py", line
215, in connect_tcp
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
File "/usr/lib64/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/af_env/lib64/python3.9/site-packages/httpcore/_exceptions.py",
line 14, in map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectError: [Errno 111] Connection refused
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File
"/root/af_env/lib64/python3.9/site-packages/airflow/executors/local_executor.py",
line 92, in _run_worker
_execute_work(log, workload)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/executors/local_executor.py",
line 120, in _execute_work
supervise(
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 1793, in supervise
process = ActivitySubprocess.start(
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 854, in start
proc._on_child_started(ti=what, dag_rel_path=dag_rel_path,
bundle_info=bundle_info)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/execution_time/supervisor.py",
line 865, in _on_child_started
ti_context = self.client.task_instances.start(ti.id, self.pid,
start_date)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/api/client.py", line
151, in start
resp = self.client.patch(f"task-instances/{id}/run",
content=body.model_dump_json())
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
1218, in patch
return self.request(
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 336, in wrapped_f
return copy(f, *args, **kw)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 475, in __call__
do = self.iter(retry_state=retry_state)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 376, in iter
result = action(retry_state)
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 418, in exc_check
raise retry_exc.reraise()
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 185, in reraise
raise self.last_attempt.result()
File "/usr/lib64/python3.9/concurrent/futures/_base.py", line 439, in
result
return self.__get_result()
File "/usr/lib64/python3.9/concurrent/futures/_base.py", line 391, in
__get_result
raise self._exception
File "/root/af_env/lib64/python3.9/site-packages/tenacity/__init__.py",
line 478, in __call__
result = fn(*args, **kwargs)
File
"/root/af_env/lib64/python3.9/site-packages/airflow/sdk/api/client.py", line
714, in request
return super().request(*args, **kwargs)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
825, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
914, in send
response = self._send_handling_auth(
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
942, in _send_handling_auth
response = self._send_handling_redirects(
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
979, in _send_handling_redirects
response = self._send_single_request(request)
File "/root/af_env/lib64/python3.9/site-packages/httpx/_client.py", line
1014, in _send_single_request
response = transport.handle_request(request)
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
250, in handle_request
resp = self._pool.handle_request(req)
File "/usr/lib64/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
118, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError: [Errno 111] Connection refused
2025-08-19 10:00:11,087 INFO - Received executor event with state failed for
task instance TaskInstanceKey(dag_id='sleep_dag', task_id='sleep_1_minutes',
run_id='scheduled__2025-08-19T02:00:00+00:00', try_number=1, map_index=-1)
2025-08-19 10:00:11,090 INFO - TaskInstance Finished: dag_id=sleep_dag,
task_id=sleep_1_minutes, run_id=scheduled__2025-08-19T02:00:00+00:00,
map_index=-1, run_start_date=None, run_end_date=None, run_duration=None,
state=queued, executor=LocalExecutor(parallelism=1), executor_state=failed,
try_number=1, max_tries=0, pool=default_pool, queue=default, priority_weight=1,
operator=BashOperator, queued_dttm=2025-08-19 02:00:01.043077+00:00,
scheduled_dttm=2025-08-19 02:00:01.030819+00:00,queued_by_job_id=6, pid=None
2025-08-19 10:00:11,092 ERROR - DAG 'sleep_dag' for task instance
<TaskInstance: sleep_dag.sleep_1_minutes scheduled__2025-08-19T02:00:00+00:00
[queued]> not found in serialized_dag table
2025-08-19 10:00:11,092 ERROR - Executor LocalExecutor(parallelism=1)
reported that the task instance <TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T02:00:00+00:00 [queued]> finished with state failed, but
the task instance's state attribute is queued. Learn more:
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally
2025-08-19 10:00:11,096 INFO - Marking task as FAILED. dag_id=sleep_dag,
task_id=sleep_1_minutes, run_id=scheduled__2025-08-19T02:00:00+00:00,
logical_date=20250819T020000, start_date=, end_date=20250819T020011
2025-08-19 10:00:12,121 INFO - Marking run <DagRun sleep_dag @ 2025-08-19
02:00:00+00:00: scheduled__2025-08-19T02:00:00+00:00, state:running, queued_at:
2025-08-19 02:00:00.996972+00:00. run_type: scheduled> failed
2025-08-19 10:00:12,122 INFO - DagRun Finished: dag_id=sleep_dag,
logical_date=2025-08-19 02:00:00+00:00,
run_id=scheduled__2025-08-19T02:00:00+00:00, run_start_date=2025-08-19
02:00:01.013940+00:00, run_end_date=2025-08-19 02:00:12.122298+00:00,
run_duration=11.108358, state=failed, run_type=scheduled,
data_interval_start=2025-08-19 02:00:00+00:00, data_interval_end=2025-08-19
02:00:00+00:00,
2025-08-19 10:00:12,126 INFO - Setting next_dagrun for sleep_dag to
2025-08-19 03:00:00+00:00, run_after=2025-08-19 03:00:00+00:00
2025-08-19 10:00:16,381 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:04:59,038 INFO - checking for stale bundle versions locally
2025-08-19 10:04:59,039 INFO - DAG bundles loaded: dags-folder, example_dags
2025-08-19 10:05:16,402 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:10:16,421 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:15:16,439 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:20:16,460 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:25:16,480 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:30:16,500 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:34:59,058 INFO - checking for stale bundle versions locally
2025-08-19 10:34:59,059 INFO - DAG bundles loaded: dags-folder, example_dags
2025-08-19 10:35:16,520 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:40:16,540 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:45:16,560 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:50:16,578 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 10:55:16,599 INFO - Adopting or resetting orphaned tasks for
active dag runs
2025-08-19 11:00:00,340 INFO - Setting next_dagrun for sleep_dag to
2025-08-19 04:00:00+00:00, run_after=2025-08-19 04:00:00+00:00
2025-08-19 11:00:00,379 INFO - 1 tasks up for execution:
<TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T03:00:00+00:00 [scheduled]>
2025-08-19 11:00:00,380 INFO - DAG sleep_dag has 0/16 running and queued
tasks
2025-08-19 11:00:00,380 INFO - Setting the following tasks to queued state:
<TaskInstance: sleep_dag.sleep_1_minutes
scheduled__2025-08-19T03:00:00+00:00 [scheduled]>
2025-08-19 11:00:00,385 INFO - Trying to enqueue tasks: [<TaskInstance:
sleep_dag.sleep_1_minutes scheduled__2025-08-19T03:00:00+00:00 [scheduled]>]
for executor: LocalExecutor(parallelism=1)
2025-08-19 11:00:10,552 ERROR - uhoh
Traceback (most recent call last):
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
101, in map_httpcore_exceptions
yield
File
"/root/af_env/lib64/python3.9/site-packages/httpx/_transports/default.py", line
250, in handle_request
resp = self._pool.handle_request(req)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection_pool.py",
line 256, in handle_request
raise exc from None
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection_pool.py",
line 236, in handle_request
response = connection.handle_request(
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
101, in handle_request
raise exc
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
78, in handle_request
stream = self._connect(request)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_sync/connection.py", line
124, in _connect
stream = self._network_backend.connect_tcp(**kwargs)
File
"/root/af_env/lib64/python3.9/site-packages/httpcore/_backends/sync.py", line
215, in connect_tcp
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
File "/usr/lib64/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/af_env/lib64/python3.9/site-packages/httpcore/_exceptions.py",
line 14, in map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectError: [Errno 111] Connection refused
> ```
>
>Web UI will encounter state mismatch error
>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
