love1angel commented on issue #58708:
URL: https://github.com/apache/airflow/issues/58708#issuecomment-3580801394
the whole trace here:
```
scheduler | 2025-11-26T10:59:10.311027Z [info ] Process exited
[supervisor] exit_code=<Negsignal.SIGKILL: -9> loc=supervisor.py:709
pid=79378 signal_sent=SIGKILL
scheduler | 2025-11-26T10:59:10.313251Z [info ] Process exited
[supervisor] exit_code=<Negsignal.SIGKILL: -9> loc=supervisor.py:709
pid=79379 signal_sent=SIGKILL
scheduler | 2025-11-26T10:59:10.311592Z [error ] uhoh
[airflow.executors.local_executor.LocalExecutor]
loc=local_executor.py:100
scheduler | Traceback (most recent call last):
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/executors/local_executor.py",
line 96, in _run_worker
scheduler | _execute_work(log, workload)
scheduler | ~~~~~~~~~~~~~^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/executors/local_executor.py",
line 124, in _execute_work
scheduler | supervise(
scheduler | ~~~~~~~~~^
scheduler | # This is the "wrong" ti type, but it duck types the same.
TODO: Create a protocol for this.
scheduler |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | ...<5 lines>...
scheduler | log_path=workload.log_path,
scheduler | ^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | )
scheduler | ^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/execution_time/supervisor.py",
line 1940, in supervise
scheduler | process = ActivitySubprocess.start(
scheduler | dag_rel_path=dag_rel_path,
scheduler | ...<4 lines>...
scheduler | subprocess_logs_to_stdout=subprocess_logs_to_stdout,
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/execution_time/supervisor.py",
line 954, in start
scheduler | return self.request(
scheduler | ~~~~~~~~~~~~^
scheduler | "PATCH",
scheduler | ^^^^^^^^
scheduler | ...<11 lines>...
scheduler | extensions=extensions,
scheduler | ^^^^^^^^^^^^^^^^^^^^^^
scheduler | )
scheduler | ^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 338, in wrapped_f
scheduler | return copy(f, *args, **kw)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 477, in __call__
scheduler | do = self.iter(retry_state=retry_state)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 378, in iter
scheduler | result = action(retry_state)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 400, in <lambda>
scheduler | self._add_action_func(lambda rs: rs.outcome.result())
scheduler | ~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/concurrent/futures/_base.py",
line 449, in result
scheduler | return self.__get_result()
scheduler | ~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/concurrent/futures/_base.py",
line 401, in __get_result
scheduler | raise self._exception
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 480, in __call__
scheduler | result = fn(*args, **kwargs)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/api/client.py",
line 885, in request
scheduler | return super().request(*args, **kwargs)
scheduler | ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 825, in request
scheduler | return self.send(request, auth=auth,
follow_redirects=follow_redirects)
scheduler | ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 914, in send
scheduler | response = self._send_handling_auth(
scheduler | request,
scheduler | ...<2 lines>...
scheduler | history=[],
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 942, in _send_handling_auth
scheduler | response = self._send_handling_redirects(
scheduler | request,
scheduler | follow_redirects=follow_redirects,
scheduler | history=history,
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 999, in _send_handling_redirects
scheduler | raise exc
scheduler | File "/home/xp/miniconda3/envs/airflow/lib/p
scheduler | ~~~~~~~~~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 999, in _send_handling_redirects
scheduler | raise exc
scheduler | File "/home/xp/miniconda3/envs/airflow/lib/p
scheduler | ~~~~~~~~~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_models.py",
line 829, in raise_for_status
scheduler | raise HTTPStatusError(message, request=request, response=self)
scheduler | httpx.HTTPStatusError: Client error '403 Forbidden' for url
'[http://localhost:8080/execution/task-instances/019abf97-326c-775f-9a7b-0218bb1df549/run'](http://localhost:8080/execution/task-instances/019abf97-326c-775f-9a7b-0218bb1df549/run%27)
scheduler | For more information check:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403
scheduler | Correlation-id=019abfd1-355a-77de-9b58-38855eb8cd38
scheduler | 2025-11-26T10:59:10.313705Z [error ] uhoh
[airflow.executors.local_executor.LocalExecutor]
loc=local_executor.py:100
scheduler | Traceback (most recent call last):
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/executors/local_executor.py",
line 96, in _run_worker
scheduler | _execute_work(log, workload)
scheduler | ~~~~~~~~~~~~~^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/executors/local_executor.py",
line 124, in _execute_work
scheduler | supervise(
scheduler | ~~~~~~~~~^
scheduler | # This is the "wrong" ti type, but it duck types the same.
TODO: Create a protocol for this.
scheduler |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | ...<5 lines>...
scheduler | log_path=workload.log_path,
scheduler | ^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | )
scheduler | ^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/execution_time/supervisor.py",
line 1940, in supervise
scheduler | process = ActivitySubprocess.start(
scheduler | dag_rel_path=dag_rel_path,
scheduler | ...<4 lines>...
scheduler | subprocess_logs_to_stdout=subprocess_logs_to_stdout,
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/execution_time/supervisor.py",
line 954, in start
scheduler | proc._on_child_started(ti=what, dag_rel_path=dag_rel_path,
bundle_info=bundle_info)
scheduler |
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/execution_time/supervisor.py",
line 965, in _on_child_started
scheduler | ti_context = self.client.task_instances.start(ti.id, self.pid,
start_date)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/api/client.py",
line 215, in start
scheduler | resp = self.client.patch(f"task-instances/{id}/run",
content=body.model_dump_json())
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 1218, in patch
scheduler | return self.request(
scheduler | ~~~~~~~~~~~~^
scheduler | "PATCH",
scheduler | ^^^^^^^^
scheduler | ...<11 lines>...
scheduler | extensions=extensions,
scheduler | ^^^^^^^^^^^^^^^^^^^^^^
scheduler | )
scheduler | ^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 338, in wrapped_f
scheduler | return copy(f, *args, **k
scheduler | do = self.iter(retry_state=retry_state)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 378, in iter
scheduler | result = action(retry_state)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 400, in <lambda>
scheduler | self._add_action_func(lambda rs: rs.outcome.result())
scheduler | ~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/concurrent/futures/_base.py",
line 449, in result
scheduler | return self.__get_result()
scheduler | ~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/concurrent/futures/_base.py",
line 401, in __get_result
scheduler | raise self._exception
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/tenacity/__init__.py",
line 480, in __call__
scheduler | result = fn(*args, **kwargs)
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/api/client.py",
line 885, in request
scheduler | return super().request(*args, **kwargs)
scheduler | ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 825, in request
scheduler | return self.send(request, auth=auth,
follow_redirects=follow_redirects)
scheduler | ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 914, in send
scheduler | response = self._send_handling_auth(
scheduler | request,
scheduler | ...<2 lines>...
scheduler | history=[],
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 942, in _send_handling_auth
scheduler | response = self._send_handling_redirects(
scheduler | request,
scheduler | follow_redirects=follow_redirects,
scheduler | history=history,
scheduler | )
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 999, in _send_handling_redirects
scheduler | raise exc
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_client.py",
line 982, in _send_handling_redirects
scheduler | hook(response)
scheduler | ~~~~^^^^^^^^^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/airflow/sdk/api/client.py",
line 186, in raise_on_4xx_5xx_with_note
scheduler | return get_json_error(response) or response.raise_for_status()
scheduler | ~~~~~~~~~~~~~~~~~~~~~~~~~^^
scheduler | File
"/home/xp/miniconda3/envs/airflow/lib/python3.13/site-packages/httpx/_models.py",
line 829, in raise_for_status
scheduler | raise HTTPStatusError(message, request=request, response=self)
scheduler | httpx.HTTPStatusError: Client error '403 Forbidden' for url
'[http://localhost:8080/execution/task-instances/019abf97-326d-772e-8bf5-34ffadb9c640/run'](http://localhost:8080/execution/task-instances/019abf97-326d-772e-8bf5-34ffadb9c640/run%27)
scheduler | For more information check:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403
scheduler | Correlation-id=019abfd1-355e-7d89-8048-0359aab549d7
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
