inkerinmaa opened a new issue, #61761:
URL: https://github.com/apache/airflow/issues/61761
### Apache Airflow version
3.1.7
### If "Other Airflow 3 version" selected, which one?
_No response_
### What happened?
API Server errors continue to appear in Airflow 3.1.7 as well. K3S
deployement via helm chart. Stack:
`INFO: 10.42.0.182:34248 - "GET / HTTP/1.1" 500 Internal Server Error
ERROR: Exception in ASGI application
+ Exception Group Traceback (most recent call last):
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/_utils.py", line
79, in collapse_excgroups
| yield
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/base.py",
line 183, in __call__
| async with anyio.create_task_group() as task_group:
| ^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/anyio/_backends/_asyncio.py",
line 783, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File
"/home/airflow/.local/lib/python3.12/site-packages/uvicorn/protocols/http/httptools_impl.py",
line 416, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/fastapi/applications.py",
line 1082, in __call__
| await super().__call__(scope, receive, send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/applications.py",
line 113, in __call__
| await self.middleware_stack(scope, receive, send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/errors.py",
line 186, in __call__
| raise exc
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/errors.py",
line 164, in __call__
| await self.app(scope, receive, _send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 29, in __call__
| await responder(scope, receive, send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 130, in __call__
| await super().__call__(scope, receive, send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 46, in __call__
| await self.app(scope, receive, self.send_with_compression)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/cors.py",
line 85, in __call__
| await self.app(scope, receive, send)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/base.py",
line 182, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| ^^^^^^^^^^^^^^^^^^^^
| File "/usr/python/lib/python3.12/contextlib.py", line 158, in
__exit__
| self.gen.throw(value)
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/_utils.py", line
85, in collapse_excgroups
| raise exc
| File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/base.py",
line 184, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/middlewares/refresh_token.py",
line 49, in dispatch
| new_user, current_user = await self._refresh_user(current_token)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/middlewares/refresh_token.py",
line 80, in _refresh_user
| user = await resolve_user_from_token(current_token)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/core_api/security.py",
line 100, in resolve_user_from_token
| return await get_auth_manager().get_user_from_token(token_str)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/managers/base_auth_manager.py",
line 111, in get_user_from_token
| return self.deserialize_user(payload)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/cachetools/_cachedmethod.py",
line 375, in __call__
| return wrapper(self._obj, *args, **kwargs)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/cachetools/_cachedmethod.py",
line 358, in wrapper
| v = method(self, *args, **kwargs)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/providers/fab/auth_manager/fab_auth_manager.py",
line 267, in deserialize_user
| return self.session.scalars(select(User).where(User.id ==
int(token["sub"]))).one()
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/scoping.py",
line 1891, in scalars
| return self._proxied.scalars(
| ^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/session.py",
line 2459, in scalars
| return self._execute_internal(
| ^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/session.py",
line 2249, in _execute_internal
| result: Result[Any] = compile_state_cls.orm_execute_statement(
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/context.py",
line 306, in orm_execute_statement
| result = conn.execute(
| ^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1419, in execute
| return meth(
| ^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/sql/elements.py",
line 527, in _execute_on_connection
| return connection._execute_clauseelement(
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1641, in _execute_clauseelement
| ret = self._execute_context(
| ^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1813, in _execute_context
| conn = self._revalidate_connection()
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 673, in _revalidate_connection
| self._invalid_transaction()
| File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 663, in _invalid_transaction
| raise exc.PendingRollbackError(
| sqlalchemy.exc.PendingRollbackError: Can't reconnect until invalid
transaction is rolled back. Please rollback() fully before proceeding
(Background on this error at: https://sqlalche.me/e/20/8s2b)
+------------------------------------
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/home/airflow/.local/lib/python3.12/site-packages/uvicorn/protocols/http/httptools_impl.py",
line 416, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/fastapi/applications.py",
line 1082, in __call__
await super().__call__(scope, receive, send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/applications.py",
line 113, in __call__
await self.middleware_stack(scope, receive, send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/errors.py",
line 186, in __call__
raise exc
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/errors.py",
line 164, in __call__
await self.app(scope, receive, _send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 29, in __call__
await responder(scope, receive, send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 130, in __call__
await super().__call__(scope, receive, send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/gzip.py",
line 46, in __call__
await self.app(scope, receive, self.send_with_compression)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/cors.py",
line 85, in __call__
await self.app(scope, receive, send)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/base.py",
line 182, in __call__
with recv_stream, send_stream, collapse_excgroups():
^^^^^^^^^^^^^^^^^^^^
File "/usr/python/lib/python3.12/contextlib.py", line 158, in __exit__
self.gen.throw(value)
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/_utils.py", line
85, in collapse_excgroups
raise exc
File
"/home/airflow/.local/lib/python3.12/site-packages/starlette/middleware/base.py",
line 184, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/middlewares/refresh_token.py",
line 49, in dispatch
new_user, current_user = await self._refresh_user(current_token)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/middlewares/refresh_token.py",
line 80, in _refresh_user
user = await resolve_user_from_token(current_token)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/core_api/security.py",
line 100, in resolve_user_from_token
return await get_auth_manager().get_user_from_token(token_str)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/auth/managers/base_auth_manager.py",
line 111, in get_user_from_token
return self.deserialize_user(payload)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/cachetools/_cachedmethod.py",
line 375, in __call__
return wrapper(self._obj, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/cachetools/_cachedmethod.py",
line 358, in wrapper
v = method(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/airflow/providers/fab/auth_manager/fab_auth_manager.py",
line 267, in deserialize_user
return self.session.scalars(select(User).where(User.id ==
int(token["sub"]))).one()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/scoping.py",
line 1891, in scalars
return self._proxied.scalars(
^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/session.py",
line 2459, in scalars
return self._execute_internal(
^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/session.py",
line 2249, in _execute_internal
result: Result[Any] = compile_state_cls.orm_execute_statement(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/orm/context.py",
line 306, in orm_execute_statement
result = conn.execute(
^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1419, in execute
return meth(
^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/sql/elements.py",
line 527, in _execute_on_connection
return connection._execute_clauseelement(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1641, in _execute_clauseelement
ret = self._execute_context(
^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 1813, in _execute_context
conn = self._revalidate_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 673, in _revalidate_connection
self._invalid_transaction()
File
"/home/airflow/.local/lib/python3.12/site-packages/sqlalchemy/engine/base.py",
line 663, in _invalid_transaction
raise exc.PendingRollbackError(
sqlalchemy.exc.PendingRollbackError: Can't reconnect until invalid
transaction is rolled back. Please rollback() fully before proceeding
(Background on this error at: https://sqlalche.me/e/20/8s2b)`
### What you think should happen instead?
_No response_
### How to reproduce
3.1.7 Airflow deployment via official Helm chart
### Operating System
Debian 12
### Versions of Apache Airflow Providers
_No response_
### Deployment
Official Apache Airflow Helm Chart
### Deployment details
_No response_
### Anything else?
_No response_
### Are you willing to submit PR?
- [ ] Yes I am willing to submit a PR!
### Code of Conduct
- [x] I agree to follow this project's [Code of
Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
