This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-1-test by this push:
new 88ad427dec7 [v3-1-test] Fix memory leak in Client via SSL context
creation (#57334) (#57374)
88ad427dec7 is described below
commit 88ad427dec7dddfce2a76256a8464283fa2c6856
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Oct 28 00:02:19 2025 +0100
[v3-1-test] Fix memory leak in Client via SSL context creation (#57334)
(#57374)
related: https://github.com/apache/airflow/issues/56641
When I performed the same memray inspection on the latest version that
includes both fixes, the first issue was clearly resolved, but the second issue
still persists.
[memray1.html](https://github.com/user-attachments/files/23160226/memray1.html)
When a Client object is created, `ctx =
ssl.create_default_context(cafile=ca_file)` continues to be executed
repeatedly, which accumulates in memory and causes a memory leak. (It appears
to be allocated as a C language object and remains in memory regardless of
Python object GC)
This PR uses caching to prevent the SSL context object from being recreated.
Here are the results after running for two hours with this change. The
memory usage, which was previously growing to tens of MBs, now stabilizes at
approximately 700KB.
[memray2.html](https://github.com/user-attachments/files/23160405/memray2.html)
(cherry picked from commit 7369e4645c17e29c2c15fe10f9ede3d5afe02a2a)
Co-authored-by: Jeongwoo Do <[email protected]>
---
task-sdk/src/airflow/sdk/api/client.py | 14 ++++++++++----
task-sdk/tests/task_sdk/api/test_client.py | 27 +++++++++++++++++++++++++++
2 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/task-sdk/src/airflow/sdk/api/client.py
b/task-sdk/src/airflow/sdk/api/client.py
index 6c1134036d3..5c891c84c8f 100644
--- a/task-sdk/src/airflow/sdk/api/client.py
+++ b/task-sdk/src/airflow/sdk/api/client.py
@@ -814,6 +814,15 @@ API_TIMEOUT = conf.getfloat("workers",
"execution_api_timeout")
class Client(httpx.Client):
+ @classmethod
+ @lru_cache()
+ def _get_ssl_context_cached(cls, ca_file: str, ca_path: str | None = None)
-> ssl.SSLContext:
+ """Cache SSL context to prevent memory growth from repeated context
creation."""
+ ctx = ssl.create_default_context(cafile=ca_file)
+ if ca_path:
+ ctx.load_verify_locations(ca_path)
+ return ctx
+
def __init__(self, *, base_url: str | None, dry_run: bool = False, token:
str, **kwargs: Any):
if (not base_url) ^ dry_run:
raise ValueError(f"Can only specify one of {base_url=} or
{dry_run=}")
@@ -826,10 +835,7 @@ class Client(httpx.Client):
kwargs.setdefault("base_url", "dry-run://server")
else:
kwargs["base_url"] = base_url
- ctx = ssl.create_default_context(cafile=certifi.where())
- if API_SSL_CERT_PATH:
- ctx.load_verify_locations(API_SSL_CERT_PATH)
- kwargs["verify"] = ctx
+ kwargs["verify"] = self._get_ssl_context_cached(certifi.where(),
API_SSL_CERT_PATH)
# Set timeout if not explicitly provided
kwargs.setdefault("timeout", API_TIMEOUT)
diff --git a/task-sdk/tests/task_sdk/api/test_client.py
b/task-sdk/tests/task_sdk/api/test_client.py
index 32a709663ac..d9c87d915c9 100644
--- a/task-sdk/tests/task_sdk/api/test_client.py
+++ b/task-sdk/tests/task_sdk/api/test_client.py
@@ -23,6 +23,7 @@ from datetime import datetime
from typing import TYPE_CHECKING
from unittest import mock
+import certifi
import httpx
import pytest
import uuid6
@@ -1366,3 +1367,29 @@ class TestHITLOperations:
assert result.params_input == {}
assert result.responded_by_user == HITLUser(id="admin", name="admin")
assert result.responded_at == timezone.datetime(2025, 7, 3, 0, 0, 0)
+
+
+class TestSSLContextCaching:
+ def setup_method(self):
+ Client._get_ssl_context_cached.cache_clear()
+
+ def teardown_method(self):
+ Client._get_ssl_context_cached.cache_clear()
+
+ def test_cache_hit_on_same_parameters(self):
+ ca_file = certifi.where()
+ ctx1 = Client._get_ssl_context_cached(ca_file, None)
+ ctx2 = Client._get_ssl_context_cached(ca_file, None)
+ assert ctx1 is ctx2
+
+ def test_cache_miss_on_different_parameters(self):
+ ca_file = certifi.where()
+
+ ctx1 = Client._get_ssl_context_cached(ca_file, None)
+ ctx2 = Client._get_ssl_context_cached(ca_file, ca_file)
+
+ info = Client._get_ssl_context_cached.cache_info()
+
+ assert ctx1 is not ctx2
+ assert info.misses == 2
+ assert info.currsize == 2
