This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-management.git
The following commit(s) were added to refs/heads/main by this push:
new f8cc98a4 Fix flaky KubernetesApiSpec by decoupling functional tests
from request timeout #216 (#842)
f8cc98a4 is described below
commit f8cc98a47aa08b0837b7ddbd3ff97980a96777df
Author: Arun Selvamani <[email protected]>
AuthorDate: Fri Jun 19 12:07:16 2026 -0700
Fix flaky KubernetesApiSpec by decoupling functional tests from request
timeout #216 (#842)
* Every request in AbstractKubernetesApiImpl.makeRequest is raced against
apiServerRequestTimeout. The first HTTP request pays Pekko HTTP
connection-pool
and JIT warm-up cost and could exceed the hard-coded 1s deadline on a
loaded CI
runner, failing functional tests with a spurious LeaseTimeoutException.
* Give the functional and auth tests a generous 10s request timeout so
first-request
warm-up no longer trips the deadline.
* Route the deliberate "timeout on ..." tests through a dedicated
short-timeout (1s)
client so they still fail fast; their relative delays scale off that
short value.
Co-authored-by: Arun Selvamani <[email protected]>
Co-authored-by: Claude Opus 4.8 <[email protected]>
---
.../lease/kubernetes/KubernetesApiSpec.scala | 48 +++++++++++++++++-----
1 file changed, 37 insertions(+), 11 deletions(-)
diff --git
a/lease-kubernetes/src/test/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiSpec.scala
b/lease-kubernetes/src/test/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiSpec.scala
index 63811d4b..3b1943cc 100644
---
a/lease-kubernetes/src/test/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiSpec.scala
+++
b/lease-kubernetes/src/test/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiSpec.scala
@@ -53,7 +53,27 @@ class KubernetesApiSpec
val wireMockServer = new WireMockServer(wireMockConfig().port(0))
wireMockServer.start()
+ // Generous request deadline for the functional tests. Every request inside
+ // `AbstractKubernetesApiImpl.makeRequest` is raced against
`apiServerRequestTimeout`, and the
+ // first HTTP request through Pekko HTTP pays connection-pool/JIT warm-up
cost (observed ~0.8s
+ // locally, more on a loaded CI box). A 1s deadline here is what made this
suite flaky: the
+ // warm-up request would lose the race and fail with a spurious
LeaseTimeoutException.
+ // See https://github.com/apache/pekko-management/issues/216
+ private val robustRequestTimeout = 10.seconds
+
val settings = new KubernetesSettings(
+ "",
+ "",
+ "localhost",
+ wireMockServer.port(),
+ namespace = Some("lease"),
+ "",
+ apiServerRequestTimeout = robustRequestTimeout,
+ secure = false,
+ bodyReadTimeout = robustRequestTimeout)
+
+ // Short deadline used ONLY by the deliberate "timeout on ..." tests, so
they fail fast.
+ val timeoutSettings = new KubernetesSettings(
"",
"",
"localhost",
@@ -65,7 +85,6 @@ class KubernetesApiSpec
WireMock.configureFor(settings.apiServerPort)
- // double the default timeout for CI
(https://github.com/apache/pekko-management/issues/216)
implicit val patience: PatienceConfig =
PatienceConfig(testKitSettings.DefaultTimeout.duration.*(2))
val underTest = new KubernetesApiImpl(system, settings) {
@@ -73,6 +92,13 @@ class KubernetesApiSpec
override protected def readConfigVarFromFilesystem(path: String, name:
String): Future[Option[String]] =
Future.successful(None)
}
+
+ // Same as `underTest`, but with the short `timeoutSettings`, for the
deliberate timeout tests.
+ val underTestTimeouts = new KubernetesApiImpl(system, timeoutSettings) {
+ // avoid touching slow CI filesystem
+ override protected def readConfigVarFromFilesystem(path: String, name:
String): Future[Option[String]] =
+ Future.successful(None)
+ }
val leaseName = "lease-1"
val client1 = "client-1"
val client2 = "client-2"
@@ -201,7 +227,7 @@ class KubernetesApiSpec
stubFor(
get(urlEqualTo(s"/apis/pekko.apache.org/v1/namespaces/lease/leases/$lease")).willReturn(
aResponse()
- .withFixedDelay((settings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
+ .withFixedDelay((timeoutSettings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
.withStatus(StatusCodes.OK.intValue)
.withHeader("Content-Type", "application/json")
.withBody(s"""
@@ -222,7 +248,7 @@ class KubernetesApiSpec
|}
""".stripMargin)))
- underTest
+ underTestTimeouts
.readOrCreateLeaseResource(lease)
.failed
.futureValue
@@ -239,11 +265,11 @@ class KubernetesApiSpec
stubFor(
post(urlEqualTo(s"/apis/pekko.apache.org/v1/namespaces/lease/leases/$lease")).willReturn(
aResponse()
- .withFixedDelay((settings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
+ .withFixedDelay((timeoutSettings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
.withStatus(StatusCodes.OK.intValue)
.withHeader("Content-Type", "application/json")))
- underTest
+ underTestTimeouts
.readOrCreateLeaseResource(lease)
.failed
.futureValue
@@ -256,11 +282,11 @@ class KubernetesApiSpec
stubFor(
put(urlEqualTo(s"/apis/pekko.apache.org/v1/namespaces/lease/leases/$lease")).willReturn(
aResponse()
- .withFixedDelay((settings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
+ .withFixedDelay((timeoutSettings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
.withStatus(StatusCodes.OK.intValue)
.withHeader("Content-Type", "application/json")))
- underTest.updateLeaseResource(lease, owner,
"1").failed.futureValue.getMessage shouldEqual
+ underTestTimeouts.updateLeaseResource(lease, owner,
"1").failed.futureValue.getMessage shouldEqual
s"Timed out updating lease [$lease] to owner [$owner]. It is not known
if the update happened. Is the API server up?"
}
@@ -269,10 +295,10 @@ class KubernetesApiSpec
stubFor(
delete(urlEqualTo(s"/apis/pekko.apache.org/v1/namespaces/lease/leases/$lease")).willReturn(
aResponse()
- .withFixedDelay((settings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
+ .withFixedDelay((timeoutSettings.apiServerRequestTimeout *
2).toMillis.toInt) // Oh noes
.withStatus(StatusCodes.OK.intValue)))
- underTest.removeLease(lease).failed.futureValue.getMessage shouldEqual
+ underTestTimeouts.removeLease(lease).failed.futureValue.getMessage
shouldEqual
s"Timed out removing lease [$lease]. It is not known if the remove
happened. Is the API server up?"
}
@@ -309,7 +335,7 @@ class KubernetesApiSpec
wireMockServer.port(),
namespace = Some("lease"),
"",
- apiServerRequestTimeout = 1.second,
+ apiServerRequestTimeout = robustRequestTimeout,
secure = false)
val toFail = new AtomicBoolean(true)
@@ -340,7 +366,7 @@ class KubernetesApiSpec
wireMockServer.port(),
namespace = Some("lease"),
"",
- apiServerRequestTimeout = 1.second,
+ apiServerRequestTimeout = robustRequestTimeout,
secure = false)
val retryUnauthorized = new KubernetesApiImpl(system, newSettings) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
