This is an automated email from the ASF dual-hosted git repository.
fanningpj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-management.git
The following commit(s) were added to refs/heads/main by this push:
new ebc7aa57 Add native leases support in integration tests (#236)
ebc7aa57 is described below
commit ebc7aa57443fc6f5204338856fd4b49e2f19566b
Author: Iilun <[email protected]>
AuthorDate: Tue May 14 18:39:43 2024 +0200
Add native leases support in integration tests (#236)
* Add native leases support in integration tests
* Fix test not failing on error
---------
Co-authored-by: Iilun <[email protected]>
---
.github/workflows/integration-tests-lease.yml | 3 +-
.../kubernetes/job-native.yml | 18 +++
lease-kubernetes-int-test/kubernetes/rbac.yml | 23 +++-
.../src/main/resources/application.conf | 2 +
.../lease/kubernetes/LeaseTestSuite.scala | 2 +
lease-kubernetes-int-test/test.sh | 22 ++--
... => AbstractKubernetesApiIntegrationTest.scala} | 26 +++--
.../kubernetes/KubernetesApiIntegrationTest.scala | 122 +--------------------
.../NativeKubernetesApiIntegrationTest.scala | 33 ++++++
9 files changed, 106 insertions(+), 145 deletions(-)
diff --git a/.github/workflows/integration-tests-lease.yml
b/.github/workflows/integration-tests-lease.yml
index fd7b2b3c..2afeaa48 100644
--- a/.github/workflows/integration-tests-lease.yml
+++ b/.github/workflows/integration-tests-lease.yml
@@ -58,7 +58,8 @@ jobs:
kubectl proxy --port=8080 &
echo 'Running tests'
sbt ";lease-kubernetes/it:test"
- ./lease-kubernetes-int-test/test.sh
+ ./lease-kubernetes-int-test/test.sh job.yml
+ ./lease-kubernetes-int-test/test.sh job-native.yml
- name: Print logs on failure
if: ${{ failure() }}
diff --git a/lease-kubernetes-int-test/kubernetes/job-native.yml
b/lease-kubernetes-int-test/kubernetes/job-native.yml
new file mode 100644
index 00000000..f349004a
--- /dev/null
+++ b/lease-kubernetes-int-test/kubernetes/job-native.yml
@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: lease-test
+spec:
+ parallelism: 1
+ completions: 1
+ backoffLimit: 0
+ template:
+ spec:
+ containers:
+ - name: lease-test
+ imagePullPolicy: Never
+ image: pekko-lease-kubernetes-int-test:latest
+ env:
+ - name: LEASE_CLASS
+ value:
"org.apache.pekko.coordination.lease.kubernetes.NativeKubernetesLease"
+ restartPolicy: Never
diff --git a/lease-kubernetes-int-test/kubernetes/rbac.yml
b/lease-kubernetes-int-test/kubernetes/rbac.yml
index a5a43de1..590ca8bf 100644
--- a/lease-kubernetes-int-test/kubernetes/rbac.yml
+++ b/lease-kubernetes-int-test/kubernetes/rbac.yml
@@ -3,7 +3,16 @@ apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: lease-access
rules:
- - apiGroups: ["akka.io"]
+ - apiGroups: ["pekko.apache.org"]
+ resources: ["leases"]
+ verbs: ["get", "create", "update", "delete", "list"]
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: native-lease-access
+rules:
+ - apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "delete", "list"]
---
@@ -21,3 +30,15 @@ roleRef:
kind: Role
name: lease-access
apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: native-lease-access
+subjects:
+ - kind: ServiceAccount
+ name: default
+roleRef:
+ kind: Role
+ name: native-lease-access
+ apiGroup: rbac.authorization.k8s.io
diff --git a/lease-kubernetes-int-test/src/main/resources/application.conf
b/lease-kubernetes-int-test/src/main/resources/application.conf
index 5b9e8cc8..2c22155f 100644
--- a/lease-kubernetes-int-test/src/main/resources/application.conf
+++ b/lease-kubernetes-int-test/src/main/resources/application.conf
@@ -14,4 +14,6 @@ pekko {
}
}
}
+ # Possibility to modify lease class via environment, if not set will take
default value
+ coordination.lease.kubernetes.lease-class = ${?LEASE_CLASS}
}
diff --git
a/lease-kubernetes-int-test/src/main/scala/org/apache/pekko/coordination/lease/kubernetes/LeaseTestSuite.scala
b/lease-kubernetes-int-test/src/main/scala/org/apache/pekko/coordination/lease/kubernetes/LeaseTestSuite.scala
index bd312bcc..60639221 100644
---
a/lease-kubernetes-int-test/src/main/scala/org/apache/pekko/coordination/lease/kubernetes/LeaseTestSuite.scala
+++
b/lease-kubernetes-int-test/src/main/scala/org/apache/pekko/coordination/lease/kubernetes/LeaseTestSuite.scala
@@ -50,9 +50,11 @@ object LeaseTestSuite {
case Success(_) =>
log.info("Test failed, see the logs")
CoordinatedShutdown(as).run(TestFailedReason)
+ System.exit(1)
case Failure(exception) =>
log.error(exception, "Test exception")
CoordinatedShutdown(as).run(TestFailedReason)
+ System.exit(1)
}
}
diff --git a/lease-kubernetes-int-test/test.sh
b/lease-kubernetes-int-test/test.sh
index 9920639b..7e5d7acd 100755
--- a/lease-kubernetes-int-test/test.sh
+++ b/lease-kubernetes-int-test/test.sh
@@ -6,12 +6,13 @@ set -exu
JOB_NAME=lease-test
PROJECT_DIR=lease-kubernetes-int-test
+JOB_YML=$1
eval $(minikube -p minikube docker-env)
sbt "lease-kubernetes-int-test / docker:publishLocal"
-kubectl apply -f $PROJECT_DIR/kubernetes/rbac.yml
-kubectl delete -f $PROJECT_DIR/kubernetes/job.yml || true
+kubectl apply -f "$PROJECT_DIR/kubernetes/rbac.yml"
+kubectl delete -f "$PROJECT_DIR/kubernetes/$JOB_YML" || true
for i in {1..10}
do
@@ -22,18 +23,13 @@ done
echo "Old jobs cleaned up. Creating new job"
-kubectl create -f $PROJECT_DIR/kubernetes/job.yml
+kubectl create -f "$PROJECT_DIR/kubernetes/$JOB_YML"
# Add in a default sleep when we know a min amount of time it'll take
-for i in {1..10}
-do
- echo "Checking for job completion"
- # format is: wait for 1/1
- # lease-test 1/1 10s 2m32s
- [ `kubectl get jobs | grep $JOB_NAME | awk '{print $2}'` == "1/1" ] && break
- sleep 5
-done
+# Waiting for the status failed or complete to occur
+echo "Checking for job completion"
+kubectl wait --for=jsonpath='{.status.conditions[0].status}'=True job/$JOB_NAME
echo "Logs for job run:"
echo "=============================="
@@ -42,11 +38,11 @@ pods=$(kubectl get pods --selector=job-name=$JOB_NAME
--output=jsonpath={.items.
echo "Pods: $pods"
for pod in $pods
do
- echo "Logging for $pod"
+ echo "Logging for $pod"
kubectl logs $pod
done
-if [ $i -eq 10 ]
+if [ $(kubectl get job/$JOB_NAME -o jsonpath='{.status.conditions[0].type}') =
"Failed" ]
then
kubectl get jobs
kubectl describe job $JOB_NAME
diff --git
a/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/AbstractKubernetesApiIntegrationTest.scala
similarity index 83%
copy from
lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
copy to
lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/AbstractKubernetesApiIntegrationTest.scala
index 6ae33a55..c840505b 100644
---
a/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
+++
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/AbstractKubernetesApiIntegrationTest.scala
@@ -1,24 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * This file is part of the Apache Pekko project, which was derived from Akka.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.pekko.coordination.lease.kubernetes
+import com.typesafe.config.ConfigFactory
import org.apache.pekko
import pekko.Done
import pekko.actor.ActorSystem
-import pekko.coordination.lease.kubernetes.internal.KubernetesApiImpl
+import pekko.coordination.lease.kubernetes.internal.AbstractKubernetesApiImpl
import pekko.testkit.TestKit
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{ BeforeAndAfterAll, CancelAfterFailure }
import org.scalatest.concurrent.{ Eventually, ScalaFutures }
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike
+import org.scalatest.{ BeforeAndAfterAll, CancelAfterFailure }
import scala.concurrent.Await
import scala.concurrent.duration._
@@ -30,7 +38,7 @@ import scala.concurrent.duration._
*
* `kubectl proxy --port=8080`
*/
-class KubernetesApiIntegrationTest extends
TestKit(ActorSystem("KubernetesApiIntegrationSpec",
+abstract class AbstractKubernetesApiIntegrationTest extends
TestKit(ActorSystem("KubernetesApiIntegrationSpec",
ConfigFactory.parseString(
"""
pekko.loglevel = DEBUG
@@ -39,6 +47,8 @@ class KubernetesApiIntegrationTest extends
TestKit(ActorSystem("KubernetesApiInt
with AnyWordSpecLike with Matchers
with ScalaFutures with BeforeAndAfterAll with CancelAfterFailure with
Eventually {
+ val underTest: AbstractKubernetesApiImpl
+
implicit val patience: PatienceConfig =
PatienceConfig(testKitSettings.DefaultTimeout.duration)
val settings = new KubernetesSettings(
@@ -50,8 +60,6 @@ class KubernetesApiIntegrationTest extends
TestKit(ActorSystem("KubernetesApiInt
"",
apiServerRequestTimeout = 1.second,
false)
-
- val underTest = new KubernetesApiImpl(system, settings)
val leaseName = "lease-1"
val client1 = "client-1"
val client2 = "client-2"
diff --git
a/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
index 6ae33a55..cbb2c641 100644
---
a/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
+++
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/KubernetesApiIntegrationTest.scala
@@ -10,18 +10,7 @@
package org.apache.pekko.coordination.lease.kubernetes
import org.apache.pekko
-import pekko.Done
-import pekko.actor.ActorSystem
import pekko.coordination.lease.kubernetes.internal.KubernetesApiImpl
-import pekko.testkit.TestKit
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{ BeforeAndAfterAll, CancelAfterFailure }
-import org.scalatest.concurrent.{ Eventually, ScalaFutures }
-import org.scalatest.matchers.should.Matchers
-import org.scalatest.wordspec.AnyWordSpecLike
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
/**
* This test requires an API server available on localhost:8080, the lease CRD
created and a namespace called lease
@@ -30,116 +19,7 @@ import scala.concurrent.duration._
*
* `kubectl proxy --port=8080`
*/
-class KubernetesApiIntegrationTest extends
TestKit(ActorSystem("KubernetesApiIntegrationSpec",
- ConfigFactory.parseString(
- """
- pekko.loglevel = DEBUG
- pekko.coordination.lease.kubernetes.lease-operation-timeout = 1.5s
- """)))
- with AnyWordSpecLike with Matchers
- with ScalaFutures with BeforeAndAfterAll with CancelAfterFailure with
Eventually {
-
- implicit val patience: PatienceConfig =
PatienceConfig(testKitSettings.DefaultTimeout.duration)
-
- val settings = new KubernetesSettings(
- "",
- "",
- "localhost",
- 8080,
- namespace = Some("lease"),
- "",
- apiServerRequestTimeout = 1.second,
- false)
+class KubernetesApiIntegrationTest extends
AbstractKubernetesApiIntegrationTest {
val underTest = new KubernetesApiImpl(system, settings)
- val leaseName = "lease-1"
- val client1 = "client-1"
- val client2 = "client-2"
- var currentVersion = ""
-
- override protected def afterAll(): Unit = {
- TestKit.shutdownActorSystem(system)
- }
-
- override protected def beforeAll(): Unit = {
- // do some operation to check the proxy is up
- eventually {
- Await.result(underTest.removeLease(leaseName), 2.second) shouldEqual Done
- }
- }
-
- "Kubernetes lease resource" should {
- "be able to be created" in {
- val leaseRecord =
underTest.readOrCreateLeaseResource(leaseName).futureValue
- leaseRecord.owner shouldEqual None
- leaseRecord.version shouldNot equal("")
- leaseRecord.version shouldNot equal(null)
- currentVersion = leaseRecord.version
- }
-
- "be able to read back with same version" in {
- val leaseRecord =
underTest.readOrCreateLeaseResource(leaseName).futureValue
- leaseRecord.version shouldEqual currentVersion
- }
-
- "be able to take a lease with no owner" in {
- val leaseRecord = underTest.updateLeaseResource(leaseName, client1,
currentVersion).futureValue
- val success: LeaseResource = leaseRecord match {
- case Right(lr) => lr
- case Left(_) => fail("There shouldn't be anyone else updating the
resource.")
- }
- success.version shouldNot equal(currentVersion)
- currentVersion = success.version
- success.owner shouldEqual Some(client1)
- }
-
- "be able to update a lease if resource version is correct" in {
- val timeUpdate = System.currentTimeMillis()
- val leaseRecord = underTest.updateLeaseResource(leaseName, client1,
currentVersion, time = timeUpdate).futureValue
- val success: LeaseResource = leaseRecord match {
- case Right(lr) => lr
- case Left(_) => fail("There shouldn't be anyone else updating the
resource.")
- }
- success.version shouldNot equal(currentVersion)
- currentVersion = success.version
- success.owner shouldEqual Some(client1)
- success.time shouldEqual timeUpdate
- }
-
- "not be able to update a lease if resource version is correct" in {
- val timeUpdate = System.currentTimeMillis()
- val leaseRecord = underTest.updateLeaseResource(leaseName, client1,
"10", time = timeUpdate).futureValue
- val failure: LeaseResource = leaseRecord match {
- case Right(_) => fail("Expected update failure (we've used an invalid
version!).")
- case Left(lr) => lr
- }
- failure.version shouldEqual currentVersion
- currentVersion = failure.version
- failure.owner shouldEqual Some(client1)
- failure.time shouldNot equal(timeUpdate)
- }
-
- "be able to remove ownership" in {
- val leaseRecord = underTest.updateLeaseResource(leaseName, "",
currentVersion).futureValue
- val success: LeaseResource = leaseRecord match {
- case Right(lr) => lr
- case Left(_) => fail("There shouldn't be anyone else updating the
resource.")
- }
- success.version shouldNot equal(currentVersion)
- currentVersion = success.version
- success.owner shouldEqual None
- }
-
- "be able to get lease once other client has removed" in {
- val leaseRecord = underTest.updateLeaseResource(leaseName, client2,
currentVersion).futureValue
- val success: LeaseResource = leaseRecord match {
- case Right(lr) => lr
- case Left(_) => fail("There shouldn't be anyone else updating the
resource.")
- }
- success.version shouldNot equal(currentVersion)
- currentVersion = success.version
- success.owner shouldEqual Some(client2)
- }
- }
-
}
diff --git
a/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/NativeKubernetesApiIntegrationTest.scala
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/NativeKubernetesApiIntegrationTest.scala
new file mode 100644
index 00000000..1296a1df
--- /dev/null
+++
b/lease-kubernetes/src/it/scala/org/apache/pekko/coordination/lease/kubernetes/NativeKubernetesApiIntegrationTest.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pekko.coordination.lease.kubernetes
+
+import org.apache.pekko
+import pekko.coordination.lease.kubernetes.internal.NativeKubernetesApiImpl
+
+/**
+ * This test requires an API server available on localhost:8080, the lease CRD
created and a namespace called lease
+ *
+ * One way of doing this is to have a kubectl proxy open:
+ *
+ * `kubectl proxy --port=8080`
+ */
+class NativeKubernetesApiIntegrationTest extends
AbstractKubernetesApiIntegrationTest {
+
+ val underTest = new NativeKubernetesApiImpl(system, settings)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
