This is an automated email from the ASF dual-hosted git repository.
ricardozanini pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/incubator-kie-kogito-serverless-operator.git
The following commit(s) were added to refs/heads/main by this push:
new a8dc5ed8 [KOGITO-10013] Migrate from Minikube to Kind for E2E tests
and export logs (#354)
a8dc5ed8 is described below
commit a8dc5ed8606c62a0a35c5b02dfeced5b0a08ee1c
Author: Jordi Gil <[email protected]>
AuthorDate: Thu Jan 18 08:28:40 2024 -0500
[KOGITO-10013] Migrate from Minikube to Kind for E2E tests and export logs
(#354)
* [KOGITO-10013] Migrate from Minikube to Kind for E2E tests
Signed-off-by: Jordi Gil <[email protected]>
* Add headers and set log retention to minimum (1 day)
Signed-off-by: Jordi Gil <[email protected]>
* Changed registry port reference to 5001 and reference to Minikube to Kind
Signed-off-by: Jordi Gil <[email protected]>
* Add makefile targets to install kind, create and delete cluster and amend
github action to use the create cluster target
Signed-off-by: Jordi Gil <[email protected]>
* Load controller built image to node with kind load docker-image
Signed-off-by: Jordi Gil <[email protected]>
* Remove podman from build process in e2e lane
Signed-off-by: Jordi Gil <[email protected]>
* Install python's docker libraries: docker and python-docker
Signed-off-by: Jordi Gil <[email protected]>
* Add docker-squash as pip egg to build the image
Signed-off-by: Jordi Gil <[email protected]>
---------
Signed-off-by: Jordi Gil <[email protected]>
---
.github/workflows/e2e.yml | 69 +++++++--------------
Makefile | 16 ++++-
hack/ci/create-kind-cluster-with-registry.sh | 91 ++++++++++++++++++++++++++++
3 files changed, 129 insertions(+), 47 deletions(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 29e67693..b528a513 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -13,11 +13,6 @@ on:
env:
GO_VERSION: 1.19
- # Kubernetes version should aligned with OCP LTS, aka OCP 4.10 (k8s 1.23)
for now
- #
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md
- # WARNING: kindest/node is not always existing with given version ...
- KUBERNETES_VERSION: v1.26.3
- MINIKUBE_VERSION: v1.31.1
DEBUG: true
jobs:
@@ -27,7 +22,7 @@ jobs:
cancel-in-progress: true
timeout-minutes: 120
runs-on: ubuntu-latest
- name: End-to-end tests (Minikube)
+ name: End-to-end tests (Kind)
steps:
- name: Install package
run: |
@@ -50,45 +45,13 @@ jobs:
go-version: ${{ env.GO_VERSION }}
cache: true
- - name: Cache Minikube Download
- id: cache-minikube-download
- uses: actions/cache@v3
- with:
- key: ${{ runner.os }}-minikube-${{ env.MINIKUBE_VERSION }}
- path: |
- ~/.minikube/download/
-
- - name: Setup Minikube cluster
- run: |
- # Install minikube at $HOME/.local/bin
- ./hack/ci/install-minikube.sh $HOME/.local/bin
- echo "$HOME/.local/bin" >> $GITHUB_PATH
- minikube start --kubernetes-version ${{ env.KUBERNETES_VERSION }}
--addons=registry --insecure-registry=localhost:5000,192.168.0.0/16 --cpus=max
--memory=max
-
- - name: Wait for Minikube up and running
+ - name: Setup Kind cluster
run: |
- minikube ssh cat /lib/systemd/system/docker.service
- kubectl get pods -A
- set -x
- MINIKUBE_COMPONENTS=(etcd kube-apiserver kube-controller-manager
kube-scheduler)
- for component in "${MINIKUBE_COMPONENTS[@]}"
- do
- echo "Check component '${component}' is in 'Running' state"
- COMPONENT_NAME=${component} timeout 60s bash -c 'kubectl get pods
-l tier=control-plane -l component=${COMPONENT_NAME} -n kube-system && while [[
"$(kubectl get pods -l tier=control-plane -l component=${COMPONENT_NAME} -n
kube-system -o jsonpath={.items[0].status.phase})" != "Running" ]] ; do sleep 2
&& kubectl get pods -l tier=control-plane -l component=${COMPONENT_NAME} -n
kube-system -o jsonpath={.items[0].status.phase}; done'
- done
+ make create-cluster
-
- - name: Wait for Minikube registry
+ - name: Set OPERATOR_IMAGE_NAME to point to Kind's local registry
run: |
- kubectl get pods -A
- timeout 60s bash -c 'kubectl get pods -l
kubernetes.io/minikube-addons=registry -l actual-registry=true -n kube-system
&& while [[ "$(kubectl get pods -l kubernetes.io/minikube-addons=registry -l
actual-registry=true -n kube-system -o jsonpath={.items[0].status.phase})" !=
"Running" ]] ; do sleep 2 && kubectl get pods -l
kubernetes.io/minikube-addons=registry -l actual-registry=true -n kube-system
-o jsonpath={.items[0].status.phase}; done'
-
-
- - name: Retrieve Minikube registry
- run: |
- minikube_registry="$(minikube ip):5000"
- echo "MINIKUBE_REGISTRY=${minikube_registry}" >> $GITHUB_ENV
- echo
"OPERATOR_IMAGE_NAME=${minikube_registry}/kogito-serverless-operator:0.0.1" >>
$GITHUB_ENV
+ echo
"OPERATOR_IMAGE_NAME=127.0.0.1:5001/kogito-serverless-operator:0.0.1" >>
$GITHUB_ENV
- name: Setup Python for cekit
uses: actions/setup-python@v4
@@ -99,12 +62,12 @@ jobs:
- name: Build operator image
run: |
pip install cekit==4.5.0
- pip install odcs podman behave lxml krb5
- make container-build BUILDER=podman IMG=${{ env.OPERATOR_IMAGE_NAME
}}
+ pip install odcs docker python-docker behave lxml krb5 docker-squash
+ make container-build BUILDER=docker IMG=${{ env.OPERATOR_IMAGE_NAME
}}
- - name: Load image in Minikube
+ - name: Load image in Kind
run: |
- podman push --tls-verify=false ${{ env.OPERATOR_IMAGE_NAME }}
+ kind load docker-image
127.0.0.1:5001/kogito-serverless-operator:0.0.1
- name: Check pods
run: |
@@ -115,3 +78,17 @@ jobs:
- name: Run tests
run: |
make test-e2e
+
+ - name: Export kind logs
+ if: always()
+ run: |
+ mkdir -p /tmp/kind/logs
+ kind export logs --loglevel=debug /tmp/kind/logs
+
+ - name: Upload kind logs
+ if: always()
+ uses: actions/upload-artifact@v3
+ with:
+ name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
+ path: /tmp/kind/logs
+ retention-days: 1
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 68254626..17e2ad05 100644
--- a/Makefile
+++ b/Makefile
@@ -248,6 +248,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
## Tool Versions
KUSTOMIZE_VERSION ?= v4.5.2
CONTROLLER_TOOLS_VERSION ?= v0.9.2
+KIND_VERSION ?= v0.20.0
KUSTOMIZE_INSTALL_SCRIPT ?=
"https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh";
.PHONY: kustomize
@@ -349,4 +350,17 @@ test-e2e: install-operator-sdk
go test ./test/e2e/* -v -ginkgo.v -timeout 30m
.PHONY: before-pr
-before-pr: test generate-all
\ No newline at end of file
+before-pr: test generate-all
+
+
+.PHONY: install-kind
+install-kind:
+ command -v kind >/dev/null || go install
sigs.k8s.io/kind@$(KIND_VERSION)
+
+.PHONY: create-cluster
+create-cluster: install-kind
+ kind get clusters | grep kind >/dev/null ||
./hack/ci/create-kind-cluster-with-registry.sh
+
+.PHONY: delete-cluster
+delete-cluster: install-kind
+ kind delete cluster && docker rm -f kind-registry
\ No newline at end of file
diff --git a/hack/ci/create-kind-cluster-with-registry.sh
b/hack/ci/create-kind-cluster-with-registry.sh
new file mode 100755
index 00000000..bb341a11
--- /dev/null
+++ b/hack/ci/create-kind-cluster-with-registry.sh
@@ -0,0 +1,91 @@
+# Copyright 2024 Apache Software Foundation (ASF)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/sh
+
+set -o errexit
+
+reg_name='kind-registry'
+reg_port='5001'
+
+
+# 1. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+#
https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster -n kind --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+ [plugins."io.containerd.grpc.v1.cri".registry]
+ config_path = "/etc/containerd/certs.d"
+EOF
+
+# 2. Wait for kube system pods to reach running state
+if ! kubectl wait -n kube-system --for=condition=ready pods --all
--timeout=120s ; then
+ echo "some pods in the system are not running"
+ kubectl get pods -A -o wide || true
+ exit 1
+fi
+
+# 3 Create registry container
+docker run \
+ -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name
"${reg_name}" \
+ -v /tmp/certs:/certs \
+ registry:2
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same
network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}'
"${reg_name}")" = 'null' ]; then
+ docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/172.18.0.3:5000"
+# retrieve IP address of the container connected to the cluster network
+IP_ADDRESS=$(docker inspect --format='{{(index (index
.NetworkSettings.Networks "kind") ).IPAddress}}' ${reg_name})
+for node in $(kind get nodes); do
+ docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+ cat <<EOF | docker exec -i "${node}" cp /dev/stdin
"${REGISTRY_DIR}/hosts.toml"
+[host."http://${IP_ADDRESS}:5000";]
+ capabilities = ["pull", "resolve", "push"]
+ skip_verify = true
+EOF
+done
+
+# 6. Document the local registry
+#
https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: local-registry-hosting
+ namespace: kube-public
+data:
+ localRegistryHosting.v1: |
+ hostFromClusterNetwork: "${IP_ADDRESS}:5000"
+ help: "https://kind.sigs.k8s.io/docs/user/local-registry/";
+EOF
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
